404 lines
17 KiB
TypeScript
404 lines
17 KiB
TypeScript
import {
|
|
RUNTIME_DURABLE_ADAPTER_MISSING,
|
|
RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED,
|
|
RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING,
|
|
RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED,
|
|
RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED,
|
|
RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED,
|
|
RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED,
|
|
RUNTIME_DURABLE_ADAPTER_UNCONFIGURED,
|
|
RUNTIME_DURABILITY_REQUIRED_EVIDENCE,
|
|
RUNTIME_STORE_KIND_POSTGRES
|
|
} from "../db/runtime-store.ts";
|
|
|
|
export const CLOUD_API_HEALTH_CONTRACT_VERSION = "v3";
|
|
|
|
export function buildCloudApiReadiness({ db, codeAgent, runtime } = {}) {
|
|
const dbReady = isDbReady(db);
|
|
const codeAgentReady = isCodeAgentReady(codeAgent);
|
|
const runtimeReady = isRuntimeReady(runtime);
|
|
const blockers = [];
|
|
let runtimeReadinessBlocker = null;
|
|
|
|
if (!dbReady) {
|
|
blockers.push(dbBlocker(db));
|
|
}
|
|
|
|
if (!codeAgentReady) {
|
|
blockers.push(codeAgentBlocker(codeAgent));
|
|
}
|
|
|
|
if (!runtimeReady) {
|
|
runtimeReadinessBlocker = runtimeBlocker(runtime);
|
|
blockers.push(runtimeReadinessBlocker);
|
|
}
|
|
const provider = buildProviderReadiness(codeAgent);
|
|
const dbDurable = buildRuntimeDurabilityReadiness({
|
|
db,
|
|
runtime,
|
|
runtimeReady,
|
|
runtimeBlocker: runtimeReadinessBlocker
|
|
});
|
|
const sessionRunner = buildSessionRunnerReadiness(codeAgent);
|
|
const codexStdio = buildCodexStdioReadiness(codeAgent);
|
|
const codeAgentReadiness = buildCodeAgentStructuredReadiness({
|
|
provider,
|
|
dbDurable,
|
|
sessionRunner,
|
|
codexStdio,
|
|
codeAgent
|
|
});
|
|
|
|
return {
|
|
contractVersion: CLOUD_API_HEALTH_CONTRACT_VERSION,
|
|
ready: blockers.length === 0,
|
|
status: blockers.length === 0 ? "ok" : "degraded",
|
|
blockerCodes: blockers.map((blocker) => blocker.code),
|
|
blockers,
|
|
components: {
|
|
db: dbReady ? "ready" : "blocked",
|
|
codeAgent: codeAgentReady ? "ready" : "blocked",
|
|
runtime: runtimeReady ? "ready" : dbReady ? "blocked" : "waiting_for_db",
|
|
provider: provider.status,
|
|
dbDurable: dbDurable.status,
|
|
sessionRunner: sessionRunner.status,
|
|
codexStdio: codexStdio.status
|
|
},
|
|
provider,
|
|
dbDurable,
|
|
sessionRunner,
|
|
codexStdio,
|
|
codeAgent: codeAgentReadiness,
|
|
durability: dbDurable
|
|
};
|
|
}
|
|
|
|
function isDbReady(db) {
|
|
return Boolean(db?.ready && db?.connected && db?.liveConnected !== false && db?.liveDbEvidence);
|
|
}
|
|
|
|
function isCodeAgentReady(codeAgent) {
|
|
return Boolean(
|
|
codeAgent?.ready === true &&
|
|
codeAgent?.longLivedSessionGate?.status === "pass" &&
|
|
codeAgent?.codexStdioFeasibility?.canStartLongLivedCodexStdio === true
|
|
);
|
|
}
|
|
|
|
function isRuntimeReady(runtime) {
|
|
if (runtime?.durable !== true) {
|
|
return false;
|
|
}
|
|
if (runtime?.ready !== true || runtime?.liveRuntimeEvidence !== true) {
|
|
return false;
|
|
}
|
|
return !["blocked", "degraded", "failed"].includes(runtime.status);
|
|
}
|
|
|
|
function dbBlocker(db) {
|
|
return {
|
|
code: db?.configReady ? "db_live_connection_blocked" : "db_runtime_config_blocked",
|
|
type: "runtime_blocker",
|
|
scope: "cloud-api-db",
|
|
status: "open",
|
|
sourceIssue: "pikasTech/HWLAB#49",
|
|
summary: db?.blocker ?? "cloud-api DB live readiness is not established",
|
|
evidence: {
|
|
configReady: Boolean(db?.configReady),
|
|
connected: Boolean(db?.connected),
|
|
liveConnected: Boolean(db?.liveConnected),
|
|
liveDbEvidence: Boolean(db?.liveDbEvidence),
|
|
connectionAttempted: Boolean(db?.connectionAttempted ?? db?.connectionChecked),
|
|
connectionResult: db?.connectionResult ?? db?.connection?.result ?? "unknown"
|
|
}
|
|
};
|
|
}
|
|
|
|
function codeAgentBlocker(codeAgent) {
|
|
const primaryBlocker = Array.isArray(codeAgent?.blockers) ? codeAgent.blockers[0] : null;
|
|
return {
|
|
code: codeAgent?.reason ?? primaryBlocker?.code ?? "code_agent_long_lived_session_blocked",
|
|
type: "agent_blocker",
|
|
scope: "code-agent-long-lived-session",
|
|
status: "open",
|
|
sourceIssue: primaryBlocker?.sourceIssue ?? "pikasTech/HWLAB#275",
|
|
summary: codeAgent?.summary ?? "Code Agent provider readiness is not established",
|
|
evidence: {
|
|
provider: codeAgent?.provider ?? "unknown",
|
|
backend: codeAgent?.backend ?? "unknown",
|
|
agentKind: codeAgent?.agentKind ?? "unknown",
|
|
capabilityStatus: codeAgent?.capabilityStatus ?? "unknown",
|
|
longLivedSessionGate: codeAgent?.longLivedSessionGate ?? null,
|
|
codexStdioFeasibility: codeAgent?.codexStdioFeasibility ?? null,
|
|
partialReady: codeAgent?.partialReady === true,
|
|
missingEnv: Array.isArray(codeAgent?.missingEnv) ? [...codeAgent.missingEnv] : [],
|
|
secretsRedacted: true
|
|
}
|
|
};
|
|
}
|
|
|
|
function runtimeBlocker(runtime) {
|
|
const durable = runtime?.durable === true;
|
|
const durableRequested = runtime?.durableRequested === true || runtime?.adapter === RUNTIME_STORE_KIND_POSTGRES;
|
|
const unhealthy = (durable || durableRequested) && ["blocked", "degraded", "failed"].includes(runtime?.status);
|
|
const blocker = runtime?.blocker;
|
|
const blockedLayer = classifyRuntimeDurabilityLayer(runtime);
|
|
return {
|
|
code: durableRequested && unhealthy ? blocker ?? "runtime_durable_adapter_unhealthy" : RUNTIME_DURABLE_ADAPTER_MISSING,
|
|
type: "runtime_blocker",
|
|
scope: "runtime-durable-adapter",
|
|
status: "open",
|
|
sourceIssue: "pikasTech/HWLAB#164",
|
|
summary: durableRequested && unhealthy
|
|
? runtime?.reason ?? "cloud-api durable runtime adapter is configured but not ready"
|
|
: "cloud-api DB is live, but L1 runtime writes still use process-local memory because no DB-backed durable adapter is configured",
|
|
evidence: {
|
|
adapter: runtime?.adapter ?? "unknown",
|
|
durable: Boolean(runtime?.durable),
|
|
durableRequested: Boolean(runtime?.durableRequested),
|
|
durableCapable: Boolean(runtime?.durableCapable),
|
|
runtimeStatus: runtime?.status ?? "unknown",
|
|
runtimeReady: Boolean(runtime?.ready),
|
|
liveRuntimeEvidence: Boolean(runtime?.liveRuntimeEvidence),
|
|
schemaReady: Boolean(runtime?.schema?.ready),
|
|
schemaChecked: Boolean(runtime?.schema?.checked),
|
|
migrationReady: Boolean(runtime?.migration?.ready),
|
|
migrationChecked: Boolean(runtime?.migration?.checked),
|
|
gates: runtime?.gates ?? null,
|
|
blockedLayer,
|
|
queryAttempted: Boolean(runtime?.connection?.queryAttempted),
|
|
queryResult: runtime?.connection?.queryResult ?? "unknown",
|
|
dbBackedAdapterRequired: true,
|
|
dbLiveEvidenceIsDurabilityEvidence: false,
|
|
requiredEvidence: runtime?.durabilityContract?.requiredEvidence ?? RUNTIME_DURABILITY_REQUIRED_EVIDENCE,
|
|
secretMaterialRead: false
|
|
},
|
|
nextTask: durableRequested
|
|
? "Apply/repair the runtime DB schema or DB auth, then rerun /health/live without printing DB secret material."
|
|
: "Set HWLAB_CLOUD_RUNTIME_ADAPTER=postgres after the runtime DB schema is available; keep HWLAB_CLOUD_DB_URL injected from Secret."
|
|
};
|
|
}
|
|
|
|
function buildRuntimeDurabilityReadiness({ db, runtime, runtimeReady, runtimeBlocker } = {}) {
|
|
const dbLiveEvidenceObserved = Boolean(db?.ready && db?.connected && db?.liveConnected !== false && db?.liveDbEvidence);
|
|
const contract = runtime?.durabilityContract ?? {};
|
|
return {
|
|
status: runtimeReady ? "ready" : "blocked",
|
|
ready: runtimeReady,
|
|
adapter: runtime?.adapter ?? "unknown",
|
|
durable: Boolean(runtime?.durable),
|
|
durableRequested: Boolean(runtime?.durableRequested || runtime?.adapter === RUNTIME_STORE_KIND_POSTGRES),
|
|
durableCapable: Boolean(runtime?.durableCapable),
|
|
runtimeStatus: runtime?.status ?? "unknown",
|
|
runtimeReady: Boolean(runtime?.ready),
|
|
liveRuntimeEvidence: Boolean(runtime?.liveRuntimeEvidence),
|
|
dbLiveEvidenceObserved,
|
|
dbLiveEvidenceIsDurabilityEvidence: false,
|
|
requiredEvidence: contract.requiredEvidence ?? RUNTIME_DURABILITY_REQUIRED_EVIDENCE,
|
|
blockedLayer: runtimeReady ? null : contract.blockedLayer ?? classifyRuntimeDurabilityLayer(runtime),
|
|
blocker: runtimeReady ? null : runtimeBlocker?.code ?? runtime?.blocker ?? RUNTIME_DURABLE_ADAPTER_MISSING,
|
|
gates: runtime?.gates ?? null,
|
|
queryAttempted: Boolean(runtime?.connection?.queryAttempted),
|
|
queryResult: runtime?.connection?.queryResult ?? "unknown",
|
|
secretMaterialRead: false
|
|
};
|
|
}
|
|
|
|
function buildProviderReadiness(codeAgent = {}) {
|
|
const fallback = codeAgent?.fallback ?? {};
|
|
const fallbackReady = fallback.ready === true || fallback.status === "available";
|
|
const tokenBoundary = codeAgent?.codexStdioFeasibility?.tokenBoundary ?? codeAgent?.codexStdio?.tokenBoundary ?? null;
|
|
const stdioTokenBoundaryReady = codeAgent?.mode === "codex-stdio" && tokenBoundary?.present === true && codeAgent?.egress?.directPublicOpenAi !== true;
|
|
const ready = fallbackReady || stdioTokenBoundaryReady;
|
|
const missingEnv = Array.isArray(codeAgent?.missingEnv) ? [...codeAgent.missingEnv] : [];
|
|
const blocker = ready ? null : providerBlockerCode(codeAgent, fallback);
|
|
return {
|
|
status: ready ? "ready" : "blocked",
|
|
ready,
|
|
configured: ready,
|
|
provider: codeAgent?.provider ?? fallback.provider ?? "unknown",
|
|
model: codeAgent?.model ?? fallback.model ?? "unknown",
|
|
backend: codeAgent?.backend ?? fallback.backend ?? "unknown",
|
|
mode: codeAgent?.mode ?? fallback.mode ?? "unknown",
|
|
evidence: "runtime_config_redacted",
|
|
egressReady: Boolean(fallback?.egress?.ready ?? codeAgent?.egress?.ready),
|
|
missingEnv,
|
|
blocker,
|
|
secretMaterialRead: false,
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
function buildSessionRunnerReadiness(codeAgent = {}) {
|
|
const stdio = codeAgent?.codexStdioFeasibility ?? codeAgent?.codexStdio ?? {};
|
|
const stdioCommandReady = stdio.commandProbe?.ready === true;
|
|
const stdioReady = (stdio.ready === true || stdio.canStartLongLivedCodexStdio === true) && stdioCommandReady;
|
|
const runner = codeAgent?.runner ?? {};
|
|
const runnerReady = stdioReady;
|
|
const status = stdioReady ? "codex_stdio_ready" : "blocked";
|
|
const blocker = runnerReady ? null : firstCode(
|
|
runner.blocker,
|
|
codeAgent?.reason,
|
|
codeAgent?.blockerCodes?.[0],
|
|
codeAgent?.blockers?.[0]?.code,
|
|
"session_runner_blocked"
|
|
);
|
|
return {
|
|
status,
|
|
ready: runnerReady,
|
|
kind: stdioReady ? stdio.kind ?? "codex-app-server-stdio-runner" : runner.kind ?? "unknown",
|
|
provider: stdioReady ? stdio.provider ?? "codex-stdio" : runner.provider ?? codeAgent?.provider ?? "unknown",
|
|
backend: stdioReady ? stdio.backend ?? "hwlab-cloud-api/codex-app-server-stdio" : runner.backend ?? codeAgent?.backend ?? "unknown",
|
|
mode: stdioReady ? "codex-app-server-stdio-long-lived" : runner.sessionMode ?? runner.mode ?? codeAgent?.sessionMode ?? "unknown",
|
|
capabilityLevel: stdioReady ? "long-lived-codex-stdio-session" : runner.capabilityLevel ?? codeAgent?.capabilityLevel ?? "unknown",
|
|
longLivedSession: stdioReady || runner.longLivedSession === true,
|
|
durableSession: stdioReady || runner.durableSession === true || runner.durable === true,
|
|
codexStdio: stdioReady || runner.codexStdio === true,
|
|
writeCapable: stdioReady || runner.writeCapable === true,
|
|
readOnly: !stdioReady && runner.codexStdio !== true && runner.writeCapable === false,
|
|
blocker,
|
|
runnerLimitations: Array.isArray(codeAgent?.runnerLimitations) ? [...codeAgent.runnerLimitations] : [],
|
|
secretMaterialRead: false
|
|
};
|
|
}
|
|
|
|
function buildCodexStdioReadiness(codeAgent = {}) {
|
|
const stdio = codeAgent?.codexStdioFeasibility ?? codeAgent?.codexStdio ?? {};
|
|
const commandReady = stdio.commandProbe?.ready === true;
|
|
const ready = (stdio.ready === true || stdio.canStartLongLivedCodexStdio === true) && commandReady;
|
|
const blockerCodes = Array.isArray(stdio.blockerCodes)
|
|
? [...stdio.blockerCodes]
|
|
: Array.isArray(stdio.blockers)
|
|
? stdio.blockers.map((blocker) => blocker?.code).filter(Boolean)
|
|
: [];
|
|
const effectiveBlockerCodes = [...blockerCodes];
|
|
if ((stdio.ready === true || stdio.canStartLongLivedCodexStdio === true) && !commandReady) {
|
|
effectiveBlockerCodes.unshift("codex_stdio_command_probe_failed");
|
|
}
|
|
const blocker = ready ? null : firstCode(effectiveBlockerCodes[0], stdio.blockers?.[0]?.code, codeAgent?.longLivedSessionGate?.blockers?.[0]?.code, "codex_stdio_feasibility_blocked");
|
|
return {
|
|
status: ready ? "ready" : "blocked",
|
|
ready,
|
|
feasible: ready,
|
|
blocker,
|
|
blockerCodes: effectiveBlockerCodes,
|
|
sourceIssue: stdio.sourceIssue ?? "pikasTech/HWLAB#275",
|
|
commandPresent: stdio.binaryOnPath === true,
|
|
commandExecutable: stdio.binary?.executable === true,
|
|
nativeDependencyPresent: stdio.binary?.nativeDependencyPresent === true,
|
|
commandProbeReady: commandReady,
|
|
commandProbe: stdio.commandProbe ?? null,
|
|
supervisorConfigured: stdio.supervisor?.configured === true,
|
|
workspaceReady: stdio.workspaceState?.exists === true && stdio.workspaceState?.readable === true,
|
|
tokenBoundaryPresent: stdio.tokenBoundary?.present === true,
|
|
egressReady: stdio.egress?.directPublicOpenAi !== true,
|
|
secretMaterialRead: false
|
|
};
|
|
}
|
|
|
|
function buildCodeAgentStructuredReadiness({ provider, dbDurable, sessionRunner, codexStdio, codeAgent } = {}) {
|
|
const currentBlockers = [
|
|
provider?.ready === false ? provider.blocker ?? "provider_config_blocked" : null,
|
|
dbDurable?.ready === false ? dbDurable.blocker ?? "runtime_durable_adapter_not_ready" : null,
|
|
sessionRunner?.ready === false ? sessionRunner.blocker ?? "session_runner_blocked" : null,
|
|
codexStdio?.ready === false ? codexStdio.blocker ?? "codex_stdio_feasibility_blocked" : null
|
|
].filter(Boolean);
|
|
const ready = provider?.ready === true &&
|
|
dbDurable?.ready === true &&
|
|
sessionRunner?.ready === true &&
|
|
codexStdio?.ready === true &&
|
|
codeAgent?.ready === true;
|
|
return {
|
|
status: ready ? "ready" : currentBlockers.length > 0 ? "blocked" : "degraded",
|
|
ready,
|
|
providerReady: provider?.ready === true,
|
|
durableDbReady: dbDurable?.ready === true,
|
|
sessionRunnerReady: sessionRunner?.ready === true,
|
|
codexStdioFeasible: codexStdio?.ready === true,
|
|
currentBlocker: currentBlockers[0] ?? null,
|
|
currentBlockers,
|
|
provider: {
|
|
status: provider?.status ?? "unknown",
|
|
ready: provider?.ready === true,
|
|
provider: provider?.provider ?? "unknown",
|
|
backend: provider?.backend ?? "unknown",
|
|
mode: provider?.mode ?? "unknown",
|
|
blocker: provider?.blocker ?? null
|
|
},
|
|
dbDurable: {
|
|
status: dbDurable?.status ?? "unknown",
|
|
ready: dbDurable?.ready === true,
|
|
adapter: dbDurable?.adapter ?? "unknown",
|
|
blocker: dbDurable?.blocker ?? null,
|
|
blockedLayer: dbDurable?.blockedLayer ?? null,
|
|
queryResult: dbDurable?.queryResult ?? "unknown"
|
|
},
|
|
sessionRunner: {
|
|
status: sessionRunner?.status ?? "unknown",
|
|
ready: sessionRunner?.ready === true,
|
|
kind: sessionRunner?.kind ?? "unknown",
|
|
mode: sessionRunner?.mode ?? "unknown",
|
|
capabilityLevel: sessionRunner?.capabilityLevel ?? "unknown",
|
|
longLivedSession: sessionRunner?.longLivedSession === true,
|
|
durableSession: sessionRunner?.durableSession === true,
|
|
codexStdio: sessionRunner?.codexStdio === true,
|
|
writeCapable: sessionRunner?.writeCapable === true,
|
|
blocker: sessionRunner?.blocker ?? null
|
|
},
|
|
codexStdio: {
|
|
status: codexStdio?.status ?? "unknown",
|
|
ready: codexStdio?.ready === true,
|
|
feasible: codexStdio?.feasible === true,
|
|
blocker: codexStdio?.blocker ?? null,
|
|
blockerCodes: Array.isArray(codexStdio?.blockerCodes) ? [...codexStdio.blockerCodes] : []
|
|
},
|
|
secretMaterialRead: false,
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
function providerBlockerCode(codeAgent = {}, fallback = {}) {
|
|
if (Array.isArray(codeAgent?.missingEnv) && codeAgent.missingEnv.length > 0) return "provider_config_blocked";
|
|
if (fallback?.egress?.directPublicOpenAi === true || codeAgent?.egress?.directPublicOpenAi === true) return "provider_egress_blocked";
|
|
return firstCode(fallback.reason, codeAgent?.reason, "provider_config_blocked");
|
|
}
|
|
|
|
function firstCode(...values) {
|
|
for (const value of values) {
|
|
if (typeof value === "string" && value.trim()) return value.trim();
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function classifyRuntimeDurabilityLayer(runtime = {}) {
|
|
const contractLayer = runtime?.durabilityContract?.blockedLayer;
|
|
if (typeof contractLayer === "string" && contractLayer.length > 0) {
|
|
return contractLayer;
|
|
}
|
|
|
|
const blocker = runtime?.blocker;
|
|
if (
|
|
blocker === RUNTIME_DURABLE_ADAPTER_MISSING ||
|
|
blocker === RUNTIME_DURABLE_ADAPTER_UNCONFIGURED ||
|
|
blocker === RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING
|
|
) {
|
|
return "adapter";
|
|
}
|
|
if (blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED) {
|
|
return "auth";
|
|
}
|
|
if (blocker === RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED) {
|
|
return "ssl";
|
|
}
|
|
if (blocker === RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED) {
|
|
return "schema";
|
|
}
|
|
if (blocker === RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED) {
|
|
return "migration";
|
|
}
|
|
if (blocker === RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED || runtime?.connection?.queryAttempted) {
|
|
return "durability_query";
|
|
}
|
|
return "adapter";
|
|
}
|