Files
pikasTech-HWLAB/internal/cloud/health-contract.ts
T

404 lines
17 KiB
TypeScript

import {
RUNTIME_DURABLE_ADAPTER_MISSING,
RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED,
RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING,
RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED,
RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED,
RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED,
RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED,
RUNTIME_DURABLE_ADAPTER_UNCONFIGURED,
RUNTIME_DURABILITY_REQUIRED_EVIDENCE,
RUNTIME_STORE_KIND_POSTGRES
} from "../db/runtime-store.ts";
export const CLOUD_API_HEALTH_CONTRACT_VERSION = "v3";
export function buildCloudApiReadiness({ db, codeAgent, runtime } = {}) {
const dbReady = isDbReady(db);
const codeAgentReady = isCodeAgentReady(codeAgent);
const runtimeReady = isRuntimeReady(runtime);
const blockers = [];
let runtimeReadinessBlocker = null;
if (!dbReady) {
blockers.push(dbBlocker(db));
}
if (!codeAgentReady) {
blockers.push(codeAgentBlocker(codeAgent));
}
if (!runtimeReady) {
runtimeReadinessBlocker = runtimeBlocker(runtime);
blockers.push(runtimeReadinessBlocker);
}
const provider = buildProviderReadiness(codeAgent);
const dbDurable = buildRuntimeDurabilityReadiness({
db,
runtime,
runtimeReady,
runtimeBlocker: runtimeReadinessBlocker
});
const sessionRunner = buildSessionRunnerReadiness(codeAgent);
const codexStdio = buildCodexStdioReadiness(codeAgent);
const codeAgentReadiness = buildCodeAgentStructuredReadiness({
provider,
dbDurable,
sessionRunner,
codexStdio,
codeAgent
});
return {
contractVersion: CLOUD_API_HEALTH_CONTRACT_VERSION,
ready: blockers.length === 0,
status: blockers.length === 0 ? "ok" : "degraded",
blockerCodes: blockers.map((blocker) => blocker.code),
blockers,
components: {
db: dbReady ? "ready" : "blocked",
codeAgent: codeAgentReady ? "ready" : "blocked",
runtime: runtimeReady ? "ready" : dbReady ? "blocked" : "waiting_for_db",
provider: provider.status,
dbDurable: dbDurable.status,
sessionRunner: sessionRunner.status,
codexStdio: codexStdio.status
},
provider,
dbDurable,
sessionRunner,
codexStdio,
codeAgent: codeAgentReadiness,
durability: dbDurable
};
}
function isDbReady(db) {
return Boolean(db?.ready && db?.connected && db?.liveConnected !== false && db?.liveDbEvidence);
}
function isCodeAgentReady(codeAgent) {
return Boolean(
codeAgent?.ready === true &&
codeAgent?.longLivedSessionGate?.status === "pass" &&
codeAgent?.codexStdioFeasibility?.canStartLongLivedCodexStdio === true
);
}
function isRuntimeReady(runtime) {
if (runtime?.durable !== true) {
return false;
}
if (runtime?.ready !== true || runtime?.liveRuntimeEvidence !== true) {
return false;
}
return !["blocked", "degraded", "failed"].includes(runtime.status);
}
function dbBlocker(db) {
return {
code: db?.configReady ? "db_live_connection_blocked" : "db_runtime_config_blocked",
type: "runtime_blocker",
scope: "cloud-api-db",
status: "open",
sourceIssue: "pikasTech/HWLAB#49",
summary: db?.blocker ?? "cloud-api DB live readiness is not established",
evidence: {
configReady: Boolean(db?.configReady),
connected: Boolean(db?.connected),
liveConnected: Boolean(db?.liveConnected),
liveDbEvidence: Boolean(db?.liveDbEvidence),
connectionAttempted: Boolean(db?.connectionAttempted ?? db?.connectionChecked),
connectionResult: db?.connectionResult ?? db?.connection?.result ?? "unknown"
}
};
}
function codeAgentBlocker(codeAgent) {
const primaryBlocker = Array.isArray(codeAgent?.blockers) ? codeAgent.blockers[0] : null;
return {
code: codeAgent?.reason ?? primaryBlocker?.code ?? "code_agent_long_lived_session_blocked",
type: "agent_blocker",
scope: "code-agent-long-lived-session",
status: "open",
sourceIssue: primaryBlocker?.sourceIssue ?? "pikasTech/HWLAB#275",
summary: codeAgent?.summary ?? "Code Agent provider readiness is not established",
evidence: {
provider: codeAgent?.provider ?? "unknown",
backend: codeAgent?.backend ?? "unknown",
agentKind: codeAgent?.agentKind ?? "unknown",
capabilityStatus: codeAgent?.capabilityStatus ?? "unknown",
longLivedSessionGate: codeAgent?.longLivedSessionGate ?? null,
codexStdioFeasibility: codeAgent?.codexStdioFeasibility ?? null,
partialReady: codeAgent?.partialReady === true,
missingEnv: Array.isArray(codeAgent?.missingEnv) ? [...codeAgent.missingEnv] : [],
secretsRedacted: true
}
};
}
function runtimeBlocker(runtime) {
const durable = runtime?.durable === true;
const durableRequested = runtime?.durableRequested === true || runtime?.adapter === RUNTIME_STORE_KIND_POSTGRES;
const unhealthy = (durable || durableRequested) && ["blocked", "degraded", "failed"].includes(runtime?.status);
const blocker = runtime?.blocker;
const blockedLayer = classifyRuntimeDurabilityLayer(runtime);
return {
code: durableRequested && unhealthy ? blocker ?? "runtime_durable_adapter_unhealthy" : RUNTIME_DURABLE_ADAPTER_MISSING,
type: "runtime_blocker",
scope: "runtime-durable-adapter",
status: "open",
sourceIssue: "pikasTech/HWLAB#164",
summary: durableRequested && unhealthy
? runtime?.reason ?? "cloud-api durable runtime adapter is configured but not ready"
: "cloud-api DB is live, but L1 runtime writes still use process-local memory because no DB-backed durable adapter is configured",
evidence: {
adapter: runtime?.adapter ?? "unknown",
durable: Boolean(runtime?.durable),
durableRequested: Boolean(runtime?.durableRequested),
durableCapable: Boolean(runtime?.durableCapable),
runtimeStatus: runtime?.status ?? "unknown",
runtimeReady: Boolean(runtime?.ready),
liveRuntimeEvidence: Boolean(runtime?.liveRuntimeEvidence),
schemaReady: Boolean(runtime?.schema?.ready),
schemaChecked: Boolean(runtime?.schema?.checked),
migrationReady: Boolean(runtime?.migration?.ready),
migrationChecked: Boolean(runtime?.migration?.checked),
gates: runtime?.gates ?? null,
blockedLayer,
queryAttempted: Boolean(runtime?.connection?.queryAttempted),
queryResult: runtime?.connection?.queryResult ?? "unknown",
dbBackedAdapterRequired: true,
dbLiveEvidenceIsDurabilityEvidence: false,
requiredEvidence: runtime?.durabilityContract?.requiredEvidence ?? RUNTIME_DURABILITY_REQUIRED_EVIDENCE,
secretMaterialRead: false
},
nextTask: durableRequested
? "Apply/repair the runtime DB schema or DB auth, then rerun /health/live without printing DB secret material."
: "Set HWLAB_CLOUD_RUNTIME_ADAPTER=postgres after the runtime DB schema is available; keep HWLAB_CLOUD_DB_URL injected from Secret."
};
}
function buildRuntimeDurabilityReadiness({ db, runtime, runtimeReady, runtimeBlocker } = {}) {
const dbLiveEvidenceObserved = Boolean(db?.ready && db?.connected && db?.liveConnected !== false && db?.liveDbEvidence);
const contract = runtime?.durabilityContract ?? {};
return {
status: runtimeReady ? "ready" : "blocked",
ready: runtimeReady,
adapter: runtime?.adapter ?? "unknown",
durable: Boolean(runtime?.durable),
durableRequested: Boolean(runtime?.durableRequested || runtime?.adapter === RUNTIME_STORE_KIND_POSTGRES),
durableCapable: Boolean(runtime?.durableCapable),
runtimeStatus: runtime?.status ?? "unknown",
runtimeReady: Boolean(runtime?.ready),
liveRuntimeEvidence: Boolean(runtime?.liveRuntimeEvidence),
dbLiveEvidenceObserved,
dbLiveEvidenceIsDurabilityEvidence: false,
requiredEvidence: contract.requiredEvidence ?? RUNTIME_DURABILITY_REQUIRED_EVIDENCE,
blockedLayer: runtimeReady ? null : contract.blockedLayer ?? classifyRuntimeDurabilityLayer(runtime),
blocker: runtimeReady ? null : runtimeBlocker?.code ?? runtime?.blocker ?? RUNTIME_DURABLE_ADAPTER_MISSING,
gates: runtime?.gates ?? null,
queryAttempted: Boolean(runtime?.connection?.queryAttempted),
queryResult: runtime?.connection?.queryResult ?? "unknown",
secretMaterialRead: false
};
}
function buildProviderReadiness(codeAgent = {}) {
const fallback = codeAgent?.fallback ?? {};
const fallbackReady = fallback.ready === true || fallback.status === "available";
const tokenBoundary = codeAgent?.codexStdioFeasibility?.tokenBoundary ?? codeAgent?.codexStdio?.tokenBoundary ?? null;
const stdioTokenBoundaryReady = codeAgent?.mode === "codex-stdio" && tokenBoundary?.present === true && codeAgent?.egress?.directPublicOpenAi !== true;
const ready = fallbackReady || stdioTokenBoundaryReady;
const missingEnv = Array.isArray(codeAgent?.missingEnv) ? [...codeAgent.missingEnv] : [];
const blocker = ready ? null : providerBlockerCode(codeAgent, fallback);
return {
status: ready ? "ready" : "blocked",
ready,
configured: ready,
provider: codeAgent?.provider ?? fallback.provider ?? "unknown",
model: codeAgent?.model ?? fallback.model ?? "unknown",
backend: codeAgent?.backend ?? fallback.backend ?? "unknown",
mode: codeAgent?.mode ?? fallback.mode ?? "unknown",
evidence: "runtime_config_redacted",
egressReady: Boolean(fallback?.egress?.ready ?? codeAgent?.egress?.ready),
missingEnv,
blocker,
secretMaterialRead: false,
valuesRedacted: true
};
}
function buildSessionRunnerReadiness(codeAgent = {}) {
const stdio = codeAgent?.codexStdioFeasibility ?? codeAgent?.codexStdio ?? {};
const stdioCommandReady = stdio.commandProbe?.ready === true;
const stdioReady = (stdio.ready === true || stdio.canStartLongLivedCodexStdio === true) && stdioCommandReady;
const runner = codeAgent?.runner ?? {};
const runnerReady = stdioReady;
const status = stdioReady ? "codex_stdio_ready" : "blocked";
const blocker = runnerReady ? null : firstCode(
runner.blocker,
codeAgent?.reason,
codeAgent?.blockerCodes?.[0],
codeAgent?.blockers?.[0]?.code,
"session_runner_blocked"
);
return {
status,
ready: runnerReady,
kind: stdioReady ? stdio.kind ?? "codex-app-server-stdio-runner" : runner.kind ?? "unknown",
provider: stdioReady ? stdio.provider ?? "codex-stdio" : runner.provider ?? codeAgent?.provider ?? "unknown",
backend: stdioReady ? stdio.backend ?? "hwlab-cloud-api/codex-app-server-stdio" : runner.backend ?? codeAgent?.backend ?? "unknown",
mode: stdioReady ? "codex-app-server-stdio-long-lived" : runner.sessionMode ?? runner.mode ?? codeAgent?.sessionMode ?? "unknown",
capabilityLevel: stdioReady ? "long-lived-codex-stdio-session" : runner.capabilityLevel ?? codeAgent?.capabilityLevel ?? "unknown",
longLivedSession: stdioReady || runner.longLivedSession === true,
durableSession: stdioReady || runner.durableSession === true || runner.durable === true,
codexStdio: stdioReady || runner.codexStdio === true,
writeCapable: stdioReady || runner.writeCapable === true,
readOnly: !stdioReady && runner.codexStdio !== true && runner.writeCapable === false,
blocker,
runnerLimitations: Array.isArray(codeAgent?.runnerLimitations) ? [...codeAgent.runnerLimitations] : [],
secretMaterialRead: false
};
}
function buildCodexStdioReadiness(codeAgent = {}) {
const stdio = codeAgent?.codexStdioFeasibility ?? codeAgent?.codexStdio ?? {};
const commandReady = stdio.commandProbe?.ready === true;
const ready = (stdio.ready === true || stdio.canStartLongLivedCodexStdio === true) && commandReady;
const blockerCodes = Array.isArray(stdio.blockerCodes)
? [...stdio.blockerCodes]
: Array.isArray(stdio.blockers)
? stdio.blockers.map((blocker) => blocker?.code).filter(Boolean)
: [];
const effectiveBlockerCodes = [...blockerCodes];
if ((stdio.ready === true || stdio.canStartLongLivedCodexStdio === true) && !commandReady) {
effectiveBlockerCodes.unshift("codex_stdio_command_probe_failed");
}
const blocker = ready ? null : firstCode(effectiveBlockerCodes[0], stdio.blockers?.[0]?.code, codeAgent?.longLivedSessionGate?.blockers?.[0]?.code, "codex_stdio_feasibility_blocked");
return {
status: ready ? "ready" : "blocked",
ready,
feasible: ready,
blocker,
blockerCodes: effectiveBlockerCodes,
sourceIssue: stdio.sourceIssue ?? "pikasTech/HWLAB#275",
commandPresent: stdio.binaryOnPath === true,
commandExecutable: stdio.binary?.executable === true,
nativeDependencyPresent: stdio.binary?.nativeDependencyPresent === true,
commandProbeReady: commandReady,
commandProbe: stdio.commandProbe ?? null,
supervisorConfigured: stdio.supervisor?.configured === true,
workspaceReady: stdio.workspaceState?.exists === true && stdio.workspaceState?.readable === true,
tokenBoundaryPresent: stdio.tokenBoundary?.present === true,
egressReady: stdio.egress?.directPublicOpenAi !== true,
secretMaterialRead: false
};
}
function buildCodeAgentStructuredReadiness({ provider, dbDurable, sessionRunner, codexStdio, codeAgent } = {}) {
const currentBlockers = [
provider?.ready === false ? provider.blocker ?? "provider_config_blocked" : null,
dbDurable?.ready === false ? dbDurable.blocker ?? "runtime_durable_adapter_not_ready" : null,
sessionRunner?.ready === false ? sessionRunner.blocker ?? "session_runner_blocked" : null,
codexStdio?.ready === false ? codexStdio.blocker ?? "codex_stdio_feasibility_blocked" : null
].filter(Boolean);
const ready = provider?.ready === true &&
dbDurable?.ready === true &&
sessionRunner?.ready === true &&
codexStdio?.ready === true &&
codeAgent?.ready === true;
return {
status: ready ? "ready" : currentBlockers.length > 0 ? "blocked" : "degraded",
ready,
providerReady: provider?.ready === true,
durableDbReady: dbDurable?.ready === true,
sessionRunnerReady: sessionRunner?.ready === true,
codexStdioFeasible: codexStdio?.ready === true,
currentBlocker: currentBlockers[0] ?? null,
currentBlockers,
provider: {
status: provider?.status ?? "unknown",
ready: provider?.ready === true,
provider: provider?.provider ?? "unknown",
backend: provider?.backend ?? "unknown",
mode: provider?.mode ?? "unknown",
blocker: provider?.blocker ?? null
},
dbDurable: {
status: dbDurable?.status ?? "unknown",
ready: dbDurable?.ready === true,
adapter: dbDurable?.adapter ?? "unknown",
blocker: dbDurable?.blocker ?? null,
blockedLayer: dbDurable?.blockedLayer ?? null,
queryResult: dbDurable?.queryResult ?? "unknown"
},
sessionRunner: {
status: sessionRunner?.status ?? "unknown",
ready: sessionRunner?.ready === true,
kind: sessionRunner?.kind ?? "unknown",
mode: sessionRunner?.mode ?? "unknown",
capabilityLevel: sessionRunner?.capabilityLevel ?? "unknown",
longLivedSession: sessionRunner?.longLivedSession === true,
durableSession: sessionRunner?.durableSession === true,
codexStdio: sessionRunner?.codexStdio === true,
writeCapable: sessionRunner?.writeCapable === true,
blocker: sessionRunner?.blocker ?? null
},
codexStdio: {
status: codexStdio?.status ?? "unknown",
ready: codexStdio?.ready === true,
feasible: codexStdio?.feasible === true,
blocker: codexStdio?.blocker ?? null,
blockerCodes: Array.isArray(codexStdio?.blockerCodes) ? [...codexStdio.blockerCodes] : []
},
secretMaterialRead: false,
valuesRedacted: true
};
}
function providerBlockerCode(codeAgent = {}, fallback = {}) {
if (Array.isArray(codeAgent?.missingEnv) && codeAgent.missingEnv.length > 0) return "provider_config_blocked";
if (fallback?.egress?.directPublicOpenAi === true || codeAgent?.egress?.directPublicOpenAi === true) return "provider_egress_blocked";
return firstCode(fallback.reason, codeAgent?.reason, "provider_config_blocked");
}
function firstCode(...values) {
for (const value of values) {
if (typeof value === "string" && value.trim()) return value.trim();
}
return null;
}
function classifyRuntimeDurabilityLayer(runtime = {}) {
const contractLayer = runtime?.durabilityContract?.blockedLayer;
if (typeof contractLayer === "string" && contractLayer.length > 0) {
return contractLayer;
}
const blocker = runtime?.blocker;
if (
blocker === RUNTIME_DURABLE_ADAPTER_MISSING ||
blocker === RUNTIME_DURABLE_ADAPTER_UNCONFIGURED ||
blocker === RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING
) {
return "adapter";
}
if (blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED) {
return "auth";
}
if (blocker === RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED) {
return "ssl";
}
if (blocker === RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED) {
return "schema";
}
if (blocker === RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED) {
return "migration";
}
if (blocker === RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED || runtime?.connection?.queryAttempted) {
return "durability_query";
}
return "adapter";
}