unidesk-code-queue-runner
989 lines
35 KiB
JavaScript
989 lines
35 KiB
JavaScript
const PASS_STATUSES = Object.freeze(["ok", "ready", "pass", "passed", "healthy", "available", "live"]);
|
|
const ERROR_STATUSES = Object.freeze(["error", "failed", "failure"]);
|
|
const RAW_DEGRADED_STATUSES = Object.freeze(["degraded"]);
|
|
const BLOCKED_STATUSES = Object.freeze(["blocked", "unavailable", "rejected"]);
|
|
const UNVERIFIED_STATUSES = Object.freeze(["", "unknown", "pending", "loading", "probing", "unverified", "not_observed"]);
|
|
const RUNTIME_DURABLE_READY_QUERY_RESULTS = Object.freeze([
|
|
"durable_readiness_ready",
|
|
"runtime_durable_ready",
|
|
"query_ready",
|
|
"readiness_ready",
|
|
"ready",
|
|
"ok",
|
|
"pass",
|
|
"passed"
|
|
]);
|
|
const RUNTIME_DURABLE_BLOCKED_QUERY_RESULTS = Object.freeze({
|
|
driver_missing: "runtime_durable_adapter_driver_missing",
|
|
ssl_negotiation_blocked: "runtime_durable_adapter_ssl_blocked",
|
|
auth_blocked: "runtime_durable_adapter_auth_blocked",
|
|
schema_blocked: "runtime_durable_adapter_schema_blocked",
|
|
migration_blocked: "runtime_durable_adapter_migration_blocked",
|
|
query_blocked: "runtime_durable_adapter_query_blocked",
|
|
not_ready: "runtime_durable_adapter_query_blocked"
|
|
});
|
|
const RUNTIME_DURABLE_GATE_LAYERS = Object.freeze(["ssl", "auth", "schema", "migration", "durability"]);
|
|
const READ_ONLY_REASON_CODES = Object.freeze([
|
|
"read_only",
|
|
"read-only",
|
|
"readonly",
|
|
"permission",
|
|
"permissions",
|
|
"forbidden",
|
|
"unauthorized",
|
|
"runtime_durable",
|
|
"runtime-durable",
|
|
"durability",
|
|
"durable",
|
|
"db_persistence",
|
|
"persistence",
|
|
"not-codex-stdio",
|
|
"not-write-capable",
|
|
"not-durable-session",
|
|
"text-chat-only",
|
|
"fallback:text-chat-only",
|
|
"codex_stdio_blocked_readonly_available",
|
|
"controlled_readonly_not_long_lived_stdio",
|
|
"codex_stdio_supervisor_disabled",
|
|
"security_blocked"
|
|
]);
|
|
|
|
export function classifyWorkbenchLiveStatus(live = {}) {
|
|
const probes = collectLiveStatusProbes(live);
|
|
const reachable = probes.some((probe) => probe.reachable);
|
|
if (!reachable) {
|
|
const failedProbe = probes.find((probe) => !probe.reachable) ?? probes[0];
|
|
return statusResult({
|
|
kind: "unverified",
|
|
label: "等待验证",
|
|
tone: "pending",
|
|
serviceId: failedProbe?.serviceId ?? "hwlab-cloud-api",
|
|
apiPath: failedProbe?.apiPath ?? "/health/live",
|
|
reasonCode: failedProbe?.reasonCode ?? "api_unverified",
|
|
detail: failedProbe
|
|
? `等待验证：${probeAttribution(failedProbe)}。`
|
|
: "等待同源 API 验证；主工作台不会把未探测状态当作通过。",
|
|
probes
|
|
});
|
|
}
|
|
|
|
const errorProbe = probes.find((probe) => probe.kind === "error");
|
|
if (errorProbe) {
|
|
return statusResult({
|
|
kind: "error",
|
|
label: "API 错误",
|
|
tone: "blocked",
|
|
serviceId: errorProbe.serviceId,
|
|
apiPath: errorProbe.apiPath,
|
|
reasonCode: errorProbe.reasonCode,
|
|
traceId: errorProbe.traceId,
|
|
detail: `错误：${probeAttribution(errorProbe)}。`,
|
|
probes
|
|
});
|
|
}
|
|
|
|
const readOnlyProbe = probes.find((probe) => probe.kind === "readonly");
|
|
if (readOnlyProbe) {
|
|
return statusResult({
|
|
kind: "readonly",
|
|
label: "只读模式",
|
|
tone: "source",
|
|
serviceId: readOnlyProbe.serviceId,
|
|
apiPath: readOnlyProbe.apiPath,
|
|
reasonCode: readOnlyProbe.reasonCode,
|
|
traceId: readOnlyProbe.traceId,
|
|
detail: `只读：${probeAttribution(readOnlyProbe)}。`,
|
|
probes
|
|
});
|
|
}
|
|
|
|
const pendingProbe = probes.find((probe) => probe.kind === "unverified");
|
|
if (pendingProbe) {
|
|
return statusResult({
|
|
kind: "unverified",
|
|
label: "等待验证",
|
|
tone: "pending",
|
|
serviceId: pendingProbe.serviceId,
|
|
apiPath: pendingProbe.apiPath,
|
|
reasonCode: pendingProbe.reasonCode,
|
|
traceId: pendingProbe.traceId,
|
|
detail: `等待验证：${probeAttribution(pendingProbe)}。`,
|
|
probes
|
|
});
|
|
}
|
|
|
|
return statusResult({
|
|
kind: "pass",
|
|
label: "API 正常",
|
|
tone: "dev-live",
|
|
serviceId: "hwlab-cloud-api",
|
|
apiPath: "/health/live",
|
|
reasonCode: "ready",
|
|
detail: "通过：hwlab-cloud-api /health/live ready；关键同源 API 已完成当前工作台只读验证。",
|
|
probes
|
|
});
|
|
}
|
|
|
|
export function collectLiveStatusProbes(live = {}) {
|
|
return [
|
|
classifyApiProbe(live.healthLive, {
|
|
serviceId: "hwlab-cloud-api",
|
|
apiPath: "/health/live",
|
|
fallbackReasonCode: "health_live_unverified"
|
|
}),
|
|
classifyApiProbe(live.restIndex, {
|
|
serviceId: "hwlab-cloud-api",
|
|
apiPath: "/v1",
|
|
fallbackReasonCode: "rest_index_unverified"
|
|
}),
|
|
classifyApiProbe(live.health, {
|
|
serviceId: "hwlab-cloud-api",
|
|
apiPath: "/json-rpc system.health",
|
|
fallbackReasonCode: "system_health_unverified"
|
|
}),
|
|
classifyApiProbe(live.adapter, {
|
|
serviceId: "hwlab-cloud-api",
|
|
apiPath: "/json-rpc cloud.adapter.describe",
|
|
fallbackReasonCode: "adapter_describe_unverified"
|
|
}),
|
|
classifyCodeAgentProbe(live),
|
|
classifyM3ControlProbe(live),
|
|
classifyM3StatusProbe(live)
|
|
].filter(Boolean);
|
|
}
|
|
|
|
function classifyApiProbe(probe, context) {
|
|
if (!probe) {
|
|
return unverifiedProbe({
|
|
...context,
|
|
reasonCode: context.fallbackReasonCode,
|
|
reason: "not_observed"
|
|
});
|
|
}
|
|
if (probe.ok !== true) {
|
|
return errorProbe({
|
|
...context,
|
|
reasonCode: probe.timeout ? "timeout" : probe.status ? `HTTP ${probe.status}` : context.fallbackReasonCode,
|
|
reason: probe.error ?? "request_failed",
|
|
httpStatus: probe.status,
|
|
traceId: traceIdFrom(probe),
|
|
evidenceSummary: evidenceSummaryFrom(probe)
|
|
});
|
|
}
|
|
|
|
const payload = probe.data;
|
|
const serviceId = payload?.serviceId ?? context.serviceId;
|
|
const durableReason = durableRuntimeReason(payload);
|
|
if (durableReason) {
|
|
return readOnlyProbe({
|
|
...context,
|
|
serviceId,
|
|
...durableReason,
|
|
rawStatus: rawStatusFrom(payload),
|
|
traceId: traceIdFrom(payload),
|
|
evidenceSummary: evidenceSummaryFrom(payload)
|
|
});
|
|
}
|
|
|
|
if (context.apiPath === "/v1" && payload?.codeAgent) {
|
|
const codeAgentProbe = classifyCodeAgentProbe({ restIndex: probe });
|
|
if (codeAgentProbe && codeAgentProbe.kind !== "pass" && codeAgentProbe.kind !== "unverified") {
|
|
return {
|
|
...codeAgentProbe,
|
|
rawStatus: rawStatusFrom(payload)
|
|
};
|
|
}
|
|
}
|
|
|
|
const blocker = firstBlocker(payload);
|
|
if (blocker) {
|
|
const reasonCode = String(blocker.code ?? blocker.reason ?? "blocked");
|
|
const reason = blocker.summary ?? blocker.message ?? blocker.reason ?? reasonCode;
|
|
if (isReadOnlyReason(reasonCode) || isReadOnlyReason(reason)) {
|
|
return readOnlyProbe({
|
|
...context,
|
|
serviceId,
|
|
reasonCode,
|
|
reason,
|
|
rawStatus: rawStatusFrom(payload),
|
|
traceId: traceIdFrom(payload),
|
|
evidenceSummary: evidenceSummaryFrom(payload)
|
|
});
|
|
}
|
|
return errorProbe({
|
|
...context,
|
|
serviceId,
|
|
reasonCode,
|
|
reason,
|
|
rawStatus: rawStatusFrom(payload),
|
|
traceId: traceIdFrom(payload),
|
|
evidenceSummary: evidenceSummaryFrom(payload)
|
|
});
|
|
}
|
|
|
|
const status = normalizedStatus(rawStatusFrom(payload));
|
|
if (ERROR_STATUSES.includes(status)) {
|
|
return errorProbe({
|
|
...context,
|
|
serviceId,
|
|
reasonCode: status,
|
|
reason: "API returned error status",
|
|
rawStatus: rawStatusFrom(payload),
|
|
traceId: traceIdFrom(payload),
|
|
evidenceSummary: evidenceSummaryFrom(payload)
|
|
});
|
|
}
|
|
if (RAW_DEGRADED_STATUSES.includes(status) || BLOCKED_STATUSES.includes(status) || readinessFalse(payload)) {
|
|
const reasonCode = readinessFalse(payload)
|
|
? "ready_false_without_blocker"
|
|
: status === "degraded"
|
|
? "api_degraded_without_attribution"
|
|
: status;
|
|
return errorProbe({
|
|
...context,
|
|
serviceId,
|
|
reasonCode,
|
|
reason: "API readiness is not pass and no concrete blocker was provided",
|
|
rawStatus: rawStatusFrom(payload),
|
|
traceId: traceIdFrom(payload),
|
|
evidenceSummary: evidenceSummaryFrom(payload)
|
|
});
|
|
}
|
|
if (UNVERIFIED_STATUSES.includes(status)) {
|
|
return unverifiedProbe({
|
|
...context,
|
|
serviceId,
|
|
reasonCode: context.fallbackReasonCode,
|
|
reason: "status_not_observed",
|
|
rawStatus: rawStatusFrom(payload),
|
|
traceId: traceIdFrom(payload),
|
|
evidenceSummary: evidenceSummaryFrom(payload)
|
|
});
|
|
}
|
|
return passProbe({
|
|
...context,
|
|
serviceId,
|
|
reasonCode: status || "ready",
|
|
reason: "ready",
|
|
rawStatus: rawStatusFrom(payload),
|
|
traceId: traceIdFrom(payload),
|
|
evidenceSummary: evidenceSummaryFrom(payload)
|
|
});
|
|
}
|
|
|
|
function classifyCodeAgentProbe(live = {}) {
|
|
const availability = live.restIndex?.data?.codeAgent ?? live.healthLive?.data?.codeAgent;
|
|
if (!availability) {
|
|
return unverifiedProbe({
|
|
serviceId: "hwlab-cloud-api",
|
|
apiPath: "/v1/agent/chat",
|
|
reasonCode: "code_agent_unverified",
|
|
reason: "availability_not_observed"
|
|
});
|
|
}
|
|
const serviceId = "hwlab-cloud-api";
|
|
const apiPath = availability.endpoint ?? "POST /v1/agent/chat";
|
|
const provider = availability.provider ?? "not_observed";
|
|
const backend = availability.backend ?? "not_observed";
|
|
const capability = availability.capabilityLevel ?? availability.runner?.capabilityLevel;
|
|
const providerStatus = availability.error?.providerStatus ?? availability.providerStatus;
|
|
const gate = availability.longLivedSessionGate;
|
|
const stdio = availability.codexStdio ?? availability.codexStdioFeasibility;
|
|
if (gate?.status === "pass" && stdio?.canStartLongLivedCodexStdio === true && availability.ready === true) {
|
|
return passProbe({
|
|
serviceId,
|
|
apiPath,
|
|
reasonCode: "codex_stdio_ready",
|
|
reason: `provider=codex-stdio; backend=${availability.codexStdio?.backend ?? backend}; capability=long-lived-codex-stdio-session`,
|
|
traceId: traceIdFrom(availability),
|
|
evidenceSummary: evidenceSummaryFrom(availability),
|
|
meta: { provider: "codex-stdio", backend: availability.codexStdio?.backend ?? backend, capability: "long-lived-codex-stdio-session" }
|
|
});
|
|
}
|
|
if (capability === "text-chat-only" || availability.runner?.kind === "openai-responses-fallback") {
|
|
return readOnlyProbe({
|
|
serviceId,
|
|
apiPath,
|
|
reasonCode: "fallback_text_chat_only",
|
|
reason: `provider=${provider}; backend=${backend}; capability=text-chat-only`,
|
|
traceId: traceIdFrom(availability),
|
|
evidenceSummary: evidenceSummaryFrom(availability),
|
|
meta: { provider, backend, capability }
|
|
});
|
|
}
|
|
if (availability.agentKind === "controlled-readonly-session-registry" || availability.partialReady === true) {
|
|
return readOnlyProbe({
|
|
serviceId,
|
|
apiPath,
|
|
reasonCode: "codex_stdio_blocked_readonly_available",
|
|
reason: `${availability.summary ?? "controlled-readonly-session-registry available; long-lived Codex stdio blocked"}; blockers=${blockerCodes(availability).join(",") || "not_observed"}`,
|
|
traceId: traceIdFrom(availability),
|
|
evidenceSummary: evidenceSummaryFrom(availability),
|
|
meta: { provider, backend, capability }
|
|
});
|
|
}
|
|
if (capability === "read-only-session-tools") {
|
|
return readOnlyProbe({
|
|
serviceId,
|
|
apiPath,
|
|
reasonCode: "code_agent_read_only_session_tools",
|
|
reason: codeAgentReadOnlySessionToolsReason(availability),
|
|
traceId: traceIdFrom(availability),
|
|
evidenceSummary: evidenceSummaryFrom(availability),
|
|
meta: { provider, backend, capability }
|
|
});
|
|
}
|
|
if (availability.status === "blocked" || availability.ready === false) {
|
|
return errorProbe({
|
|
serviceId,
|
|
apiPath,
|
|
reasonCode: providerStatus ? `provider_http_${providerStatus}` : availability.reason ?? availability.blocker ?? "code_agent_blocked",
|
|
reason: availability.summary ?? `provider=${provider}; backend=${backend}; capability=${capability ?? "unknown"}`,
|
|
traceId: traceIdFrom(availability),
|
|
meta: { provider, backend, capability }
|
|
});
|
|
}
|
|
if (availability.runner?.ready === true && availability.runner?.writeCapable === false) {
|
|
return readOnlyProbe({
|
|
serviceId,
|
|
apiPath,
|
|
reasonCode: "codex_readonly_runner",
|
|
reason: `runner=${availability.runner.kind ?? "hwlab-readonly-runner"}; capability=${capability ?? "read-only-tools"}`,
|
|
traceId: traceIdFrom(availability),
|
|
evidenceSummary: evidenceSummaryFrom(availability),
|
|
meta: { provider, backend, capability }
|
|
});
|
|
}
|
|
if (availability.status === "available" || availability.ready === true) {
|
|
return passProbe({
|
|
serviceId,
|
|
apiPath,
|
|
reasonCode: capability ?? "available",
|
|
reason: `provider=${provider}; backend=${backend}`,
|
|
traceId: traceIdFrom(availability),
|
|
evidenceSummary: evidenceSummaryFrom(availability),
|
|
meta: { provider, backend, capability }
|
|
});
|
|
}
|
|
return unverifiedProbe({
|
|
serviceId,
|
|
apiPath,
|
|
reasonCode: "code_agent_unverified",
|
|
reason: `provider=${provider}; backend=${backend}`,
|
|
traceId: traceIdFrom(availability),
|
|
evidenceSummary: evidenceSummaryFrom(availability),
|
|
meta: { provider, backend, capability }
|
|
});
|
|
}
|
|
|
|
function blockerCodes(availability) {
|
|
if (Array.isArray(availability?.blockerCodes)) return availability.blockerCodes.filter(Boolean);
|
|
if (Array.isArray(availability?.blockers)) return availability.blockers.map((blocker) => blocker?.code).filter(Boolean);
|
|
return [];
|
|
}
|
|
|
|
function classifyM3ControlProbe(live = {}) {
|
|
const probe = live.m3Control;
|
|
const context = {
|
|
serviceId: "hwlab-cloud-api",
|
|
apiPath: "/v1/m3/io",
|
|
fallbackReasonCode: "m3_io_unverified"
|
|
};
|
|
if (!probe) {
|
|
return unverifiedProbe({ ...context, reasonCode: context.fallbackReasonCode, reason: "not_observed" });
|
|
}
|
|
if (probe.ok !== true) {
|
|
return errorProbe({
|
|
...context,
|
|
reasonCode: probe.timeout ? "timeout" : probe.status ? `HTTP ${probe.status}` : "m3_io_request_failed",
|
|
reason: probe.error ?? "M3 IO readiness request failed",
|
|
httpStatus: probe.status,
|
|
traceId: traceIdFrom(probe)
|
|
});
|
|
}
|
|
const data = probe.data;
|
|
const readiness = data?.readiness ?? {};
|
|
if (data?.status === "available" && readiness.status === "ready" && readiness.controlReady === true) {
|
|
return passProbe({
|
|
...context,
|
|
reasonCode: "m3_io_ready",
|
|
reason: "cloud-api -> gateway-simu -> box-simu -> hwlab-patch-panel readiness ready",
|
|
rawStatus: data.status,
|
|
traceId: traceIdFrom(data),
|
|
evidenceSummary: evidenceSummaryFrom(data)
|
|
});
|
|
}
|
|
const blocker = readiness.blocker ?? data?.blocker;
|
|
return errorProbe({
|
|
...context,
|
|
serviceId: serviceForM3Layer(blocker?.layer) ?? context.serviceId,
|
|
reasonCode: blocker?.code ?? data?.status ?? "m3_io_not_ready",
|
|
reason: blocker?.zh ?? data?.blockedReason ?? "M3 IO readiness is not ready",
|
|
rawStatus: data?.status,
|
|
traceId: traceIdFrom(data),
|
|
evidenceSummary: evidenceSummaryFrom(data)
|
|
});
|
|
}
|
|
|
|
function classifyM3StatusProbe(live = {}) {
|
|
const probe = live.m3Status;
|
|
const context = {
|
|
serviceId: "hwlab-cloud-api",
|
|
apiPath: "/v1/m3/status",
|
|
fallbackReasonCode: "m3_status_unverified"
|
|
};
|
|
if (!probe) {
|
|
return unverifiedProbe({ ...context, reasonCode: context.fallbackReasonCode, reason: "not_observed" });
|
|
}
|
|
if (probe.ok !== true) {
|
|
return errorProbe({
|
|
...context,
|
|
reasonCode: probe.timeout ? "timeout" : probe.status ? `HTTP ${probe.status}` : "m3_status_request_failed",
|
|
reason: probe.error ?? "M3 status aggregation failed",
|
|
httpStatus: probe.status,
|
|
traceId: traceIdFrom(probe)
|
|
});
|
|
}
|
|
const data = probe.data;
|
|
const blocker = data?.blocker;
|
|
if (data?.status === "error") {
|
|
return errorProbe({
|
|
...context,
|
|
serviceId: serviceForM3Layer(blocker?.layer) ?? context.serviceId,
|
|
reasonCode: blocker?.code ?? "m3_status_error",
|
|
reason: blocker?.zh ?? data?.summary ?? "M3 status returned error",
|
|
rawStatus: data.status,
|
|
traceId: traceIdFrom(data),
|
|
evidenceSummary: evidenceSummaryFrom(data)
|
|
});
|
|
}
|
|
if (data?.status === "blocked" && blocker) {
|
|
const serviceId = serviceForM3Layer(blocker.layer) ?? (isReadOnlyReason(blocker.code) ? context.serviceId : context.serviceId);
|
|
const factory = isReadOnlyReason(blocker.code) || isReadOnlyReason(blocker.zh) ? readOnlyProbe : errorProbe;
|
|
return factory({
|
|
...context,
|
|
serviceId,
|
|
reasonCode: blocker.code ?? "m3_status_blocked",
|
|
reason: blocker.zh ?? data.summary ?? "M3 status blocked",
|
|
rawStatus: data.status,
|
|
traceId: traceIdFrom(data),
|
|
evidenceSummary: evidenceSummaryFrom(data)
|
|
});
|
|
}
|
|
if (data?.status === "blocked") {
|
|
return unverifiedProbe({
|
|
...context,
|
|
reasonCode: "m3_status_waiting_trusted_record",
|
|
reason: data.summary ?? "M3 status is waiting for trusted persistence evidence",
|
|
rawStatus: data.status,
|
|
traceId: traceIdFrom(data),
|
|
evidenceSummary: evidenceSummaryFrom(data)
|
|
});
|
|
}
|
|
if (data?.status === "live" || data?.sourceKind === "DEV-LIVE") {
|
|
return passProbe({
|
|
...context,
|
|
reasonCode: data.status ?? "dev_live",
|
|
reason: data.summary ?? "M3 status observed through cloud-api",
|
|
rawStatus: data.status,
|
|
traceId: traceIdFrom(data),
|
|
evidenceSummary: evidenceSummaryFrom(data)
|
|
});
|
|
}
|
|
return unverifiedProbe({
|
|
...context,
|
|
reasonCode: "m3_status_unverified",
|
|
reason: data?.summary ?? "M3 status not yet verified",
|
|
rawStatus: data?.status,
|
|
traceId: traceIdFrom(data),
|
|
evidenceSummary: evidenceSummaryFrom(data)
|
|
});
|
|
}
|
|
|
|
function durableRuntimeReason(payload) {
|
|
if (!hasRuntimeDurableSignal(payload)) return null;
|
|
const classification = classifyRuntimeDurableReadiness(payload);
|
|
if (classification.ready) return null;
|
|
if (
|
|
classification.status === "blocked" &&
|
|
(
|
|
isReadOnlyReason(classification.reasonCode) ||
|
|
String(classification.reasonCode).endsWith("_blocked") ||
|
|
normalizedStatus(classification.evidence?.blockedLayer).includes("durability")
|
|
)
|
|
) {
|
|
return {
|
|
reasonCode: classification.reasonCode,
|
|
reason: classification.reason,
|
|
layer: classification.evidence?.blockedLayer || "runtime-durable",
|
|
classification
|
|
};
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function hasRuntimeDurableSignal(payload = {}) {
|
|
const runtime = payload?.runtime;
|
|
const durability = payload?.readiness?.durability;
|
|
const dbRuntimeReadiness = payload?.db?.runtimeReadiness;
|
|
return Boolean(
|
|
blockerCodeStartingWith(payload, "runtime_durable_adapter_") ||
|
|
hasOwnNonEmpty(runtime, ["adapter", "durable", "durableRequested", "ready", "blocker", "liveRuntimeEvidence", "connection", "durabilityContract", "gates"]) ||
|
|
hasOwnNonEmpty(durability, ["adapter", "durable", "ready", "blocker", "blockedLayer", "queryResult", "liveRuntimeEvidence", "gates"]) ||
|
|
hasOwnNonEmpty(dbRuntimeReadiness, ["adapter", "durable", "ready", "blocker", "blockedLayer", "queryResult", "liveRuntimeEvidence"])
|
|
);
|
|
}
|
|
|
|
function hasOwnNonEmpty(object, fields) {
|
|
if (!object || typeof object !== "object") return false;
|
|
return fields.some((field) => Object.hasOwn(object, field) && object[field] !== undefined && object[field] !== null && object[field] !== "");
|
|
}
|
|
|
|
export function classifyRuntimeDurableReadiness(payload = {}, options = {}) {
|
|
const runtime = payload?.runtime ?? {};
|
|
const durability = payload?.readiness?.durability ?? {};
|
|
const dbRuntimeReadiness = payload?.db?.runtimeReadiness ?? {};
|
|
const gates = runtime?.gates ?? durability?.gates ?? {};
|
|
const gateBlocker = firstBlockedRuntimeGate(gates);
|
|
const queryResult = firstNonEmpty(
|
|
runtime?.connection?.queryResult,
|
|
durability?.queryResult,
|
|
dbRuntimeReadiness?.queryResult
|
|
);
|
|
const queryBlocker = runtimeDurableBlockerFromQueryResult(queryResult);
|
|
const readyQueryResult = isRuntimeDurableReadyQueryResult(queryResult);
|
|
const explicitDurabilityReady = durabilityReadySignal({
|
|
runtime,
|
|
durability,
|
|
dbRuntimeReadiness,
|
|
gateBlocker,
|
|
queryResult,
|
|
readyQueryResult,
|
|
queryBlocker
|
|
});
|
|
const directBlocker = firstNonEmpty(
|
|
explicitDurabilityReady ? "" : runtime?.blocker,
|
|
durability?.blocker,
|
|
dbRuntimeReadiness?.blocker,
|
|
explicitDurabilityReady ? "" : blockerCodeStartingWith(payload, "runtime_durable_adapter_")
|
|
);
|
|
const reasonCode = firstNonEmpty(directBlocker, gateBlocker?.blocker, queryBlocker);
|
|
const blockedLayer = firstNonEmpty(
|
|
runtime?.durabilityContract?.blockedLayer,
|
|
durability?.blockedLayer,
|
|
dbRuntimeReadiness?.blockedLayer,
|
|
gateBlocker?.layer,
|
|
runtimeDurableLayerFromBlocker(reasonCode),
|
|
runtimeDurableLayerFromQueryResult(queryResult)
|
|
);
|
|
const adapter = firstNonEmpty(runtime?.adapter, durability?.adapter, dbRuntimeReadiness?.adapter, "unknown");
|
|
const runtimeDurable = runtime?.durable === true || durability?.durable === true || dbRuntimeReadiness?.durable === true;
|
|
const runtimeExplicitlyNonDurable = runtime?.durable === false || durability?.durable === false || dbRuntimeReadiness?.durable === false;
|
|
const runtimeReady = runtime?.ready === true || durability?.runtimeReady === true || dbRuntimeReadiness?.ready === true;
|
|
const durabilityReady = durability?.ready === true || dbRuntimeReadiness?.ready === true || runtimeReady;
|
|
const liveRuntimeEvidence =
|
|
runtime?.liveRuntimeEvidence === true ||
|
|
durability?.liveRuntimeEvidence === true ||
|
|
dbRuntimeReadiness?.liveRuntimeEvidence === true;
|
|
const rawStatus = rawStatusFrom(payload);
|
|
const postgresRequired = options.requirePostgresAdapter === true;
|
|
const adapterReady = !postgresRequired || adapter === "postgres";
|
|
const ready = adapterReady &&
|
|
runtimeDurable &&
|
|
runtimeReady &&
|
|
durabilityReady &&
|
|
liveRuntimeEvidence &&
|
|
!reasonCode &&
|
|
!blockedLayer &&
|
|
(!queryResult || readyQueryResult);
|
|
|
|
if (ready) {
|
|
return {
|
|
status: "ready",
|
|
ready: true,
|
|
classification: "runtime_durable_ready",
|
|
reasonCode: "runtime_durable_ready",
|
|
reason: `runtime durable ready; adapter=${adapter}; queryResult=${queryResult || "not_observed"}`,
|
|
impact: "M3 durable postflight may run because runtime persistence readiness is green.",
|
|
safeNextAction: "Continue with repo-owned DEV CD postflight; do not read or print Secret values.",
|
|
retryable: true,
|
|
readonly: false,
|
|
evidence: runtimeDurableEvidence({
|
|
adapter,
|
|
runtimeDurable,
|
|
runtimeReady,
|
|
durabilityReady,
|
|
liveRuntimeEvidence,
|
|
blockedLayer: null,
|
|
queryResult,
|
|
readyQueryResult,
|
|
reasonCode: null,
|
|
gates,
|
|
rawStatus,
|
|
postgresRequired
|
|
})
|
|
};
|
|
}
|
|
|
|
const nonReadyReasonCode = firstNonEmpty(
|
|
reasonCode,
|
|
postgresRequired && adapter !== "postgres" ? "runtime_durable_adapter_not_postgres" : "",
|
|
runtimeExplicitlyNonDurable ? "runtime_durable_false" : "",
|
|
runtimeDurable ? "" : "runtime_durable_adapter_missing",
|
|
runtimeReady ? "" : "runtime_durable_runtime_not_ready",
|
|
liveRuntimeEvidence ? "" : "runtime_durable_live_evidence_missing",
|
|
isBlockedRuntimeStatus(rawStatus) ? rawStatus : "",
|
|
"runtime_durable_adapter_not_ready"
|
|
);
|
|
const layer = firstNonEmpty(
|
|
blockedLayer,
|
|
runtimeDurableLayerFromBlocker(nonReadyReasonCode),
|
|
runtimeExplicitlyNonDurable ? "runtime-durable" : "",
|
|
"runtime-durable"
|
|
);
|
|
return {
|
|
status: "blocked",
|
|
ready: false,
|
|
classification: nonReadyReasonCode,
|
|
reasonCode: nonReadyReasonCode,
|
|
reason: runtimeDurableBlockedReason({ reasonCode: nonReadyReasonCode, adapter, layer, queryResult }),
|
|
impact: "M3 true/false IO durable postflight is skipped or blocked so DEV CD cannot prove evidence persistence.",
|
|
safeNextAction: runtimeDurableSafeNextAction(nonReadyReasonCode, layer),
|
|
retryable: true,
|
|
readonly: true,
|
|
evidence: runtimeDurableEvidence({
|
|
adapter,
|
|
runtimeDurable,
|
|
runtimeReady,
|
|
durabilityReady,
|
|
liveRuntimeEvidence,
|
|
blockedLayer: layer,
|
|
queryResult,
|
|
readyQueryResult,
|
|
reasonCode: nonReadyReasonCode,
|
|
gates,
|
|
rawStatus,
|
|
postgresRequired
|
|
})
|
|
};
|
|
}
|
|
|
|
function durabilityReadySignal({
|
|
runtime,
|
|
durability,
|
|
dbRuntimeReadiness,
|
|
gateBlocker,
|
|
queryResult,
|
|
readyQueryResult,
|
|
queryBlocker
|
|
}) {
|
|
if (gateBlocker || queryBlocker) return false;
|
|
if (queryResult && !readyQueryResult) return false;
|
|
const ready = durability?.ready === true ||
|
|
dbRuntimeReadiness?.ready === true ||
|
|
runtime?.durabilityContract?.ready === true;
|
|
if (!ready) return false;
|
|
return !firstNonEmpty(
|
|
durability?.blocker,
|
|
dbRuntimeReadiness?.blocker,
|
|
runtime?.durabilityContract?.blocker,
|
|
durability?.blockedLayer,
|
|
dbRuntimeReadiness?.blockedLayer,
|
|
runtime?.durabilityContract?.blockedLayer
|
|
);
|
|
}
|
|
|
|
function isBlockedRuntimeStatus(status) {
|
|
return [...ERROR_STATUSES, ...BLOCKED_STATUSES, ...RAW_DEGRADED_STATUSES].includes(normalizedStatus(status));
|
|
}
|
|
|
|
function firstBlockedRuntimeGate(gates = {}) {
|
|
for (const layer of RUNTIME_DURABLE_GATE_LAYERS) {
|
|
const gate = gates?.[layer];
|
|
const status = normalizedStatus(gate?.status);
|
|
if (gate?.ready === false || status === "blocked" || status === "failed") {
|
|
return {
|
|
layer,
|
|
blocker: firstNonEmpty(gate?.blocker, runtimeDurableBlockerForLayer(layer))
|
|
};
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function runtimeDurableBlockerForLayer(layer) {
|
|
const text = normalizedStatus(layer);
|
|
if (text === "ssl") return "runtime_durable_adapter_ssl_blocked";
|
|
if (text === "auth") return "runtime_durable_adapter_auth_blocked";
|
|
if (text === "schema") return "runtime_durable_adapter_schema_blocked";
|
|
if (text === "migration") return "runtime_durable_adapter_migration_blocked";
|
|
if (text === "durability" || text === "durability_query") return "runtime_durable_adapter_query_blocked";
|
|
return "runtime_durable_adapter_blocked";
|
|
}
|
|
|
|
function runtimeDurableLayerFromBlocker(blocker) {
|
|
const text = normalizedStatus(blocker);
|
|
if (!text) return "";
|
|
if (text.includes("ssl")) return "ssl";
|
|
if (text.includes("auth")) return "auth";
|
|
if (text.includes("schema")) return "schema";
|
|
if (text.includes("migration")) return "migration";
|
|
if (text.includes("query")) return "durability_query";
|
|
if (text.includes("missing") || text.includes("unconfigured") || text.includes("driver")) return "adapter";
|
|
return "";
|
|
}
|
|
|
|
function runtimeDurableLayerFromQueryResult(queryResult) {
|
|
const text = normalizedStatus(queryResult);
|
|
if (!text || isRuntimeDurableReadyQueryResult(text)) return "";
|
|
if (text.includes("ssl")) return "ssl";
|
|
if (text.includes("auth")) return "auth";
|
|
if (text.includes("schema")) return "schema";
|
|
if (text.includes("migration")) return "migration";
|
|
if (text.includes("query")) return "durability_query";
|
|
if (text.includes("driver")) return "adapter";
|
|
return "";
|
|
}
|
|
|
|
function runtimeDurableBlockerFromQueryResult(queryResult) {
|
|
const text = normalizedStatus(queryResult);
|
|
if (!text || isRuntimeDurableReadyQueryResult(text)) return "";
|
|
return RUNTIME_DURABLE_BLOCKED_QUERY_RESULTS[text] ?? "";
|
|
}
|
|
|
|
function isRuntimeDurableReadyQueryResult(queryResult) {
|
|
const text = normalizedStatus(queryResult);
|
|
return RUNTIME_DURABLE_READY_QUERY_RESULTS.includes(text);
|
|
}
|
|
|
|
function runtimeDurableEvidence({
|
|
adapter,
|
|
runtimeDurable,
|
|
runtimeReady,
|
|
durabilityReady,
|
|
liveRuntimeEvidence,
|
|
blockedLayer,
|
|
queryResult,
|
|
readyQueryResult,
|
|
reasonCode,
|
|
gates,
|
|
rawStatus,
|
|
postgresRequired
|
|
}) {
|
|
return {
|
|
adapter,
|
|
durable: runtimeDurable,
|
|
runtimeReady,
|
|
durabilityReady,
|
|
liveRuntimeEvidence,
|
|
blocker: reasonCode,
|
|
blockedLayer,
|
|
queryResult: queryResult || null,
|
|
readyQueryResult,
|
|
rawStatus: rawStatus || null,
|
|
postgresRequired,
|
|
gates: summarizeRuntimeGateStatuses(gates),
|
|
secretMaterialRead: false
|
|
};
|
|
}
|
|
|
|
function summarizeRuntimeGateStatuses(gates = {}) {
|
|
return Object.fromEntries(RUNTIME_DURABLE_GATE_LAYERS.map((layer) => [
|
|
layer,
|
|
{
|
|
checked: gates?.[layer]?.checked === true,
|
|
ready: gates?.[layer]?.ready === true,
|
|
status: gates?.[layer]?.status ?? "unknown",
|
|
blocker: gates?.[layer]?.blocker ?? null
|
|
}
|
|
]));
|
|
}
|
|
|
|
function runtimeDurableBlockedReason({ reasonCode, adapter, layer, queryResult }) {
|
|
if (reasonCode === "runtime_durable_false") {
|
|
return `runtime durable=false; adapter=${adapter}; queryResult=${queryResult || "not_observed"}`;
|
|
}
|
|
if (reasonCode === "runtime_durable_adapter_not_postgres") {
|
|
return `runtime durable adapter is not postgres; adapter=${adapter}; queryResult=${queryResult || "not_observed"}`;
|
|
}
|
|
return `runtime durable blocked at ${layer || "runtime-durable"}; blocker=${reasonCode}; queryResult=${queryResult || "unknown"}`;
|
|
}
|
|
|
|
function runtimeDurableSafeNextAction(reasonCode, layer) {
|
|
const text = normalizedStatus(`${reasonCode} ${layer}`);
|
|
if (text.includes("auth")) return "Repair DEV DB auth/SecretRef metadata through repo-owned provisioning, then rerun postflight without printing Secret values.";
|
|
if (text.includes("schema") || text.includes("migration")) return "Run repo-owned DEV runtime provisioning and migration, then rerun postflight.";
|
|
if (text.includes("ssl")) return "Align DEV DB SSL mode with the repo-owned DEV contract, then rerun postflight.";
|
|
if (text.includes("query")) return "Inspect durable runtime query readiness, preserve evidence persistence, and rerun postflight.";
|
|
if (text.includes("postgres")) return "Run DEV with HWLAB_CLOUD_RUNTIME_ADAPTER=postgres and DB URLs injected only through SecretRefs.";
|
|
return "Restore durable runtime readiness through repo-owned DEV CD steps, then rerun postflight.";
|
|
}
|
|
|
|
function firstBlocker(payload) {
|
|
const blockers = [
|
|
...(Array.isArray(payload?.blockers) ? payload.blockers : []),
|
|
...(Array.isArray(payload?.readiness?.blockers) ? payload.readiness.blockers : [])
|
|
].filter(Boolean);
|
|
if (blockers.length > 0) return blockers[0];
|
|
const code = firstNonEmpty(
|
|
...(Array.isArray(payload?.blockerCodes) ? payload.blockerCodes : []),
|
|
...(Array.isArray(payload?.readiness?.blockerCodes) ? payload.readiness.blockerCodes : [])
|
|
);
|
|
return code ? { code, summary: code } : null;
|
|
}
|
|
|
|
function blockerCodeStartingWith(payload, prefix) {
|
|
return [
|
|
...(Array.isArray(payload?.blockerCodes) ? payload.blockerCodes : []),
|
|
...(Array.isArray(payload?.readiness?.blockerCodes) ? payload.readiness.blockerCodes : [])
|
|
].find((code) => String(code ?? "").startsWith(prefix));
|
|
}
|
|
|
|
function readinessFalse(payload) {
|
|
return [
|
|
payload?.ready,
|
|
payload?.readiness?.ready,
|
|
payload?.readiness?.durability?.ready,
|
|
payload?.runtime?.ready
|
|
].some((value) => value === false);
|
|
}
|
|
|
|
function rawStatusFrom(payload) {
|
|
return payload?.status ?? payload?.readiness?.status ?? payload?.runtime?.status ?? payload?.readiness?.durability?.status ?? "";
|
|
}
|
|
|
|
function statusResult(result) {
|
|
return {
|
|
serviceId: null,
|
|
apiPath: null,
|
|
reasonCode: null,
|
|
traceId: null,
|
|
internalRawStatuses: result.probes
|
|
.map((probe) => probe.rawStatus)
|
|
.filter((status) => status !== undefined && status !== null && String(status).trim() !== ""),
|
|
...result
|
|
};
|
|
}
|
|
|
|
function passProbe(options) {
|
|
return probe({ kind: "pass", reachable: true, ...options });
|
|
}
|
|
|
|
function readOnlyProbe(options) {
|
|
return probe({ kind: "readonly", reachable: true, ...options });
|
|
}
|
|
|
|
function errorProbe(options) {
|
|
return probe({ kind: "error", reachable: Boolean(options.httpStatus ? options.httpStatus >= 200 && options.httpStatus < 600 : true), ...options });
|
|
}
|
|
|
|
function unverifiedProbe(options) {
|
|
return probe({ kind: "unverified", reachable: false, ...options });
|
|
}
|
|
|
|
function probe(options) {
|
|
return {
|
|
kind: options.kind,
|
|
reachable: options.reachable,
|
|
serviceId: options.serviceId,
|
|
apiPath: options.apiPath,
|
|
reasonCode: String(options.reasonCode ?? options.kind),
|
|
reason: String(options.reason ?? options.reasonCode ?? options.kind),
|
|
httpStatus: options.httpStatus ?? null,
|
|
rawStatus: options.rawStatus ?? null,
|
|
traceId: options.traceId ?? null,
|
|
evidenceSummary: options.evidenceSummary ?? null,
|
|
meta: options.meta ?? null
|
|
};
|
|
}
|
|
|
|
function probeAttribution(probe) {
|
|
return [
|
|
`${probe.serviceId} ${probe.apiPath}`,
|
|
probe.httpStatus ? `HTTP ${probe.httpStatus}` : null,
|
|
displayReasonCode(probe.reasonCode),
|
|
probe.reason && probe.reason !== probe.reasonCode ? displayReasonText(probe.reason) : null,
|
|
probe.traceId ? `trace=${probe.traceId}` : null,
|
|
probe.evidenceSummary
|
|
].filter(Boolean).join(" / ");
|
|
}
|
|
|
|
function displayReasonCode(reasonCode) {
|
|
const text = String(reasonCode ?? "");
|
|
return text.startsWith("runtime_durable_adapter_") ? "runtime durable" : text;
|
|
}
|
|
|
|
function displayReasonText(reason) {
|
|
return String(reason ?? "").replace(/\bruntime_durable_adapter_[a-z0-9_]+\b/giu, "runtime durable");
|
|
}
|
|
|
|
function normalizedStatus(value) {
|
|
return String(value ?? "").trim().toLowerCase();
|
|
}
|
|
|
|
function isReadOnlyReason(value) {
|
|
const text = normalizedStatus(value);
|
|
return READ_ONLY_REASON_CODES.some((code) => text.includes(code));
|
|
}
|
|
|
|
function codeAgentReadOnlySessionToolsReason(availability) {
|
|
const flags = [
|
|
...(Array.isArray(availability?.runnerLimitations) ? availability.runnerLimitations : []),
|
|
...(Array.isArray(availability?.runner?.limitations) ? availability.runner.limitations : [])
|
|
].filter(Boolean);
|
|
const issue = firstNonEmpty(availability?.issue, availability?.trackingIssue, "#317");
|
|
return `capability=read-only-session-tools; ${issue} full pass not complete${flags.length > 0 ? `; ${flags.join(",")}` : ""}`;
|
|
}
|
|
|
|
function traceIdFrom(value) {
|
|
return firstNonEmpty(
|
|
value?.traceId,
|
|
value?.meta?.traceId,
|
|
value?.data?.traceId,
|
|
value?.data?.meta?.traceId,
|
|
value?.data?.trust?.traceId,
|
|
value?.request?.traceId,
|
|
value?.operation?.traceId,
|
|
value?.result?.traceId,
|
|
value?.trust?.traceId,
|
|
value?.evidenceState?.traceId,
|
|
value?.metadata?.traceId,
|
|
value?.error?.traceId,
|
|
value?.session?.lastTraceId,
|
|
value?.session?.currentTraceId,
|
|
value?.runnerTrace?.traceId,
|
|
value?.providerTrace?.traceId
|
|
);
|
|
}
|
|
|
|
function evidenceSummaryFrom(value) {
|
|
const payload = value?.data && typeof value.data === "object" ? value.data : value;
|
|
const parts = [
|
|
["operation", firstNonEmpty(payload?.operationId, payload?.operation?.operationId, payload?.result?.operationId, payload?.trust?.operationId)],
|
|
["audit", firstNonEmpty(payload?.auditId, payload?.audit?.auditId, payload?.auditEvent?.auditId, payload?.trust?.auditId)],
|
|
["evidence", firstNonEmpty(payload?.evidenceId, payload?.evidence?.evidenceId, payload?.evidenceRecord?.evidenceId, payload?.trust?.evidenceId)],
|
|
["session", firstNonEmpty(payload?.session?.sessionId, payload?.sessionId)]
|
|
]
|
|
.filter(([, id]) => id)
|
|
.slice(0, 4)
|
|
.map(([label, id]) => `${label}=${id}`);
|
|
return parts.length > 0 ? parts.join(" ") : null;
|
|
}
|
|
|
|
function serviceForM3Layer(layer) {
|
|
const text = normalizedStatus(layer);
|
|
if (!text) return null;
|
|
if (text.includes("patch")) return "hwlab-patch-panel";
|
|
if (text.includes("gateway")) return "hwlab-gateway-simu";
|
|
if (text.includes("box") || text.includes("resource")) return "hwlab-box-simu";
|
|
if (text.includes("runtime") || text.includes("durable")) return "hwlab-cloud-api";
|
|
return null;
|
|
}
|
|
|
|
function firstNonEmpty(...values) {
|
|
for (const value of values) {
|
|
if (value !== undefined && value !== null && String(value).trim() !== "") return String(value).trim();
|
|
}
|
|
return "";
|
|
}
|