fix: report current M4 M5 runtime blockers
This commit is contained in:
+128
-103
@@ -23,10 +23,12 @@ import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT, ENVIRONMENT_DEV } from "../interna
|
||||
import {
|
||||
collectD601K3sNativeEvidence,
|
||||
collectPublicEntrypoints,
|
||||
classifyCloudApiLiveReadiness,
|
||||
d601K3sReadonlyCommand,
|
||||
d601Kubeconfig,
|
||||
devNamespace,
|
||||
requestJson,
|
||||
runtimeDurabilityFromHealth,
|
||||
workerServerDryRunCommand
|
||||
} from "./src/dev-m4-agent-loop-smoke-lib.mjs";
|
||||
|
||||
@@ -78,6 +80,14 @@ function statusForResult(result) {
|
||||
return result.blocked || (result.additionalBlockers?.length ?? 0) > 0 ? "blocked" : "pass";
|
||||
}
|
||||
|
||||
function dbLiveReadyForResult(result) {
|
||||
return result.dbLiveReady === true || (!result.blocked && result.dbLiveReady !== false);
|
||||
}
|
||||
|
||||
function runtimeDurableReadyForResult(result) {
|
||||
return result.runtimeDurableReady === true || (!result.blocked && result.runtimeDurableReady !== false);
|
||||
}
|
||||
|
||||
function blockerForResult(result) {
|
||||
const blockers = [];
|
||||
if (result.blocked && result.blockerType) {
|
||||
@@ -86,7 +96,8 @@ function blockerForResult(result) {
|
||||
scope: result.blockerScope ?? "devPreconditions",
|
||||
status: "open",
|
||||
summary: result.blockerSummary,
|
||||
classification: result.blockedClassification ?? "other"
|
||||
classification: result.blockedClassification ?? "other",
|
||||
...(result.sourceIssue ? { sourceIssue: result.sourceIssue } : {})
|
||||
});
|
||||
}
|
||||
for (const blocker of result.additionalBlockers ?? []) {
|
||||
@@ -119,11 +130,13 @@ function buildDryRunEvidence({ status = "not_run", details = [] } = {}) {
|
||||
}
|
||||
|
||||
function buildComponentEvidence(fixture, dryRun, result) {
|
||||
const dbLiveReady = dbLiveReadyForResult(result);
|
||||
const runtimeDurableReady = runtimeDurableReadyForResult(result);
|
||||
const dbLiveStatus = result.blockedClassification === "DB live"
|
||||
? "blocked"
|
||||
: result.blocked
|
||||
? "not_run"
|
||||
: "pass";
|
||||
: dbLiveReady
|
||||
? "pass"
|
||||
: "not_run";
|
||||
const m3DependencyStatus = result.blockedClassification === "hardware loop"
|
||||
? "blocked"
|
||||
: result.blocked
|
||||
@@ -164,11 +177,26 @@ function buildComponentEvidence(fixture, dryRun, result) {
|
||||
: "Trace/evidence/cleanup live closure was not attempted before preflight blockers were classified."
|
||||
},
|
||||
dbLive: {
|
||||
level: "BLOCKED",
|
||||
level: dbLiveStatus === "pass" ? "DEV-LIVE" : "BLOCKED",
|
||||
status: dbLiveStatus,
|
||||
summary: dbLiveStatus === "blocked"
|
||||
? "Live scheduling is stopped at cloud-api DB readiness; DB health is classified separately from local dry-run evidence."
|
||||
: "Live cloud-api DB readiness did not block before the current M4 classification point."
|
||||
: dbLiveStatus === "pass"
|
||||
? "Current read-only /health/live evidence reports DB ready=true, connected=true, and liveDbEvidence=true; DB live is not the active M4 blocker."
|
||||
: "Live cloud-api DB readiness did not block before the current M4 classification point."
|
||||
},
|
||||
runtimeDurableAdapter: {
|
||||
level: runtimeDurableReady ? "DEV-LIVE" : "BLOCKED",
|
||||
status: result.blockerScope === "runtime-durable-adapter"
|
||||
? "blocked"
|
||||
: runtimeDurableReady
|
||||
? "pass"
|
||||
: "not_run",
|
||||
summary: result.blockerScope === "runtime-durable-adapter"
|
||||
? "Live scheduling is stopped at runtime durable adapter readiness; DB live evidence alone is not runtime durability evidence."
|
||||
: runtimeDurableReady
|
||||
? "Current read-only /health/live evidence reports runtime.adapter=postgres, durable=true, ready=true, and liveRuntimeEvidence=true."
|
||||
: "Runtime durable adapter readiness did not produce the active M4 blocker."
|
||||
},
|
||||
m3HardwareLoopDependency: {
|
||||
level: "BLOCKED",
|
||||
@@ -182,12 +210,25 @@ function buildComponentEvidence(fixture, dryRun, result) {
|
||||
}
|
||||
|
||||
function buildDependencyState(result) {
|
||||
const dbLiveReady = dbLiveReadyForResult(result);
|
||||
const runtimeDurableReady = runtimeDurableReadyForResult(result);
|
||||
return {
|
||||
dbLive: {
|
||||
status: result.blockedClassification === "DB live" ? "blocked" : result.blocked ? "not_run" : "pass",
|
||||
status: result.blockedClassification === "DB live" ? "blocked" : dbLiveReady ? "pass" : "not_run",
|
||||
summary: result.blockedClassification === "DB live"
|
||||
? result.blockerSummary
|
||||
: "DB live readiness did not produce the active M4 blocker."
|
||||
: dbLiveReady
|
||||
? "cloud-api /health/live reports DB ready=true, connected=true, and liveDbEvidence=true."
|
||||
: "DB live readiness did not produce the active M4 blocker."
|
||||
},
|
||||
runtimeDurableAdapter: {
|
||||
status: result.blockerScope === "runtime-durable-adapter" ? "blocked" : runtimeDurableReady ? "pass" : "not_run",
|
||||
requiredEvidence: "Postgres runtime adapter schema, migration ledger, and read-query readiness with liveRuntimeEvidence=true.",
|
||||
summary: result.blockerScope === "runtime-durable-adapter"
|
||||
? result.blockerSummary
|
||||
: runtimeDurableReady
|
||||
? "cloud-api /health/live reports runtime.adapter=postgres, durable=true, ready=true, and liveRuntimeEvidence=true."
|
||||
: "Runtime durable adapter readiness did not produce the active M4 blocker."
|
||||
},
|
||||
m3HardwareLoop: {
|
||||
status: result.blockedClassification === "hardware loop" ? "blocked" : result.blocked ? "not_run" : "pass",
|
||||
@@ -284,6 +325,7 @@ function buildReport(
|
||||
`DEV browser route remains frozen at ${DEV_FRONTEND_ENDPOINT}`,
|
||||
`D601 native k3s access uses KUBECONFIG=${d601Kubeconfig} and namespace ${devNamespace}`,
|
||||
"No secret reads, real deployment, or long-running agent task is attempted",
|
||||
"DB live readiness and runtime durable adapter readiness are separate gates; DB live alone cannot promote M4",
|
||||
"Agent manager, worker, and skills are reachable only through the cloud API boundary",
|
||||
"M4 hardware assistance must go through cloud-api/hardware API and patch-panel; direct box/gateway access is forbidden"
|
||||
],
|
||||
@@ -416,105 +458,104 @@ async function runLivePreflight(fixture, dryRun) {
|
||||
blockerSummary: "DEV live probe did not present the expected HWLAB cloud API identity.",
|
||||
summary: "Blocked because the live DEV boundary did not expose the expected cloud API identity."
|
||||
};
|
||||
} else if (isDbLiveBlocked(liveBody)) {
|
||||
liveEvidence.push(`live:${liveBody.serviceId}:${liveBody.status}:db-blocked`);
|
||||
result = {
|
||||
blocked: true,
|
||||
blockerType: "runtime_blocker",
|
||||
blockerScope: "db-live",
|
||||
blockedClassification: "DB live",
|
||||
blockerSummary: summarizeDbBlocker(liveBody),
|
||||
summary: "Blocked at DB live readiness before scheduling a DEV agent task."
|
||||
};
|
||||
} else {
|
||||
liveEvidence.push(`live:${liveBody.serviceId}:${liveBody.status}`);
|
||||
|
||||
const rpcHealth = await requestJson(`${DEV_ENDPOINT}/rpc`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
body: JSON.stringify({
|
||||
jsonrpc: "2.0",
|
||||
id: "req_dev_m4_health",
|
||||
method: "system.health",
|
||||
params: {},
|
||||
meta: {
|
||||
traceId: "trc_dev_m4_health",
|
||||
serviceId: "hwlab-cli",
|
||||
environment: ENVIRONMENT_DEV
|
||||
}
|
||||
})
|
||||
});
|
||||
const rpcHealthBody = rpcHealth.body || {};
|
||||
liveEvidence.push(`rpc-health:${rpcHealthBody.result?.serviceId ?? "unknown"}`);
|
||||
if (rpcHealth.statusCode < 200 || rpcHealth.statusCode >= 300 || rpcHealthBody.error) {
|
||||
result = {
|
||||
blocked: true,
|
||||
blockerType: "agent_blocker",
|
||||
blockerScope: "agent-runtime",
|
||||
blockedClassification: "agent runtime",
|
||||
blockerSummary: rpcHealthBody.error?.message ?? `system.health returned HTTP ${rpcHealth.statusCode}.`,
|
||||
summary: "Blocked at agent runtime health before attempting hardware dispatch."
|
||||
};
|
||||
const liveReadiness = classifyCloudApiLiveReadiness(liveBody);
|
||||
if (liveReadiness.blocked) {
|
||||
liveEvidence.push(`live:${liveBody.serviceId}:${liveBody.status}:${liveReadiness.evidenceSuffix}`);
|
||||
const runtime = runtimeDurabilityFromHealth(liveBody);
|
||||
liveEvidence.push(`runtime-durable-adapter:adapter=${runtime.adapter}:blocker=${runtime.blocker}:migration=${runtime.migration.requiredMigrationId}:migrationChecked=${runtime.migration.checked}:migrationReady=${runtime.migration.ready}:queryResult=${runtime.queryResult}`);
|
||||
result = liveReadiness;
|
||||
} else {
|
||||
liveEvidence.push(`live:${liveBody.serviceId}:${liveBody.status}`);
|
||||
const runtime = runtimeDurabilityFromHealth(liveBody);
|
||||
liveEvidence.push(`runtime-durable-adapter:adapter=${runtime.adapter}:ready=${runtime.ready}:migration=${runtime.migration.requiredMigrationId}:queryResult=${runtime.queryResult}`);
|
||||
|
||||
const rpcHardware = await requestJson(`${DEV_ENDPOINT}/rpc`, {
|
||||
const rpcHealth = await requestJson(`${DEV_ENDPOINT}/rpc`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
body: JSON.stringify({
|
||||
jsonrpc: "2.0",
|
||||
id: "req_dev_m4_hw",
|
||||
method: "hardware.operation.request",
|
||||
params: {
|
||||
projectId: fixture.projectId,
|
||||
input: {
|
||||
requestedPath: "DO1 -> hwlab-patch-panel -> DI1",
|
||||
directBoxGatewayAccess: false,
|
||||
source: "dev-m4-agent-loop-preflight"
|
||||
}
|
||||
},
|
||||
id: "req_dev_m4_health",
|
||||
method: "system.health",
|
||||
params: {},
|
||||
meta: {
|
||||
traceId: "trc_dev_m4_hw",
|
||||
traceId: "trc_dev_m4_health",
|
||||
serviceId: "hwlab-cli",
|
||||
environment: ENVIRONMENT_DEV
|
||||
}
|
||||
})
|
||||
});
|
||||
const rpcHardwareBody = rpcHardware.body || {};
|
||||
const accepted = rpcHardwareBody.result?.accepted === true;
|
||||
const status = rpcHardwareBody.result?.status || rpcHardwareBody.error?.message || "unknown";
|
||||
liveEvidence.push(`rpc-hardware:${status}`);
|
||||
|
||||
if (!accepted) {
|
||||
const rpcHealthBody = rpcHealth.body || {};
|
||||
liveEvidence.push(`rpc-health:${rpcHealthBody.result?.serviceId ?? "unknown"}`);
|
||||
if (rpcHealth.statusCode < 200 || rpcHealth.statusCode >= 300 || rpcHealthBody.error) {
|
||||
result = {
|
||||
blocked: true,
|
||||
blockerType: "agent_blocker",
|
||||
blockerScope: "hardware-loop",
|
||||
blockedClassification: "hardware loop",
|
||||
blockerSummary: "hardware.operation.request did not accept the patch-panel-bounded M3 request.",
|
||||
summary: "Blocked by M3 readiness because M4 did not observe an accepted DO1 -> patch-panel -> DI1 hardware request."
|
||||
};
|
||||
} else if (!rpcHardwareBody.result?.auditId || !rpcHardwareBody.result?.evidenceId) {
|
||||
result = {
|
||||
blocked: true,
|
||||
blockerType: "observability_blocker",
|
||||
blockerScope: "evidence-audit",
|
||||
blockedClassification: "evidence/audit",
|
||||
blockerSummary: "hardware.operation.request accepted but did not return audit/evidence identifiers.",
|
||||
summary: "Blocked because accepted DEV agent hardware dispatch lacks audit/evidence closure."
|
||||
blockerScope: "agent-runtime",
|
||||
blockedClassification: "agent runtime",
|
||||
blockerSummary: rpcHealthBody.error?.message ?? `system.health returned HTTP ${rpcHealth.statusCode}.`,
|
||||
summary: "Blocked at agent runtime health before attempting hardware dispatch."
|
||||
};
|
||||
} else {
|
||||
result = {
|
||||
blocked: false,
|
||||
blockerType: null,
|
||||
blockerScope: null,
|
||||
blockedClassification: "none",
|
||||
blockerSummary: "",
|
||||
summary: "Live DEV preflight observed a non-skeleton agent hardware path."
|
||||
};
|
||||
const rpcHardware = await requestJson(`${DEV_ENDPOINT}/rpc`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
body: JSON.stringify({
|
||||
jsonrpc: "2.0",
|
||||
id: "req_dev_m4_hw",
|
||||
method: "hardware.operation.request",
|
||||
params: {
|
||||
projectId: fixture.projectId,
|
||||
input: {
|
||||
requestedPath: "DO1 -> hwlab-patch-panel -> DI1",
|
||||
directBoxGatewayAccess: false,
|
||||
source: "dev-m4-agent-loop-preflight"
|
||||
}
|
||||
},
|
||||
meta: {
|
||||
traceId: "trc_dev_m4_hw",
|
||||
serviceId: "hwlab-cli",
|
||||
environment: ENVIRONMENT_DEV
|
||||
}
|
||||
})
|
||||
});
|
||||
const rpcHardwareBody = rpcHardware.body || {};
|
||||
const accepted = rpcHardwareBody.result?.accepted === true;
|
||||
const status = rpcHardwareBody.result?.status || rpcHardwareBody.error?.message || "unknown";
|
||||
liveEvidence.push(`rpc-hardware:${status}`);
|
||||
|
||||
if (!accepted) {
|
||||
result = {
|
||||
blocked: true,
|
||||
blockerType: "agent_blocker",
|
||||
blockerScope: "hardware-loop",
|
||||
blockedClassification: "hardware loop",
|
||||
blockerSummary: "hardware.operation.request did not accept the patch-panel-bounded M3 request.",
|
||||
summary: "Blocked by M3 readiness because M4 did not observe an accepted DO1 -> patch-panel -> DI1 hardware request."
|
||||
};
|
||||
} else if (!rpcHardwareBody.result?.auditId || !rpcHardwareBody.result?.evidenceId) {
|
||||
result = {
|
||||
blocked: true,
|
||||
blockerType: "observability_blocker",
|
||||
blockerScope: "evidence-audit",
|
||||
blockedClassification: "evidence/audit",
|
||||
blockerSummary: "hardware.operation.request accepted but did not return audit/evidence identifiers.",
|
||||
summary: "Blocked because accepted DEV agent hardware dispatch lacks audit/evidence closure."
|
||||
};
|
||||
} else {
|
||||
result = {
|
||||
blocked: false,
|
||||
blockerType: null,
|
||||
blockerScope: null,
|
||||
blockedClassification: "none",
|
||||
blockerSummary: "",
|
||||
summary: "Live DEV preflight observed a non-skeleton agent hardware path."
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -575,20 +616,4 @@ async function main() {
|
||||
await runLivePreflight(fixture, await runDryRun(fixture));
|
||||
}
|
||||
|
||||
function isDbLiveBlocked(body) {
|
||||
const db = body?.db;
|
||||
if (!db || typeof db !== "object") {
|
||||
return false;
|
||||
}
|
||||
return db.connected === false || db.ready === false || ["blocked", "degraded"].includes(db.status);
|
||||
}
|
||||
|
||||
function summarizeDbBlocker(body) {
|
||||
const db = body?.db ?? {};
|
||||
const missing = Array.isArray(db.missingEnv) && db.missingEnv.length > 0
|
||||
? ` missing ${db.missingEnv.join(", ")}`
|
||||
: "";
|
||||
return `cloud-api /health/live reports DB ${db.status ?? "not ready"}; connected=${db.connected ?? "unknown"}; ready=${db.ready ?? "unknown"}.${missing}`;
|
||||
}
|
||||
|
||||
await main();
|
||||
|
||||
@@ -68,7 +68,9 @@ assert.equal(liveBlocked.exitCode, 2);
|
||||
const liveBlockedJson = JSON.parse(liveBlocked.stderr);
|
||||
assert.equal(liveBlockedJson.code, "BLOCKED");
|
||||
assert.equal(liveBlockedJson.blockers[0].type, "runtime_blocker");
|
||||
assert.equal(liveBlockedJson.blockers[0].scope, "db-live");
|
||||
assert.equal(liveBlockedJson.blockers[0].scope, "runtime-durable-adapter");
|
||||
assert.match(liveBlockedJson.blockers[0].summary, /runtime_durable_adapter_query_blocked/);
|
||||
assert.ok(liveBlockedJson.blockers.some((blocker) => blocker.scope === "skills-commit-version-injection"));
|
||||
assert.ok(liveBlockedJson.blockers.some((blocker) => blocker.scope === "m3-hardware-loop-runtime"));
|
||||
|
||||
assert.equal(gateSummary.endpoint, summary.endpoint);
|
||||
|
||||
@@ -5,6 +5,12 @@ import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { activeReportLifecycle, reportIsHistorical, reportLifecycleState } from "../../internal/dev-report-lifecycle.mjs";
|
||||
import {
|
||||
RUNTIME_DURABLE_ADAPTER_MISSING,
|
||||
RUNTIME_DURABILITY_REQUIRED_EVIDENCE,
|
||||
RUNTIME_STORE_KIND_POSTGRES
|
||||
} from "../../internal/db/runtime-store.mjs";
|
||||
import { CLOUD_CORE_MIGRATION_ID } from "../../internal/db/schema.mjs";
|
||||
import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT, ENVIRONMENT_DEV, SERVICE_IDS } from "../../internal/protocol/index.mjs";
|
||||
|
||||
export const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
@@ -189,7 +195,7 @@ function normalizeBlocker(blocker, sourceReport, fallbackPriority = "P3") {
|
||||
scope: blocker.scope,
|
||||
status: blocker.status ?? "open",
|
||||
source: sourceReport.path,
|
||||
sourceIssue: sourceReport.issue,
|
||||
sourceIssue: blocker.sourceIssue ?? sourceReport.issue,
|
||||
summary: oneLine(blocker.summary),
|
||||
...(nextTask ? { nextTask } : {})
|
||||
};
|
||||
@@ -198,7 +204,7 @@ function normalizeBlocker(blocker, sourceReport, fallbackPriority = "P3") {
|
||||
function dedupeBlockers(blockers) {
|
||||
const byKey = new Map();
|
||||
for (const blocker of blockers) {
|
||||
const key = `${blocker.type}:${blocker.scope}:${blocker.summary}`;
|
||||
const key = `${blocker.type}:${blocker.scope}`;
|
||||
const existing = byKey.get(key);
|
||||
if (!existing) {
|
||||
byKey.set(key, blocker);
|
||||
@@ -310,6 +316,26 @@ function applyPriority(blocker) {
|
||||
};
|
||||
}
|
||||
|
||||
if (blocker.scope === "runtime-durable-adapter") {
|
||||
return {
|
||||
...blocker,
|
||||
priority: "P1",
|
||||
unblockOrder: 5,
|
||||
unblocks: [issue(37), issue(39), issue(164)],
|
||||
rationale: "M4 and M5 cannot claim live agent or MVP evidence until the cloud-api postgres runtime adapter proves schema, migration, and read-query durability; DB live evidence alone is not durability evidence."
|
||||
};
|
||||
}
|
||||
|
||||
if (blocker.scope === "skills-commit-version-injection" || blocker.scope === "agent-mgr-health") {
|
||||
return {
|
||||
...blocker,
|
||||
priority: "P1",
|
||||
unblockOrder: 5,
|
||||
unblocks: [issue(37), issue(39), issue(164)],
|
||||
rationale: "M4/M5 agent evidence needs deployed manager, skills, and worker paths to expose explicit skills commit and version; session-mode-only worker templates are not enough."
|
||||
};
|
||||
}
|
||||
|
||||
if (isEdgeOrFrpBlocker(blocker)) {
|
||||
return {
|
||||
...blocker,
|
||||
@@ -629,18 +655,27 @@ function collectBlockers(reports) {
|
||||
if (isStaleLegacyIngressBlocker(blocker, report, reports)) {
|
||||
continue;
|
||||
}
|
||||
if (isStaleReadinessBlocker(blocker, reports)) {
|
||||
continue;
|
||||
}
|
||||
blockers.push(normalizeBlocker(blocker, report));
|
||||
}
|
||||
for (const blocker of report.artifactPublish?.blockers ?? []) {
|
||||
if (isStaleLegacyIngressBlocker(blocker, report, reports)) {
|
||||
continue;
|
||||
}
|
||||
if (isStaleReadinessBlocker(blocker, reports)) {
|
||||
continue;
|
||||
}
|
||||
blockers.push(normalizeBlocker(blocker, report));
|
||||
}
|
||||
for (const blocker of report.devDeployApply?.remainingBlockers ?? []) {
|
||||
if (isStaleLegacyIngressBlocker(blocker, report, reports)) {
|
||||
continue;
|
||||
}
|
||||
if (isStaleReadinessBlocker(blocker, reports)) {
|
||||
continue;
|
||||
}
|
||||
blockers.push(normalizeBlocker(blocker, report));
|
||||
}
|
||||
}
|
||||
@@ -722,6 +757,29 @@ function isStaleLegacyIngressBlocker(blocker, sourceReport, reports) {
|
||||
sourceReport.path !== reportPaths.devM5Gate;
|
||||
}
|
||||
|
||||
function isStaleReadinessBlocker(blocker, reports) {
|
||||
const scope = blocker.scope ?? "";
|
||||
const db = cloudApiDbStatus(reports);
|
||||
const provider = codeAgentProviderStatus(reports);
|
||||
if (
|
||||
(
|
||||
scope === "cloud-api-db" ||
|
||||
scope === "cloud-api-db-health-gate" ||
|
||||
scope === "cloud-api-db-live" ||
|
||||
scope === "db-live"
|
||||
) &&
|
||||
db.ready &&
|
||||
db.connected &&
|
||||
db.liveDbEvidence
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if ((scope === "code-agent-provider-secret" || scope === "code-agent-provider") && provider.ready) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function deriveMilestones(evidence, blockers) {
|
||||
const evidenceByMilestone = new Map();
|
||||
for (const item of evidence) {
|
||||
@@ -786,7 +844,7 @@ function buildMilestoneLevelClassification(milestones) {
|
||||
M1: "Local smoke is green; it is not DEV-LIVE.",
|
||||
M2: "Current public frontend/API route evidence is DEV-LIVE for route/front-end reachability only.",
|
||||
M3: "DEV-LIVE hardware trusted loop is blocked; local/source shape is not acceptance.",
|
||||
M4: "DEV-LIVE agent loop is blocked at DB live readiness; local smoke is not acceptance.",
|
||||
M4: "DEV-LIVE agent loop is blocked at runtime durable adapter or skills injection readiness; local smoke is not acceptance.",
|
||||
M5: "Dry-run is green, but bounded DEV-LIVE MVP e2e is blocked."
|
||||
};
|
||||
|
||||
@@ -918,6 +976,18 @@ function buildDoD(reports, milestones, blockers) {
|
||||
}
|
||||
|
||||
function cloudApiDbStatus(reports) {
|
||||
const liveDb = reports.devM4Agent.publicEntrypoints?.api?.live?.db ?? null;
|
||||
if (liveDb?.ready === true && liveDb?.connected === true && liveDb?.liveDbEvidence === true) {
|
||||
return {
|
||||
status: liveDb.status ?? "ok",
|
||||
ready: true,
|
||||
connected: true,
|
||||
configReady: liveDb.configReady === true,
|
||||
connectionChecked: liveDb.connectionChecked === true,
|
||||
endpointSource: liveDb.endpointSource ?? liveDb.connection?.endpointSource ?? "secret-url-host",
|
||||
liveDbEvidence: true
|
||||
};
|
||||
}
|
||||
const db = reports.devEdgeHealth.edgeHealth?.contracts?.deploy?.cloudApiDb ?? {};
|
||||
return {
|
||||
status: db.status ?? "unknown",
|
||||
@@ -930,10 +1000,91 @@ function cloudApiDbStatus(reports) {
|
||||
};
|
||||
}
|
||||
|
||||
function codeAgentProviderStatus(reports) {
|
||||
const codeAgent = reports.devM4Agent.publicEntrypoints?.api?.live?.codeAgent ??
|
||||
reports.devM4Agent.publicEntrypoints?.api?.health?.codeAgent ??
|
||||
null;
|
||||
return {
|
||||
ready: Boolean(codeAgent?.ready || codeAgent?.status === "ready" || codeAgent?.status === "available"),
|
||||
status: codeAgent?.status ?? "unknown",
|
||||
provider: codeAgent?.provider ?? "unknown"
|
||||
};
|
||||
}
|
||||
|
||||
function runtimeDurabilityStatusFromReport(report = {}) {
|
||||
const runtime = report.runtime ??
|
||||
report.livePreflight?.runtime ??
|
||||
report.publicEntrypoints?.api?.live?.runtime ??
|
||||
report.readiness?.durability ??
|
||||
{};
|
||||
const dependency = report.dependencyState?.runtimeDurableAdapter ?? {};
|
||||
const component = report.componentEvidence?.runtimeDurableAdapter ?? {};
|
||||
const blocker = report.blockers?.find((item) => item.scope === "runtime-durable-adapter") ?? null;
|
||||
const migration = runtime.migration ?? {};
|
||||
return {
|
||||
adapter: runtime.adapter ?? "unknown",
|
||||
durable: runtime.durable === true,
|
||||
durableRequested: Boolean(
|
||||
runtime.durableRequested ||
|
||||
runtime.adapter === RUNTIME_STORE_KIND_POSTGRES ||
|
||||
blocker ||
|
||||
dependency.status === "blocked" ||
|
||||
component.status === "blocked"
|
||||
),
|
||||
ready: Boolean(runtime.ready === true && runtime.liveRuntimeEvidence === true && runtime.durable === true),
|
||||
runtimeStatus: runtime.status ?? (blocker ? "blocked" : "unknown"),
|
||||
blocker: blocker?.classification ?? runtime.blocker ?? report.readiness?.durability?.blocker ?? RUNTIME_DURABLE_ADAPTER_MISSING,
|
||||
blockedLayer: runtime.durabilityContract?.blockedLayer ?? report.readiness?.durability?.blockedLayer ?? null,
|
||||
requiredEvidence: runtime.durabilityContract?.requiredEvidence ??
|
||||
report.readiness?.durability?.requiredEvidence ??
|
||||
RUNTIME_DURABILITY_REQUIRED_EVIDENCE,
|
||||
migration: {
|
||||
checked: migration.checked ?? null,
|
||||
ready: migration.ready ?? null,
|
||||
missing: migration.missing ?? null,
|
||||
requiredMigrationId: migration.requiredMigrationId ?? CLOUD_CORE_MIGRATION_ID,
|
||||
appliedMigrationId: migration.appliedMigrationId ?? null
|
||||
},
|
||||
queryAttempted: runtime.connection?.queryAttempted ?? null,
|
||||
queryResult: runtime.connection?.queryResult ?? "unknown",
|
||||
summary: blocker?.summary ?? dependency.summary ?? component.summary ?? "No runtime durable adapter evidence was attached."
|
||||
};
|
||||
}
|
||||
|
||||
function runtimeDurabilityStatus(reports) {
|
||||
return runtimeDurabilityStatusFromReport(reports.devM4Agent);
|
||||
}
|
||||
|
||||
function m4SkillsInjectionStatus(reports) {
|
||||
const native = reports.devM4Agent.d601K3sNative ?? {};
|
||||
const injection = native.skillsInjection ?? {};
|
||||
const blockers = reports.devM4Agent.blockers ?? [];
|
||||
const injectionBlocker = blockers.find((blocker) => blocker.scope === "skills-commit-version-injection") ?? null;
|
||||
const managerBlocker = blockers.find((blocker) => blocker.scope === "agent-mgr-health") ?? null;
|
||||
const missing = Array.isArray(injection.missing) ? injection.missing : [];
|
||||
const managerMissing = native.agentManager?.health?.missingSkills ?? [];
|
||||
return {
|
||||
status: injectionBlocker || managerBlocker || injection.status === "blocked" ? "blocked" : "pass",
|
||||
serviceCommitId: injection.serviceCommitId ?? null,
|
||||
serviceVersion: injection.serviceVersion ?? null,
|
||||
workerDryRunCommitId: injection.workerDryRunCommitId ?? null,
|
||||
workerDryRunVersion: injection.workerDryRunVersion ?? null,
|
||||
missing: [...new Set([...missing, ...managerMissing])],
|
||||
summary: injectionBlocker?.summary ??
|
||||
managerBlocker?.summary ??
|
||||
(injection.status === "blocked"
|
||||
? `DEV skills injection is incomplete: missing ${missing.join(", ")}.`
|
||||
: "DEV skills commit and version injection evidence is present.")
|
||||
};
|
||||
}
|
||||
|
||||
function buildCurrentDevLayering(reports, blockers) {
|
||||
const m2EndpointLive = hasCurrentM2EndpointEvidence(reports.devM2Smoke);
|
||||
const edgeLive = hasCurrentLiveEdgeEvidence(reports.devEdgeHealth);
|
||||
const cloudDb = cloudApiDbStatus(reports);
|
||||
const provider = codeAgentProviderStatus(reports);
|
||||
const runtimeDurability = runtimeDurabilityStatus(reports);
|
||||
const skillsInjection = m4SkillsInjectionStatus(reports);
|
||||
const dbReady = cloudDb.ready && cloudDb.connected && cloudDb.endpointSource === "secret-url-host" && cloudDb.liveDbEvidence;
|
||||
const m3Live = statusIsPass(reports.devM3Hardware.liveOperation?.status);
|
||||
const m4Live = statusIsPass(reports.devM4Agent.livePreflight?.status);
|
||||
@@ -971,6 +1122,41 @@ function buildCurrentDevLayering(reports, blockers) {
|
||||
evidence: reports.devM5Gate.devPreconditions?.evidence?.filter((line) => line.includes("/health/live") || line.includes("DB")) ?? [],
|
||||
nextRequired: "Provide live DB connection evidence through redacted health output; route reachability alone is insufficient."
|
||||
},
|
||||
runtimeDurableAdapter: {
|
||||
label: "Runtime durable adapter",
|
||||
status: runtimeDurability.ready ? "pass" : "blocked",
|
||||
evidenceLevel: runtimeDurability.ready ? "DEV-LIVE" : "BLOCKED",
|
||||
summary: `runtime adapter=${runtimeDurability.adapter}; durableRequested=${runtimeDurability.durableRequested}; durable=${runtimeDurability.durable}; ready=${runtimeDurability.ready}; blocker=${runtimeDurability.blocker}; migration=${runtimeDurability.migration.requiredMigrationId} checked=${runtimeDurability.migration.checked ?? "unknown"} ready=${runtimeDurability.migration.ready ?? "unknown"}; queryResult=${runtimeDurability.queryResult}.`,
|
||||
evidence: [
|
||||
`requiredEvidence=${runtimeDurability.requiredEvidence}`,
|
||||
`blockedLayer=${runtimeDurability.blockedLayer ?? "unknown"}`,
|
||||
`migration=${runtimeDurability.migration.requiredMigrationId}`,
|
||||
`queryAttempted=${runtimeDurability.queryAttempted ?? "unknown"}`
|
||||
],
|
||||
nextRequired: "Repair the postgres durable runtime adapter query/migration readiness and rerun health/M4 preflight without treating DB live as durability proof."
|
||||
},
|
||||
agentSkillsInjection: {
|
||||
label: "Agent skills injection",
|
||||
status: skillsInjection.status,
|
||||
evidenceLevel: skillsInjection.status === "pass" ? "DEV-LIVE" : "BLOCKED",
|
||||
summary: skillsInjection.summary,
|
||||
evidence: [
|
||||
`skills.commit=${skillsInjection.serviceCommitId ?? "missing"}`,
|
||||
`skills.version=${skillsInjection.serviceVersion ?? "missing"}`,
|
||||
`worker.commit=${skillsInjection.workerDryRunCommitId ?? "missing"}`,
|
||||
`worker.version=${skillsInjection.workerDryRunVersion ?? "missing"}`,
|
||||
`missing=${skillsInjection.missing.join(",") || "none"}`
|
||||
],
|
||||
nextRequired: "Inject explicit skill commit and version into hwlab-agent-mgr, hwlab-agent-skills, and scheduled worker jobs before claiming M4/M5 agent evidence."
|
||||
},
|
||||
codeAgentProvider: {
|
||||
label: "Code Agent provider",
|
||||
status: provider.ready ? "pass" : "blocked",
|
||||
evidenceLevel: provider.ready ? "DEV-LIVE" : "BLOCKED",
|
||||
summary: `code-agent provider=${provider.provider}; status=${provider.status}; ready=${provider.ready}.`,
|
||||
evidence: [`provider=${provider.provider}`, `status=${provider.status}`],
|
||||
nextRequired: "Keep provider-backed Code Agent readiness separate from DB live and runtime durable adapter readiness."
|
||||
},
|
||||
d601RunnerObservability: {
|
||||
label: "D601 runner observability",
|
||||
status: reports.d601Observability.runnerKubeconfigReadable === false
|
||||
@@ -1005,7 +1191,7 @@ function buildCurrentDevLayering(reports, blockers) {
|
||||
evidenceLevel: m4Live ? "DEV-LIVE" : "BLOCKED",
|
||||
summary: reports.devM4Agent.livePreflight?.summary ?? reports.devM4Agent.devPreconditions?.summary ?? "No live M4 observation was recorded.",
|
||||
evidence: reports.devM4Agent.livePreflight?.evidence ?? [],
|
||||
nextRequired: "Do not schedule or claim the agent loop as live until DB live and required runtime/evidence preconditions pass."
|
||||
nextRequired: "Do not schedule or claim the agent loop as live until DB live, runtime durable adapter, skills injection, and required evidence preconditions pass."
|
||||
},
|
||||
artifactDesiredStateSource: {
|
||||
label: "artifact/desired-state source",
|
||||
@@ -1024,6 +1210,8 @@ function buildCurrentDevLayering(reports, blockers) {
|
||||
|
||||
function buildMilestoneBlockerClassification(reports) {
|
||||
const cloudDb = cloudApiDbStatus(reports);
|
||||
const runtimeDurability = runtimeDurabilityStatus(reports);
|
||||
const skillsInjection = m4SkillsInjectionStatus(reports);
|
||||
const m3Live = statusIsPass(reports.devM3Hardware.liveOperation?.status);
|
||||
const m4Live = statusIsPass(reports.devM4Agent.livePreflight?.status);
|
||||
const m3ServiceDiscoveryBlocked = reports.devM3Hardware.blockers?.some((blocker) =>
|
||||
@@ -1045,6 +1233,23 @@ function buildMilestoneBlockerClassification(reports) {
|
||||
? "patch-panel-m3-wiring-missing"
|
||||
: "hardware-loop-runtime";
|
||||
|
||||
const m4RuntimeDurableBlocked = reports.devM4Agent.blockers?.some((blocker) =>
|
||||
blocker.scope === "runtime-durable-adapter"
|
||||
) === true || (!m4Live && runtimeDurability.durableRequested && !runtimeDurability.ready);
|
||||
const m4SkillsInjectionBlocked = skillsInjection.status === "blocked";
|
||||
const m4DbLiveBlocked = reports.devM4Agent.blockers?.some((blocker) =>
|
||||
blocker.scope === "db-live"
|
||||
) === true;
|
||||
const m4BlockerClass = m4Live
|
||||
? "cleared"
|
||||
: m4RuntimeDurableBlocked
|
||||
? runtimeDurability.blocker
|
||||
: m4SkillsInjectionBlocked
|
||||
? "skills-commit-version-injection"
|
||||
: m4DbLiveBlocked
|
||||
? "db-live-readiness"
|
||||
: "agent-loop-live-preflight";
|
||||
|
||||
return [
|
||||
{
|
||||
milestone: "M3",
|
||||
@@ -1083,31 +1288,51 @@ function buildMilestoneBlockerClassification(reports) {
|
||||
milestone: "M4",
|
||||
status: m4Live ? "pass" : "blocked",
|
||||
currentLevel: m4Live ? "DEV-LIVE" : "BLOCKED",
|
||||
blockerClass: m4Live ? "cleared" : "db-live-readiness",
|
||||
dependency: "Cloud API /health/live must report DB ready=true, connected=true, endpointSource=secret-url-host, and liveDbEvidence=true before live agent scheduling/evidence closure.",
|
||||
blockerClass: m4BlockerClass,
|
||||
dependency: m4RuntimeDurableBlocked
|
||||
? "Cloud API /health/live must prove the postgres durable runtime adapter schema, 0001_cloud_core_skeleton migration ledger, and read-query readiness before live agent scheduling/evidence closure."
|
||||
: m4SkillsInjectionBlocked
|
||||
? "Agent manager, skills service, and worker jobs must expose explicit skill commit and version fields before M4 live evidence can be trusted."
|
||||
: "Cloud API /health/live must report DB ready=true, connected=true, endpointSource=secret-url-host, and liveDbEvidence=true before live agent scheduling/evidence closure.",
|
||||
evidence: [
|
||||
`db.status=${cloudDb.status}`,
|
||||
`db.ready=${cloudDb.ready}`,
|
||||
`db.connected=${cloudDb.connected}`,
|
||||
`db.endpointSource=${cloudDb.endpointSource}`,
|
||||
`db.liveDbEvidence=${cloudDb.liveDbEvidence}`
|
||||
`db.liveDbEvidence=${cloudDb.liveDbEvidence}`,
|
||||
`runtime.adapter=${runtimeDurability.adapter}`,
|
||||
`runtime.blocker=${runtimeDurability.blocker}`,
|
||||
`runtime.migration=${runtimeDurability.migration.requiredMigrationId}`,
|
||||
`runtime.migration.ready=${runtimeDurability.migration.ready ?? "unknown"}`,
|
||||
`runtime.queryResult=${runtimeDurability.queryResult}`,
|
||||
`skills.missing=${skillsInjection.missing.join(",") || "none"}`
|
||||
],
|
||||
nextRequired: "Repair DB live readiness and rerun the M4 live preflight without scheduling a DEV agent task before preconditions pass.",
|
||||
nonPromotionReason: "Frontend revision and read-only route reachability do not prove DB-backed agent runtime readiness."
|
||||
nextRequired: m4RuntimeDurableBlocked
|
||||
? "Repair the runtime durable adapter query/migration readiness and rerun the M4 live preflight without scheduling a DEV agent task before preconditions pass."
|
||||
: m4SkillsInjectionBlocked
|
||||
? "Inject skill commit/version into agent manager, skills service, and worker jobs, then rerun the M4 read-only/live preflight."
|
||||
: "Repair DB live readiness and rerun the M4 live preflight without scheduling a DEV agent task before preconditions pass.",
|
||||
nonPromotionReason: "Frontend revision and read-only route reachability do not prove DB-backed durable agent runtime readiness or skill version injection."
|
||||
},
|
||||
{
|
||||
milestone: "M5",
|
||||
status: "blocked",
|
||||
currentLevel: "BLOCKED",
|
||||
blockerClass: "composite-db-m3-m4-live",
|
||||
dependency: "M5 needs DB live readiness, M3 trusted-loop DEV evidence, M4 live preflight/evidence closure, and current source/artifact coverage.",
|
||||
blockerClass: m4RuntimeDurableBlocked
|
||||
? "composite-runtime-durable-m3-m4-live"
|
||||
: m4SkillsInjectionBlocked
|
||||
? "composite-skills-injection-m3-m4-live"
|
||||
: "composite-db-m3-m4-live",
|
||||
dependency: "M5 needs DB live readiness, runtime durable adapter readiness, skills commit/version injection, M3 trusted-loop DEV evidence, M4 live preflight/evidence closure, and current source/artifact coverage.",
|
||||
evidence: [
|
||||
"M5 dry-run is green",
|
||||
`db.ready=${cloudDb.ready}`,
|
||||
`runtime.blocker=${runtimeDurability.blocker}`,
|
||||
`skills.status=${skillsInjection.status}`,
|
||||
`m3.live=${m3Live}`,
|
||||
`m4.live=${m4Live}`
|
||||
],
|
||||
nextRequired: "After DB/M3/M4 blockers are cleared, run only the bounded DEV MVP live gate command with explicit DEV/non-PROD confirmations.",
|
||||
nextRequired: "After DB/runtime durable adapter/skills/M3/M4 blockers are cleared, run only the bounded DEV MVP live gate command with explicit DEV/non-PROD confirmations.",
|
||||
nonPromotionReason: "No frontend, route-only, local, or dry-run evidence is allowed to stand in for bounded DEV-LIVE MVP e2e acceptance."
|
||||
}
|
||||
];
|
||||
@@ -1165,6 +1390,8 @@ function fallbackAction(scope) {
|
||||
if (scope.includes("edge") || scope.includes("ingress") || scope.includes("frp")) return "Repair frp/master-edge/D601 router path and rerun read-only DEV edge health.";
|
||||
if (scope.includes("cloud-api-db")) return "Configure DEV cloud-api DB env readiness and Secret URL host connectivity, then rerun health/preflight without exposing secrets.";
|
||||
if (scope === "db-live") return "Repair DEV cloud-api DB live readiness, then rerun the read-only health and M4 preflight reports without exposing secret values.";
|
||||
if (scope === "runtime-durable-adapter") return "Repair DEV cloud-api postgres durable runtime adapter schema/migration/read-query readiness, then rerun /health/live and the M4 preflight without exposing secret values.";
|
||||
if (scope === "skills-commit-version-injection" || scope === "agent-mgr-health") return "Inject explicit skills commit and version into agent manager, skills service, and worker jobs, then rerun the M4 read-only/live preflight.";
|
||||
if (scope === "m3-patch-panel-wiring") return "Load/apply DEV patch-panel wiring for res_boxsimu_1:DO1 -> res_boxsimu_2:DI1, then rerun the bounded DEV M3 smoke.";
|
||||
if (scope === "m3-box-simu-identity") return "Fix DEV box-simu instance identity so direct endpoints expose distinct res_boxsimu_1 and res_boxsimu_2 resources.";
|
||||
if (scope === "m3-gateway-simu-identity") return "Fix DEV gateway-simu instance identity so direct endpoints expose two distinct gateway sessions.";
|
||||
@@ -1181,6 +1408,8 @@ function evidenceRequiredFor(scope) {
|
||||
if (scope.includes("kubectl") || scope.includes("k3s")) return "Read-only kubectl/k3s report proving pods/services/configmaps are observable in hwlab-dev without reading Secrets.";
|
||||
if (scope.includes("cloud-api-db")) return "Cloud API health/live output showing DB env ready, endpointSource=secret-url-host, redacted secret references, and no secret material.";
|
||||
if (scope === "db-live") return "Cloud API /health/live output with ready=true, connected=true, endpointSource=secret-url-host, liveDbEvidence=true, and redacted secret references.";
|
||||
if (scope === "runtime-durable-adapter") return "Cloud API /health/live output with runtime.adapter=postgres, durable=true, ready=true, liveRuntimeEvidence=true, migration 0001_cloud_core_skeleton ready, and read-query readiness passing.";
|
||||
if (scope === "skills-commit-version-injection" || scope === "agent-mgr-health") return "Read-only hwlab-agent-mgr/hwlab-agent-skills health plus worker dry-run manifest showing skillCommitId/skillVersion and HWLAB_SKILL_COMMIT_ID/HWLAB_SKILL_VERSION injected.";
|
||||
if (scope === "m3-patch-panel-wiring") return "Read-only patch-panel /status and /wiring showing active res_boxsimu_1:DO1 -> res_boxsimu_2:DI1 before a bounded write/read smoke records operation, trace, audit, and evidence IDs.";
|
||||
if (scope === "m3-box-simu-identity") return "Read-only direct box-simu /health/live and /status output showing distinct res_boxsimu_1 and res_boxsimu_2 resources.";
|
||||
if (scope === "m3-gateway-simu-identity") return "Read-only direct gateway-simu /health/live and /status output showing two distinct gateway identities/sessions.";
|
||||
@@ -1299,7 +1528,7 @@ export async function buildReport() {
|
||||
green: dod.green,
|
||||
reason: dod.green
|
||||
? "All #9 DEV DoD checks are green."
|
||||
: `Frontend revision ${latestFrontendRevision} is usable in degraded/read-only mode and EDGE/ROUTE DEV-LIVE evidence exists on :16666/:16667, but M5 remains blocked by artifact source drift, API/runtime durability degradation, missing M3 trusted loop operation evidence, and blocked M4 agent-loop preflight.`
|
||||
: `Frontend revision ${latestFrontendRevision} is usable in degraded/read-only mode and EDGE/ROUTE DEV-LIVE evidence exists on :16666/:16667, but M5 remains blocked by artifact source drift, runtime durable adapter or skills injection readiness, missing M3 trusted loop operation evidence, and blocked M4 agent-loop preflight.`
|
||||
},
|
||||
latestFrontendDevFact: {
|
||||
...frontendDevFact,
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import { buildReport } from "./dev-evidence-blocker-aggregator.mjs";
|
||||
|
||||
test("aggregator reports M4 current blocker as durable runtime or skills injection, not stale DB live", async () => {
|
||||
const report = await buildReport();
|
||||
const m4 = report.milestoneBlockerClassification.find((item) => item.milestone === "M4");
|
||||
const m5 = report.milestoneBlockerClassification.find((item) => item.milestone === "M5");
|
||||
const dbLayer = report.currentDevLayering.dbLive;
|
||||
const dbCheck = report.dod.checks.find((item) => item.id === "cloud-api-db-ready");
|
||||
|
||||
assert.ok(m4, "M4 blocker classification is present");
|
||||
assert.ok(m5, "M5 blocker classification is present");
|
||||
assert.equal(dbLayer.status, "pass");
|
||||
assert.equal(dbLayer.evidenceLevel, "DEV-LIVE");
|
||||
assert.equal(dbCheck.status, "pass");
|
||||
assert.equal(dbCheck.evidenceLevel, "DEV-LIVE");
|
||||
assert.equal(
|
||||
report.blockers.some((blocker) =>
|
||||
["cloud-api-db", "cloud-api-db-health-gate", "cloud-api-db-live", "db-live"].includes(blocker.scope)
|
||||
),
|
||||
false,
|
||||
"DB live blockers must be stale once current DB live evidence is pass"
|
||||
);
|
||||
assert.notEqual(m4.blockerClass, "db-live-readiness");
|
||||
assert.ok(
|
||||
["runtime_durable_adapter_query_blocked", "skills-commit-version-injection"].includes(m4.blockerClass),
|
||||
`unexpected M4 blocker class ${m4.blockerClass}`
|
||||
);
|
||||
assert.match(m4.dependency, /durable runtime adapter|skill commit and version/u);
|
||||
assert.match(m4.nonPromotionReason, /durable agent runtime readiness|skill version injection/u);
|
||||
assert.match(m5.dependency, /runtime durable adapter readiness, skills commit\/version injection/u);
|
||||
});
|
||||
@@ -9,6 +9,12 @@ import {
|
||||
AGENT_SKILLS_SERVICE_ID,
|
||||
AGENT_WORKER_SERVICE_ID
|
||||
} from "../../internal/agent/index.mjs";
|
||||
import {
|
||||
RUNTIME_DURABLE_ADAPTER_MISSING,
|
||||
RUNTIME_DURABILITY_REQUIRED_EVIDENCE,
|
||||
RUNTIME_STORE_KIND_POSTGRES
|
||||
} from "../../internal/db/runtime-store.mjs";
|
||||
import { CLOUD_CORE_MIGRATION_ID } from "../../internal/db/schema.mjs";
|
||||
import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs";
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
@@ -20,6 +26,183 @@ export const d601K3sReadonlyCommand =
|
||||
export const workerServerDryRunCommand =
|
||||
"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev create -f - --dry-run=server -o json";
|
||||
|
||||
export function isDbLiveReady(body) {
|
||||
const db = body?.db;
|
||||
return Boolean(db?.ready && db?.connected && db?.liveConnected !== false && db?.liveDbEvidence);
|
||||
}
|
||||
|
||||
export function isDbLiveBlocked(body) {
|
||||
const db = body?.db;
|
||||
if (!db || typeof db !== "object") {
|
||||
return false;
|
||||
}
|
||||
if (isDbLiveReady(body)) {
|
||||
return false;
|
||||
}
|
||||
return db.connected === false ||
|
||||
db.ready === false ||
|
||||
db.liveConnected === false ||
|
||||
db.liveDbEvidence === false ||
|
||||
["blocked", "degraded", "failed"].includes(db.status);
|
||||
}
|
||||
|
||||
export function summarizeDbBlocker(body) {
|
||||
const db = body?.db ?? {};
|
||||
const missing = Array.isArray(db.missingEnv) && db.missingEnv.length > 0
|
||||
? ` missing ${db.missingEnv.join(", ")}`
|
||||
: "";
|
||||
return `cloud-api /health/live reports DB ${db.status ?? "not ready"}; connected=${db.connected ?? "unknown"}; ready=${db.ready ?? "unknown"}; liveDbEvidence=${db.liveDbEvidence ?? "unknown"}.${missing}`;
|
||||
}
|
||||
|
||||
export function runtimeDurabilityFromHealth(body) {
|
||||
const runtime = body?.runtime ?? {};
|
||||
const durability = body?.readiness?.durability ?? {};
|
||||
const migration = runtime.migration ?? {};
|
||||
const migrationGate = runtime.gates?.migration ?? durability.gates?.migration ?? {};
|
||||
const blocker = runtime.blocker ??
|
||||
durability.blocker ??
|
||||
(Array.isArray(body?.blockerCodes)
|
||||
? body.blockerCodes.find((code) => String(code).startsWith("runtime_durable_adapter_"))
|
||||
: null) ??
|
||||
RUNTIME_DURABLE_ADAPTER_MISSING;
|
||||
const durableRequested = Boolean(
|
||||
runtime.durableRequested ||
|
||||
durability.durableRequested ||
|
||||
runtime.adapter === RUNTIME_STORE_KIND_POSTGRES ||
|
||||
durability.adapter === RUNTIME_STORE_KIND_POSTGRES
|
||||
);
|
||||
const ready = Boolean(
|
||||
runtime.durable === true &&
|
||||
runtime.ready === true &&
|
||||
runtime.liveRuntimeEvidence === true &&
|
||||
durability.ready !== false
|
||||
);
|
||||
|
||||
return {
|
||||
adapter: runtime.adapter ?? durability.adapter ?? "unknown",
|
||||
durable: Boolean(runtime.durable),
|
||||
durableRequested,
|
||||
ready,
|
||||
runtimeStatus: runtime.status ?? durability.runtimeStatus ?? "unknown",
|
||||
blocker,
|
||||
blockedLayer: runtime.durabilityContract?.blockedLayer ?? durability.blockedLayer ?? null,
|
||||
requiredEvidence: runtime.durabilityContract?.requiredEvidence ??
|
||||
durability.requiredEvidence ??
|
||||
RUNTIME_DURABILITY_REQUIRED_EVIDENCE,
|
||||
migration: {
|
||||
checked: migration.checked ?? migrationGate.checked ?? false,
|
||||
ready: migration.ready ?? migrationGate.ready ?? false,
|
||||
missing: migration.missing ?? (migration.ready === false ? true : null),
|
||||
requiredMigrationId: migration.requiredMigrationId ?? CLOUD_CORE_MIGRATION_ID,
|
||||
appliedMigrationId: migration.appliedMigrationId ?? null
|
||||
},
|
||||
queryAttempted: Boolean(runtime.connection?.queryAttempted ?? durability.queryAttempted),
|
||||
queryResult: runtime.connection?.queryResult ?? durability.queryResult ?? "unknown"
|
||||
};
|
||||
}
|
||||
|
||||
export function isRuntimeDurableAdapterBlocked(body) {
|
||||
const runtime = runtimeDurabilityFromHealth(body);
|
||||
if (!runtime.durableRequested) {
|
||||
return false;
|
||||
}
|
||||
return !runtime.ready ||
|
||||
["blocked", "degraded", "failed"].includes(runtime.runtimeStatus) ||
|
||||
String(runtime.blocker).startsWith("runtime_durable_adapter_");
|
||||
}
|
||||
|
||||
export function summarizeRuntimeDurableBlocker(body) {
|
||||
const runtime = runtimeDurabilityFromHealth(body);
|
||||
const migration = runtime.migration;
|
||||
return `cloud-api durable runtime adapter is blocked: ${runtime.blocker}; adapter=${runtime.adapter}; durable=${runtime.durable}; ready=${runtime.ready}; migration=${migration.requiredMigrationId} checked=${migration.checked} ready=${migration.ready} missing=${migration.missing ?? "unknown"}; queryResult=${runtime.queryResult}.`;
|
||||
}
|
||||
|
||||
export function publicDbLiveSummary(body) {
|
||||
const db = body?.db ?? null;
|
||||
if (!db || typeof db !== "object") {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
status: db.status ?? null,
|
||||
connected: db.connected ?? null,
|
||||
liveConnected: db.liveConnected ?? null,
|
||||
ready: db.ready ?? null,
|
||||
configReady: db.configReady ?? null,
|
||||
connectionChecked: db.connectionChecked ?? null,
|
||||
connectionAttempted: db.connectionAttempted ?? null,
|
||||
connectionResult: db.connectionResult ?? null,
|
||||
endpointSource: db.endpointSource ?? db.connection?.endpointSource ?? null,
|
||||
liveDbEvidence: db.liveDbEvidence ?? null,
|
||||
evidence: db.evidence ?? null
|
||||
};
|
||||
}
|
||||
|
||||
export function publicRuntimeDurabilitySummary(body) {
|
||||
if (!body?.runtime && !body?.readiness?.durability) {
|
||||
return null;
|
||||
}
|
||||
const runtime = runtimeDurabilityFromHealth(body);
|
||||
return {
|
||||
adapter: runtime.adapter,
|
||||
durable: runtime.durable,
|
||||
durableRequested: runtime.durableRequested,
|
||||
ready: runtime.ready,
|
||||
status: runtime.runtimeStatus,
|
||||
blocker: runtime.blocker,
|
||||
blockedLayer: runtime.blockedLayer,
|
||||
requiredEvidence: runtime.requiredEvidence,
|
||||
liveRuntimeEvidence: Boolean(runtime.ready),
|
||||
migration: runtime.migration,
|
||||
queryAttempted: runtime.queryAttempted,
|
||||
queryResult: runtime.queryResult
|
||||
};
|
||||
}
|
||||
|
||||
export function classifyCloudApiLiveReadiness(body) {
|
||||
if (isDbLiveBlocked(body)) {
|
||||
return {
|
||||
blocked: true,
|
||||
blockerType: "runtime_blocker",
|
||||
blockerScope: "db-live",
|
||||
sourceIssue: "pikasTech/HWLAB#49",
|
||||
blockedClassification: "DB live",
|
||||
blockerSummary: summarizeDbBlocker(body),
|
||||
summary: "Blocked at DB live readiness before scheduling a DEV agent task.",
|
||||
evidenceSuffix: "db-blocked",
|
||||
dbLiveReady: false,
|
||||
runtimeDurableReady: false
|
||||
};
|
||||
}
|
||||
|
||||
if (isRuntimeDurableAdapterBlocked(body)) {
|
||||
const runtime = runtimeDurabilityFromHealth(body);
|
||||
return {
|
||||
blocked: true,
|
||||
blockerType: "runtime_blocker",
|
||||
blockerScope: "runtime-durable-adapter",
|
||||
sourceIssue: "pikasTech/HWLAB#164",
|
||||
blockedClassification: runtime.blocker,
|
||||
blockerSummary: summarizeRuntimeDurableBlocker(body),
|
||||
summary: "Blocked at runtime durable adapter readiness before scheduling a DEV agent task.",
|
||||
evidenceSuffix: `runtime-durable-adapter-blocked:${runtime.blocker}`,
|
||||
dbLiveReady: isDbLiveReady(body),
|
||||
runtimeDurableReady: false
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
blocked: false,
|
||||
blockerType: null,
|
||||
blockerScope: null,
|
||||
blockedClassification: "none",
|
||||
blockerSummary: "",
|
||||
summary: "Live DEV preflight observed DB live and durable runtime adapter readiness.",
|
||||
evidenceSuffix: null,
|
||||
dbLiveReady: isDbLiveReady(body),
|
||||
runtimeDurableReady: true
|
||||
};
|
||||
}
|
||||
|
||||
export function requestJson(urlString, { method = "GET", headers = {}, body, timeoutMs = 5000 } = {}) {
|
||||
const url = new URL(urlString);
|
||||
const client = url.protocol === "https:" ? httpsRequest : httpRequest;
|
||||
@@ -348,12 +531,9 @@ export async function collectPublicEntrypoints() {
|
||||
serviceId: apiLive.body?.serviceId ?? null,
|
||||
environment: apiLive.body?.environment ?? null,
|
||||
status: apiLive.body?.status ?? null,
|
||||
db: apiLive.body?.db ? {
|
||||
status: apiLive.body.db.status ?? null,
|
||||
connected: apiLive.body.db.connected ?? null,
|
||||
ready: apiLive.body.db.ready ?? null,
|
||||
evidence: apiLive.body.db.evidence ?? null
|
||||
} : null,
|
||||
db: publicDbLiveSummary(apiLive.body),
|
||||
runtime: publicRuntimeDurabilitySummary(apiLive.body),
|
||||
blockerCodes: Array.isArray(apiLive.body?.blockerCodes) ? apiLive.body.blockerCodes : [],
|
||||
error: apiLive.error ?? null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import {
|
||||
classifyCloudApiLiveReadiness,
|
||||
publicDbLiveSummary,
|
||||
publicRuntimeDurabilitySummary,
|
||||
runtimeDurabilityFromHealth
|
||||
} from "./dev-m4-agent-loop-smoke-lib.mjs";
|
||||
|
||||
test("M4 live readiness classifies durable adapter query blocker after DB live is ready", () => {
|
||||
const payload = {
|
||||
db: {
|
||||
ready: true,
|
||||
connected: true,
|
||||
liveConnected: true,
|
||||
liveDbEvidence: true,
|
||||
status: "ok"
|
||||
},
|
||||
runtime: {
|
||||
adapter: "postgres",
|
||||
durable: false,
|
||||
durableRequested: true,
|
||||
ready: false,
|
||||
status: "blocked",
|
||||
blocker: "runtime_durable_adapter_query_blocked",
|
||||
reason: "durable runtime read query is blocked",
|
||||
liveRuntimeEvidence: false,
|
||||
connection: {
|
||||
queryAttempted: true,
|
||||
queryResult: "query_blocked"
|
||||
},
|
||||
migration: {
|
||||
checked: true,
|
||||
ready: false,
|
||||
missing: true,
|
||||
requiredMigrationId: "0001_cloud_core_skeleton"
|
||||
}
|
||||
},
|
||||
readiness: {
|
||||
durability: {
|
||||
blockedLayer: "durability_query",
|
||||
requiredEvidence: "runtime_adapter_schema_migration_read_query"
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const classification = classifyCloudApiLiveReadiness(payload);
|
||||
assert.equal(classification.blocked, true);
|
||||
assert.equal(classification.blockerScope, "runtime-durable-adapter");
|
||||
assert.equal(classification.blockedClassification, "runtime_durable_adapter_query_blocked");
|
||||
assert.match(classification.blockerSummary, /0001_cloud_core_skeleton/u);
|
||||
assert.match(classification.evidenceSuffix, /runtime_durable_adapter_query_blocked/u);
|
||||
|
||||
const runtime = runtimeDurabilityFromHealth(payload);
|
||||
assert.equal(runtime.blocker, "runtime_durable_adapter_query_blocked");
|
||||
assert.equal(runtime.migration.checked, true);
|
||||
assert.equal(runtime.migration.ready, false);
|
||||
assert.equal(runtime.queryResult, "query_blocked");
|
||||
});
|
||||
|
||||
test("M4 live readiness keeps DB live blocker when DB is not connected", () => {
|
||||
const classification = classifyCloudApiLiveReadiness({
|
||||
db: {
|
||||
ready: false,
|
||||
connected: false,
|
||||
liveDbEvidence: false,
|
||||
status: "degraded"
|
||||
},
|
||||
runtime: {
|
||||
adapter: "postgres",
|
||||
durableRequested: true,
|
||||
blocker: "runtime_durable_adapter_query_blocked"
|
||||
}
|
||||
});
|
||||
|
||||
assert.equal(classification.blockerScope, "db-live");
|
||||
assert.equal(classification.blockedClassification, "DB live");
|
||||
});
|
||||
|
||||
test("public DB live summary preserves DB pass fields apart from durable adapter blocker", () => {
|
||||
const summary = publicDbLiveSummary({
|
||||
db: {
|
||||
status: "ok",
|
||||
connected: true,
|
||||
liveConnected: true,
|
||||
ready: true,
|
||||
configReady: true,
|
||||
connectionChecked: true,
|
||||
endpointSource: "secret-url-host",
|
||||
liveDbEvidence: true,
|
||||
evidence: "live_db_tcp_connection_ready"
|
||||
},
|
||||
runtime: {
|
||||
adapter: "postgres",
|
||||
durable: false,
|
||||
ready: false,
|
||||
blocker: "runtime_durable_adapter_query_blocked"
|
||||
}
|
||||
});
|
||||
|
||||
assert.equal(summary.status, "ok");
|
||||
assert.equal(summary.connected, true);
|
||||
assert.equal(summary.liveConnected, true);
|
||||
assert.equal(summary.ready, true);
|
||||
assert.equal(summary.configReady, true);
|
||||
assert.equal(summary.connectionChecked, true);
|
||||
assert.equal(summary.endpointSource, "secret-url-host");
|
||||
assert.equal(summary.liveDbEvidence, true);
|
||||
assert.equal(summary.evidence, "live_db_tcp_connection_ready");
|
||||
});
|
||||
|
||||
test("public runtime durability summary preserves durable adapter fields separately from DB live", () => {
|
||||
const summary = publicRuntimeDurabilitySummary({
|
||||
db: {
|
||||
ready: true,
|
||||
connected: true,
|
||||
liveConnected: true,
|
||||
liveDbEvidence: true
|
||||
},
|
||||
runtime: {
|
||||
adapter: "postgres",
|
||||
durable: false,
|
||||
durableRequested: true,
|
||||
ready: false,
|
||||
status: "blocked",
|
||||
blocker: "runtime_durable_adapter_query_blocked",
|
||||
liveRuntimeEvidence: false,
|
||||
connection: {
|
||||
queryAttempted: true,
|
||||
queryResult: "query_blocked"
|
||||
},
|
||||
migration: {
|
||||
checked: false,
|
||||
ready: false,
|
||||
missing: true,
|
||||
requiredMigrationId: "0001_cloud_core_skeleton"
|
||||
},
|
||||
durabilityContract: {
|
||||
blockedLayer: "durability_query",
|
||||
requiredEvidence: "runtime_adapter_schema_migration_read_query"
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
assert.equal(summary.adapter, "postgres");
|
||||
assert.equal(summary.durableRequested, true);
|
||||
assert.equal(summary.durable, false);
|
||||
assert.equal(summary.ready, false);
|
||||
assert.equal(summary.liveRuntimeEvidence, false);
|
||||
assert.equal(summary.blocker, "runtime_durable_adapter_query_blocked");
|
||||
assert.equal(summary.blockedLayer, "durability_query");
|
||||
assert.equal(summary.requiredEvidence, "runtime_adapter_schema_migration_read_query");
|
||||
assert.equal(summary.queryAttempted, true);
|
||||
assert.equal(summary.queryResult, "query_blocked");
|
||||
assert.equal(summary.migration.requiredMigrationId, "0001_cloud_core_skeleton");
|
||||
});
|
||||
@@ -528,6 +528,8 @@ test("accepts the current blocked M5 aggregator non-promotion contract", async (
|
||||
|
||||
test("rejects aggregator DB acceptance promoted from frontend route evidence", async () => {
|
||||
const report = cloneBaseAggregatorReport();
|
||||
const m4Source = report.sourceReports.devM4Agent;
|
||||
m4Source.path = "reports/dev-gate/dev-edge-health.json";
|
||||
const check = report.dod.checks.find((item) => item.id === "cloud-api-db-ready");
|
||||
Object.assign(check, {
|
||||
status: "pass",
|
||||
@@ -542,6 +544,24 @@ test("rejects aggregator DB acceptance promoted from frontend route evidence", a
|
||||
await assertRejected(report, /DB DEV-LIVE acceptance source reports/);
|
||||
});
|
||||
|
||||
test("rejects aggregator stale DB runtime-env blocker when DB live is pass", async () => {
|
||||
const report = cloneBaseAggregatorReport();
|
||||
report.blockers.push({
|
||||
id: "dev-gate-preflight:cloud-api-db-health-gate",
|
||||
priority: "P1",
|
||||
type: "runtime_blocker",
|
||||
scope: "cloud-api-db-health-gate",
|
||||
status: "open",
|
||||
source: "reports/dev-gate/dev-preflight-report.json",
|
||||
sourceIssue: "pikasTech/HWLAB#34",
|
||||
summary: "cloud-api DB runtime env is not ready; missing HWLAB_CLOUD_DB_URL, HWLAB_CLOUD_DB_SSL_MODE.",
|
||||
unblockOrder: 5,
|
||||
unblocks: ["pikasTech/HWLAB#34", "pikasTech/HWLAB#33", "pikasTech/HWLAB#39"],
|
||||
rationale: "Stale DB runtime-env blocker fixture."
|
||||
});
|
||||
await assertRejected(report, /stale DB live\/runtime-env blocker/);
|
||||
});
|
||||
|
||||
test("rejects aggregator M3 acceptance promoted from source evidence", async () => {
|
||||
const report = cloneBaseAggregatorReport();
|
||||
const sourceEvidence = report.evidence.find((item) =>
|
||||
|
||||
@@ -2468,9 +2468,14 @@ function observedLiveIdentifier(value) {
|
||||
|
||||
function aggregatorLiveStateFromSources(sources) {
|
||||
const cloudApiDb = sources.devEdgeHealth.edgeHealth?.contracts?.deploy?.cloudApiDb ?? {};
|
||||
const dbLive = cloudApiDb.ready === true &&
|
||||
const edgeDbLive = cloudApiDb.ready === true &&
|
||||
(cloudApiDb.connected === true || cloudApiDb.liveConnected === true) &&
|
||||
cloudApiDb.liveDbEvidence === true;
|
||||
const m4LiveDb = sources.devM4Agent.publicEntrypoints?.api?.live?.db ?? {};
|
||||
const m4DbLive = m4LiveDb.ready === true &&
|
||||
(m4LiveDb.connected === true || m4LiveDb.liveConnected === true) &&
|
||||
m4LiveDb.liveDbEvidence === true;
|
||||
const dbLive = edgeDbLive || m4DbLive;
|
||||
const m3Operation = sources.devM3Hardware.liveOperation ?? {};
|
||||
const m3Live = statusIsPass(m3Operation.status) &&
|
||||
["operationId", "traceId", "auditId", "evidenceId"].every((field) =>
|
||||
@@ -2547,11 +2552,37 @@ function assertProtectedAggregatorDevLiveEvidence(item, label, liveState, assume
|
||||
);
|
||||
}
|
||||
|
||||
function assertNoStaleDbRuntimeBlocker(report, label, liveState) {
|
||||
if (!liveState.dbLive) {
|
||||
return;
|
||||
}
|
||||
|
||||
const staleDbScopes = new Set([
|
||||
"cloud-api-db",
|
||||
"cloud-api-db-health-gate",
|
||||
"cloud-api-db-live",
|
||||
"db-live"
|
||||
]);
|
||||
const staleBlocker = report.blockers?.find((blocker) => staleDbScopes.has(blocker.scope));
|
||||
assert.equal(
|
||||
staleBlocker,
|
||||
undefined,
|
||||
`${label}.blockers must not keep stale DB live/runtime-env blocker ${staleBlocker?.scope ?? ""} when DB live evidence is pass`
|
||||
);
|
||||
|
||||
const dbCheck = findById(report.dod.checks, "cloud-api-db-ready");
|
||||
assertObject(dbCheck, `${label}.dod.checks.cloud-api-db-ready`);
|
||||
assert.equal(dbCheck.status, "pass", `${label}.dod.checks.cloud-api-db-ready.status must be pass when DB live evidence is proven`);
|
||||
assert.equal(dbCheck.evidenceLevel, "DEV-LIVE", `${label}.dod.checks.cloud-api-db-ready.evidenceLevel must be DEV-LIVE when DB live evidence is proven`);
|
||||
}
|
||||
|
||||
async function assertAggregatorNonPromotionBoundaries(report, label) {
|
||||
const sources = await readAggregatorSourceReports(report, label);
|
||||
const liveState = aggregatorLiveStateFromSources(sources);
|
||||
const anyProtectedLiveBlocked = !liveState.dbLive || !liveState.m3Live || !liveState.m4Live || !liveState.m5Live;
|
||||
|
||||
assertNoStaleDbRuntimeBlocker(report, label, liveState);
|
||||
|
||||
assertObject(report.latestFrontendDevFact, `${label}.latestFrontendDevFact`);
|
||||
assert.equal(
|
||||
report.latestFrontendDevFact.promotesM3M4M5,
|
||||
|
||||
Reference in New Issue
Block a user