Merge pull request #336 from pikasTech/fix/db-durable-runtime-311

Harden durable runtime DEV gates
This commit is contained in:
Lyon
2026-05-23 20:42:42 +08:00
committed by GitHub
7 changed files with 355 additions and 19 deletions
+11 -2
View File
@@ -1237,12 +1237,21 @@ async function kubectlCommandResult(ctx, kubectlContext, args, options = {}) {
}
function parseLastJsonObject(text) {
const lines = String(text ?? "").trim().split(/\r?\n/u).reverse();
const normalized = String(text ?? "").trim();
const lines = normalized.split(/\r?\n/u).reverse();
for (const line of lines) {
const parsed = parseJsonMaybe(line.trim());
if (parsed && typeof parsed === "object") return parsed;
}
return parseJsonMaybe(text);
const starts = [];
for (let index = 0; index < normalized.length; index += 1) {
if (normalized[index] === "{") starts.push(index);
}
for (const start of starts.reverse()) {
const parsed = parseJsonMaybe(normalized.slice(start));
if (parsed && typeof parsed === "object") return parsed;
}
return parseJsonMaybe(normalized);
}
function stepBlocker(stepResult) {
+49 -3
View File
@@ -108,7 +108,8 @@ function makeRunCommand({
targetCommitId = publishedCommit,
headCommitId = targetCommitId,
extraGitRefs = {},
desiredStatePromotionStatus = "pass"
desiredStatePromotionStatus = "pass",
runtimeJobLogSuffix = ""
} = {}) {
let lease = heldLock ? leaseFromLock(heldLock) : null;
const assertLeaseMicroTime = (value) => {
@@ -168,14 +169,14 @@ function makeRunCommand({
if (command.includes("kubectl") && args.includes("logs") && args.some((arg) => String(arg).startsWith("job/hwlab-runtime-provision"))) {
return {
code: 0,
stdout: `${JSON.stringify({ conclusion: { status: "ready" }, safety: { secretValuesPrinted: false } }, null, 2)}\n`,
stdout: `${runtimeJobLogSuffix}${JSON.stringify({ conclusion: { status: "ready" }, safety: { secretValuesPrinted: false } }, null, 2)}\n`,
stderr: ""
};
}
if (command.includes("kubectl") && args.includes("logs") && args.some((arg) => String(arg).startsWith("job/hwlab-runtime-migrate"))) {
return {
code: 0,
stdout: `${JSON.stringify({ conclusion: { status: "ready" }, safety: { secretValuesPrinted: false } }, null, 2)}\n`,
stdout: `${runtimeJobLogSuffix}${JSON.stringify({ conclusion: { status: "ready" }, safety: { secretValuesPrinted: false } }, null, 2)}\n`,
stderr: ""
};
}
@@ -696,11 +697,56 @@ test("transaction runs phases, allows internal side-effect env, releases lock, a
assert.equal(postflightCall?.env.HWLAB_CD_TRANSACTION_ID, publishCall.env.HWLAB_CD_TRANSACTION_ID);
assert.deepEqual(postflightCall.args.slice(0, 4), ["scripts/dev-runtime-postflight.mjs", "--live", "--confirm-dev", "--confirmed-non-production"]);
assert.equal(refreshCall.args[refreshCall.args.indexOf("--target-ref") + 1], "abc1234abc1234abc1234abc1234abc1234abc1");
assert.ok(
commandLog.findIndex((entry) => entry.input.includes("hwlab-runtime-provision")) <
commandLog.findIndex((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs"))
);
assert.ok(
commandLog.findIndex((entry) => entry.input.includes("hwlab-runtime-migrate")) <
commandLog.findIndex((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs"))
);
const writtenReport = JSON.parse(await readFile(path.join(repoRoot, "reports/dev-gate/dev-cd-apply.json"), "utf8"));
assert.equal(writtenReport.devCdApply.liveVerify.summary.checked, 2);
});
test("runtime maintenance job reports are parsed after redacting secret-like log output", async () => {
const repoRoot = await makeRepo();
const commandLog = [];
let output = "";
const code = await runDevCdApply([
"--apply",
"--confirm-dev",
"--confirmed-non-production",
"--owner-task-id",
"task-redaction",
"--kubeconfig",
"/tmp/kubeconfig",
"--write-report",
"--full-output"
], {
repoRoot,
env: {},
runCommand: makeRunCommand({
commandLog,
runtimeJobLogSuffix: "debug postgresql://hwlab:SECRET_DB@db.example:5432/hwlab password=SECRET_PASSWORD token=SECRET_TOKEN\n"
}),
httpGetJson: makeHttpGetJson(),
now: () => new Date(iso(100000)),
stdout: { write: (chunk) => { output += chunk; } }
});
const report = JSON.parse(output);
const serialized = JSON.stringify(report);
assert.equal(code, 0);
assert.equal(report.status, "pass");
assert.equal(serialized.includes("SECRET_DB"), false);
assert.equal(serialized.includes("SECRET_PASSWORD"), false);
assert.equal(serialized.includes("SECRET_TOKEN"), false);
assert.equal(report.devCdApply.steps.find((step) => step.id === "runtime-db-provisioning").stdoutJson.safety.secretValuesPrinted, false);
assert.equal(report.devCdApply.steps.find((step) => step.id === "runtime-db-migration").stdoutJson.safety.secretValuesPrinted, false);
});
test("transaction can apply an already published deploy-json promotion without republishing control HEAD", async () => {
const repoRoot = await makeRepo({ commitId: "abc1234" });
const commandLog = [];
+62 -1
View File
@@ -179,6 +179,62 @@ test("live dry-run classifies pg_hba no-encryption rejection as SSL with ssl dis
assert.equal(JSON.stringify(report).includes("pg_hba.conf"), false);
});
test("live dry-run classifies missing runtime schema before migration ledger", async () => {
const report = await buildDevRuntimeMigrationReport(
parseArgs(["--dry-run", "--allow-live-db-read", "--confirm-dev"]),
{
env: {
HWLAB_CLOUD_DB_SSL_MODE: "disable"
},
queryClient: {
async query(sql) {
assert.match(sql, /information_schema\.columns/u);
return { rows: [] };
}
},
now: () => "2026-05-23T00:00:00.000Z"
}
);
assert.equal(report.conclusion.status, "blocked");
assert.equal(report.actions.liveDbReadAttempted, true);
assert.equal(report.actions.liveDbWriteAttempted, false);
assert.equal(report.gates.auth.status, "ready");
assert.equal(report.gates.schema.status, "blocked");
assert.equal(report.gates.migration.status, "not_checked");
assert.equal(report.gates.readiness.status, "not_checked");
assert.equal(report.blockers[0].scope, "runtime-migration-schema");
assert.equal(report.blockers[0].evidence.blocker, "runtime_durable_adapter_schema_blocked");
assert.equal(report.runtime.schema.ready, false);
assert.ok(report.runtime.schema.missingTables.includes("gateway_sessions"));
});
test("live dry-run classifies durability read query blocker after schema and ledger are ready", async () => {
const report = await buildDevRuntimeMigrationReport(
parseArgs(["--dry-run", "--allow-live-db-read", "--confirm-dev"]),
{
env: {
HWLAB_CLOUD_DB_SSL_MODE: "disable"
},
queryClient: createFakeQueryClient({ migrationReady: true, countErrorCode: "57014" }),
now: () => "2026-05-23T00:00:00.000Z"
}
);
assert.equal(report.conclusion.status, "blocked");
assert.equal(report.actions.liveDbReadAttempted, true);
assert.equal(report.actions.liveDbWriteAttempted, false);
assert.equal(report.gates.auth.status, "ready");
assert.equal(report.gates.schema.status, "ready");
assert.equal(report.gates.migration.status, "ready");
assert.equal(report.gates.readiness.status, "blocked");
assert.equal(report.blockers[0].scope, "runtime-migration-readiness");
assert.equal(report.blockers[0].evidence.blocker, "runtime_durable_adapter_query_blocked");
assert.equal(report.blockers[0].evidence.queryResult, "query_blocked");
assert.equal(report.runtime.migration.ready, true);
assert.equal(report.runtime.connection.queryResult, "query_blocked");
});
test("apply mode records ledger and verifies durable runtime readiness", async () => {
const queryClient = createFakeQueryClient({ migrationReady: false });
const report = await buildDevRuntimeMigrationReport(
@@ -283,7 +339,7 @@ test("apply mode uses a dedicated pg client transaction", async () => {
assert.ok(pools[1].calls.some((call) => call.type === "end"));
});
function createFakeQueryClient({ migrationReady }) {
function createFakeQueryClient({ migrationReady, countErrorCode = null }) {
const state = {
migrationReady,
calls: []
@@ -317,6 +373,11 @@ function createFakeQueryClient({ migrationReady }) {
};
}
if (sql.startsWith("SELECT COUNT(*)::int AS count FROM ")) {
if (countErrorCode) {
const error = new Error("durability count blocked");
error.code = countErrorCode;
throw error;
}
return { rows: [{ count: 0 }] };
}
throw new Error(`unexpected query: ${sql}`);
+97 -5
View File
@@ -14,6 +14,12 @@ const defaultReportPath = "reports/dev-gate/dev-runtime-postflight-report.json";
const issue = "pikasTech/HWLAB#311";
const defaultHealthUrl = `${DEV_ENDPOINT}/health/live`;
const defaultV1Url = `${DEV_ENDPOINT}/v1`;
const expectedM3Sequence = Object.freeze([
{ id: "write-do1-true", action: "do.write", resultValue: true },
{ id: "read-di1-true", action: "di.read", resultValue: true },
{ id: "write-do1-false", action: "do.write", resultValue: false },
{ id: "read-di1-false", action: "di.read", resultValue: false }
]);
export async function runDevRuntimePostflightCli(argv = process.argv.slice(2), options = {}) {
const args = parseArgs(argv);
@@ -241,11 +247,14 @@ function summarizeApiPayload(check, response) {
m3IoControl: json.m3IoControl
? {
route: json.m3IoControl.route ?? null,
enabled: json.m3IoControl.enabled === true,
contractVersion: json.m3IoControl.contractVersion ?? null
status: json.m3IoControl.status ?? null,
enabled: json.m3IoControl.enabled === true || json.m3IoControl.status === "available",
contractVersion: json.m3IoControl.contractVersion ?? null,
directFrontendGatewayOrBoxAccess: json.m3IoControl.boundaries?.directFrontendGatewayOrBoxAccess === true,
genericHardwareRpcExposedToFrontend: json.m3IoControl.boundaries?.genericHardwareRpcExposedToFrontend === true
}
: null,
error: response.error ?? null
error: response.error ? redactFailureText(response.error) : null
};
}
@@ -297,6 +306,7 @@ function summarizeDb(db = {}) {
function summarizeM3Report(report = {}) {
const operations = Array.isArray(report.liveOperations) ? report.liveOperations : [];
const persistenceContract = summarizeM3PersistenceContract(operations);
return {
mode: report.mode ?? "unknown",
status: report.summary?.status ?? report.status ?? "unknown",
@@ -304,6 +314,8 @@ function summarizeM3Report(report = {}) {
trustedGreen: report.summary?.trustedGreen === true,
result: report.summary?.result ?? null,
operationCount: operations.length,
persisted: persistenceContract.ready,
persistenceContract,
operations: operations.map((operation) => ({
id: operation.id,
action: operation.action,
@@ -316,13 +328,47 @@ function summarizeM3Report(report = {}) {
evidenceState: {
status: operation.evidenceState?.status ?? null,
durable: operation.evidenceState?.durable === true,
sourceKind: operation.evidenceState?.sourceKind ?? null
sourceKind: operation.evidenceState?.sourceKind ?? null,
writeStatus: operation.evidenceState?.writeStatus ?? null
},
blocker: operation.blocker ?? null
blocker: sanitizeM3Blocker(operation.blocker)
}))
};
}
function summarizeM3PersistenceContract(operations = []) {
const observed = new Map(operations.map((operation) => [operation.id, operation]));
const sequence = expectedM3Sequence.map((expected) => {
const operation = observed.get(expected.id) ?? {};
const evidenceState = operation.evidenceState ?? {};
const idsPresent = Boolean(operation.operationId && operation.traceId && operation.auditId && operation.evidenceId);
const valueMatches = operation.resultValue === expected.resultValue;
const evidenceGreen = evidenceState.status === "green" &&
evidenceState.durable === true &&
evidenceState.sourceKind === "DEV-LIVE";
const persisted = evidenceState.writeStatus === "persisted";
return {
id: expected.id,
action: expected.action,
expectedValue: expected.resultValue,
observedValue: Object.hasOwn(operation, "resultValue") ? operation.resultValue : null,
status: operation.status ?? "missing",
idsPresent,
valueMatches,
evidenceGreen,
persisted
};
});
return {
contractVersion: "m3-do1-true-false-persisted-v1",
requiredRoute: "res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1",
requiredValues: [true, true, false, false],
ready: operations.length === expectedM3Sequence.length &&
sequence.every((item) => item.status === "succeeded" && item.idsPresent && item.valueMatches && item.evidenceGreen && item.persisted),
sequence
};
}
function addApiReadinessBlockers(report) {
for (const [scope, payload] of [["api-health-live", report.apiHealth], ["api-v1", report.apiV1]]) {
if (!payload.reachable) {
@@ -338,10 +384,36 @@ function addApiReadinessBlockers(report) {
blockerCodes: payload.blockerCodes
});
}
if (scope === "api-v1" && !apiV1M3ContractReady(payload)) {
addBlocker(report, "runtime_blocker", scope, "GET /v1 did not expose the controlled M3 IO route required by durable postflight.", {
m3IoControl: payload.m3IoControl,
requiredRoute: "/v1/m3/io",
requiredContractVersion: "m3-io-control-v1"
});
}
}
}
function apiV1M3ContractReady(payload) {
const control = payload.m3IoControl;
return payload.check === "GET /v1" &&
control?.route === "/v1/m3/io" &&
control?.contractVersion === "m3-io-control-v1" &&
control?.enabled === true &&
control?.directFrontendGatewayOrBoxAccess !== true &&
control?.genericHardwareRpcExposedToFrontend !== true;
}
function addM3Blockers(report) {
if (report.m3?.persistenceContract?.ready !== true) {
addBlocker(report, "runtime_blocker", "m3-durable-postflight", "M3 DO1 true/false durable postflight did not prove persisted operation/audit/evidence for every step.", {
persistenceContract: report.m3?.persistenceContract ?? null,
trustedGreen: report.m3?.trustedGreen === true,
operationCount: report.m3?.operationCount ?? 0
});
return;
}
if (report.m3?.trustedGreen !== true || report.m3?.status !== "pass") {
addBlocker(report, "runtime_blocker", "m3-durable-postflight", "M3 true/false durable postflight did not produce trusted green persisted evidence.", {
classification: report.m3?.classification ?? "unknown",
@@ -433,6 +505,26 @@ function summarizeReport(report) {
};
}
function sanitizeM3Blocker(blocker) {
if (!blocker || typeof blocker !== "object") {
return typeof blocker === "string" ? redactFailureText(blocker) : blocker ?? null;
}
return sanitizeRedactedValue(blocker);
}
function sanitizeRedactedValue(value) {
if (typeof value === "string") {
return redactFailureText(value);
}
if (Array.isArray(value)) {
return value.map(sanitizeRedactedValue);
}
if (value && typeof value === "object") {
return Object.fromEntries(Object.entries(value).map(([key, nested]) => [key, sanitizeRedactedValue(nested)]));
}
return value;
}
async function writeReport(report, reportPath, root) {
const absolute = path.resolve(root, reportPath);
await mkdir(path.dirname(absolute), { recursive: true });
+106 -3
View File
@@ -76,6 +76,53 @@ test("live mode requires /v1 durable readiness before M3 writes", async () => {
assert.equal(report.blockers.some((blocker) => blocker.scope === "api-v1"), true);
});
test("live mode requires /v1 to expose the controlled M3 IO route before M3 writes", async () => {
const report = await buildDevRuntimePostflightReport(
parseArgs(["--live", "--confirm-dev", "--confirmed-non-production"]),
{
httpGetJson: async (url) => {
const payload = apiPayload({ ready: true });
if (url.endsWith("/v1")) {
delete payload.m3IoControl;
}
return { ok: true, status: 200, json: payload };
},
m3Runner: async () => {
throw new Error("M3 runner should not run when /v1 omits the M3 IO contract");
},
now: () => "2026-05-23T00:00:00.000Z"
}
);
assert.equal(report.conclusion.status, "blocked");
assert.equal(report.actions.m3MutationAttempted, false);
assert.equal(report.blockers.some((blocker) =>
blocker.scope === "api-v1" &&
blocker.summary.includes("controlled M3 IO route")
), true);
});
test("live mode does not mistake DB live evidence for durable runtime readiness", async () => {
const report = await buildDevRuntimePostflightReport(
parseArgs(["--live", "--confirm-dev", "--confirmed-non-production"]),
{
httpGetJson: async () => ({ ok: true, status: 200, json: tcpOnlyPayload() }),
m3Runner: async () => {
throw new Error("M3 runner should not run when only TCP/SecretRef DB readiness is green");
},
now: () => "2026-05-23T00:00:00.000Z"
}
);
assert.equal(report.conclusion.status, "blocked");
assert.equal(report.apiHealth.db.connected, true);
assert.equal(report.apiHealth.db.liveDbEvidence, true);
assert.equal(report.apiHealth.runtime.durable, false);
assert.equal(report.apiHealth.readiness.durabilityReady, false);
assert.equal(report.actions.m3MutationAttempted, false);
assert.equal(report.blockers.some((blocker) => blocker.scope === "api-health-live"), true);
});
test("live mode records M3 true/false durable green evidence", async () => {
const report = await buildDevRuntimePostflightReport(
parseArgs(["--live", "--confirm-dev", "--confirmed-non-production"]),
@@ -91,6 +138,14 @@ test("live mode records M3 true/false durable green evidence", async () => {
assert.equal(report.actions.m3MutationAllowed, true);
assert.equal(report.m3.trustedGreen, true);
assert.equal(report.m3.operationCount, 4);
assert.equal(report.m3.persisted, true);
assert.equal(report.m3.persistenceContract.ready, true);
assert.deepEqual(report.m3.persistenceContract.sequence.map((item) => [item.id, item.observedValue, item.persisted]), [
["write-do1-true", true, true],
["read-di1-true", true, true],
["write-do1-false", false, true],
["read-di1-false", false, true]
]);
assert.deepEqual(report.m3.operations.map((operation) => operation.resultValue), [true, true, false, false]);
assert.equal(report.blockers.length, 0);
});
@@ -108,7 +163,26 @@ test("live mode blocks when M3 operation evidence is not durable green", async (
);
assert.equal(report.conclusion.status, "blocked");
assert.equal(report.blockers.some((blocker) => blocker.scope === "m3-durable-evidence"), true);
assert.equal(report.blockers.some((blocker) => blocker.scope === "m3-durable-postflight"), true);
assert.equal(report.m3.persistenceContract.ready, false);
});
test("live mode blocks when M3 true/false persisted sequence is incomplete", async () => {
const m3 = m3GreenReport();
m3.liveOperations = m3.liveOperations.slice(0, 3);
const report = await buildDevRuntimePostflightReport(
parseArgs(["--live", "--confirm-dev", "--confirmed-non-production"]),
{
httpGetJson: async () => ({ ok: true, status: 200, json: apiPayload({ ready: true }) }),
m3Runner: async () => m3,
now: () => "2026-05-23T00:00:00.000Z"
}
);
assert.equal(report.conclusion.status, "blocked");
assert.equal(report.m3.persistenceContract.ready, false);
assert.equal(report.m3.persistenceContract.sequence.at(-1).status, "missing");
assert.equal(report.blockers.some((blocker) => blocker.scope === "m3-durable-postflight"), true);
});
function apiPayload({ ready }) {
@@ -160,10 +234,38 @@ function apiPayload({ ready }) {
}
},
blockerCodes: ready ? []
: ["runtime_durable_adapter_auth_blocked"]
: ["runtime_durable_adapter_auth_blocked"],
m3IoControl: {
route: "/v1/m3/io",
status: "available",
contractVersion: "m3-io-control-v1",
boundaries: {
directFrontendGatewayOrBoxAccess: false,
genericHardwareRpcExposedToFrontend: false
}
}
};
}
function tcpOnlyPayload() {
const payload = apiPayload({ ready: false });
payload.db.ready = true;
payload.db.connected = true;
payload.db.liveDbEvidence = true;
payload.db.connectionResult = "connected";
payload.db.runtimeReadiness.blocker = "runtime_durable_adapter_query_blocked";
payload.db.runtimeReadiness.queryResult = "query_blocked";
payload.runtime.blocker = "runtime_durable_adapter_query_blocked";
payload.runtime.durabilityContract.blockedLayer = "durability_query";
payload.runtime.gates.auth = gate(true);
payload.runtime.gates.schema = gate(true);
payload.runtime.gates.migration = gate(true);
payload.runtime.gates.durability = gate(false, "runtime_durable_adapter_query_blocked");
payload.readiness.durability.blockedLayer = "durability_query";
payload.blockerCodes = ["runtime_durable_adapter_query_blocked"];
return payload;
}
function gate(ready, blocker = null) {
return {
checked: true,
@@ -195,7 +297,8 @@ function m3GreenReport() {
evidenceState: {
status: "green",
durable: true,
sourceKind: "DEV-LIVE"
sourceKind: "DEV-LIVE",
writeStatus: "persisted"
}
}))
};
@@ -58,6 +58,14 @@ test("live dry-run classifies missing target role separately from missing databa
assert.equal(report.blockers.some((blocker) => blocker.scope === "runtime-provisioning-role"), true);
assert.equal(report.blockers.some((blocker) => blocker.scope === "runtime-provisioning-database"), false);
assert.equal(report.blockers.some((blocker) => blocker.scope === "runtime-provisioning-auth"), true);
assert.equal(
report.blockers.find((blocker) => blocker.scope === "runtime-provisioning-role").evidence.affectedRuntimeBlocker,
"runtime_durable_adapter_auth_blocked"
);
assert.equal(
report.blockers.find((blocker) => blocker.scope === "runtime-provisioning-auth").evidence.blocker,
"runtime_durable_adapter_auth_blocked"
);
assertNoFixtureSecrets(report);
});
@@ -77,6 +85,10 @@ test("live dry-run classifies missing target database separately from target rol
assert.equal(report.provisioning.targetDatabase.exists, false);
assert.equal(report.blockers.some((blocker) => blocker.scope === "runtime-provisioning-role"), false);
assert.equal(report.blockers.some((blocker) => blocker.scope === "runtime-provisioning-database"), true);
assert.equal(
report.blockers.find((blocker) => blocker.scope === "runtime-provisioning-database").evidence.affectedRuntimeBlocker,
"runtime_durable_adapter_auth_blocked"
);
});
test("live dry-run separates SSL from auth/schema/migration", async () => {