feat: add RPT-004 MVP E2E harness
This commit is contained in:
Code Queue Review
+4
-2
File diff suppressed because one or more lines are too long
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env node
|
||||
import {
|
||||
formatRpt004Failure,
|
||||
runRpt004Cli
|
||||
} from "./src/rpt004-mvp-e2e-harness.mjs";
|
||||
|
||||
try {
|
||||
process.exitCode = await runRpt004Cli(process.argv.slice(2));
|
||||
} catch (error) {
|
||||
process.stdout.write(`${JSON.stringify(formatRpt004Failure(error), null, 2)}\n`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,214 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import {
|
||||
classifyCodeAgentSkillPath,
|
||||
classifyM3DurableEvidence,
|
||||
classifyRpt004Conclusion,
|
||||
parseArgs,
|
||||
validateLiveBoundary
|
||||
} from "./rpt004-mvp-e2e-harness.mjs";
|
||||
|
||||
test("RPT-004 CLI accepts the required live URL/API URL shape", () => {
|
||||
const args = parseArgs([
|
||||
"--live",
|
||||
"--url",
|
||||
"http://74.48.78.17:16666/",
|
||||
"--api-url",
|
||||
"http://74.48.78.17:16667/"
|
||||
]);
|
||||
|
||||
assert.equal(args.live, true);
|
||||
assert.equal(args.url, "http://74.48.78.17:16666/");
|
||||
assert.equal(args.apiUrl, "http://74.48.78.17:16667/");
|
||||
assert.equal(args.writeReport, true);
|
||||
assert.doesNotThrow(() => validateLiveBoundary(args));
|
||||
});
|
||||
|
||||
test("RPT-004 live boundary rejects deprecated or swapped active DEV ports", () => {
|
||||
assert.throws(
|
||||
() => validateLiveBoundary({ url: "http://74.48.78.17:6666/", apiUrl: "http://74.48.78.17:16667/" }),
|
||||
/16666/u
|
||||
);
|
||||
assert.throws(
|
||||
() => validateLiveBoundary({ url: "http://74.48.78.17:16667/", apiUrl: "http://74.48.78.17:16666/" }),
|
||||
/16666/u
|
||||
);
|
||||
});
|
||||
|
||||
test("RPT-004 classifies deployed revision drift as NOT_CURRENT, never MVP PASS", () => {
|
||||
const conclusion = classifyRpt004Conclusion([
|
||||
check("expected-artifact-current", "pass"),
|
||||
check("live-revision-current", "not_current"),
|
||||
check("live-health-ready", "pass"),
|
||||
check("16666-default-chinese-workbench", "pass")
|
||||
]);
|
||||
|
||||
assert.equal(conclusion.status, "not_current");
|
||||
assert.equal(conclusion.classification, "NOT_CURRENT");
|
||||
assert.equal(conclusion.mvpPass, false);
|
||||
});
|
||||
|
||||
test("RPT-004 classifies degraded live health as BLOCKED, never MVP PASS", () => {
|
||||
const conclusion = classifyRpt004Conclusion([
|
||||
check("expected-artifact-current", "pass"),
|
||||
check("live-revision-current", "pass"),
|
||||
check("live-health-ready", "blocked"),
|
||||
check("16666-default-chinese-workbench", "pass"),
|
||||
check("m3-v1-io-trusted-loop", "not_run")
|
||||
]);
|
||||
|
||||
assert.equal(conclusion.status, "blocked");
|
||||
assert.equal(conclusion.classification, "BLOCKED");
|
||||
assert.equal(conclusion.mvpPass, false);
|
||||
});
|
||||
|
||||
test("RPT-004 reports MVP_PASS only when every required live dimension passes", () => {
|
||||
const conclusion = classifyRpt004Conclusion([
|
||||
check("expected-artifact-current", "pass"),
|
||||
check("live-revision-current", "pass"),
|
||||
check("live-health-ready", "pass"),
|
||||
check("16666-default-chinese-workbench", "pass"),
|
||||
check("layout-smoke-hit-target", "pass"),
|
||||
check("code-agent-skill-path", "pass"),
|
||||
check("m3-v1-io-trusted-loop", "pass"),
|
||||
check("operation-audit-evidence-durable-green", "pass")
|
||||
]);
|
||||
|
||||
assert.equal(conclusion.status, "pass");
|
||||
assert.equal(conclusion.classification, "MVP_PASS");
|
||||
assert.equal(conclusion.mvpPass, true);
|
||||
});
|
||||
|
||||
test("Code Agent OpenAI fallback cannot satisfy skill path/toolCalls readiness", () => {
|
||||
const result = classifyCodeAgentSkillPath({
|
||||
status: "completed",
|
||||
provider: "openai-responses",
|
||||
model: "gpt-5.5",
|
||||
backend: "hwlab-cloud-api/openai-responses",
|
||||
implementationType: "openai-responses-fallback",
|
||||
runner: {
|
||||
kind: "openai-responses-fallback"
|
||||
},
|
||||
reply: {
|
||||
content: "text answer"
|
||||
},
|
||||
toolCalls: [],
|
||||
skills: {
|
||||
status: "not_requested",
|
||||
items: []
|
||||
}
|
||||
}, { httpStatus: 200, httpOk: true });
|
||||
|
||||
assert.equal(result.status, "blocked");
|
||||
assert.match(result.summary, /openai-fallback/u);
|
||||
});
|
||||
|
||||
test("Code Agent Codex stdio skill discovery passes with completed toolCalls and repo skill", () => {
|
||||
const result = classifyCodeAgentSkillPath({
|
||||
status: "completed",
|
||||
conversationId: "cnv_test",
|
||||
sessionId: "ses_test",
|
||||
messageId: "msg_test",
|
||||
traceId: "trc_test",
|
||||
provider: "codex-stdio",
|
||||
model: "gpt-5.5",
|
||||
backend: "hwlab-cloud-api/codex-mcp-stdio-runner",
|
||||
workspace: "/workspace/hwlab",
|
||||
sandbox: "workspace-write",
|
||||
capabilityLevel: "long-lived-codex-stdio-session",
|
||||
sessionMode: "codex-mcp-stdio-long-lived",
|
||||
implementationType: "repo-owned-codex-mcp-stdio-session",
|
||||
session: {
|
||||
sessionId: "ses_test",
|
||||
status: "idle",
|
||||
idleTimeoutMs: 1800000,
|
||||
lastTraceId: "trc_test",
|
||||
longLivedSession: true,
|
||||
codexStdio: true
|
||||
},
|
||||
sessionReuse: {
|
||||
reused: false,
|
||||
turn: 1
|
||||
},
|
||||
runner: {
|
||||
kind: "codex-mcp-stdio-runner",
|
||||
workspace: "/workspace/hwlab",
|
||||
sandbox: "workspace-write",
|
||||
sessionMode: "codex-mcp-stdio-long-lived",
|
||||
implementationType: "repo-owned-codex-mcp-stdio-session",
|
||||
codexStdio: true,
|
||||
writeCapable: true,
|
||||
durableSession: true
|
||||
},
|
||||
runnerTrace: {
|
||||
runnerKind: "codex-mcp-stdio-runner"
|
||||
},
|
||||
longLivedSessionGate: {
|
||||
status: "pass",
|
||||
pass: true
|
||||
},
|
||||
toolCalls: [{
|
||||
name: "skills.discover",
|
||||
status: "completed",
|
||||
type: "file-read"
|
||||
}],
|
||||
skills: {
|
||||
status: "ready",
|
||||
items: [{
|
||||
name: "hwlab-agent-runtime",
|
||||
source: "/workspace/hwlab/skills/hwlab-agent-runtime/SKILL.md"
|
||||
}],
|
||||
count: 1
|
||||
}
|
||||
}, { httpStatus: 200, httpOk: true });
|
||||
|
||||
assert.equal(result.status, "pass");
|
||||
});
|
||||
|
||||
test("M3 durable evidence requires operation, audit, evidence, DEV-LIVE green, and persisted writes", () => {
|
||||
const green = classifyM3DurableEvidence(greenOperations());
|
||||
assert.equal(green.status, "pass");
|
||||
|
||||
const blockedOps = greenOperations();
|
||||
blockedOps[2].evidenceState.writeStatus = "written_non_durable";
|
||||
const blocked = classifyM3DurableEvidence(blockedOps);
|
||||
assert.equal(blocked.status, "blocked");
|
||||
assert.equal(blocked.observations.sequence[2].persisted, false);
|
||||
});
|
||||
|
||||
function check(id, status) {
|
||||
return {
|
||||
id,
|
||||
status,
|
||||
summary: `${id} ${status}`
|
||||
};
|
||||
}
|
||||
|
||||
function greenOperations() {
|
||||
return [
|
||||
operation("write-do1-true", "do.write", true),
|
||||
operation("read-di1-true", "di.read", true),
|
||||
operation("write-do1-false", "do.write", false),
|
||||
operation("read-di1-false", "di.read", false)
|
||||
];
|
||||
}
|
||||
|
||||
function operation(id, action, value) {
|
||||
return {
|
||||
id,
|
||||
action,
|
||||
status: "succeeded",
|
||||
operationId: `op_${id}`,
|
||||
traceId: `trc_${id}`,
|
||||
auditId: `aud_${id}`,
|
||||
evidenceId: `evd_${id}`,
|
||||
resultValue: value,
|
||||
evidenceState: {
|
||||
status: "green",
|
||||
sourceKind: "DEV-LIVE",
|
||||
durable: true,
|
||||
writeStatus: "persisted"
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -32,7 +32,8 @@ const issueFamily = Object.freeze({
|
||||
DEV_EVIDENCE_BLOCKER_AGGREGATOR: "pikasTech/HWLAB#41",
|
||||
DEV_M5_GATE_AGGREGATOR_V2: "pikasTech/HWLAB#58",
|
||||
DEV_CLOUD_WORKBENCH_LIVE: "pikasTech/HWLAB#7",
|
||||
DEV_CLOUD_WORKBENCH_LAYOUT: "pikasTech/HWLAB#273"
|
||||
DEV_CLOUD_WORKBENCH_LAYOUT: "pikasTech/HWLAB#273",
|
||||
RPT004_MVP_E2E: "pikasTech/HWLAB#316"
|
||||
});
|
||||
const allowedIssues = new Set([
|
||||
contractIssue,
|
||||
@@ -526,6 +527,10 @@ async function validateReport(relativePath) {
|
||||
await validateAggregatorV2Report(relativePath, report);
|
||||
return;
|
||||
}
|
||||
if (report.reportKind === "rpt-004-mvp-e2e-harness" || report.taskId === "rpt004-mvp-e2e-harness") {
|
||||
await validateRpt004MvpE2eReport(report, label);
|
||||
return;
|
||||
}
|
||||
|
||||
if (report.issue === requiredDevM3Issue || report.taskId === "dev-m3-hardware-loop") {
|
||||
await validateDevM3Report(report, label);
|
||||
@@ -772,6 +777,137 @@ async function validateReport(relativePath) {
|
||||
}
|
||||
}
|
||||
|
||||
async function validateRpt004MvpE2eReport(report, label) {
|
||||
for (const field of [
|
||||
"$schema",
|
||||
"$id",
|
||||
"reportVersion",
|
||||
"reportKind",
|
||||
"issue",
|
||||
"taskId",
|
||||
"acceptanceLevel",
|
||||
"devOnly",
|
||||
"prodDisabled",
|
||||
"mode",
|
||||
"status",
|
||||
"conclusion",
|
||||
"sourceContract",
|
||||
"validationCommands",
|
||||
"devPreconditions",
|
||||
"target",
|
||||
"expected",
|
||||
"dimensions",
|
||||
"checks",
|
||||
"blockers",
|
||||
"safety"
|
||||
]) {
|
||||
assert.ok(Object.hasOwn(report, field), `${label} missing ${field}`);
|
||||
}
|
||||
|
||||
assert.equal(report.reportVersion, "v1", `${label}.reportVersion`);
|
||||
assert.equal(report.reportKind, "rpt-004-mvp-e2e-harness", `${label}.reportKind`);
|
||||
assert.equal(report.issue, issueFamily.RPT004_MVP_E2E, `${label}.issue`);
|
||||
assert.equal(report.taskId, "rpt004-mvp-e2e-harness", `${label}.taskId`);
|
||||
assert.equal(report.acceptanceLevel, "rpt004_mvp_e2e", `${label}.acceptanceLevel`);
|
||||
assert.equal(report.devOnly, true, `${label}.devOnly`);
|
||||
assert.equal(report.prodDisabled, true, `${label}.prodDisabled`);
|
||||
assert.ok(["source-check", "dev-live"].includes(report.mode), `${label}.mode`);
|
||||
assert.ok(["pass", "blocked", "not_current"].includes(report.status), `${label}.status`);
|
||||
|
||||
assertObject(report.conclusion, `${label}.conclusion`);
|
||||
assert.ok(
|
||||
["CHECK_ONLY", "BLOCKED", "NOT_CURRENT", "MVP_PASS"].includes(report.conclusion.classification),
|
||||
`${label}.conclusion.classification`
|
||||
);
|
||||
assert.equal(typeof report.conclusion.mvpPass, "boolean", `${label}.conclusion.mvpPass`);
|
||||
if (report.status !== "pass" || report.conclusion.classification !== "MVP_PASS") {
|
||||
assert.equal(report.conclusion.mvpPass, false, `${label}.conclusion.mvpPass must stay false outside MVP_PASS`);
|
||||
}
|
||||
if (report.conclusion.classification === "NOT_CURRENT") {
|
||||
assert.equal(report.status, "not_current", `${label}.status for NOT_CURRENT`);
|
||||
}
|
||||
|
||||
assertCommandList(report.validationCommands, `${label}.validationCommands`, { minLength: 5 });
|
||||
for (const requiredCommand of [
|
||||
"node --check scripts/rpt004-mvp-e2e-harness.mjs",
|
||||
"node --check scripts/src/rpt004-mvp-e2e-harness.mjs",
|
||||
"node --test scripts/src/rpt004-mvp-e2e-harness.test.mjs",
|
||||
"node scripts/rpt004-mvp-e2e-harness.mjs --check --no-write",
|
||||
"node scripts/rpt004-mvp-e2e-harness.mjs --live --url http://74.48.78.17:16666/ --api-url http://74.48.78.17:16667/"
|
||||
]) {
|
||||
assert.ok(
|
||||
report.validationCommands.includes(requiredCommand),
|
||||
`${label}.validationCommands missing ${requiredCommand}`
|
||||
);
|
||||
}
|
||||
|
||||
assertObject(report.target, `${label}.target`);
|
||||
assert.equal(report.target.environment, "dev", `${label}.target.environment`);
|
||||
assert.equal(report.target.browserUrl, "http://74.48.78.17:16666/", `${label}.target.browserUrl`);
|
||||
assert.equal(report.target.apiUrl, "http://74.48.78.17:16667/", `${label}.target.apiUrl`);
|
||||
assert.equal(report.target.m3IoUrl, "http://74.48.78.17:16667/v1/m3/io", `${label}.target.m3IoUrl`);
|
||||
|
||||
assertObject(report.expected, `${label}.expected`);
|
||||
assert.match(report.expected.commitId, /^[a-f0-9]{7,40}$/u, `${label}.expected.commitId`);
|
||||
assert.equal(report.expected.serviceCount, report.expected.requiredServiceCount, `${label}.expected.serviceCount`);
|
||||
|
||||
const requiredDimensions = [
|
||||
"artifact",
|
||||
"revision",
|
||||
"health",
|
||||
"workbench",
|
||||
"layout",
|
||||
"m3TrustedLoop",
|
||||
"durableEvidence",
|
||||
"codeAgentSkillPath"
|
||||
];
|
||||
assertObject(report.dimensions, `${label}.dimensions`);
|
||||
for (const id of requiredDimensions) {
|
||||
assert.ok(Object.hasOwn(report.dimensions, id), `${label}.dimensions missing ${id}`);
|
||||
}
|
||||
|
||||
assertArray(report.checks, `${label}.checks`);
|
||||
const minimumCheckCount = report.mode === "dev-live" ? 4 : 2;
|
||||
assert.ok(
|
||||
report.checks.length >= minimumCheckCount,
|
||||
`${label}.checks must include at least ${minimumCheckCount} entries`
|
||||
);
|
||||
const checkIds = new Set(report.checks.map((check) => check.id));
|
||||
assert.ok(checkIds.has("expected-artifact-current"), `${label}.checks missing expected-artifact-current`);
|
||||
if (report.mode === "dev-live") {
|
||||
for (const id of [
|
||||
"live-revision-current",
|
||||
"live-health-ready",
|
||||
"16666-default-chinese-workbench",
|
||||
"layout-smoke-hit-target",
|
||||
"code-agent-skill-path",
|
||||
"m3-v1-io-trusted-loop",
|
||||
"operation-audit-evidence-durable-green"
|
||||
]) {
|
||||
assert.ok(checkIds.has(id) || report.status !== "pass", `${label}.checks missing ${id}`);
|
||||
}
|
||||
}
|
||||
for (const [index, check] of report.checks.entries()) {
|
||||
assertObject(check, `${label}.checks[${index}]`);
|
||||
assertString(check.id, `${label}.checks[${index}].id`);
|
||||
assert.ok(["pass", "blocked", "not_current", "not_run"].includes(check.status), `${label}.checks[${index}].status`);
|
||||
assertString(check.summary, `${label}.checks[${index}].summary`);
|
||||
}
|
||||
|
||||
assertBlockers(report.blockers, `${label}.blockers`);
|
||||
assertObject(report.safety, `${label}.safety`);
|
||||
assert.equal(report.safety.devOnly, true, `${label}.safety.devOnly`);
|
||||
assert.equal(report.safety.prodTouched, false, `${label}.safety.prodTouched`);
|
||||
assert.equal(report.safety.deployApplyAttempted, false, `${label}.safety.deployApplyAttempted`);
|
||||
assert.equal(report.safety.rolloutAttempted, false, `${label}.safety.rolloutAttempted`);
|
||||
assert.equal(report.safety.secretValuesPrinted, false, `${label}.safety.secretValuesPrinted`);
|
||||
assert.equal(report.safety.openAiFallbackCountsAsPass, false, `${label}.safety.openAiFallbackCountsAsPass`);
|
||||
|
||||
if (report.artifacts?.artifactReport) {
|
||||
assert.match(report.artifacts.artifactReport, /not applicable/u, `${label}.artifacts.artifactReport`);
|
||||
}
|
||||
}
|
||||
|
||||
async function assertDocumentSet(documentsValue, label, requiredDocs) {
|
||||
assertStringArray(documentsValue, label, { minLength: requiredDocs.length });
|
||||
const documents = new Set(documentsValue);
|
||||
|
||||
Reference in New Issue
Block a user