Merge pull request #351 from pikasTech/feat/rpt004-mvp-e2e-harness
feat: add RPT-004 MVP E2E harness
This commit is contained in:
Lyon
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,214 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import {
|
||||
classifyCodeAgentSkillPath,
|
||||
classifyM3DurableEvidence,
|
||||
classifyRpt004Conclusion,
|
||||
parseArgs,
|
||||
validateLiveBoundary
|
||||
} from "./rpt004-mvp-e2e-harness.mjs";
|
||||
|
||||
test("RPT-004 CLI accepts the required live URL/API URL shape", () => {
|
||||
const args = parseArgs([
|
||||
"--live",
|
||||
"--url",
|
||||
"http://74.48.78.17:16666/",
|
||||
"--api-url",
|
||||
"http://74.48.78.17:16667/"
|
||||
]);
|
||||
|
||||
assert.equal(args.live, true);
|
||||
assert.equal(args.url, "http://74.48.78.17:16666/");
|
||||
assert.equal(args.apiUrl, "http://74.48.78.17:16667/");
|
||||
assert.equal(args.writeReport, true);
|
||||
assert.doesNotThrow(() => validateLiveBoundary(args));
|
||||
});
|
||||
|
||||
test("RPT-004 live boundary rejects deprecated or swapped active DEV ports", () => {
|
||||
assert.throws(
|
||||
() => validateLiveBoundary({ url: "http://74.48.78.17:6666/", apiUrl: "http://74.48.78.17:16667/" }),
|
||||
/16666/u
|
||||
);
|
||||
assert.throws(
|
||||
() => validateLiveBoundary({ url: "http://74.48.78.17:16667/", apiUrl: "http://74.48.78.17:16666/" }),
|
||||
/16666/u
|
||||
);
|
||||
});
|
||||
|
||||
test("RPT-004 classifies deployed revision drift as NOT_CURRENT, never MVP PASS", () => {
|
||||
const conclusion = classifyRpt004Conclusion([
|
||||
check("expected-artifact-current", "pass"),
|
||||
check("live-revision-current", "not_current"),
|
||||
check("live-health-ready", "pass"),
|
||||
check("16666-default-chinese-workbench", "pass")
|
||||
]);
|
||||
|
||||
assert.equal(conclusion.status, "not_current");
|
||||
assert.equal(conclusion.classification, "NOT_CURRENT");
|
||||
assert.equal(conclusion.mvpPass, false);
|
||||
});
|
||||
|
||||
test("RPT-004 classifies degraded live health as BLOCKED, never MVP PASS", () => {
|
||||
const conclusion = classifyRpt004Conclusion([
|
||||
check("expected-artifact-current", "pass"),
|
||||
check("live-revision-current", "pass"),
|
||||
check("live-health-ready", "blocked"),
|
||||
check("16666-default-chinese-workbench", "pass"),
|
||||
check("m3-v1-io-trusted-loop", "not_run")
|
||||
]);
|
||||
|
||||
assert.equal(conclusion.status, "blocked");
|
||||
assert.equal(conclusion.classification, "BLOCKED");
|
||||
assert.equal(conclusion.mvpPass, false);
|
||||
});
|
||||
|
||||
test("RPT-004 reports MVP_PASS only when every required live dimension passes", () => {
|
||||
const conclusion = classifyRpt004Conclusion([
|
||||
check("expected-artifact-current", "pass"),
|
||||
check("live-revision-current", "pass"),
|
||||
check("live-health-ready", "pass"),
|
||||
check("16666-default-chinese-workbench", "pass"),
|
||||
check("layout-smoke-hit-target", "pass"),
|
||||
check("code-agent-skill-path", "pass"),
|
||||
check("m3-v1-io-trusted-loop", "pass"),
|
||||
check("operation-audit-evidence-durable-green", "pass")
|
||||
]);
|
||||
|
||||
assert.equal(conclusion.status, "pass");
|
||||
assert.equal(conclusion.classification, "MVP_PASS");
|
||||
assert.equal(conclusion.mvpPass, true);
|
||||
});
|
||||
|
||||
test("Code Agent OpenAI fallback cannot satisfy skill path/toolCalls readiness", () => {
|
||||
const result = classifyCodeAgentSkillPath({
|
||||
status: "completed",
|
||||
provider: "openai-responses",
|
||||
model: "gpt-5.5",
|
||||
backend: "hwlab-cloud-api/openai-responses",
|
||||
implementationType: "openai-responses-fallback",
|
||||
runner: {
|
||||
kind: "openai-responses-fallback"
|
||||
},
|
||||
reply: {
|
||||
content: "text answer"
|
||||
},
|
||||
toolCalls: [],
|
||||
skills: {
|
||||
status: "not_requested",
|
||||
items: []
|
||||
}
|
||||
}, { httpStatus: 200, httpOk: true });
|
||||
|
||||
assert.equal(result.status, "blocked");
|
||||
assert.match(result.summary, /openai-fallback/u);
|
||||
});
|
||||
|
||||
test("Code Agent Codex stdio skill discovery passes with completed toolCalls and repo skill", () => {
|
||||
const result = classifyCodeAgentSkillPath({
|
||||
status: "completed",
|
||||
conversationId: "cnv_test",
|
||||
sessionId: "ses_test",
|
||||
messageId: "msg_test",
|
||||
traceId: "trc_test",
|
||||
provider: "codex-stdio",
|
||||
model: "gpt-5.5",
|
||||
backend: "hwlab-cloud-api/codex-mcp-stdio-runner",
|
||||
workspace: "/workspace/hwlab",
|
||||
sandbox: "workspace-write",
|
||||
capabilityLevel: "long-lived-codex-stdio-session",
|
||||
sessionMode: "codex-mcp-stdio-long-lived",
|
||||
implementationType: "repo-owned-codex-mcp-stdio-session",
|
||||
session: {
|
||||
sessionId: "ses_test",
|
||||
status: "idle",
|
||||
idleTimeoutMs: 1800000,
|
||||
lastTraceId: "trc_test",
|
||||
longLivedSession: true,
|
||||
codexStdio: true
|
||||
},
|
||||
sessionReuse: {
|
||||
reused: false,
|
||||
turn: 1
|
||||
},
|
||||
runner: {
|
||||
kind: "codex-mcp-stdio-runner",
|
||||
workspace: "/workspace/hwlab",
|
||||
sandbox: "workspace-write",
|
||||
sessionMode: "codex-mcp-stdio-long-lived",
|
||||
implementationType: "repo-owned-codex-mcp-stdio-session",
|
||||
codexStdio: true,
|
||||
writeCapable: true,
|
||||
durableSession: true
|
||||
},
|
||||
runnerTrace: {
|
||||
runnerKind: "codex-mcp-stdio-runner"
|
||||
},
|
||||
longLivedSessionGate: {
|
||||
status: "pass",
|
||||
pass: true
|
||||
},
|
||||
toolCalls: [{
|
||||
name: "skills.discover",
|
||||
status: "completed",
|
||||
type: "file-read"
|
||||
}],
|
||||
skills: {
|
||||
status: "ready",
|
||||
items: [{
|
||||
name: "hwlab-agent-runtime",
|
||||
source: "/workspace/hwlab/skills/hwlab-agent-runtime/SKILL.md"
|
||||
}],
|
||||
count: 1
|
||||
}
|
||||
}, { httpStatus: 200, httpOk: true });
|
||||
|
||||
assert.equal(result.status, "pass");
|
||||
});
|
||||
|
||||
test("M3 durable evidence requires operation, audit, evidence, DEV-LIVE green, and persisted writes", () => {
|
||||
const green = classifyM3DurableEvidence(greenOperations());
|
||||
assert.equal(green.status, "pass");
|
||||
|
||||
const blockedOps = greenOperations();
|
||||
blockedOps[2].evidenceState.writeStatus = "written_non_durable";
|
||||
const blocked = classifyM3DurableEvidence(blockedOps);
|
||||
assert.equal(blocked.status, "blocked");
|
||||
assert.equal(blocked.observations.sequence[2].persisted, false);
|
||||
});
|
||||
|
||||
function check(id, status) {
|
||||
return {
|
||||
id,
|
||||
status,
|
||||
summary: `${id} ${status}`
|
||||
};
|
||||
}
|
||||
|
||||
function greenOperations() {
|
||||
return [
|
||||
operation("write-do1-true", "do.write", true),
|
||||
operation("read-di1-true", "di.read", true),
|
||||
operation("write-do1-false", "do.write", false),
|
||||
operation("read-di1-false", "di.read", false)
|
||||
];
|
||||
}
|
||||
|
||||
function operation(id, action, value) {
|
||||
return {
|
||||
id,
|
||||
action,
|
||||
status: "succeeded",
|
||||
operationId: `op_${id}`,
|
||||
traceId: `trc_${id}`,
|
||||
auditId: `aud_${id}`,
|
||||
evidenceId: `evd_${id}`,
|
||||
resultValue: value,
|
||||
evidenceState: {
|
||||
status: "green",
|
||||
sourceKind: "DEV-LIVE",
|
||||
durable: true,
|
||||
writeStatus: "persisted"
|
||||
}
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user