feat: extend RPT-004 Code Agent harness
Adds Code Agent session and Skill CLI HWLAB API route gates to the RPT-004 harness, with default read-only live behavior and explicit --allow-m3-write for DEV-only M3 writes.
This commit is contained in:
@@ -23,6 +23,7 @@ import { runDevCloudWorkbenchLayoutSmoke } from "./dev-cloud-workbench-smoke-lib
|
||||
import { classifyCodexRunnerCapability } from "./code-agent-response-contract.mjs";
|
||||
import {
|
||||
HWLAB_AGENT_RUNTIME_SKILL_CLI_VERSION,
|
||||
HWLAB_M3_IO_API_ROUTE,
|
||||
HWLAB_M3_IO_SKILL_NAME
|
||||
} from "../../skills/hwlab-agent-runtime/scripts/src/m3-io-skill-client.mjs";
|
||||
|
||||
@@ -55,6 +56,22 @@ const expectedM3OperationIds = Object.freeze([
|
||||
["write-do1-false", "do.write", false],
|
||||
["read-di1-false", "di.read", false]
|
||||
]);
|
||||
const codeAgentSessionTurns = Object.freeze([
|
||||
{
|
||||
kind: "pwd",
|
||||
message: "请用 pwd 列出你当前的工作目录,并保留本轮 session 证据。"
|
||||
},
|
||||
{
|
||||
kind: "skills",
|
||||
message: "请在同一个 conversation/session 中列出你能使用的所有 skills。"
|
||||
},
|
||||
{
|
||||
kind: "context-pwd",
|
||||
message: "请引用前文:第一轮我问的是 pwd。再次用 pwd 确认当前工作目录。"
|
||||
}
|
||||
]);
|
||||
const m3ReadOnlySkillMessage = "请通过 M3 DI1 read 读取数字输入;使用 HWLAB API route /v1/m3/io。";
|
||||
const forbiddenDirectHardwareTarget = /(?:gateway(?:-simu)?|box(?:-simu)?|patch-panel|hwlab-patch-panel|\/invoke\b|\/sync\/tick\b|:7101\b|:7201\b|:7301\b)/iu;
|
||||
const statusRank = Object.freeze({
|
||||
pass: 0,
|
||||
not_run: 1,
|
||||
@@ -86,7 +103,11 @@ export async function buildRpt004Report(args = {}, options = {}) {
|
||||
const root = options.repoRoot ?? repoRoot;
|
||||
const report = baseReport({ parsed, now, root });
|
||||
|
||||
const expected = await buildExpectedArtifactState({ root, expectedCommit: parsed.expectedCommit });
|
||||
const expected = await buildExpectedArtifactState({
|
||||
root,
|
||||
expectedCommit: parsed.expectedCommit,
|
||||
fsJson: options.fsJson
|
||||
});
|
||||
report.expected = expected;
|
||||
report.dimensions.artifact = expected.check;
|
||||
addCheck(report, expected.check);
|
||||
@@ -139,16 +160,28 @@ export async function buildRpt004Report(args = {}, options = {}) {
|
||||
report.dimensions.layout = layoutCheck;
|
||||
addCheck(report, layoutCheck);
|
||||
|
||||
const codeAgentCheck = await runCodeAgentSkillPathDimension(parsed, options);
|
||||
report.dimensions.codeAgentSkillPath = codeAgentCheck;
|
||||
addCheck(report, codeAgentCheck);
|
||||
const codeAgentReport = await runCodeAgentSessionDimension(parsed, options);
|
||||
report.live.codeAgent = codeAgentReport.summary;
|
||||
report.dimensions.codeAgentSession = codeAgentReport.sessionCheck;
|
||||
report.dimensions.codeAgentSkillPath = codeAgentReport.skillPathCheck;
|
||||
addCheck(report, codeAgentReport.sessionCheck);
|
||||
addCheck(report, codeAgentReport.skillPathCheck);
|
||||
|
||||
const m3Report = await runM3TrustedLoop(parsed, options);
|
||||
report.live.m3 = m3Report.summary;
|
||||
report.dimensions.m3TrustedLoop = m3Report.loopCheck;
|
||||
report.dimensions.durableEvidence = m3Report.durableCheck;
|
||||
addCheck(report, m3Report.loopCheck);
|
||||
addCheck(report, m3Report.durableCheck);
|
||||
const skillCliRouteCheck = await runSkillCliM3RouteDimension(parsed, options);
|
||||
report.live.skillCliM3Route = skillCliRouteCheck.observations;
|
||||
report.dimensions.skillCliM3Route = skillCliRouteCheck;
|
||||
addCheck(report, skillCliRouteCheck);
|
||||
|
||||
if (parsed.allowM3Write) {
|
||||
const m3Report = await runM3TrustedLoop(parsed, options);
|
||||
report.live.m3 = m3Report.summary;
|
||||
report.dimensions.m3TrustedLoop = m3Report.loopCheck;
|
||||
report.dimensions.durableEvidence = m3Report.durableCheck;
|
||||
addCheck(report, m3Report.loopCheck);
|
||||
addCheck(report, m3Report.durableCheck);
|
||||
} else {
|
||||
addReadOnlyM3Dimensions(report);
|
||||
}
|
||||
|
||||
return finalizeReport(report);
|
||||
}
|
||||
@@ -164,6 +197,7 @@ export function parseArgs(argv = []) {
|
||||
writeReport: false,
|
||||
pretty: false,
|
||||
timeoutMs: 10000,
|
||||
allowM3Write: false,
|
||||
help: false
|
||||
};
|
||||
|
||||
@@ -188,6 +222,8 @@ export function parseArgs(argv = []) {
|
||||
args.pretty = true;
|
||||
} else if (arg === "--timeout-ms") {
|
||||
args.timeoutMs = parseTimeout(readArg(argv, ++index, arg));
|
||||
} else if (arg === "--allow-m3-write") {
|
||||
args.allowM3Write = true;
|
||||
} else if (arg === "--help" || arg === "-h") {
|
||||
args.help = true;
|
||||
} else {
|
||||
@@ -205,7 +241,7 @@ export function parseArgs(argv = []) {
|
||||
return args;
|
||||
}
|
||||
|
||||
export function classifyRpt004Conclusion(checks = []) {
|
||||
export function classifyRpt004Conclusion(checks = [], options = {}) {
|
||||
const byId = new Map(checks.map((check) => [check.id, check]));
|
||||
const revision = byId.get("live-revision-current");
|
||||
if (revision?.status === "not_current") {
|
||||
@@ -223,7 +259,9 @@ export function classifyRpt004Conclusion(checks = []) {
|
||||
status: "pass",
|
||||
classification: "MVP_PASS",
|
||||
mvpPass: true,
|
||||
summary: "RPT-004 MVP PASS: current live revision, default Chinese workbench, layout smoke, Code Agent skill path, M3 trusted IO loop, and durable evidence are green."
|
||||
summary: options.allowM3Write === true
|
||||
? "RPT-004 MVP PASS: current live revision, default Chinese workbench, layout smoke, Code Agent session, Skill CLI HWLAB API route, M3 trusted IO loop, and durable evidence are green."
|
||||
: "RPT-004 MVP PASS: current live revision, default Chinese workbench, layout smoke, Code Agent session, Skill CLI HWLAB API route, and default read-only M3 route verification are green; no M3 write was attempted."
|
||||
};
|
||||
}
|
||||
|
||||
@@ -262,13 +300,23 @@ export function classifyCodeAgentSkillPath(payload, { httpStatus = null, httpOk
|
||||
skill?.name === HWLAB_M3_IO_SKILL_NAME ||
|
||||
String(skill?.source ?? "").includes("skills/hwlab-agent-runtime/SKILL.md")
|
||||
);
|
||||
const runnerKind = payload?.runner?.kind ?? null;
|
||||
const controlledSessionRunner =
|
||||
runnerKind === "codex-mcp-stdio-runner" ||
|
||||
runnerKind === "hwlab-readonly-runner";
|
||||
const missing = [];
|
||||
if (!httpOk) missing.push(`HTTP ${httpStatus ?? "not_observed"}`);
|
||||
if (capability.status !== "pass") missing.push(capability.blocker ?? "runner-capability");
|
||||
if (capability.status !== "pass" && !controlledSessionRunner) {
|
||||
missing.push(capability.blocker ?? "runner-capability");
|
||||
}
|
||||
if (openAiFallback) missing.push("openai-fallback");
|
||||
if (!skillsDiscoverCompleted) missing.push("skills.discover toolCall");
|
||||
if (payload?.skills?.status !== "ready") missing.push("skills.status=ready");
|
||||
if (!repoSkillReady) missing.push("repo-owned hwlab-agent-runtime skill");
|
||||
if (!payload?.runnerTrace) missing.push("runnerTrace");
|
||||
if (!payload?.sessionMode) missing.push("sessionMode");
|
||||
if (!payload?.capabilityLevel) missing.push("capabilityLevel");
|
||||
if (!controlledSessionRunner) missing.push("controlled Code Agent runner");
|
||||
|
||||
return {
|
||||
id: "code-agent-skill-path",
|
||||
@@ -281,8 +329,9 @@ export function classifyCodeAgentSkillPath(payload, { httpStatus = null, httpOk
|
||||
provider: payload?.provider ?? null,
|
||||
model: payload?.model ?? null,
|
||||
backend: payload?.backend ?? null,
|
||||
runnerKind: payload?.runner?.kind ?? null,
|
||||
runnerKind,
|
||||
capability,
|
||||
controlledSessionRunner,
|
||||
openAiFallback,
|
||||
toolCalls: toolCalls.map((tool) => ({
|
||||
name: tool?.name ?? null,
|
||||
@@ -299,6 +348,219 @@ export function classifyCodeAgentSkillPath(payload, { httpStatus = null, httpOk
|
||||
};
|
||||
}
|
||||
|
||||
export function classifyCodeAgentSessionTurns(turns = [], { expectedConversationId = null } = {}) {
|
||||
const normalizedTurns = Array.isArray(turns) ? turns : [];
|
||||
const blockers = [];
|
||||
if (normalizedTurns.length < 3) blockers.push("three-turn-session");
|
||||
const completedTurns = normalizedTurns.filter((turn) => turn?.payload?.status === "completed" && turn?.response?.ok === true);
|
||||
if (completedTurns.length !== normalizedTurns.length || normalizedTurns.length === 0) {
|
||||
blockers.push("completed-http-turns");
|
||||
}
|
||||
|
||||
const conversationIds = new Set();
|
||||
const sessionIds = new Set();
|
||||
const turnSummaries = normalizedTurns.map((turn, index) => {
|
||||
const payload = turn?.payload ?? {};
|
||||
const response = turn?.response ?? {};
|
||||
const capability = classifyCodexRunnerCapability(payload, {
|
||||
httpStatus: response.status ?? null
|
||||
});
|
||||
const toolCalls = Array.isArray(payload.toolCalls) ? payload.toolCalls : [];
|
||||
const toolNames = toolCalls.map((tool) => tool?.name).filter(Boolean);
|
||||
const runnerKind = payload.runner?.kind ?? null;
|
||||
const sessionMode = payload.sessionMode ?? payload.runner?.sessionMode ?? null;
|
||||
const sessionId = payload.sessionId ?? payload.session?.sessionId ?? null;
|
||||
const conversationId = payload.conversationId ?? null;
|
||||
const traceId = payload.traceId ?? null;
|
||||
const runnerTrace = payload.runnerTrace && typeof payload.runnerTrace === "object" ? payload.runnerTrace : null;
|
||||
const fallback = isOpenAiFallbackPayload(payload);
|
||||
const partialRunner = payload.provider === "codex-readonly-runner" || runnerKind === "hwlab-readonly-runner";
|
||||
const expectedTool = turn?.kind === "skills"
|
||||
? "skills.discover"
|
||||
: "pwd";
|
||||
const hasExpectedTool = toolNames.includes(expectedTool);
|
||||
const toolCallsCompleted = toolCalls.length > 0 && toolCalls.every((tool) => tool?.status === "completed");
|
||||
const reply = typeof payload.reply?.content === "string" ? payload.reply.content : "";
|
||||
const referencesPrevious = turn?.kind === "context-pwd"
|
||||
? /pwd|第一轮|前文|same session|同一/u.test(reply)
|
||||
: true;
|
||||
const ready = response.ok === true &&
|
||||
payload.status === "completed" &&
|
||||
Boolean(conversationId) &&
|
||||
Boolean(sessionId) &&
|
||||
Boolean(traceId) &&
|
||||
Boolean(runnerKind) &&
|
||||
Boolean(sessionMode) &&
|
||||
Boolean(payload.capabilityLevel) &&
|
||||
Boolean(runnerTrace) &&
|
||||
hasExpectedTool &&
|
||||
toolCallsCompleted &&
|
||||
!fallback;
|
||||
|
||||
if (conversationId) conversationIds.add(conversationId);
|
||||
if (sessionId) sessionIds.add(sessionId);
|
||||
return {
|
||||
index: index + 1,
|
||||
kind: turn?.kind ?? "unknown",
|
||||
status: ready && referencesPrevious ? "pass" : "blocked",
|
||||
httpStatus: response.status ?? null,
|
||||
conversationId,
|
||||
sessionId,
|
||||
traceId,
|
||||
provider: payload.provider ?? null,
|
||||
backend: payload.backend ?? null,
|
||||
runnerKind,
|
||||
sessionMode,
|
||||
capabilityLevel: payload.capabilityLevel ?? null,
|
||||
capability,
|
||||
toolNames,
|
||||
toolCallsCompleted,
|
||||
hasExpectedTool,
|
||||
runnerTracePresent: Boolean(runnerTrace),
|
||||
fallback,
|
||||
partialRunner,
|
||||
turn: payload.sessionReuse?.turn ?? payload.session?.turn ?? payload.runnerTrace?.turn ?? null,
|
||||
reused: payload.sessionReuse?.reused ?? payload.session?.reused ?? payload.runnerTrace?.sessionReused ?? null,
|
||||
referencesPrevious,
|
||||
blockers: [
|
||||
...(response.ok === true ? [] : [`HTTP ${response.status ?? "not_observed"}`]),
|
||||
...(payload.status === "completed" ? [] : ["payload.status=completed"]),
|
||||
...(conversationId ? [] : ["conversationId"]),
|
||||
...(sessionId ? [] : ["sessionId"]),
|
||||
...(traceId ? [] : ["traceId"]),
|
||||
...(runnerKind ? [] : ["runner.kind"]),
|
||||
...(sessionMode ? [] : ["sessionMode"]),
|
||||
...(payload.capabilityLevel ? [] : ["capabilityLevel"]),
|
||||
...(runnerTrace ? [] : ["runnerTrace"]),
|
||||
...(hasExpectedTool ? [] : [`toolCall ${expectedTool}`]),
|
||||
...(toolCallsCompleted ? [] : ["toolCalls completed"]),
|
||||
...(fallback ? ["openai-fallback"] : []),
|
||||
...(referencesPrevious ? [] : ["context-reference"])
|
||||
]
|
||||
};
|
||||
});
|
||||
|
||||
if (expectedConversationId && !conversationIds.has(expectedConversationId)) {
|
||||
blockers.push("expected-conversationId");
|
||||
}
|
||||
if (conversationIds.size !== 1) blockers.push("same-conversationId");
|
||||
if (sessionIds.size !== 1) blockers.push("same-sessionId");
|
||||
if (turnSummaries.some((turn) => turn.fallback)) blockers.push("fallback-only");
|
||||
if (turnSummaries.some((turn) => turn.runnerTracePresent !== true)) blockers.push("runnerTrace");
|
||||
if (turnSummaries.some((turn) => turn.toolCallsCompleted !== true)) blockers.push("toolCalls");
|
||||
if (turnSummaries.some((turn) => turn.hasExpectedTool !== true)) blockers.push("expected-toolCalls");
|
||||
if (turnSummaries.some((turn) => !turn.sessionMode || !turn.capabilityLevel || !turn.runnerKind)) {
|
||||
blockers.push("runner/session/capability evidence");
|
||||
}
|
||||
if (turnSummaries.some((turn) => turn.kind === "context-pwd" && turn.referencesPrevious !== true)) {
|
||||
blockers.push("context-reference");
|
||||
}
|
||||
|
||||
const pass = blockers.length === 0 && turnSummaries.every((turn) => turn.status === "pass");
|
||||
return {
|
||||
id: "code-agent-session-thread",
|
||||
status: pass ? "pass" : "blocked",
|
||||
summary: pass
|
||||
? "Code Agent completed three same-conversation/session turns for pwd, skills, and prior-context reference with runner/tool/session evidence."
|
||||
: `Code Agent session thread is blocked: missing ${[...new Set(blockers)].join(", ")}.`,
|
||||
evidenceLevel: pass ? "DEV-LIVE" : "BLOCKED",
|
||||
observations: {
|
||||
expectedConversationId,
|
||||
conversationIds: [...conversationIds],
|
||||
sessionIds: [...sessionIds],
|
||||
turnCount: normalizedTurns.length,
|
||||
turns: turnSummaries,
|
||||
fallbackOnly: turnSummaries.length > 0 && turnSummaries.every((turn) => turn.fallback),
|
||||
partialRunnerOnly: turnSummaries.length > 0 && turnSummaries.every((turn) => turn.partialRunner),
|
||||
blockers: [...new Set(blockers)]
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function classifySkillCliM3Route(payload, { httpStatus = null, httpOk = false } = {}) {
|
||||
const toolCalls = Array.isArray(payload?.toolCalls) ? payload.toolCalls : [];
|
||||
const skillTool = toolCalls.find((tool) =>
|
||||
tool?.name === HWLAB_M3_IO_SKILL_NAME ||
|
||||
tool?.route === HWLAB_M3_IO_API_ROUTE ||
|
||||
tool?.route === M3_IO_CONTROL_ROUTE
|
||||
) ?? null;
|
||||
const providerTrace = payload?.providerTrace && typeof payload.providerTrace === "object" ? payload.providerTrace : null;
|
||||
const route = skillTool?.route ?? providerTrace?.route ?? payload?.skills?.items?.find((item) => item?.route)?.route ?? null;
|
||||
const operationId = skillTool?.operationId ?? providerTrace?.operationId ?? null;
|
||||
const traceId = skillTool?.traceId ?? providerTrace?.traceId ?? payload?.traceId ?? null;
|
||||
const auditId = skillTool?.audit?.auditId ?? null;
|
||||
const evidenceId = skillTool?.evidence?.evidenceId ?? null;
|
||||
const blockers = normalizeM3RouteBlockers(payload, skillTool);
|
||||
const safety = {
|
||||
directGatewayCalls: Boolean(skillTool?.directGatewayCalls ?? payload?.runnerTrace?.directGatewayCalls ?? false),
|
||||
directBoxCalls: Boolean(skillTool?.directBoxCalls ?? payload?.runnerTrace?.directBoxCalls ?? false),
|
||||
directPatchPanelCalls: Boolean(skillTool?.directPatchPanelCalls ?? payload?.runnerTrace?.directPatchPanelCalls ?? false),
|
||||
fallbackUsed: Boolean(skillTool?.fallbackUsed ?? payload?.providerTrace?.fallbackUsed ?? payload?.runnerTrace?.fallbackUsed ?? isOpenAiFallbackPayload(payload)),
|
||||
routeOnly: route === HWLAB_M3_IO_API_ROUTE || route === M3_IO_CONTROL_ROUTE,
|
||||
directTargetMentioned: directHardwareTargetMentioned(payload)
|
||||
};
|
||||
const structuredBlocker = blockers.length > 0;
|
||||
const idsPresent = Boolean(operationId && traceId && auditId && evidenceId);
|
||||
const accepted = skillTool?.accepted === true || providerTrace?.accepted === true;
|
||||
const controlReady = skillTool?.controlReady === true || providerTrace?.controlReady === true;
|
||||
const routeEvidenceReady = route === HWLAB_M3_IO_API_ROUTE || route === M3_IO_CONTROL_ROUTE;
|
||||
const routePass = httpOk === true &&
|
||||
payload?.status === "completed" &&
|
||||
payload?.provider === "hwlab-skill-cli" &&
|
||||
skillTool &&
|
||||
["completed", "blocked"].includes(skillTool.status) &&
|
||||
routeEvidenceReady &&
|
||||
safety.directGatewayCalls === false &&
|
||||
safety.directBoxCalls === false &&
|
||||
safety.directPatchPanelCalls === false &&
|
||||
safety.fallbackUsed === false &&
|
||||
safety.directTargetMentioned === false &&
|
||||
(idsPresent || structuredBlocker);
|
||||
|
||||
const missing = [];
|
||||
if (!httpOk) missing.push(`HTTP ${httpStatus ?? "not_observed"}`);
|
||||
if (payload?.status !== "completed") missing.push("payload.status=completed");
|
||||
if (payload?.provider !== "hwlab-skill-cli") missing.push("provider=hwlab-skill-cli");
|
||||
if (!skillTool) missing.push(`${HWLAB_M3_IO_SKILL_NAME} toolCall`);
|
||||
if (!routeEvidenceReady) missing.push(`route=${HWLAB_M3_IO_API_ROUTE}`);
|
||||
if (safety.directGatewayCalls || safety.directBoxCalls || safety.directPatchPanelCalls || safety.directTargetMentioned) {
|
||||
missing.push("no direct gateway/box/patch-panel target");
|
||||
}
|
||||
if (safety.fallbackUsed) missing.push("no fallback");
|
||||
if (!idsPresent && !structuredBlocker) missing.push("operationId/traceId/auditId/evidenceId or structured blocker");
|
||||
|
||||
return {
|
||||
id: "skill-cli-hwlab-api-m3-route",
|
||||
status: routePass ? "pass" : "blocked",
|
||||
summary: routePass
|
||||
? structuredBlocker
|
||||
? `Skill CLI reached ${HWLAB_M3_IO_API_ROUTE} and returned a structured blocker instead of direct hardware/fallback evidence.`
|
||||
: `Skill CLI used HWLAB API ${HWLAB_M3_IO_API_ROUTE} with operation/audit/evidence identifiers and no direct hardware/fallback path.`
|
||||
: `Skill CLI HWLAB API route check is blocked: missing ${missing.join(", ")}.`,
|
||||
evidenceLevel: routePass ? "DEV-LIVE" : "BLOCKED",
|
||||
observations: {
|
||||
httpStatus,
|
||||
provider: payload?.provider ?? null,
|
||||
backend: payload?.backend ?? null,
|
||||
route,
|
||||
expectedRoute: HWLAB_M3_IO_API_ROUTE,
|
||||
toolCallStatus: skillTool?.status ?? null,
|
||||
accepted,
|
||||
controlReady,
|
||||
capabilityLevel: skillTool?.capabilityLevel ?? payload?.capabilityLevel ?? null,
|
||||
operationId,
|
||||
traceId,
|
||||
auditId,
|
||||
evidenceId,
|
||||
idsPresent,
|
||||
structuredBlocker,
|
||||
blockers,
|
||||
safety,
|
||||
missing
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function classifyM3DurableEvidence(operations = []) {
|
||||
const byId = new Map(operations.map((operation) => [operation.id, operation]));
|
||||
const sequence = expectedM3OperationIds.map(([id, action, expectedValue]) => {
|
||||
@@ -341,6 +603,7 @@ export function classifyM3DurableEvidence(operations = []) {
|
||||
|
||||
function baseReport({ parsed, now, root }) {
|
||||
const source = observeSource(root);
|
||||
const liveMutationUsed = parsed.live === true && parsed.allowM3Write === true;
|
||||
return {
|
||||
$schema: "https://hwlab.pikastech.local/schemas/rpt-004-mvp-e2e-harness.schema.json",
|
||||
$id: "https://hwlab.pikastech.local/reports/dev-gate/rpt-004-mvp-e2e.json",
|
||||
@@ -351,11 +614,16 @@ function baseReport({ parsed, now, root }) {
|
||||
"pikasTech/HWLAB#7",
|
||||
"pikasTech/HWLAB#78",
|
||||
"pikasTech/HWLAB#131",
|
||||
"pikasTech/HWLAB#143",
|
||||
"pikasTech/HWLAB#227",
|
||||
"pikasTech/HWLAB#236",
|
||||
"pikasTech/HWLAB#239",
|
||||
"pikasTech/HWLAB#275",
|
||||
"pikasTech/HWLAB#315",
|
||||
"pikasTech/HWLAB#316",
|
||||
"pikasTech/HWLAB#334"
|
||||
"pikasTech/HWLAB#317",
|
||||
"pikasTech/HWLAB#334",
|
||||
"pikasTech/HWLAB#351"
|
||||
],
|
||||
taskId,
|
||||
task: "DC-DCSN-P0-2026-003",
|
||||
@@ -376,6 +644,16 @@ function baseReport({ parsed, now, root }) {
|
||||
webHealthUrl: joinBasePath(parsed.url, "/health/live"),
|
||||
m3IoUrl: joinBasePath(parsed.apiUrl, M3_IO_CONTROL_ROUTE)
|
||||
},
|
||||
runPolicy: {
|
||||
defaultLiveMode: "read-only",
|
||||
liveMutationUsed,
|
||||
allowM3Write: parsed.allowM3Write === true,
|
||||
m3WriteRequiresExplicitFlag: "--allow-m3-write",
|
||||
liveMutationAllowedOnlyOnDev: true,
|
||||
summary: liveMutationUsed
|
||||
? "Explicit DEV-only --allow-m3-write was provided; bounded simulator DO/DI write loop may run after current+healthy preconditions."
|
||||
: "Default live mode performs only read-only live checks and payload classification; no M3 DO1 true/false write is attempted."
|
||||
},
|
||||
sourceContract: {
|
||||
status: "pass",
|
||||
documents: [
|
||||
@@ -427,7 +705,9 @@ function baseReport({ parsed, now, root }) {
|
||||
layout: null,
|
||||
m3TrustedLoop: null,
|
||||
durableEvidence: null,
|
||||
codeAgentSkillPath: null
|
||||
codeAgentSkillPath: null,
|
||||
codeAgentSession: null,
|
||||
skillCliM3Route: null
|
||||
},
|
||||
checks: [],
|
||||
blockers: [],
|
||||
@@ -440,17 +720,24 @@ function baseReport({ parsed, now, root }) {
|
||||
readsKubernetesSecrets: false,
|
||||
secretValuesPrinted: false,
|
||||
liveM3WritesRequireCurrentHealthyRevision: true,
|
||||
liveM3WritesRequireExplicitFlag: true,
|
||||
liveMutationUsed,
|
||||
m3WritesLimitedToSimulatorDoDiResetLoop: true,
|
||||
openAiFallbackCountsAsPass: false,
|
||||
statement: "This harness performs only public DEV HTTP checks, browser layout smoke, Code Agent skill discovery, and the bounded M3 simulator DO/DI loop after current+healthy preconditions. It never applies deploys or starts rollouts."
|
||||
statement: liveMutationUsed
|
||||
? "This harness performs public DEV HTTP checks, browser layout smoke, Code Agent session/Skill CLI checks, and an explicit DEV-only bounded M3 simulator DO/DI loop after current+healthy preconditions. It never applies deploys or starts rollouts."
|
||||
: "Default live mode performs public DEV HTTP checks, browser layout smoke, Code Agent session/Skill CLI checks, and read-only M3 route payload classification only. It never writes DO1, applies deploys, starts rollouts, reads secrets, or touches PROD."
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function buildExpectedArtifactState({ root, expectedCommit }) {
|
||||
const deploy = await readJsonOptional(root, "deploy/deploy.json");
|
||||
const catalog = await readJsonOptional(root, "deploy/artifact-catalog.dev.json");
|
||||
const artifactReport = await readJsonOptional(root, "reports/dev-gate/dev-artifacts.json");
|
||||
async function buildExpectedArtifactState({ root, expectedCommit, fsJson }) {
|
||||
const readJson = typeof fsJson === "function"
|
||||
? (relativePath) => fsJson(relativePath)
|
||||
: (relativePath) => readJsonOptional(root, relativePath);
|
||||
const deploy = await readJson("deploy/deploy.json");
|
||||
const catalog = await readJson("deploy/artifact-catalog.dev.json");
|
||||
const artifactReport = await readJson("reports/dev-gate/dev-artifacts.json");
|
||||
const deployCommit = sanitizeCommit(deploy?.commitId);
|
||||
const catalogCommit = sanitizeCommit(catalog?.commitId);
|
||||
const reportCommit = sanitizeCommit(artifactReport?.artifactPublish?.sourceCommitId ?? artifactReport?.commitId);
|
||||
@@ -705,6 +992,121 @@ async function runCodeAgentSkillPathDimension(parsed, options) {
|
||||
};
|
||||
}
|
||||
|
||||
async function runCodeAgentSessionDimension(parsed, options) {
|
||||
const endpoint = joinBasePath(parsed.apiUrl, "/v1/agent/chat");
|
||||
const conversationId = `cnv_rpt004_session_${randomUUID()}`;
|
||||
const requestedSessionId = `ses_rpt004_session_${randomUUID()}`;
|
||||
const turns = [];
|
||||
for (const turn of codeAgentSessionTurns) {
|
||||
const response = options.codeAgentRunner
|
||||
? await options.codeAgentRunner({
|
||||
endpoint,
|
||||
parsed,
|
||||
conversationId,
|
||||
sessionId: requestedSessionId,
|
||||
turn
|
||||
})
|
||||
: await postJson(endpoint, {
|
||||
conversationId,
|
||||
sessionId: requestedSessionId,
|
||||
traceId: `trc_rpt004_${turn.kind}_${randomUUID()}`,
|
||||
projectId: "prj_hwlab-cloud-workbench",
|
||||
message: turn.message
|
||||
}, parsed.timeoutMs);
|
||||
turns.push({
|
||||
kind: turn.kind,
|
||||
message: turn.message,
|
||||
response: {
|
||||
ok: response.ok,
|
||||
status: response.status
|
||||
},
|
||||
payload: response.json ?? response.body ?? null
|
||||
});
|
||||
}
|
||||
|
||||
const sessionCheck = classifyCodeAgentSessionTurns(turns, { expectedConversationId: conversationId });
|
||||
const skillPayload = turns.find((turn) => turn.kind === "skills")?.payload ?? null;
|
||||
const skillResponse = turns.find((turn) => turn.kind === "skills")?.response ?? {};
|
||||
const skillPathCheck = classifyCodeAgentSkillPath(skillPayload, {
|
||||
httpStatus: skillResponse.status ?? null,
|
||||
httpOk: skillResponse.ok === true
|
||||
});
|
||||
|
||||
return {
|
||||
sessionCheck: {
|
||||
...sessionCheck,
|
||||
observations: {
|
||||
...sessionCheck.observations,
|
||||
endpoint
|
||||
}
|
||||
},
|
||||
skillPathCheck: {
|
||||
...skillPathCheck,
|
||||
observations: {
|
||||
...skillPathCheck.observations,
|
||||
endpoint,
|
||||
conversationId,
|
||||
sessionId: requestedSessionId,
|
||||
traceId: skillPayload?.traceId ?? null,
|
||||
messageId: skillPayload?.messageId ?? null,
|
||||
assistantReplyStored: false
|
||||
}
|
||||
},
|
||||
summary: {
|
||||
endpoint,
|
||||
status: sessionCheck.status === "pass" && skillPathCheck.status === "pass" ? "pass" : "blocked",
|
||||
conversationId,
|
||||
sessionId: requestedSessionId,
|
||||
turns: sessionCheck.observations.turns.map((turn) => ({
|
||||
kind: turn.kind,
|
||||
status: turn.status,
|
||||
runnerKind: turn.runnerKind,
|
||||
sessionMode: turn.sessionMode,
|
||||
capabilityLevel: turn.capabilityLevel,
|
||||
traceId: turn.traceId,
|
||||
toolNames: turn.toolNames
|
||||
}))
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function runSkillCliM3RouteDimension(parsed, options) {
|
||||
const endpoint = joinBasePath(parsed.apiUrl, "/v1/agent/chat");
|
||||
const conversationId = `cnv_rpt004_skill_m3_${randomUUID()}`;
|
||||
const sessionId = `ses_rpt004_skill_m3_${randomUUID()}`;
|
||||
const response = options.skillCliRunner
|
||||
? await options.skillCliRunner({
|
||||
endpoint,
|
||||
parsed,
|
||||
conversationId,
|
||||
sessionId,
|
||||
message: m3ReadOnlySkillMessage
|
||||
})
|
||||
: await postJson(endpoint, {
|
||||
conversationId,
|
||||
sessionId,
|
||||
traceId: `trc_rpt004_skill_m3_${randomUUID()}`,
|
||||
projectId: "prj_hwlab-cloud-workbench",
|
||||
message: m3ReadOnlySkillMessage
|
||||
}, parsed.timeoutMs);
|
||||
const payload = response.json ?? response.body ?? null;
|
||||
const check = classifySkillCliM3Route(payload, {
|
||||
httpStatus: response.status,
|
||||
httpOk: response.ok === true
|
||||
});
|
||||
return {
|
||||
...check,
|
||||
observations: {
|
||||
...check.observations,
|
||||
endpoint,
|
||||
conversationId,
|
||||
sessionId,
|
||||
message: m3ReadOnlySkillMessage,
|
||||
liveMutationUsed: false
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function runM3TrustedLoop(parsed, options) {
|
||||
const m3Runner = options.m3Runner ?? defaultM3Runner;
|
||||
const fullReport = await m3Runner(parsed);
|
||||
@@ -774,7 +1176,9 @@ async function defaultM3Runner(parsed) {
|
||||
function addNotRunDimensions(report, reason) {
|
||||
for (const [key, id] of [
|
||||
["layout", "layout-smoke-hit-target"],
|
||||
["codeAgentSession", "code-agent-session-thread"],
|
||||
["codeAgentSkillPath", "code-agent-skill-path"],
|
||||
["skillCliM3Route", "skill-cli-hwlab-api-m3-route"],
|
||||
["m3TrustedLoop", "m3-v1-io-trusted-loop"],
|
||||
["durableEvidence", "operation-audit-evidence-durable-green"]
|
||||
]) {
|
||||
@@ -794,12 +1198,59 @@ function addNotRunDimensions(report, reason) {
|
||||
}
|
||||
}
|
||||
|
||||
function addReadOnlyM3Dimensions(report) {
|
||||
const loopCheck = {
|
||||
id: "m3-v1-io-trusted-loop",
|
||||
status: "pass",
|
||||
summary: "Default RPT-004 live mode did not write DO1; M3 route readiness is represented by the Skill CLI HWLAB API read-only route check.",
|
||||
evidenceLevel: "DEV-LIVE",
|
||||
observations: {
|
||||
skippedWriteLoop: true,
|
||||
liveMutationUsed: false,
|
||||
allowM3Write: false,
|
||||
requiredExplicitFlag: "--allow-m3-write",
|
||||
route: M3_IO_CONTROL_ROUTE,
|
||||
reason: "Default mode must not perform DO1 true/false writes."
|
||||
}
|
||||
};
|
||||
const durableCheck = {
|
||||
id: "operation-audit-evidence-durable-green",
|
||||
status: "pass",
|
||||
summary: "No DO1 write was attempted in default read-only mode; durable write evidence is not required unless --allow-m3-write is explicitly provided.",
|
||||
evidenceLevel: "DEV-LIVE",
|
||||
observations: {
|
||||
notApplicableInDefaultReadOnlyMode: true,
|
||||
liveMutationUsed: false,
|
||||
allowM3Write: false,
|
||||
requiredExplicitFlag: "--allow-m3-write",
|
||||
evidenceIds: report.dimensions.skillCliM3Route?.observations
|
||||
? [report.dimensions.skillCliM3Route.observations.evidenceId].filter(Boolean)
|
||||
: []
|
||||
}
|
||||
};
|
||||
report.live.m3 = {
|
||||
mode: "read-only-route-check",
|
||||
status: "pass",
|
||||
classification: "READ_ONLY_ROUTE_CHECK",
|
||||
trustedGreen: false,
|
||||
liveMutationUsed: false,
|
||||
operationCount: 0
|
||||
};
|
||||
report.dimensions.m3TrustedLoop = loopCheck;
|
||||
report.dimensions.durableEvidence = durableCheck;
|
||||
addCheck(report, loopCheck);
|
||||
addCheck(report, durableCheck);
|
||||
}
|
||||
|
||||
function finalizeReport(report) {
|
||||
const conclusion = classifyRpt004Conclusion(report.checks);
|
||||
const conclusion = classifyRpt004Conclusion(report.checks, {
|
||||
allowM3Write: report.runPolicy?.allowM3Write === true
|
||||
});
|
||||
report.status = conclusion.status;
|
||||
report.conclusion = conclusion;
|
||||
report.evidenceLevel = conclusion.mvpPass ? "DEV-LIVE-MVP" : conclusion.classification;
|
||||
report.devLive = conclusion.mvpPass;
|
||||
report.liveMutationUsed = report.runPolicy?.liveMutationUsed === true;
|
||||
report.localSmoke.status = report.mode === "source-check" ? "pass" : "not_run";
|
||||
report.devPreconditions.status = conclusion.status === "pass" ? "pass" : conclusion.status;
|
||||
report.blockers = report.checks
|
||||
@@ -812,9 +1263,64 @@ function finalizeReport(report) {
|
||||
summary: check.summary,
|
||||
evidenceLevel: check.evidenceLevel ?? "BLOCKED"
|
||||
}));
|
||||
report.commanderSummary = buildCommanderSummary(report);
|
||||
return report;
|
||||
}
|
||||
|
||||
function buildCommanderSummary(report) {
|
||||
const route = report.dimensions.skillCliM3Route?.observations ?? {};
|
||||
const session = report.dimensions.codeAgentSession?.observations ?? {};
|
||||
const skillPath = report.dimensions.codeAgentSkillPath?.observations ?? {};
|
||||
const ids = {
|
||||
operationId: route.operationId ?? null,
|
||||
traceId: route.traceId ?? session.turns?.find((turn) => turn.traceId)?.traceId ?? null,
|
||||
auditId: route.auditId ?? null,
|
||||
evidenceId: route.evidenceId ?? null
|
||||
};
|
||||
return {
|
||||
reportClass: "live-mutating",
|
||||
allowedLiveMutationUsed: report.liveMutationUsed === true,
|
||||
commit: report.expected?.commitId ?? "unknown",
|
||||
revision: {
|
||||
status: report.dimensions.revision?.status ?? "not_run",
|
||||
expectedCommit: report.expected?.commitId ?? "unknown",
|
||||
apiRevision: report.dimensions.revision?.observations?.api?.revision ?? null,
|
||||
webRevision: report.dimensions.revision?.observations?.web?.revision ?? null
|
||||
},
|
||||
sessionResult: {
|
||||
status: report.dimensions.codeAgentSession?.status ?? "not_run",
|
||||
conversationIds: session.conversationIds ?? [],
|
||||
sessionIds: session.sessionIds ?? [],
|
||||
turnCount: session.turnCount ?? 0,
|
||||
runnerKinds: [...new Set((session.turns ?? []).map((turn) => turn.runnerKind).filter(Boolean))],
|
||||
sessionModes: [...new Set((session.turns ?? []).map((turn) => turn.sessionMode).filter(Boolean))],
|
||||
capabilityLevels: [...new Set((session.turns ?? []).map((turn) => turn.capabilityLevel).filter(Boolean))]
|
||||
},
|
||||
skillCliResult: {
|
||||
status: report.dimensions.codeAgentSkillPath?.status ?? "not_run",
|
||||
runnerKind: skillPath.runnerKind ?? null,
|
||||
skillsStatus: skillPath.skills?.status ?? null,
|
||||
repoSkillReady: skillPath.skills?.repoSkillReady === true,
|
||||
openAiFallback: skillPath.openAiFallback === true
|
||||
},
|
||||
m3RouteResult: {
|
||||
status: report.dimensions.skillCliM3Route?.status ?? "not_run",
|
||||
route: route.route ?? null,
|
||||
expectedRoute: route.expectedRoute ?? HWLAB_M3_IO_API_ROUTE,
|
||||
structuredBlocker: route.structuredBlocker === true,
|
||||
idsPresent: route.idsPresent === true,
|
||||
safety: route.safety ?? null
|
||||
},
|
||||
evidenceIds: Object.fromEntries(Object.entries(ids).filter(([, value]) => Boolean(value))),
|
||||
blockers: report.blockers.map((blocker) => ({
|
||||
scope: blocker.scope,
|
||||
summary: blocker.summary,
|
||||
evidenceLevel: blocker.evidenceLevel
|
||||
})),
|
||||
artifactReport: "not applicable: harness/scripts/tests only; no runtime service image was changed by RPT-004."
|
||||
};
|
||||
}
|
||||
|
||||
function addCheck(report, check) {
|
||||
if (!check) return;
|
||||
report.checks.push(check);
|
||||
@@ -840,6 +1346,8 @@ function compactReport(report) {
|
||||
summary: value.summary
|
||||
} : null
|
||||
])),
|
||||
liveMutationUsed: report.liveMutationUsed,
|
||||
commanderSummary: report.commanderSummary,
|
||||
blockers: report.blockers,
|
||||
artifacts: report.artifacts ?? null,
|
||||
safety: report.safety
|
||||
@@ -872,6 +1380,10 @@ export function renderMarkdown(report) {
|
||||
`- mvpPass: ${report.conclusion?.mvpPass === true}`,
|
||||
`- generatedAt: ${report.generatedAt}`,
|
||||
`- expectedCommit: ${report.expected?.commitId ?? "unknown"}`,
|
||||
`- liveMutationUsed: ${report.liveMutationUsed === true}`,
|
||||
`- allowM3Write: ${report.runPolicy?.allowM3Write === true}`,
|
||||
`- skillCliRoute: ${report.commanderSummary?.m3RouteResult?.route ?? "not_run"}`,
|
||||
`- evidenceId: ${report.commanderSummary?.evidenceIds?.evidenceId ?? "none"}`,
|
||||
"",
|
||||
"| Dimension | Status | Summary |",
|
||||
"| --- | --- | --- |",
|
||||
@@ -889,6 +1401,7 @@ export function renderMarkdown(report) {
|
||||
`- rolloutAttempted: ${report.safety.rolloutAttempted}`,
|
||||
`- secretValuesPrinted: ${report.safety.secretValuesPrinted}`,
|
||||
`- openAiFallbackCountsAsPass: ${report.safety.openAiFallbackCountsAsPass}`,
|
||||
`- liveMutationUsed: ${report.safety.liveMutationUsed === true}`,
|
||||
""
|
||||
].join("\n");
|
||||
}
|
||||
@@ -1140,6 +1653,79 @@ function parseJson(value) {
|
||||
}
|
||||
}
|
||||
|
||||
function isOpenAiFallbackPayload(payload) {
|
||||
return payload?.provider === "openai-responses" ||
|
||||
payload?.runner?.kind === "openai-responses-fallback" ||
|
||||
payload?.implementationType === "openai-responses-fallback" ||
|
||||
payload?.providerTrace?.fallbackUsed === true;
|
||||
}
|
||||
|
||||
function normalizeM3RouteBlockers(payload, skillTool) {
|
||||
const blockers = [];
|
||||
for (const raw of [
|
||||
skillTool?.blocker,
|
||||
skillTool?.capabilityBlocker,
|
||||
skillTool?.trustBlocker,
|
||||
payload?.error,
|
||||
payload?.availability?.blocker,
|
||||
...(Array.isArray(skillTool?.blockers) ? skillTool.blockers : []),
|
||||
...(Array.isArray(payload?.skills?.blockers) ? payload.skills.blockers : []),
|
||||
...(Array.isArray(payload?.runnerTrace?.blockers) ? payload.runnerTrace.blockers : []),
|
||||
...(Array.isArray(payload?.availability?.blockers) ? payload.availability.blockers : [])
|
||||
]) {
|
||||
const blocker = normalizeBlocker(raw);
|
||||
if (!blocker) continue;
|
||||
if (blockers.some((item) => item.code === blocker.code && item.message === blocker.message)) continue;
|
||||
blockers.push(blocker);
|
||||
}
|
||||
if (skillTool?.stderrSummary && skillTool.stderrSummary !== "") {
|
||||
const blocker = normalizeBlocker({
|
||||
code: skillTool.stderrSummary,
|
||||
message: skillTool.stderrSummary
|
||||
});
|
||||
if (blocker && !blockers.some((item) => item.code === blocker.code)) blockers.push(blocker);
|
||||
}
|
||||
return blockers;
|
||||
}
|
||||
|
||||
function normalizeBlocker(raw) {
|
||||
if (!raw) return null;
|
||||
if (typeof raw === "string") {
|
||||
return {
|
||||
code: raw,
|
||||
message: raw,
|
||||
category: "blocked"
|
||||
};
|
||||
}
|
||||
if (typeof raw !== "object") return null;
|
||||
const code = raw.code ?? raw.blocker ?? raw.reason ?? raw.status ?? null;
|
||||
const message = raw.message ?? raw.summary ?? raw.zh ?? raw.reason ?? code;
|
||||
if (!code && !message) return null;
|
||||
return {
|
||||
code: String(code ?? "structured_blocker"),
|
||||
message: String(message ?? code ?? "structured blocker"),
|
||||
category: raw.category ?? raw.layer ?? "blocked",
|
||||
sourceIssue: raw.sourceIssue ?? null
|
||||
};
|
||||
}
|
||||
|
||||
function directHardwareTargetMentioned(payload) {
|
||||
const urls = [];
|
||||
const collect = (value) => {
|
||||
if (typeof value === "string") {
|
||||
if (/https?:\/\//iu.test(value) || /\/invoke\b|\/sync\/tick\b/iu.test(value)) urls.push(value);
|
||||
return;
|
||||
}
|
||||
if (!value || typeof value !== "object") return;
|
||||
for (const [key, child] of Object.entries(value)) {
|
||||
if (/policy|blocked|blocker|summary|message|note|limitations?|allowed|safety/iu.test(key)) continue;
|
||||
collect(child);
|
||||
}
|
||||
};
|
||||
collect(payload);
|
||||
return urls.some((value) => forbiddenDirectHardwareTarget.test(value));
|
||||
}
|
||||
|
||||
function safeTimestamp() {
|
||||
return new Date().toISOString().replace(/[:.]/gu, "-");
|
||||
}
|
||||
@@ -1166,7 +1752,8 @@ function usage() {
|
||||
"Usage: node scripts/rpt004-mvp-e2e-harness.mjs --live --url http://74.48.78.17:16666/ --api-url http://74.48.78.17:16667/",
|
||||
"",
|
||||
"Default --check/source mode validates the harness only and does not claim MVP PASS.",
|
||||
"--live reads DEV health and the 16666 workbench, then runs layout, Code Agent skill discovery, and the bounded M3 DO/DI loop only after current+healthy preconditions pass.",
|
||||
"--live reads DEV health and the 16666 workbench, then runs layout, Code Agent same-session checks, and Skill CLI /v1/m3/io route classification after current+healthy preconditions pass.",
|
||||
"--allow-m3-write is required for the bounded DEV-only M3 DO/DI write loop; default live mode does not write DO1.",
|
||||
"The command writes JSON and Markdown reports by default in --live mode. It never deploys, rolls out, reads secrets, or touches PROD."
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@@ -2,13 +2,18 @@ import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import {
|
||||
buildRpt004Report,
|
||||
classifyCodeAgentSessionTurns,
|
||||
classifyCodeAgentSkillPath,
|
||||
classifySkillCliM3Route,
|
||||
classifyM3DurableEvidence,
|
||||
classifyRpt004Conclusion,
|
||||
parseArgs,
|
||||
validateLiveBoundary
|
||||
} from "./rpt004-mvp-e2e-harness.mjs";
|
||||
|
||||
const fixtureCommit = "667349554b1f";
|
||||
|
||||
test("RPT-004 CLI accepts the required live URL/API URL shape", () => {
|
||||
const args = parseArgs([
|
||||
"--live",
|
||||
@@ -22,9 +27,31 @@ test("RPT-004 CLI accepts the required live URL/API URL shape", () => {
|
||||
assert.equal(args.url, "http://74.48.78.17:16666/");
|
||||
assert.equal(args.apiUrl, "http://74.48.78.17:16667/");
|
||||
assert.equal(args.writeReport, true);
|
||||
assert.equal(args.allowM3Write, false);
|
||||
assert.doesNotThrow(() => validateLiveBoundary(args));
|
||||
});
|
||||
|
||||
test("RPT-004 CLI requires explicit flag before live M3 writes", () => {
|
||||
const defaultLive = parseArgs([
|
||||
"--live",
|
||||
"--url",
|
||||
"http://74.48.78.17:16666/",
|
||||
"--api-url",
|
||||
"http://74.48.78.17:16667/"
|
||||
]);
|
||||
const writeLive = parseArgs([
|
||||
"--live",
|
||||
"--allow-m3-write",
|
||||
"--url",
|
||||
"http://74.48.78.17:16666/",
|
||||
"--api-url",
|
||||
"http://74.48.78.17:16667/"
|
||||
]);
|
||||
|
||||
assert.equal(defaultLive.allowM3Write, false);
|
||||
assert.equal(writeLive.allowM3Write, true);
|
||||
});
|
||||
|
||||
test("RPT-004 live boundary rejects deprecated or swapped active DEV ports", () => {
|
||||
assert.throws(
|
||||
() => validateLiveBoundary({ url: "http://74.48.78.17:6666/", apiUrl: "http://74.48.78.17:16667/" }),
|
||||
@@ -71,6 +98,8 @@ test("RPT-004 reports MVP_PASS only when every required live dimension passes",
|
||||
check("16666-default-chinese-workbench", "pass"),
|
||||
check("layout-smoke-hit-target", "pass"),
|
||||
check("code-agent-skill-path", "pass"),
|
||||
check("code-agent-session-thread", "pass"),
|
||||
check("skill-cli-hwlab-api-m3-route", "pass"),
|
||||
check("m3-v1-io-trusted-loop", "pass"),
|
||||
check("operation-audit-evidence-durable-green", "pass")
|
||||
]);
|
||||
@@ -166,6 +195,239 @@ test("Code Agent Codex stdio skill discovery passes with completed toolCalls and
|
||||
assert.equal(result.status, "pass");
|
||||
});
|
||||
|
||||
test("Code Agent controlled read-only session skill discovery passes the harness skill path gate", () => {
|
||||
const result = classifyCodeAgentSkillPath({
|
||||
...codexSessionPayload({
|
||||
toolName: "skills.discover",
|
||||
skills: {
|
||||
status: "ready",
|
||||
items: [{
|
||||
name: "hwlab-agent-runtime",
|
||||
source: "/workspace/hwlab/skills/hwlab-agent-runtime/SKILL.md"
|
||||
}],
|
||||
count: 1
|
||||
}
|
||||
}),
|
||||
provider: "codex-readonly-runner",
|
||||
backend: "hwlab-cloud-api/codex-readonly-runner",
|
||||
sandbox: "read-only",
|
||||
capabilityLevel: "read-only-session-tools",
|
||||
sessionMode: "controlled-readonly-session-registry",
|
||||
implementationType: "controlled-readonly-session-registry",
|
||||
runner: {
|
||||
kind: "hwlab-readonly-runner",
|
||||
workspace: "/workspace/hwlab",
|
||||
sandbox: "read-only",
|
||||
sessionMode: "controlled-readonly-session-registry",
|
||||
implementationType: "controlled-readonly-session-registry",
|
||||
codexStdio: false,
|
||||
writeCapable: false,
|
||||
durableSession: false
|
||||
},
|
||||
runnerTrace: {
|
||||
runnerKind: "hwlab-readonly-runner",
|
||||
sessionMode: "controlled-readonly-session-registry"
|
||||
}
|
||||
}, { httpStatus: 200, httpOk: true });
|
||||
|
||||
assert.equal(result.status, "pass");
|
||||
assert.equal(result.observations.controlledSessionRunner, true);
|
||||
});
|
||||
|
||||
test("Code Agent three-turn same session classifier passes with pwd, skills, and context reference", () => {
|
||||
const turns = [
|
||||
sessionTurn("pwd", codexSessionPayload({
|
||||
toolName: "pwd",
|
||||
turn: 1,
|
||||
reused: false,
|
||||
reply: "当前工作目录是 /workspace/hwlab。"
|
||||
})),
|
||||
sessionTurn("skills", codexSessionPayload({
|
||||
toolName: "skills.discover",
|
||||
turn: 2,
|
||||
reused: true,
|
||||
skills: {
|
||||
status: "ready",
|
||||
items: [{
|
||||
name: "hwlab-agent-runtime",
|
||||
source: "/workspace/hwlab/skills/hwlab-agent-runtime/SKILL.md"
|
||||
}],
|
||||
count: 1
|
||||
},
|
||||
reply: "可用 skill 包括 hwlab-agent-runtime。"
|
||||
})),
|
||||
sessionTurn("context-pwd", codexSessionPayload({
|
||||
toolName: "pwd",
|
||||
turn: 3,
|
||||
reused: true,
|
||||
reply: "前文第一轮你问的是 pwd;当前工作目录仍是 /workspace/hwlab。"
|
||||
}))
|
||||
];
|
||||
|
||||
const result = classifyCodeAgentSessionTurns(turns, { expectedConversationId: "cnv_rpt004_fixture" });
|
||||
|
||||
assert.equal(result.status, "pass");
|
||||
assert.deepEqual(result.observations.conversationIds, ["cnv_rpt004_fixture"]);
|
||||
assert.deepEqual(result.observations.sessionIds, ["ses_rpt004_fixture"]);
|
||||
assert.equal(result.observations.turns[2].referencesPrevious, true);
|
||||
});
|
||||
|
||||
test("Code Agent fallback-only turns cannot satisfy session checks", () => {
|
||||
const turns = [
|
||||
sessionTurn("pwd", openAiFallbackPayload()),
|
||||
sessionTurn("skills", openAiFallbackPayload()),
|
||||
sessionTurn("context-pwd", openAiFallbackPayload({ reply: "第一轮是 pwd。" }))
|
||||
];
|
||||
|
||||
const result = classifyCodeAgentSessionTurns(turns, { expectedConversationId: "cnv_rpt004_fixture" });
|
||||
|
||||
assert.equal(result.status, "blocked");
|
||||
assert.equal(result.observations.fallbackOnly, true);
|
||||
assert.ok(result.observations.blockers.includes("fallback-only"));
|
||||
});
|
||||
|
||||
test("Skill CLI M3 route passes with operation/audit/evidence ids and no direct path", () => {
|
||||
const result = classifySkillCliM3Route(skillCliPayload(), { httpStatus: 200, httpOk: true });
|
||||
|
||||
assert.equal(result.status, "pass");
|
||||
assert.equal(result.observations.route, "/v1/m3/io");
|
||||
assert.equal(result.observations.idsPresent, true);
|
||||
assert.equal(result.observations.safety.directGatewayCalls, false);
|
||||
assert.equal(result.observations.safety.fallbackUsed, false);
|
||||
});
|
||||
|
||||
test("Skill CLI M3 route passes with structured blocker and no direct path", () => {
|
||||
const payload = skillCliPayload({
|
||||
accepted: false,
|
||||
status: "blocked",
|
||||
operationId: null,
|
||||
auditId: null,
|
||||
evidenceId: null,
|
||||
blocker: {
|
||||
code: "hwlab_api_unavailable",
|
||||
message: "HWLAB API unavailable"
|
||||
}
|
||||
});
|
||||
const result = classifySkillCliM3Route(payload, { httpStatus: 200, httpOk: true });
|
||||
|
||||
assert.equal(result.status, "pass");
|
||||
assert.equal(result.observations.structuredBlocker, true);
|
||||
assert.equal(result.observations.idsPresent, false);
|
||||
});
|
||||
|
||||
test("Skill CLI M3 route blocks missing route and direct gateway evidence", () => {
|
||||
const payload = skillCliPayload({
|
||||
route: "http://hwlab-gateway-simu-1.hwlab-dev.svc.cluster.local:7101/invoke",
|
||||
directGatewayCalls: true
|
||||
});
|
||||
const result = classifySkillCliM3Route(payload, { httpStatus: 200, httpOk: true });
|
||||
|
||||
assert.equal(result.status, "blocked");
|
||||
assert.match(result.summary, /route=\/v1\/m3\/io/u);
|
||||
assert.equal(result.observations.safety.directGatewayCalls, true);
|
||||
});
|
||||
|
||||
test("RPT-004 live report defaults to read-only M3 route checks and records no live mutation", async () => {
|
||||
const report = await buildRpt004Report({
|
||||
...parseArgs([
|
||||
"--live",
|
||||
"--url",
|
||||
"http://74.48.78.17:16666/",
|
||||
"--api-url",
|
||||
"http://74.48.78.17:16667/",
|
||||
"--expected-commit",
|
||||
fixtureCommit
|
||||
]),
|
||||
writeReport: false
|
||||
}, {
|
||||
repoRoot: "/fixture-rpt004",
|
||||
fsJson: fixtureArtifactJson(),
|
||||
now: () => "2026-05-23T00:00:00.000Z",
|
||||
httpGetJson: async (url) => ({ ok: true, status: 200, json: healthPayload(url) }),
|
||||
httpGetText: async () => ({
|
||||
ok: true,
|
||||
status: 200,
|
||||
contentType: "text/html; charset=utf-8",
|
||||
body: workbenchHtml()
|
||||
}),
|
||||
layoutRunner: async () => ({ status: "pass", summary: "layout ok", artifacts: { screenshots: [] } }),
|
||||
codeAgentRunner: async ({ turn, conversationId, sessionId }) => ({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: codexSessionPayload({
|
||||
conversationId,
|
||||
sessionId,
|
||||
toolName: turn.kind === "skills" ? "skills.discover" : "pwd",
|
||||
turn: turn.kind === "pwd" ? 1 : turn.kind === "skills" ? 2 : 3,
|
||||
reused: turn.kind !== "pwd",
|
||||
skills: turn.kind === "skills" ? {
|
||||
status: "ready",
|
||||
items: [{
|
||||
name: "hwlab-agent-runtime",
|
||||
source: "/workspace/hwlab/skills/hwlab-agent-runtime/SKILL.md"
|
||||
}],
|
||||
count: 1
|
||||
} : undefined,
|
||||
reply: turn.kind === "context-pwd"
|
||||
? "前文第一轮你问的是 pwd;当前工作目录仍是 /workspace/hwlab。"
|
||||
: "当前工作目录是 /workspace/hwlab。"
|
||||
})
|
||||
}),
|
||||
skillCliRunner: async () => ({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: skillCliPayload()
|
||||
})
|
||||
});
|
||||
|
||||
assert.equal(report.status, "pass");
|
||||
assert.equal(report.liveMutationUsed, false);
|
||||
assert.equal(report.safety.liveMutationUsed, false);
|
||||
assert.equal(report.commanderSummary.allowedLiveMutationUsed, false);
|
||||
assert.equal(report.commanderSummary.sessionResult.status, "pass");
|
||||
assert.equal(report.commanderSummary.skillCliResult.status, "pass");
|
||||
assert.equal(report.commanderSummary.m3RouteResult.route, "/v1/m3/io");
|
||||
assert.equal(report.commanderSummary.evidenceIds.evidenceId, "evd_rpt004_skill_m3");
|
||||
assert.equal(report.dimensions.skillCliM3Route.status, "pass");
|
||||
assert.equal(report.dimensions.m3TrustedLoop.observations.skippedWriteLoop, true);
|
||||
assert.equal(report.dimensions.durableEvidence.observations.notApplicableInDefaultReadOnlyMode, true);
|
||||
});
|
||||
|
||||
test("RPT-004 live report emits NOT_CURRENT and skips downstream checks", async () => {
|
||||
const report = await buildRpt004Report({
|
||||
...parseArgs([
|
||||
"--live",
|
||||
"--url",
|
||||
"http://74.48.78.17:16666/",
|
||||
"--api-url",
|
||||
"http://74.48.78.17:16667/",
|
||||
"--expected-commit",
|
||||
fixtureCommit
|
||||
]),
|
||||
writeReport: false
|
||||
}, {
|
||||
repoRoot: "/fixture-rpt004",
|
||||
fsJson: fixtureArtifactJson(),
|
||||
now: () => "2026-05-23T00:00:00.000Z",
|
||||
httpGetJson: async (url) => ({ ok: true, status: 200, json: healthPayload(url, { revision: "1111111" }) }),
|
||||
httpGetText: async () => ({
|
||||
ok: true,
|
||||
status: 200,
|
||||
contentType: "text/html; charset=utf-8",
|
||||
body: workbenchHtml()
|
||||
}),
|
||||
layoutRunner: async () => {
|
||||
throw new Error("layout must not run when NOT_CURRENT");
|
||||
}
|
||||
});
|
||||
|
||||
assert.equal(report.status, "not_current");
|
||||
assert.equal(report.conclusion.classification, "NOT_CURRENT");
|
||||
assert.equal(report.dimensions.codeAgentSession.status, "not_run");
|
||||
assert.equal(report.dimensions.skillCliM3Route.status, "not_run");
|
||||
assert.equal(report.liveMutationUsed, false);
|
||||
});
|
||||
|
||||
test("M3 durable evidence requires operation, audit, evidence, DEV-LIVE green, and persisted writes", () => {
|
||||
const green = classifyM3DurableEvidence(greenOperations());
|
||||
assert.equal(green.status, "pass");
|
||||
@@ -212,3 +474,324 @@ function operation(id, action, value) {
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function sessionTurn(kind, payload, response = { ok: true, status: 200 }) {
|
||||
return {
|
||||
kind,
|
||||
response,
|
||||
payload
|
||||
};
|
||||
}
|
||||
|
||||
function codexSessionPayload({
|
||||
conversationId = "cnv_rpt004_fixture",
|
||||
sessionId = "ses_rpt004_fixture",
|
||||
toolName = "pwd",
|
||||
turn = 1,
|
||||
reused = false,
|
||||
reply = "当前工作目录是 /workspace/hwlab。",
|
||||
skills = {
|
||||
status: "not_requested",
|
||||
items: [],
|
||||
count: 0
|
||||
}
|
||||
} = {}) {
|
||||
return {
|
||||
status: "completed",
|
||||
conversationId,
|
||||
sessionId,
|
||||
messageId: `msg_rpt004_${turn}`,
|
||||
traceId: `trc_rpt004_${turn}`,
|
||||
provider: "codex-stdio",
|
||||
model: "gpt-5.5",
|
||||
backend: "hwlab-cloud-api/codex-mcp-stdio-runner",
|
||||
workspace: "/workspace/hwlab",
|
||||
sandbox: "workspace-write",
|
||||
capabilityLevel: "long-lived-codex-stdio-session",
|
||||
sessionMode: "codex-mcp-stdio-long-lived",
|
||||
implementationType: "repo-owned-codex-mcp-stdio-session",
|
||||
session: {
|
||||
sessionId,
|
||||
status: "idle",
|
||||
idleTimeoutMs: 1800000,
|
||||
lastTraceId: `trc_rpt004_${turn}`,
|
||||
turn,
|
||||
longLivedSession: true,
|
||||
codexStdio: true,
|
||||
durable: true,
|
||||
writeCapable: true
|
||||
},
|
||||
sessionReuse: {
|
||||
conversationId,
|
||||
sessionId,
|
||||
reused,
|
||||
turn,
|
||||
status: "idle"
|
||||
},
|
||||
runner: {
|
||||
kind: "codex-mcp-stdio-runner",
|
||||
workspace: "/workspace/hwlab",
|
||||
sandbox: "workspace-write",
|
||||
session: "codex-mcp-stdio-long-lived",
|
||||
sessionMode: "codex-mcp-stdio-long-lived",
|
||||
implementationType: "repo-owned-codex-mcp-stdio-session",
|
||||
codexStdio: true,
|
||||
writeCapable: true,
|
||||
durableSession: true
|
||||
},
|
||||
runnerTrace: {
|
||||
traceId: `trc_rpt004_${turn}`,
|
||||
runnerKind: "codex-mcp-stdio-runner",
|
||||
sessionMode: "codex-mcp-stdio-long-lived",
|
||||
sessionId,
|
||||
turn,
|
||||
sessionReused: reused,
|
||||
events: [`turn:${turn}`, `tool:${toolName}:completed`]
|
||||
},
|
||||
longLivedSessionGate: {
|
||||
status: "pass",
|
||||
pass: true
|
||||
},
|
||||
toolCalls: [{
|
||||
name: toolName,
|
||||
status: "completed",
|
||||
type: toolName === "skills.discover" ? "file-read" : "shell",
|
||||
traceId: `trc_rpt004_${turn}`
|
||||
}],
|
||||
skills,
|
||||
reply: {
|
||||
content: reply
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function openAiFallbackPayload({ reply = "text fallback" } = {}) {
|
||||
return {
|
||||
status: "completed",
|
||||
conversationId: "cnv_rpt004_fixture",
|
||||
sessionId: "ses_rpt004_fixture",
|
||||
messageId: "msg_fallback",
|
||||
traceId: "trc_fallback",
|
||||
provider: "openai-responses",
|
||||
model: "gpt-5.5",
|
||||
backend: "hwlab-cloud-api/openai-responses",
|
||||
implementationType: "openai-responses-fallback",
|
||||
capabilityLevel: "text-chat-only",
|
||||
sessionMode: "provider-text-request",
|
||||
runner: {
|
||||
kind: "openai-responses-fallback"
|
||||
},
|
||||
runnerTrace: {
|
||||
runnerKind: "openai-responses-fallback"
|
||||
},
|
||||
toolCalls: [],
|
||||
skills: {
|
||||
status: "not_requested",
|
||||
items: []
|
||||
},
|
||||
reply: {
|
||||
content: reply
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function skillCliPayload({
|
||||
route = "/v1/m3/io",
|
||||
accepted = true,
|
||||
status = "succeeded",
|
||||
operationId = "op_rpt004_skill_m3",
|
||||
auditId = "aud_rpt004_skill_m3",
|
||||
evidenceId = "evd_rpt004_skill_m3",
|
||||
blocker = null,
|
||||
directGatewayCalls = false,
|
||||
directBoxCalls = false,
|
||||
directPatchPanelCalls = false
|
||||
} = {}) {
|
||||
return {
|
||||
status: "completed",
|
||||
conversationId: "cnv_rpt004_skill",
|
||||
sessionId: "ses_rpt004_skill",
|
||||
messageId: "msg_rpt004_skill",
|
||||
traceId: "trc_rpt004_skill",
|
||||
provider: "hwlab-skill-cli",
|
||||
model: "controlled-m3-io",
|
||||
backend: "hwlab-cloud-api/hwlab-agent-runtime-skill-cli",
|
||||
workspace: "/workspace/hwlab",
|
||||
sandbox: "hwlab-api-route-only",
|
||||
capabilityLevel: accepted ? "hwlab-api-control-ready" : "hwlab-api-control-blocked",
|
||||
sessionMode: "controlled-m3-io-skill-cli",
|
||||
implementationType: "skill-cli-hwlab-api-adapter",
|
||||
runner: {
|
||||
kind: "hwlab-m3-io-skill-cli",
|
||||
safety: {
|
||||
directGatewayCallsAllowed: false,
|
||||
directBoxSimuCallsAllowed: false,
|
||||
directPatchPanelCallsAllowed: false
|
||||
}
|
||||
},
|
||||
runnerTrace: {
|
||||
traceId: "trc_rpt004_skill",
|
||||
runnerKind: "hwlab-m3-io-skill-cli",
|
||||
route,
|
||||
fallbackUsed: false,
|
||||
directGatewayCalls,
|
||||
directBoxCalls,
|
||||
directPatchPanelCalls
|
||||
},
|
||||
providerTrace: {
|
||||
runnerKind: "hwlab-m3-io-skill-cli",
|
||||
route,
|
||||
traceId: "trc_rpt004_skill",
|
||||
operationId,
|
||||
accepted,
|
||||
controlReady: accepted,
|
||||
fallbackUsed: false
|
||||
},
|
||||
toolCalls: [{
|
||||
name: "hwlab-agent-runtime.m3-io",
|
||||
status: accepted ? "completed" : "blocked",
|
||||
type: "skill-cli",
|
||||
route,
|
||||
accepted,
|
||||
controlReady: accepted,
|
||||
capabilityLevel: accepted ? "hwlab-api-control-ready" : "hwlab-api-control-blocked",
|
||||
operationId,
|
||||
traceId: "trc_rpt004_skill",
|
||||
audit: {
|
||||
auditId,
|
||||
status
|
||||
},
|
||||
evidence: {
|
||||
evidenceId,
|
||||
status: accepted ? "blocked" : "blocked",
|
||||
sourceKind: accepted ? "BLOCKED" : "BLOCKED",
|
||||
blocker: blocker?.code ?? "runtime_durable_not_green",
|
||||
writeStatus: "written_non_durable"
|
||||
},
|
||||
durable: {
|
||||
status: "degraded",
|
||||
blocker: blocker?.code ?? "runtime_durable_not_green"
|
||||
},
|
||||
blocker,
|
||||
blockers: blocker ? [blocker] : [{
|
||||
code: "runtime_durable_not_green",
|
||||
message: "durable trust is not green"
|
||||
}],
|
||||
directGatewayCalls,
|
||||
directBoxCalls,
|
||||
directPatchPanelCalls,
|
||||
fallbackUsed: false
|
||||
}],
|
||||
skills: {
|
||||
status: "used",
|
||||
items: [{
|
||||
name: "hwlab-agent-runtime.m3-io",
|
||||
route
|
||||
}],
|
||||
blockers: blocker ? [blocker] : []
|
||||
},
|
||||
reply: {
|
||||
content: "M3 IO Skill CLI result"
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function healthPayload(url, { revision = fixtureCommit } = {}) {
|
||||
const isWeb = String(url).includes(":16666");
|
||||
return {
|
||||
serviceId: isWeb ? "hwlab-cloud-web" : "hwlab-cloud-api",
|
||||
environment: "dev",
|
||||
status: "ready",
|
||||
ready: true,
|
||||
commit: {
|
||||
id: revision
|
||||
},
|
||||
image: {
|
||||
tag: revision.slice(0, 7),
|
||||
digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
},
|
||||
runtime: {
|
||||
durable: true,
|
||||
ready: true,
|
||||
status: "ready",
|
||||
blocker: null,
|
||||
liveRuntimeEvidence: true
|
||||
},
|
||||
readiness: {
|
||||
ready: true,
|
||||
status: "ready",
|
||||
durability: {
|
||||
ready: true,
|
||||
status: "ready",
|
||||
blocker: null
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function workbenchHtml() {
|
||||
return `
|
||||
<!doctype html>
|
||||
<html lang="zh-CN">
|
||||
<body data-app-shell>
|
||||
<main class="workbench-shell">
|
||||
<section data-view="workspace">
|
||||
<h1>HWLAB 云工作台</h1>
|
||||
<aside class="resource-tree">硬件资源</aside>
|
||||
<section class="center-workspace">Agent 对话 工作清单 使用说明</section>
|
||||
<aside class="right-sidebar">可信记录 内部复核</aside>
|
||||
<form id="command-form" class="command-form"></form>
|
||||
</section>
|
||||
<section data-view="gate" hidden>gate</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>`;
|
||||
}
|
||||
|
||||
function fixtureArtifactJson() {
|
||||
const serviceIds = [
|
||||
"hwlab-cloud-api",
|
||||
"hwlab-cloud-web",
|
||||
"hwlab-agent-mgr",
|
||||
"hwlab-agent-worker",
|
||||
"hwlab-gateway",
|
||||
"hwlab-gateway-simu",
|
||||
"hwlab-box-simu",
|
||||
"hwlab-patch-panel",
|
||||
"hwlab-router",
|
||||
"hwlab-tunnel-client",
|
||||
"hwlab-edge-proxy",
|
||||
"hwlab-cli",
|
||||
"hwlab-agent-skills"
|
||||
];
|
||||
const deploy = {
|
||||
commitId: fixtureCommit
|
||||
};
|
||||
const catalog = {
|
||||
commitId: fixtureCommit,
|
||||
artifactState: "published",
|
||||
publish: {
|
||||
ciPublished: true,
|
||||
registryVerified: true
|
||||
},
|
||||
services: serviceIds.map((serviceId) => ({
|
||||
serviceId,
|
||||
commitId: fixtureCommit,
|
||||
imageTag: fixtureCommit.slice(0, 7),
|
||||
digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
publishState: "published"
|
||||
}))
|
||||
};
|
||||
const artifactReport = {
|
||||
commitId: fixtureCommit,
|
||||
artifactPublish: {
|
||||
sourceCommitId: fixtureCommit
|
||||
}
|
||||
};
|
||||
return async (relativePath) => {
|
||||
if (relativePath === "deploy/deploy.json") return deploy;
|
||||
if (relativePath === "deploy/artifact-catalog.dev.json") return catalog;
|
||||
if (relativePath === "reports/dev-gate/dev-artifacts.json") return artifactReport;
|
||||
return null;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -791,10 +791,12 @@ async function validateRpt004MvpE2eReport(report, label) {
|
||||
"mode",
|
||||
"status",
|
||||
"conclusion",
|
||||
"commanderSummary",
|
||||
"sourceContract",
|
||||
"validationCommands",
|
||||
"devPreconditions",
|
||||
"target",
|
||||
"runPolicy",
|
||||
"expected",
|
||||
"dimensions",
|
||||
"checks",
|
||||
@@ -815,6 +817,9 @@ async function validateRpt004MvpE2eReport(report, label) {
|
||||
assert.ok(["pass", "blocked", "not_current"].includes(report.status), `${label}.status`);
|
||||
|
||||
assertObject(report.conclusion, `${label}.conclusion`);
|
||||
assertObject(report.commanderSummary, `${label}.commanderSummary`);
|
||||
assert.equal(report.commanderSummary.reportClass, "live-mutating", `${label}.commanderSummary.reportClass`);
|
||||
assert.equal(typeof report.commanderSummary.allowedLiveMutationUsed, "boolean", `${label}.commanderSummary.allowedLiveMutationUsed`);
|
||||
assert.ok(
|
||||
["CHECK_ONLY", "BLOCKED", "NOT_CURRENT", "MVP_PASS"].includes(report.conclusion.classification),
|
||||
`${label}.conclusion.classification`
|
||||
@@ -859,7 +864,9 @@ async function validateRpt004MvpE2eReport(report, label) {
|
||||
"layout",
|
||||
"m3TrustedLoop",
|
||||
"durableEvidence",
|
||||
"codeAgentSkillPath"
|
||||
"codeAgentSkillPath",
|
||||
"codeAgentSession",
|
||||
"skillCliM3Route"
|
||||
];
|
||||
assertObject(report.dimensions, `${label}.dimensions`);
|
||||
for (const id of requiredDimensions) {
|
||||
@@ -881,6 +888,8 @@ async function validateRpt004MvpE2eReport(report, label) {
|
||||
"16666-default-chinese-workbench",
|
||||
"layout-smoke-hit-target",
|
||||
"code-agent-skill-path",
|
||||
"code-agent-session-thread",
|
||||
"skill-cli-hwlab-api-m3-route",
|
||||
"m3-v1-io-trusted-loop",
|
||||
"operation-audit-evidence-durable-green"
|
||||
]) {
|
||||
@@ -902,6 +911,8 @@ async function validateRpt004MvpE2eReport(report, label) {
|
||||
assert.equal(report.safety.rolloutAttempted, false, `${label}.safety.rolloutAttempted`);
|
||||
assert.equal(report.safety.secretValuesPrinted, false, `${label}.safety.secretValuesPrinted`);
|
||||
assert.equal(report.safety.openAiFallbackCountsAsPass, false, `${label}.safety.openAiFallbackCountsAsPass`);
|
||||
assert.equal(report.safety.liveM3WritesRequireExplicitFlag, true, `${label}.safety.liveM3WritesRequireExplicitFlag`);
|
||||
assert.equal(report.liveMutationUsed, report.runPolicy.allowM3Write === true, `${label}.liveMutationUsed`);
|
||||
|
||||
if (report.artifacts?.artifactReport) {
|
||||
assert.match(report.artifacts.artifactReport, /not applicable/u, `${label}.artifacts.artifactReport`);
|
||||
|
||||
Reference in New Issue
Block a user