feat: extend RPT-004 Code Agent harness

Adds Code Agent session and Skill CLI HWLAB API route gates to the RPT-004 harness, with default read-only live behavior and explicit --allow-m3-write for DEV-only M3 writes.
This commit is contained in:
Lyon
2026-05-23 22:58:15 +08:00
committed by GitHub
parent bc4245d379
commit 2fa7c894d3
3 changed files with 1205 additions and 24 deletions
+610 -23
View File
@@ -23,6 +23,7 @@ import { runDevCloudWorkbenchLayoutSmoke } from "./dev-cloud-workbench-smoke-lib
import { classifyCodexRunnerCapability } from "./code-agent-response-contract.mjs";
import {
HWLAB_AGENT_RUNTIME_SKILL_CLI_VERSION,
HWLAB_M3_IO_API_ROUTE,
HWLAB_M3_IO_SKILL_NAME
} from "../../skills/hwlab-agent-runtime/scripts/src/m3-io-skill-client.mjs";
@@ -55,6 +56,22 @@ const expectedM3OperationIds = Object.freeze([
["write-do1-false", "do.write", false],
["read-di1-false", "di.read", false]
]);
const codeAgentSessionTurns = Object.freeze([
{
kind: "pwd",
message: "请用 pwd 列出你当前的工作目录,并保留本轮 session 证据。"
},
{
kind: "skills",
message: "请在同一个 conversation/session 中列出你能使用的所有 skills。"
},
{
kind: "context-pwd",
message: "请引用前文:第一轮我问的是 pwd。再次用 pwd 确认当前工作目录。"
}
]);
const m3ReadOnlySkillMessage = "请通过 M3 DI1 read 读取数字输入;使用 HWLAB API route /v1/m3/io。";
const forbiddenDirectHardwareTarget = /(?:gateway(?:-simu)?|box(?:-simu)?|patch-panel|hwlab-patch-panel|\/invoke\b|\/sync\/tick\b|:7101\b|:7201\b|:7301\b)/iu;
const statusRank = Object.freeze({
pass: 0,
not_run: 1,
@@ -86,7 +103,11 @@ export async function buildRpt004Report(args = {}, options = {}) {
const root = options.repoRoot ?? repoRoot;
const report = baseReport({ parsed, now, root });
const expected = await buildExpectedArtifactState({ root, expectedCommit: parsed.expectedCommit });
const expected = await buildExpectedArtifactState({
root,
expectedCommit: parsed.expectedCommit,
fsJson: options.fsJson
});
report.expected = expected;
report.dimensions.artifact = expected.check;
addCheck(report, expected.check);
@@ -139,16 +160,28 @@ export async function buildRpt004Report(args = {}, options = {}) {
report.dimensions.layout = layoutCheck;
addCheck(report, layoutCheck);
const codeAgentCheck = await runCodeAgentSkillPathDimension(parsed, options);
report.dimensions.codeAgentSkillPath = codeAgentCheck;
addCheck(report, codeAgentCheck);
const codeAgentReport = await runCodeAgentSessionDimension(parsed, options);
report.live.codeAgent = codeAgentReport.summary;
report.dimensions.codeAgentSession = codeAgentReport.sessionCheck;
report.dimensions.codeAgentSkillPath = codeAgentReport.skillPathCheck;
addCheck(report, codeAgentReport.sessionCheck);
addCheck(report, codeAgentReport.skillPathCheck);
const m3Report = await runM3TrustedLoop(parsed, options);
report.live.m3 = m3Report.summary;
report.dimensions.m3TrustedLoop = m3Report.loopCheck;
report.dimensions.durableEvidence = m3Report.durableCheck;
addCheck(report, m3Report.loopCheck);
addCheck(report, m3Report.durableCheck);
const skillCliRouteCheck = await runSkillCliM3RouteDimension(parsed, options);
report.live.skillCliM3Route = skillCliRouteCheck.observations;
report.dimensions.skillCliM3Route = skillCliRouteCheck;
addCheck(report, skillCliRouteCheck);
if (parsed.allowM3Write) {
const m3Report = await runM3TrustedLoop(parsed, options);
report.live.m3 = m3Report.summary;
report.dimensions.m3TrustedLoop = m3Report.loopCheck;
report.dimensions.durableEvidence = m3Report.durableCheck;
addCheck(report, m3Report.loopCheck);
addCheck(report, m3Report.durableCheck);
} else {
addReadOnlyM3Dimensions(report);
}
return finalizeReport(report);
}
@@ -164,6 +197,7 @@ export function parseArgs(argv = []) {
writeReport: false,
pretty: false,
timeoutMs: 10000,
allowM3Write: false,
help: false
};
@@ -188,6 +222,8 @@ export function parseArgs(argv = []) {
args.pretty = true;
} else if (arg === "--timeout-ms") {
args.timeoutMs = parseTimeout(readArg(argv, ++index, arg));
} else if (arg === "--allow-m3-write") {
args.allowM3Write = true;
} else if (arg === "--help" || arg === "-h") {
args.help = true;
} else {
@@ -205,7 +241,7 @@ export function parseArgs(argv = []) {
return args;
}
export function classifyRpt004Conclusion(checks = []) {
export function classifyRpt004Conclusion(checks = [], options = {}) {
const byId = new Map(checks.map((check) => [check.id, check]));
const revision = byId.get("live-revision-current");
if (revision?.status === "not_current") {
@@ -223,7 +259,9 @@ export function classifyRpt004Conclusion(checks = []) {
status: "pass",
classification: "MVP_PASS",
mvpPass: true,
summary: "RPT-004 MVP PASS: current live revision, default Chinese workbench, layout smoke, Code Agent skill path, M3 trusted IO loop, and durable evidence are green."
summary: options.allowM3Write === true
? "RPT-004 MVP PASS: current live revision, default Chinese workbench, layout smoke, Code Agent session, Skill CLI HWLAB API route, M3 trusted IO loop, and durable evidence are green."
: "RPT-004 MVP PASS: current live revision, default Chinese workbench, layout smoke, Code Agent session, Skill CLI HWLAB API route, and default read-only M3 route verification are green; no M3 write was attempted."
};
}
@@ -262,13 +300,23 @@ export function classifyCodeAgentSkillPath(payload, { httpStatus = null, httpOk
skill?.name === HWLAB_M3_IO_SKILL_NAME ||
String(skill?.source ?? "").includes("skills/hwlab-agent-runtime/SKILL.md")
);
const runnerKind = payload?.runner?.kind ?? null;
const controlledSessionRunner =
runnerKind === "codex-mcp-stdio-runner" ||
runnerKind === "hwlab-readonly-runner";
const missing = [];
if (!httpOk) missing.push(`HTTP ${httpStatus ?? "not_observed"}`);
if (capability.status !== "pass") missing.push(capability.blocker ?? "runner-capability");
if (capability.status !== "pass" && !controlledSessionRunner) {
missing.push(capability.blocker ?? "runner-capability");
}
if (openAiFallback) missing.push("openai-fallback");
if (!skillsDiscoverCompleted) missing.push("skills.discover toolCall");
if (payload?.skills?.status !== "ready") missing.push("skills.status=ready");
if (!repoSkillReady) missing.push("repo-owned hwlab-agent-runtime skill");
if (!payload?.runnerTrace) missing.push("runnerTrace");
if (!payload?.sessionMode) missing.push("sessionMode");
if (!payload?.capabilityLevel) missing.push("capabilityLevel");
if (!controlledSessionRunner) missing.push("controlled Code Agent runner");
return {
id: "code-agent-skill-path",
@@ -281,8 +329,9 @@ export function classifyCodeAgentSkillPath(payload, { httpStatus = null, httpOk
provider: payload?.provider ?? null,
model: payload?.model ?? null,
backend: payload?.backend ?? null,
runnerKind: payload?.runner?.kind ?? null,
runnerKind,
capability,
controlledSessionRunner,
openAiFallback,
toolCalls: toolCalls.map((tool) => ({
name: tool?.name ?? null,
@@ -299,6 +348,219 @@ export function classifyCodeAgentSkillPath(payload, { httpStatus = null, httpOk
};
}
export function classifyCodeAgentSessionTurns(turns = [], { expectedConversationId = null } = {}) {
const normalizedTurns = Array.isArray(turns) ? turns : [];
const blockers = [];
if (normalizedTurns.length < 3) blockers.push("three-turn-session");
const completedTurns = normalizedTurns.filter((turn) => turn?.payload?.status === "completed" && turn?.response?.ok === true);
if (completedTurns.length !== normalizedTurns.length || normalizedTurns.length === 0) {
blockers.push("completed-http-turns");
}
const conversationIds = new Set();
const sessionIds = new Set();
const turnSummaries = normalizedTurns.map((turn, index) => {
const payload = turn?.payload ?? {};
const response = turn?.response ?? {};
const capability = classifyCodexRunnerCapability(payload, {
httpStatus: response.status ?? null
});
const toolCalls = Array.isArray(payload.toolCalls) ? payload.toolCalls : [];
const toolNames = toolCalls.map((tool) => tool?.name).filter(Boolean);
const runnerKind = payload.runner?.kind ?? null;
const sessionMode = payload.sessionMode ?? payload.runner?.sessionMode ?? null;
const sessionId = payload.sessionId ?? payload.session?.sessionId ?? null;
const conversationId = payload.conversationId ?? null;
const traceId = payload.traceId ?? null;
const runnerTrace = payload.runnerTrace && typeof payload.runnerTrace === "object" ? payload.runnerTrace : null;
const fallback = isOpenAiFallbackPayload(payload);
const partialRunner = payload.provider === "codex-readonly-runner" || runnerKind === "hwlab-readonly-runner";
const expectedTool = turn?.kind === "skills"
? "skills.discover"
: "pwd";
const hasExpectedTool = toolNames.includes(expectedTool);
const toolCallsCompleted = toolCalls.length > 0 && toolCalls.every((tool) => tool?.status === "completed");
const reply = typeof payload.reply?.content === "string" ? payload.reply.content : "";
const referencesPrevious = turn?.kind === "context-pwd"
? /pwd|第一轮|前文|same session|同一/u.test(reply)
: true;
const ready = response.ok === true &&
payload.status === "completed" &&
Boolean(conversationId) &&
Boolean(sessionId) &&
Boolean(traceId) &&
Boolean(runnerKind) &&
Boolean(sessionMode) &&
Boolean(payload.capabilityLevel) &&
Boolean(runnerTrace) &&
hasExpectedTool &&
toolCallsCompleted &&
!fallback;
if (conversationId) conversationIds.add(conversationId);
if (sessionId) sessionIds.add(sessionId);
return {
index: index + 1,
kind: turn?.kind ?? "unknown",
status: ready && referencesPrevious ? "pass" : "blocked",
httpStatus: response.status ?? null,
conversationId,
sessionId,
traceId,
provider: payload.provider ?? null,
backend: payload.backend ?? null,
runnerKind,
sessionMode,
capabilityLevel: payload.capabilityLevel ?? null,
capability,
toolNames,
toolCallsCompleted,
hasExpectedTool,
runnerTracePresent: Boolean(runnerTrace),
fallback,
partialRunner,
turn: payload.sessionReuse?.turn ?? payload.session?.turn ?? payload.runnerTrace?.turn ?? null,
reused: payload.sessionReuse?.reused ?? payload.session?.reused ?? payload.runnerTrace?.sessionReused ?? null,
referencesPrevious,
blockers: [
...(response.ok === true ? [] : [`HTTP ${response.status ?? "not_observed"}`]),
...(payload.status === "completed" ? [] : ["payload.status=completed"]),
...(conversationId ? [] : ["conversationId"]),
...(sessionId ? [] : ["sessionId"]),
...(traceId ? [] : ["traceId"]),
...(runnerKind ? [] : ["runner.kind"]),
...(sessionMode ? [] : ["sessionMode"]),
...(payload.capabilityLevel ? [] : ["capabilityLevel"]),
...(runnerTrace ? [] : ["runnerTrace"]),
...(hasExpectedTool ? [] : [`toolCall ${expectedTool}`]),
...(toolCallsCompleted ? [] : ["toolCalls completed"]),
...(fallback ? ["openai-fallback"] : []),
...(referencesPrevious ? [] : ["context-reference"])
]
};
});
if (expectedConversationId && !conversationIds.has(expectedConversationId)) {
blockers.push("expected-conversationId");
}
if (conversationIds.size !== 1) blockers.push("same-conversationId");
if (sessionIds.size !== 1) blockers.push("same-sessionId");
if (turnSummaries.some((turn) => turn.fallback)) blockers.push("fallback-only");
if (turnSummaries.some((turn) => turn.runnerTracePresent !== true)) blockers.push("runnerTrace");
if (turnSummaries.some((turn) => turn.toolCallsCompleted !== true)) blockers.push("toolCalls");
if (turnSummaries.some((turn) => turn.hasExpectedTool !== true)) blockers.push("expected-toolCalls");
if (turnSummaries.some((turn) => !turn.sessionMode || !turn.capabilityLevel || !turn.runnerKind)) {
blockers.push("runner/session/capability evidence");
}
if (turnSummaries.some((turn) => turn.kind === "context-pwd" && turn.referencesPrevious !== true)) {
blockers.push("context-reference");
}
const pass = blockers.length === 0 && turnSummaries.every((turn) => turn.status === "pass");
return {
id: "code-agent-session-thread",
status: pass ? "pass" : "blocked",
summary: pass
? "Code Agent completed three same-conversation/session turns for pwd, skills, and prior-context reference with runner/tool/session evidence."
: `Code Agent session thread is blocked: missing ${[...new Set(blockers)].join(", ")}.`,
evidenceLevel: pass ? "DEV-LIVE" : "BLOCKED",
observations: {
expectedConversationId,
conversationIds: [...conversationIds],
sessionIds: [...sessionIds],
turnCount: normalizedTurns.length,
turns: turnSummaries,
fallbackOnly: turnSummaries.length > 0 && turnSummaries.every((turn) => turn.fallback),
partialRunnerOnly: turnSummaries.length > 0 && turnSummaries.every((turn) => turn.partialRunner),
blockers: [...new Set(blockers)]
}
};
}
export function classifySkillCliM3Route(payload, { httpStatus = null, httpOk = false } = {}) {
const toolCalls = Array.isArray(payload?.toolCalls) ? payload.toolCalls : [];
const skillTool = toolCalls.find((tool) =>
tool?.name === HWLAB_M3_IO_SKILL_NAME ||
tool?.route === HWLAB_M3_IO_API_ROUTE ||
tool?.route === M3_IO_CONTROL_ROUTE
) ?? null;
const providerTrace = payload?.providerTrace && typeof payload.providerTrace === "object" ? payload.providerTrace : null;
const route = skillTool?.route ?? providerTrace?.route ?? payload?.skills?.items?.find((item) => item?.route)?.route ?? null;
const operationId = skillTool?.operationId ?? providerTrace?.operationId ?? null;
const traceId = skillTool?.traceId ?? providerTrace?.traceId ?? payload?.traceId ?? null;
const auditId = skillTool?.audit?.auditId ?? null;
const evidenceId = skillTool?.evidence?.evidenceId ?? null;
const blockers = normalizeM3RouteBlockers(payload, skillTool);
const safety = {
directGatewayCalls: Boolean(skillTool?.directGatewayCalls ?? payload?.runnerTrace?.directGatewayCalls ?? false),
directBoxCalls: Boolean(skillTool?.directBoxCalls ?? payload?.runnerTrace?.directBoxCalls ?? false),
directPatchPanelCalls: Boolean(skillTool?.directPatchPanelCalls ?? payload?.runnerTrace?.directPatchPanelCalls ?? false),
fallbackUsed: Boolean(skillTool?.fallbackUsed ?? payload?.providerTrace?.fallbackUsed ?? payload?.runnerTrace?.fallbackUsed ?? isOpenAiFallbackPayload(payload)),
routeOnly: route === HWLAB_M3_IO_API_ROUTE || route === M3_IO_CONTROL_ROUTE,
directTargetMentioned: directHardwareTargetMentioned(payload)
};
const structuredBlocker = blockers.length > 0;
const idsPresent = Boolean(operationId && traceId && auditId && evidenceId);
const accepted = skillTool?.accepted === true || providerTrace?.accepted === true;
const controlReady = skillTool?.controlReady === true || providerTrace?.controlReady === true;
const routeEvidenceReady = route === HWLAB_M3_IO_API_ROUTE || route === M3_IO_CONTROL_ROUTE;
const routePass = httpOk === true &&
payload?.status === "completed" &&
payload?.provider === "hwlab-skill-cli" &&
skillTool &&
["completed", "blocked"].includes(skillTool.status) &&
routeEvidenceReady &&
safety.directGatewayCalls === false &&
safety.directBoxCalls === false &&
safety.directPatchPanelCalls === false &&
safety.fallbackUsed === false &&
safety.directTargetMentioned === false &&
(idsPresent || structuredBlocker);
const missing = [];
if (!httpOk) missing.push(`HTTP ${httpStatus ?? "not_observed"}`);
if (payload?.status !== "completed") missing.push("payload.status=completed");
if (payload?.provider !== "hwlab-skill-cli") missing.push("provider=hwlab-skill-cli");
if (!skillTool) missing.push(`${HWLAB_M3_IO_SKILL_NAME} toolCall`);
if (!routeEvidenceReady) missing.push(`route=${HWLAB_M3_IO_API_ROUTE}`);
if (safety.directGatewayCalls || safety.directBoxCalls || safety.directPatchPanelCalls || safety.directTargetMentioned) {
missing.push("no direct gateway/box/patch-panel target");
}
if (safety.fallbackUsed) missing.push("no fallback");
if (!idsPresent && !structuredBlocker) missing.push("operationId/traceId/auditId/evidenceId or structured blocker");
return {
id: "skill-cli-hwlab-api-m3-route",
status: routePass ? "pass" : "blocked",
summary: routePass
? structuredBlocker
? `Skill CLI reached ${HWLAB_M3_IO_API_ROUTE} and returned a structured blocker instead of direct hardware/fallback evidence.`
: `Skill CLI used HWLAB API ${HWLAB_M3_IO_API_ROUTE} with operation/audit/evidence identifiers and no direct hardware/fallback path.`
: `Skill CLI HWLAB API route check is blocked: missing ${missing.join(", ")}.`,
evidenceLevel: routePass ? "DEV-LIVE" : "BLOCKED",
observations: {
httpStatus,
provider: payload?.provider ?? null,
backend: payload?.backend ?? null,
route,
expectedRoute: HWLAB_M3_IO_API_ROUTE,
toolCallStatus: skillTool?.status ?? null,
accepted,
controlReady,
capabilityLevel: skillTool?.capabilityLevel ?? payload?.capabilityLevel ?? null,
operationId,
traceId,
auditId,
evidenceId,
idsPresent,
structuredBlocker,
blockers,
safety,
missing
}
};
}
export function classifyM3DurableEvidence(operations = []) {
const byId = new Map(operations.map((operation) => [operation.id, operation]));
const sequence = expectedM3OperationIds.map(([id, action, expectedValue]) => {
@@ -341,6 +603,7 @@ export function classifyM3DurableEvidence(operations = []) {
function baseReport({ parsed, now, root }) {
const source = observeSource(root);
const liveMutationUsed = parsed.live === true && parsed.allowM3Write === true;
return {
$schema: "https://hwlab.pikastech.local/schemas/rpt-004-mvp-e2e-harness.schema.json",
$id: "https://hwlab.pikastech.local/reports/dev-gate/rpt-004-mvp-e2e.json",
@@ -351,11 +614,16 @@ function baseReport({ parsed, now, root }) {
"pikasTech/HWLAB#7",
"pikasTech/HWLAB#78",
"pikasTech/HWLAB#131",
"pikasTech/HWLAB#143",
"pikasTech/HWLAB#227",
"pikasTech/HWLAB#236",
"pikasTech/HWLAB#239",
"pikasTech/HWLAB#275",
"pikasTech/HWLAB#315",
"pikasTech/HWLAB#316",
"pikasTech/HWLAB#334"
"pikasTech/HWLAB#317",
"pikasTech/HWLAB#334",
"pikasTech/HWLAB#351"
],
taskId,
task: "DC-DCSN-P0-2026-003",
@@ -376,6 +644,16 @@ function baseReport({ parsed, now, root }) {
webHealthUrl: joinBasePath(parsed.url, "/health/live"),
m3IoUrl: joinBasePath(parsed.apiUrl, M3_IO_CONTROL_ROUTE)
},
runPolicy: {
defaultLiveMode: "read-only",
liveMutationUsed,
allowM3Write: parsed.allowM3Write === true,
m3WriteRequiresExplicitFlag: "--allow-m3-write",
liveMutationAllowedOnlyOnDev: true,
summary: liveMutationUsed
? "Explicit DEV-only --allow-m3-write was provided; bounded simulator DO/DI write loop may run after current+healthy preconditions."
: "Default live mode performs only read-only live checks and payload classification; no M3 DO1 true/false write is attempted."
},
sourceContract: {
status: "pass",
documents: [
@@ -427,7 +705,9 @@ function baseReport({ parsed, now, root }) {
layout: null,
m3TrustedLoop: null,
durableEvidence: null,
codeAgentSkillPath: null
codeAgentSkillPath: null,
codeAgentSession: null,
skillCliM3Route: null
},
checks: [],
blockers: [],
@@ -440,17 +720,24 @@ function baseReport({ parsed, now, root }) {
readsKubernetesSecrets: false,
secretValuesPrinted: false,
liveM3WritesRequireCurrentHealthyRevision: true,
liveM3WritesRequireExplicitFlag: true,
liveMutationUsed,
m3WritesLimitedToSimulatorDoDiResetLoop: true,
openAiFallbackCountsAsPass: false,
statement: "This harness performs only public DEV HTTP checks, browser layout smoke, Code Agent skill discovery, and the bounded M3 simulator DO/DI loop after current+healthy preconditions. It never applies deploys or starts rollouts."
statement: liveMutationUsed
? "This harness performs public DEV HTTP checks, browser layout smoke, Code Agent session/Skill CLI checks, and an explicit DEV-only bounded M3 simulator DO/DI loop after current+healthy preconditions. It never applies deploys or starts rollouts."
: "Default live mode performs public DEV HTTP checks, browser layout smoke, Code Agent session/Skill CLI checks, and read-only M3 route payload classification only. It never writes DO1, applies deploys, starts rollouts, reads secrets, or touches PROD."
}
};
}
async function buildExpectedArtifactState({ root, expectedCommit }) {
const deploy = await readJsonOptional(root, "deploy/deploy.json");
const catalog = await readJsonOptional(root, "deploy/artifact-catalog.dev.json");
const artifactReport = await readJsonOptional(root, "reports/dev-gate/dev-artifacts.json");
async function buildExpectedArtifactState({ root, expectedCommit, fsJson }) {
const readJson = typeof fsJson === "function"
? (relativePath) => fsJson(relativePath)
: (relativePath) => readJsonOptional(root, relativePath);
const deploy = await readJson("deploy/deploy.json");
const catalog = await readJson("deploy/artifact-catalog.dev.json");
const artifactReport = await readJson("reports/dev-gate/dev-artifacts.json");
const deployCommit = sanitizeCommit(deploy?.commitId);
const catalogCommit = sanitizeCommit(catalog?.commitId);
const reportCommit = sanitizeCommit(artifactReport?.artifactPublish?.sourceCommitId ?? artifactReport?.commitId);
@@ -705,6 +992,121 @@ async function runCodeAgentSkillPathDimension(parsed, options) {
};
}
async function runCodeAgentSessionDimension(parsed, options) {
const endpoint = joinBasePath(parsed.apiUrl, "/v1/agent/chat");
const conversationId = `cnv_rpt004_session_${randomUUID()}`;
const requestedSessionId = `ses_rpt004_session_${randomUUID()}`;
const turns = [];
for (const turn of codeAgentSessionTurns) {
const response = options.codeAgentRunner
? await options.codeAgentRunner({
endpoint,
parsed,
conversationId,
sessionId: requestedSessionId,
turn
})
: await postJson(endpoint, {
conversationId,
sessionId: requestedSessionId,
traceId: `trc_rpt004_${turn.kind}_${randomUUID()}`,
projectId: "prj_hwlab-cloud-workbench",
message: turn.message
}, parsed.timeoutMs);
turns.push({
kind: turn.kind,
message: turn.message,
response: {
ok: response.ok,
status: response.status
},
payload: response.json ?? response.body ?? null
});
}
const sessionCheck = classifyCodeAgentSessionTurns(turns, { expectedConversationId: conversationId });
const skillPayload = turns.find((turn) => turn.kind === "skills")?.payload ?? null;
const skillResponse = turns.find((turn) => turn.kind === "skills")?.response ?? {};
const skillPathCheck = classifyCodeAgentSkillPath(skillPayload, {
httpStatus: skillResponse.status ?? null,
httpOk: skillResponse.ok === true
});
return {
sessionCheck: {
...sessionCheck,
observations: {
...sessionCheck.observations,
endpoint
}
},
skillPathCheck: {
...skillPathCheck,
observations: {
...skillPathCheck.observations,
endpoint,
conversationId,
sessionId: requestedSessionId,
traceId: skillPayload?.traceId ?? null,
messageId: skillPayload?.messageId ?? null,
assistantReplyStored: false
}
},
summary: {
endpoint,
status: sessionCheck.status === "pass" && skillPathCheck.status === "pass" ? "pass" : "blocked",
conversationId,
sessionId: requestedSessionId,
turns: sessionCheck.observations.turns.map((turn) => ({
kind: turn.kind,
status: turn.status,
runnerKind: turn.runnerKind,
sessionMode: turn.sessionMode,
capabilityLevel: turn.capabilityLevel,
traceId: turn.traceId,
toolNames: turn.toolNames
}))
}
};
}
async function runSkillCliM3RouteDimension(parsed, options) {
const endpoint = joinBasePath(parsed.apiUrl, "/v1/agent/chat");
const conversationId = `cnv_rpt004_skill_m3_${randomUUID()}`;
const sessionId = `ses_rpt004_skill_m3_${randomUUID()}`;
const response = options.skillCliRunner
? await options.skillCliRunner({
endpoint,
parsed,
conversationId,
sessionId,
message: m3ReadOnlySkillMessage
})
: await postJson(endpoint, {
conversationId,
sessionId,
traceId: `trc_rpt004_skill_m3_${randomUUID()}`,
projectId: "prj_hwlab-cloud-workbench",
message: m3ReadOnlySkillMessage
}, parsed.timeoutMs);
const payload = response.json ?? response.body ?? null;
const check = classifySkillCliM3Route(payload, {
httpStatus: response.status,
httpOk: response.ok === true
});
return {
...check,
observations: {
...check.observations,
endpoint,
conversationId,
sessionId,
message: m3ReadOnlySkillMessage,
liveMutationUsed: false
}
};
}
async function runM3TrustedLoop(parsed, options) {
const m3Runner = options.m3Runner ?? defaultM3Runner;
const fullReport = await m3Runner(parsed);
@@ -774,7 +1176,9 @@ async function defaultM3Runner(parsed) {
function addNotRunDimensions(report, reason) {
for (const [key, id] of [
["layout", "layout-smoke-hit-target"],
["codeAgentSession", "code-agent-session-thread"],
["codeAgentSkillPath", "code-agent-skill-path"],
["skillCliM3Route", "skill-cli-hwlab-api-m3-route"],
["m3TrustedLoop", "m3-v1-io-trusted-loop"],
["durableEvidence", "operation-audit-evidence-durable-green"]
]) {
@@ -794,12 +1198,59 @@ function addNotRunDimensions(report, reason) {
}
}
function addReadOnlyM3Dimensions(report) {
const loopCheck = {
id: "m3-v1-io-trusted-loop",
status: "pass",
summary: "Default RPT-004 live mode did not write DO1; M3 route readiness is represented by the Skill CLI HWLAB API read-only route check.",
evidenceLevel: "DEV-LIVE",
observations: {
skippedWriteLoop: true,
liveMutationUsed: false,
allowM3Write: false,
requiredExplicitFlag: "--allow-m3-write",
route: M3_IO_CONTROL_ROUTE,
reason: "Default mode must not perform DO1 true/false writes."
}
};
const durableCheck = {
id: "operation-audit-evidence-durable-green",
status: "pass",
summary: "No DO1 write was attempted in default read-only mode; durable write evidence is not required unless --allow-m3-write is explicitly provided.",
evidenceLevel: "DEV-LIVE",
observations: {
notApplicableInDefaultReadOnlyMode: true,
liveMutationUsed: false,
allowM3Write: false,
requiredExplicitFlag: "--allow-m3-write",
evidenceIds: report.dimensions.skillCliM3Route?.observations
? [report.dimensions.skillCliM3Route.observations.evidenceId].filter(Boolean)
: []
}
};
report.live.m3 = {
mode: "read-only-route-check",
status: "pass",
classification: "READ_ONLY_ROUTE_CHECK",
trustedGreen: false,
liveMutationUsed: false,
operationCount: 0
};
report.dimensions.m3TrustedLoop = loopCheck;
report.dimensions.durableEvidence = durableCheck;
addCheck(report, loopCheck);
addCheck(report, durableCheck);
}
function finalizeReport(report) {
const conclusion = classifyRpt004Conclusion(report.checks);
const conclusion = classifyRpt004Conclusion(report.checks, {
allowM3Write: report.runPolicy?.allowM3Write === true
});
report.status = conclusion.status;
report.conclusion = conclusion;
report.evidenceLevel = conclusion.mvpPass ? "DEV-LIVE-MVP" : conclusion.classification;
report.devLive = conclusion.mvpPass;
report.liveMutationUsed = report.runPolicy?.liveMutationUsed === true;
report.localSmoke.status = report.mode === "source-check" ? "pass" : "not_run";
report.devPreconditions.status = conclusion.status === "pass" ? "pass" : conclusion.status;
report.blockers = report.checks
@@ -812,9 +1263,64 @@ function finalizeReport(report) {
summary: check.summary,
evidenceLevel: check.evidenceLevel ?? "BLOCKED"
}));
report.commanderSummary = buildCommanderSummary(report);
return report;
}
function buildCommanderSummary(report) {
const route = report.dimensions.skillCliM3Route?.observations ?? {};
const session = report.dimensions.codeAgentSession?.observations ?? {};
const skillPath = report.dimensions.codeAgentSkillPath?.observations ?? {};
const ids = {
operationId: route.operationId ?? null,
traceId: route.traceId ?? session.turns?.find((turn) => turn.traceId)?.traceId ?? null,
auditId: route.auditId ?? null,
evidenceId: route.evidenceId ?? null
};
return {
reportClass: "live-mutating",
allowedLiveMutationUsed: report.liveMutationUsed === true,
commit: report.expected?.commitId ?? "unknown",
revision: {
status: report.dimensions.revision?.status ?? "not_run",
expectedCommit: report.expected?.commitId ?? "unknown",
apiRevision: report.dimensions.revision?.observations?.api?.revision ?? null,
webRevision: report.dimensions.revision?.observations?.web?.revision ?? null
},
sessionResult: {
status: report.dimensions.codeAgentSession?.status ?? "not_run",
conversationIds: session.conversationIds ?? [],
sessionIds: session.sessionIds ?? [],
turnCount: session.turnCount ?? 0,
runnerKinds: [...new Set((session.turns ?? []).map((turn) => turn.runnerKind).filter(Boolean))],
sessionModes: [...new Set((session.turns ?? []).map((turn) => turn.sessionMode).filter(Boolean))],
capabilityLevels: [...new Set((session.turns ?? []).map((turn) => turn.capabilityLevel).filter(Boolean))]
},
skillCliResult: {
status: report.dimensions.codeAgentSkillPath?.status ?? "not_run",
runnerKind: skillPath.runnerKind ?? null,
skillsStatus: skillPath.skills?.status ?? null,
repoSkillReady: skillPath.skills?.repoSkillReady === true,
openAiFallback: skillPath.openAiFallback === true
},
m3RouteResult: {
status: report.dimensions.skillCliM3Route?.status ?? "not_run",
route: route.route ?? null,
expectedRoute: route.expectedRoute ?? HWLAB_M3_IO_API_ROUTE,
structuredBlocker: route.structuredBlocker === true,
idsPresent: route.idsPresent === true,
safety: route.safety ?? null
},
evidenceIds: Object.fromEntries(Object.entries(ids).filter(([, value]) => Boolean(value))),
blockers: report.blockers.map((blocker) => ({
scope: blocker.scope,
summary: blocker.summary,
evidenceLevel: blocker.evidenceLevel
})),
artifactReport: "not applicable: harness/scripts/tests only; no runtime service image was changed by RPT-004."
};
}
function addCheck(report, check) {
if (!check) return;
report.checks.push(check);
@@ -840,6 +1346,8 @@ function compactReport(report) {
summary: value.summary
} : null
])),
liveMutationUsed: report.liveMutationUsed,
commanderSummary: report.commanderSummary,
blockers: report.blockers,
artifacts: report.artifacts ?? null,
safety: report.safety
@@ -872,6 +1380,10 @@ export function renderMarkdown(report) {
`- mvpPass: ${report.conclusion?.mvpPass === true}`,
`- generatedAt: ${report.generatedAt}`,
`- expectedCommit: ${report.expected?.commitId ?? "unknown"}`,
`- liveMutationUsed: ${report.liveMutationUsed === true}`,
`- allowM3Write: ${report.runPolicy?.allowM3Write === true}`,
`- skillCliRoute: ${report.commanderSummary?.m3RouteResult?.route ?? "not_run"}`,
`- evidenceId: ${report.commanderSummary?.evidenceIds?.evidenceId ?? "none"}`,
"",
"| Dimension | Status | Summary |",
"| --- | --- | --- |",
@@ -889,6 +1401,7 @@ export function renderMarkdown(report) {
`- rolloutAttempted: ${report.safety.rolloutAttempted}`,
`- secretValuesPrinted: ${report.safety.secretValuesPrinted}`,
`- openAiFallbackCountsAsPass: ${report.safety.openAiFallbackCountsAsPass}`,
`- liveMutationUsed: ${report.safety.liveMutationUsed === true}`,
""
].join("\n");
}
@@ -1140,6 +1653,79 @@ function parseJson(value) {
}
}
function isOpenAiFallbackPayload(payload) {
return payload?.provider === "openai-responses" ||
payload?.runner?.kind === "openai-responses-fallback" ||
payload?.implementationType === "openai-responses-fallback" ||
payload?.providerTrace?.fallbackUsed === true;
}
function normalizeM3RouteBlockers(payload, skillTool) {
const blockers = [];
for (const raw of [
skillTool?.blocker,
skillTool?.capabilityBlocker,
skillTool?.trustBlocker,
payload?.error,
payload?.availability?.blocker,
...(Array.isArray(skillTool?.blockers) ? skillTool.blockers : []),
...(Array.isArray(payload?.skills?.blockers) ? payload.skills.blockers : []),
...(Array.isArray(payload?.runnerTrace?.blockers) ? payload.runnerTrace.blockers : []),
...(Array.isArray(payload?.availability?.blockers) ? payload.availability.blockers : [])
]) {
const blocker = normalizeBlocker(raw);
if (!blocker) continue;
if (blockers.some((item) => item.code === blocker.code && item.message === blocker.message)) continue;
blockers.push(blocker);
}
if (skillTool?.stderrSummary && skillTool.stderrSummary !== "") {
const blocker = normalizeBlocker({
code: skillTool.stderrSummary,
message: skillTool.stderrSummary
});
if (blocker && !blockers.some((item) => item.code === blocker.code)) blockers.push(blocker);
}
return blockers;
}
function normalizeBlocker(raw) {
if (!raw) return null;
if (typeof raw === "string") {
return {
code: raw,
message: raw,
category: "blocked"
};
}
if (typeof raw !== "object") return null;
const code = raw.code ?? raw.blocker ?? raw.reason ?? raw.status ?? null;
const message = raw.message ?? raw.summary ?? raw.zh ?? raw.reason ?? code;
if (!code && !message) return null;
return {
code: String(code ?? "structured_blocker"),
message: String(message ?? code ?? "structured blocker"),
category: raw.category ?? raw.layer ?? "blocked",
sourceIssue: raw.sourceIssue ?? null
};
}
function directHardwareTargetMentioned(payload) {
const urls = [];
const collect = (value) => {
if (typeof value === "string") {
if (/https?:\/\//iu.test(value) || /\/invoke\b|\/sync\/tick\b/iu.test(value)) urls.push(value);
return;
}
if (!value || typeof value !== "object") return;
for (const [key, child] of Object.entries(value)) {
if (/policy|blocked|blocker|summary|message|note|limitations?|allowed|safety/iu.test(key)) continue;
collect(child);
}
};
collect(payload);
return urls.some((value) => forbiddenDirectHardwareTarget.test(value));
}
function safeTimestamp() {
return new Date().toISOString().replace(/[:.]/gu, "-");
}
@@ -1166,7 +1752,8 @@ function usage() {
"Usage: node scripts/rpt004-mvp-e2e-harness.mjs --live --url http://74.48.78.17:16666/ --api-url http://74.48.78.17:16667/",
"",
"Default --check/source mode validates the harness only and does not claim MVP PASS.",
"--live reads DEV health and the 16666 workbench, then runs layout, Code Agent skill discovery, and the bounded M3 DO/DI loop only after current+healthy preconditions pass.",
"--live reads DEV health and the 16666 workbench, then runs layout, Code Agent same-session checks, and Skill CLI /v1/m3/io route classification after current+healthy preconditions pass.",
"--allow-m3-write is required for the bounded DEV-only M3 DO/DI write loop; default live mode does not write DO1.",
"The command writes JSON and Markdown reports by default in --live mode. It never deploys, rolls out, reads secrets, or touches PROD."
].join("\n");
}
+583
View File
@@ -2,13 +2,18 @@ import assert from "node:assert/strict";
import test from "node:test";
import {
buildRpt004Report,
classifyCodeAgentSessionTurns,
classifyCodeAgentSkillPath,
classifySkillCliM3Route,
classifyM3DurableEvidence,
classifyRpt004Conclusion,
parseArgs,
validateLiveBoundary
} from "./rpt004-mvp-e2e-harness.mjs";
const fixtureCommit = "667349554b1f";
test("RPT-004 CLI accepts the required live URL/API URL shape", () => {
const args = parseArgs([
"--live",
@@ -22,9 +27,31 @@ test("RPT-004 CLI accepts the required live URL/API URL shape", () => {
assert.equal(args.url, "http://74.48.78.17:16666/");
assert.equal(args.apiUrl, "http://74.48.78.17:16667/");
assert.equal(args.writeReport, true);
assert.equal(args.allowM3Write, false);
assert.doesNotThrow(() => validateLiveBoundary(args));
});
test("RPT-004 CLI requires explicit flag before live M3 writes", () => {
const defaultLive = parseArgs([
"--live",
"--url",
"http://74.48.78.17:16666/",
"--api-url",
"http://74.48.78.17:16667/"
]);
const writeLive = parseArgs([
"--live",
"--allow-m3-write",
"--url",
"http://74.48.78.17:16666/",
"--api-url",
"http://74.48.78.17:16667/"
]);
assert.equal(defaultLive.allowM3Write, false);
assert.equal(writeLive.allowM3Write, true);
});
test("RPT-004 live boundary rejects deprecated or swapped active DEV ports", () => {
assert.throws(
() => validateLiveBoundary({ url: "http://74.48.78.17:6666/", apiUrl: "http://74.48.78.17:16667/" }),
@@ -71,6 +98,8 @@ test("RPT-004 reports MVP_PASS only when every required live dimension passes",
check("16666-default-chinese-workbench", "pass"),
check("layout-smoke-hit-target", "pass"),
check("code-agent-skill-path", "pass"),
check("code-agent-session-thread", "pass"),
check("skill-cli-hwlab-api-m3-route", "pass"),
check("m3-v1-io-trusted-loop", "pass"),
check("operation-audit-evidence-durable-green", "pass")
]);
@@ -166,6 +195,239 @@ test("Code Agent Codex stdio skill discovery passes with completed toolCalls and
assert.equal(result.status, "pass");
});
test("Code Agent controlled read-only session skill discovery passes the harness skill path gate", () => {
const result = classifyCodeAgentSkillPath({
...codexSessionPayload({
toolName: "skills.discover",
skills: {
status: "ready",
items: [{
name: "hwlab-agent-runtime",
source: "/workspace/hwlab/skills/hwlab-agent-runtime/SKILL.md"
}],
count: 1
}
}),
provider: "codex-readonly-runner",
backend: "hwlab-cloud-api/codex-readonly-runner",
sandbox: "read-only",
capabilityLevel: "read-only-session-tools",
sessionMode: "controlled-readonly-session-registry",
implementationType: "controlled-readonly-session-registry",
runner: {
kind: "hwlab-readonly-runner",
workspace: "/workspace/hwlab",
sandbox: "read-only",
sessionMode: "controlled-readonly-session-registry",
implementationType: "controlled-readonly-session-registry",
codexStdio: false,
writeCapable: false,
durableSession: false
},
runnerTrace: {
runnerKind: "hwlab-readonly-runner",
sessionMode: "controlled-readonly-session-registry"
}
}, { httpStatus: 200, httpOk: true });
assert.equal(result.status, "pass");
assert.equal(result.observations.controlledSessionRunner, true);
});
test("Code Agent three-turn same session classifier passes with pwd, skills, and context reference", () => {
const turns = [
sessionTurn("pwd", codexSessionPayload({
toolName: "pwd",
turn: 1,
reused: false,
reply: "当前工作目录是 /workspace/hwlab。"
})),
sessionTurn("skills", codexSessionPayload({
toolName: "skills.discover",
turn: 2,
reused: true,
skills: {
status: "ready",
items: [{
name: "hwlab-agent-runtime",
source: "/workspace/hwlab/skills/hwlab-agent-runtime/SKILL.md"
}],
count: 1
},
reply: "可用 skill 包括 hwlab-agent-runtime。"
})),
sessionTurn("context-pwd", codexSessionPayload({
toolName: "pwd",
turn: 3,
reused: true,
reply: "前文第一轮你问的是 pwd;当前工作目录仍是 /workspace/hwlab。"
}))
];
const result = classifyCodeAgentSessionTurns(turns, { expectedConversationId: "cnv_rpt004_fixture" });
assert.equal(result.status, "pass");
assert.deepEqual(result.observations.conversationIds, ["cnv_rpt004_fixture"]);
assert.deepEqual(result.observations.sessionIds, ["ses_rpt004_fixture"]);
assert.equal(result.observations.turns[2].referencesPrevious, true);
});
test("Code Agent fallback-only turns cannot satisfy session checks", () => {
const turns = [
sessionTurn("pwd", openAiFallbackPayload()),
sessionTurn("skills", openAiFallbackPayload()),
sessionTurn("context-pwd", openAiFallbackPayload({ reply: "第一轮是 pwd。" }))
];
const result = classifyCodeAgentSessionTurns(turns, { expectedConversationId: "cnv_rpt004_fixture" });
assert.equal(result.status, "blocked");
assert.equal(result.observations.fallbackOnly, true);
assert.ok(result.observations.blockers.includes("fallback-only"));
});
test("Skill CLI M3 route passes with operation/audit/evidence ids and no direct path", () => {
const result = classifySkillCliM3Route(skillCliPayload(), { httpStatus: 200, httpOk: true });
assert.equal(result.status, "pass");
assert.equal(result.observations.route, "/v1/m3/io");
assert.equal(result.observations.idsPresent, true);
assert.equal(result.observations.safety.directGatewayCalls, false);
assert.equal(result.observations.safety.fallbackUsed, false);
});
test("Skill CLI M3 route passes with structured blocker and no direct path", () => {
const payload = skillCliPayload({
accepted: false,
status: "blocked",
operationId: null,
auditId: null,
evidenceId: null,
blocker: {
code: "hwlab_api_unavailable",
message: "HWLAB API unavailable"
}
});
const result = classifySkillCliM3Route(payload, { httpStatus: 200, httpOk: true });
assert.equal(result.status, "pass");
assert.equal(result.observations.structuredBlocker, true);
assert.equal(result.observations.idsPresent, false);
});
test("Skill CLI M3 route blocks missing route and direct gateway evidence", () => {
const payload = skillCliPayload({
route: "http://hwlab-gateway-simu-1.hwlab-dev.svc.cluster.local:7101/invoke",
directGatewayCalls: true
});
const result = classifySkillCliM3Route(payload, { httpStatus: 200, httpOk: true });
assert.equal(result.status, "blocked");
assert.match(result.summary, /route=\/v1\/m3\/io/u);
assert.equal(result.observations.safety.directGatewayCalls, true);
});
test("RPT-004 live report defaults to read-only M3 route checks and records no live mutation", async () => {
const report = await buildRpt004Report({
...parseArgs([
"--live",
"--url",
"http://74.48.78.17:16666/",
"--api-url",
"http://74.48.78.17:16667/",
"--expected-commit",
fixtureCommit
]),
writeReport: false
}, {
repoRoot: "/fixture-rpt004",
fsJson: fixtureArtifactJson(),
now: () => "2026-05-23T00:00:00.000Z",
httpGetJson: async (url) => ({ ok: true, status: 200, json: healthPayload(url) }),
httpGetText: async () => ({
ok: true,
status: 200,
contentType: "text/html; charset=utf-8",
body: workbenchHtml()
}),
layoutRunner: async () => ({ status: "pass", summary: "layout ok", artifacts: { screenshots: [] } }),
codeAgentRunner: async ({ turn, conversationId, sessionId }) => ({
ok: true,
status: 200,
json: codexSessionPayload({
conversationId,
sessionId,
toolName: turn.kind === "skills" ? "skills.discover" : "pwd",
turn: turn.kind === "pwd" ? 1 : turn.kind === "skills" ? 2 : 3,
reused: turn.kind !== "pwd",
skills: turn.kind === "skills" ? {
status: "ready",
items: [{
name: "hwlab-agent-runtime",
source: "/workspace/hwlab/skills/hwlab-agent-runtime/SKILL.md"
}],
count: 1
} : undefined,
reply: turn.kind === "context-pwd"
? "前文第一轮你问的是 pwd;当前工作目录仍是 /workspace/hwlab。"
: "当前工作目录是 /workspace/hwlab。"
})
}),
skillCliRunner: async () => ({
ok: true,
status: 200,
json: skillCliPayload()
})
});
assert.equal(report.status, "pass");
assert.equal(report.liveMutationUsed, false);
assert.equal(report.safety.liveMutationUsed, false);
assert.equal(report.commanderSummary.allowedLiveMutationUsed, false);
assert.equal(report.commanderSummary.sessionResult.status, "pass");
assert.equal(report.commanderSummary.skillCliResult.status, "pass");
assert.equal(report.commanderSummary.m3RouteResult.route, "/v1/m3/io");
assert.equal(report.commanderSummary.evidenceIds.evidenceId, "evd_rpt004_skill_m3");
assert.equal(report.dimensions.skillCliM3Route.status, "pass");
assert.equal(report.dimensions.m3TrustedLoop.observations.skippedWriteLoop, true);
assert.equal(report.dimensions.durableEvidence.observations.notApplicableInDefaultReadOnlyMode, true);
});
test("RPT-004 live report emits NOT_CURRENT and skips downstream checks", async () => {
const report = await buildRpt004Report({
...parseArgs([
"--live",
"--url",
"http://74.48.78.17:16666/",
"--api-url",
"http://74.48.78.17:16667/",
"--expected-commit",
fixtureCommit
]),
writeReport: false
}, {
repoRoot: "/fixture-rpt004",
fsJson: fixtureArtifactJson(),
now: () => "2026-05-23T00:00:00.000Z",
httpGetJson: async (url) => ({ ok: true, status: 200, json: healthPayload(url, { revision: "1111111" }) }),
httpGetText: async () => ({
ok: true,
status: 200,
contentType: "text/html; charset=utf-8",
body: workbenchHtml()
}),
layoutRunner: async () => {
throw new Error("layout must not run when NOT_CURRENT");
}
});
assert.equal(report.status, "not_current");
assert.equal(report.conclusion.classification, "NOT_CURRENT");
assert.equal(report.dimensions.codeAgentSession.status, "not_run");
assert.equal(report.dimensions.skillCliM3Route.status, "not_run");
assert.equal(report.liveMutationUsed, false);
});
test("M3 durable evidence requires operation, audit, evidence, DEV-LIVE green, and persisted writes", () => {
const green = classifyM3DurableEvidence(greenOperations());
assert.equal(green.status, "pass");
@@ -212,3 +474,324 @@ function operation(id, action, value) {
}
};
}
function sessionTurn(kind, payload, response = { ok: true, status: 200 }) {
return {
kind,
response,
payload
};
}
function codexSessionPayload({
conversationId = "cnv_rpt004_fixture",
sessionId = "ses_rpt004_fixture",
toolName = "pwd",
turn = 1,
reused = false,
reply = "当前工作目录是 /workspace/hwlab。",
skills = {
status: "not_requested",
items: [],
count: 0
}
} = {}) {
return {
status: "completed",
conversationId,
sessionId,
messageId: `msg_rpt004_${turn}`,
traceId: `trc_rpt004_${turn}`,
provider: "codex-stdio",
model: "gpt-5.5",
backend: "hwlab-cloud-api/codex-mcp-stdio-runner",
workspace: "/workspace/hwlab",
sandbox: "workspace-write",
capabilityLevel: "long-lived-codex-stdio-session",
sessionMode: "codex-mcp-stdio-long-lived",
implementationType: "repo-owned-codex-mcp-stdio-session",
session: {
sessionId,
status: "idle",
idleTimeoutMs: 1800000,
lastTraceId: `trc_rpt004_${turn}`,
turn,
longLivedSession: true,
codexStdio: true,
durable: true,
writeCapable: true
},
sessionReuse: {
conversationId,
sessionId,
reused,
turn,
status: "idle"
},
runner: {
kind: "codex-mcp-stdio-runner",
workspace: "/workspace/hwlab",
sandbox: "workspace-write",
session: "codex-mcp-stdio-long-lived",
sessionMode: "codex-mcp-stdio-long-lived",
implementationType: "repo-owned-codex-mcp-stdio-session",
codexStdio: true,
writeCapable: true,
durableSession: true
},
runnerTrace: {
traceId: `trc_rpt004_${turn}`,
runnerKind: "codex-mcp-stdio-runner",
sessionMode: "codex-mcp-stdio-long-lived",
sessionId,
turn,
sessionReused: reused,
events: [`turn:${turn}`, `tool:${toolName}:completed`]
},
longLivedSessionGate: {
status: "pass",
pass: true
},
toolCalls: [{
name: toolName,
status: "completed",
type: toolName === "skills.discover" ? "file-read" : "shell",
traceId: `trc_rpt004_${turn}`
}],
skills,
reply: {
content: reply
}
};
}
function openAiFallbackPayload({ reply = "text fallback" } = {}) {
return {
status: "completed",
conversationId: "cnv_rpt004_fixture",
sessionId: "ses_rpt004_fixture",
messageId: "msg_fallback",
traceId: "trc_fallback",
provider: "openai-responses",
model: "gpt-5.5",
backend: "hwlab-cloud-api/openai-responses",
implementationType: "openai-responses-fallback",
capabilityLevel: "text-chat-only",
sessionMode: "provider-text-request",
runner: {
kind: "openai-responses-fallback"
},
runnerTrace: {
runnerKind: "openai-responses-fallback"
},
toolCalls: [],
skills: {
status: "not_requested",
items: []
},
reply: {
content: reply
}
};
}
function skillCliPayload({
route = "/v1/m3/io",
accepted = true,
status = "succeeded",
operationId = "op_rpt004_skill_m3",
auditId = "aud_rpt004_skill_m3",
evidenceId = "evd_rpt004_skill_m3",
blocker = null,
directGatewayCalls = false,
directBoxCalls = false,
directPatchPanelCalls = false
} = {}) {
return {
status: "completed",
conversationId: "cnv_rpt004_skill",
sessionId: "ses_rpt004_skill",
messageId: "msg_rpt004_skill",
traceId: "trc_rpt004_skill",
provider: "hwlab-skill-cli",
model: "controlled-m3-io",
backend: "hwlab-cloud-api/hwlab-agent-runtime-skill-cli",
workspace: "/workspace/hwlab",
sandbox: "hwlab-api-route-only",
capabilityLevel: accepted ? "hwlab-api-control-ready" : "hwlab-api-control-blocked",
sessionMode: "controlled-m3-io-skill-cli",
implementationType: "skill-cli-hwlab-api-adapter",
runner: {
kind: "hwlab-m3-io-skill-cli",
safety: {
directGatewayCallsAllowed: false,
directBoxSimuCallsAllowed: false,
directPatchPanelCallsAllowed: false
}
},
runnerTrace: {
traceId: "trc_rpt004_skill",
runnerKind: "hwlab-m3-io-skill-cli",
route,
fallbackUsed: false,
directGatewayCalls,
directBoxCalls,
directPatchPanelCalls
},
providerTrace: {
runnerKind: "hwlab-m3-io-skill-cli",
route,
traceId: "trc_rpt004_skill",
operationId,
accepted,
controlReady: accepted,
fallbackUsed: false
},
toolCalls: [{
name: "hwlab-agent-runtime.m3-io",
status: accepted ? "completed" : "blocked",
type: "skill-cli",
route,
accepted,
controlReady: accepted,
capabilityLevel: accepted ? "hwlab-api-control-ready" : "hwlab-api-control-blocked",
operationId,
traceId: "trc_rpt004_skill",
audit: {
auditId,
status
},
evidence: {
evidenceId,
status: accepted ? "blocked" : "blocked",
sourceKind: accepted ? "BLOCKED" : "BLOCKED",
blocker: blocker?.code ?? "runtime_durable_not_green",
writeStatus: "written_non_durable"
},
durable: {
status: "degraded",
blocker: blocker?.code ?? "runtime_durable_not_green"
},
blocker,
blockers: blocker ? [blocker] : [{
code: "runtime_durable_not_green",
message: "durable trust is not green"
}],
directGatewayCalls,
directBoxCalls,
directPatchPanelCalls,
fallbackUsed: false
}],
skills: {
status: "used",
items: [{
name: "hwlab-agent-runtime.m3-io",
route
}],
blockers: blocker ? [blocker] : []
},
reply: {
content: "M3 IO Skill CLI result"
}
};
}
function healthPayload(url, { revision = fixtureCommit } = {}) {
const isWeb = String(url).includes(":16666");
return {
serviceId: isWeb ? "hwlab-cloud-web" : "hwlab-cloud-api",
environment: "dev",
status: "ready",
ready: true,
commit: {
id: revision
},
image: {
tag: revision.slice(0, 7),
digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
},
runtime: {
durable: true,
ready: true,
status: "ready",
blocker: null,
liveRuntimeEvidence: true
},
readiness: {
ready: true,
status: "ready",
durability: {
ready: true,
status: "ready",
blocker: null
}
}
};
}
function workbenchHtml() {
return `
<!doctype html>
<html lang="zh-CN">
<body data-app-shell>
<main class="workbench-shell">
<section data-view="workspace">
<h1>HWLAB 云工作台</h1>
<aside class="resource-tree">硬件资源</aside>
<section class="center-workspace">Agent 对话 工作清单 使用说明</section>
<aside class="right-sidebar">可信记录 内部复核</aside>
<form id="command-form" class="command-form"></form>
</section>
<section data-view="gate" hidden>gate</section>
</main>
</body>
</html>`;
}
function fixtureArtifactJson() {
const serviceIds = [
"hwlab-cloud-api",
"hwlab-cloud-web",
"hwlab-agent-mgr",
"hwlab-agent-worker",
"hwlab-gateway",
"hwlab-gateway-simu",
"hwlab-box-simu",
"hwlab-patch-panel",
"hwlab-router",
"hwlab-tunnel-client",
"hwlab-edge-proxy",
"hwlab-cli",
"hwlab-agent-skills"
];
const deploy = {
commitId: fixtureCommit
};
const catalog = {
commitId: fixtureCommit,
artifactState: "published",
publish: {
ciPublished: true,
registryVerified: true
},
services: serviceIds.map((serviceId) => ({
serviceId,
commitId: fixtureCommit,
imageTag: fixtureCommit.slice(0, 7),
digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
publishState: "published"
}))
};
const artifactReport = {
commitId: fixtureCommit,
artifactPublish: {
sourceCommitId: fixtureCommit
}
};
return async (relativePath) => {
if (relativePath === "deploy/deploy.json") return deploy;
if (relativePath === "deploy/artifact-catalog.dev.json") return catalog;
if (relativePath === "reports/dev-gate/dev-artifacts.json") return artifactReport;
return null;
};
}
+12 -1
View File
@@ -791,10 +791,12 @@ async function validateRpt004MvpE2eReport(report, label) {
"mode",
"status",
"conclusion",
"commanderSummary",
"sourceContract",
"validationCommands",
"devPreconditions",
"target",
"runPolicy",
"expected",
"dimensions",
"checks",
@@ -815,6 +817,9 @@ async function validateRpt004MvpE2eReport(report, label) {
assert.ok(["pass", "blocked", "not_current"].includes(report.status), `${label}.status`);
assertObject(report.conclusion, `${label}.conclusion`);
assertObject(report.commanderSummary, `${label}.commanderSummary`);
assert.equal(report.commanderSummary.reportClass, "live-mutating", `${label}.commanderSummary.reportClass`);
assert.equal(typeof report.commanderSummary.allowedLiveMutationUsed, "boolean", `${label}.commanderSummary.allowedLiveMutationUsed`);
assert.ok(
["CHECK_ONLY", "BLOCKED", "NOT_CURRENT", "MVP_PASS"].includes(report.conclusion.classification),
`${label}.conclusion.classification`
@@ -859,7 +864,9 @@ async function validateRpt004MvpE2eReport(report, label) {
"layout",
"m3TrustedLoop",
"durableEvidence",
"codeAgentSkillPath"
"codeAgentSkillPath",
"codeAgentSession",
"skillCliM3Route"
];
assertObject(report.dimensions, `${label}.dimensions`);
for (const id of requiredDimensions) {
@@ -881,6 +888,8 @@ async function validateRpt004MvpE2eReport(report, label) {
"16666-default-chinese-workbench",
"layout-smoke-hit-target",
"code-agent-skill-path",
"code-agent-session-thread",
"skill-cli-hwlab-api-m3-route",
"m3-v1-io-trusted-loop",
"operation-audit-evidence-durable-green"
]) {
@@ -902,6 +911,8 @@ async function validateRpt004MvpE2eReport(report, label) {
assert.equal(report.safety.rolloutAttempted, false, `${label}.safety.rolloutAttempted`);
assert.equal(report.safety.secretValuesPrinted, false, `${label}.safety.secretValuesPrinted`);
assert.equal(report.safety.openAiFallbackCountsAsPass, false, `${label}.safety.openAiFallbackCountsAsPass`);
assert.equal(report.safety.liveM3WritesRequireExplicitFlag, true, `${label}.safety.liveM3WritesRequireExplicitFlag`);
assert.equal(report.liveMutationUsed, report.runPolicy.allowM3Write === true, `${label}.liveMutationUsed`);
if (report.artifacts?.artifactReport) {
assert.match(report.artifacts.artifactReport, /not applicable/u, `${label}.artifacts.artifactReport`);