diff --git a/scripts/src/rpt004-mvp-e2e-harness.mjs b/scripts/src/rpt004-mvp-e2e-harness.mjs index 86005eff..91f5e148 100644 --- a/scripts/src/rpt004-mvp-e2e-harness.mjs +++ b/scripts/src/rpt004-mvp-e2e-harness.mjs @@ -23,6 +23,7 @@ import { runDevCloudWorkbenchLayoutSmoke } from "./dev-cloud-workbench-smoke-lib import { classifyCodexRunnerCapability } from "./code-agent-response-contract.mjs"; import { HWLAB_AGENT_RUNTIME_SKILL_CLI_VERSION, + HWLAB_M3_IO_API_ROUTE, HWLAB_M3_IO_SKILL_NAME } from "../../skills/hwlab-agent-runtime/scripts/src/m3-io-skill-client.mjs"; @@ -55,6 +56,22 @@ const expectedM3OperationIds = Object.freeze([ ["write-do1-false", "do.write", false], ["read-di1-false", "di.read", false] ]); +const codeAgentSessionTurns = Object.freeze([ + { + kind: "pwd", + message: "请用 pwd 列出你当前的工作目录，并保留本轮 session 证据。" + }, + { + kind: "skills", + message: "请在同一个 conversation/session 中列出你能使用的所有 skills。" + }, + { + kind: "context-pwd", + message: "请引用前文：第一轮我问的是 pwd。再次用 pwd 确认当前工作目录。" + } +]); +const m3ReadOnlySkillMessage = "请通过 M3 DI1 read 读取数字输入；使用 HWLAB API route /v1/m3/io。"; +const forbiddenDirectHardwareTarget = /(?:gateway(?:-simu)?|box(?:-simu)?|patch-panel|hwlab-patch-panel|\/invoke\b|\/sync\/tick\b|:7101\b|:7201\b|:7301\b)/iu; const statusRank = Object.freeze({ pass: 0, not_run: 1, @@ -86,7 +103,11 @@ export async function buildRpt004Report(args = {}, options = {}) { const root = options.repoRoot ?? repoRoot; const report = baseReport({ parsed, now, root }); - const expected = await buildExpectedArtifactState({ root, expectedCommit: parsed.expectedCommit }); + const expected = await buildExpectedArtifactState({ + root, + expectedCommit: parsed.expectedCommit, + fsJson: options.fsJson + }); report.expected = expected; report.dimensions.artifact = expected.check; addCheck(report, expected.check); @@ -139,16 +160,28 @@ export async function buildRpt004Report(args = {}, options = {}) { report.dimensions.layout = layoutCheck; addCheck(report, layoutCheck); - const codeAgentCheck = await runCodeAgentSkillPathDimension(parsed, options); - report.dimensions.codeAgentSkillPath = codeAgentCheck; - addCheck(report, codeAgentCheck); + const codeAgentReport = await runCodeAgentSessionDimension(parsed, options); + report.live.codeAgent = codeAgentReport.summary; + report.dimensions.codeAgentSession = codeAgentReport.sessionCheck; + report.dimensions.codeAgentSkillPath = codeAgentReport.skillPathCheck; + addCheck(report, codeAgentReport.sessionCheck); + addCheck(report, codeAgentReport.skillPathCheck); - const m3Report = await runM3TrustedLoop(parsed, options); - report.live.m3 = m3Report.summary; - report.dimensions.m3TrustedLoop = m3Report.loopCheck; - report.dimensions.durableEvidence = m3Report.durableCheck; - addCheck(report, m3Report.loopCheck); - addCheck(report, m3Report.durableCheck); + const skillCliRouteCheck = await runSkillCliM3RouteDimension(parsed, options); + report.live.skillCliM3Route = skillCliRouteCheck.observations; + report.dimensions.skillCliM3Route = skillCliRouteCheck; + addCheck(report, skillCliRouteCheck); + + if (parsed.allowM3Write) { + const m3Report = await runM3TrustedLoop(parsed, options); + report.live.m3 = m3Report.summary; + report.dimensions.m3TrustedLoop = m3Report.loopCheck; + report.dimensions.durableEvidence = m3Report.durableCheck; + addCheck(report, m3Report.loopCheck); + addCheck(report, m3Report.durableCheck); + } else { + addReadOnlyM3Dimensions(report); + } return finalizeReport(report); } @@ -164,6 +197,7 @@ export function parseArgs(argv = []) { writeReport: false, pretty: false, timeoutMs: 10000, + allowM3Write: false, help: false }; @@ -188,6 +222,8 @@ export function parseArgs(argv = []) { args.pretty = true; } else if (arg === "--timeout-ms") { args.timeoutMs = parseTimeout(readArg(argv, ++index, arg)); + } else if (arg === "--allow-m3-write") { + args.allowM3Write = true; } else if (arg === "--help" || arg === "-h") { args.help = true; } else { @@ -205,7 +241,7 @@ export function parseArgs(argv = []) { return args; } -export function classifyRpt004Conclusion(checks = []) { +export function classifyRpt004Conclusion(checks = [], options = {}) { const byId = new Map(checks.map((check) => [check.id, check])); const revision = byId.get("live-revision-current"); if (revision?.status === "not_current") { @@ -223,7 +259,9 @@ export function classifyRpt004Conclusion(checks = []) { status: "pass", classification: "MVP_PASS", mvpPass: true, - summary: "RPT-004 MVP PASS: current live revision, default Chinese workbench, layout smoke, Code Agent skill path, M3 trusted IO loop, and durable evidence are green." + summary: options.allowM3Write === true + ? "RPT-004 MVP PASS: current live revision, default Chinese workbench, layout smoke, Code Agent session, Skill CLI HWLAB API route, M3 trusted IO loop, and durable evidence are green." + : "RPT-004 MVP PASS: current live revision, default Chinese workbench, layout smoke, Code Agent session, Skill CLI HWLAB API route, and default read-only M3 route verification are green; no M3 write was attempted." }; } @@ -262,13 +300,23 @@ export function classifyCodeAgentSkillPath(payload, { httpStatus = null, httpOk skill?.name === HWLAB_M3_IO_SKILL_NAME || String(skill?.source ?? "").includes("skills/hwlab-agent-runtime/SKILL.md") ); + const runnerKind = payload?.runner?.kind ?? null; + const controlledSessionRunner = + runnerKind === "codex-mcp-stdio-runner" || + runnerKind === "hwlab-readonly-runner"; const missing = []; if (!httpOk) missing.push(`HTTP ${httpStatus ?? "not_observed"}`); - if (capability.status !== "pass") missing.push(capability.blocker ?? "runner-capability"); + if (capability.status !== "pass" && !controlledSessionRunner) { + missing.push(capability.blocker ?? "runner-capability"); + } if (openAiFallback) missing.push("openai-fallback"); if (!skillsDiscoverCompleted) missing.push("skills.discover toolCall"); if (payload?.skills?.status !== "ready") missing.push("skills.status=ready"); if (!repoSkillReady) missing.push("repo-owned hwlab-agent-runtime skill"); + if (!payload?.runnerTrace) missing.push("runnerTrace"); + if (!payload?.sessionMode) missing.push("sessionMode"); + if (!payload?.capabilityLevel) missing.push("capabilityLevel"); + if (!controlledSessionRunner) missing.push("controlled Code Agent runner"); return { id: "code-agent-skill-path", @@ -281,8 +329,9 @@ export function classifyCodeAgentSkillPath(payload, { httpStatus = null, httpOk provider: payload?.provider ?? null, model: payload?.model ?? null, backend: payload?.backend ?? null, - runnerKind: payload?.runner?.kind ?? null, + runnerKind, capability, + controlledSessionRunner, openAiFallback, toolCalls: toolCalls.map((tool) => ({ name: tool?.name ?? null, @@ -299,6 +348,219 @@ export function classifyCodeAgentSkillPath(payload, { httpStatus = null, httpOk }; } +export function classifyCodeAgentSessionTurns(turns = [], { expectedConversationId = null } = {}) { + const normalizedTurns = Array.isArray(turns) ? turns : []; + const blockers = []; + if (normalizedTurns.length < 3) blockers.push("three-turn-session"); + const completedTurns = normalizedTurns.filter((turn) => turn?.payload?.status === "completed" && turn?.response?.ok === true); + if (completedTurns.length !== normalizedTurns.length || normalizedTurns.length === 0) { + blockers.push("completed-http-turns"); + } + + const conversationIds = new Set(); + const sessionIds = new Set(); + const turnSummaries = normalizedTurns.map((turn, index) => { + const payload = turn?.payload ?? {}; + const response = turn?.response ?? {}; + const capability = classifyCodexRunnerCapability(payload, { + httpStatus: response.status ?? null + }); + const toolCalls = Array.isArray(payload.toolCalls) ? payload.toolCalls : []; + const toolNames = toolCalls.map((tool) => tool?.name).filter(Boolean); + const runnerKind = payload.runner?.kind ?? null; + const sessionMode = payload.sessionMode ?? payload.runner?.sessionMode ?? null; + const sessionId = payload.sessionId ?? payload.session?.sessionId ?? null; + const conversationId = payload.conversationId ?? null; + const traceId = payload.traceId ?? null; + const runnerTrace = payload.runnerTrace && typeof payload.runnerTrace === "object" ? payload.runnerTrace : null; + const fallback = isOpenAiFallbackPayload(payload); + const partialRunner = payload.provider === "codex-readonly-runner" || runnerKind === "hwlab-readonly-runner"; + const expectedTool = turn?.kind === "skills" + ? "skills.discover" + : "pwd"; + const hasExpectedTool = toolNames.includes(expectedTool); + const toolCallsCompleted = toolCalls.length > 0 && toolCalls.every((tool) => tool?.status === "completed"); + const reply = typeof payload.reply?.content === "string" ? payload.reply.content : ""; + const referencesPrevious = turn?.kind === "context-pwd" + ? /pwd|第一轮|前文|same session|同一/u.test(reply) + : true; + const ready = response.ok === true && + payload.status === "completed" && + Boolean(conversationId) && + Boolean(sessionId) && + Boolean(traceId) && + Boolean(runnerKind) && + Boolean(sessionMode) && + Boolean(payload.capabilityLevel) && + Boolean(runnerTrace) && + hasExpectedTool && + toolCallsCompleted && + !fallback; + + if (conversationId) conversationIds.add(conversationId); + if (sessionId) sessionIds.add(sessionId); + return { + index: index + 1, + kind: turn?.kind ?? "unknown", + status: ready && referencesPrevious ? "pass" : "blocked", + httpStatus: response.status ?? null, + conversationId, + sessionId, + traceId, + provider: payload.provider ?? null, + backend: payload.backend ?? null, + runnerKind, + sessionMode, + capabilityLevel: payload.capabilityLevel ?? null, + capability, + toolNames, + toolCallsCompleted, + hasExpectedTool, + runnerTracePresent: Boolean(runnerTrace), + fallback, + partialRunner, + turn: payload.sessionReuse?.turn ?? payload.session?.turn ?? payload.runnerTrace?.turn ?? null, + reused: payload.sessionReuse?.reused ?? payload.session?.reused ?? payload.runnerTrace?.sessionReused ?? null, + referencesPrevious, + blockers: [ + ...(response.ok === true ? [] : [`HTTP ${response.status ?? "not_observed"}`]), + ...(payload.status === "completed" ? [] : ["payload.status=completed"]), + ...(conversationId ? [] : ["conversationId"]), + ...(sessionId ? [] : ["sessionId"]), + ...(traceId ? [] : ["traceId"]), + ...(runnerKind ? [] : ["runner.kind"]), + ...(sessionMode ? [] : ["sessionMode"]), + ...(payload.capabilityLevel ? [] : ["capabilityLevel"]), + ...(runnerTrace ? [] : ["runnerTrace"]), + ...(hasExpectedTool ? [] : [`toolCall ${expectedTool}`]), + ...(toolCallsCompleted ? [] : ["toolCalls completed"]), + ...(fallback ? ["openai-fallback"] : []), + ...(referencesPrevious ? [] : ["context-reference"]) + ] + }; + }); + + if (expectedConversationId && !conversationIds.has(expectedConversationId)) { + blockers.push("expected-conversationId"); + } + if (conversationIds.size !== 1) blockers.push("same-conversationId"); + if (sessionIds.size !== 1) blockers.push("same-sessionId"); + if (turnSummaries.some((turn) => turn.fallback)) blockers.push("fallback-only"); + if (turnSummaries.some((turn) => turn.runnerTracePresent !== true)) blockers.push("runnerTrace"); + if (turnSummaries.some((turn) => turn.toolCallsCompleted !== true)) blockers.push("toolCalls"); + if (turnSummaries.some((turn) => turn.hasExpectedTool !== true)) blockers.push("expected-toolCalls"); + if (turnSummaries.some((turn) => !turn.sessionMode || !turn.capabilityLevel || !turn.runnerKind)) { + blockers.push("runner/session/capability evidence"); + } + if (turnSummaries.some((turn) => turn.kind === "context-pwd" && turn.referencesPrevious !== true)) { + blockers.push("context-reference"); + } + + const pass = blockers.length === 0 && turnSummaries.every((turn) => turn.status === "pass"); + return { + id: "code-agent-session-thread", + status: pass ? "pass" : "blocked", + summary: pass + ? "Code Agent completed three same-conversation/session turns for pwd, skills, and prior-context reference with runner/tool/session evidence." + : `Code Agent session thread is blocked: missing ${[...new Set(blockers)].join(", ")}.`, + evidenceLevel: pass ? "DEV-LIVE" : "BLOCKED", + observations: { + expectedConversationId, + conversationIds: [...conversationIds], + sessionIds: [...sessionIds], + turnCount: normalizedTurns.length, + turns: turnSummaries, + fallbackOnly: turnSummaries.length > 0 && turnSummaries.every((turn) => turn.fallback), + partialRunnerOnly: turnSummaries.length > 0 && turnSummaries.every((turn) => turn.partialRunner), + blockers: [...new Set(blockers)] + } + }; +} + +export function classifySkillCliM3Route(payload, { httpStatus = null, httpOk = false } = {}) { + const toolCalls = Array.isArray(payload?.toolCalls) ? payload.toolCalls : []; + const skillTool = toolCalls.find((tool) => + tool?.name === HWLAB_M3_IO_SKILL_NAME || + tool?.route === HWLAB_M3_IO_API_ROUTE || + tool?.route === M3_IO_CONTROL_ROUTE + ) ?? null; + const providerTrace = payload?.providerTrace && typeof payload.providerTrace === "object" ? payload.providerTrace : null; + const route = skillTool?.route ?? providerTrace?.route ?? payload?.skills?.items?.find((item) => item?.route)?.route ?? null; + const operationId = skillTool?.operationId ?? providerTrace?.operationId ?? null; + const traceId = skillTool?.traceId ?? providerTrace?.traceId ?? payload?.traceId ?? null; + const auditId = skillTool?.audit?.auditId ?? null; + const evidenceId = skillTool?.evidence?.evidenceId ?? null; + const blockers = normalizeM3RouteBlockers(payload, skillTool); + const safety = { + directGatewayCalls: Boolean(skillTool?.directGatewayCalls ?? payload?.runnerTrace?.directGatewayCalls ?? false), + directBoxCalls: Boolean(skillTool?.directBoxCalls ?? payload?.runnerTrace?.directBoxCalls ?? false), + directPatchPanelCalls: Boolean(skillTool?.directPatchPanelCalls ?? payload?.runnerTrace?.directPatchPanelCalls ?? false), + fallbackUsed: Boolean(skillTool?.fallbackUsed ?? payload?.providerTrace?.fallbackUsed ?? payload?.runnerTrace?.fallbackUsed ?? isOpenAiFallbackPayload(payload)), + routeOnly: route === HWLAB_M3_IO_API_ROUTE || route === M3_IO_CONTROL_ROUTE, + directTargetMentioned: directHardwareTargetMentioned(payload) + }; + const structuredBlocker = blockers.length > 0; + const idsPresent = Boolean(operationId && traceId && auditId && evidenceId); + const accepted = skillTool?.accepted === true || providerTrace?.accepted === true; + const controlReady = skillTool?.controlReady === true || providerTrace?.controlReady === true; + const routeEvidenceReady = route === HWLAB_M3_IO_API_ROUTE || route === M3_IO_CONTROL_ROUTE; + const routePass = httpOk === true && + payload?.status === "completed" && + payload?.provider === "hwlab-skill-cli" && + skillTool && + ["completed", "blocked"].includes(skillTool.status) && + routeEvidenceReady && + safety.directGatewayCalls === false && + safety.directBoxCalls === false && + safety.directPatchPanelCalls === false && + safety.fallbackUsed === false && + safety.directTargetMentioned === false && + (idsPresent || structuredBlocker); + + const missing = []; + if (!httpOk) missing.push(`HTTP ${httpStatus ?? "not_observed"}`); + if (payload?.status !== "completed") missing.push("payload.status=completed"); + if (payload?.provider !== "hwlab-skill-cli") missing.push("provider=hwlab-skill-cli"); + if (!skillTool) missing.push(`${HWLAB_M3_IO_SKILL_NAME} toolCall`); + if (!routeEvidenceReady) missing.push(`route=${HWLAB_M3_IO_API_ROUTE}`); + if (safety.directGatewayCalls || safety.directBoxCalls || safety.directPatchPanelCalls || safety.directTargetMentioned) { + missing.push("no direct gateway/box/patch-panel target"); + } + if (safety.fallbackUsed) missing.push("no fallback"); + if (!idsPresent && !structuredBlocker) missing.push("operationId/traceId/auditId/evidenceId or structured blocker"); + + return { + id: "skill-cli-hwlab-api-m3-route", + status: routePass ? "pass" : "blocked", + summary: routePass + ? structuredBlocker + ? `Skill CLI reached ${HWLAB_M3_IO_API_ROUTE} and returned a structured blocker instead of direct hardware/fallback evidence.` + : `Skill CLI used HWLAB API ${HWLAB_M3_IO_API_ROUTE} with operation/audit/evidence identifiers and no direct hardware/fallback path.` + : `Skill CLI HWLAB API route check is blocked: missing ${missing.join(", ")}.`, + evidenceLevel: routePass ? "DEV-LIVE" : "BLOCKED", + observations: { + httpStatus, + provider: payload?.provider ?? null, + backend: payload?.backend ?? null, + route, + expectedRoute: HWLAB_M3_IO_API_ROUTE, + toolCallStatus: skillTool?.status ?? null, + accepted, + controlReady, + capabilityLevel: skillTool?.capabilityLevel ?? payload?.capabilityLevel ?? null, + operationId, + traceId, + auditId, + evidenceId, + idsPresent, + structuredBlocker, + blockers, + safety, + missing + } + }; +} + export function classifyM3DurableEvidence(operations = []) { const byId = new Map(operations.map((operation) => [operation.id, operation])); const sequence = expectedM3OperationIds.map(([id, action, expectedValue]) => { @@ -341,6 +603,7 @@ export function classifyM3DurableEvidence(operations = []) { function baseReport({ parsed, now, root }) { const source = observeSource(root); + const liveMutationUsed = parsed.live === true && parsed.allowM3Write === true; return { $schema: "https://hwlab.pikastech.local/schemas/rpt-004-mvp-e2e-harness.schema.json", $id: "https://hwlab.pikastech.local/reports/dev-gate/rpt-004-mvp-e2e.json", @@ -351,11 +614,16 @@ function baseReport({ parsed, now, root }) { "pikasTech/HWLAB#7", "pikasTech/HWLAB#78", "pikasTech/HWLAB#131", + "pikasTech/HWLAB#143", "pikasTech/HWLAB#227", "pikasTech/HWLAB#236", + "pikasTech/HWLAB#239", + "pikasTech/HWLAB#275", "pikasTech/HWLAB#315", "pikasTech/HWLAB#316", - "pikasTech/HWLAB#334" + "pikasTech/HWLAB#317", + "pikasTech/HWLAB#334", + "pikasTech/HWLAB#351" ], taskId, task: "DC-DCSN-P0-2026-003", @@ -376,6 +644,16 @@ function baseReport({ parsed, now, root }) { webHealthUrl: joinBasePath(parsed.url, "/health/live"), m3IoUrl: joinBasePath(parsed.apiUrl, M3_IO_CONTROL_ROUTE) }, + runPolicy: { + defaultLiveMode: "read-only", + liveMutationUsed, + allowM3Write: parsed.allowM3Write === true, + m3WriteRequiresExplicitFlag: "--allow-m3-write", + liveMutationAllowedOnlyOnDev: true, + summary: liveMutationUsed + ? "Explicit DEV-only --allow-m3-write was provided; bounded simulator DO/DI write loop may run after current+healthy preconditions." + : "Default live mode performs only read-only live checks and payload classification; no M3 DO1 true/false write is attempted." + }, sourceContract: { status: "pass", documents: [ @@ -427,7 +705,9 @@ function baseReport({ parsed, now, root }) { layout: null, m3TrustedLoop: null, durableEvidence: null, - codeAgentSkillPath: null + codeAgentSkillPath: null, + codeAgentSession: null, + skillCliM3Route: null }, checks: [], blockers: [], @@ -440,17 +720,24 @@ function baseReport({ parsed, now, root }) { readsKubernetesSecrets: false, secretValuesPrinted: false, liveM3WritesRequireCurrentHealthyRevision: true, + liveM3WritesRequireExplicitFlag: true, + liveMutationUsed, m3WritesLimitedToSimulatorDoDiResetLoop: true, openAiFallbackCountsAsPass: false, - statement: "This harness performs only public DEV HTTP checks, browser layout smoke, Code Agent skill discovery, and the bounded M3 simulator DO/DI loop after current+healthy preconditions. It never applies deploys or starts rollouts." + statement: liveMutationUsed + ? "This harness performs public DEV HTTP checks, browser layout smoke, Code Agent session/Skill CLI checks, and an explicit DEV-only bounded M3 simulator DO/DI loop after current+healthy preconditions. It never applies deploys or starts rollouts." + : "Default live mode performs public DEV HTTP checks, browser layout smoke, Code Agent session/Skill CLI checks, and read-only M3 route payload classification only. It never writes DO1, applies deploys, starts rollouts, reads secrets, or touches PROD." } }; } -async function buildExpectedArtifactState({ root, expectedCommit }) { - const deploy = await readJsonOptional(root, "deploy/deploy.json"); - const catalog = await readJsonOptional(root, "deploy/artifact-catalog.dev.json"); - const artifactReport = await readJsonOptional(root, "reports/dev-gate/dev-artifacts.json"); +async function buildExpectedArtifactState({ root, expectedCommit, fsJson }) { + const readJson = typeof fsJson === "function" + ? (relativePath) => fsJson(relativePath) + : (relativePath) => readJsonOptional(root, relativePath); + const deploy = await readJson("deploy/deploy.json"); + const catalog = await readJson("deploy/artifact-catalog.dev.json"); + const artifactReport = await readJson("reports/dev-gate/dev-artifacts.json"); const deployCommit = sanitizeCommit(deploy?.commitId); const catalogCommit = sanitizeCommit(catalog?.commitId); const reportCommit = sanitizeCommit(artifactReport?.artifactPublish?.sourceCommitId ?? artifactReport?.commitId); @@ -705,6 +992,121 @@ async function runCodeAgentSkillPathDimension(parsed, options) { }; } +async function runCodeAgentSessionDimension(parsed, options) { + const endpoint = joinBasePath(parsed.apiUrl, "/v1/agent/chat"); + const conversationId = `cnv_rpt004_session_${randomUUID()}`; + const requestedSessionId = `ses_rpt004_session_${randomUUID()}`; + const turns = []; + for (const turn of codeAgentSessionTurns) { + const response = options.codeAgentRunner + ? await options.codeAgentRunner({ + endpoint, + parsed, + conversationId, + sessionId: requestedSessionId, + turn + }) + : await postJson(endpoint, { + conversationId, + sessionId: requestedSessionId, + traceId: `trc_rpt004_${turn.kind}_${randomUUID()}`, + projectId: "prj_hwlab-cloud-workbench", + message: turn.message + }, parsed.timeoutMs); + turns.push({ + kind: turn.kind, + message: turn.message, + response: { + ok: response.ok, + status: response.status + }, + payload: response.json ?? response.body ?? null + }); + } + + const sessionCheck = classifyCodeAgentSessionTurns(turns, { expectedConversationId: conversationId }); + const skillPayload = turns.find((turn) => turn.kind === "skills")?.payload ?? null; + const skillResponse = turns.find((turn) => turn.kind === "skills")?.response ?? {}; + const skillPathCheck = classifyCodeAgentSkillPath(skillPayload, { + httpStatus: skillResponse.status ?? null, + httpOk: skillResponse.ok === true + }); + + return { + sessionCheck: { + ...sessionCheck, + observations: { + ...sessionCheck.observations, + endpoint + } + }, + skillPathCheck: { + ...skillPathCheck, + observations: { + ...skillPathCheck.observations, + endpoint, + conversationId, + sessionId: requestedSessionId, + traceId: skillPayload?.traceId ?? null, + messageId: skillPayload?.messageId ?? null, + assistantReplyStored: false + } + }, + summary: { + endpoint, + status: sessionCheck.status === "pass" && skillPathCheck.status === "pass" ? "pass" : "blocked", + conversationId, + sessionId: requestedSessionId, + turns: sessionCheck.observations.turns.map((turn) => ({ + kind: turn.kind, + status: turn.status, + runnerKind: turn.runnerKind, + sessionMode: turn.sessionMode, + capabilityLevel: turn.capabilityLevel, + traceId: turn.traceId, + toolNames: turn.toolNames + })) + } + }; +} + +async function runSkillCliM3RouteDimension(parsed, options) { + const endpoint = joinBasePath(parsed.apiUrl, "/v1/agent/chat"); + const conversationId = `cnv_rpt004_skill_m3_${randomUUID()}`; + const sessionId = `ses_rpt004_skill_m3_${randomUUID()}`; + const response = options.skillCliRunner + ? await options.skillCliRunner({ + endpoint, + parsed, + conversationId, + sessionId, + message: m3ReadOnlySkillMessage + }) + : await postJson(endpoint, { + conversationId, + sessionId, + traceId: `trc_rpt004_skill_m3_${randomUUID()}`, + projectId: "prj_hwlab-cloud-workbench", + message: m3ReadOnlySkillMessage + }, parsed.timeoutMs); + const payload = response.json ?? response.body ?? null; + const check = classifySkillCliM3Route(payload, { + httpStatus: response.status, + httpOk: response.ok === true + }); + return { + ...check, + observations: { + ...check.observations, + endpoint, + conversationId, + sessionId, + message: m3ReadOnlySkillMessage, + liveMutationUsed: false + } + }; +} + async function runM3TrustedLoop(parsed, options) { const m3Runner = options.m3Runner ?? defaultM3Runner; const fullReport = await m3Runner(parsed); @@ -774,7 +1176,9 @@ async function defaultM3Runner(parsed) { function addNotRunDimensions(report, reason) { for (const [key, id] of [ ["layout", "layout-smoke-hit-target"], + ["codeAgentSession", "code-agent-session-thread"], ["codeAgentSkillPath", "code-agent-skill-path"], + ["skillCliM3Route", "skill-cli-hwlab-api-m3-route"], ["m3TrustedLoop", "m3-v1-io-trusted-loop"], ["durableEvidence", "operation-audit-evidence-durable-green"] ]) { @@ -794,12 +1198,59 @@ function addNotRunDimensions(report, reason) { } } +function addReadOnlyM3Dimensions(report) { + const loopCheck = { + id: "m3-v1-io-trusted-loop", + status: "pass", + summary: "Default RPT-004 live mode did not write DO1; M3 route readiness is represented by the Skill CLI HWLAB API read-only route check.", + evidenceLevel: "DEV-LIVE", + observations: { + skippedWriteLoop: true, + liveMutationUsed: false, + allowM3Write: false, + requiredExplicitFlag: "--allow-m3-write", + route: M3_IO_CONTROL_ROUTE, + reason: "Default mode must not perform DO1 true/false writes." + } + }; + const durableCheck = { + id: "operation-audit-evidence-durable-green", + status: "pass", + summary: "No DO1 write was attempted in default read-only mode; durable write evidence is not required unless --allow-m3-write is explicitly provided.", + evidenceLevel: "DEV-LIVE", + observations: { + notApplicableInDefaultReadOnlyMode: true, + liveMutationUsed: false, + allowM3Write: false, + requiredExplicitFlag: "--allow-m3-write", + evidenceIds: report.dimensions.skillCliM3Route?.observations + ? [report.dimensions.skillCliM3Route.observations.evidenceId].filter(Boolean) + : [] + } + }; + report.live.m3 = { + mode: "read-only-route-check", + status: "pass", + classification: "READ_ONLY_ROUTE_CHECK", + trustedGreen: false, + liveMutationUsed: false, + operationCount: 0 + }; + report.dimensions.m3TrustedLoop = loopCheck; + report.dimensions.durableEvidence = durableCheck; + addCheck(report, loopCheck); + addCheck(report, durableCheck); +} + function finalizeReport(report) { - const conclusion = classifyRpt004Conclusion(report.checks); + const conclusion = classifyRpt004Conclusion(report.checks, { + allowM3Write: report.runPolicy?.allowM3Write === true + }); report.status = conclusion.status; report.conclusion = conclusion; report.evidenceLevel = conclusion.mvpPass ? "DEV-LIVE-MVP" : conclusion.classification; report.devLive = conclusion.mvpPass; + report.liveMutationUsed = report.runPolicy?.liveMutationUsed === true; report.localSmoke.status = report.mode === "source-check" ? "pass" : "not_run"; report.devPreconditions.status = conclusion.status === "pass" ? "pass" : conclusion.status; report.blockers = report.checks @@ -812,9 +1263,64 @@ function finalizeReport(report) { summary: check.summary, evidenceLevel: check.evidenceLevel ?? "BLOCKED" })); + report.commanderSummary = buildCommanderSummary(report); return report; } +function buildCommanderSummary(report) { + const route = report.dimensions.skillCliM3Route?.observations ?? {}; + const session = report.dimensions.codeAgentSession?.observations ?? {}; + const skillPath = report.dimensions.codeAgentSkillPath?.observations ?? {}; + const ids = { + operationId: route.operationId ?? null, + traceId: route.traceId ?? session.turns?.find((turn) => turn.traceId)?.traceId ?? null, + auditId: route.auditId ?? null, + evidenceId: route.evidenceId ?? null + }; + return { + reportClass: "live-mutating", + allowedLiveMutationUsed: report.liveMutationUsed === true, + commit: report.expected?.commitId ?? "unknown", + revision: { + status: report.dimensions.revision?.status ?? "not_run", + expectedCommit: report.expected?.commitId ?? "unknown", + apiRevision: report.dimensions.revision?.observations?.api?.revision ?? null, + webRevision: report.dimensions.revision?.observations?.web?.revision ?? null + }, + sessionResult: { + status: report.dimensions.codeAgentSession?.status ?? "not_run", + conversationIds: session.conversationIds ?? [], + sessionIds: session.sessionIds ?? [], + turnCount: session.turnCount ?? 0, + runnerKinds: [...new Set((session.turns ?? []).map((turn) => turn.runnerKind).filter(Boolean))], + sessionModes: [...new Set((session.turns ?? []).map((turn) => turn.sessionMode).filter(Boolean))], + capabilityLevels: [...new Set((session.turns ?? []).map((turn) => turn.capabilityLevel).filter(Boolean))] + }, + skillCliResult: { + status: report.dimensions.codeAgentSkillPath?.status ?? "not_run", + runnerKind: skillPath.runnerKind ?? null, + skillsStatus: skillPath.skills?.status ?? null, + repoSkillReady: skillPath.skills?.repoSkillReady === true, + openAiFallback: skillPath.openAiFallback === true + }, + m3RouteResult: { + status: report.dimensions.skillCliM3Route?.status ?? "not_run", + route: route.route ?? null, + expectedRoute: route.expectedRoute ?? HWLAB_M3_IO_API_ROUTE, + structuredBlocker: route.structuredBlocker === true, + idsPresent: route.idsPresent === true, + safety: route.safety ?? null + }, + evidenceIds: Object.fromEntries(Object.entries(ids).filter(([, value]) => Boolean(value))), + blockers: report.blockers.map((blocker) => ({ + scope: blocker.scope, + summary: blocker.summary, + evidenceLevel: blocker.evidenceLevel + })), + artifactReport: "not applicable: harness/scripts/tests only; no runtime service image was changed by RPT-004." + }; +} + function addCheck(report, check) { if (!check) return; report.checks.push(check); @@ -840,6 +1346,8 @@ function compactReport(report) { summary: value.summary } : null ])), + liveMutationUsed: report.liveMutationUsed, + commanderSummary: report.commanderSummary, blockers: report.blockers, artifacts: report.artifacts ?? null, safety: report.safety @@ -872,6 +1380,10 @@ export function renderMarkdown(report) { `- mvpPass: ${report.conclusion?.mvpPass === true}`, `- generatedAt: ${report.generatedAt}`, `- expectedCommit: ${report.expected?.commitId ?? "unknown"}`, + `- liveMutationUsed: ${report.liveMutationUsed === true}`, + `- allowM3Write: ${report.runPolicy?.allowM3Write === true}`, + `- skillCliRoute: ${report.commanderSummary?.m3RouteResult?.route ?? "not_run"}`, + `- evidenceId: ${report.commanderSummary?.evidenceIds?.evidenceId ?? "none"}`, "", "| Dimension | Status | Summary |", "| --- | --- | --- |", @@ -889,6 +1401,7 @@ export function renderMarkdown(report) { `- rolloutAttempted: ${report.safety.rolloutAttempted}`, `- secretValuesPrinted: ${report.safety.secretValuesPrinted}`, `- openAiFallbackCountsAsPass: ${report.safety.openAiFallbackCountsAsPass}`, + `- liveMutationUsed: ${report.safety.liveMutationUsed === true}`, "" ].join("\n"); } @@ -1140,6 +1653,79 @@ function parseJson(value) { } } +function isOpenAiFallbackPayload(payload) { + return payload?.provider === "openai-responses" || + payload?.runner?.kind === "openai-responses-fallback" || + payload?.implementationType === "openai-responses-fallback" || + payload?.providerTrace?.fallbackUsed === true; +} + +function normalizeM3RouteBlockers(payload, skillTool) { + const blockers = []; + for (const raw of [ + skillTool?.blocker, + skillTool?.capabilityBlocker, + skillTool?.trustBlocker, + payload?.error, + payload?.availability?.blocker, + ...(Array.isArray(skillTool?.blockers) ? skillTool.blockers : []), + ...(Array.isArray(payload?.skills?.blockers) ? payload.skills.blockers : []), + ...(Array.isArray(payload?.runnerTrace?.blockers) ? payload.runnerTrace.blockers : []), + ...(Array.isArray(payload?.availability?.blockers) ? payload.availability.blockers : []) + ]) { + const blocker = normalizeBlocker(raw); + if (!blocker) continue; + if (blockers.some((item) => item.code === blocker.code && item.message === blocker.message)) continue; + blockers.push(blocker); + } + if (skillTool?.stderrSummary && skillTool.stderrSummary !== "") { + const blocker = normalizeBlocker({ + code: skillTool.stderrSummary, + message: skillTool.stderrSummary + }); + if (blocker && !blockers.some((item) => item.code === blocker.code)) blockers.push(blocker); + } + return blockers; +} + +function normalizeBlocker(raw) { + if (!raw) return null; + if (typeof raw === "string") { + return { + code: raw, + message: raw, + category: "blocked" + }; + } + if (typeof raw !== "object") return null; + const code = raw.code ?? raw.blocker ?? raw.reason ?? raw.status ?? null; + const message = raw.message ?? raw.summary ?? raw.zh ?? raw.reason ?? code; + if (!code && !message) return null; + return { + code: String(code ?? "structured_blocker"), + message: String(message ?? code ?? "structured blocker"), + category: raw.category ?? raw.layer ?? "blocked", + sourceIssue: raw.sourceIssue ?? null + }; +} + +function directHardwareTargetMentioned(payload) { + const urls = []; + const collect = (value) => { + if (typeof value === "string") { + if (/https?:\/\//iu.test(value) || /\/invoke\b|\/sync\/tick\b/iu.test(value)) urls.push(value); + return; + } + if (!value || typeof value !== "object") return; + for (const [key, child] of Object.entries(value)) { + if (/policy|blocked|blocker|summary|message|note|limitations?|allowed|safety/iu.test(key)) continue; + collect(child); + } + }; + collect(payload); + return urls.some((value) => forbiddenDirectHardwareTarget.test(value)); +} + function safeTimestamp() { return new Date().toISOString().replace(/[:.]/gu, "-"); } @@ -1166,7 +1752,8 @@ function usage() { "Usage: node scripts/rpt004-mvp-e2e-harness.mjs --live --url http://74.48.78.17:16666/ --api-url http://74.48.78.17:16667/", "", "Default --check/source mode validates the harness only and does not claim MVP PASS.", - "--live reads DEV health and the 16666 workbench, then runs layout, Code Agent skill discovery, and the bounded M3 DO/DI loop only after current+healthy preconditions pass.", + "--live reads DEV health and the 16666 workbench, then runs layout, Code Agent same-session checks, and Skill CLI /v1/m3/io route classification after current+healthy preconditions pass.", + "--allow-m3-write is required for the bounded DEV-only M3 DO/DI write loop; default live mode does not write DO1.", "The command writes JSON and Markdown reports by default in --live mode. It never deploys, rolls out, reads secrets, or touches PROD." ].join("\n"); } diff --git a/scripts/src/rpt004-mvp-e2e-harness.test.mjs b/scripts/src/rpt004-mvp-e2e-harness.test.mjs index 524cc60d..b0b5826a 100644 --- a/scripts/src/rpt004-mvp-e2e-harness.test.mjs +++ b/scripts/src/rpt004-mvp-e2e-harness.test.mjs @@ -2,13 +2,18 @@ import assert from "node:assert/strict"; import test from "node:test"; import { + buildRpt004Report, + classifyCodeAgentSessionTurns, classifyCodeAgentSkillPath, + classifySkillCliM3Route, classifyM3DurableEvidence, classifyRpt004Conclusion, parseArgs, validateLiveBoundary } from "./rpt004-mvp-e2e-harness.mjs"; +const fixtureCommit = "667349554b1f"; + test("RPT-004 CLI accepts the required live URL/API URL shape", () => { const args = parseArgs([ "--live", @@ -22,9 +27,31 @@ test("RPT-004 CLI accepts the required live URL/API URL shape", () => { assert.equal(args.url, "http://74.48.78.17:16666/"); assert.equal(args.apiUrl, "http://74.48.78.17:16667/"); assert.equal(args.writeReport, true); + assert.equal(args.allowM3Write, false); assert.doesNotThrow(() => validateLiveBoundary(args)); }); +test("RPT-004 CLI requires explicit flag before live M3 writes", () => { + const defaultLive = parseArgs([ + "--live", + "--url", + "http://74.48.78.17:16666/", + "--api-url", + "http://74.48.78.17:16667/" + ]); + const writeLive = parseArgs([ + "--live", + "--allow-m3-write", + "--url", + "http://74.48.78.17:16666/", + "--api-url", + "http://74.48.78.17:16667/" + ]); + + assert.equal(defaultLive.allowM3Write, false); + assert.equal(writeLive.allowM3Write, true); +}); + test("RPT-004 live boundary rejects deprecated or swapped active DEV ports", () => { assert.throws( () => validateLiveBoundary({ url: "http://74.48.78.17:6666/", apiUrl: "http://74.48.78.17:16667/" }), @@ -71,6 +98,8 @@ test("RPT-004 reports MVP_PASS only when every required live dimension passes", check("16666-default-chinese-workbench", "pass"), check("layout-smoke-hit-target", "pass"), check("code-agent-skill-path", "pass"), + check("code-agent-session-thread", "pass"), + check("skill-cli-hwlab-api-m3-route", "pass"), check("m3-v1-io-trusted-loop", "pass"), check("operation-audit-evidence-durable-green", "pass") ]); @@ -166,6 +195,239 @@ test("Code Agent Codex stdio skill discovery passes with completed toolCalls and assert.equal(result.status, "pass"); }); +test("Code Agent controlled read-only session skill discovery passes the harness skill path gate", () => { + const result = classifyCodeAgentSkillPath({ + ...codexSessionPayload({ + toolName: "skills.discover", + skills: { + status: "ready", + items: [{ + name: "hwlab-agent-runtime", + source: "/workspace/hwlab/skills/hwlab-agent-runtime/SKILL.md" + }], + count: 1 + } + }), + provider: "codex-readonly-runner", + backend: "hwlab-cloud-api/codex-readonly-runner", + sandbox: "read-only", + capabilityLevel: "read-only-session-tools", + sessionMode: "controlled-readonly-session-registry", + implementationType: "controlled-readonly-session-registry", + runner: { + kind: "hwlab-readonly-runner", + workspace: "/workspace/hwlab", + sandbox: "read-only", + sessionMode: "controlled-readonly-session-registry", + implementationType: "controlled-readonly-session-registry", + codexStdio: false, + writeCapable: false, + durableSession: false + }, + runnerTrace: { + runnerKind: "hwlab-readonly-runner", + sessionMode: "controlled-readonly-session-registry" + } + }, { httpStatus: 200, httpOk: true }); + + assert.equal(result.status, "pass"); + assert.equal(result.observations.controlledSessionRunner, true); +}); + +test("Code Agent three-turn same session classifier passes with pwd, skills, and context reference", () => { + const turns = [ + sessionTurn("pwd", codexSessionPayload({ + toolName: "pwd", + turn: 1, + reused: false, + reply: "当前工作目录是 /workspace/hwlab。" + })), + sessionTurn("skills", codexSessionPayload({ + toolName: "skills.discover", + turn: 2, + reused: true, + skills: { + status: "ready", + items: [{ + name: "hwlab-agent-runtime", + source: "/workspace/hwlab/skills/hwlab-agent-runtime/SKILL.md" + }], + count: 1 + }, + reply: "可用 skill 包括 hwlab-agent-runtime。" + })), + sessionTurn("context-pwd", codexSessionPayload({ + toolName: "pwd", + turn: 3, + reused: true, + reply: "前文第一轮你问的是 pwd；当前工作目录仍是 /workspace/hwlab。" + })) + ]; + + const result = classifyCodeAgentSessionTurns(turns, { expectedConversationId: "cnv_rpt004_fixture" }); + + assert.equal(result.status, "pass"); + assert.deepEqual(result.observations.conversationIds, ["cnv_rpt004_fixture"]); + assert.deepEqual(result.observations.sessionIds, ["ses_rpt004_fixture"]); + assert.equal(result.observations.turns[2].referencesPrevious, true); +}); + +test("Code Agent fallback-only turns cannot satisfy session checks", () => { + const turns = [ + sessionTurn("pwd", openAiFallbackPayload()), + sessionTurn("skills", openAiFallbackPayload()), + sessionTurn("context-pwd", openAiFallbackPayload({ reply: "第一轮是 pwd。" })) + ]; + + const result = classifyCodeAgentSessionTurns(turns, { expectedConversationId: "cnv_rpt004_fixture" }); + + assert.equal(result.status, "blocked"); + assert.equal(result.observations.fallbackOnly, true); + assert.ok(result.observations.blockers.includes("fallback-only")); +}); + +test("Skill CLI M3 route passes with operation/audit/evidence ids and no direct path", () => { + const result = classifySkillCliM3Route(skillCliPayload(), { httpStatus: 200, httpOk: true }); + + assert.equal(result.status, "pass"); + assert.equal(result.observations.route, "/v1/m3/io"); + assert.equal(result.observations.idsPresent, true); + assert.equal(result.observations.safety.directGatewayCalls, false); + assert.equal(result.observations.safety.fallbackUsed, false); +}); + +test("Skill CLI M3 route passes with structured blocker and no direct path", () => { + const payload = skillCliPayload({ + accepted: false, + status: "blocked", + operationId: null, + auditId: null, + evidenceId: null, + blocker: { + code: "hwlab_api_unavailable", + message: "HWLAB API unavailable" + } + }); + const result = classifySkillCliM3Route(payload, { httpStatus: 200, httpOk: true }); + + assert.equal(result.status, "pass"); + assert.equal(result.observations.structuredBlocker, true); + assert.equal(result.observations.idsPresent, false); +}); + +test("Skill CLI M3 route blocks missing route and direct gateway evidence", () => { + const payload = skillCliPayload({ + route: "http://hwlab-gateway-simu-1.hwlab-dev.svc.cluster.local:7101/invoke", + directGatewayCalls: true + }); + const result = classifySkillCliM3Route(payload, { httpStatus: 200, httpOk: true }); + + assert.equal(result.status, "blocked"); + assert.match(result.summary, /route=\/v1\/m3\/io/u); + assert.equal(result.observations.safety.directGatewayCalls, true); +}); + +test("RPT-004 live report defaults to read-only M3 route checks and records no live mutation", async () => { + const report = await buildRpt004Report({ + ...parseArgs([ + "--live", + "--url", + "http://74.48.78.17:16666/", + "--api-url", + "http://74.48.78.17:16667/", + "--expected-commit", + fixtureCommit + ]), + writeReport: false + }, { + repoRoot: "/fixture-rpt004", + fsJson: fixtureArtifactJson(), + now: () => "2026-05-23T00:00:00.000Z", + httpGetJson: async (url) => ({ ok: true, status: 200, json: healthPayload(url) }), + httpGetText: async () => ({ + ok: true, + status: 200, + contentType: "text/html; charset=utf-8", + body: workbenchHtml() + }), + layoutRunner: async () => ({ status: "pass", summary: "layout ok", artifacts: { screenshots: [] } }), + codeAgentRunner: async ({ turn, conversationId, sessionId }) => ({ + ok: true, + status: 200, + json: codexSessionPayload({ + conversationId, + sessionId, + toolName: turn.kind === "skills" ? "skills.discover" : "pwd", + turn: turn.kind === "pwd" ? 1 : turn.kind === "skills" ? 2 : 3, + reused: turn.kind !== "pwd", + skills: turn.kind === "skills" ? { + status: "ready", + items: [{ + name: "hwlab-agent-runtime", + source: "/workspace/hwlab/skills/hwlab-agent-runtime/SKILL.md" + }], + count: 1 + } : undefined, + reply: turn.kind === "context-pwd" + ? "前文第一轮你问的是 pwd；当前工作目录仍是 /workspace/hwlab。" + : "当前工作目录是 /workspace/hwlab。" + }) + }), + skillCliRunner: async () => ({ + ok: true, + status: 200, + json: skillCliPayload() + }) + }); + + assert.equal(report.status, "pass"); + assert.equal(report.liveMutationUsed, false); + assert.equal(report.safety.liveMutationUsed, false); + assert.equal(report.commanderSummary.allowedLiveMutationUsed, false); + assert.equal(report.commanderSummary.sessionResult.status, "pass"); + assert.equal(report.commanderSummary.skillCliResult.status, "pass"); + assert.equal(report.commanderSummary.m3RouteResult.route, "/v1/m3/io"); + assert.equal(report.commanderSummary.evidenceIds.evidenceId, "evd_rpt004_skill_m3"); + assert.equal(report.dimensions.skillCliM3Route.status, "pass"); + assert.equal(report.dimensions.m3TrustedLoop.observations.skippedWriteLoop, true); + assert.equal(report.dimensions.durableEvidence.observations.notApplicableInDefaultReadOnlyMode, true); +}); + +test("RPT-004 live report emits NOT_CURRENT and skips downstream checks", async () => { + const report = await buildRpt004Report({ + ...parseArgs([ + "--live", + "--url", + "http://74.48.78.17:16666/", + "--api-url", + "http://74.48.78.17:16667/", + "--expected-commit", + fixtureCommit + ]), + writeReport: false + }, { + repoRoot: "/fixture-rpt004", + fsJson: fixtureArtifactJson(), + now: () => "2026-05-23T00:00:00.000Z", + httpGetJson: async (url) => ({ ok: true, status: 200, json: healthPayload(url, { revision: "1111111" }) }), + httpGetText: async () => ({ + ok: true, + status: 200, + contentType: "text/html; charset=utf-8", + body: workbenchHtml() + }), + layoutRunner: async () => { + throw new Error("layout must not run when NOT_CURRENT"); + } + }); + + assert.equal(report.status, "not_current"); + assert.equal(report.conclusion.classification, "NOT_CURRENT"); + assert.equal(report.dimensions.codeAgentSession.status, "not_run"); + assert.equal(report.dimensions.skillCliM3Route.status, "not_run"); + assert.equal(report.liveMutationUsed, false); +}); + test("M3 durable evidence requires operation, audit, evidence, DEV-LIVE green, and persisted writes", () => { const green = classifyM3DurableEvidence(greenOperations()); assert.equal(green.status, "pass"); @@ -212,3 +474,324 @@ function operation(id, action, value) { } }; } + +function sessionTurn(kind, payload, response = { ok: true, status: 200 }) { + return { + kind, + response, + payload + }; +} + +function codexSessionPayload({ + conversationId = "cnv_rpt004_fixture", + sessionId = "ses_rpt004_fixture", + toolName = "pwd", + turn = 1, + reused = false, + reply = "当前工作目录是 /workspace/hwlab。", + skills = { + status: "not_requested", + items: [], + count: 0 + } +} = {}) { + return { + status: "completed", + conversationId, + sessionId, + messageId: `msg_rpt004_${turn}`, + traceId: `trc_rpt004_${turn}`, + provider: "codex-stdio", + model: "gpt-5.5", + backend: "hwlab-cloud-api/codex-mcp-stdio-runner", + workspace: "/workspace/hwlab", + sandbox: "workspace-write", + capabilityLevel: "long-lived-codex-stdio-session", + sessionMode: "codex-mcp-stdio-long-lived", + implementationType: "repo-owned-codex-mcp-stdio-session", + session: { + sessionId, + status: "idle", + idleTimeoutMs: 1800000, + lastTraceId: `trc_rpt004_${turn}`, + turn, + longLivedSession: true, + codexStdio: true, + durable: true, + writeCapable: true + }, + sessionReuse: { + conversationId, + sessionId, + reused, + turn, + status: "idle" + }, + runner: { + kind: "codex-mcp-stdio-runner", + workspace: "/workspace/hwlab", + sandbox: "workspace-write", + session: "codex-mcp-stdio-long-lived", + sessionMode: "codex-mcp-stdio-long-lived", + implementationType: "repo-owned-codex-mcp-stdio-session", + codexStdio: true, + writeCapable: true, + durableSession: true + }, + runnerTrace: { + traceId: `trc_rpt004_${turn}`, + runnerKind: "codex-mcp-stdio-runner", + sessionMode: "codex-mcp-stdio-long-lived", + sessionId, + turn, + sessionReused: reused, + events: [`turn:${turn}`, `tool:${toolName}:completed`] + }, + longLivedSessionGate: { + status: "pass", + pass: true + }, + toolCalls: [{ + name: toolName, + status: "completed", + type: toolName === "skills.discover" ? "file-read" : "shell", + traceId: `trc_rpt004_${turn}` + }], + skills, + reply: { + content: reply + } + }; +} + +function openAiFallbackPayload({ reply = "text fallback" } = {}) { + return { + status: "completed", + conversationId: "cnv_rpt004_fixture", + sessionId: "ses_rpt004_fixture", + messageId: "msg_fallback", + traceId: "trc_fallback", + provider: "openai-responses", + model: "gpt-5.5", + backend: "hwlab-cloud-api/openai-responses", + implementationType: "openai-responses-fallback", + capabilityLevel: "text-chat-only", + sessionMode: "provider-text-request", + runner: { + kind: "openai-responses-fallback" + }, + runnerTrace: { + runnerKind: "openai-responses-fallback" + }, + toolCalls: [], + skills: { + status: "not_requested", + items: [] + }, + reply: { + content: reply + } + }; +} + +function skillCliPayload({ + route = "/v1/m3/io", + accepted = true, + status = "succeeded", + operationId = "op_rpt004_skill_m3", + auditId = "aud_rpt004_skill_m3", + evidenceId = "evd_rpt004_skill_m3", + blocker = null, + directGatewayCalls = false, + directBoxCalls = false, + directPatchPanelCalls = false +} = {}) { + return { + status: "completed", + conversationId: "cnv_rpt004_skill", + sessionId: "ses_rpt004_skill", + messageId: "msg_rpt004_skill", + traceId: "trc_rpt004_skill", + provider: "hwlab-skill-cli", + model: "controlled-m3-io", + backend: "hwlab-cloud-api/hwlab-agent-runtime-skill-cli", + workspace: "/workspace/hwlab", + sandbox: "hwlab-api-route-only", + capabilityLevel: accepted ? "hwlab-api-control-ready" : "hwlab-api-control-blocked", + sessionMode: "controlled-m3-io-skill-cli", + implementationType: "skill-cli-hwlab-api-adapter", + runner: { + kind: "hwlab-m3-io-skill-cli", + safety: { + directGatewayCallsAllowed: false, + directBoxSimuCallsAllowed: false, + directPatchPanelCallsAllowed: false + } + }, + runnerTrace: { + traceId: "trc_rpt004_skill", + runnerKind: "hwlab-m3-io-skill-cli", + route, + fallbackUsed: false, + directGatewayCalls, + directBoxCalls, + directPatchPanelCalls + }, + providerTrace: { + runnerKind: "hwlab-m3-io-skill-cli", + route, + traceId: "trc_rpt004_skill", + operationId, + accepted, + controlReady: accepted, + fallbackUsed: false + }, + toolCalls: [{ + name: "hwlab-agent-runtime.m3-io", + status: accepted ? "completed" : "blocked", + type: "skill-cli", + route, + accepted, + controlReady: accepted, + capabilityLevel: accepted ? "hwlab-api-control-ready" : "hwlab-api-control-blocked", + operationId, + traceId: "trc_rpt004_skill", + audit: { + auditId, + status + }, + evidence: { + evidenceId, + status: accepted ? "blocked" : "blocked", + sourceKind: accepted ? "BLOCKED" : "BLOCKED", + blocker: blocker?.code ?? "runtime_durable_not_green", + writeStatus: "written_non_durable" + }, + durable: { + status: "degraded", + blocker: blocker?.code ?? "runtime_durable_not_green" + }, + blocker, + blockers: blocker ? [blocker] : [{ + code: "runtime_durable_not_green", + message: "durable trust is not green" + }], + directGatewayCalls, + directBoxCalls, + directPatchPanelCalls, + fallbackUsed: false + }], + skills: { + status: "used", + items: [{ + name: "hwlab-agent-runtime.m3-io", + route + }], + blockers: blocker ? [blocker] : [] + }, + reply: { + content: "M3 IO Skill CLI result" + } + }; +} + +function healthPayload(url, { revision = fixtureCommit } = {}) { + const isWeb = String(url).includes(":16666"); + return { + serviceId: isWeb ? "hwlab-cloud-web" : "hwlab-cloud-api", + environment: "dev", + status: "ready", + ready: true, + commit: { + id: revision + }, + image: { + tag: revision.slice(0, 7), + digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + runtime: { + durable: true, + ready: true, + status: "ready", + blocker: null, + liveRuntimeEvidence: true + }, + readiness: { + ready: true, + status: "ready", + durability: { + ready: true, + status: "ready", + blocker: null + } + } + }; +} + +function workbenchHtml() { + return ` + + + +
+
+

HWLAB 云工作台

+ +
Agent 对话 工作清单 使用说明
+ +
+
+ +
+ +`; +} + +function fixtureArtifactJson() { + const serviceIds = [ + "hwlab-cloud-api", + "hwlab-cloud-web", + "hwlab-agent-mgr", + "hwlab-agent-worker", + "hwlab-gateway", + "hwlab-gateway-simu", + "hwlab-box-simu", + "hwlab-patch-panel", + "hwlab-router", + "hwlab-tunnel-client", + "hwlab-edge-proxy", + "hwlab-cli", + "hwlab-agent-skills" + ]; + const deploy = { + commitId: fixtureCommit + }; + const catalog = { + commitId: fixtureCommit, + artifactState: "published", + publish: { + ciPublished: true, + registryVerified: true + }, + services: serviceIds.map((serviceId) => ({ + serviceId, + commitId: fixtureCommit, + imageTag: fixtureCommit.slice(0, 7), + digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + publishState: "published" + })) + }; + const artifactReport = { + commitId: fixtureCommit, + artifactPublish: { + sourceCommitId: fixtureCommit + } + }; + return async (relativePath) => { + if (relativePath === "deploy/deploy.json") return deploy; + if (relativePath === "deploy/artifact-catalog.dev.json") return catalog; + if (relativePath === "reports/dev-gate/dev-artifacts.json") return artifactReport; + return null; + }; +} diff --git a/scripts/validate-dev-gate-report.mjs b/scripts/validate-dev-gate-report.mjs index 6c446693..f30ed292 100644 --- a/scripts/validate-dev-gate-report.mjs +++ b/scripts/validate-dev-gate-report.mjs @@ -791,10 +791,12 @@ async function validateRpt004MvpE2eReport(report, label) { "mode", "status", "conclusion", + "commanderSummary", "sourceContract", "validationCommands", "devPreconditions", "target", + "runPolicy", "expected", "dimensions", "checks", @@ -815,6 +817,9 @@ async function validateRpt004MvpE2eReport(report, label) { assert.ok(["pass", "blocked", "not_current"].includes(report.status), `${label}.status`); assertObject(report.conclusion, `${label}.conclusion`); + assertObject(report.commanderSummary, `${label}.commanderSummary`); + assert.equal(report.commanderSummary.reportClass, "live-mutating", `${label}.commanderSummary.reportClass`); + assert.equal(typeof report.commanderSummary.allowedLiveMutationUsed, "boolean", `${label}.commanderSummary.allowedLiveMutationUsed`); assert.ok( ["CHECK_ONLY", "BLOCKED", "NOT_CURRENT", "MVP_PASS"].includes(report.conclusion.classification), `${label}.conclusion.classification` @@ -859,7 +864,9 @@ async function validateRpt004MvpE2eReport(report, label) { "layout", "m3TrustedLoop", "durableEvidence", - "codeAgentSkillPath" + "codeAgentSkillPath", + "codeAgentSession", + "skillCliM3Route" ]; assertObject(report.dimensions, `${label}.dimensions`); for (const id of requiredDimensions) { @@ -881,6 +888,8 @@ async function validateRpt004MvpE2eReport(report, label) { "16666-default-chinese-workbench", "layout-smoke-hit-target", "code-agent-skill-path", + "code-agent-session-thread", + "skill-cli-hwlab-api-m3-route", "m3-v1-io-trusted-loop", "operation-audit-evidence-durable-green" ]) { @@ -902,6 +911,8 @@ async function validateRpt004MvpE2eReport(report, label) { assert.equal(report.safety.rolloutAttempted, false, `${label}.safety.rolloutAttempted`); assert.equal(report.safety.secretValuesPrinted, false, `${label}.safety.secretValuesPrinted`); assert.equal(report.safety.openAiFallbackCountsAsPass, false, `${label}.safety.openAiFallbackCountsAsPass`); + assert.equal(report.safety.liveM3WritesRequireExplicitFlag, true, `${label}.safety.liveM3WritesRequireExplicitFlag`); + assert.equal(report.liveMutationUsed, report.runPolicy.allowM3Write === true, `${label}.liveMutationUsed`); if (report.artifacts?.artifactReport) { assert.match(report.artifacts.artifactReport, /not applicable/u, `${label}.artifacts.artifactReport`);