lyon
1813 lines
70 KiB
JavaScript
1813 lines
70 KiB
JavaScript
import { createHash, randomUUID } from "node:crypto";
|
|
import { execFileSync } from "node:child_process";
|
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
import { activeReportLifecycle } from "../../internal/dev-report-lifecycle.mjs";
|
|
import {
|
|
DEV_ENDPOINT,
|
|
DEV_FRONTEND_ENDPOINT,
|
|
ENVIRONMENT_DEV,
|
|
SERVICE_IDS
|
|
} from "../../internal/protocol/index.mjs";
|
|
import {
|
|
M3_IO_CHAIN,
|
|
M3_IO_CONTROL_ROUTE
|
|
} from "../../internal/cloud/m3-io-control.mjs";
|
|
import {
|
|
buildM3IoControlSourceReport,
|
|
runM3IoControlLiveReport
|
|
} from "./m3-io-control-e2e.mjs";
|
|
import { runDevCloudWorkbenchLayoutSmoke } from "./dev-cloud-workbench-smoke-lib.mjs";
|
|
import { classifyCodexRunnerCapability } from "./code-agent-response-contract.mjs";
|
|
import {
|
|
HWLAB_AGENT_RUNTIME_SKILL_CLI_VERSION,
|
|
HWLAB_M3_IO_API_ROUTE,
|
|
HWLAB_M3_IO_SKILL_NAME
|
|
} from "../../skills/hwlab-agent-runtime/scripts/src/m3-io-skill-client.mjs";
|
|
import { ensureNotRepoReportsPath, tempReportPath } from "./report-paths.mjs";
|
|
|
|
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
|
const defaultJsonReportPath = tempReportPath("rpt-004-mvp-e2e.json");
|
|
const defaultMarkdownReportPath = tempReportPath("rpt-004-mvp-e2e.md");
|
|
const defaultArtifactReportPath = tempReportPath("dev-artifacts.json");
|
|
const issue = "pikasTech/HWLAB#316";
|
|
const taskId = "rpt004-mvp-e2e-harness";
|
|
const reportKind = "rpt-004-mvp-e2e-harness";
|
|
const requiredWorkbenchLabels = Object.freeze([
|
|
"HWLAB 云工作台",
|
|
"硬件资源",
|
|
"Agent 对话",
|
|
"可信记录",
|
|
"内部复核",
|
|
"使用说明"
|
|
]);
|
|
const workbenchMarkers = Object.freeze([
|
|
"data-app-shell",
|
|
"workbench-shell",
|
|
"resource-tree",
|
|
"center-workspace",
|
|
"right-sidebar",
|
|
"command-form"
|
|
]);
|
|
const expectedM3OperationIds = Object.freeze([
|
|
["write-do1-true", "do.write", true],
|
|
["read-di1-true", "di.read", true],
|
|
["write-do1-false", "do.write", false],
|
|
["read-di1-false", "di.read", false]
|
|
]);
|
|
const codeAgentSessionTurns = Object.freeze([
|
|
{
|
|
kind: "pwd",
|
|
message: "请用 pwd 列出你当前的工作目录，并保留本轮 session 证据。"
|
|
},
|
|
{
|
|
kind: "skills",
|
|
message: "请在同一个 conversation/session 中列出你能使用的所有 skills。"
|
|
},
|
|
{
|
|
kind: "context-pwd",
|
|
message: "请引用前文：第一轮我问的是 pwd。再次用 pwd 确认当前工作目录。"
|
|
}
|
|
]);
|
|
const m3ReadOnlySkillMessage = "请通过 M3 DI1 read 读取数字输入；使用 HWLAB API route /v1/m3/io。";
|
|
const forbiddenDirectHardwareTarget = /(?:gateway(?:-simu)?|box(?:-simu)?|patch-panel|hwlab-patch-panel|\/invoke\b|\/sync\/tick\b|:7101\b|:7201\b|:7301\b)/iu;
|
|
const statusRank = Object.freeze({
|
|
pass: 0,
|
|
not_run: 1,
|
|
blocked: 2,
|
|
degraded: 2,
|
|
not_current: 3
|
|
});
|
|
|
|
export async function runRpt004Cli(argv = [], options = {}) {
|
|
const args = parseArgs(argv);
|
|
if (args.help) {
|
|
process.stdout.write(`${usage()}\n`);
|
|
return 0;
|
|
}
|
|
|
|
const report = await buildRpt004Report(args, options);
|
|
if (args.writeReport) {
|
|
await writeReports(report, args, options.repoRoot ?? repoRoot);
|
|
}
|
|
|
|
const printable = args.pretty || args.writeReport ? report : compactReport(report);
|
|
process.stdout.write(`${JSON.stringify(printable, null, 2)}\n`);
|
|
return report.status === "pass" || args.live !== true ? 0 : 2;
|
|
}
|
|
|
|
export async function buildRpt004Report(args = {}, options = {}) {
|
|
const parsed = normalizeArgs(args);
|
|
const now = options.now ?? (() => new Date().toISOString());
|
|
const root = options.repoRoot ?? repoRoot;
|
|
const report = baseReport({ parsed, now, root });
|
|
|
|
const expected = await buildExpectedArtifactState({
|
|
root,
|
|
expectedCommit: parsed.expectedCommit,
|
|
fsJson: options.fsJson
|
|
});
|
|
report.expected = expected;
|
|
report.dimensions.artifact = expected.check;
|
|
addCheck(report, expected.check);
|
|
|
|
if (!parsed.live) {
|
|
addCheck(report, sourceHarnessCheck(root));
|
|
addNotRunDimensions(report, "RPT-004 source check did not touch live DEV endpoints.");
|
|
return finalizeReport(report);
|
|
}
|
|
|
|
validateLiveBoundary(parsed);
|
|
const httpGetJson = options.httpGetJson ?? getJson;
|
|
const httpGetText = options.httpGetText ?? getText;
|
|
const apiHealthUrl = joinBasePath(parsed.apiUrl, "/health/live");
|
|
const webHealthUrl = joinBasePath(parsed.url, "/health/live");
|
|
const [apiHealth, webHealth, html] = await Promise.all([
|
|
httpGetJson(apiHealthUrl, parsed.timeoutMs),
|
|
httpGetJson(webHealthUrl, parsed.timeoutMs),
|
|
httpGetText(parsed.url, parsed.timeoutMs)
|
|
]);
|
|
|
|
report.live.apiHealth = summarizeHealthResponse("hwlab-cloud-api", apiHealthUrl, apiHealth);
|
|
report.live.webHealth = summarizeHealthResponse("hwlab-cloud-web", webHealthUrl, webHealth);
|
|
report.live.indexHtml = summarizeHtmlResponse(parsed.url, html);
|
|
|
|
const revisionCheck = classifyLiveRevision({
|
|
expected,
|
|
apiHealth: report.live.apiHealth,
|
|
webHealth: report.live.webHealth
|
|
});
|
|
const healthCheck = classifyLiveHealth(report.live.apiHealth, report.live.webHealth);
|
|
const workbenchCheck = classifyDefaultChineseWorkbench(report.live.indexHtml);
|
|
report.dimensions.revision = revisionCheck;
|
|
report.dimensions.health = healthCheck;
|
|
report.dimensions.workbench = workbenchCheck;
|
|
addCheck(report, revisionCheck);
|
|
addCheck(report, healthCheck);
|
|
addCheck(report, workbenchCheck);
|
|
|
|
const currentAndHealthy = revisionCheck.status === "pass" && healthCheck.status === "pass";
|
|
if (!currentAndHealthy) {
|
|
const reason = revisionCheck.status !== "pass"
|
|
? "live revision is not current"
|
|
: "live health is degraded or blocked";
|
|
addNotRunDimensions(report, `RPT-004 stopped before downstream smoke because ${reason}.`);
|
|
return finalizeReport(report);
|
|
}
|
|
|
|
const layoutCheck = await runLayoutDimension(parsed, options);
|
|
report.dimensions.layout = layoutCheck;
|
|
addCheck(report, layoutCheck);
|
|
|
|
const codeAgentReport = await runCodeAgentSessionDimension(parsed, options);
|
|
report.live.codeAgent = codeAgentReport.summary;
|
|
report.dimensions.codeAgentSession = codeAgentReport.sessionCheck;
|
|
report.dimensions.codeAgentSkillPath = codeAgentReport.skillPathCheck;
|
|
addCheck(report, codeAgentReport.sessionCheck);
|
|
addCheck(report, codeAgentReport.skillPathCheck);
|
|
|
|
const skillCliRouteCheck = await runSkillCliM3RouteDimension(parsed, options);
|
|
report.live.skillCliM3Route = skillCliRouteCheck.observations;
|
|
report.dimensions.skillCliM3Route = skillCliRouteCheck;
|
|
addCheck(report, skillCliRouteCheck);
|
|
|
|
if (parsed.allowM3Write) {
|
|
const m3Report = await runM3TrustedLoop(parsed, options);
|
|
report.live.m3 = m3Report.summary;
|
|
report.dimensions.m3TrustedLoop = m3Report.loopCheck;
|
|
report.dimensions.durableEvidence = m3Report.durableCheck;
|
|
addCheck(report, m3Report.loopCheck);
|
|
addCheck(report, m3Report.durableCheck);
|
|
} else {
|
|
addReadOnlyM3Dimensions(report);
|
|
}
|
|
|
|
return finalizeReport(report);
|
|
}
|
|
|
|
export function parseArgs(argv = []) {
|
|
const args = {
|
|
live: false,
|
|
url: `${DEV_FRONTEND_ENDPOINT}/`,
|
|
apiUrl: `${DEV_ENDPOINT}/`,
|
|
expectedCommit: null,
|
|
reportPath: defaultJsonReportPath,
|
|
markdownPath: defaultMarkdownReportPath,
|
|
writeReport: false,
|
|
pretty: false,
|
|
timeoutMs: 10000,
|
|
allowM3Write: false,
|
|
help: false
|
|
};
|
|
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const arg = argv[index];
|
|
if (arg === "--check") args.live = false;
|
|
else if (arg === "--live") args.live = true;
|
|
else if (arg === "--url") args.url = readArg(argv, ++index, arg);
|
|
else if (arg === "--api-url") args.apiUrl = readArg(argv, ++index, arg);
|
|
else if (arg === "--expected-commit") args.expectedCommit = readArg(argv, ++index, arg);
|
|
else if (arg === "--report") {
|
|
args.reportPath = readArg(argv, ++index, arg);
|
|
args.writeReport = true;
|
|
} else if (arg === "--markdown") {
|
|
args.markdownPath = readArg(argv, ++index, arg);
|
|
args.writeReport = true;
|
|
} else if (arg === "--no-write") {
|
|
args.writeReport = false;
|
|
} else if (arg === "--pretty") {
|
|
args.pretty = true;
|
|
} else if (arg === "--timeout-ms") {
|
|
args.timeoutMs = parseTimeout(readArg(argv, ++index, arg));
|
|
} else if (arg === "--allow-m3-write") {
|
|
args.allowM3Write = true;
|
|
} else if (arg === "--help" || arg === "-h") {
|
|
args.help = true;
|
|
} else {
|
|
throw new Error(`unknown argument: ${arg}`);
|
|
}
|
|
}
|
|
|
|
if (args.live && !argv.includes("--no-write")) {
|
|
args.writeReport = true;
|
|
}
|
|
args.expectedCommit = args.expectedCommit ? sanitizeCommit(args.expectedCommit) : null;
|
|
if (args.expectedCommit === "unknown") {
|
|
throw new Error("--expected-commit must be a 7-40 character Git SHA");
|
|
}
|
|
return args;
|
|
}
|
|
|
|
export function classifyRpt004Conclusion(checks = [], options = {}) {
|
|
const byId = new Map(checks.map((check) => [check.id, check]));
|
|
const revision = byId.get("live-revision-current");
|
|
if (revision?.status === "not_current") {
|
|
return {
|
|
status: "not_current",
|
|
classification: "NOT_CURRENT",
|
|
mvpPass: false,
|
|
summary: "RPT-004 is NOT_CURRENT: live 16666/16667 revision does not match the expected DEV artifact commit."
|
|
};
|
|
}
|
|
|
|
const worst = worstStatus(checks.map((check) => check.status));
|
|
if (worst === "pass") {
|
|
return {
|
|
status: "pass",
|
|
classification: "MVP_PASS",
|
|
mvpPass: true,
|
|
summary: options.allowM3Write === true
|
|
? "RPT-004 MVP PASS: current live revision, default Chinese workbench, layout smoke, Code Agent session, Skill CLI HWLAB API route, M3 trusted IO loop, and durable evidence are green."
|
|
: "RPT-004 MVP PASS: current live revision, default Chinese workbench, layout smoke, Code Agent session, Skill CLI HWLAB API route, and default read-only M3 route verification are green; no M3 write was attempted."
|
|
};
|
|
}
|
|
|
|
const sourceOnly = checks.some((check) => check.id === "rpt004-source-contract") &&
|
|
checks.every((check) => ["pass", "not_run"].includes(check.status));
|
|
if (sourceOnly) {
|
|
return {
|
|
status: "pass",
|
|
classification: "CHECK_ONLY",
|
|
mvpPass: false,
|
|
summary: "RPT-004 harness source check passed; no live DEV smoke ran, so this is not MVP PASS."
|
|
};
|
|
}
|
|
|
|
return {
|
|
status: "blocked",
|
|
classification: "BLOCKED",
|
|
mvpPass: false,
|
|
summary: "RPT-004 is BLOCKED: at least one required live dimension is blocked, degraded, skipped, or not run."
|
|
};
|
|
}
|
|
|
|
export function classifyCodeAgentSkillPath(payload, { httpStatus = null, httpOk = false } = {}) {
|
|
const capability = classifyCodexRunnerCapability(payload, { httpStatus });
|
|
const toolCalls = Array.isArray(payload?.toolCalls) ? payload.toolCalls : [];
|
|
const skillItems = Array.isArray(payload?.skills?.items) ? payload.skills.items : [];
|
|
const openAiFallback =
|
|
payload?.provider === "openai-responses" ||
|
|
payload?.runner?.kind === "openai-responses-fallback" ||
|
|
payload?.implementationType === "openai-responses-fallback";
|
|
const skillsDiscoverCompleted = toolCalls.some((tool) =>
|
|
tool?.name === "skills.discover" && tool?.status === "completed"
|
|
);
|
|
const repoSkillReady = skillItems.some((skill) =>
|
|
skill?.name === "hwlab-agent-runtime" ||
|
|
skill?.name === HWLAB_M3_IO_SKILL_NAME ||
|
|
String(skill?.source ?? "").includes("skills/hwlab-agent-runtime/SKILL.md")
|
|
);
|
|
const runnerKind = payload?.runner?.kind ?? null;
|
|
const controlledSessionRunner =
|
|
runnerKind === "codex-app-server-stdio-runner" ||
|
|
runnerKind === "hwlab-readonly-runner";
|
|
const missing = [];
|
|
if (!httpOk) missing.push(`HTTP ${httpStatus ?? "not_observed"}`);
|
|
if (capability.status !== "pass" && !controlledSessionRunner) {
|
|
missing.push(capability.blocker ?? "runner-capability");
|
|
}
|
|
if (openAiFallback) missing.push("openai-fallback");
|
|
if (!skillsDiscoverCompleted) missing.push("skills.discover toolCall");
|
|
if (payload?.skills?.status !== "ready") missing.push("skills.status=ready");
|
|
if (!repoSkillReady) missing.push("repo-owned hwlab-agent-runtime skill");
|
|
if (!payload?.runnerTrace) missing.push("runnerTrace");
|
|
if (!payload?.sessionMode) missing.push("sessionMode");
|
|
if (!payload?.capabilityLevel) missing.push("capabilityLevel");
|
|
if (!controlledSessionRunner) missing.push("controlled Code Agent runner");
|
|
|
|
return {
|
|
id: "code-agent-skill-path",
|
|
status: missing.length === 0 ? "pass" : "blocked",
|
|
summary: missing.length === 0
|
|
? "Code Agent skill discovery used the repo-owned Codex stdio runner with completed toolCalls and hwlab-agent-runtime skill evidence."
|
|
: `Code Agent skill path is blocked: missing ${missing.join(", ")}.`,
|
|
evidenceLevel: missing.length === 0 ? "DEV-LIVE" : "BLOCKED",
|
|
observations: {
|
|
provider: payload?.provider ?? null,
|
|
model: payload?.model ?? null,
|
|
backend: payload?.backend ?? null,
|
|
runnerKind,
|
|
capability,
|
|
controlledSessionRunner,
|
|
openAiFallback,
|
|
toolCalls: toolCalls.map((tool) => ({
|
|
name: tool?.name ?? null,
|
|
status: tool?.status ?? null,
|
|
type: tool?.type ?? null
|
|
})),
|
|
skills: {
|
|
status: payload?.skills?.status ?? null,
|
|
count: payload?.skills?.count ?? (Array.isArray(payload?.skills?.items) ? payload.skills.items.length : null),
|
|
repoSkillReady
|
|
},
|
|
assistantReplyStored: false
|
|
}
|
|
};
|
|
}
|
|
|
|
export function classifyCodeAgentSessionTurns(turns = [], { expectedConversationId = null } = {}) {
|
|
const normalizedTurns = Array.isArray(turns) ? turns : [];
|
|
const blockers = [];
|
|
if (normalizedTurns.length < 3) blockers.push("three-turn-session");
|
|
const completedTurns = normalizedTurns.filter((turn) => turn?.payload?.status === "completed" && turn?.response?.ok === true);
|
|
if (completedTurns.length !== normalizedTurns.length || normalizedTurns.length === 0) {
|
|
blockers.push("completed-http-turns");
|
|
}
|
|
|
|
const conversationIds = new Set();
|
|
const sessionIds = new Set();
|
|
const turnSummaries = normalizedTurns.map((turn, index) => {
|
|
const payload = turn?.payload ?? {};
|
|
const response = turn?.response ?? {};
|
|
const capability = classifyCodexRunnerCapability(payload, {
|
|
httpStatus: response.status ?? null
|
|
});
|
|
const toolCalls = Array.isArray(payload.toolCalls) ? payload.toolCalls : [];
|
|
const toolNames = toolCalls.map((tool) => tool?.name).filter(Boolean);
|
|
const runnerKind = payload.runner?.kind ?? null;
|
|
const sessionMode = payload.sessionMode ?? payload.runner?.sessionMode ?? null;
|
|
const sessionId = payload.sessionId ?? payload.session?.sessionId ?? null;
|
|
const conversationId = payload.conversationId ?? null;
|
|
const traceId = payload.traceId ?? null;
|
|
const runnerTrace = payload.runnerTrace && typeof payload.runnerTrace === "object" ? payload.runnerTrace : null;
|
|
const fallback = isOpenAiFallbackPayload(payload);
|
|
const partialRunner = payload.provider === "codex-readonly-runner" || runnerKind === "hwlab-readonly-runner";
|
|
const expectedTool = turn?.kind === "skills"
|
|
? "skills.discover"
|
|
: "pwd";
|
|
const hasExpectedTool = toolNames.includes(expectedTool);
|
|
const toolCallsCompleted = toolCalls.length > 0 && toolCalls.every((tool) => tool?.status === "completed");
|
|
const reply = typeof payload.reply?.content === "string" ? payload.reply.content : "";
|
|
const referencesPrevious = turn?.kind === "context-pwd"
|
|
? /pwd|第一轮|前文|same session|同一/u.test(reply)
|
|
: true;
|
|
const ready = response.ok === true &&
|
|
payload.status === "completed" &&
|
|
Boolean(conversationId) &&
|
|
Boolean(sessionId) &&
|
|
Boolean(traceId) &&
|
|
Boolean(runnerKind) &&
|
|
Boolean(sessionMode) &&
|
|
Boolean(payload.capabilityLevel) &&
|
|
Boolean(runnerTrace) &&
|
|
hasExpectedTool &&
|
|
toolCallsCompleted &&
|
|
!fallback;
|
|
|
|
if (conversationId) conversationIds.add(conversationId);
|
|
if (sessionId) sessionIds.add(sessionId);
|
|
return {
|
|
index: index + 1,
|
|
kind: turn?.kind ?? "unknown",
|
|
status: ready && referencesPrevious ? "pass" : "blocked",
|
|
httpStatus: response.status ?? null,
|
|
conversationId,
|
|
sessionId,
|
|
traceId,
|
|
provider: payload.provider ?? null,
|
|
backend: payload.backend ?? null,
|
|
runnerKind,
|
|
sessionMode,
|
|
capabilityLevel: payload.capabilityLevel ?? null,
|
|
capability,
|
|
toolNames,
|
|
toolCallsCompleted,
|
|
hasExpectedTool,
|
|
runnerTracePresent: Boolean(runnerTrace),
|
|
fallback,
|
|
partialRunner,
|
|
turn: payload.sessionReuse?.turn ?? payload.session?.turn ?? payload.runnerTrace?.turn ?? null,
|
|
reused: payload.sessionReuse?.reused ?? payload.session?.reused ?? payload.runnerTrace?.sessionReused ?? null,
|
|
referencesPrevious,
|
|
blockers: [
|
|
...(response.ok === true ? [] : [`HTTP ${response.status ?? "not_observed"}`]),
|
|
...(payload.status === "completed" ? [] : ["payload.status=completed"]),
|
|
...(conversationId ? [] : ["conversationId"]),
|
|
...(sessionId ? [] : ["sessionId"]),
|
|
...(traceId ? [] : ["traceId"]),
|
|
...(runnerKind ? [] : ["runner.kind"]),
|
|
...(sessionMode ? [] : ["sessionMode"]),
|
|
...(payload.capabilityLevel ? [] : ["capabilityLevel"]),
|
|
...(runnerTrace ? [] : ["runnerTrace"]),
|
|
...(hasExpectedTool ? [] : [`toolCall ${expectedTool}`]),
|
|
...(toolCallsCompleted ? [] : ["toolCalls completed"]),
|
|
...(fallback ? ["openai-fallback"] : []),
|
|
...(referencesPrevious ? [] : ["context-reference"])
|
|
]
|
|
};
|
|
});
|
|
|
|
if (expectedConversationId && !conversationIds.has(expectedConversationId)) {
|
|
blockers.push("expected-conversationId");
|
|
}
|
|
if (conversationIds.size !== 1) blockers.push("same-conversationId");
|
|
if (sessionIds.size !== 1) blockers.push("same-sessionId");
|
|
if (turnSummaries.some((turn) => turn.fallback)) blockers.push("fallback-only");
|
|
if (turnSummaries.some((turn) => turn.runnerTracePresent !== true)) blockers.push("runnerTrace");
|
|
if (turnSummaries.some((turn) => turn.toolCallsCompleted !== true)) blockers.push("toolCalls");
|
|
if (turnSummaries.some((turn) => turn.hasExpectedTool !== true)) blockers.push("expected-toolCalls");
|
|
if (turnSummaries.some((turn) => !turn.sessionMode || !turn.capabilityLevel || !turn.runnerKind)) {
|
|
blockers.push("runner/session/capability evidence");
|
|
}
|
|
if (turnSummaries.some((turn) => turn.kind === "context-pwd" && turn.referencesPrevious !== true)) {
|
|
blockers.push("context-reference");
|
|
}
|
|
|
|
const pass = blockers.length === 0 && turnSummaries.every((turn) => turn.status === "pass");
|
|
return {
|
|
id: "code-agent-session-thread",
|
|
status: pass ? "pass" : "blocked",
|
|
summary: pass
|
|
? "Code Agent completed three same-conversation/session turns for pwd, skills, and prior-context reference with runner/tool/session evidence."
|
|
: `Code Agent session thread is blocked: missing ${[...new Set(blockers)].join(", ")}.`,
|
|
evidenceLevel: pass ? "DEV-LIVE" : "BLOCKED",
|
|
observations: {
|
|
expectedConversationId,
|
|
conversationIds: [...conversationIds],
|
|
sessionIds: [...sessionIds],
|
|
turnCount: normalizedTurns.length,
|
|
turns: turnSummaries,
|
|
fallbackOnly: turnSummaries.length > 0 && turnSummaries.every((turn) => turn.fallback),
|
|
partialRunnerOnly: turnSummaries.length > 0 && turnSummaries.every((turn) => turn.partialRunner),
|
|
blockers: [...new Set(blockers)]
|
|
}
|
|
};
|
|
}
|
|
|
|
export function classifySkillCliM3Route(payload, { httpStatus = null, httpOk = false } = {}) {
|
|
const toolCalls = Array.isArray(payload?.toolCalls) ? payload.toolCalls : [];
|
|
const skillTool = toolCalls.find((tool) =>
|
|
tool?.name === HWLAB_M3_IO_SKILL_NAME ||
|
|
tool?.route === HWLAB_M3_IO_API_ROUTE ||
|
|
tool?.route === M3_IO_CONTROL_ROUTE
|
|
) ?? null;
|
|
const providerTrace = payload?.providerTrace && typeof payload.providerTrace === "object" ? payload.providerTrace : null;
|
|
const route = skillTool?.route ?? providerTrace?.route ?? payload?.skills?.items?.find((item) => item?.route)?.route ?? null;
|
|
const method = skillTool?.method ?? providerTrace?.method ?? null;
|
|
const operationId = skillTool?.operationId ?? providerTrace?.operationId ?? null;
|
|
const traceId = skillTool?.traceId ?? providerTrace?.traceId ?? payload?.traceId ?? null;
|
|
const auditId = skillTool?.auditId ?? skillTool?.audit?.auditId ?? providerTrace?.auditId ?? null;
|
|
const evidenceId = skillTool?.evidenceId ?? skillTool?.evidence?.evidenceId ?? providerTrace?.evidenceId ?? null;
|
|
const readback = skillTool?.readback ?? providerTrace?.readback ?? skillTool?.result?.targetReadback ?? null;
|
|
const blockers = normalizeM3RouteBlockers(payload, skillTool);
|
|
const safety = {
|
|
directGatewayCalls: Boolean(skillTool?.directGatewayCalls ?? payload?.runnerTrace?.directGatewayCalls ?? false),
|
|
directBoxCalls: Boolean(skillTool?.directBoxCalls ?? payload?.runnerTrace?.directBoxCalls ?? false),
|
|
directPatchPanelCalls: Boolean(skillTool?.directPatchPanelCalls ?? payload?.runnerTrace?.directPatchPanelCalls ?? false),
|
|
fallbackUsed: Boolean(skillTool?.fallbackUsed ?? payload?.providerTrace?.fallbackUsed ?? payload?.runnerTrace?.fallbackUsed ?? isOpenAiFallbackPayload(payload)),
|
|
routeOnly: route === HWLAB_M3_IO_API_ROUTE || route === M3_IO_CONTROL_ROUTE,
|
|
directTargetMentioned: directHardwareTargetMentioned(payload)
|
|
};
|
|
const structuredBlocker = blockers.length > 0;
|
|
const idsPresent = Boolean(operationId && traceId && auditId && evidenceId);
|
|
const accepted = skillTool?.accepted === true || providerTrace?.accepted === true;
|
|
const controlReady = skillTool?.controlReady === true || providerTrace?.controlReady === true;
|
|
const routeEvidenceReady = route === HWLAB_M3_IO_API_ROUTE || route === M3_IO_CONTROL_ROUTE;
|
|
const routePass = httpOk === true &&
|
|
payload?.status === "completed" &&
|
|
payload?.provider === "legacy-skill-cli" &&
|
|
skillTool &&
|
|
["completed", "blocked"].includes(skillTool.status) &&
|
|
routeEvidenceReady &&
|
|
method === "POST" &&
|
|
safety.directGatewayCalls === false &&
|
|
safety.directBoxCalls === false &&
|
|
safety.directPatchPanelCalls === false &&
|
|
safety.fallbackUsed === false &&
|
|
safety.directTargetMentioned === false &&
|
|
(idsPresent || structuredBlocker);
|
|
|
|
const missing = [];
|
|
if (!httpOk) missing.push(`HTTP ${httpStatus ?? "not_observed"}`);
|
|
if (payload?.status !== "completed") missing.push("payload.status=completed");
|
|
if (payload?.provider !== "legacy-skill-cli") missing.push("provider=legacy-skill-cli");
|
|
if (!skillTool) missing.push(`${HWLAB_M3_IO_SKILL_NAME} toolCall`);
|
|
if (!routeEvidenceReady) missing.push(`route=${HWLAB_M3_IO_API_ROUTE}`);
|
|
if (method !== "POST") missing.push("method=POST");
|
|
if (safety.directGatewayCalls || safety.directBoxCalls || safety.directPatchPanelCalls || safety.directTargetMentioned) {
|
|
missing.push("no direct gateway/box/patch-panel target");
|
|
}
|
|
if (safety.fallbackUsed) missing.push("no fallback");
|
|
if (!idsPresent && !structuredBlocker) missing.push("operationId/traceId/auditId/evidenceId or structured blocker");
|
|
|
|
return {
|
|
id: "skill-cli-hwlab-api-m3-route",
|
|
status: routePass ? "pass" : "blocked",
|
|
summary: routePass
|
|
? structuredBlocker
|
|
? `Skill CLI reached ${HWLAB_M3_IO_API_ROUTE} and returned a structured blocker instead of direct hardware/fallback evidence.`
|
|
: `Skill CLI used HWLAB API ${HWLAB_M3_IO_API_ROUTE} with operation/audit/evidence identifiers and no direct hardware/fallback path.`
|
|
: `Skill CLI HWLAB API route check is blocked: missing ${missing.join(", ")}.`,
|
|
evidenceLevel: routePass ? "DEV-LIVE" : "BLOCKED",
|
|
observations: {
|
|
httpStatus,
|
|
provider: payload?.provider ?? null,
|
|
backend: payload?.backend ?? null,
|
|
route,
|
|
expectedRoute: HWLAB_M3_IO_API_ROUTE,
|
|
method,
|
|
toolCallStatus: skillTool?.status ?? null,
|
|
accepted,
|
|
controlReady,
|
|
capabilityLevel: skillTool?.capabilityLevel ?? payload?.capabilityLevel ?? null,
|
|
operationId,
|
|
traceId,
|
|
auditId,
|
|
evidenceId,
|
|
readback,
|
|
idsPresent,
|
|
structuredBlocker,
|
|
blockers,
|
|
safety,
|
|
missing
|
|
}
|
|
};
|
|
}
|
|
|
|
export function classifyM3DurableEvidence(operations = []) {
|
|
const byId = new Map(operations.map((operation) => [operation.id, operation]));
|
|
const sequence = expectedM3OperationIds.map(([id, action, expectedValue]) => {
|
|
const operation = byId.get(id) ?? {};
|
|
const evidenceState = operation.evidenceState ?? {};
|
|
return {
|
|
id,
|
|
action,
|
|
expectedValue,
|
|
observedValue: Object.hasOwn(operation, "resultValue") ? operation.resultValue : null,
|
|
status: operation.status ?? "missing",
|
|
idsPresent: Boolean(operation.operationId && operation.traceId && operation.auditId && operation.evidenceId),
|
|
valueMatches: operation.resultValue === expectedValue,
|
|
durableGreen: evidenceState.status === "green" &&
|
|
evidenceState.sourceKind === "DEV-LIVE" &&
|
|
evidenceState.durable === true,
|
|
persisted: evidenceState.writeStatus === "persisted"
|
|
};
|
|
});
|
|
const ready = sequence.every((step) =>
|
|
step.status === "succeeded" &&
|
|
step.idsPresent &&
|
|
step.valueMatches &&
|
|
step.durableGreen &&
|
|
step.persisted
|
|
);
|
|
return {
|
|
id: "operation-audit-evidence-durable-green",
|
|
status: ready ? "pass" : "blocked",
|
|
summary: ready
|
|
? "Every M3 DO/DI step produced operationId, traceId, auditId, evidenceId, DEV-LIVE green evidence, and persisted write state."
|
|
: "M3 operation/audit/evidence durable green contract is incomplete.",
|
|
evidenceLevel: ready ? "DEV-LIVE" : "BLOCKED",
|
|
observations: {
|
|
requiredRoute: `${M3_IO_CHAIN.sourceResourceId}:${M3_IO_CHAIN.sourcePort} -> ${M3_IO_CHAIN.patchPanelServiceId} -> ${M3_IO_CHAIN.targetResourceId}:${M3_IO_CHAIN.targetPort}`,
|
|
sequence
|
|
}
|
|
};
|
|
}
|
|
|
|
function baseReport({ parsed, now, root }) {
|
|
const source = observeSource(root);
|
|
const liveMutationUsed = parsed.live === true && parsed.allowM3Write === true;
|
|
return {
|
|
$schema: "https://hwlab.pikastech.local/schemas/rpt-004-mvp-e2e-harness.schema.json",
|
|
$id: "https://hwlab.pikastech.local/dev-gate/rpt-004-mvp-e2e.json",
|
|
reportVersion: "v1",
|
|
reportKind,
|
|
issue,
|
|
supports: [
|
|
"pikasTech/HWLAB#7",
|
|
"pikasTech/HWLAB#78",
|
|
"pikasTech/HWLAB#131",
|
|
"pikasTech/HWLAB#143",
|
|
"pikasTech/HWLAB#227",
|
|
"pikasTech/HWLAB#236",
|
|
"pikasTech/HWLAB#239",
|
|
"pikasTech/HWLAB#275",
|
|
"pikasTech/HWLAB#315",
|
|
"pikasTech/HWLAB#316",
|
|
"pikasTech/HWLAB#317",
|
|
"pikasTech/HWLAB#334",
|
|
"pikasTech/HWLAB#351"
|
|
],
|
|
taskId,
|
|
task: "DC-DCSN-P0-2026-003",
|
|
acceptanceLevel: "rpt004_mvp_e2e",
|
|
devOnly: true,
|
|
prodDisabled: true,
|
|
reportLifecycle: activeReportLifecycle("RPT-004 live MVP E2E harness for active DEV 16666/16667 verification."),
|
|
mode: parsed.live ? "dev-live" : "source-check",
|
|
status: "blocked",
|
|
generatedAt: now(),
|
|
commitId: source.reportCommitId,
|
|
source,
|
|
target: {
|
|
environment: ENVIRONMENT_DEV,
|
|
browserUrl: parsed.url,
|
|
apiUrl: parsed.apiUrl,
|
|
apiHealthUrl: joinBasePath(parsed.apiUrl, "/health/live"),
|
|
webHealthUrl: joinBasePath(parsed.url, "/health/live"),
|
|
m3IoUrl: joinBasePath(parsed.apiUrl, M3_IO_CONTROL_ROUTE)
|
|
},
|
|
runPolicy: {
|
|
defaultLiveMode: "read-only",
|
|
liveMutationUsed,
|
|
allowM3Write: parsed.allowM3Write === true,
|
|
m3WriteRequiresExplicitFlag: "--allow-m3-write",
|
|
liveMutationAllowedOnlyOnDev: true,
|
|
summary: liveMutationUsed
|
|
? "Explicit DEV-only --allow-m3-write was provided; bounded simulator DO/DI write loop may run after current+healthy preconditions."
|
|
: "Default live mode performs only read-only live checks and payload classification; no M3 DO1 true/false write is attempted."
|
|
},
|
|
sourceContract: {
|
|
status: "pass",
|
|
documents: [
|
|
"AGENTS.md",
|
|
"docs/reference/dev-runtime-boundary.md",
|
|
"docs/reference/cloud-workbench.md",
|
|
"docs/reference/code-agent-chat-readiness.md",
|
|
"skills/hwlab-agent-runtime/SKILL.md"
|
|
],
|
|
summary: "RPT-004 keeps layout, live revision, M3 trusted IO, durable evidence, and Code Agent skill-path checks in one repo-owned entrypoint."
|
|
},
|
|
validationCommands: [
|
|
"node --check scripts/rpt004-mvp-e2e-harness.mjs",
|
|
"node --check scripts/src/rpt004-mvp-e2e-harness.mjs",
|
|
"node --test scripts/src/rpt004-mvp-e2e-harness.test.mjs",
|
|
"node scripts/rpt004-mvp-e2e-harness.mjs --check --no-write",
|
|
"node scripts/rpt004-mvp-e2e-harness.mjs --live --url http://74.48.78.17:16666/ --api-url http://74.48.78.17:16667/",
|
|
`node scripts/validate-dev-gate-report.mjs ${defaultJsonReportPath}`
|
|
],
|
|
localSmoke: {
|
|
status: "not_run",
|
|
commands: ["node scripts/rpt004-mvp-e2e-harness.mjs --check --no-write"],
|
|
evidence: ["Source-check mode validates the harness without touching live DEV."],
|
|
summary: "Local/source check is not a substitute for RPT-004 live MVP PASS."
|
|
},
|
|
dryRun: {
|
|
status: "not_applicable",
|
|
commands: ["node scripts/rpt004-mvp-e2e-harness.mjs --check --no-write"],
|
|
evidence: ["RPT-004 is a bounded live smoke harness, not a deployment dry-run."],
|
|
summary: "No deploy apply, rollout, kubectl mutation, or PROD action is performed."
|
|
},
|
|
devPreconditions: {
|
|
status: parsed.live ? "checked-live" : "source-only",
|
|
requirements: [
|
|
"16666 and 16667 must be the active DEV endpoints.",
|
|
"Live API and Cloud Web revisions must match the expected DEV artifact commit.",
|
|
"API /health/live must be ready and durable before the M3 trusted IO loop runs.",
|
|
"OpenAI Responses text fallback cannot satisfy the Code Agent skill-path/toolCalls gate."
|
|
],
|
|
summary: "RPT-004 stops before M3 live writes when revision or health preconditions are not green."
|
|
},
|
|
expected: null,
|
|
live: {},
|
|
dimensions: {
|
|
artifact: null,
|
|
revision: null,
|
|
health: null,
|
|
workbench: null,
|
|
layout: null,
|
|
m3TrustedLoop: null,
|
|
durableEvidence: null,
|
|
codeAgentSkillPath: null,
|
|
codeAgentSession: null,
|
|
skillCliM3Route: null
|
|
},
|
|
checks: [],
|
|
blockers: [],
|
|
safety: {
|
|
devOnly: true,
|
|
prodTouched: false,
|
|
deployApplyAttempted: false,
|
|
rolloutAttempted: false,
|
|
kubectlMutationAttempted: false,
|
|
readsKubernetesSecrets: false,
|
|
secretValuesPrinted: false,
|
|
liveM3WritesRequireCurrentHealthyRevision: true,
|
|
liveM3WritesRequireExplicitFlag: true,
|
|
liveMutationUsed,
|
|
m3WritesLimitedToSimulatorDoDiResetLoop: true,
|
|
openAiFallbackCountsAsPass: false,
|
|
statement: liveMutationUsed
|
|
? "This harness performs public DEV HTTP checks, browser layout smoke, Code Agent session/Skill CLI checks, and an explicit DEV-only bounded M3 simulator DO/DI loop after current+healthy preconditions. It never applies deploys or starts rollouts."
|
|
: "Default live mode performs public DEV HTTP checks, browser layout smoke, Code Agent session/Skill CLI checks, and read-only M3 route payload classification only. It never writes DO1, applies deploys, starts rollouts, reads secrets, or touches PROD."
|
|
}
|
|
};
|
|
}
|
|
|
|
async function buildExpectedArtifactState({ root, expectedCommit, fsJson }) {
|
|
const readJson = typeof fsJson === "function"
|
|
? (relativePath) => fsJson(relativePath)
|
|
: (relativePath) => readJsonOptional(root, relativePath);
|
|
const deploy = await readJson("deploy/deploy.json");
|
|
const catalog = await readJson("deploy/artifact-catalog.dev.json");
|
|
const artifactReport = await readJson(defaultArtifactReportPath);
|
|
const deployCommit = sanitizeCommit(deploy?.commitId);
|
|
const catalogCommit = sanitizeCommit(catalog?.commitId);
|
|
const reportCommit = sanitizeCommit(artifactReport?.artifactPublish?.sourceCommitId ?? artifactReport?.commitId);
|
|
const commitId = expectedCommit ?? firstKnown(deployCommit, catalogCommit, reportCommit);
|
|
const services = Array.isArray(catalog?.services) ? catalog.services : [];
|
|
const serviceStates = services.map((service) => {
|
|
const serviceCommit = sanitizeCommit(service?.commitId);
|
|
return {
|
|
serviceId: service?.serviceId ?? "unknown",
|
|
commitId: serviceCommit,
|
|
imageTag: String(service?.imageTag ?? imageTagFromImage(service?.image) ?? "unknown"),
|
|
digest: String(service?.digest ?? "unknown"),
|
|
publishState: service?.publishState ?? "unknown",
|
|
commitMatches: commitEquivalent(serviceCommit, commitId),
|
|
digestPresent: /^sha256:[a-f0-9]{64}$/u.test(String(service?.digest ?? ""))
|
|
};
|
|
});
|
|
const missingServices = SERVICE_IDS.filter((serviceId) => !serviceStates.some((service) => service.serviceId === serviceId));
|
|
const mismatches = [];
|
|
if (!commitId || commitId === "unknown") mismatches.push("expected commit not observed");
|
|
if (deployCommit !== "unknown" && !commitEquivalent(deployCommit, commitId)) mismatches.push("deploy commit mismatch");
|
|
if (catalogCommit !== "unknown" && !commitEquivalent(catalogCommit, commitId)) mismatches.push("catalog commit mismatch");
|
|
if (reportCommit !== "unknown" && !commitEquivalent(reportCommit, commitId)) mismatches.push("artifact report commit mismatch");
|
|
if (catalog?.artifactState !== "published") mismatches.push("artifact catalog is not published");
|
|
if (catalog?.publish?.ciPublished !== true || catalog?.publish?.registryVerified !== true) {
|
|
mismatches.push("artifact catalog publish flags are not green");
|
|
}
|
|
if (missingServices.length > 0) mismatches.push(`missing services: ${missingServices.join(",")}`);
|
|
for (const service of serviceStates) {
|
|
if (service.publishState !== "published") mismatches.push(`${service.serviceId} is not published`);
|
|
if (!service.commitMatches) mismatches.push(`${service.serviceId} commit mismatch`);
|
|
if (!service.digestPresent) mismatches.push(`${service.serviceId} digest missing`);
|
|
}
|
|
|
|
const status = mismatches.length === 0 ? "pass" : "blocked";
|
|
return {
|
|
commitId,
|
|
shortCommitId: commitId === "unknown" ? "unknown" : commitId.slice(0, 7),
|
|
imageTag: commitId === "unknown" ? "unknown" : commitId.slice(0, 7),
|
|
deployCommit,
|
|
catalogCommit,
|
|
artifactReportCommit: reportCommit,
|
|
serviceCount: serviceStates.length,
|
|
requiredServiceCount: SERVICE_IDS.length,
|
|
services: serviceStates,
|
|
check: {
|
|
id: "expected-artifact-current",
|
|
status,
|
|
summary: status === "pass"
|
|
? `Expected DEV artifact commit ${commitId.slice(0, 7)} is published for all required services.`
|
|
: `Expected DEV artifact state is blocked: ${mismatches.join("; ")}.`,
|
|
evidenceLevel: status === "pass" ? "SOURCE" : "BLOCKED",
|
|
observations: {
|
|
commitId,
|
|
deployCommit,
|
|
catalogCommit,
|
|
artifactReportCommit: reportCommit,
|
|
serviceCount: serviceStates.length,
|
|
missingServices,
|
|
mismatches
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
function sourceHarnessCheck(root) {
|
|
const requiredFiles = [
|
|
"scripts/rpt004-mvp-e2e-harness.mjs",
|
|
"scripts/src/rpt004-mvp-e2e-harness.mjs",
|
|
"scripts/src/rpt004-mvp-e2e-harness.test.mjs",
|
|
"scripts/dev-cloud-workbench-layout-smoke.mjs",
|
|
"scripts/m3-io-control-e2e.mjs",
|
|
"scripts/code-agent-chat-smoke.mjs",
|
|
"skills/hwlab-agent-runtime/SKILL.md"
|
|
];
|
|
const missing = requiredFiles.filter((file) => !pathExists(root, file));
|
|
return {
|
|
id: "rpt004-source-contract",
|
|
status: missing.length === 0 ? "pass" : "blocked",
|
|
summary: missing.length === 0
|
|
? "RPT-004 source harness and reused smoke entrypoints are present."
|
|
: `RPT-004 source harness is missing files: ${missing.join(", ")}.`,
|
|
evidenceLevel: missing.length === 0 ? "SOURCE" : "BLOCKED",
|
|
observations: {
|
|
requiredFiles,
|
|
missing
|
|
}
|
|
};
|
|
}
|
|
|
|
function classifyLiveRevision({ expected, apiHealth, webHealth }) {
|
|
const mismatches = [];
|
|
if (expected.check.status !== "pass") mismatches.push("expected artifact state is not green");
|
|
if (!apiHealth.reachable) mismatches.push("api health not reachable");
|
|
if (!webHealth.reachable) mismatches.push("web health not reachable");
|
|
if (apiHealth.environment !== ENVIRONMENT_DEV) mismatches.push(`api environment=${apiHealth.environment}`);
|
|
if (webHealth.environment !== ENVIRONMENT_DEV) mismatches.push(`web environment=${webHealth.environment}`);
|
|
if (!commitEquivalent(apiHealth.revision, expected.commitId) && apiHealth.imageTag !== expected.imageTag) {
|
|
mismatches.push(`api revision ${apiHealth.revision}/${apiHealth.imageTag} != ${expected.shortCommitId}`);
|
|
}
|
|
if (!commitEquivalent(webHealth.revision, expected.commitId) && webHealth.imageTag !== expected.imageTag) {
|
|
mismatches.push(`web revision ${webHealth.revision}/${webHealth.imageTag} != ${expected.shortCommitId}`);
|
|
}
|
|
const status = mismatches.length === 0 ? "pass" : "not_current";
|
|
return {
|
|
id: "live-revision-current",
|
|
status,
|
|
summary: status === "pass"
|
|
? `Live 16666/16667 revisions match expected artifact commit ${expected.shortCommitId}.`
|
|
: `Live revision is NOT_CURRENT: ${mismatches.join("; ")}.`,
|
|
evidenceLevel: status === "pass" ? "DEV-LIVE" : "BLOCKED",
|
|
observations: {
|
|
expectedCommit: expected.commitId,
|
|
expectedImageTag: expected.imageTag,
|
|
api: {
|
|
endpoint: apiHealth.endpoint,
|
|
serviceId: apiHealth.serviceId,
|
|
revision: apiHealth.revision,
|
|
imageTag: apiHealth.imageTag,
|
|
digest: apiHealth.digest
|
|
},
|
|
web: {
|
|
endpoint: webHealth.endpoint,
|
|
serviceId: webHealth.serviceId,
|
|
revision: webHealth.revision,
|
|
imageTag: webHealth.imageTag,
|
|
digest: webHealth.digest
|
|
},
|
|
mismatches
|
|
}
|
|
};
|
|
}
|
|
|
|
function classifyLiveHealth(apiHealth, webHealth) {
|
|
const blockers = [];
|
|
if (!apiHealth.reachable) blockers.push("api /health/live unreachable");
|
|
if (!webHealth.reachable) blockers.push("web /health/live unreachable");
|
|
if (!["ready", "ok"].includes(apiHealth.healthStatus) || apiHealth.ready !== true) {
|
|
blockers.push(`api health ${apiHealth.healthStatus}; ready=${apiHealth.ready}`);
|
|
}
|
|
if (apiHealth.runtime.durable !== true || apiHealth.runtime.ready !== true || apiHealth.readiness.durability.ready !== true) {
|
|
blockers.push(`runtime durable not green: ${apiHealth.runtime.status}/${apiHealth.runtime.blocker}`);
|
|
}
|
|
if (!["ok", "ready"].includes(webHealth.healthStatus)) {
|
|
blockers.push(`web health ${webHealth.healthStatus}`);
|
|
}
|
|
return {
|
|
id: "live-health-ready",
|
|
status: blockers.length === 0 ? "pass" : "blocked",
|
|
summary: blockers.length === 0
|
|
? "Live health is ready and runtime durable readiness is green."
|
|
: `Live health is BLOCKED/degraded: ${blockers.join("; ")}.`,
|
|
evidenceLevel: blockers.length === 0 ? "DEV-LIVE" : "BLOCKED",
|
|
observations: {
|
|
apiStatus: apiHealth.healthStatus,
|
|
apiReady: apiHealth.ready,
|
|
runtime: apiHealth.runtime,
|
|
readiness: apiHealth.readiness,
|
|
blockerCodes: apiHealth.blockerCodes,
|
|
webStatus: webHealth.healthStatus,
|
|
blockers
|
|
}
|
|
};
|
|
}
|
|
|
|
function classifyDefaultChineseWorkbench(html) {
|
|
const missingLabels = requiredWorkbenchLabels.filter((label) => !html.text.includes(label));
|
|
const missingMarkers = workbenchMarkers.filter((marker) => !html.text.includes(marker));
|
|
const workspaceDefault = /data-view=["']workspace["'][^>]*>/u.test(html.text) &&
|
|
!/<section[^>]*data-view=["']workspace["'][^>]*hidden/u.test(html.text);
|
|
const gateHidden = /data-view=["']gate["'][^>]*hidden/u.test(html.text);
|
|
const blockers = [];
|
|
if (!html.ok) blockers.push(`index HTML HTTP ${html.httpStatus ?? "not_observed"}`);
|
|
if (missingLabels.length > 0) blockers.push(`missing Chinese labels: ${missingLabels.join(",")}`);
|
|
if (missingMarkers.length > 0) blockers.push(`missing workbench markers: ${missingMarkers.join(",")}`);
|
|
if (!workspaceDefault) blockers.push("workspace view is not the default visible view");
|
|
if (!gateHidden) blockers.push("internal gate view is not hidden by default");
|
|
return {
|
|
id: "16666-default-chinese-workbench",
|
|
status: blockers.length === 0 ? "pass" : "blocked",
|
|
summary: blockers.length === 0
|
|
? "16666 default route serves the Chinese user workbench."
|
|
: `16666 default Chinese workbench check is blocked: ${blockers.join("; ")}.`,
|
|
evidenceLevel: blockers.length === 0 ? "DEV-LIVE" : "BLOCKED",
|
|
observations: {
|
|
url: html.url,
|
|
httpStatus: html.httpStatus,
|
|
contentType: html.contentType,
|
|
missingLabels,
|
|
missingMarkers,
|
|
workspaceDefault,
|
|
gateHidden,
|
|
bodyHash: html.bodyHash
|
|
}
|
|
};
|
|
}
|
|
|
|
async function runLayoutDimension(parsed, options) {
|
|
const layoutRunner = options.layoutRunner ?? runDevCloudWorkbenchLayoutSmoke;
|
|
const layout = await layoutRunner({
|
|
live: true,
|
|
urlExplicit: true,
|
|
url: parsed.url,
|
|
timeoutMs: parsed.timeoutMs,
|
|
artifactDir: path.join("tmp", "rpt004-layout", safeTimestamp())
|
|
});
|
|
const pass = layout.status === "pass";
|
|
return {
|
|
id: "layout-smoke-hit-target",
|
|
status: pass ? "pass" : "blocked",
|
|
summary: pass
|
|
? "Live layout smoke passed across desktop/mobile hit target, overflow, and route checks."
|
|
: `Live layout smoke is blocked: ${layout.summary ?? layout.status}.`,
|
|
evidenceLevel: pass ? "DEV-LIVE-LAYOUT" : "BLOCKED",
|
|
observations: {
|
|
status: layout.status,
|
|
sourceMode: layout.sourceMode,
|
|
url: layout.url,
|
|
failureCount: Array.isArray(layout.failures) ? layout.failures.length : 0,
|
|
blockers: layout.blockers ?? [],
|
|
screenshotDir: layout.artifacts?.screenshotDir ?? null,
|
|
screenshotCount: Array.isArray(layout.artifacts?.screenshots) ? layout.artifacts.screenshots.length : 0
|
|
}
|
|
};
|
|
}
|
|
|
|
async function runCodeAgentSkillPathDimension(parsed, options) {
|
|
const endpoint = joinBasePath(parsed.apiUrl, "/v1/agent/chat");
|
|
const response = options.codeAgentRunner
|
|
? await options.codeAgentRunner({ endpoint, parsed })
|
|
: await postJson(endpoint, {
|
|
conversationId: `cnv_rpt004_skills_${randomUUID()}`,
|
|
traceId: `trc_rpt004_skills_${randomUUID()}`,
|
|
projectId: "prj_hwlab-cloud-workbench",
|
|
message: "列出你能使用的所有skill"
|
|
}, parsed.timeoutMs);
|
|
const payload = response.json ?? response.body ?? null;
|
|
const check = classifyCodeAgentSkillPath(payload, {
|
|
httpStatus: response.status,
|
|
httpOk: response.ok
|
|
});
|
|
return {
|
|
...check,
|
|
observations: {
|
|
...check.observations,
|
|
endpoint,
|
|
httpStatus: response.status,
|
|
traceId: payload?.traceId ?? null,
|
|
messageId: payload?.messageId ?? null,
|
|
assistantReplyStored: false
|
|
}
|
|
};
|
|
}
|
|
|
|
async function runCodeAgentSessionDimension(parsed, options) {
|
|
const endpoint = joinBasePath(parsed.apiUrl, "/v1/agent/chat");
|
|
const conversationId = `cnv_rpt004_session_${randomUUID()}`;
|
|
const requestedSessionId = `ses_rpt004_session_${randomUUID()}`;
|
|
const turns = [];
|
|
for (const turn of codeAgentSessionTurns) {
|
|
const response = options.codeAgentRunner
|
|
? await options.codeAgentRunner({
|
|
endpoint,
|
|
parsed,
|
|
conversationId,
|
|
sessionId: requestedSessionId,
|
|
turn
|
|
})
|
|
: await postJson(endpoint, {
|
|
conversationId,
|
|
sessionId: requestedSessionId,
|
|
traceId: `trc_rpt004_${turn.kind}_${randomUUID()}`,
|
|
projectId: "prj_hwlab-cloud-workbench",
|
|
message: turn.message
|
|
}, parsed.timeoutMs);
|
|
turns.push({
|
|
kind: turn.kind,
|
|
message: turn.message,
|
|
response: {
|
|
ok: response.ok,
|
|
status: response.status
|
|
},
|
|
payload: response.json ?? response.body ?? null
|
|
});
|
|
}
|
|
|
|
const sessionCheck = classifyCodeAgentSessionTurns(turns, { expectedConversationId: conversationId });
|
|
const skillPayload = turns.find((turn) => turn.kind === "skills")?.payload ?? null;
|
|
const skillResponse = turns.find((turn) => turn.kind === "skills")?.response ?? {};
|
|
const skillPathCheck = classifyCodeAgentSkillPath(skillPayload, {
|
|
httpStatus: skillResponse.status ?? null,
|
|
httpOk: skillResponse.ok === true
|
|
});
|
|
|
|
return {
|
|
sessionCheck: {
|
|
...sessionCheck,
|
|
observations: {
|
|
...sessionCheck.observations,
|
|
endpoint
|
|
}
|
|
},
|
|
skillPathCheck: {
|
|
...skillPathCheck,
|
|
observations: {
|
|
...skillPathCheck.observations,
|
|
endpoint,
|
|
conversationId,
|
|
sessionId: requestedSessionId,
|
|
traceId: skillPayload?.traceId ?? null,
|
|
messageId: skillPayload?.messageId ?? null,
|
|
assistantReplyStored: false
|
|
}
|
|
},
|
|
summary: {
|
|
endpoint,
|
|
status: sessionCheck.status === "pass" && skillPathCheck.status === "pass" ? "pass" : "blocked",
|
|
conversationId,
|
|
sessionId: requestedSessionId,
|
|
turns: sessionCheck.observations.turns.map((turn) => ({
|
|
kind: turn.kind,
|
|
status: turn.status,
|
|
runnerKind: turn.runnerKind,
|
|
sessionMode: turn.sessionMode,
|
|
capabilityLevel: turn.capabilityLevel,
|
|
traceId: turn.traceId,
|
|
toolNames: turn.toolNames
|
|
}))
|
|
}
|
|
};
|
|
}
|
|
|
|
async function runSkillCliM3RouteDimension(parsed, options) {
|
|
const endpoint = joinBasePath(parsed.apiUrl, "/v1/agent/chat");
|
|
const conversationId = `cnv_rpt004_skill_m3_${randomUUID()}`;
|
|
const sessionId = `ses_rpt004_skill_m3_${randomUUID()}`;
|
|
const response = options.skillCliRunner
|
|
? await options.skillCliRunner({
|
|
endpoint,
|
|
parsed,
|
|
conversationId,
|
|
sessionId,
|
|
message: m3ReadOnlySkillMessage
|
|
})
|
|
: await postJson(endpoint, {
|
|
conversationId,
|
|
sessionId,
|
|
traceId: `trc_rpt004_skill_m3_${randomUUID()}`,
|
|
projectId: "prj_hwlab-cloud-workbench",
|
|
message: m3ReadOnlySkillMessage
|
|
}, parsed.timeoutMs);
|
|
const payload = response.json ?? response.body ?? null;
|
|
const check = classifySkillCliM3Route(payload, {
|
|
httpStatus: response.status,
|
|
httpOk: response.ok === true
|
|
});
|
|
return {
|
|
...check,
|
|
observations: {
|
|
...check.observations,
|
|
endpoint,
|
|
conversationId,
|
|
sessionId,
|
|
message: m3ReadOnlySkillMessage,
|
|
liveMutationUsed: false
|
|
}
|
|
};
|
|
}
|
|
|
|
async function runM3TrustedLoop(parsed, options) {
|
|
const m3Runner = options.m3Runner ?? defaultM3Runner;
|
|
const fullReport = await m3Runner(parsed);
|
|
const operations = Array.isArray(fullReport.liveOperations) ? fullReport.liveOperations : [];
|
|
const durableCheck = classifyM3DurableEvidence(operations);
|
|
const trustedGreen = fullReport.summary?.trustedGreen === true &&
|
|
fullReport.summary?.status === "pass" &&
|
|
durableCheck.status === "pass";
|
|
const loopCheck = {
|
|
id: "m3-v1-io-trusted-loop",
|
|
status: trustedGreen ? "pass" : "blocked",
|
|
summary: trustedGreen
|
|
? "M3 /v1/m3/io DO1 true/read/false/read trusted loop passed through the patch-panel route."
|
|
: `M3 /v1/m3/io trusted loop is blocked: ${fullReport.summary?.classification ?? "unknown"}.`,
|
|
evidenceLevel: trustedGreen ? "DEV-LIVE" : "BLOCKED",
|
|
observations: {
|
|
route: M3_IO_CONTROL_ROUTE,
|
|
requiredRoute: `${M3_IO_CHAIN.sourceResourceId}:${M3_IO_CHAIN.sourcePort} -> ${M3_IO_CHAIN.patchPanelServiceId} -> ${M3_IO_CHAIN.targetResourceId}:${M3_IO_CHAIN.targetPort}`,
|
|
classification: fullReport.summary?.classification ?? null,
|
|
trustedGreen: fullReport.summary?.trustedGreen === true,
|
|
operationCount: operations.length,
|
|
operations: operations.map((operation) => ({
|
|
id: operation.id,
|
|
action: operation.action,
|
|
status: operation.status,
|
|
resultValue: Object.hasOwn(operation, "resultValue") ? operation.resultValue : null,
|
|
operationId: operation.operationId ?? null,
|
|
traceId: operation.traceId ?? null,
|
|
auditId: operation.auditId ?? null,
|
|
evidenceId: operation.evidenceId ?? null,
|
|
evidenceState: {
|
|
status: operation.evidenceState?.status ?? null,
|
|
sourceKind: operation.evidenceState?.sourceKind ?? null,
|
|
durable: operation.evidenceState?.durable === true,
|
|
writeStatus: operation.evidenceState?.writeStatus ?? null
|
|
}
|
|
}))
|
|
}
|
|
};
|
|
return {
|
|
loopCheck,
|
|
durableCheck,
|
|
summary: {
|
|
mode: fullReport.mode,
|
|
status: fullReport.summary?.status ?? "unknown",
|
|
classification: fullReport.summary?.classification ?? null,
|
|
trustedGreen: fullReport.summary?.trustedGreen === true,
|
|
operationCount: operations.length
|
|
}
|
|
};
|
|
}
|
|
|
|
async function defaultM3Runner(parsed) {
|
|
const sourceReport = await buildM3IoControlSourceReport({ repoRoot });
|
|
return runM3IoControlLiveReport({
|
|
flags: new Set(["--live", "--confirm-dev", "--confirmed-non-production"]),
|
|
live: true,
|
|
confirmDev: true,
|
|
confirmedNonProduction: true,
|
|
target: "api",
|
|
frontendUrl: parsed.url,
|
|
apiUrl: joinBasePath(parsed.apiUrl, M3_IO_CONTROL_ROUTE),
|
|
timeoutMs: parsed.timeoutMs
|
|
}, sourceReport);
|
|
}
|
|
|
|
function addNotRunDimensions(report, reason) {
|
|
for (const [key, id] of [
|
|
["layout", "layout-smoke-hit-target"],
|
|
["codeAgentSession", "code-agent-session-thread"],
|
|
["codeAgentSkillPath", "code-agent-skill-path"],
|
|
["skillCliM3Route", "skill-cli-hwlab-api-m3-route"],
|
|
["m3TrustedLoop", "m3-v1-io-trusted-loop"],
|
|
["durableEvidence", "operation-audit-evidence-durable-green"]
|
|
]) {
|
|
if (report.dimensions[key]) continue;
|
|
const check = {
|
|
id,
|
|
status: "not_run",
|
|
summary: reason,
|
|
evidenceLevel: "BLOCKED",
|
|
observations: {
|
|
skipped: true,
|
|
reason
|
|
}
|
|
};
|
|
report.dimensions[key] = check;
|
|
addCheck(report, check);
|
|
}
|
|
}
|
|
|
|
function addReadOnlyM3Dimensions(report) {
|
|
const loopCheck = {
|
|
id: "m3-v1-io-trusted-loop",
|
|
status: "pass",
|
|
summary: "Default RPT-004 live mode did not write DO1; M3 route readiness is represented by the Skill CLI HWLAB API read-only route check.",
|
|
evidenceLevel: "DEV-LIVE",
|
|
observations: {
|
|
skippedWriteLoop: true,
|
|
liveMutationUsed: false,
|
|
allowM3Write: false,
|
|
requiredExplicitFlag: "--allow-m3-write",
|
|
route: M3_IO_CONTROL_ROUTE,
|
|
reason: "Default mode must not perform DO1 true/false writes."
|
|
}
|
|
};
|
|
const durableCheck = {
|
|
id: "operation-audit-evidence-durable-green",
|
|
status: "pass",
|
|
summary: "No DO1 write was attempted in default read-only mode; durable write evidence is not required unless --allow-m3-write is explicitly provided.",
|
|
evidenceLevel: "DEV-LIVE",
|
|
observations: {
|
|
notApplicableInDefaultReadOnlyMode: true,
|
|
liveMutationUsed: false,
|
|
allowM3Write: false,
|
|
requiredExplicitFlag: "--allow-m3-write",
|
|
evidenceIds: report.dimensions.skillCliM3Route?.observations
|
|
? [report.dimensions.skillCliM3Route.observations.evidenceId].filter(Boolean)
|
|
: []
|
|
}
|
|
};
|
|
report.live.m3 = {
|
|
mode: "read-only-route-check",
|
|
status: "pass",
|
|
classification: "READ_ONLY_ROUTE_CHECK",
|
|
trustedGreen: false,
|
|
liveMutationUsed: false,
|
|
operationCount: 0
|
|
};
|
|
report.dimensions.m3TrustedLoop = loopCheck;
|
|
report.dimensions.durableEvidence = durableCheck;
|
|
addCheck(report, loopCheck);
|
|
addCheck(report, durableCheck);
|
|
}
|
|
|
|
function finalizeReport(report) {
|
|
const conclusion = classifyRpt004Conclusion(report.checks, {
|
|
allowM3Write: report.runPolicy?.allowM3Write === true
|
|
});
|
|
report.status = conclusion.status;
|
|
report.conclusion = conclusion;
|
|
report.evidenceLevel = conclusion.mvpPass ? "DEV-LIVE-MVP" : conclusion.classification;
|
|
report.devLive = conclusion.mvpPass;
|
|
report.liveMutationUsed = report.runPolicy?.liveMutationUsed === true;
|
|
report.localSmoke.status = report.mode === "source-check" ? "pass" : "not_run";
|
|
report.devPreconditions.status = conclusion.status === "pass" ? "pass" : conclusion.status;
|
|
report.blockers = report.checks
|
|
.filter((check) => !["pass"].includes(check.status))
|
|
.filter((check) => !(conclusion.classification === "CHECK_ONLY" && check.status === "not_run"))
|
|
.map((check) => ({
|
|
type: check.status === "not_current" ? "runtime_blocker" : "observability_blocker",
|
|
scope: check.id,
|
|
status: "open",
|
|
summary: check.summary,
|
|
evidenceLevel: check.evidenceLevel ?? "BLOCKED"
|
|
}));
|
|
report.commanderSummary = buildCommanderSummary(report);
|
|
return report;
|
|
}
|
|
|
|
function buildCommanderSummary(report) {
|
|
const route = report.dimensions.skillCliM3Route?.observations ?? {};
|
|
const session = report.dimensions.codeAgentSession?.observations ?? {};
|
|
const skillPath = report.dimensions.codeAgentSkillPath?.observations ?? {};
|
|
const ids = {
|
|
operationId: route.operationId ?? null,
|
|
traceId: route.traceId ?? session.turns?.find((turn) => turn.traceId)?.traceId ?? null,
|
|
auditId: route.auditId ?? null,
|
|
evidenceId: route.evidenceId ?? null
|
|
};
|
|
const routeEvidenceContract = {
|
|
route: route.route ?? null,
|
|
expectedRoute: route.expectedRoute ?? HWLAB_M3_IO_API_ROUTE,
|
|
method: route.method ?? null,
|
|
operationId: ids.operationId,
|
|
traceId: ids.traceId,
|
|
auditId: ids.auditId,
|
|
evidenceId: ids.evidenceId,
|
|
accepted: route.accepted ?? null,
|
|
status: route.toolCallStatus ?? null,
|
|
controlReady: route.controlReady ?? null,
|
|
readback: route.readback ?? null,
|
|
visible: route.route === HWLAB_M3_IO_API_ROUTE &&
|
|
route.method === "POST" &&
|
|
Boolean(ids.operationId && ids.traceId && ids.auditId && ids.evidenceId)
|
|
};
|
|
return {
|
|
reportClass: "live-mutating",
|
|
allowedLiveMutationUsed: report.liveMutationUsed === true,
|
|
commit: report.expected?.commitId ?? "unknown",
|
|
revision: {
|
|
status: report.dimensions.revision?.status ?? "not_run",
|
|
expectedCommit: report.expected?.commitId ?? "unknown",
|
|
apiRevision: report.dimensions.revision?.observations?.api?.revision ?? null,
|
|
webRevision: report.dimensions.revision?.observations?.web?.revision ?? null
|
|
},
|
|
sessionResult: {
|
|
status: report.dimensions.codeAgentSession?.status ?? "not_run",
|
|
conversationIds: session.conversationIds ?? [],
|
|
sessionIds: session.sessionIds ?? [],
|
|
turnCount: session.turnCount ?? 0,
|
|
runnerKinds: [...new Set((session.turns ?? []).map((turn) => turn.runnerKind).filter(Boolean))],
|
|
sessionModes: [...new Set((session.turns ?? []).map((turn) => turn.sessionMode).filter(Boolean))],
|
|
capabilityLevels: [...new Set((session.turns ?? []).map((turn) => turn.capabilityLevel).filter(Boolean))]
|
|
},
|
|
skillCliResult: {
|
|
status: report.dimensions.codeAgentSkillPath?.status ?? "not_run",
|
|
runnerKind: skillPath.runnerKind ?? null,
|
|
skillsStatus: skillPath.skills?.status ?? null,
|
|
repoSkillReady: skillPath.skills?.repoSkillReady === true,
|
|
openAiFallback: skillPath.openAiFallback === true
|
|
},
|
|
m3RouteResult: {
|
|
status: report.dimensions.skillCliM3Route?.status ?? "not_run",
|
|
route: route.route ?? null,
|
|
expectedRoute: route.expectedRoute ?? HWLAB_M3_IO_API_ROUTE,
|
|
method: route.method ?? null,
|
|
structuredBlocker: route.structuredBlocker === true,
|
|
idsPresent: route.idsPresent === true,
|
|
accepted: route.accepted ?? null,
|
|
controlReady: route.controlReady ?? null,
|
|
readback: route.readback ?? null,
|
|
safety: route.safety ?? null
|
|
},
|
|
exactRouteEvidenceContract: routeEvidenceContract,
|
|
evidenceIds: Object.fromEntries(Object.entries(ids).filter(([, value]) => Boolean(value))),
|
|
blockers: report.blockers.map((blocker) => ({
|
|
scope: blocker.scope,
|
|
summary: blocker.summary,
|
|
evidenceLevel: blocker.evidenceLevel
|
|
})),
|
|
artifactReport: "not applicable: harness/scripts/tests only; no runtime service image was changed by RPT-004."
|
|
};
|
|
}
|
|
|
|
function addCheck(report, check) {
|
|
if (!check) return;
|
|
report.checks.push(check);
|
|
}
|
|
|
|
function compactReport(report) {
|
|
return {
|
|
reportKind: report.reportKind,
|
|
taskId: report.taskId,
|
|
mode: report.mode,
|
|
status: report.status,
|
|
conclusion: report.conclusion,
|
|
expected: {
|
|
commitId: report.expected?.commitId,
|
|
imageTag: report.expected?.imageTag,
|
|
serviceCount: report.expected?.serviceCount
|
|
},
|
|
target: report.target,
|
|
dimensions: Object.fromEntries(Object.entries(report.dimensions).map(([key, value]) => [
|
|
key,
|
|
value ? {
|
|
status: value.status,
|
|
summary: value.summary
|
|
} : null
|
|
])),
|
|
liveMutationUsed: report.liveMutationUsed,
|
|
commanderSummary: report.commanderSummary,
|
|
blockers: report.blockers,
|
|
artifacts: report.artifacts ?? null,
|
|
safety: report.safety
|
|
};
|
|
}
|
|
|
|
async function writeReports(report, args, root) {
|
|
const jsonPath = ensureNotRepoReportsPath(root, args.reportPath, "--report");
|
|
const markdownPath = ensureNotRepoReportsPath(root, args.markdownPath, "--markdown");
|
|
report.artifacts = {
|
|
jsonReportPath: relativeOrAbsolute(jsonPath, root),
|
|
markdownReportPath: relativeOrAbsolute(markdownPath, root),
|
|
artifactReport: "not applicable: harness/scripts/tests only; no runtime service image was changed by RPT-004."
|
|
};
|
|
await mkdir(path.dirname(jsonPath), { recursive: true });
|
|
await mkdir(path.dirname(markdownPath), { recursive: true });
|
|
await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
|
await writeFile(markdownPath, renderMarkdown(report), "utf8");
|
|
}
|
|
|
|
export function renderMarkdown(report) {
|
|
const rows = Object.entries(report.dimensions).map(([key, check]) =>
|
|
`| ${key} | ${check?.status ?? "missing"} | ${escapeMarkdown(check?.summary ?? "missing")} |`
|
|
);
|
|
return [
|
|
"# RPT-004 MVP E2E Harness Report",
|
|
"",
|
|
`- status: ${report.status}`,
|
|
`- classification: ${report.conclusion?.classification ?? "unknown"}`,
|
|
`- mvpPass: ${report.conclusion?.mvpPass === true}`,
|
|
`- generatedAt: ${report.generatedAt}`,
|
|
`- expectedCommit: ${report.expected?.commitId ?? "unknown"}`,
|
|
`- liveMutationUsed: ${report.liveMutationUsed === true}`,
|
|
`- allowM3Write: ${report.runPolicy?.allowM3Write === true}`,
|
|
`- skillCliRoute: ${report.commanderSummary?.m3RouteResult?.route ?? "not_run"}`,
|
|
`- skillCliMethod: ${report.commanderSummary?.m3RouteResult?.method ?? "not_run"}`,
|
|
`- operationId: ${report.commanderSummary?.evidenceIds?.operationId ?? "none"}`,
|
|
`- traceId: ${report.commanderSummary?.evidenceIds?.traceId ?? "none"}`,
|
|
`- auditId: ${report.commanderSummary?.evidenceIds?.auditId ?? "none"}`,
|
|
`- evidenceId: ${report.commanderSummary?.evidenceIds?.evidenceId ?? "none"}`,
|
|
"",
|
|
"| Dimension | Status | Summary |",
|
|
"| --- | --- | --- |",
|
|
...rows,
|
|
"",
|
|
"## Blockers",
|
|
"",
|
|
...(report.blockers.length === 0
|
|
? ["- none"]
|
|
: report.blockers.map((blocker) => `- ${blocker.scope}: ${blocker.summary}`)),
|
|
"",
|
|
"## Safety",
|
|
"",
|
|
`- deployApplyAttempted: ${report.safety.deployApplyAttempted}`,
|
|
`- rolloutAttempted: ${report.safety.rolloutAttempted}`,
|
|
`- secretValuesPrinted: ${report.safety.secretValuesPrinted}`,
|
|
`- openAiFallbackCountsAsPass: ${report.safety.openAiFallbackCountsAsPass}`,
|
|
`- liveMutationUsed: ${report.safety.liveMutationUsed === true}`,
|
|
""
|
|
].join("\n");
|
|
}
|
|
|
|
function summarizeHealthResponse(serviceId, endpoint, response) {
|
|
const json = response.json ?? {};
|
|
const revision = sanitizeCommit(json.commit?.id ?? json.commitId ?? json.revision);
|
|
const imageTag = sanitizeRuntimeString(json.image?.tag ?? imageTagFromImage(json.image?.reference));
|
|
return {
|
|
endpoint,
|
|
reachable: response.ok === true && Boolean(response.json),
|
|
httpStatus: response.status,
|
|
serviceId: sanitizeRuntimeString(json.serviceId ?? serviceId),
|
|
environment: sanitizeRuntimeString(json.environment),
|
|
healthStatus: sanitizeRuntimeString(json.status),
|
|
ready: json.ready === true,
|
|
revision,
|
|
imageTag,
|
|
digest: sanitizeRuntimeString(json.image?.digest),
|
|
observedAt: sanitizeTimestamp(json.observedAt),
|
|
runtime: {
|
|
durable: json.runtime?.durable === true,
|
|
ready: json.runtime?.ready === true,
|
|
status: sanitizeRuntimeString(json.runtime?.status),
|
|
blocker: sanitizeRuntimeString(json.runtime?.blocker),
|
|
liveRuntimeEvidence: json.runtime?.liveRuntimeEvidence === true
|
|
},
|
|
readiness: {
|
|
ready: json.readiness?.ready === true,
|
|
status: sanitizeRuntimeString(json.readiness?.status),
|
|
durability: {
|
|
ready: json.readiness?.durability?.ready === true,
|
|
status: sanitizeRuntimeString(json.readiness?.durability?.status),
|
|
blocker: sanitizeRuntimeString(json.readiness?.durability?.blocker),
|
|
blockedLayer: sanitizeRuntimeString(json.readiness?.durability?.blockedLayer),
|
|
queryResult: sanitizeRuntimeString(json.readiness?.durability?.queryResult)
|
|
}
|
|
},
|
|
blockerCodes: Array.isArray(json.blockerCodes)
|
|
? json.blockerCodes.filter((item) => typeof item === "string")
|
|
: [],
|
|
error: response.error ? redactFailureText(response.error) : null
|
|
};
|
|
}
|
|
|
|
function summarizeHtmlResponse(url, response) {
|
|
const text = typeof response.body === "string" ? response.body : "";
|
|
return {
|
|
url,
|
|
ok: response.ok === true && response.status === 200 && /text\/html/iu.test(response.contentType ?? ""),
|
|
httpStatus: response.status,
|
|
contentType: response.contentType ?? "",
|
|
text,
|
|
bodyHash: createHash("sha256").update(text).digest("hex").slice(0, 16),
|
|
error: response.error ? redactFailureText(response.error) : null
|
|
};
|
|
}
|
|
|
|
async function getJson(url, timeoutMs) {
|
|
const response = await request(url, { method: "GET", timeoutMs, headers: { accept: "application/json" } });
|
|
return {
|
|
...response,
|
|
json: parseJson(response.body)
|
|
};
|
|
}
|
|
|
|
async function getText(url, timeoutMs) {
|
|
return request(url, { method: "GET", timeoutMs, headers: { accept: "text/html,*/*" } });
|
|
}
|
|
|
|
async function postJson(url, body, timeoutMs) {
|
|
const response = await request(url, {
|
|
method: "POST",
|
|
timeoutMs,
|
|
headers: {
|
|
accept: "application/json",
|
|
"content-type": "application/json"
|
|
},
|
|
body: JSON.stringify(body)
|
|
});
|
|
return {
|
|
...response,
|
|
json: parseJson(response.body)
|
|
};
|
|
}
|
|
|
|
async function request(url, { method, timeoutMs, headers, body } = {}) {
|
|
const controller = new AbortController();
|
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
try {
|
|
const response = await fetch(url, {
|
|
method,
|
|
headers,
|
|
body,
|
|
signal: controller.signal
|
|
});
|
|
const text = await response.text();
|
|
return {
|
|
ok: response.ok,
|
|
status: response.status,
|
|
contentType: response.headers.get("content-type") ?? "",
|
|
body: text,
|
|
error: response.ok ? null : response.statusText
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
status: 0,
|
|
contentType: "",
|
|
body: "",
|
|
error: error.name === "AbortError" ? `request timed out after ${timeoutMs}ms` : error.message
|
|
};
|
|
} finally {
|
|
clearTimeout(timer);
|
|
}
|
|
}
|
|
|
|
export function validateLiveBoundary(parsed) {
|
|
for (const [name, value, expectedPort] of [
|
|
["--url", parsed.url, 16666],
|
|
["--api-url", parsed.apiUrl, 16667]
|
|
]) {
|
|
const url = new URL(value);
|
|
const port = Number(url.port || (url.protocol === "http:" ? 80 : 443));
|
|
if (url.protocol !== "http:" || url.hostname !== "74.48.78.17" || port !== expectedPort) {
|
|
throw new Error(`${name} must target active DEV http://74.48.78.17:${expectedPort}/`);
|
|
}
|
|
if (port === 6666 || port === 6667) {
|
|
throw new Error(`${name} must use active DEV ports 16666/16667, not deprecated ${port}`);
|
|
}
|
|
if (/\bprod(?:uction)?\b/iu.test(url.href)) {
|
|
throw new Error(`${name} must not target PROD`);
|
|
}
|
|
}
|
|
}
|
|
|
|
function normalizeArgs(args) {
|
|
return {
|
|
...parseArgs([]),
|
|
...args,
|
|
timeoutMs: args.timeoutMs ?? parseArgs([]).timeoutMs
|
|
};
|
|
}
|
|
|
|
function observeSource(root) {
|
|
const commitId = runGit(root, ["rev-parse", "--verify", "HEAD"]) ?? "unknown";
|
|
const shortCommitId = commitId === "unknown" ? "unknown" : commitId.slice(0, 12);
|
|
const status = runGit(root, ["status", "--porcelain"]);
|
|
const dirty = status === null ? null : status.length > 0;
|
|
return {
|
|
status: commitId === "unknown" ? "unknown" : "observed",
|
|
commitId,
|
|
shortCommitId,
|
|
reportCommitId: commitId !== "unknown" && dirty === false ? shortCommitId : "unknown",
|
|
worktreeState: dirty === null ? "unknown" : dirty ? "dirty" : "clean"
|
|
};
|
|
}
|
|
|
|
function runGit(root, args) {
|
|
try {
|
|
return execFileSync("git", args, {
|
|
cwd: root,
|
|
encoding: "utf8",
|
|
stdio: ["ignore", "pipe", "ignore"],
|
|
timeout: 5000
|
|
}).trim();
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function readJsonOptional(root, relativePath) {
|
|
try {
|
|
return JSON.parse(await readFile(path.join(root, relativePath), "utf8"));
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function pathExists(root, relativePath) {
|
|
try {
|
|
execFileSync("test", ["-f", path.join(root, relativePath)], { stdio: "ignore", timeout: 1000 });
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
function joinBasePath(value, targetPath) {
|
|
const base = new URL(value);
|
|
return new URL(targetPath, `${base.protocol}//${base.host}/`).toString();
|
|
}
|
|
|
|
function readArg(argv, index, flag) {
|
|
const value = argv[index];
|
|
if (!value || value.startsWith("--")) {
|
|
throw new Error(`${flag} requires a value`);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function parseTimeout(value) {
|
|
const parsed = Number.parseInt(value, 10);
|
|
if (!Number.isInteger(parsed) || parsed <= 0) {
|
|
throw new Error("--timeout-ms must be a positive integer");
|
|
}
|
|
return Math.min(parsed, 180000);
|
|
}
|
|
|
|
function worstStatus(statuses) {
|
|
return statuses
|
|
.filter(Boolean)
|
|
.sort((a, b) => (statusRank[b] ?? 2) - (statusRank[a] ?? 2))[0] ?? "not_run";
|
|
}
|
|
|
|
function firstKnown(...values) {
|
|
return values.find((value) => value && value !== "unknown") ?? "unknown";
|
|
}
|
|
|
|
function commitEquivalent(actual, expected) {
|
|
if (!actual || !expected || actual === "unknown" || expected === "unknown") return false;
|
|
return actual === expected || actual.startsWith(expected) || expected.startsWith(actual);
|
|
}
|
|
|
|
function sanitizeCommit(value) {
|
|
if (typeof value !== "string") return "unknown";
|
|
const trimmed = value.trim().toLowerCase();
|
|
return /^[a-f0-9]{7,40}$/u.test(trimmed) ? trimmed : "unknown";
|
|
}
|
|
|
|
function sanitizeRuntimeString(value) {
|
|
return typeof value === "string" && value.trim() ? value.trim() : "unknown";
|
|
}
|
|
|
|
function sanitizeTimestamp(value) {
|
|
return typeof value === "string" && !Number.isNaN(Date.parse(value)) ? value : new Date().toISOString();
|
|
}
|
|
|
|
function imageTagFromImage(image) {
|
|
if (typeof image !== "string") return null;
|
|
return image.match(/:([^:/]+)$/u)?.[1] ?? null;
|
|
}
|
|
|
|
function parseJson(value) {
|
|
try {
|
|
return value ? JSON.parse(value) : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function isOpenAiFallbackPayload(payload) {
|
|
return payload?.provider === "openai-responses" ||
|
|
payload?.runner?.kind === "openai-responses-fallback" ||
|
|
payload?.implementationType === "openai-responses-fallback" ||
|
|
payload?.providerTrace?.fallbackUsed === true;
|
|
}
|
|
|
|
function normalizeM3RouteBlockers(payload, skillTool) {
|
|
const blockers = [];
|
|
for (const raw of [
|
|
skillTool?.blocker,
|
|
skillTool?.capabilityBlocker,
|
|
skillTool?.trustBlocker,
|
|
payload?.error,
|
|
payload?.availability?.blocker,
|
|
...(Array.isArray(skillTool?.blockers) ? skillTool.blockers : []),
|
|
...(Array.isArray(payload?.skills?.blockers) ? payload.skills.blockers : []),
|
|
...(Array.isArray(payload?.runnerTrace?.blockers) ? payload.runnerTrace.blockers : []),
|
|
...(Array.isArray(payload?.availability?.blockers) ? payload.availability.blockers : [])
|
|
]) {
|
|
const blocker = normalizeBlocker(raw);
|
|
if (!blocker) continue;
|
|
if (blockers.some((item) => item.code === blocker.code && item.message === blocker.message)) continue;
|
|
blockers.push(blocker);
|
|
}
|
|
if (skillTool?.stderrSummary && skillTool.stderrSummary !== "") {
|
|
const blocker = normalizeBlocker({
|
|
code: skillTool.stderrSummary,
|
|
message: skillTool.stderrSummary
|
|
});
|
|
if (blocker && !blockers.some((item) => item.code === blocker.code)) blockers.push(blocker);
|
|
}
|
|
return blockers;
|
|
}
|
|
|
|
function normalizeBlocker(raw) {
|
|
if (!raw) return null;
|
|
if (typeof raw === "string") {
|
|
return {
|
|
code: raw,
|
|
message: raw,
|
|
category: "blocked"
|
|
};
|
|
}
|
|
if (typeof raw !== "object") return null;
|
|
const code = raw.code ?? raw.blocker ?? raw.reason ?? raw.status ?? null;
|
|
const message = raw.message ?? raw.summary ?? raw.zh ?? raw.reason ?? code;
|
|
if (!code && !message) return null;
|
|
return {
|
|
code: String(code ?? "structured_blocker"),
|
|
message: String(message ?? code ?? "structured blocker"),
|
|
category: raw.category ?? raw.layer ?? "blocked",
|
|
sourceIssue: raw.sourceIssue ?? null
|
|
};
|
|
}
|
|
|
|
function directHardwareTargetMentioned(payload) {
|
|
const urls = [];
|
|
const collect = (value) => {
|
|
if (typeof value === "string") {
|
|
if (/https?:\/\//iu.test(value) || /\/invoke\b|\/sync\/tick\b/iu.test(value)) urls.push(value);
|
|
return;
|
|
}
|
|
if (!value || typeof value !== "object") return;
|
|
for (const [key, child] of Object.entries(value)) {
|
|
if (/policy|blocked|blocker|summary|message|note|limitations?|allowed|safety/iu.test(key)) continue;
|
|
collect(child);
|
|
}
|
|
};
|
|
collect(payload);
|
|
return urls.some((value) => forbiddenDirectHardwareTarget.test(value));
|
|
}
|
|
|
|
function safeTimestamp() {
|
|
return new Date().toISOString().replace(/[:.]/gu, "-");
|
|
}
|
|
|
|
function relativeOrAbsolute(absolutePath, root) {
|
|
const relative = path.relative(root, absolutePath);
|
|
return relative && !relative.startsWith("..") && !path.isAbsolute(relative) ? relative : absolutePath;
|
|
}
|
|
|
|
function escapeMarkdown(value) {
|
|
return String(value ?? "").replace(/\|/gu, "\\|").replace(/\n/gu, " ");
|
|
}
|
|
|
|
function redactFailureText(value) {
|
|
return String(value ?? "")
|
|
.replace(/postgres(?:ql)?:\/\/[^\s"'<>]+/giu, "[redacted-postgres-url]")
|
|
.replace(/(password\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]")
|
|
.replace(/(token\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]")
|
|
.replace(/(OPENAI_API_KEY\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]");
|
|
}
|
|
|
|
function usage() {
|
|
return [
|
|
"Usage: node scripts/rpt004-mvp-e2e-harness.mjs --live --url http://74.48.78.17:16666/ --api-url http://74.48.78.17:16667/",
|
|
"",
|
|
"Default --check/source mode validates the harness only and does not claim MVP PASS.",
|
|
"--live reads DEV health and the 16666 workbench, then runs layout, Code Agent same-session checks, and Skill CLI /v1/m3/io route classification after current+healthy preconditions pass.",
|
|
"--allow-m3-write is required for the bounded DEV-only M3 DO/DI write loop; default live mode does not write DO1.",
|
|
"The command writes JSON and Markdown reports by default in --live mode. It never deploys, rolls out, reads secrets, or touches PROD."
|
|
].join("\n");
|
|
}
|
|
|
|
export function formatRpt004Failure(error) {
|
|
const message = redactFailureText(error instanceof Error ? error.message : String(error));
|
|
return {
|
|
reportKind,
|
|
issue,
|
|
taskId,
|
|
status: "blocked",
|
|
conclusion: {
|
|
status: "blocked",
|
|
classification: "BLOCKED",
|
|
mvpPass: false,
|
|
summary: "RPT-004 harness failed before producing a full report."
|
|
},
|
|
error: message,
|
|
safety: {
|
|
prodTouched: false,
|
|
deployApplyAttempted: false,
|
|
rolloutAttempted: false,
|
|
secretValuesPrinted: false
|
|
}
|
|
};
|
|
}
|