fix: show Code Agent completion evidence
Host commander review: merged PR #171 after diff review. Scope is user-visible Code Agent completion evidence and smoke guards that reject echo/mock/stub completions; live smoke is read-only aside from the minimal chat request and does not print reply text or secrets. No runtime mutation, no service restart, and no M3/M4/M5 acceptance claim.
This commit is contained in:
Lyon
@@ -10,6 +10,8 @@ import {
|
||||
|
||||
const defaultLiveUrl = "http://74.48.78.17:16666/";
|
||||
const defaultLiveMessage = "请用一句话回复 HWLAB Code Agent readiness。";
|
||||
const trustedLiveProviders = new Set(["openai-responses", "codex-cli"]);
|
||||
const untrustedProviderPattern = /\b(?:echo|mock|fixture|stub|sample)\b/iu;
|
||||
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
|
||||
@@ -68,6 +70,19 @@ async function runLocalContractSmoke() {
|
||||
assert.equal(sourceReadiness.devLiveReplyPass, false);
|
||||
logOk("local stub completion is not #143 DEV-LIVE pass");
|
||||
|
||||
const echoReadiness = classifyCodeAgentChatReadiness({
|
||||
...completed,
|
||||
provider: "echo-mock",
|
||||
model: "mock-model",
|
||||
backend: "hwlab-cloud-api/echo-mock"
|
||||
}, { realDevLive: true, httpStatus: 200 });
|
||||
assert.equal(echoReadiness.status, "blocked");
|
||||
assert.equal(echoReadiness.level, "BLOCKED/untrusted-completion");
|
||||
assert.equal(echoReadiness.blocker, "untrusted-completion");
|
||||
assert.equal(echoReadiness.devLiveReplyPass, false);
|
||||
assert.match(echoReadiness.reason, /echo\/mock\/stub/u);
|
||||
logOk("echo/mock completion is not real DEV-LIVE pass");
|
||||
|
||||
const failed = await handleCodeAgentChat(
|
||||
{
|
||||
traceId: "trc_code-agent-chat-smoke-failed",
|
||||
@@ -186,6 +201,16 @@ async function runLiveReadinessSmoke(args) {
|
||||
function classifyCodeAgentChatReadiness(payload, { realDevLive = false, httpStatus = null } = {}) {
|
||||
const assistantReply = typeof payload?.reply?.content === "string" ? payload.reply.content.trim() : "";
|
||||
if (payload?.status === "completed" && assistantReply) {
|
||||
const evidence = inspectRealCompletionEvidence(payload);
|
||||
if (!evidence.ok) {
|
||||
return {
|
||||
status: "blocked",
|
||||
level: realDevLive ? "BLOCKED/untrusted-completion" : "SOURCE",
|
||||
blocker: "untrusted-completion",
|
||||
devLiveReplyPass: false,
|
||||
reason: `completed response is missing real provider/model/trace/conversation evidence or looks like echo/mock/stub: ${evidence.reason}`
|
||||
};
|
||||
}
|
||||
if (realDevLive) {
|
||||
return {
|
||||
status: "pass",
|
||||
@@ -228,6 +253,28 @@ function classifyCodeAgentChatReadiness(payload, { realDevLive = false, httpStat
|
||||
};
|
||||
}
|
||||
|
||||
function inspectRealCompletionEvidence(payload) {
|
||||
const provider = String(payload?.provider ?? "").trim().toLowerCase();
|
||||
const backend = String(payload?.backend ?? "").trim().toLowerCase();
|
||||
const missing = [];
|
||||
if (!payload?.conversationId && !payload?.sessionId) missing.push("conversationId");
|
||||
if (!payload?.traceId) missing.push("traceId");
|
||||
if (!payload?.messageId) missing.push("messageId");
|
||||
if (!payload?.provider) missing.push("provider");
|
||||
if (!payload?.model) missing.push("model");
|
||||
if (!payload?.backend) missing.push("backend");
|
||||
if (missing.length > 0) {
|
||||
return { ok: false, reason: `missing ${missing.join(",")}` };
|
||||
}
|
||||
if (!trustedLiveProviders.has(provider)) {
|
||||
return { ok: false, reason: `provider ${payload.provider} is not an accepted real provider` };
|
||||
}
|
||||
if (untrustedProviderPattern.test(`${provider} ${backend}`)) {
|
||||
return { ok: false, reason: `provider/backend looks non-real: ${payload.provider}/${payload.backend}` };
|
||||
}
|
||||
return { ok: true, reason: "provider/model/trace/conversation evidence present" };
|
||||
}
|
||||
|
||||
function summarizePayload(payload) {
|
||||
const assistantReply = typeof payload?.reply?.content === "string" ? payload.reply.content.trim() : "";
|
||||
const error = payload?.error
|
||||
|
||||
@@ -83,6 +83,7 @@ const requiredTrustedRecordTerms = Object.freeze([
|
||||
"audit.event.query + SOURCE 回退",
|
||||
"evidence.record.query + SOURCE 回退",
|
||||
"conversationId、traceId 和 messageId",
|
||||
"provider/model/trace/conversation",
|
||||
"失败原因=",
|
||||
"只读 audit 缺少 M3 patch-panel live report / operation / trace / evidence 绑定",
|
||||
"只读 evidence 缺少 M3 patch-panel live report / operation / trace / audit 绑定",
|
||||
@@ -119,6 +120,20 @@ const workbenchMarkers = Object.freeze([
|
||||
"command-form"
|
||||
]);
|
||||
|
||||
const requiredCodeAgentEvidenceTerms = Object.freeze([
|
||||
"provider",
|
||||
"model",
|
||||
"backend",
|
||||
"conversation",
|
||||
"trace",
|
||||
"message",
|
||||
"providerTrace",
|
||||
"openai-responses",
|
||||
"codex-cli",
|
||||
"echo/mock/stub",
|
||||
"untrusted_completion"
|
||||
]);
|
||||
|
||||
const forbiddenWritePatterns = Object.freeze([
|
||||
/callRpc\(\s*["']hardware\./u,
|
||||
/hardware\.operation\.request/u,
|
||||
@@ -271,6 +286,11 @@ function runStaticSmoke() {
|
||||
evidence: ["BLOCKED 凭证缺口", "provider_unavailable", "OPENAI_API_KEY", "hwlab-code-agent-provider/openai-api-key", "completed -> dev-live guard"]
|
||||
});
|
||||
|
||||
addCheck(checks, blockers, "code-agent-completed-evidence-visible", hasCodeAgentCompletedEvidenceVisibility(files), "Completed Code Agent replies expose provider/model/trace/conversation evidence and reject echo/mock/stub completions.", {
|
||||
blocker: "observability_blocker",
|
||||
evidence: requiredCodeAgentEvidenceTerms
|
||||
});
|
||||
|
||||
addCheck(checks, blockers, "no-hardware-write-api", noForbiddenWriteSurface(source), "Workbench source does not expose hardware write APIs or live mutation commands.", {
|
||||
blocker: "safety_blocker",
|
||||
evidence: forbiddenWritePatterns.map(String)
|
||||
@@ -657,10 +677,14 @@ function hasCodeAgentReadinessVisibility({ html, app }) {
|
||||
/hwlab-code-agent-provider\/openai-api-key/u.test(app) &&
|
||||
/只有真实 completed 回复才标 DEV-LIVE/u.test(app) &&
|
||||
/不能因为只有 conversationId 标为开发实况/u.test(app) &&
|
||||
/Boolean\(message\.provider\)/u.test(app) &&
|
||||
/Boolean\(message\.model\)/u.test(app) &&
|
||||
/Boolean\(message\.backend\)/u.test(app) &&
|
||||
/status === "completed" \? "dev-live"/u.test(app) &&
|
||||
/isTrustedCodeAgentProvider/u.test(app) &&
|
||||
/Boolean\(value\?\.model\)/u.test(app) &&
|
||||
/Boolean\(value\?\.backend\)/u.test(app) &&
|
||||
/Boolean\(value\?\.traceId\)/u.test(app) &&
|
||||
/Boolean\(value\?\.conversationId \|\| value\?\.sessionId\)/u.test(app) &&
|
||||
/isRealCompletedChatMessage/u.test(app) &&
|
||||
/isRealCompletedChatResult/u.test(app) &&
|
||||
/hasRealCodeAgentEvidence/u.test(app) &&
|
||||
!/status === "failed" \? "dev-live"/u.test(app) &&
|
||||
!/tone:\s*state\.conversationId\s*\?\s*"dev-live"/u.test(app) &&
|
||||
!/provider_unavailable[\s\S]{0,120}tone-[\w-]*dev-live/iu.test(source) &&
|
||||
@@ -668,6 +692,32 @@ function hasCodeAgentReadinessVisibility({ html, app }) {
|
||||
);
|
||||
}
|
||||
|
||||
function hasCodeAgentCompletedEvidenceVisibility({ app }) {
|
||||
const messageEvidenceBody = functionBody(app, "messageEvidenceFields");
|
||||
const realEvidenceBody = functionBody(app, "hasRealCodeAgentEvidence");
|
||||
return (
|
||||
requiredCodeAgentEvidenceTerms.every((term) => app.includes(term)) &&
|
||||
/conversationId:\s*result\.conversationId \|\| result\.sessionId \|\| state\.conversationId/u.test(app) &&
|
||||
/providerTrace:\s*result\.providerTrace/u.test(app) &&
|
||||
/function\s+providerTraceSummary\s*\(/u.test(app) &&
|
||||
/recordField\("provider",\s*message\.provider\)/u.test(messageEvidenceBody) &&
|
||||
/recordField\("model",\s*message\.model\)/u.test(messageEvidenceBody) &&
|
||||
/recordField\("backend",\s*message\.backend\)/u.test(messageEvidenceBody) &&
|
||||
/recordField\("conversation",\s*message\.conversationId\)/u.test(messageEvidenceBody) &&
|
||||
/recordField\("trace",\s*message\.traceId\)/u.test(messageEvidenceBody) &&
|
||||
/recordField\("message",\s*message\.messageId\)/u.test(messageEvidenceBody) &&
|
||||
/providerTraceSummary\(message\.providerTrace\)/u.test(messageEvidenceBody) &&
|
||||
/isTrustedCodeAgentProvider\(value\?\.provider\)/u.test(realEvidenceBody) &&
|
||||
/TRUSTED_CODE_AGENT_PROVIDERS/u.test(app) &&
|
||||
/UNTRUSTED_CODE_AGENT_PROVIDER_PATTERN/u.test(realEvidenceBody) &&
|
||||
/value\?\.conversationId \|\| value\?\.sessionId/u.test(realEvidenceBody) &&
|
||||
/isRealCompletedChatResult\(result\)/u.test(app) &&
|
||||
/realCompleted \? "completed" : "failed"/u.test(app) &&
|
||||
/Code Agent 完成证据不足/u.test(app) &&
|
||||
/本次不会标记为真实完成/u.test(app)
|
||||
);
|
||||
}
|
||||
|
||||
function noForbiddenWriteSurface(source) {
|
||||
return forbiddenWritePatterns.every((pattern) => !pattern.test(source));
|
||||
}
|
||||
@@ -848,7 +898,18 @@ async function fetchText(url) {
|
||||
}
|
||||
|
||||
function liveHtmlLooksLikeWorkbench(html) {
|
||||
return /HWLAB 云工作台/u.test(html) && labelsPresent(html, workbenchMarkers) && !/<body[^>]*>\s*<(?:main|section)[^>]*(?:gate|diagnostics|status|help)/iu.test(html);
|
||||
const workspaceTag = firstTagForDataView(html, "workspace");
|
||||
const gateTag = firstTagForDataView(html, "gate");
|
||||
const helpTag = firstTagForDataView(html, "help");
|
||||
return (
|
||||
/HWLAB 云工作台/u.test(html) &&
|
||||
labelsPresent(html, workbenchMarkers) &&
|
||||
workspaceTag !== null &&
|
||||
!/\bhidden\b/u.test(workspaceTag) &&
|
||||
gateTag !== null &&
|
||||
/\bhidden\b/u.test(gateTag) &&
|
||||
(helpTag === null || /\bhidden\b/u.test(helpTag))
|
||||
);
|
||||
}
|
||||
|
||||
async function inspectLiveDom(url) {
|
||||
@@ -892,7 +953,7 @@ async function inspectLiveDom(url) {
|
||||
dom.htmlOverflow === "hidden" &&
|
||||
dom.workspaceHidden === false &&
|
||||
dom.gateHidden === true &&
|
||||
dom.diagnosticsHidden === true &&
|
||||
(dom.diagnosticsHidden === true || dom.diagnosticsHidden === null) &&
|
||||
dom.outerScrollLocked &&
|
||||
dom.labelsPresent;
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user