diff --git a/scripts/code-agent-chat-smoke.mjs b/scripts/code-agent-chat-smoke.mjs index 41450690..c45e243f 100644 --- a/scripts/code-agent-chat-smoke.mjs +++ b/scripts/code-agent-chat-smoke.mjs @@ -10,6 +10,8 @@ import { const defaultLiveUrl = "http://74.48.78.17:16666/"; const defaultLiveMessage = "请用一句话回复 HWLAB Code Agent readiness。"; +const trustedLiveProviders = new Set(["openai-responses", "codex-cli"]); +const untrustedProviderPattern = /\b(?:echo|mock|fixture|stub|sample)\b/iu; const args = parseArgs(process.argv.slice(2)); @@ -68,6 +70,19 @@ async function runLocalContractSmoke() { assert.equal(sourceReadiness.devLiveReplyPass, false); logOk("local stub completion is not #143 DEV-LIVE pass"); + const echoReadiness = classifyCodeAgentChatReadiness({ + ...completed, + provider: "echo-mock", + model: "mock-model", + backend: "hwlab-cloud-api/echo-mock" + }, { realDevLive: true, httpStatus: 200 }); + assert.equal(echoReadiness.status, "blocked"); + assert.equal(echoReadiness.level, "BLOCKED/untrusted-completion"); + assert.equal(echoReadiness.blocker, "untrusted-completion"); + assert.equal(echoReadiness.devLiveReplyPass, false); + assert.match(echoReadiness.reason, /echo\/mock\/stub/u); + logOk("echo/mock completion is not real DEV-LIVE pass"); + const failed = await handleCodeAgentChat( { traceId: "trc_code-agent-chat-smoke-failed", @@ -186,6 +201,16 @@ async function runLiveReadinessSmoke(args) { function classifyCodeAgentChatReadiness(payload, { realDevLive = false, httpStatus = null } = {}) { const assistantReply = typeof payload?.reply?.content === "string" ? payload.reply.content.trim() : ""; if (payload?.status === "completed" && assistantReply) { + const evidence = inspectRealCompletionEvidence(payload); + if (!evidence.ok) { + return { + status: "blocked", + level: realDevLive ? "BLOCKED/untrusted-completion" : "SOURCE", + blocker: "untrusted-completion", + devLiveReplyPass: false, + reason: `completed response is missing real provider/model/trace/conversation evidence or looks like echo/mock/stub: ${evidence.reason}` + }; + } if (realDevLive) { return { status: "pass", @@ -224,6 +249,28 @@ function classifyCodeAgentChatReadiness(payload, { realDevLive = false, httpStat }; } +function inspectRealCompletionEvidence(payload) { + const provider = String(payload?.provider ?? "").trim().toLowerCase(); + const backend = String(payload?.backend ?? "").trim().toLowerCase(); + const missing = []; + if (!payload?.conversationId && !payload?.sessionId) missing.push("conversationId"); + if (!payload?.traceId) missing.push("traceId"); + if (!payload?.messageId) missing.push("messageId"); + if (!payload?.provider) missing.push("provider"); + if (!payload?.model) missing.push("model"); + if (!payload?.backend) missing.push("backend"); + if (missing.length > 0) { + return { ok: false, reason: `missing ${missing.join(",")}` }; + } + if (!trustedLiveProviders.has(provider)) { + return { ok: false, reason: `provider ${payload.provider} is not an accepted real provider` }; + } + if (untrustedProviderPattern.test(`${provider} ${backend}`)) { + return { ok: false, reason: `provider/backend looks non-real: ${payload.provider}/${payload.backend}` }; + } + return { ok: true, reason: "provider/model/trace/conversation evidence present" }; +} + function summarizePayload(payload) { const assistantReply = typeof payload?.reply?.content === "string" ? payload.reply.content.trim() : ""; const error = payload?.error diff --git a/scripts/src/dev-cloud-workbench-smoke-lib.mjs b/scripts/src/dev-cloud-workbench-smoke-lib.mjs index 0ed04fb7..66f048e3 100644 --- a/scripts/src/dev-cloud-workbench-smoke-lib.mjs +++ b/scripts/src/dev-cloud-workbench-smoke-lib.mjs @@ -83,6 +83,7 @@ const requiredTrustedRecordTerms = Object.freeze([ "audit.event.query + SOURCE 回退", "evidence.record.query + SOURCE 回退", "conversationId、traceId 和 messageId", + "provider/model/trace/conversation", "失败原因=", "只读 audit 缺少 M3 patch-panel live report / operation / trace / evidence 绑定", "只读 evidence 缺少 M3 patch-panel live report / operation / trace / audit 绑定", @@ -119,6 +120,20 @@ const workbenchMarkers = Object.freeze([ "command-form" ]); +const requiredCodeAgentEvidenceTerms = Object.freeze([ + "provider", + "model", + "backend", + "conversation", + "trace", + "message", + "providerTrace", + "openai-responses", + "codex-cli", + "echo/mock/stub", + "untrusted_completion" +]); + const forbiddenWritePatterns = Object.freeze([ /callRpc\(\s*["']hardware\./u, /hardware\.operation\.request/u, @@ -271,6 +286,11 @@ function runStaticSmoke() { evidence: ["BLOCKED 凭证缺口", "provider_unavailable", "OPENAI_API_KEY", "hwlab-code-agent-provider/openai-api-key", "completed -> dev-live guard"] }); + addCheck(checks, blockers, "code-agent-completed-evidence-visible", hasCodeAgentCompletedEvidenceVisibility(files), "Completed Code Agent replies expose provider/model/trace/conversation evidence and reject echo/mock/stub completions.", { + blocker: "observability_blocker", + evidence: requiredCodeAgentEvidenceTerms + }); + addCheck(checks, blockers, "no-hardware-write-api", noForbiddenWriteSurface(source), "Workbench source does not expose hardware write APIs or live mutation commands.", { blocker: "safety_blocker", evidence: forbiddenWritePatterns.map(String) @@ -657,10 +677,14 @@ function hasCodeAgentReadinessVisibility({ html, app }) { /hwlab-code-agent-provider\/openai-api-key/u.test(app) && /只有真实 completed 回复才标 DEV-LIVE/u.test(app) && /不能因为只有 conversationId 标为开发实况/u.test(app) && - /Boolean\(message\.provider\)/u.test(app) && - /Boolean\(message\.model\)/u.test(app) && - /Boolean\(message\.backend\)/u.test(app) && - /status === "completed" \? "dev-live"/u.test(app) && + /isTrustedCodeAgentProvider/u.test(app) && + /Boolean\(value\?\.model\)/u.test(app) && + /Boolean\(value\?\.backend\)/u.test(app) && + /Boolean\(value\?\.traceId\)/u.test(app) && + /Boolean\(value\?\.conversationId \|\| value\?\.sessionId\)/u.test(app) && + /isRealCompletedChatMessage/u.test(app) && + /isRealCompletedChatResult/u.test(app) && + /hasRealCodeAgentEvidence/u.test(app) && !/status === "failed" \? "dev-live"/u.test(app) && !/tone:\s*state\.conversationId\s*\?\s*"dev-live"/u.test(app) && !/provider_unavailable[\s\S]{0,120}tone-[\w-]*dev-live/iu.test(source) && @@ -668,6 +692,32 @@ function hasCodeAgentReadinessVisibility({ html, app }) { ); } +function hasCodeAgentCompletedEvidenceVisibility({ app }) { + const messageEvidenceBody = functionBody(app, "messageEvidenceFields"); + const realEvidenceBody = functionBody(app, "hasRealCodeAgentEvidence"); + return ( + requiredCodeAgentEvidenceTerms.every((term) => app.includes(term)) && + /conversationId:\s*result\.conversationId \|\| result\.sessionId \|\| state\.conversationId/u.test(app) && + /providerTrace:\s*result\.providerTrace/u.test(app) && + /function\s+providerTraceSummary\s*\(/u.test(app) && + /recordField\("provider",\s*message\.provider\)/u.test(messageEvidenceBody) && + /recordField\("model",\s*message\.model\)/u.test(messageEvidenceBody) && + /recordField\("backend",\s*message\.backend\)/u.test(messageEvidenceBody) && + /recordField\("conversation",\s*message\.conversationId\)/u.test(messageEvidenceBody) && + /recordField\("trace",\s*message\.traceId\)/u.test(messageEvidenceBody) && + /recordField\("message",\s*message\.messageId\)/u.test(messageEvidenceBody) && + /providerTraceSummary\(message\.providerTrace\)/u.test(messageEvidenceBody) && + /isTrustedCodeAgentProvider\(value\?\.provider\)/u.test(realEvidenceBody) && + /TRUSTED_CODE_AGENT_PROVIDERS/u.test(app) && + /UNTRUSTED_CODE_AGENT_PROVIDER_PATTERN/u.test(realEvidenceBody) && + /value\?\.conversationId \|\| value\?\.sessionId/u.test(realEvidenceBody) && + /isRealCompletedChatResult\(result\)/u.test(app) && + /realCompleted \? "completed" : "failed"/u.test(app) && + /Code Agent 完成证据不足/u.test(app) && + /本次不会标记为真实完成/u.test(app) + ); +} + function noForbiddenWriteSurface(source) { return forbiddenWritePatterns.every((pattern) => !pattern.test(source)); } @@ -848,7 +898,18 @@ async function fetchText(url) { } function liveHtmlLooksLikeWorkbench(html) { - return /HWLAB 云工作台/u.test(html) && labelsPresent(html, workbenchMarkers) && !/]*>\s*<(?:main|section)[^>]*(?:gate|diagnostics|status|help)/iu.test(html); + const workspaceTag = firstTagForDataView(html, "workspace"); + const gateTag = firstTagForDataView(html, "gate"); + const helpTag = firstTagForDataView(html, "help"); + return ( + /HWLAB 云工作台/u.test(html) && + labelsPresent(html, workbenchMarkers) && + workspaceTag !== null && + !/\bhidden\b/u.test(workspaceTag) && + gateTag !== null && + /\bhidden\b/u.test(gateTag) && + (helpTag === null || /\bhidden\b/u.test(helpTag)) + ); } async function inspectLiveDom(url) { @@ -892,7 +953,7 @@ async function inspectLiveDom(url) { dom.htmlOverflow === "hidden" && dom.workspaceHidden === false && dom.gateHidden === true && - dom.diagnosticsHidden === true && + (dom.diagnosticsHidden === true || dom.diagnosticsHidden === null) && dom.outerScrollLocked && dom.labelsPresent; return { diff --git a/web/hwlab-cloud-web/app.mjs b/web/hwlab-cloud-web/app.mjs index adc97bfb..68c852ad 100644 --- a/web/hwlab-cloud-web/app.mjs +++ b/web/hwlab-cloud-web/app.mjs @@ -48,6 +48,8 @@ const M3_TRUSTED_ROUTE = Object.freeze({ const LIVE_M3_ID_FIELDS = Object.freeze(["operationId", "traceId", "auditId", "evidenceId"]); const LIVE_M3_PASS_STATUSES = Object.freeze(["pass", "passed", "verified", "succeeded", "trusted", "completed"]); const NON_LIVE_ID_VALUES = Object.freeze(["", "n/a", "none", "null", "undefined", "not_observed", "not-observed"]); +const TRUSTED_CODE_AGENT_PROVIDERS = Object.freeze(["openai-responses", "codex-cli"]); +const UNTRUSTED_CODE_AGENT_PROVIDER_PATTERN = /\b(?:echo|mock|fixture|stub|sample)\b/iu; const viewIds = new Set([...document.querySelectorAll("[data-view]")].map((view) => view.dataset.view)); let rpcSequence = 0; @@ -281,27 +283,37 @@ function initCommandBar() { const result = await sendAgentMessage(value, state.conversationId, traceId); state.conversationId = result.conversationId || result.sessionId || state.conversationId; const index = state.chatMessages.findIndex((message) => message.id === pendingMessage.id); - const status = result.status === "completed" ? "completed" : "failed"; + const realCompleted = isRealCompletedChatResult(result); + const status = realCompleted ? "completed" : "failed"; state.chatMessages[index] = { ...pendingMessage, - title: result.status === "completed" ? sourceTitle(result) : "Code Agent 返回失败", - text: result.status === "completed" + title: realCompleted ? sourceTitle(result) : result.status === "completed" ? "Code Agent 完成证据不足" : "Code Agent 返回失败", + text: realCompleted ? result.reply?.content || "Code Agent 没有返回文本。" + : result.status === "completed" + ? untrustedCompletionMessage(result) : failureMessage(result), status, traceId: result.traceId, + conversationId: result.conversationId || result.sessionId || state.conversationId, messageId: result.messageId, provider: result.provider, model: result.model, backend: result.backend, + providerTrace: result.providerTrace, updatedAt: result.updatedAt, - error: result.error, + error: result.error ?? (result.status === "completed" && !realCompleted + ? { + code: "untrusted_completion", + message: "completed 回复缺少真实 provider/model/trace/conversation 证据，或 provider 属于 echo/mock/stub。" + } + : undefined), availability: result.availability }; if (result.availability) { state.codeAgentAvailability = result.availability; } - if (result.status !== "completed") { + if (!realCompleted) { el.commandInput.value = value; } renderAgentChatStatus(status, result); @@ -1320,14 +1332,33 @@ function messageCard(message) { article.append(textSpan(roleLabel(message.role), "message-role")); article.append(textSpan(message.title, "message-title")); article.append(textSpan(message.text, "message-copy")); - const meta = [message.provider, message.model, message.backend, message.traceId].filter(Boolean).join(" / "); - const messageMeta = [message.messageId, meta].filter(Boolean).join(" / "); + const messageMeta = messageEvidenceFields(message).join(" / "); if (messageMeta) { article.append(textSpan(messageMeta, "message-meta")); } return article; } +function messageEvidenceFields(message) { + return [ + recordField("provider", message.provider), + recordField("model", message.model), + recordField("backend", message.backend), + recordField("conversation", message.conversationId), + recordField("trace", message.traceId), + recordField("message", message.messageId), + providerTraceSummary(message.providerTrace) + ].filter(Boolean); +} + +function providerTraceSummary(providerTrace) { + if (!providerTrace || typeof providerTrace !== "object") return null; + const responseId = providerTrace.responseId ?? providerTrace.id; + if (responseId) return recordField("providerTrace", responseId); + if (providerTrace.command) return "providerTrace=codex-cli"; + return null; +} + function statusCard(item) { const article = document.createElement("article"); article.className = "status-card"; @@ -1461,8 +1492,8 @@ function codeAgentPromptText(availability) { function codeAgentControlSummary(availability) { if (latestCompletedAgentMessage()) { return state.conversationId - ? `当前会话 ${state.conversationId}；最近一次真实 completed 回复来自 cloud-api 受控 Code Agent 接口。` - : "最近一次真实 completed 回复来自 cloud-api 受控 Code Agent 接口。"; + ? `当前会话 ${state.conversationId}；最近一次真实 completed 回复来自 cloud-api 受控 Code Agent 接口，并带有 provider/model/trace/conversation 证据。` + : "最近一次真实 completed 回复来自 cloud-api 受控 Code Agent 接口，并带有 provider/model/trace/conversation 证据。"; } if (availability?.status === "blocked") { return codeAgentBlockedSummary(availability); @@ -1474,6 +1505,11 @@ function codeAgentBlockedSummary(availability) { return availability?.summary ?? "真实后端已接入，但当前 DEV provider Secret hwlab-code-agent-provider/openai-api-key 未注入，因此不能返回真实回复。"; } +function untrustedCompletionMessage(result) { + const provider = result?.provider ?? "unknown"; + return `Code Agent 返回 completed，但缺少真实 provider/model/trace/conversation 证据，或 provider=${provider} 属于 echo/mock/stub；本次不会标记为真实完成。`; +} + function codeAgentAvailabilitySummary() { if (state.codeAgentAvailability?.status === "blocked") { return "同源 API 可响应；Code Agent 为 BLOCKED/凭证缺口，真实后端已接入但 DEV provider Secret hwlab-code-agent-provider/openai-api-key 未注入。"; @@ -1487,14 +1523,35 @@ function codeAgentAvailabilitySummary() { function latestCompletedAgentMessage() { return [...state.chatMessages].reverse().find((message) => message.role === "agent" && - message.status === "completed" && - !message.error && - Boolean(message.provider) && - Boolean(message.model) && - Boolean(message.backend) + isRealCompletedChatMessage(message) ) ?? null; } +function isRealCompletedChatResult(result) { + const assistantReply = typeof result?.reply?.content === "string" ? result.reply.content.trim() : ""; + return result?.status === "completed" && assistantReply.length > 0 && hasRealCodeAgentEvidence(result); +} + +function isRealCompletedChatMessage(message) { + return message?.status === "completed" && !message.error && hasRealCodeAgentEvidence(message); +} + +function hasRealCodeAgentEvidence(value) { + return ( + isTrustedCodeAgentProvider(value?.provider) && + Boolean(value?.model) && + Boolean(value?.backend) && + Boolean(value?.traceId) && + Boolean(value?.conversationId || value?.sessionId) && + !UNTRUSTED_CODE_AGENT_PROVIDER_PATTERN.test(`${value?.provider ?? ""} ${value?.backend ?? ""}`) + ); +} + +function isTrustedCodeAgentProvider(provider) { + const normalized = String(provider ?? "").trim().toLowerCase(); + return TRUSTED_CODE_AGENT_PROVIDERS.includes(normalized); +} + function latestChatResult() { return [...state.chatMessages].reverse().find((message) => message.role === "agent") ?? null; } diff --git a/web/hwlab-cloud-web/scripts/check.mjs b/web/hwlab-cloud-web/scripts/check.mjs index b4f8379a..b07b36f8 100644 --- a/web/hwlab-cloud-web/scripts/check.mjs +++ b/web/hwlab-cloud-web/scripts/check.mjs @@ -50,6 +50,7 @@ const requiredTrustedRecordTerms = Object.freeze([ "audit.event.query + SOURCE 回退", "evidence.record.query + SOURCE 回退", "conversationId、traceId 和 messageId", + "provider/model/trace/conversation", "失败原因=", "只读 audit 缺少 M3 patch-panel live report / operation / trace / evidence 绑定", "只读 evidence 缺少 M3 patch-panel live report / operation / trace / audit 绑定", @@ -283,9 +284,23 @@ assert.match(app, /OPENAI_API_KEY/); assert.match(app, /hwlab-code-agent-provider\/openai-api-key/); assert.match(app, /只有真实 completed 回复才标 DEV-LIVE/); assert.match(app, /不能因为只有 conversationId 标为开发实况/); -assert.match(app, /Boolean\(message\.provider\)/); -assert.match(app, /Boolean\(message\.model\)/); -assert.match(app, /Boolean\(message\.backend\)/); +assert.match(app, /TRUSTED_CODE_AGENT_PROVIDERS/); +assert.match(app, /UNTRUSTED_CODE_AGENT_PROVIDER_PATTERN/); +assert.match(app, /isRealCompletedChatResult/); +assert.match(app, /isRealCompletedChatMessage/); +assert.match(app, /hasRealCodeAgentEvidence/); +assert.match(app, /isTrustedCodeAgentProvider\(value\?\.provider\)/); +assert.match(app, /Boolean\(value\?\.model\)/); +assert.match(app, /Boolean\(value\?\.backend\)/); +assert.match(app, /Boolean\(value\?\.traceId\)/); +assert.match(app, /Boolean\(value\?\.conversationId \|\| value\?\.sessionId\)/); +assert.match(app, /providerTrace:\s*result\.providerTrace/); +assert.match(app, /providerTraceSummary\(message\.providerTrace\)/); +assert.match(app, /recordField\("conversation", message\.conversationId\)/); +assert.match(app, /recordField\("provider", message\.provider\)/); +assert.match(app, /Code Agent 完成证据不足/); +assert.match(app, /本次不会标记为真实完成/); +assert.match(app, /untrusted_completion/); assert.match(app, /status === "completed" \? "dev-live"/); assert.doesNotMatch(app, /status === "failed" \? "dev-live"/); assert.doesNotMatch(`${html}\n${app}`, /provider_unavailable[\s\S]{0,120}tone-[\w-]*dev-live/iu);