diff --git a/scripts/code-agent-chat-smoke.mjs b/scripts/code-agent-chat-smoke.mjs index c722c14f..6b4b8d32 100644 --- a/scripts/code-agent-chat-smoke.mjs +++ b/scripts/code-agent-chat-smoke.mjs @@ -7,11 +7,14 @@ import { handleCodeAgentChat, validateCodeAgentChatSchema } from "../internal/cloud/code-agent-chat.mjs"; +import { + classifyCodeAgentChatReadiness, + isHttpNon2xxStatus, + summarizeCodeAgentPayload +} from "./src/code-agent-response-contract.mjs"; const defaultLiveUrl = "http://74.48.78.17:16666/"; const defaultLiveMessage = "请用一句话回复 HWLAB Code Agent readiness。"; -const trustedLiveProviders = new Set(["openai-responses", "codex-cli"]); -const untrustedProviderPattern = /\b(?:echo|mock|fixture|stub|sample)\b/iu; const args = parseArgs(process.argv.slice(2)); @@ -129,6 +132,46 @@ async function runLocalContractSmoke() { assert.equal(credentialReadiness.devLiveReplyPass, false); logOk("provider credential blocker readiness"); + const upstreamBlocked = classifyCodeAgentChatReadiness({ + ...failed, + error: { + code: "provider_unavailable", + message: "OpenAI Responses returned HTTP 502: request rejected", + providerStatus: 502, + missingEnv: [] + }, + availability: { + status: "available" + } + }, { realDevLive: true, httpStatus: 200 }); + assert.equal(upstreamBlocked.status, "blocked"); + assert.equal(upstreamBlocked.level, "BLOCKED/provider"); + assert.equal(upstreamBlocked.blocker, "provider-upstream"); + assert.equal(upstreamBlocked.providerStatus, 502); + assert.equal(upstreamBlocked.devLiveReplyPass, false); + logOk("provider 502 blocker readiness"); + + const completedLivePayload = { + ...completed, + provider: "openai-responses", + model: "gpt-5.5", + backend: "hwlab-cloud-api/openai-responses" + }; + const completedHttp200Readiness = classifyCodeAgentChatReadiness(completedLivePayload, { realDevLive: true, httpStatus: 200 }); + assert.equal(completedHttp200Readiness.status, "pass"); + assert.equal(completedHttp200Readiness.devLiveReplyPass, true); + logOk("HTTP 200 completion-looking payload can pass"); + + for (const status of [400, 401, 403, 429, 500, 502, 503, 504]) { + const readiness = classifyCodeAgentChatReadiness(completedLivePayload, { realDevLive: true, httpStatus: status }); + assert.equal(readiness.status, "blocked", `HTTP ${status} must block`); + assert.equal(readiness.blocker, "provider-upstream", `HTTP ${status} blocker`); + assert.equal(readiness.providerStatus, status, `HTTP ${status} providerStatus`); + assert.equal(readiness.devLiveReplyPass, false, `HTTP ${status} devLiveReplyPass`); + assert.match(readiness.reason, /HTTP non-2xx|upstream response/u, `HTTP ${status} reason`); + } + logOk("non-2xx completion-looking payloads are blocked"); + process.stdout.write("[code-agent-chat-smoke] passed\n"); } @@ -158,7 +201,7 @@ async function runLiveReadinessSmoke(args) { transportError = error instanceof Error ? error.message : String(error); } - const readiness = payload + const readiness = payload || isHttpNon2xxStatus(httpStatus) ? classifyCodeAgentChatReadiness(payload, { realDevLive: true, httpStatus }) : { status: "blocked", @@ -168,7 +211,7 @@ async function runLiveReadinessSmoke(args) { reason: "real DEV /v1/agent/chat did not return a JSON Code Agent payload" }; - if (payload) { + if (payload && readiness.status === "pass") { try { validateCodeAgentChatSchema(payload); } catch (error) { @@ -186,7 +229,7 @@ async function runLiveReadinessSmoke(args) { url: endpoint.href, httpStatus, readiness, - response: summarizePayload(payload), + response: summarizePayload(payload, { httpStatus, traceId }), ...(transportError ? { transportError } : {}), safety: { prodTouched: false, @@ -198,101 +241,19 @@ async function runLiveReadinessSmoke(args) { }; } -function classifyCodeAgentChatReadiness(payload, { realDevLive = false, httpStatus = null } = {}) { - const assistantReply = typeof payload?.reply?.content === "string" ? payload.reply.content.trim() : ""; - if (payload?.status === "completed" && assistantReply) { - const evidence = inspectRealCompletionEvidence(payload); - if (!evidence.ok) { - return { - status: "blocked", - level: realDevLive ? "BLOCKED/untrusted-completion" : "SOURCE", - blocker: "untrusted-completion", - devLiveReplyPass: false, - reason: `completed response is missing real provider/model/trace/conversation evidence or looks like echo/mock/stub: ${evidence.reason}` - }; - } - if (realDevLive) { - return { - status: "pass", - level: "#143 DEV-LIVE reply pass", - blocker: null, - devLiveReplyPass: true, - reason: "real DEV /v1/agent/chat completed with a non-empty assistant reply" - }; - } - return { - status: "blocked", - level: "SOURCE", - blocker: "not-dev-live", - devLiveReplyPass: false, - reason: "mock, fixture, local stub, or source-only completion cannot be counted as #143 DEV-LIVE reply pass" - }; - } - - const missingEnv = Array.isArray(payload?.error?.missingEnv) ? payload.error.missingEnv : []; - if ( - payload?.status === "failed" && - payload.error?.code === "provider_unavailable" && - (missingEnv.includes("OPENAI_API_KEY") || missingEnv.includes("HWLAB_CODE_AGENT_OPENAI_BASE_URL")) - ) { - return { - status: "blocked", - level: "BLOCKED/credential", - blocker: missingEnv.includes("OPENAI_API_KEY") ? "credential" : "provider-config", - devLiveReplyPass: false, - reason: "provider_unavailable with missing Code Agent provider env means the provider contract is incomplete" - }; - } - +function summarizePayload(payload, options = {}) { + const responseSummary = summarizeCodeAgentPayload(payload, options); + const assistantReply = responseSummary?.hasReply === true && typeof payload?.reply?.content === "string" + ? payload.reply.content.trim() + : ""; return { - status: "blocked", - level: "BLOCKED", - blocker: "runtime", - devLiveReplyPass: false, - reason: `Code Agent chat did not produce a completed non-empty assistant reply${httpStatus ? ` (HTTP ${httpStatus})` : ""}` - }; -} - -function inspectRealCompletionEvidence(payload) { - const provider = String(payload?.provider ?? "").trim().toLowerCase(); - const backend = String(payload?.backend ?? "").trim().toLowerCase(); - const missing = []; - if (!payload?.conversationId && !payload?.sessionId) missing.push("conversationId"); - if (!payload?.traceId) missing.push("traceId"); - if (!payload?.messageId) missing.push("messageId"); - if (!payload?.provider) missing.push("provider"); - if (!payload?.model) missing.push("model"); - if (!payload?.backend) missing.push("backend"); - if (missing.length > 0) { - return { ok: false, reason: `missing ${missing.join(",")}` }; - } - if (!trustedLiveProviders.has(provider)) { - return { ok: false, reason: `provider ${payload.provider} is not an accepted real provider` }; - } - if (untrustedProviderPattern.test(`${provider} ${backend}`)) { - return { ok: false, reason: `provider/backend looks non-real: ${payload.provider}/${payload.backend}` }; - } - return { ok: true, reason: "provider/model/trace/conversation evidence present" }; -} - -function summarizePayload(payload) { - const assistantReply = typeof payload?.reply?.content === "string" ? payload.reply.content.trim() : ""; - const error = payload?.error - ? { - code: payload.error.code, - missingEnv: payload.error.missingEnv, - missingCommands: payload.error.missingCommands, - providerStatus: payload.error.providerStatus - } - : null; - return { - status: payload?.status ?? null, - provider: payload?.provider ?? null, - model: payload?.model ?? null, - backend: payload?.backend ?? null, - assistantReplyNonEmpty: assistantReply.length > 0, + status: responseSummary?.status ?? null, + provider: responseSummary?.provider ?? null, + model: responseSummary?.model ?? null, + backend: responseSummary?.backend ?? null, + assistantReplyNonEmpty: responseSummary?.hasReply === true, assistantReplyLength: assistantReply.length, - error + error: responseSummary?.error ?? null }; } diff --git a/scripts/dev-cloud-workbench-smoke.test.mjs b/scripts/dev-cloud-workbench-smoke.test.mjs index 4e6fac52..04982a53 100644 --- a/scripts/dev-cloud-workbench-smoke.test.mjs +++ b/scripts/dev-cloud-workbench-smoke.test.mjs @@ -2,9 +2,14 @@ import assert from "node:assert/strict"; import test from "node:test"; import { + classifyCodeAgentBrowserJourney, classifyLiveDeploymentIdentity, - classifyLiveWebAssetIdentity + classifyLiveWebAssetIdentity, + sanitizeAgentChatBody } from "./src/dev-cloud-workbench-smoke-lib.mjs"; +import { + classifyCodeAgentChatReadiness +} from "./src/code-agent-response-contract.mjs"; const sourceIdentity = Object.freeze({ status: "observed", @@ -100,3 +105,133 @@ test("live web asset identity blocks stale deployed primary assets", () => { assert.deepEqual(stale.mismatches, ["styles.css", "app.mjs"]); assert.match(stale.summary, /Deployment drift/u); }); + +test("Code Agent browser classifier blocks provider 502 response shapes", () => { + const summary = sanitizeAgentChatBody({ + status: "completed", + provider: "openai-responses", + model: "gpt-5.5", + backend: "hwlab-cloud-api/openai-responses", + traceId: "trc_provider_502", + reply: { + content: "must not count" + }, + error: { + code: "provider_unavailable", + message: "OpenAI Responses returned HTTP 502: request rejected", + providerStatus: 502, + missingEnv: [] + } + }, { httpStatus: 502 }); + + assert.equal(summary.status, "failed"); + assert.equal(summary.hasReply, false); + assert.equal(summary.error.code, "provider_unavailable"); + assert.equal(summary.error.providerStatus, 502); + + const classification = classifyCodeAgentBrowserJourney({ + responseSummary: summary, + httpOk: false, + httpStatus: 502, + ui: { + agentChatStatus: "DEV-LIVE 回复", + completedMessageVisible: true, + failedMessageVisible: false + } + }); + assert.equal(classification.status, "blocked"); + assert.equal(classification.blocker, "provider-upstream"); +}); + +test("Code Agent readiness classifier blocks completed payloads over any non-2xx HTTP status", () => { + const completedPayload = { + conversationId: "cnv_completed_non_2xx", + sessionId: "cnv_completed_non_2xx", + messageId: "msg_completed_non_2xx", + status: "completed", + provider: "openai-responses", + model: "gpt-5.5", + backend: "hwlab-cloud-api/openai-responses", + traceId: "trc_completed_non_2xx", + reply: { + content: "must only count over HTTP 2xx" + } + }; + + const accepted = classifyCodeAgentChatReadiness(completedPayload, { realDevLive: true, httpStatus: 200 }); + assert.equal(accepted.status, "pass"); + assert.equal(accepted.devLiveReplyPass, true); + + for (const httpStatus of [400, 401, 403, 429, 500, 502, 503, 504]) { + const blocked = classifyCodeAgentChatReadiness(completedPayload, { realDevLive: true, httpStatus }); + assert.equal(blocked.status, "blocked", `HTTP ${httpStatus} must block`); + assert.equal(blocked.blocker, "provider-upstream", `HTTP ${httpStatus} blocker`); + assert.equal(blocked.devLiveReplyPass, false, `HTTP ${httpStatus} must not pass DEV-LIVE`); + assert.equal(blocked.providerStatus, httpStatus, `HTTP ${httpStatus} providerStatus`); + assert.match(blocked.reason, /HTTP non-2xx|upstream response/u, `HTTP ${httpStatus} reason`); + } +}); + +test("Code Agent browser classifier accepts only completed HTTP success plus completed UI and real provider evidence", () => { + const summary = sanitizeAgentChatBody({ + status: "completed", + provider: "openai-responses", + model: "gpt-5.5", + backend: "hwlab-cloud-api/openai-responses", + traceId: "trc_completed", + reply: { + content: "ok" + } + }, { httpStatus: 200 }); + + const pass = classifyCodeAgentBrowserJourney({ + responseSummary: summary, + httpOk: true, + httpStatus: 200, + ui: { + agentChatStatus: "DEV-LIVE 回复", + completedMessageVisible: true, + failedMessageVisible: false + } + }); + assert.equal(pass.status, "pass"); + + const blocked = classifyCodeAgentBrowserJourney({ + responseSummary: summary, + httpOk: false, + httpStatus: 500, + ui: { + agentChatStatus: "DEV-LIVE 回复", + completedMessageVisible: true, + failedMessageVisible: false + } + }); + assert.equal(blocked.status, "blocked"); + assert.equal(blocked.blocker, "provider-upstream"); +}); + +test("Code Agent browser classifier blocks completed payloads without backend evidence", () => { + const summary = sanitizeAgentChatBody({ + status: "completed", + provider: "openai-responses", + model: "gpt-5.5", + traceId: "trc_missing_backend", + reply: { + content: "must not count" + } + }, { httpStatus: 200 }); + + const classification = classifyCodeAgentBrowserJourney({ + responseSummary: summary, + httpOk: true, + httpStatus: 200, + ui: { + agentChatStatus: "DEV-LIVE 回复", + completedMessageVisible: true, + failedMessageVisible: false + } + }); + + assert.equal(classification.status, "blocked"); + assert.equal(classification.blocker, "untrusted-completion"); +}); diff --git a/scripts/src/code-agent-response-contract.mjs b/scripts/src/code-agent-response-contract.mjs new file mode 100644 index 00000000..a24e2976 --- /dev/null +++ b/scripts/src/code-agent-response-contract.mjs @@ -0,0 +1,302 @@ +const trustedLiveProviders = new Set(["openai-responses", "codex-cli"]); +const untrustedProviderPattern = /\b(?:echo|mock|fixture|stub|sample)\b/iu; + +export function classifyCodeAgentChatReadiness(payload, { realDevLive = false, httpStatus = null } = {}) { + const providerBlock = classifyCodeAgentProviderBlock(payload, { httpStatus }); + if (providerBlock.blocked) { + return { + status: "blocked", + level: providerBlock.level, + blocker: providerBlock.blocker, + devLiveReplyPass: false, + reason: providerBlock.reason, + ...(providerBlock.providerStatus != null ? { providerStatus: providerBlock.providerStatus } : {}) + }; + } + + const assistantReply = typeof payload?.reply?.content === "string" ? payload.reply.content.trim() : ""; + if (payload?.status === "completed" && assistantReply) { + const evidence = inspectRealCompletionEvidence(payload); + if (!evidence.ok) { + return { + status: "blocked", + level: realDevLive ? "BLOCKED/untrusted-completion" : "SOURCE", + blocker: "untrusted-completion", + devLiveReplyPass: false, + reason: `completed response is missing real provider/model/trace/conversation evidence or looks like echo/mock/stub: ${evidence.reason}` + }; + } + if (realDevLive) { + return { + status: "pass", + level: "#143 DEV-LIVE reply pass", + blocker: null, + devLiveReplyPass: true, + reason: "real DEV /v1/agent/chat completed with a non-empty assistant reply" + }; + } + return { + status: "blocked", + level: "SOURCE", + blocker: "not-dev-live", + devLiveReplyPass: false, + reason: "mock, fixture, local stub, or source-only completion cannot be counted as #143 DEV-LIVE reply pass" + }; + } + + const suffix = httpStatus ? ` (HTTP ${httpStatus})` : ""; + return { + status: "blocked", + level: "BLOCKED", + blocker: "runtime", + devLiveReplyPass: false, + reason: `Code Agent chat did not produce a completed non-empty assistant reply${suffix}` + }; +} + +export function classifyCodeAgentProviderBlock(payload, { httpStatus = null } = {}) { + const payloadStatus = stringOrNull(payload?.status); + const errorCode = stringOrNull(payload?.error?.code); + const availabilityStatus = stringOrNull(payload?.availability?.status); + const missingEnv = Array.isArray(payload?.error?.missingEnv) + ? payload.error.missingEnv.filter((item) => typeof item === "string") + : []; + const providerStatus = numericStatus(payload?.error?.providerStatus); + const responseStatus = numericStatus(httpStatus); + const providerHttpBlocked = isHttpNon2xxStatus(providerStatus); + const responseHttpBlocked = isHttpNon2xxStatus(responseStatus); + const httpBlocked = providerHttpBlocked || responseHttpBlocked; + const blockingHttpStatus = providerHttpBlocked ? providerStatus : responseHttpBlocked ? responseStatus : providerStatus ?? responseStatus; + const providerUnavailable = + errorCode === "provider_unavailable" || + availabilityStatus === "blocked" || + payloadStatus === "blocked" || + httpBlocked; + + if (!providerUnavailable) { + return { blocked: false }; + } + + if (httpBlocked) { + return { + blocked: true, + level: "BLOCKED/provider", + blocker: "provider-upstream", + providerStatus: blockingHttpStatus, + missingEnv, + reason: `Code Agent HTTP non-2xx status ${blockingHttpStatus} means the upstream response is blocked, not completed` + }; + } + + if (missingEnv.includes("OPENAI_API_KEY")) { + return { + blocked: true, + level: "BLOCKED/credential", + blocker: "credential", + providerStatus: blockingHttpStatus, + missingEnv, + reason: "provider_unavailable with missing OPENAI_API_KEY means the provider credential contract is incomplete" + }; + } + + if (missingEnv.includes("HWLAB_CODE_AGENT_OPENAI_BASE_URL")) { + return { + blocked: true, + level: "BLOCKED/provider-config", + blocker: "provider-config", + providerStatus: blockingHttpStatus, + missingEnv, + reason: "provider_unavailable with missing HWLAB_CODE_AGENT_OPENAI_BASE_URL means the provider egress/base-url contract is incomplete" + }; + } + + return { + blocked: true, + level: "BLOCKED/provider", + blocker: "provider-unavailable", + providerStatus: blockingHttpStatus, + missingEnv, + reason: "provider_unavailable or availability.status=blocked means the provider path is blocked, not completed" + }; +} + +export function summarizeCodeAgentPayload(payload, { httpStatus = null, traceId = null } = {}) { + const providerBlock = classifyCodeAgentProviderBlock(payload, { httpStatus }); + const hasObjectPayload = payload && typeof payload === "object" && !Array.isArray(payload); + + if (!hasObjectPayload) { + if (!providerBlock.blocked) return null; + return { + status: "failed", + sourceKind: null, + evidenceLevel: null, + provider: "unknown", + model: "unknown", + backend: "unknown", + traceId: traceId || "unknown", + hasReply: false, + error: { + code: "provider_unavailable", + missingEnv: [], + missingCommands: [], + providerStatus: providerBlock.providerStatus + } + }; + } + + const error = normalizeCodeAgentError(payload.error); + const assistantReply = typeof payload.reply?.content === "string" ? payload.reply.content.trim() : ""; + const provider = stringOrNull(payload.provider) ?? (providerBlock.blocked ? "unknown" : null); + const model = stringOrNull(payload.model) ?? (providerBlock.blocked ? "unknown" : null); + const backend = stringOrNull(payload.backend) ?? (providerBlock.blocked ? "unknown" : null); + const observedTraceId = stringOrNull(payload.traceId) ?? traceId ?? (providerBlock.blocked ? "unknown" : null); + + if (providerBlock.blocked) { + const blockedError = error + ? { + ...error, + code: error.code ?? "provider_unavailable", + providerStatus: error.providerStatus ?? providerBlock.providerStatus + } + : { + code: "provider_unavailable", + missingEnv: providerBlock.missingEnv ?? [], + missingCommands: [], + providerStatus: providerBlock.providerStatus + }; + return { + status: "failed", + sourceKind: stringOrNull(payload.sourceKind), + evidenceLevel: stringOrNull(payload.evidenceLevel), + provider, + model, + backend, + traceId: observedTraceId, + hasReply: false, + error: blockedError + }; + } + + return { + status: stringOrNull(payload.status), + sourceKind: stringOrNull(payload.sourceKind), + evidenceLevel: stringOrNull(payload.evidenceLevel), + provider, + model, + backend, + traceId: observedTraceId, + hasReply: assistantReply.length > 0, + error + }; +} + +export function classifyCodeAgentBrowserJourney({ responseSummary, httpOk = false, httpStatus = null, ui = null } = {}) { + const providerBlock = classifyCodeAgentProviderBlock(responseSummary, { httpStatus }); + const uiCompleted = ui?.agentChatStatus === "DEV-LIVE 回复" || ui?.agentChatStatus === "已回复" || ui?.completedMessageVisible === true; + const uiFailed = ui?.failedMessageVisible === true || ui?.agentChatStatus === "发送失败" || ui?.agentChatStatus === "BLOCKED 凭证缺口"; + const evidence = inspectSanitizedCompletionEvidence(responseSummary); + const backendCompleted = + httpOk && + responseSummary?.status === "completed" && + responseSummary?.hasReply === true && + responseSummary.error === null && + evidence.ok; + + if (backendCompleted && uiCompleted && !uiFailed) { + return { + status: "pass", + blocker: null, + reason: "completed Code Agent payload over HTTP 2xx with real provider/backend/model/trace/reply evidence and completed UI state" + }; + } + + if (providerBlock.blocked) { + return { + status: "blocked", + blocker: providerBlock.blocker, + reason: providerBlock.reason, + ...(providerBlock.providerStatus != null ? { providerStatus: providerBlock.providerStatus } : {}) + }; + } + + if (responseSummary?.status === "completed") { + return { + status: "blocked", + blocker: "untrusted-completion", + reason: `Code Agent completed-looking payload was not paired with HTTP success, completed UI, and real provider evidence: ${evidence.reason}` + }; + } + + return { + status: "blocked", + blocker: "runtime", + reason: "Code Agent browser journey did not produce a completed non-error response" + }; +} + +export function inspectRealCompletionEvidence(payload) { + const provider = String(payload?.provider ?? "").trim().toLowerCase(); + const backend = String(payload?.backend ?? "").trim().toLowerCase(); + const missing = []; + if (!payload?.conversationId && !payload?.sessionId) missing.push("conversationId"); + if (!payload?.traceId) missing.push("traceId"); + if (!payload?.messageId) missing.push("messageId"); + if (!payload?.provider) missing.push("provider"); + if (!payload?.model) missing.push("model"); + if (!payload?.backend) missing.push("backend"); + if (missing.length > 0) { + return { ok: false, reason: `missing ${missing.join(",")}` }; + } + if (!trustedLiveProviders.has(provider)) { + return { ok: false, reason: `provider ${payload.provider} is not an accepted real provider` }; + } + if (untrustedProviderPattern.test(`${provider} ${backend}`)) { + return { ok: false, reason: `provider/backend looks non-real: ${payload.provider}/${payload.backend}` }; + } + return { ok: true, reason: "provider/model/trace/conversation evidence present" }; +} + +export function normalizeCodeAgentError(error) { + if (!error || typeof error !== "object") return null; + return { + code: stringOrNull(error.code), + missingEnv: Array.isArray(error.missingEnv) ? error.missingEnv.filter((item) => typeof item === "string") : [], + missingCommands: Array.isArray(error.missingCommands) ? error.missingCommands.filter((item) => typeof item === "string") : [], + providerStatus: numericStatus(error.providerStatus) + }; +} + +export function isHttpNon2xxStatus(value) { + const status = numericStatus(value); + return status !== null && (status < 200 || status > 299); +} + +function inspectSanitizedCompletionEvidence(summary) { + const provider = String(summary?.provider ?? "").trim().toLowerCase(); + const backend = String(summary?.backend ?? "").trim().toLowerCase(); + const missing = []; + if (!summary?.provider) missing.push("provider"); + if (!summary?.model) missing.push("model"); + if (!summary?.backend) missing.push("backend"); + if (!summary?.traceId) missing.push("traceId"); + if (summary?.hasReply !== true) missing.push("reply"); + if (missing.length > 0) { + return { ok: false, reason: `missing ${missing.join(",")}` }; + } + if (!trustedLiveProviders.has(provider)) { + return { ok: false, reason: `provider ${summary.provider} is not an accepted real provider` }; + } + if (untrustedProviderPattern.test(`${provider} ${backend}`)) { + return { ok: false, reason: `provider/backend looks non-real: ${summary.provider}/${summary.backend}` }; + } + return { ok: true, reason: "real provider/backend/model/trace/reply evidence present" }; +} + +function stringOrNull(value) { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +function numericStatus(value) { + if (typeof value !== "number" || !Number.isInteger(value)) return null; + return value; +} diff --git a/scripts/src/dev-cloud-workbench-smoke-lib.mjs b/scripts/src/dev-cloud-workbench-smoke-lib.mjs index f8ce7cd7..92409afb 100644 --- a/scripts/src/dev-cloud-workbench-smoke-lib.mjs +++ b/scripts/src/dev-cloud-workbench-smoke-lib.mjs @@ -7,6 +7,12 @@ import { fileURLToPath } from "node:url"; import { gateSummary } from "../../web/hwlab-cloud-web/gate-summary.mjs"; import { runtime } from "../../web/hwlab-cloud-web/runtime.mjs"; +import { + classifyCodeAgentBrowserJourney, + summarizeCodeAgentPayload +} from "./code-agent-response-contract.mjs"; + +export { classifyCodeAgentBrowserJourney }; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); const webRoot = path.join(repoRoot, "web/hwlab-cloud-web"); @@ -510,7 +516,7 @@ async function runLiveSmoke(args) { hardwareWriteApis: false, sourceIsDevLive: false, liveMode: "browser-user-journey-with-code-agent-post", - retainedApiFields: ["status", "provider", "model", "backend", "traceId", "hasReply", "error.code", "error.missingEnv"], + retainedApiFields: ["status", "provider", "model", "backend", "traceId", "hasReply", "error.code", "error.missingEnv", "error.providerStatus"], statement: "Live smoke opens the deployed workbench and sends one controlled Code Agent message; it does not call hardware write APIs and must not be used to infer M3 DEV-LIVE hardware-loop acceptance." } }); @@ -1667,7 +1673,7 @@ async function inspectLiveUserJourney(url) { method: response.request().method(), status: response.status(), urlPath: new URL(response.url()).pathname, - body: sanitizeAgentChatBody(body) + body: sanitizeAgentChatBody(body, { httpStatus: response.status() }) }); }); @@ -1698,28 +1704,30 @@ async function inspectLiveUserJourney(url) { } await page.waitForTimeout(500); const ui = await inspectJourneyUi(page); - const responseSummary = sanitizeAgentChatBody(responseBody); - const backendCompleted = - response.ok() && - responseSummary?.status === "completed" && - Boolean(responseSummary.provider) && - Boolean(responseSummary.model) && - Boolean(responseSummary.traceId); - const uiCompleted = ui.agentChatStatus === "DEV-LIVE 回复" || ui.completedMessageVisible; - const pass = backendCompleted && uiCompleted && !ui.failedMessageVisible; + const responseSummary = sanitizeAgentChatBody(responseBody, { httpStatus: response.status() }); + const classification = classifyCodeAgentBrowserJourney({ + responseSummary, + httpOk: response.ok(), + httpStatus: response.status(), + ui + }); + const pass = classification.status === "pass"; return { status: pass ? "pass" : "blocked", summary: pass ? "Deployed browser journey opened the workbench, sent a Code Agent message, and observed a completed UI response." - : "Deployed browser journey did not observe a completed non-error Code Agent response.", + : classification.blocker === "provider-upstream" || classification.blocker === "provider-unavailable" + ? "Deployed browser journey reached Code Agent, but the provider returned a blocked/unavailable state instead of a completed reply." + : "Deployed browser journey did not observe a completed non-error Code Agent response.", evidence: [ `POST /v1/agent/chat HTTP ${response.status()}`, `status=${responseSummary?.status ?? "unknown"}`, `provider=${responseSummary?.provider ?? "unknown"}`, `model=${responseSummary?.model ?? "unknown"}`, `traceId=${responseSummary?.traceId ?? "unknown"}`, - `uiStatus=${ui.agentChatStatus ?? "unknown"}` + `uiStatus=${ui.agentChatStatus ?? "unknown"}`, + `blocker=${classification.blocker ?? "none"}` ], observations: { controls, @@ -1729,6 +1737,7 @@ async function inspectLiveUserJourney(url) { urlPath: new URL(response.url()).pathname }, response: responseSummary, + classification, ui, networkEvents: agentResponses } @@ -1811,24 +1820,8 @@ async function inspectDefaultApiStatus(page) { }); } -function sanitizeAgentChatBody(body) { - if (!body || typeof body !== "object") return null; - return { - status: typeof body.status === "string" ? body.status : null, - sourceKind: typeof body.sourceKind === "string" ? body.sourceKind : null, - evidenceLevel: typeof body.evidenceLevel === "string" ? body.evidenceLevel : null, - provider: typeof body.provider === "string" ? body.provider : null, - model: typeof body.model === "string" ? body.model : null, - backend: typeof body.backend === "string" ? body.backend : null, - traceId: typeof body.traceId === "string" ? body.traceId : null, - hasReply: Boolean(body.reply?.content), - error: body.error && typeof body.error === "object" - ? { - code: typeof body.error.code === "string" ? body.error.code : null, - missingEnv: Array.isArray(body.error.missingEnv) ? body.error.missingEnv.filter((item) => typeof item === "string") : [] - } - : null - }; +export function sanitizeAgentChatBody(body, options = {}) { + return summarizeCodeAgentPayload(body, options); } export async function runDevCloudWorkbenchLocalAgentFixtureSmoke() { @@ -1867,7 +1860,7 @@ export async function runDevCloudWorkbenchLocalAgentFixtureSmoke() { method: response.request().method(), status: response.status(), urlPath: new URL(response.url()).pathname, - body: sanitizeAgentChatBody(body) + body: sanitizeAgentChatBody(body, { httpStatus: response.status() }) }); }); @@ -1894,7 +1887,7 @@ export async function runDevCloudWorkbenchLocalAgentFixtureSmoke() { { timeout: 8000 } ); const ui = await inspectJourneyUi(page); - const responseSummary = sanitizeAgentChatBody(responseBody); + const responseSummary = sanitizeAgentChatBody(responseBody, { httpStatus: response.status() }); const pass = response.ok() && responseSummary?.status === "completed" && diff --git a/scripts/validate-dev-gate-report-guard.test.mjs b/scripts/validate-dev-gate-report-guard.test.mjs index 68464e36..d314622c 100644 --- a/scripts/validate-dev-gate-report-guard.test.mjs +++ b/scripts/validate-dev-gate-report-guard.test.mjs @@ -103,7 +103,96 @@ test("rejects accepted workbench evidence that mixes provider_unavailable blocke code: "provider_unavailable", missingEnv: ["OPENAI_API_KEY"] }; - await assertRejected(report, /provider_unavailable or OPENAI_API_KEY blockers/); + await assertRejected(report, /provider_unavailable, OPENAI_API_KEY, or providerStatus blockers/); +}); + +test("accepts blocked workbench evidence for provider HTTP 502 without promoting completion", async () => { + const report = cloneBaseReport(); + const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey"); + report.status = "blocked"; + report.evidenceLevel = "BLOCKED"; + report.devLive = false; + report.devPreconditions.status = "blocked"; + report.devPreconditions.summary = "Deployed browser journey is blocked; see checks and blockers."; + report.blockers = [ + { + type: "runtime_blocker", + scope: "live-code-agent-browser-journey", + status: "open", + summary: "Code Agent provider returned provider_unavailable with provider HTTP 502." + } + ]; + report.safety.retainedApiFields = [ + "status", + "provider", + "model", + "backend", + "traceId", + "hasReply", + "error.code", + "error.missingEnv", + "error.providerStatus" + ]; + journey.status = "blocked"; + journey.summary = "Deployed browser journey reached Code Agent, but the provider returned a blocked/unavailable state instead of a completed reply."; + journey.evidence = [ + "POST /v1/agent/chat HTTP 200", + "status=failed", + "provider=openai-responses", + "model=gpt-5.5", + "traceId=trc_provider_502", + "uiStatus=发送失败", + "blocker=provider-upstream" + ]; + journey.observations.request.status = 200; + journey.observations.response = { + status: "failed", + provider: "openai-responses", + model: "gpt-5.5", + backend: "hwlab-cloud-api/openai-responses", + traceId: "trc_provider_502", + hasReply: false, + error: { + code: "provider_unavailable", + missingEnv: [], + providerStatus: 502 + } + }; + journey.observations.classification = { + status: "blocked", + blocker: "provider-upstream", + providerStatus: 502, + reason: "Code Agent HTTP non-2xx status 502 means the upstream response is blocked, not completed" + }; + journey.observations.ui.agentChatStatus = "发送失败"; + journey.observations.ui.inputCleared = false; + journey.observations.ui.completedMessageVisible = false; + journey.observations.ui.failedMessageVisible = true; + journey.observations.networkEvents = [ + { + method: "POST", + status: 200, + urlPath: "/v1/agent/chat", + body: journey.observations.response + } + ]; + + await withReport(report, async (reportPath) => { + const result = runValidator(reportPath); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /validated 1 dev-gate report JSON file/); + }); +}); + +test("rejects accepted workbench evidence that hides providerStatus 502 in network events", async () => { + const report = cloneBaseReport(); + const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey"); + journey.observations.networkEvents[0].body.error = { + code: null, + missingEnv: [], + providerStatus: 502 + }; + await assertRejected(report, /providerStatus blockers/); }); test("rejects accepted workbench evidence that uses the synthetic gpt-5 placeholder model", async () => { diff --git a/scripts/validate-dev-gate-report.mjs b/scripts/validate-dev-gate-report.mjs index 342b9f3d..1c344dd2 100644 --- a/scripts/validate-dev-gate-report.mjs +++ b/scripts/validate-dev-gate-report.mjs @@ -1469,7 +1469,10 @@ async function validateDevCloudWorkbenchLiveReport(report, label, raw) { ); } assertObject(journey.observations.ui, `${label}.live-code-agent-browser-journey.ui`); - assert.equal(journey.observations.ui.agentChatStatus, "已回复", `${label}.live-code-agent-browser-journey.ui.agentChatStatus`); + assert.ok( + ["已回复", "DEV-LIVE 回复"].includes(journey.observations.ui.agentChatStatus), + `${label}.live-code-agent-browser-journey.ui.agentChatStatus` + ); assert.equal(journey.observations.ui.completedMessageVisible, true, `${label}.live-code-agent-browser-journey.ui.completedMessageVisible`); assert.equal(journey.observations.ui.failedMessageVisible, false, `${label}.live-code-agent-browser-journey.ui.failedMessageVisible`); @@ -1480,24 +1483,26 @@ async function validateDevCloudWorkbenchLiveReport(report, label, raw) { assert.equal(report.safety[field], false, `${label}.safety.${field}`); } assert.equal(report.safety.liveMode, "browser-user-journey-with-code-agent-post", `${label}.safety.liveMode`); - assert.deepEqual( - report.safety.retainedApiFields, - ["status", "provider", "model", "backend", "traceId", "hasReply", "error.code", "error.missingEnv"], - `${label}.safety.retainedApiFields` - ); + assertCodeAgentRetainedApiFields(report.safety.retainedApiFields, `${label}.safety.retainedApiFields`); assertString(report.safety.statement, `${label}.safety.statement`); assertFreshDevCloudWorkbenchLiveEvidence(report, label, raw, journey); } function assertDevCloudWorkbenchCodeAgentBlocked(report, label, journey) { const response = journey.observations.response; - assert.equal(response.status, "failed", `${label}.live-code-agent-browser-journey.response.status`); + assert.ok(["failed", "blocked"].includes(response.status), `${label}.live-code-agent-browser-journey.response.status`); assert.equal(response.hasReply, false, `${label}.live-code-agent-browser-journey.response.hasReply`); assertObject(response.error, `${label}.live-code-agent-browser-journey.response.error`); assert.ok( - ["provider_unavailable", "untrusted_completion"].includes(response.error.code), + ["provider_unavailable", "provider_blocked", "untrusted_completion"].includes(response.error.code), `${label}.live-code-agent-browser-journey.response.error.code` ); + if (response.error.providerStatus !== null && response.error.providerStatus !== undefined) { + assert.ok( + Number.isInteger(response.error.providerStatus) && response.error.providerStatus >= 100 && response.error.providerStatus <= 599, + `${label}.live-code-agent-browser-journey.response.error.providerStatus` + ); + } for (const field of ["provider", "model", "backend", "traceId"]) { assertString(response[field], `${label}.live-code-agent-browser-journey.response.${field}`); } @@ -1510,7 +1515,10 @@ function assertDevCloudWorkbenchCodeAgentBlocked(report, label, journey) { } assertObject(journey.observations.ui, `${label}.live-code-agent-browser-journey.ui`); - assert.equal(journey.observations.ui.agentChatStatus, "发送失败", `${label}.live-code-agent-browser-journey.ui.agentChatStatus`); + assert.ok( + ["发送失败", "BLOCKED 凭证缺口"].includes(journey.observations.ui.agentChatStatus), + `${label}.live-code-agent-browser-journey.ui.agentChatStatus` + ); assert.equal(journey.observations.ui.completedMessageVisible, false, `${label}.live-code-agent-browser-journey.ui.completedMessageVisible`); assert.equal(journey.observations.ui.failedMessageVisible, true, `${label}.live-code-agent-browser-journey.ui.failedMessageVisible`); assertBlockers(report.blockers, `${label}.blockers`); @@ -1524,14 +1532,23 @@ function assertDevCloudWorkbenchCodeAgentBlocked(report, label, journey) { assert.equal(report.safety[field], false, `${label}.safety.${field}`); } assert.equal(report.safety.liveMode, "browser-user-journey-with-code-agent-post", `${label}.safety.liveMode`); - assert.deepEqual( - report.safety.retainedApiFields, - ["status", "provider", "model", "backend", "traceId", "hasReply", "error.code", "error.missingEnv"], - `${label}.safety.retainedApiFields` - ); + assertCodeAgentRetainedApiFields(report.safety.retainedApiFields, `${label}.safety.retainedApiFields`); assertString(report.safety.statement, `${label}.safety.statement`); } +function assertCodeAgentRetainedApiFields(value, label) { + assertArray(value, label); + for (const field of ["status", "provider", "model", "backend", "traceId", "hasReply", "error.code", "error.missingEnv"]) { + assert.ok(value.includes(field), `${label} missing ${field}`); + } + for (const field of value) { + assert.ok( + ["status", "provider", "model", "backend", "traceId", "hasReply", "error.code", "error.missingEnv", "error.providerStatus"].includes(field), + `${label} contains unexpected retained field ${field}` + ); + } +} + function assertDevCloudWorkbenchDeploymentPreflight(report, label, checks) { for (const requiredCheck of [ "live-runtime-current-main", @@ -1727,9 +1744,9 @@ function assertFreshDevCloudWorkbenchLiveEvidence(report, label, raw, journey) { const journeyText = JSON.stringify(journey.observations); assert.equal( - /provider_unavailable|OPENAI_API_KEY/u.test(journeyText), + /provider_unavailable|OPENAI_API_KEY|providerStatus|provider HTTP|HTTP 502/u.test(journeyText), false, - `${label}.live-code-agent-browser-journey must not mix provider_unavailable or OPENAI_API_KEY blockers into accepted evidence` + `${label}.live-code-agent-browser-journey must not mix provider_unavailable, OPENAI_API_KEY, or providerStatus blockers into accepted evidence` ); assertAcceptedCodeAgentPayload( diff --git a/web/hwlab-cloud-web/app.mjs b/web/hwlab-cloud-web/app.mjs index b583f83a..aff22325 100644 --- a/web/hwlab-cloud-web/app.mjs +++ b/web/hwlab-cloud-web/app.mjs @@ -396,6 +396,9 @@ async function sendAgentMessage(message, conversationId, traceId = nextProtocolI }); if (!response.ok) { + if (isBlockedAgentResponse(response.data, response.status)) { + return normalizeBlockedAgentResult(response.data, response.status, traceId, response.error); + } const error = new Error(response.error || "Code Agent 请求失败"); error.traceId = traceId; throw error; @@ -441,9 +444,27 @@ async function fetchJson(path, options = {}) { signal: controller.signal }); const text = await response.text(); - const data = text ? JSON.parse(text) : null; + let data = null; + try { + data = text ? JSON.parse(text) : null; + } catch (error) { + if (response.ok) throw error; + return { + ok: false, + path, + status: response.status, + data: null, + error: `${path} HTTP ${response.status}:响应不是 JSON` + }; + } if (!response.ok) { - throw new Error(`${path} HTTP ${response.status}:${messageFromPayload(data)}`); + return { + ok: false, + path, + status: response.status, + data, + error: `${path} HTTP ${response.status}:${messageFromPayload(data)}` + }; } return { ok: true, path, status: response.status, data }; } catch (error) { @@ -1456,7 +1477,9 @@ function classifyCodeAgentCompletion(result) { function failureMessage(result) { if (result?.error?.code === "provider_unavailable" || result?.availability?.status === "blocked") { - const availability = result.availability ?? codeAgentAvailabilityFromResult(result); + const availability = result.error?.providerStatus || result.availability?.status !== "blocked" + ? codeAgentAvailabilityFromResult(result) + : result.availability ?? codeAgentAvailabilityFromResult(result); const missing = Array.isArray(result.error?.missingEnv) && result.error.missingEnv.length > 0 ? ` 当前缺口:${result.error.missingEnv.join("、")}。` : ""; @@ -1612,25 +1635,66 @@ function messageFromPayload(payload) { return payload.error?.message || payload.error?.code || payload.message || JSON.stringify(payload).slice(0, 160); } +function normalizeBlockedAgentResult(payload, httpStatus, traceId, fallbackMessage) { + const { reply, ...safePayload } = payload; + const providerStatus = payload.error?.providerStatus ?? httpStatus; + const error = { + ...(payload.error && typeof payload.error === "object" ? payload.error : {}), + code: payload.error?.code ?? "provider_unavailable", + message: payload.error?.message ?? fallbackMessage ?? `Code Agent provider HTTP ${httpStatus}`, + providerStatus + }; + return { + ...safePayload, + status: "failed", + traceId: payload.traceId || traceId, + error, + availability: payload.availability ?? codeAgentAvailabilityFromResult({ error }) + }; +} + +function isBlockedAgentResponse(payload, httpStatus) { + return Boolean( + payload && + typeof payload === "object" && + ( + isHttpNon2xxStatus(httpStatus) || + isHttpNon2xxStatus(payload.error?.providerStatus) || + payload.status === "blocked" || + payload.error?.code === "provider_unavailable" || + payload.availability?.status === "blocked" + ) + ); +} + +function isHttpNon2xxStatus(value) { + return typeof value === "number" && Number.isInteger(value) && (value < 200 || value > 299); +} + function codeAgentAvailabilityFrom(live) { return live?.restIndex?.data?.codeAgent ?? live?.healthLive?.data?.codeAgent ?? null; } function codeAgentAvailabilityFromResult(result) { + const missingEnv = Array.isArray(result?.error?.missingEnv) ? result.error.missingEnv : []; + const providerStatus = result?.error?.providerStatus; + const credentialBlocked = missingEnv.includes("OPENAI_API_KEY"); return { status: "blocked", - blocker: "凭证缺口", + blocker: credentialBlocked ? "凭证缺口" : "provider_unavailable", reason: result?.error?.code ?? "provider_unavailable", - summary: "真实后端已接入,但当前 DEV provider Secret hwlab-code-agent-provider/openai-api-key 未注入,因此不能返回真实回复。", - missingEnv: result?.error?.missingEnv ?? [], - secretRefs: [ + summary: providerStatus + ? `真实后端已接入,但当前 provider 上游返回 HTTP ${providerStatus};本次保持 BLOCKED/provider_unavailable,不会冒充真实 Code Agent 回复。` + : "真实后端已接入,但当前 DEV provider Secret hwlab-code-agent-provider/openai-api-key 未注入,因此不能返回真实回复。", + missingEnv, + secretRefs: credentialBlocked ? [ { env: "OPENAI_API_KEY", secretName: "hwlab-code-agent-provider", secretKey: "openai-api-key", redacted: true } - ] + ] : [] }; } diff --git a/web/hwlab-cloud-web/scripts/check.mjs b/web/hwlab-cloud-web/scripts/check.mjs index 71de11aa..b01c14d1 100644 --- a/web/hwlab-cloud-web/scripts/check.mjs +++ b/web/hwlab-cloud-web/scripts/check.mjs @@ -320,6 +320,10 @@ assert.doesNotMatch(html, /主流程[\s\S]*添加草稿/u); assert.match(app, /sendAgentMessage/); assert.match(app, /fetchJson\("\/v1\/agent\/chat"/); assert.match(app, /Code Agent 调用失败/); +assert.match(app, /normalizeBlockedAgentResult/); +assert.match(app, /isBlockedAgentResponse/); +assert.match(app, /isHttpNon2xxStatus/); +assert.match(app, /providerStatus/); assert.match(app, /codeAgentAvailability/); assert.match(app, /codeAgentAvailabilityFrom/); assert.match(app, /latestCompletedAgentMessage/);