fix: block code agent non-2xx readiness
This commit is contained in:
@@ -7,11 +7,14 @@ import {
|
|||||||
handleCodeAgentChat,
|
handleCodeAgentChat,
|
||||||
validateCodeAgentChatSchema
|
validateCodeAgentChatSchema
|
||||||
} from "../internal/cloud/code-agent-chat.mjs";
|
} from "../internal/cloud/code-agent-chat.mjs";
|
||||||
|
import {
|
||||||
|
classifyCodeAgentChatReadiness,
|
||||||
|
isHttpNon2xxStatus,
|
||||||
|
summarizeCodeAgentPayload
|
||||||
|
} from "./src/code-agent-response-contract.mjs";
|
||||||
|
|
||||||
const defaultLiveUrl = "http://74.48.78.17:16666/";
|
const defaultLiveUrl = "http://74.48.78.17:16666/";
|
||||||
const defaultLiveMessage = "请用一句话回复 HWLAB Code Agent readiness。";
|
const defaultLiveMessage = "请用一句话回复 HWLAB Code Agent readiness。";
|
||||||
const trustedLiveProviders = new Set(["openai-responses", "codex-cli"]);
|
|
||||||
const untrustedProviderPattern = /\b(?:echo|mock|fixture|stub|sample)\b/iu;
|
|
||||||
|
|
||||||
const args = parseArgs(process.argv.slice(2));
|
const args = parseArgs(process.argv.slice(2));
|
||||||
|
|
||||||
@@ -129,6 +132,46 @@ async function runLocalContractSmoke() {
|
|||||||
assert.equal(credentialReadiness.devLiveReplyPass, false);
|
assert.equal(credentialReadiness.devLiveReplyPass, false);
|
||||||
logOk("provider credential blocker readiness");
|
logOk("provider credential blocker readiness");
|
||||||
|
|
||||||
|
const upstreamBlocked = classifyCodeAgentChatReadiness({
|
||||||
|
...failed,
|
||||||
|
error: {
|
||||||
|
code: "provider_unavailable",
|
||||||
|
message: "OpenAI Responses returned HTTP 502: request rejected",
|
||||||
|
providerStatus: 502,
|
||||||
|
missingEnv: []
|
||||||
|
},
|
||||||
|
availability: {
|
||||||
|
status: "available"
|
||||||
|
}
|
||||||
|
}, { realDevLive: true, httpStatus: 200 });
|
||||||
|
assert.equal(upstreamBlocked.status, "blocked");
|
||||||
|
assert.equal(upstreamBlocked.level, "BLOCKED/provider");
|
||||||
|
assert.equal(upstreamBlocked.blocker, "provider-upstream");
|
||||||
|
assert.equal(upstreamBlocked.providerStatus, 502);
|
||||||
|
assert.equal(upstreamBlocked.devLiveReplyPass, false);
|
||||||
|
logOk("provider 502 blocker readiness");
|
||||||
|
|
||||||
|
const completedLivePayload = {
|
||||||
|
...completed,
|
||||||
|
provider: "openai-responses",
|
||||||
|
model: "gpt-5.5",
|
||||||
|
backend: "hwlab-cloud-api/openai-responses"
|
||||||
|
};
|
||||||
|
const completedHttp200Readiness = classifyCodeAgentChatReadiness(completedLivePayload, { realDevLive: true, httpStatus: 200 });
|
||||||
|
assert.equal(completedHttp200Readiness.status, "pass");
|
||||||
|
assert.equal(completedHttp200Readiness.devLiveReplyPass, true);
|
||||||
|
logOk("HTTP 200 completion-looking payload can pass");
|
||||||
|
|
||||||
|
for (const status of [400, 401, 403, 429, 500, 502, 503, 504]) {
|
||||||
|
const readiness = classifyCodeAgentChatReadiness(completedLivePayload, { realDevLive: true, httpStatus: status });
|
||||||
|
assert.equal(readiness.status, "blocked", `HTTP ${status} must block`);
|
||||||
|
assert.equal(readiness.blocker, "provider-upstream", `HTTP ${status} blocker`);
|
||||||
|
assert.equal(readiness.providerStatus, status, `HTTP ${status} providerStatus`);
|
||||||
|
assert.equal(readiness.devLiveReplyPass, false, `HTTP ${status} devLiveReplyPass`);
|
||||||
|
assert.match(readiness.reason, /HTTP non-2xx|upstream response/u, `HTTP ${status} reason`);
|
||||||
|
}
|
||||||
|
logOk("non-2xx completion-looking payloads are blocked");
|
||||||
|
|
||||||
process.stdout.write("[code-agent-chat-smoke] passed\n");
|
process.stdout.write("[code-agent-chat-smoke] passed\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -158,7 +201,7 @@ async function runLiveReadinessSmoke(args) {
|
|||||||
transportError = error instanceof Error ? error.message : String(error);
|
transportError = error instanceof Error ? error.message : String(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
const readiness = payload
|
const readiness = payload || isHttpNon2xxStatus(httpStatus)
|
||||||
? classifyCodeAgentChatReadiness(payload, { realDevLive: true, httpStatus })
|
? classifyCodeAgentChatReadiness(payload, { realDevLive: true, httpStatus })
|
||||||
: {
|
: {
|
||||||
status: "blocked",
|
status: "blocked",
|
||||||
@@ -168,7 +211,7 @@ async function runLiveReadinessSmoke(args) {
|
|||||||
reason: "real DEV /v1/agent/chat did not return a JSON Code Agent payload"
|
reason: "real DEV /v1/agent/chat did not return a JSON Code Agent payload"
|
||||||
};
|
};
|
||||||
|
|
||||||
if (payload) {
|
if (payload && readiness.status === "pass") {
|
||||||
try {
|
try {
|
||||||
validateCodeAgentChatSchema(payload);
|
validateCodeAgentChatSchema(payload);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -186,7 +229,7 @@ async function runLiveReadinessSmoke(args) {
|
|||||||
url: endpoint.href,
|
url: endpoint.href,
|
||||||
httpStatus,
|
httpStatus,
|
||||||
readiness,
|
readiness,
|
||||||
response: summarizePayload(payload),
|
response: summarizePayload(payload, { httpStatus, traceId }),
|
||||||
...(transportError ? { transportError } : {}),
|
...(transportError ? { transportError } : {}),
|
||||||
safety: {
|
safety: {
|
||||||
prodTouched: false,
|
prodTouched: false,
|
||||||
@@ -198,101 +241,19 @@ async function runLiveReadinessSmoke(args) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function classifyCodeAgentChatReadiness(payload, { realDevLive = false, httpStatus = null } = {}) {
|
function summarizePayload(payload, options = {}) {
|
||||||
const assistantReply = typeof payload?.reply?.content === "string" ? payload.reply.content.trim() : "";
|
const responseSummary = summarizeCodeAgentPayload(payload, options);
|
||||||
if (payload?.status === "completed" && assistantReply) {
|
const assistantReply = responseSummary?.hasReply === true && typeof payload?.reply?.content === "string"
|
||||||
const evidence = inspectRealCompletionEvidence(payload);
|
? payload.reply.content.trim()
|
||||||
if (!evidence.ok) {
|
: "";
|
||||||
return {
|
|
||||||
status: "blocked",
|
|
||||||
level: realDevLive ? "BLOCKED/untrusted-completion" : "SOURCE",
|
|
||||||
blocker: "untrusted-completion",
|
|
||||||
devLiveReplyPass: false,
|
|
||||||
reason: `completed response is missing real provider/model/trace/conversation evidence or looks like echo/mock/stub: ${evidence.reason}`
|
|
||||||
};
|
|
||||||
}
|
|
||||||
if (realDevLive) {
|
|
||||||
return {
|
|
||||||
status: "pass",
|
|
||||||
level: "#143 DEV-LIVE reply pass",
|
|
||||||
blocker: null,
|
|
||||||
devLiveReplyPass: true,
|
|
||||||
reason: "real DEV /v1/agent/chat completed with a non-empty assistant reply"
|
|
||||||
};
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
status: "blocked",
|
|
||||||
level: "SOURCE",
|
|
||||||
blocker: "not-dev-live",
|
|
||||||
devLiveReplyPass: false,
|
|
||||||
reason: "mock, fixture, local stub, or source-only completion cannot be counted as #143 DEV-LIVE reply pass"
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const missingEnv = Array.isArray(payload?.error?.missingEnv) ? payload.error.missingEnv : [];
|
|
||||||
if (
|
|
||||||
payload?.status === "failed" &&
|
|
||||||
payload.error?.code === "provider_unavailable" &&
|
|
||||||
(missingEnv.includes("OPENAI_API_KEY") || missingEnv.includes("HWLAB_CODE_AGENT_OPENAI_BASE_URL"))
|
|
||||||
) {
|
|
||||||
return {
|
|
||||||
status: "blocked",
|
|
||||||
level: "BLOCKED/credential",
|
|
||||||
blocker: missingEnv.includes("OPENAI_API_KEY") ? "credential" : "provider-config",
|
|
||||||
devLiveReplyPass: false,
|
|
||||||
reason: "provider_unavailable with missing Code Agent provider env means the provider contract is incomplete"
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
status: "blocked",
|
status: responseSummary?.status ?? null,
|
||||||
level: "BLOCKED",
|
provider: responseSummary?.provider ?? null,
|
||||||
blocker: "runtime",
|
model: responseSummary?.model ?? null,
|
||||||
devLiveReplyPass: false,
|
backend: responseSummary?.backend ?? null,
|
||||||
reason: `Code Agent chat did not produce a completed non-empty assistant reply${httpStatus ? ` (HTTP ${httpStatus})` : ""}`
|
assistantReplyNonEmpty: responseSummary?.hasReply === true,
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function inspectRealCompletionEvidence(payload) {
|
|
||||||
const provider = String(payload?.provider ?? "").trim().toLowerCase();
|
|
||||||
const backend = String(payload?.backend ?? "").trim().toLowerCase();
|
|
||||||
const missing = [];
|
|
||||||
if (!payload?.conversationId && !payload?.sessionId) missing.push("conversationId");
|
|
||||||
if (!payload?.traceId) missing.push("traceId");
|
|
||||||
if (!payload?.messageId) missing.push("messageId");
|
|
||||||
if (!payload?.provider) missing.push("provider");
|
|
||||||
if (!payload?.model) missing.push("model");
|
|
||||||
if (!payload?.backend) missing.push("backend");
|
|
||||||
if (missing.length > 0) {
|
|
||||||
return { ok: false, reason: `missing ${missing.join(",")}` };
|
|
||||||
}
|
|
||||||
if (!trustedLiveProviders.has(provider)) {
|
|
||||||
return { ok: false, reason: `provider ${payload.provider} is not an accepted real provider` };
|
|
||||||
}
|
|
||||||
if (untrustedProviderPattern.test(`${provider} ${backend}`)) {
|
|
||||||
return { ok: false, reason: `provider/backend looks non-real: ${payload.provider}/${payload.backend}` };
|
|
||||||
}
|
|
||||||
return { ok: true, reason: "provider/model/trace/conversation evidence present" };
|
|
||||||
}
|
|
||||||
|
|
||||||
function summarizePayload(payload) {
|
|
||||||
const assistantReply = typeof payload?.reply?.content === "string" ? payload.reply.content.trim() : "";
|
|
||||||
const error = payload?.error
|
|
||||||
? {
|
|
||||||
code: payload.error.code,
|
|
||||||
missingEnv: payload.error.missingEnv,
|
|
||||||
missingCommands: payload.error.missingCommands,
|
|
||||||
providerStatus: payload.error.providerStatus
|
|
||||||
}
|
|
||||||
: null;
|
|
||||||
return {
|
|
||||||
status: payload?.status ?? null,
|
|
||||||
provider: payload?.provider ?? null,
|
|
||||||
model: payload?.model ?? null,
|
|
||||||
backend: payload?.backend ?? null,
|
|
||||||
assistantReplyNonEmpty: assistantReply.length > 0,
|
|
||||||
assistantReplyLength: assistantReply.length,
|
assistantReplyLength: assistantReply.length,
|
||||||
error
|
error: responseSummary?.error ?? null
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,9 +2,14 @@ import assert from "node:assert/strict";
|
|||||||
import test from "node:test";
|
import test from "node:test";
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
classifyCodeAgentBrowserJourney,
|
||||||
classifyLiveDeploymentIdentity,
|
classifyLiveDeploymentIdentity,
|
||||||
classifyLiveWebAssetIdentity
|
classifyLiveWebAssetIdentity,
|
||||||
|
sanitizeAgentChatBody
|
||||||
} from "./src/dev-cloud-workbench-smoke-lib.mjs";
|
} from "./src/dev-cloud-workbench-smoke-lib.mjs";
|
||||||
|
import {
|
||||||
|
classifyCodeAgentChatReadiness
|
||||||
|
} from "./src/code-agent-response-contract.mjs";
|
||||||
|
|
||||||
const sourceIdentity = Object.freeze({
|
const sourceIdentity = Object.freeze({
|
||||||
status: "observed",
|
status: "observed",
|
||||||
@@ -100,3 +105,133 @@ test("live web asset identity blocks stale deployed primary assets", () => {
|
|||||||
assert.deepEqual(stale.mismatches, ["styles.css", "app.mjs"]);
|
assert.deepEqual(stale.mismatches, ["styles.css", "app.mjs"]);
|
||||||
assert.match(stale.summary, /Deployment drift/u);
|
assert.match(stale.summary, /Deployment drift/u);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("Code Agent browser classifier blocks provider 502 response shapes", () => {
|
||||||
|
const summary = sanitizeAgentChatBody({
|
||||||
|
status: "completed",
|
||||||
|
provider: "openai-responses",
|
||||||
|
model: "gpt-5.5",
|
||||||
|
backend: "hwlab-cloud-api/openai-responses",
|
||||||
|
traceId: "trc_provider_502",
|
||||||
|
reply: {
|
||||||
|
content: "must not count"
|
||||||
|
},
|
||||||
|
error: {
|
||||||
|
code: "provider_unavailable",
|
||||||
|
message: "OpenAI Responses returned HTTP 502: request rejected",
|
||||||
|
providerStatus: 502,
|
||||||
|
missingEnv: []
|
||||||
|
}
|
||||||
|
}, { httpStatus: 502 });
|
||||||
|
|
||||||
|
assert.equal(summary.status, "failed");
|
||||||
|
assert.equal(summary.hasReply, false);
|
||||||
|
assert.equal(summary.error.code, "provider_unavailable");
|
||||||
|
assert.equal(summary.error.providerStatus, 502);
|
||||||
|
|
||||||
|
const classification = classifyCodeAgentBrowserJourney({
|
||||||
|
responseSummary: summary,
|
||||||
|
httpOk: false,
|
||||||
|
httpStatus: 502,
|
||||||
|
ui: {
|
||||||
|
agentChatStatus: "DEV-LIVE 回复",
|
||||||
|
completedMessageVisible: true,
|
||||||
|
failedMessageVisible: false
|
||||||
|
}
|
||||||
|
});
|
||||||
|
assert.equal(classification.status, "blocked");
|
||||||
|
assert.equal(classification.blocker, "provider-upstream");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Code Agent readiness classifier blocks completed payloads over any non-2xx HTTP status", () => {
|
||||||
|
const completedPayload = {
|
||||||
|
conversationId: "cnv_completed_non_2xx",
|
||||||
|
sessionId: "cnv_completed_non_2xx",
|
||||||
|
messageId: "msg_completed_non_2xx",
|
||||||
|
status: "completed",
|
||||||
|
provider: "openai-responses",
|
||||||
|
model: "gpt-5.5",
|
||||||
|
backend: "hwlab-cloud-api/openai-responses",
|
||||||
|
traceId: "trc_completed_non_2xx",
|
||||||
|
reply: {
|
||||||
|
content: "must only count over HTTP 2xx"
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const accepted = classifyCodeAgentChatReadiness(completedPayload, { realDevLive: true, httpStatus: 200 });
|
||||||
|
assert.equal(accepted.status, "pass");
|
||||||
|
assert.equal(accepted.devLiveReplyPass, true);
|
||||||
|
|
||||||
|
for (const httpStatus of [400, 401, 403, 429, 500, 502, 503, 504]) {
|
||||||
|
const blocked = classifyCodeAgentChatReadiness(completedPayload, { realDevLive: true, httpStatus });
|
||||||
|
assert.equal(blocked.status, "blocked", `HTTP ${httpStatus} must block`);
|
||||||
|
assert.equal(blocked.blocker, "provider-upstream", `HTTP ${httpStatus} blocker`);
|
||||||
|
assert.equal(blocked.devLiveReplyPass, false, `HTTP ${httpStatus} must not pass DEV-LIVE`);
|
||||||
|
assert.equal(blocked.providerStatus, httpStatus, `HTTP ${httpStatus} providerStatus`);
|
||||||
|
assert.match(blocked.reason, /HTTP non-2xx|upstream response/u, `HTTP ${httpStatus} reason`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Code Agent browser classifier accepts only completed HTTP success plus completed UI and real provider evidence", () => {
|
||||||
|
const summary = sanitizeAgentChatBody({
|
||||||
|
status: "completed",
|
||||||
|
provider: "openai-responses",
|
||||||
|
model: "gpt-5.5",
|
||||||
|
backend: "hwlab-cloud-api/openai-responses",
|
||||||
|
traceId: "trc_completed",
|
||||||
|
reply: {
|
||||||
|
content: "ok"
|
||||||
|
}
|
||||||
|
}, { httpStatus: 200 });
|
||||||
|
|
||||||
|
const pass = classifyCodeAgentBrowserJourney({
|
||||||
|
responseSummary: summary,
|
||||||
|
httpOk: true,
|
||||||
|
httpStatus: 200,
|
||||||
|
ui: {
|
||||||
|
agentChatStatus: "DEV-LIVE 回复",
|
||||||
|
completedMessageVisible: true,
|
||||||
|
failedMessageVisible: false
|
||||||
|
}
|
||||||
|
});
|
||||||
|
assert.equal(pass.status, "pass");
|
||||||
|
|
||||||
|
const blocked = classifyCodeAgentBrowserJourney({
|
||||||
|
responseSummary: summary,
|
||||||
|
httpOk: false,
|
||||||
|
httpStatus: 500,
|
||||||
|
ui: {
|
||||||
|
agentChatStatus: "DEV-LIVE 回复",
|
||||||
|
completedMessageVisible: true,
|
||||||
|
failedMessageVisible: false
|
||||||
|
}
|
||||||
|
});
|
||||||
|
assert.equal(blocked.status, "blocked");
|
||||||
|
assert.equal(blocked.blocker, "provider-upstream");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Code Agent browser classifier blocks completed payloads without backend evidence", () => {
|
||||||
|
const summary = sanitizeAgentChatBody({
|
||||||
|
status: "completed",
|
||||||
|
provider: "openai-responses",
|
||||||
|
model: "gpt-5.5",
|
||||||
|
traceId: "trc_missing_backend",
|
||||||
|
reply: {
|
||||||
|
content: "must not count"
|
||||||
|
}
|
||||||
|
}, { httpStatus: 200 });
|
||||||
|
|
||||||
|
const classification = classifyCodeAgentBrowserJourney({
|
||||||
|
responseSummary: summary,
|
||||||
|
httpOk: true,
|
||||||
|
httpStatus: 200,
|
||||||
|
ui: {
|
||||||
|
agentChatStatus: "DEV-LIVE 回复",
|
||||||
|
completedMessageVisible: true,
|
||||||
|
failedMessageVisible: false
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.equal(classification.status, "blocked");
|
||||||
|
assert.equal(classification.blocker, "untrusted-completion");
|
||||||
|
});
|
||||||
|
|||||||
@@ -0,0 +1,302 @@
|
|||||||
|
const trustedLiveProviders = new Set(["openai-responses", "codex-cli"]);
|
||||||
|
const untrustedProviderPattern = /\b(?:echo|mock|fixture|stub|sample)\b/iu;
|
||||||
|
|
||||||
|
export function classifyCodeAgentChatReadiness(payload, { realDevLive = false, httpStatus = null } = {}) {
|
||||||
|
const providerBlock = classifyCodeAgentProviderBlock(payload, { httpStatus });
|
||||||
|
if (providerBlock.blocked) {
|
||||||
|
return {
|
||||||
|
status: "blocked",
|
||||||
|
level: providerBlock.level,
|
||||||
|
blocker: providerBlock.blocker,
|
||||||
|
devLiveReplyPass: false,
|
||||||
|
reason: providerBlock.reason,
|
||||||
|
...(providerBlock.providerStatus != null ? { providerStatus: providerBlock.providerStatus } : {})
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const assistantReply = typeof payload?.reply?.content === "string" ? payload.reply.content.trim() : "";
|
||||||
|
if (payload?.status === "completed" && assistantReply) {
|
||||||
|
const evidence = inspectRealCompletionEvidence(payload);
|
||||||
|
if (!evidence.ok) {
|
||||||
|
return {
|
||||||
|
status: "blocked",
|
||||||
|
level: realDevLive ? "BLOCKED/untrusted-completion" : "SOURCE",
|
||||||
|
blocker: "untrusted-completion",
|
||||||
|
devLiveReplyPass: false,
|
||||||
|
reason: `completed response is missing real provider/model/trace/conversation evidence or looks like echo/mock/stub: ${evidence.reason}`
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (realDevLive) {
|
||||||
|
return {
|
||||||
|
status: "pass",
|
||||||
|
level: "#143 DEV-LIVE reply pass",
|
||||||
|
blocker: null,
|
||||||
|
devLiveReplyPass: true,
|
||||||
|
reason: "real DEV /v1/agent/chat completed with a non-empty assistant reply"
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
status: "blocked",
|
||||||
|
level: "SOURCE",
|
||||||
|
blocker: "not-dev-live",
|
||||||
|
devLiveReplyPass: false,
|
||||||
|
reason: "mock, fixture, local stub, or source-only completion cannot be counted as #143 DEV-LIVE reply pass"
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const suffix = httpStatus ? ` (HTTP ${httpStatus})` : "";
|
||||||
|
return {
|
||||||
|
status: "blocked",
|
||||||
|
level: "BLOCKED",
|
||||||
|
blocker: "runtime",
|
||||||
|
devLiveReplyPass: false,
|
||||||
|
reason: `Code Agent chat did not produce a completed non-empty assistant reply${suffix}`
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function classifyCodeAgentProviderBlock(payload, { httpStatus = null } = {}) {
|
||||||
|
const payloadStatus = stringOrNull(payload?.status);
|
||||||
|
const errorCode = stringOrNull(payload?.error?.code);
|
||||||
|
const availabilityStatus = stringOrNull(payload?.availability?.status);
|
||||||
|
const missingEnv = Array.isArray(payload?.error?.missingEnv)
|
||||||
|
? payload.error.missingEnv.filter((item) => typeof item === "string")
|
||||||
|
: [];
|
||||||
|
const providerStatus = numericStatus(payload?.error?.providerStatus);
|
||||||
|
const responseStatus = numericStatus(httpStatus);
|
||||||
|
const providerHttpBlocked = isHttpNon2xxStatus(providerStatus);
|
||||||
|
const responseHttpBlocked = isHttpNon2xxStatus(responseStatus);
|
||||||
|
const httpBlocked = providerHttpBlocked || responseHttpBlocked;
|
||||||
|
const blockingHttpStatus = providerHttpBlocked ? providerStatus : responseHttpBlocked ? responseStatus : providerStatus ?? responseStatus;
|
||||||
|
const providerUnavailable =
|
||||||
|
errorCode === "provider_unavailable" ||
|
||||||
|
availabilityStatus === "blocked" ||
|
||||||
|
payloadStatus === "blocked" ||
|
||||||
|
httpBlocked;
|
||||||
|
|
||||||
|
if (!providerUnavailable) {
|
||||||
|
return { blocked: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (httpBlocked) {
|
||||||
|
return {
|
||||||
|
blocked: true,
|
||||||
|
level: "BLOCKED/provider",
|
||||||
|
blocker: "provider-upstream",
|
||||||
|
providerStatus: blockingHttpStatus,
|
||||||
|
missingEnv,
|
||||||
|
reason: `Code Agent HTTP non-2xx status ${blockingHttpStatus} means the upstream response is blocked, not completed`
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (missingEnv.includes("OPENAI_API_KEY")) {
|
||||||
|
return {
|
||||||
|
blocked: true,
|
||||||
|
level: "BLOCKED/credential",
|
||||||
|
blocker: "credential",
|
||||||
|
providerStatus: blockingHttpStatus,
|
||||||
|
missingEnv,
|
||||||
|
reason: "provider_unavailable with missing OPENAI_API_KEY means the provider credential contract is incomplete"
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (missingEnv.includes("HWLAB_CODE_AGENT_OPENAI_BASE_URL")) {
|
||||||
|
return {
|
||||||
|
blocked: true,
|
||||||
|
level: "BLOCKED/provider-config",
|
||||||
|
blocker: "provider-config",
|
||||||
|
providerStatus: blockingHttpStatus,
|
||||||
|
missingEnv,
|
||||||
|
reason: "provider_unavailable with missing HWLAB_CODE_AGENT_OPENAI_BASE_URL means the provider egress/base-url contract is incomplete"
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
blocked: true,
|
||||||
|
level: "BLOCKED/provider",
|
||||||
|
blocker: "provider-unavailable",
|
||||||
|
providerStatus: blockingHttpStatus,
|
||||||
|
missingEnv,
|
||||||
|
reason: "provider_unavailable or availability.status=blocked means the provider path is blocked, not completed"
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function summarizeCodeAgentPayload(payload, { httpStatus = null, traceId = null } = {}) {
|
||||||
|
const providerBlock = classifyCodeAgentProviderBlock(payload, { httpStatus });
|
||||||
|
const hasObjectPayload = payload && typeof payload === "object" && !Array.isArray(payload);
|
||||||
|
|
||||||
|
if (!hasObjectPayload) {
|
||||||
|
if (!providerBlock.blocked) return null;
|
||||||
|
return {
|
||||||
|
status: "failed",
|
||||||
|
sourceKind: null,
|
||||||
|
evidenceLevel: null,
|
||||||
|
provider: "unknown",
|
||||||
|
model: "unknown",
|
||||||
|
backend: "unknown",
|
||||||
|
traceId: traceId || "unknown",
|
||||||
|
hasReply: false,
|
||||||
|
error: {
|
||||||
|
code: "provider_unavailable",
|
||||||
|
missingEnv: [],
|
||||||
|
missingCommands: [],
|
||||||
|
providerStatus: providerBlock.providerStatus
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const error = normalizeCodeAgentError(payload.error);
|
||||||
|
const assistantReply = typeof payload.reply?.content === "string" ? payload.reply.content.trim() : "";
|
||||||
|
const provider = stringOrNull(payload.provider) ?? (providerBlock.blocked ? "unknown" : null);
|
||||||
|
const model = stringOrNull(payload.model) ?? (providerBlock.blocked ? "unknown" : null);
|
||||||
|
const backend = stringOrNull(payload.backend) ?? (providerBlock.blocked ? "unknown" : null);
|
||||||
|
const observedTraceId = stringOrNull(payload.traceId) ?? traceId ?? (providerBlock.blocked ? "unknown" : null);
|
||||||
|
|
||||||
|
if (providerBlock.blocked) {
|
||||||
|
const blockedError = error
|
||||||
|
? {
|
||||||
|
...error,
|
||||||
|
code: error.code ?? "provider_unavailable",
|
||||||
|
providerStatus: error.providerStatus ?? providerBlock.providerStatus
|
||||||
|
}
|
||||||
|
: {
|
||||||
|
code: "provider_unavailable",
|
||||||
|
missingEnv: providerBlock.missingEnv ?? [],
|
||||||
|
missingCommands: [],
|
||||||
|
providerStatus: providerBlock.providerStatus
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
status: "failed",
|
||||||
|
sourceKind: stringOrNull(payload.sourceKind),
|
||||||
|
evidenceLevel: stringOrNull(payload.evidenceLevel),
|
||||||
|
provider,
|
||||||
|
model,
|
||||||
|
backend,
|
||||||
|
traceId: observedTraceId,
|
||||||
|
hasReply: false,
|
||||||
|
error: blockedError
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
status: stringOrNull(payload.status),
|
||||||
|
sourceKind: stringOrNull(payload.sourceKind),
|
||||||
|
evidenceLevel: stringOrNull(payload.evidenceLevel),
|
||||||
|
provider,
|
||||||
|
model,
|
||||||
|
backend,
|
||||||
|
traceId: observedTraceId,
|
||||||
|
hasReply: assistantReply.length > 0,
|
||||||
|
error
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function classifyCodeAgentBrowserJourney({ responseSummary, httpOk = false, httpStatus = null, ui = null } = {}) {
|
||||||
|
const providerBlock = classifyCodeAgentProviderBlock(responseSummary, { httpStatus });
|
||||||
|
const uiCompleted = ui?.agentChatStatus === "DEV-LIVE 回复" || ui?.agentChatStatus === "已回复" || ui?.completedMessageVisible === true;
|
||||||
|
const uiFailed = ui?.failedMessageVisible === true || ui?.agentChatStatus === "发送失败" || ui?.agentChatStatus === "BLOCKED 凭证缺口";
|
||||||
|
const evidence = inspectSanitizedCompletionEvidence(responseSummary);
|
||||||
|
const backendCompleted =
|
||||||
|
httpOk &&
|
||||||
|
responseSummary?.status === "completed" &&
|
||||||
|
responseSummary?.hasReply === true &&
|
||||||
|
responseSummary.error === null &&
|
||||||
|
evidence.ok;
|
||||||
|
|
||||||
|
if (backendCompleted && uiCompleted && !uiFailed) {
|
||||||
|
return {
|
||||||
|
status: "pass",
|
||||||
|
blocker: null,
|
||||||
|
reason: "completed Code Agent payload over HTTP 2xx with real provider/backend/model/trace/reply evidence and completed UI state"
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (providerBlock.blocked) {
|
||||||
|
return {
|
||||||
|
status: "blocked",
|
||||||
|
blocker: providerBlock.blocker,
|
||||||
|
reason: providerBlock.reason,
|
||||||
|
...(providerBlock.providerStatus != null ? { providerStatus: providerBlock.providerStatus } : {})
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (responseSummary?.status === "completed") {
|
||||||
|
return {
|
||||||
|
status: "blocked",
|
||||||
|
blocker: "untrusted-completion",
|
||||||
|
reason: `Code Agent completed-looking payload was not paired with HTTP success, completed UI, and real provider evidence: ${evidence.reason}`
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
status: "blocked",
|
||||||
|
blocker: "runtime",
|
||||||
|
reason: "Code Agent browser journey did not produce a completed non-error response"
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function inspectRealCompletionEvidence(payload) {
|
||||||
|
const provider = String(payload?.provider ?? "").trim().toLowerCase();
|
||||||
|
const backend = String(payload?.backend ?? "").trim().toLowerCase();
|
||||||
|
const missing = [];
|
||||||
|
if (!payload?.conversationId && !payload?.sessionId) missing.push("conversationId");
|
||||||
|
if (!payload?.traceId) missing.push("traceId");
|
||||||
|
if (!payload?.messageId) missing.push("messageId");
|
||||||
|
if (!payload?.provider) missing.push("provider");
|
||||||
|
if (!payload?.model) missing.push("model");
|
||||||
|
if (!payload?.backend) missing.push("backend");
|
||||||
|
if (missing.length > 0) {
|
||||||
|
return { ok: false, reason: `missing ${missing.join(",")}` };
|
||||||
|
}
|
||||||
|
if (!trustedLiveProviders.has(provider)) {
|
||||||
|
return { ok: false, reason: `provider ${payload.provider} is not an accepted real provider` };
|
||||||
|
}
|
||||||
|
if (untrustedProviderPattern.test(`${provider} ${backend}`)) {
|
||||||
|
return { ok: false, reason: `provider/backend looks non-real: ${payload.provider}/${payload.backend}` };
|
||||||
|
}
|
||||||
|
return { ok: true, reason: "provider/model/trace/conversation evidence present" };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeCodeAgentError(error) {
|
||||||
|
if (!error || typeof error !== "object") return null;
|
||||||
|
return {
|
||||||
|
code: stringOrNull(error.code),
|
||||||
|
missingEnv: Array.isArray(error.missingEnv) ? error.missingEnv.filter((item) => typeof item === "string") : [],
|
||||||
|
missingCommands: Array.isArray(error.missingCommands) ? error.missingCommands.filter((item) => typeof item === "string") : [],
|
||||||
|
providerStatus: numericStatus(error.providerStatus)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isHttpNon2xxStatus(value) {
|
||||||
|
const status = numericStatus(value);
|
||||||
|
return status !== null && (status < 200 || status > 299);
|
||||||
|
}
|
||||||
|
|
||||||
|
function inspectSanitizedCompletionEvidence(summary) {
|
||||||
|
const provider = String(summary?.provider ?? "").trim().toLowerCase();
|
||||||
|
const backend = String(summary?.backend ?? "").trim().toLowerCase();
|
||||||
|
const missing = [];
|
||||||
|
if (!summary?.provider) missing.push("provider");
|
||||||
|
if (!summary?.model) missing.push("model");
|
||||||
|
if (!summary?.backend) missing.push("backend");
|
||||||
|
if (!summary?.traceId) missing.push("traceId");
|
||||||
|
if (summary?.hasReply !== true) missing.push("reply");
|
||||||
|
if (missing.length > 0) {
|
||||||
|
return { ok: false, reason: `missing ${missing.join(",")}` };
|
||||||
|
}
|
||||||
|
if (!trustedLiveProviders.has(provider)) {
|
||||||
|
return { ok: false, reason: `provider ${summary.provider} is not an accepted real provider` };
|
||||||
|
}
|
||||||
|
if (untrustedProviderPattern.test(`${provider} ${backend}`)) {
|
||||||
|
return { ok: false, reason: `provider/backend looks non-real: ${summary.provider}/${summary.backend}` };
|
||||||
|
}
|
||||||
|
return { ok: true, reason: "real provider/backend/model/trace/reply evidence present" };
|
||||||
|
}
|
||||||
|
|
||||||
|
function stringOrNull(value) {
|
||||||
|
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function numericStatus(value) {
|
||||||
|
if (typeof value !== "number" || !Number.isInteger(value)) return null;
|
||||||
|
return value;
|
||||||
|
}
|
||||||
@@ -7,6 +7,12 @@ import { fileURLToPath } from "node:url";
|
|||||||
|
|
||||||
import { gateSummary } from "../../web/hwlab-cloud-web/gate-summary.mjs";
|
import { gateSummary } from "../../web/hwlab-cloud-web/gate-summary.mjs";
|
||||||
import { runtime } from "../../web/hwlab-cloud-web/runtime.mjs";
|
import { runtime } from "../../web/hwlab-cloud-web/runtime.mjs";
|
||||||
|
import {
|
||||||
|
classifyCodeAgentBrowserJourney,
|
||||||
|
summarizeCodeAgentPayload
|
||||||
|
} from "./code-agent-response-contract.mjs";
|
||||||
|
|
||||||
|
export { classifyCodeAgentBrowserJourney };
|
||||||
|
|
||||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||||
const webRoot = path.join(repoRoot, "web/hwlab-cloud-web");
|
const webRoot = path.join(repoRoot, "web/hwlab-cloud-web");
|
||||||
@@ -510,7 +516,7 @@ async function runLiveSmoke(args) {
|
|||||||
hardwareWriteApis: false,
|
hardwareWriteApis: false,
|
||||||
sourceIsDevLive: false,
|
sourceIsDevLive: false,
|
||||||
liveMode: "browser-user-journey-with-code-agent-post",
|
liveMode: "browser-user-journey-with-code-agent-post",
|
||||||
retainedApiFields: ["status", "provider", "model", "backend", "traceId", "hasReply", "error.code", "error.missingEnv"],
|
retainedApiFields: ["status", "provider", "model", "backend", "traceId", "hasReply", "error.code", "error.missingEnv", "error.providerStatus"],
|
||||||
statement: "Live smoke opens the deployed workbench and sends one controlled Code Agent message; it does not call hardware write APIs and must not be used to infer M3 DEV-LIVE hardware-loop acceptance."
|
statement: "Live smoke opens the deployed workbench and sends one controlled Code Agent message; it does not call hardware write APIs and must not be used to infer M3 DEV-LIVE hardware-loop acceptance."
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -1667,7 +1673,7 @@ async function inspectLiveUserJourney(url) {
|
|||||||
method: response.request().method(),
|
method: response.request().method(),
|
||||||
status: response.status(),
|
status: response.status(),
|
||||||
urlPath: new URL(response.url()).pathname,
|
urlPath: new URL(response.url()).pathname,
|
||||||
body: sanitizeAgentChatBody(body)
|
body: sanitizeAgentChatBody(body, { httpStatus: response.status() })
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -1698,28 +1704,30 @@ async function inspectLiveUserJourney(url) {
|
|||||||
}
|
}
|
||||||
await page.waitForTimeout(500);
|
await page.waitForTimeout(500);
|
||||||
const ui = await inspectJourneyUi(page);
|
const ui = await inspectJourneyUi(page);
|
||||||
const responseSummary = sanitizeAgentChatBody(responseBody);
|
const responseSummary = sanitizeAgentChatBody(responseBody, { httpStatus: response.status() });
|
||||||
const backendCompleted =
|
const classification = classifyCodeAgentBrowserJourney({
|
||||||
response.ok() &&
|
responseSummary,
|
||||||
responseSummary?.status === "completed" &&
|
httpOk: response.ok(),
|
||||||
Boolean(responseSummary.provider) &&
|
httpStatus: response.status(),
|
||||||
Boolean(responseSummary.model) &&
|
ui
|
||||||
Boolean(responseSummary.traceId);
|
});
|
||||||
const uiCompleted = ui.agentChatStatus === "DEV-LIVE 回复" || ui.completedMessageVisible;
|
const pass = classification.status === "pass";
|
||||||
const pass = backendCompleted && uiCompleted && !ui.failedMessageVisible;
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
status: pass ? "pass" : "blocked",
|
status: pass ? "pass" : "blocked",
|
||||||
summary: pass
|
summary: pass
|
||||||
? "Deployed browser journey opened the workbench, sent a Code Agent message, and observed a completed UI response."
|
? "Deployed browser journey opened the workbench, sent a Code Agent message, and observed a completed UI response."
|
||||||
: "Deployed browser journey did not observe a completed non-error Code Agent response.",
|
: classification.blocker === "provider-upstream" || classification.blocker === "provider-unavailable"
|
||||||
|
? "Deployed browser journey reached Code Agent, but the provider returned a blocked/unavailable state instead of a completed reply."
|
||||||
|
: "Deployed browser journey did not observe a completed non-error Code Agent response.",
|
||||||
evidence: [
|
evidence: [
|
||||||
`POST /v1/agent/chat HTTP ${response.status()}`,
|
`POST /v1/agent/chat HTTP ${response.status()}`,
|
||||||
`status=${responseSummary?.status ?? "unknown"}`,
|
`status=${responseSummary?.status ?? "unknown"}`,
|
||||||
`provider=${responseSummary?.provider ?? "unknown"}`,
|
`provider=${responseSummary?.provider ?? "unknown"}`,
|
||||||
`model=${responseSummary?.model ?? "unknown"}`,
|
`model=${responseSummary?.model ?? "unknown"}`,
|
||||||
`traceId=${responseSummary?.traceId ?? "unknown"}`,
|
`traceId=${responseSummary?.traceId ?? "unknown"}`,
|
||||||
`uiStatus=${ui.agentChatStatus ?? "unknown"}`
|
`uiStatus=${ui.agentChatStatus ?? "unknown"}`,
|
||||||
|
`blocker=${classification.blocker ?? "none"}`
|
||||||
],
|
],
|
||||||
observations: {
|
observations: {
|
||||||
controls,
|
controls,
|
||||||
@@ -1729,6 +1737,7 @@ async function inspectLiveUserJourney(url) {
|
|||||||
urlPath: new URL(response.url()).pathname
|
urlPath: new URL(response.url()).pathname
|
||||||
},
|
},
|
||||||
response: responseSummary,
|
response: responseSummary,
|
||||||
|
classification,
|
||||||
ui,
|
ui,
|
||||||
networkEvents: agentResponses
|
networkEvents: agentResponses
|
||||||
}
|
}
|
||||||
@@ -1811,24 +1820,8 @@ async function inspectDefaultApiStatus(page) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function sanitizeAgentChatBody(body) {
|
export function sanitizeAgentChatBody(body, options = {}) {
|
||||||
if (!body || typeof body !== "object") return null;
|
return summarizeCodeAgentPayload(body, options);
|
||||||
return {
|
|
||||||
status: typeof body.status === "string" ? body.status : null,
|
|
||||||
sourceKind: typeof body.sourceKind === "string" ? body.sourceKind : null,
|
|
||||||
evidenceLevel: typeof body.evidenceLevel === "string" ? body.evidenceLevel : null,
|
|
||||||
provider: typeof body.provider === "string" ? body.provider : null,
|
|
||||||
model: typeof body.model === "string" ? body.model : null,
|
|
||||||
backend: typeof body.backend === "string" ? body.backend : null,
|
|
||||||
traceId: typeof body.traceId === "string" ? body.traceId : null,
|
|
||||||
hasReply: Boolean(body.reply?.content),
|
|
||||||
error: body.error && typeof body.error === "object"
|
|
||||||
? {
|
|
||||||
code: typeof body.error.code === "string" ? body.error.code : null,
|
|
||||||
missingEnv: Array.isArray(body.error.missingEnv) ? body.error.missingEnv.filter((item) => typeof item === "string") : []
|
|
||||||
}
|
|
||||||
: null
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function runDevCloudWorkbenchLocalAgentFixtureSmoke() {
|
export async function runDevCloudWorkbenchLocalAgentFixtureSmoke() {
|
||||||
@@ -1867,7 +1860,7 @@ export async function runDevCloudWorkbenchLocalAgentFixtureSmoke() {
|
|||||||
method: response.request().method(),
|
method: response.request().method(),
|
||||||
status: response.status(),
|
status: response.status(),
|
||||||
urlPath: new URL(response.url()).pathname,
|
urlPath: new URL(response.url()).pathname,
|
||||||
body: sanitizeAgentChatBody(body)
|
body: sanitizeAgentChatBody(body, { httpStatus: response.status() })
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -1894,7 +1887,7 @@ export async function runDevCloudWorkbenchLocalAgentFixtureSmoke() {
|
|||||||
{ timeout: 8000 }
|
{ timeout: 8000 }
|
||||||
);
|
);
|
||||||
const ui = await inspectJourneyUi(page);
|
const ui = await inspectJourneyUi(page);
|
||||||
const responseSummary = sanitizeAgentChatBody(responseBody);
|
const responseSummary = sanitizeAgentChatBody(responseBody, { httpStatus: response.status() });
|
||||||
const pass =
|
const pass =
|
||||||
response.ok() &&
|
response.ok() &&
|
||||||
responseSummary?.status === "completed" &&
|
responseSummary?.status === "completed" &&
|
||||||
|
|||||||
@@ -103,7 +103,96 @@ test("rejects accepted workbench evidence that mixes provider_unavailable blocke
|
|||||||
code: "provider_unavailable",
|
code: "provider_unavailable",
|
||||||
missingEnv: ["OPENAI_API_KEY"]
|
missingEnv: ["OPENAI_API_KEY"]
|
||||||
};
|
};
|
||||||
await assertRejected(report, /provider_unavailable or OPENAI_API_KEY blockers/);
|
await assertRejected(report, /provider_unavailable, OPENAI_API_KEY, or providerStatus blockers/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("accepts blocked workbench evidence for provider HTTP 502 without promoting completion", async () => {
|
||||||
|
const report = cloneBaseReport();
|
||||||
|
const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
|
||||||
|
report.status = "blocked";
|
||||||
|
report.evidenceLevel = "BLOCKED";
|
||||||
|
report.devLive = false;
|
||||||
|
report.devPreconditions.status = "blocked";
|
||||||
|
report.devPreconditions.summary = "Deployed browser journey is blocked; see checks and blockers.";
|
||||||
|
report.blockers = [
|
||||||
|
{
|
||||||
|
type: "runtime_blocker",
|
||||||
|
scope: "live-code-agent-browser-journey",
|
||||||
|
status: "open",
|
||||||
|
summary: "Code Agent provider returned provider_unavailable with provider HTTP 502."
|
||||||
|
}
|
||||||
|
];
|
||||||
|
report.safety.retainedApiFields = [
|
||||||
|
"status",
|
||||||
|
"provider",
|
||||||
|
"model",
|
||||||
|
"backend",
|
||||||
|
"traceId",
|
||||||
|
"hasReply",
|
||||||
|
"error.code",
|
||||||
|
"error.missingEnv",
|
||||||
|
"error.providerStatus"
|
||||||
|
];
|
||||||
|
journey.status = "blocked";
|
||||||
|
journey.summary = "Deployed browser journey reached Code Agent, but the provider returned a blocked/unavailable state instead of a completed reply.";
|
||||||
|
journey.evidence = [
|
||||||
|
"POST /v1/agent/chat HTTP 200",
|
||||||
|
"status=failed",
|
||||||
|
"provider=openai-responses",
|
||||||
|
"model=gpt-5.5",
|
||||||
|
"traceId=trc_provider_502",
|
||||||
|
"uiStatus=发送失败",
|
||||||
|
"blocker=provider-upstream"
|
||||||
|
];
|
||||||
|
journey.observations.request.status = 200;
|
||||||
|
journey.observations.response = {
|
||||||
|
status: "failed",
|
||||||
|
provider: "openai-responses",
|
||||||
|
model: "gpt-5.5",
|
||||||
|
backend: "hwlab-cloud-api/openai-responses",
|
||||||
|
traceId: "trc_provider_502",
|
||||||
|
hasReply: false,
|
||||||
|
error: {
|
||||||
|
code: "provider_unavailable",
|
||||||
|
missingEnv: [],
|
||||||
|
providerStatus: 502
|
||||||
|
}
|
||||||
|
};
|
||||||
|
journey.observations.classification = {
|
||||||
|
status: "blocked",
|
||||||
|
blocker: "provider-upstream",
|
||||||
|
providerStatus: 502,
|
||||||
|
reason: "Code Agent HTTP non-2xx status 502 means the upstream response is blocked, not completed"
|
||||||
|
};
|
||||||
|
journey.observations.ui.agentChatStatus = "发送失败";
|
||||||
|
journey.observations.ui.inputCleared = false;
|
||||||
|
journey.observations.ui.completedMessageVisible = false;
|
||||||
|
journey.observations.ui.failedMessageVisible = true;
|
||||||
|
journey.observations.networkEvents = [
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
status: 200,
|
||||||
|
urlPath: "/v1/agent/chat",
|
||||||
|
body: journey.observations.response
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
|
await withReport(report, async (reportPath) => {
|
||||||
|
const result = runValidator(reportPath);
|
||||||
|
assert.equal(result.status, 0, result.stderr);
|
||||||
|
assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("rejects accepted workbench evidence that hides providerStatus 502 in network events", async () => {
|
||||||
|
const report = cloneBaseReport();
|
||||||
|
const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
|
||||||
|
journey.observations.networkEvents[0].body.error = {
|
||||||
|
code: null,
|
||||||
|
missingEnv: [],
|
||||||
|
providerStatus: 502
|
||||||
|
};
|
||||||
|
await assertRejected(report, /providerStatus blockers/);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("rejects accepted workbench evidence that uses the synthetic gpt-5 placeholder model", async () => {
|
test("rejects accepted workbench evidence that uses the synthetic gpt-5 placeholder model", async () => {
|
||||||
|
|||||||
@@ -1469,7 +1469,10 @@ async function validateDevCloudWorkbenchLiveReport(report, label, raw) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
assertObject(journey.observations.ui, `${label}.live-code-agent-browser-journey.ui`);
|
assertObject(journey.observations.ui, `${label}.live-code-agent-browser-journey.ui`);
|
||||||
assert.equal(journey.observations.ui.agentChatStatus, "已回复", `${label}.live-code-agent-browser-journey.ui.agentChatStatus`);
|
assert.ok(
|
||||||
|
["已回复", "DEV-LIVE 回复"].includes(journey.observations.ui.agentChatStatus),
|
||||||
|
`${label}.live-code-agent-browser-journey.ui.agentChatStatus`
|
||||||
|
);
|
||||||
assert.equal(journey.observations.ui.completedMessageVisible, true, `${label}.live-code-agent-browser-journey.ui.completedMessageVisible`);
|
assert.equal(journey.observations.ui.completedMessageVisible, true, `${label}.live-code-agent-browser-journey.ui.completedMessageVisible`);
|
||||||
assert.equal(journey.observations.ui.failedMessageVisible, false, `${label}.live-code-agent-browser-journey.ui.failedMessageVisible`);
|
assert.equal(journey.observations.ui.failedMessageVisible, false, `${label}.live-code-agent-browser-journey.ui.failedMessageVisible`);
|
||||||
|
|
||||||
@@ -1480,24 +1483,26 @@ async function validateDevCloudWorkbenchLiveReport(report, label, raw) {
|
|||||||
assert.equal(report.safety[field], false, `${label}.safety.${field}`);
|
assert.equal(report.safety[field], false, `${label}.safety.${field}`);
|
||||||
}
|
}
|
||||||
assert.equal(report.safety.liveMode, "browser-user-journey-with-code-agent-post", `${label}.safety.liveMode`);
|
assert.equal(report.safety.liveMode, "browser-user-journey-with-code-agent-post", `${label}.safety.liveMode`);
|
||||||
assert.deepEqual(
|
assertCodeAgentRetainedApiFields(report.safety.retainedApiFields, `${label}.safety.retainedApiFields`);
|
||||||
report.safety.retainedApiFields,
|
|
||||||
["status", "provider", "model", "backend", "traceId", "hasReply", "error.code", "error.missingEnv"],
|
|
||||||
`${label}.safety.retainedApiFields`
|
|
||||||
);
|
|
||||||
assertString(report.safety.statement, `${label}.safety.statement`);
|
assertString(report.safety.statement, `${label}.safety.statement`);
|
||||||
assertFreshDevCloudWorkbenchLiveEvidence(report, label, raw, journey);
|
assertFreshDevCloudWorkbenchLiveEvidence(report, label, raw, journey);
|
||||||
}
|
}
|
||||||
|
|
||||||
function assertDevCloudWorkbenchCodeAgentBlocked(report, label, journey) {
|
function assertDevCloudWorkbenchCodeAgentBlocked(report, label, journey) {
|
||||||
const response = journey.observations.response;
|
const response = journey.observations.response;
|
||||||
assert.equal(response.status, "failed", `${label}.live-code-agent-browser-journey.response.status`);
|
assert.ok(["failed", "blocked"].includes(response.status), `${label}.live-code-agent-browser-journey.response.status`);
|
||||||
assert.equal(response.hasReply, false, `${label}.live-code-agent-browser-journey.response.hasReply`);
|
assert.equal(response.hasReply, false, `${label}.live-code-agent-browser-journey.response.hasReply`);
|
||||||
assertObject(response.error, `${label}.live-code-agent-browser-journey.response.error`);
|
assertObject(response.error, `${label}.live-code-agent-browser-journey.response.error`);
|
||||||
assert.ok(
|
assert.ok(
|
||||||
["provider_unavailable", "untrusted_completion"].includes(response.error.code),
|
["provider_unavailable", "provider_blocked", "untrusted_completion"].includes(response.error.code),
|
||||||
`${label}.live-code-agent-browser-journey.response.error.code`
|
`${label}.live-code-agent-browser-journey.response.error.code`
|
||||||
);
|
);
|
||||||
|
if (response.error.providerStatus !== null && response.error.providerStatus !== undefined) {
|
||||||
|
assert.ok(
|
||||||
|
Number.isInteger(response.error.providerStatus) && response.error.providerStatus >= 100 && response.error.providerStatus <= 599,
|
||||||
|
`${label}.live-code-agent-browser-journey.response.error.providerStatus`
|
||||||
|
);
|
||||||
|
}
|
||||||
for (const field of ["provider", "model", "backend", "traceId"]) {
|
for (const field of ["provider", "model", "backend", "traceId"]) {
|
||||||
assertString(response[field], `${label}.live-code-agent-browser-journey.response.${field}`);
|
assertString(response[field], `${label}.live-code-agent-browser-journey.response.${field}`);
|
||||||
}
|
}
|
||||||
@@ -1510,7 +1515,10 @@ function assertDevCloudWorkbenchCodeAgentBlocked(report, label, journey) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
assertObject(journey.observations.ui, `${label}.live-code-agent-browser-journey.ui`);
|
assertObject(journey.observations.ui, `${label}.live-code-agent-browser-journey.ui`);
|
||||||
assert.equal(journey.observations.ui.agentChatStatus, "发送失败", `${label}.live-code-agent-browser-journey.ui.agentChatStatus`);
|
assert.ok(
|
||||||
|
["发送失败", "BLOCKED 凭证缺口"].includes(journey.observations.ui.agentChatStatus),
|
||||||
|
`${label}.live-code-agent-browser-journey.ui.agentChatStatus`
|
||||||
|
);
|
||||||
assert.equal(journey.observations.ui.completedMessageVisible, false, `${label}.live-code-agent-browser-journey.ui.completedMessageVisible`);
|
assert.equal(journey.observations.ui.completedMessageVisible, false, `${label}.live-code-agent-browser-journey.ui.completedMessageVisible`);
|
||||||
assert.equal(journey.observations.ui.failedMessageVisible, true, `${label}.live-code-agent-browser-journey.ui.failedMessageVisible`);
|
assert.equal(journey.observations.ui.failedMessageVisible, true, `${label}.live-code-agent-browser-journey.ui.failedMessageVisible`);
|
||||||
assertBlockers(report.blockers, `${label}.blockers`);
|
assertBlockers(report.blockers, `${label}.blockers`);
|
||||||
@@ -1524,14 +1532,23 @@ function assertDevCloudWorkbenchCodeAgentBlocked(report, label, journey) {
|
|||||||
assert.equal(report.safety[field], false, `${label}.safety.${field}`);
|
assert.equal(report.safety[field], false, `${label}.safety.${field}`);
|
||||||
}
|
}
|
||||||
assert.equal(report.safety.liveMode, "browser-user-journey-with-code-agent-post", `${label}.safety.liveMode`);
|
assert.equal(report.safety.liveMode, "browser-user-journey-with-code-agent-post", `${label}.safety.liveMode`);
|
||||||
assert.deepEqual(
|
assertCodeAgentRetainedApiFields(report.safety.retainedApiFields, `${label}.safety.retainedApiFields`);
|
||||||
report.safety.retainedApiFields,
|
|
||||||
["status", "provider", "model", "backend", "traceId", "hasReply", "error.code", "error.missingEnv"],
|
|
||||||
`${label}.safety.retainedApiFields`
|
|
||||||
);
|
|
||||||
assertString(report.safety.statement, `${label}.safety.statement`);
|
assertString(report.safety.statement, `${label}.safety.statement`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function assertCodeAgentRetainedApiFields(value, label) {
|
||||||
|
assertArray(value, label);
|
||||||
|
for (const field of ["status", "provider", "model", "backend", "traceId", "hasReply", "error.code", "error.missingEnv"]) {
|
||||||
|
assert.ok(value.includes(field), `${label} missing ${field}`);
|
||||||
|
}
|
||||||
|
for (const field of value) {
|
||||||
|
assert.ok(
|
||||||
|
["status", "provider", "model", "backend", "traceId", "hasReply", "error.code", "error.missingEnv", "error.providerStatus"].includes(field),
|
||||||
|
`${label} contains unexpected retained field ${field}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function assertDevCloudWorkbenchDeploymentPreflight(report, label, checks) {
|
function assertDevCloudWorkbenchDeploymentPreflight(report, label, checks) {
|
||||||
for (const requiredCheck of [
|
for (const requiredCheck of [
|
||||||
"live-runtime-current-main",
|
"live-runtime-current-main",
|
||||||
@@ -1727,9 +1744,9 @@ function assertFreshDevCloudWorkbenchLiveEvidence(report, label, raw, journey) {
|
|||||||
|
|
||||||
const journeyText = JSON.stringify(journey.observations);
|
const journeyText = JSON.stringify(journey.observations);
|
||||||
assert.equal(
|
assert.equal(
|
||||||
/provider_unavailable|OPENAI_API_KEY/u.test(journeyText),
|
/provider_unavailable|OPENAI_API_KEY|providerStatus|provider HTTP|HTTP 502/u.test(journeyText),
|
||||||
false,
|
false,
|
||||||
`${label}.live-code-agent-browser-journey must not mix provider_unavailable or OPENAI_API_KEY blockers into accepted evidence`
|
`${label}.live-code-agent-browser-journey must not mix provider_unavailable, OPENAI_API_KEY, or providerStatus blockers into accepted evidence`
|
||||||
);
|
);
|
||||||
|
|
||||||
assertAcceptedCodeAgentPayload(
|
assertAcceptedCodeAgentPayload(
|
||||||
|
|||||||
@@ -396,6 +396,9 @@ async function sendAgentMessage(message, conversationId, traceId = nextProtocolI
|
|||||||
});
|
});
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
|
if (isBlockedAgentResponse(response.data, response.status)) {
|
||||||
|
return normalizeBlockedAgentResult(response.data, response.status, traceId, response.error);
|
||||||
|
}
|
||||||
const error = new Error(response.error || "Code Agent 请求失败");
|
const error = new Error(response.error || "Code Agent 请求失败");
|
||||||
error.traceId = traceId;
|
error.traceId = traceId;
|
||||||
throw error;
|
throw error;
|
||||||
@@ -441,9 +444,27 @@ async function fetchJson(path, options = {}) {
|
|||||||
signal: controller.signal
|
signal: controller.signal
|
||||||
});
|
});
|
||||||
const text = await response.text();
|
const text = await response.text();
|
||||||
const data = text ? JSON.parse(text) : null;
|
let data = null;
|
||||||
|
try {
|
||||||
|
data = text ? JSON.parse(text) : null;
|
||||||
|
} catch (error) {
|
||||||
|
if (response.ok) throw error;
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
path,
|
||||||
|
status: response.status,
|
||||||
|
data: null,
|
||||||
|
error: `${path} HTTP ${response.status}:响应不是 JSON`
|
||||||
|
};
|
||||||
|
}
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
throw new Error(`${path} HTTP ${response.status}:${messageFromPayload(data)}`);
|
return {
|
||||||
|
ok: false,
|
||||||
|
path,
|
||||||
|
status: response.status,
|
||||||
|
data,
|
||||||
|
error: `${path} HTTP ${response.status}:${messageFromPayload(data)}`
|
||||||
|
};
|
||||||
}
|
}
|
||||||
return { ok: true, path, status: response.status, data };
|
return { ok: true, path, status: response.status, data };
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -1456,7 +1477,9 @@ function classifyCodeAgentCompletion(result) {
|
|||||||
|
|
||||||
function failureMessage(result) {
|
function failureMessage(result) {
|
||||||
if (result?.error?.code === "provider_unavailable" || result?.availability?.status === "blocked") {
|
if (result?.error?.code === "provider_unavailable" || result?.availability?.status === "blocked") {
|
||||||
const availability = result.availability ?? codeAgentAvailabilityFromResult(result);
|
const availability = result.error?.providerStatus || result.availability?.status !== "blocked"
|
||||||
|
? codeAgentAvailabilityFromResult(result)
|
||||||
|
: result.availability ?? codeAgentAvailabilityFromResult(result);
|
||||||
const missing = Array.isArray(result.error?.missingEnv) && result.error.missingEnv.length > 0
|
const missing = Array.isArray(result.error?.missingEnv) && result.error.missingEnv.length > 0
|
||||||
? ` 当前缺口:${result.error.missingEnv.join("、")}。`
|
? ` 当前缺口:${result.error.missingEnv.join("、")}。`
|
||||||
: "";
|
: "";
|
||||||
@@ -1612,25 +1635,66 @@ function messageFromPayload(payload) {
|
|||||||
return payload.error?.message || payload.error?.code || payload.message || JSON.stringify(payload).slice(0, 160);
|
return payload.error?.message || payload.error?.code || payload.message || JSON.stringify(payload).slice(0, 160);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function normalizeBlockedAgentResult(payload, httpStatus, traceId, fallbackMessage) {
|
||||||
|
const { reply, ...safePayload } = payload;
|
||||||
|
const providerStatus = payload.error?.providerStatus ?? httpStatus;
|
||||||
|
const error = {
|
||||||
|
...(payload.error && typeof payload.error === "object" ? payload.error : {}),
|
||||||
|
code: payload.error?.code ?? "provider_unavailable",
|
||||||
|
message: payload.error?.message ?? fallbackMessage ?? `Code Agent provider HTTP ${httpStatus}`,
|
||||||
|
providerStatus
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
...safePayload,
|
||||||
|
status: "failed",
|
||||||
|
traceId: payload.traceId || traceId,
|
||||||
|
error,
|
||||||
|
availability: payload.availability ?? codeAgentAvailabilityFromResult({ error })
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function isBlockedAgentResponse(payload, httpStatus) {
|
||||||
|
return Boolean(
|
||||||
|
payload &&
|
||||||
|
typeof payload === "object" &&
|
||||||
|
(
|
||||||
|
isHttpNon2xxStatus(httpStatus) ||
|
||||||
|
isHttpNon2xxStatus(payload.error?.providerStatus) ||
|
||||||
|
payload.status === "blocked" ||
|
||||||
|
payload.error?.code === "provider_unavailable" ||
|
||||||
|
payload.availability?.status === "blocked"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isHttpNon2xxStatus(value) {
|
||||||
|
return typeof value === "number" && Number.isInteger(value) && (value < 200 || value > 299);
|
||||||
|
}
|
||||||
|
|
||||||
function codeAgentAvailabilityFrom(live) {
|
function codeAgentAvailabilityFrom(live) {
|
||||||
return live?.restIndex?.data?.codeAgent ?? live?.healthLive?.data?.codeAgent ?? null;
|
return live?.restIndex?.data?.codeAgent ?? live?.healthLive?.data?.codeAgent ?? null;
|
||||||
}
|
}
|
||||||
|
|
||||||
function codeAgentAvailabilityFromResult(result) {
|
function codeAgentAvailabilityFromResult(result) {
|
||||||
|
const missingEnv = Array.isArray(result?.error?.missingEnv) ? result.error.missingEnv : [];
|
||||||
|
const providerStatus = result?.error?.providerStatus;
|
||||||
|
const credentialBlocked = missingEnv.includes("OPENAI_API_KEY");
|
||||||
return {
|
return {
|
||||||
status: "blocked",
|
status: "blocked",
|
||||||
blocker: "凭证缺口",
|
blocker: credentialBlocked ? "凭证缺口" : "provider_unavailable",
|
||||||
reason: result?.error?.code ?? "provider_unavailable",
|
reason: result?.error?.code ?? "provider_unavailable",
|
||||||
summary: "真实后端已接入,但当前 DEV provider Secret hwlab-code-agent-provider/openai-api-key 未注入,因此不能返回真实回复。",
|
summary: providerStatus
|
||||||
missingEnv: result?.error?.missingEnv ?? [],
|
? `真实后端已接入,但当前 provider 上游返回 HTTP ${providerStatus};本次保持 BLOCKED/provider_unavailable,不会冒充真实 Code Agent 回复。`
|
||||||
secretRefs: [
|
: "真实后端已接入,但当前 DEV provider Secret hwlab-code-agent-provider/openai-api-key 未注入,因此不能返回真实回复。",
|
||||||
|
missingEnv,
|
||||||
|
secretRefs: credentialBlocked ? [
|
||||||
{
|
{
|
||||||
env: "OPENAI_API_KEY",
|
env: "OPENAI_API_KEY",
|
||||||
secretName: "hwlab-code-agent-provider",
|
secretName: "hwlab-code-agent-provider",
|
||||||
secretKey: "openai-api-key",
|
secretKey: "openai-api-key",
|
||||||
redacted: true
|
redacted: true
|
||||||
}
|
}
|
||||||
]
|
] : []
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -320,6 +320,10 @@ assert.doesNotMatch(html, /主流程[\s\S]*添加草稿/u);
|
|||||||
assert.match(app, /sendAgentMessage/);
|
assert.match(app, /sendAgentMessage/);
|
||||||
assert.match(app, /fetchJson\("\/v1\/agent\/chat"/);
|
assert.match(app, /fetchJson\("\/v1\/agent\/chat"/);
|
||||||
assert.match(app, /Code Agent 调用失败/);
|
assert.match(app, /Code Agent 调用失败/);
|
||||||
|
assert.match(app, /normalizeBlockedAgentResult/);
|
||||||
|
assert.match(app, /isBlockedAgentResponse/);
|
||||||
|
assert.match(app, /isHttpNon2xxStatus/);
|
||||||
|
assert.match(app, /providerStatus/);
|
||||||
assert.match(app, /codeAgentAvailability/);
|
assert.match(app, /codeAgentAvailability/);
|
||||||
assert.match(app, /codeAgentAvailabilityFrom/);
|
assert.match(app, /codeAgentAvailabilityFrom/);
|
||||||
assert.match(app, /latestCompletedAgentMessage/);
|
assert.match(app, /latestCompletedAgentMessage/);
|
||||||
|
|||||||
Reference in New Issue
Block a user