fix: block code agent non-2xx readiness
This commit is contained in:
Code Queue Review
@@ -7,11 +7,14 @@ import {
|
||||
handleCodeAgentChat,
|
||||
validateCodeAgentChatSchema
|
||||
} from "../internal/cloud/code-agent-chat.mjs";
|
||||
import {
|
||||
classifyCodeAgentChatReadiness,
|
||||
isHttpNon2xxStatus,
|
||||
summarizeCodeAgentPayload
|
||||
} from "./src/code-agent-response-contract.mjs";
|
||||
|
||||
const defaultLiveUrl = "http://74.48.78.17:16666/";
|
||||
const defaultLiveMessage = "请用一句话回复 HWLAB Code Agent readiness。";
|
||||
const trustedLiveProviders = new Set(["openai-responses", "codex-cli"]);
|
||||
const untrustedProviderPattern = /\b(?:echo|mock|fixture|stub|sample)\b/iu;
|
||||
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
|
||||
@@ -129,6 +132,46 @@ async function runLocalContractSmoke() {
|
||||
assert.equal(credentialReadiness.devLiveReplyPass, false);
|
||||
logOk("provider credential blocker readiness");
|
||||
|
||||
const upstreamBlocked = classifyCodeAgentChatReadiness({
|
||||
...failed,
|
||||
error: {
|
||||
code: "provider_unavailable",
|
||||
message: "OpenAI Responses returned HTTP 502: request rejected",
|
||||
providerStatus: 502,
|
||||
missingEnv: []
|
||||
},
|
||||
availability: {
|
||||
status: "available"
|
||||
}
|
||||
}, { realDevLive: true, httpStatus: 200 });
|
||||
assert.equal(upstreamBlocked.status, "blocked");
|
||||
assert.equal(upstreamBlocked.level, "BLOCKED/provider");
|
||||
assert.equal(upstreamBlocked.blocker, "provider-upstream");
|
||||
assert.equal(upstreamBlocked.providerStatus, 502);
|
||||
assert.equal(upstreamBlocked.devLiveReplyPass, false);
|
||||
logOk("provider 502 blocker readiness");
|
||||
|
||||
const completedLivePayload = {
|
||||
...completed,
|
||||
provider: "openai-responses",
|
||||
model: "gpt-5.5",
|
||||
backend: "hwlab-cloud-api/openai-responses"
|
||||
};
|
||||
const completedHttp200Readiness = classifyCodeAgentChatReadiness(completedLivePayload, { realDevLive: true, httpStatus: 200 });
|
||||
assert.equal(completedHttp200Readiness.status, "pass");
|
||||
assert.equal(completedHttp200Readiness.devLiveReplyPass, true);
|
||||
logOk("HTTP 200 completion-looking payload can pass");
|
||||
|
||||
for (const status of [400, 401, 403, 429, 500, 502, 503, 504]) {
|
||||
const readiness = classifyCodeAgentChatReadiness(completedLivePayload, { realDevLive: true, httpStatus: status });
|
||||
assert.equal(readiness.status, "blocked", `HTTP ${status} must block`);
|
||||
assert.equal(readiness.blocker, "provider-upstream", `HTTP ${status} blocker`);
|
||||
assert.equal(readiness.providerStatus, status, `HTTP ${status} providerStatus`);
|
||||
assert.equal(readiness.devLiveReplyPass, false, `HTTP ${status} devLiveReplyPass`);
|
||||
assert.match(readiness.reason, /HTTP non-2xx|upstream response/u, `HTTP ${status} reason`);
|
||||
}
|
||||
logOk("non-2xx completion-looking payloads are blocked");
|
||||
|
||||
process.stdout.write("[code-agent-chat-smoke] passed\n");
|
||||
}
|
||||
|
||||
@@ -158,7 +201,7 @@ async function runLiveReadinessSmoke(args) {
|
||||
transportError = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
const readiness = payload
|
||||
const readiness = payload || isHttpNon2xxStatus(httpStatus)
|
||||
? classifyCodeAgentChatReadiness(payload, { realDevLive: true, httpStatus })
|
||||
: {
|
||||
status: "blocked",
|
||||
@@ -168,7 +211,7 @@ async function runLiveReadinessSmoke(args) {
|
||||
reason: "real DEV /v1/agent/chat did not return a JSON Code Agent payload"
|
||||
};
|
||||
|
||||
if (payload) {
|
||||
if (payload && readiness.status === "pass") {
|
||||
try {
|
||||
validateCodeAgentChatSchema(payload);
|
||||
} catch (error) {
|
||||
@@ -186,7 +229,7 @@ async function runLiveReadinessSmoke(args) {
|
||||
url: endpoint.href,
|
||||
httpStatus,
|
||||
readiness,
|
||||
response: summarizePayload(payload),
|
||||
response: summarizePayload(payload, { httpStatus, traceId }),
|
||||
...(transportError ? { transportError } : {}),
|
||||
safety: {
|
||||
prodTouched: false,
|
||||
@@ -198,101 +241,19 @@ async function runLiveReadinessSmoke(args) {
|
||||
};
|
||||
}
|
||||
|
||||
function classifyCodeAgentChatReadiness(payload, { realDevLive = false, httpStatus = null } = {}) {
|
||||
const assistantReply = typeof payload?.reply?.content === "string" ? payload.reply.content.trim() : "";
|
||||
if (payload?.status === "completed" && assistantReply) {
|
||||
const evidence = inspectRealCompletionEvidence(payload);
|
||||
if (!evidence.ok) {
|
||||
return {
|
||||
status: "blocked",
|
||||
level: realDevLive ? "BLOCKED/untrusted-completion" : "SOURCE",
|
||||
blocker: "untrusted-completion",
|
||||
devLiveReplyPass: false,
|
||||
reason: `completed response is missing real provider/model/trace/conversation evidence or looks like echo/mock/stub: ${evidence.reason}`
|
||||
};
|
||||
}
|
||||
if (realDevLive) {
|
||||
return {
|
||||
status: "pass",
|
||||
level: "#143 DEV-LIVE reply pass",
|
||||
blocker: null,
|
||||
devLiveReplyPass: true,
|
||||
reason: "real DEV /v1/agent/chat completed with a non-empty assistant reply"
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: "blocked",
|
||||
level: "SOURCE",
|
||||
blocker: "not-dev-live",
|
||||
devLiveReplyPass: false,
|
||||
reason: "mock, fixture, local stub, or source-only completion cannot be counted as #143 DEV-LIVE reply pass"
|
||||
};
|
||||
}
|
||||
|
||||
const missingEnv = Array.isArray(payload?.error?.missingEnv) ? payload.error.missingEnv : [];
|
||||
if (
|
||||
payload?.status === "failed" &&
|
||||
payload.error?.code === "provider_unavailable" &&
|
||||
(missingEnv.includes("OPENAI_API_KEY") || missingEnv.includes("HWLAB_CODE_AGENT_OPENAI_BASE_URL"))
|
||||
) {
|
||||
return {
|
||||
status: "blocked",
|
||||
level: "BLOCKED/credential",
|
||||
blocker: missingEnv.includes("OPENAI_API_KEY") ? "credential" : "provider-config",
|
||||
devLiveReplyPass: false,
|
||||
reason: "provider_unavailable with missing Code Agent provider env means the provider contract is incomplete"
|
||||
};
|
||||
}
|
||||
|
||||
function summarizePayload(payload, options = {}) {
|
||||
const responseSummary = summarizeCodeAgentPayload(payload, options);
|
||||
const assistantReply = responseSummary?.hasReply === true && typeof payload?.reply?.content === "string"
|
||||
? payload.reply.content.trim()
|
||||
: "";
|
||||
return {
|
||||
status: "blocked",
|
||||
level: "BLOCKED",
|
||||
blocker: "runtime",
|
||||
devLiveReplyPass: false,
|
||||
reason: `Code Agent chat did not produce a completed non-empty assistant reply${httpStatus ? ` (HTTP ${httpStatus})` : ""}`
|
||||
};
|
||||
}
|
||||
|
||||
function inspectRealCompletionEvidence(payload) {
|
||||
const provider = String(payload?.provider ?? "").trim().toLowerCase();
|
||||
const backend = String(payload?.backend ?? "").trim().toLowerCase();
|
||||
const missing = [];
|
||||
if (!payload?.conversationId && !payload?.sessionId) missing.push("conversationId");
|
||||
if (!payload?.traceId) missing.push("traceId");
|
||||
if (!payload?.messageId) missing.push("messageId");
|
||||
if (!payload?.provider) missing.push("provider");
|
||||
if (!payload?.model) missing.push("model");
|
||||
if (!payload?.backend) missing.push("backend");
|
||||
if (missing.length > 0) {
|
||||
return { ok: false, reason: `missing ${missing.join(",")}` };
|
||||
}
|
||||
if (!trustedLiveProviders.has(provider)) {
|
||||
return { ok: false, reason: `provider ${payload.provider} is not an accepted real provider` };
|
||||
}
|
||||
if (untrustedProviderPattern.test(`${provider} ${backend}`)) {
|
||||
return { ok: false, reason: `provider/backend looks non-real: ${payload.provider}/${payload.backend}` };
|
||||
}
|
||||
return { ok: true, reason: "provider/model/trace/conversation evidence present" };
|
||||
}
|
||||
|
||||
function summarizePayload(payload) {
|
||||
const assistantReply = typeof payload?.reply?.content === "string" ? payload.reply.content.trim() : "";
|
||||
const error = payload?.error
|
||||
? {
|
||||
code: payload.error.code,
|
||||
missingEnv: payload.error.missingEnv,
|
||||
missingCommands: payload.error.missingCommands,
|
||||
providerStatus: payload.error.providerStatus
|
||||
}
|
||||
: null;
|
||||
return {
|
||||
status: payload?.status ?? null,
|
||||
provider: payload?.provider ?? null,
|
||||
model: payload?.model ?? null,
|
||||
backend: payload?.backend ?? null,
|
||||
assistantReplyNonEmpty: assistantReply.length > 0,
|
||||
status: responseSummary?.status ?? null,
|
||||
provider: responseSummary?.provider ?? null,
|
||||
model: responseSummary?.model ?? null,
|
||||
backend: responseSummary?.backend ?? null,
|
||||
assistantReplyNonEmpty: responseSummary?.hasReply === true,
|
||||
assistantReplyLength: assistantReply.length,
|
||||
error
|
||||
error: responseSummary?.error ?? null
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user