fix: block code agent non-2xx readiness

2026-05-23 02:39:41 +00:00
parent 892a0276bd
commit 6961baedb7
8 changed files with 722 additions and 157 deletions
@@ -0,0 +1,302 @@
+const trustedLiveProviders = new Set(["openai-responses", "codex-cli"]);
+const untrustedProviderPattern = /\b(?:echo|mock|fixture|stub|sample)\b/iu;
+
+export function classifyCodeAgentChatReadiness(payload, { realDevLive = false, httpStatus = null } = {}) {
+  const providerBlock = classifyCodeAgentProviderBlock(payload, { httpStatus });
+  if (providerBlock.blocked) {
+    return {
+      status: "blocked",
+      level: providerBlock.level,
+      blocker: providerBlock.blocker,
+      devLiveReplyPass: false,
+      reason: providerBlock.reason,
+      ...(providerBlock.providerStatus != null ? { providerStatus: providerBlock.providerStatus } : {})
+    };
+  }
+
+  const assistantReply = typeof payload?.reply?.content === "string" ? payload.reply.content.trim() : "";
+  if (payload?.status === "completed" && assistantReply) {
+    const evidence = inspectRealCompletionEvidence(payload);
+    if (!evidence.ok) {
+      return {
+        status: "blocked",
+        level: realDevLive ? "BLOCKED/untrusted-completion" : "SOURCE",
+        blocker: "untrusted-completion",
+        devLiveReplyPass: false,
+        reason: `completed response is missing real provider/model/trace/conversation evidence or looks like echo/mock/stub: ${evidence.reason}`
+      };
+    }
+    if (realDevLive) {
+      return {
+        status: "pass",
+        level: "#143 DEV-LIVE reply pass",
+        blocker: null,
+        devLiveReplyPass: true,
+        reason: "real DEV /v1/agent/chat completed with a non-empty assistant reply"
+      };
+    }
+    return {
+      status: "blocked",
+      level: "SOURCE",
+      blocker: "not-dev-live",
+      devLiveReplyPass: false,
+      reason: "mock, fixture, local stub, or source-only completion cannot be counted as #143 DEV-LIVE reply pass"
+    };
+  }
+
+  const suffix = httpStatus ? ` (HTTP ${httpStatus})` : "";
+  return {
+    status: "blocked",
+    level: "BLOCKED",
+    blocker: "runtime",
+    devLiveReplyPass: false,
+    reason: `Code Agent chat did not produce a completed non-empty assistant reply${suffix}`
+  };
+}
+
+export function classifyCodeAgentProviderBlock(payload, { httpStatus = null } = {}) {
+  const payloadStatus = stringOrNull(payload?.status);
+  const errorCode = stringOrNull(payload?.error?.code);
+  const availabilityStatus = stringOrNull(payload?.availability?.status);
+  const missingEnv = Array.isArray(payload?.error?.missingEnv)
+    ? payload.error.missingEnv.filter((item) => typeof item === "string")
+    : [];
+  const providerStatus = numericStatus(payload?.error?.providerStatus);
+  const responseStatus = numericStatus(httpStatus);
+  const providerHttpBlocked = isHttpNon2xxStatus(providerStatus);
+  const responseHttpBlocked = isHttpNon2xxStatus(responseStatus);
+  const httpBlocked = providerHttpBlocked || responseHttpBlocked;
+  const blockingHttpStatus = providerHttpBlocked ? providerStatus : responseHttpBlocked ? responseStatus : providerStatus ?? responseStatus;
+  const providerUnavailable =
+    errorCode === "provider_unavailable" ||
+    availabilityStatus === "blocked" ||
+    payloadStatus === "blocked" ||
+    httpBlocked;
+
+  if (!providerUnavailable) {
+    return { blocked: false };
+  }
+
+  if (httpBlocked) {
+    return {
+      blocked: true,
+      level: "BLOCKED/provider",
+      blocker: "provider-upstream",
+      providerStatus: blockingHttpStatus,
+      missingEnv,
+      reason: `Code Agent HTTP non-2xx status ${blockingHttpStatus} means the upstream response is blocked, not completed`
+    };
+  }
+
+  if (missingEnv.includes("OPENAI_API_KEY")) {
+    return {
+      blocked: true,
+      level: "BLOCKED/credential",
+      blocker: "credential",
+      providerStatus: blockingHttpStatus,
+      missingEnv,
+      reason: "provider_unavailable with missing OPENAI_API_KEY means the provider credential contract is incomplete"
+    };
+  }
+
+  if (missingEnv.includes("HWLAB_CODE_AGENT_OPENAI_BASE_URL")) {
+    return {
+      blocked: true,
+      level: "BLOCKED/provider-config",
+      blocker: "provider-config",
+      providerStatus: blockingHttpStatus,
+      missingEnv,
+      reason: "provider_unavailable with missing HWLAB_CODE_AGENT_OPENAI_BASE_URL means the provider egress/base-url contract is incomplete"
+    };
+  }
+
+  return {
+    blocked: true,
+    level: "BLOCKED/provider",
+    blocker: "provider-unavailable",
+    providerStatus: blockingHttpStatus,
+    missingEnv,
+    reason: "provider_unavailable or availability.status=blocked means the provider path is blocked, not completed"
+  };
+}
+
+export function summarizeCodeAgentPayload(payload, { httpStatus = null, traceId = null } = {}) {
+  const providerBlock = classifyCodeAgentProviderBlock(payload, { httpStatus });
+  const hasObjectPayload = payload && typeof payload === "object" && !Array.isArray(payload);
+
+  if (!hasObjectPayload) {
+    if (!providerBlock.blocked) return null;
+    return {
+      status: "failed",
+      sourceKind: null,
+      evidenceLevel: null,
+      provider: "unknown",
+      model: "unknown",
+      backend: "unknown",
+      traceId: traceId || "unknown",
+      hasReply: false,
+      error: {
+        code: "provider_unavailable",
+        missingEnv: [],
+        missingCommands: [],
+        providerStatus: providerBlock.providerStatus
+      }
+    };
+  }
+
+  const error = normalizeCodeAgentError(payload.error);
+  const assistantReply = typeof payload.reply?.content === "string" ? payload.reply.content.trim() : "";
+  const provider = stringOrNull(payload.provider) ?? (providerBlock.blocked ? "unknown" : null);
+  const model = stringOrNull(payload.model) ?? (providerBlock.blocked ? "unknown" : null);
+  const backend = stringOrNull(payload.backend) ?? (providerBlock.blocked ? "unknown" : null);
+  const observedTraceId = stringOrNull(payload.traceId) ?? traceId ?? (providerBlock.blocked ? "unknown" : null);
+
+  if (providerBlock.blocked) {
+    const blockedError = error
+      ? {
+          ...error,
+          code: error.code ?? "provider_unavailable",
+          providerStatus: error.providerStatus ?? providerBlock.providerStatus
+        }
+      : {
+          code: "provider_unavailable",
+          missingEnv: providerBlock.missingEnv ?? [],
+          missingCommands: [],
+          providerStatus: providerBlock.providerStatus
+        };
+    return {
+      status: "failed",
+      sourceKind: stringOrNull(payload.sourceKind),
+      evidenceLevel: stringOrNull(payload.evidenceLevel),
+      provider,
+      model,
+      backend,
+      traceId: observedTraceId,
+      hasReply: false,
+      error: blockedError
+    };
+  }
+
+  return {
+    status: stringOrNull(payload.status),
+    sourceKind: stringOrNull(payload.sourceKind),
+    evidenceLevel: stringOrNull(payload.evidenceLevel),
+    provider,
+    model,
+    backend,
+    traceId: observedTraceId,
+    hasReply: assistantReply.length > 0,
+    error
+  };
+}
+
+export function classifyCodeAgentBrowserJourney({ responseSummary, httpOk = false, httpStatus = null, ui = null } = {}) {
+  const providerBlock = classifyCodeAgentProviderBlock(responseSummary, { httpStatus });
+  const uiCompleted = ui?.agentChatStatus === "DEV-LIVE 回复" || ui?.agentChatStatus === "已回复" || ui?.completedMessageVisible === true;
+  const uiFailed = ui?.failedMessageVisible === true || ui?.agentChatStatus === "发送失败" || ui?.agentChatStatus === "BLOCKED 凭证缺口";
+  const evidence = inspectSanitizedCompletionEvidence(responseSummary);
+  const backendCompleted =
+    httpOk &&
+    responseSummary?.status === "completed" &&
+    responseSummary?.hasReply === true &&
+    responseSummary.error === null &&
+    evidence.ok;
+
+  if (backendCompleted && uiCompleted && !uiFailed) {
+    return {
+      status: "pass",
+      blocker: null,
+      reason: "completed Code Agent payload over HTTP 2xx with real provider/backend/model/trace/reply evidence and completed UI state"
+    };
+  }
+
+  if (providerBlock.blocked) {
+    return {
+      status: "blocked",
+      blocker: providerBlock.blocker,
+      reason: providerBlock.reason,
+      ...(providerBlock.providerStatus != null ? { providerStatus: providerBlock.providerStatus } : {})
+    };
+  }
+
+  if (responseSummary?.status === "completed") {
+    return {
+      status: "blocked",
+      blocker: "untrusted-completion",
+      reason: `Code Agent completed-looking payload was not paired with HTTP success, completed UI, and real provider evidence: ${evidence.reason}`
+    };
+  }
+
+  return {
+    status: "blocked",
+    blocker: "runtime",
+    reason: "Code Agent browser journey did not produce a completed non-error response"
+  };
+}
+
+export function inspectRealCompletionEvidence(payload) {
+  const provider = String(payload?.provider ?? "").trim().toLowerCase();
+  const backend = String(payload?.backend ?? "").trim().toLowerCase();
+  const missing = [];
+  if (!payload?.conversationId && !payload?.sessionId) missing.push("conversationId");
+  if (!payload?.traceId) missing.push("traceId");
+  if (!payload?.messageId) missing.push("messageId");
+  if (!payload?.provider) missing.push("provider");
+  if (!payload?.model) missing.push("model");
+  if (!payload?.backend) missing.push("backend");
+  if (missing.length > 0) {
+    return { ok: false, reason: `missing ${missing.join(",")}` };
+  }
+  if (!trustedLiveProviders.has(provider)) {
+    return { ok: false, reason: `provider ${payload.provider} is not an accepted real provider` };
+  }
+  if (untrustedProviderPattern.test(`${provider} ${backend}`)) {
+    return { ok: false, reason: `provider/backend looks non-real: ${payload.provider}/${payload.backend}` };
+  }
+  return { ok: true, reason: "provider/model/trace/conversation evidence present" };
+}
+
+export function normalizeCodeAgentError(error) {
+  if (!error || typeof error !== "object") return null;
+  return {
+    code: stringOrNull(error.code),
+    missingEnv: Array.isArray(error.missingEnv) ? error.missingEnv.filter((item) => typeof item === "string") : [],
+    missingCommands: Array.isArray(error.missingCommands) ? error.missingCommands.filter((item) => typeof item === "string") : [],
+    providerStatus: numericStatus(error.providerStatus)
+  };
+}
+
+export function isHttpNon2xxStatus(value) {
+  const status = numericStatus(value);
+  return status !== null && (status < 200 || status > 299);
+}
+
+function inspectSanitizedCompletionEvidence(summary) {
+  const provider = String(summary?.provider ?? "").trim().toLowerCase();
+  const backend = String(summary?.backend ?? "").trim().toLowerCase();
+  const missing = [];
+  if (!summary?.provider) missing.push("provider");
+  if (!summary?.model) missing.push("model");
+  if (!summary?.backend) missing.push("backend");
+  if (!summary?.traceId) missing.push("traceId");
+  if (summary?.hasReply !== true) missing.push("reply");
+  if (missing.length > 0) {
+    return { ok: false, reason: `missing ${missing.join(",")}` };
+  }
+  if (!trustedLiveProviders.has(provider)) {
+    return { ok: false, reason: `provider ${summary.provider} is not an accepted real provider` };
+  }
+  if (untrustedProviderPattern.test(`${provider} ${backend}`)) {
+    return { ok: false, reason: `provider/backend looks non-real: ${summary.provider}/${summary.backend}` };
+  }
+  return { ok: true, reason: "real provider/backend/model/trace/reply evidence present" };
+}
+
+function stringOrNull(value) {
+  return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
+}
+
+function numericStatus(value) {
+  if (typeof value !== "number" || !Number.isInteger(value)) return null;
+  return value;
+}
@@ -7,6 +7,12 @@ import { fileURLToPath } from "node:url";

 import { gateSummary } from "../../web/hwlab-cloud-web/gate-summary.mjs";
 import { runtime } from "../../web/hwlab-cloud-web/runtime.mjs";
+import {
+  classifyCodeAgentBrowserJourney,
+  summarizeCodeAgentPayload
+} from "./code-agent-response-contract.mjs";
+
+export { classifyCodeAgentBrowserJourney };

 const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
 const webRoot = path.join(repoRoot, "web/hwlab-cloud-web");
@@ -510,7 +516,7 @@ async function runLiveSmoke(args) {
      hardwareWriteApis: false,
      sourceIsDevLive: false,
      liveMode: "browser-user-journey-with-code-agent-post",
-      retainedApiFields: ["status", "provider", "model", "backend", "traceId", "hasReply", "error.code", "error.missingEnv"],
+      retainedApiFields: ["status", "provider", "model", "backend", "traceId", "hasReply", "error.code", "error.missingEnv", "error.providerStatus"],
      statement: "Live smoke opens the deployed workbench and sends one controlled Code Agent message; it does not call hardware write APIs and must not be used to infer M3 DEV-LIVE hardware-loop acceptance."
    }
  });
@@ -1667,7 +1673,7 @@ async function inspectLiveUserJourney(url) {
        method: response.request().method(),
        status: response.status(),
        urlPath: new URL(response.url()).pathname,
-        body: sanitizeAgentChatBody(body)
+        body: sanitizeAgentChatBody(body, { httpStatus: response.status() })
      });
    });

@@ -1698,28 +1704,30 @@ async function inspectLiveUserJourney(url) {
    }
    await page.waitForTimeout(500);
    const ui = await inspectJourneyUi(page);
-    const responseSummary = sanitizeAgentChatBody(responseBody);
-    const backendCompleted =
-      response.ok() &&
-      responseSummary?.status === "completed" &&
-      Boolean(responseSummary.provider) &&
-      Boolean(responseSummary.model) &&
-      Boolean(responseSummary.traceId);
-    const uiCompleted = ui.agentChatStatus === "DEV-LIVE 回复" || ui.completedMessageVisible;
-    const pass = backendCompleted && uiCompleted && !ui.failedMessageVisible;
+    const responseSummary = sanitizeAgentChatBody(responseBody, { httpStatus: response.status() });
+    const classification = classifyCodeAgentBrowserJourney({
+      responseSummary,
+      httpOk: response.ok(),
+      httpStatus: response.status(),
+      ui
+    });
+    const pass = classification.status === "pass";

    return {
      status: pass ? "pass" : "blocked",
      summary: pass
        ? "Deployed browser journey opened the workbench, sent a Code Agent message, and observed a completed UI response."
-        : "Deployed browser journey did not observe a completed non-error Code Agent response.",
+        : classification.blocker === "provider-upstream" || classification.blocker === "provider-unavailable"
+          ? "Deployed browser journey reached Code Agent, but the provider returned a blocked/unavailable state instead of a completed reply."
+          : "Deployed browser journey did not observe a completed non-error Code Agent response.",
      evidence: [
        `POST /v1/agent/chat HTTP ${response.status()}`,
        `status=${responseSummary?.status ?? "unknown"}`,
        `provider=${responseSummary?.provider ?? "unknown"}`,
        `model=${responseSummary?.model ?? "unknown"}`,
        `traceId=${responseSummary?.traceId ?? "unknown"}`,
-        `uiStatus=${ui.agentChatStatus ?? "unknown"}`
+        `uiStatus=${ui.agentChatStatus ?? "unknown"}`,
+        `blocker=${classification.blocker ?? "none"}`
      ],
      observations: {
        controls,
@@ -1729,6 +1737,7 @@ async function inspectLiveUserJourney(url) {
          urlPath: new URL(response.url()).pathname
        },
        response: responseSummary,
+        classification,
        ui,
        networkEvents: agentResponses
      }
@@ -1811,24 +1820,8 @@ async function inspectDefaultApiStatus(page) {
  });
 }

-function sanitizeAgentChatBody(body) {
-  if (!body || typeof body !== "object") return null;
-  return {
-    status: typeof body.status === "string" ? body.status : null,
-    sourceKind: typeof body.sourceKind === "string" ? body.sourceKind : null,
-    evidenceLevel: typeof body.evidenceLevel === "string" ? body.evidenceLevel : null,
-    provider: typeof body.provider === "string" ? body.provider : null,
-    model: typeof body.model === "string" ? body.model : null,
-    backend: typeof body.backend === "string" ? body.backend : null,
-    traceId: typeof body.traceId === "string" ? body.traceId : null,
-    hasReply: Boolean(body.reply?.content),
-    error: body.error && typeof body.error === "object"
-      ? {
-          code: typeof body.error.code === "string" ? body.error.code : null,
-          missingEnv: Array.isArray(body.error.missingEnv) ? body.error.missingEnv.filter((item) => typeof item === "string") : []
-        }
-      : null
-  };
+export function sanitizeAgentChatBody(body, options = {}) {
+  return summarizeCodeAgentPayload(body, options);
 }

 export async function runDevCloudWorkbenchLocalAgentFixtureSmoke() {
@@ -1867,7 +1860,7 @@ export async function runDevCloudWorkbenchLocalAgentFixtureSmoke() {
        method: response.request().method(),
        status: response.status(),
        urlPath: new URL(response.url()).pathname,
-        body: sanitizeAgentChatBody(body)
+        body: sanitizeAgentChatBody(body, { httpStatus: response.status() })
      });
    });

@@ -1894,7 +1887,7 @@ export async function runDevCloudWorkbenchLocalAgentFixtureSmoke() {
      { timeout: 8000 }
    );
    const ui = await inspectJourneyUi(page);
-    const responseSummary = sanitizeAgentChatBody(responseBody);
+    const responseSummary = sanitizeAgentChatBody(responseBody, { httpStatus: response.status() });
    const pass =
      response.ok() &&
      responseSummary?.status === "completed" &&