From b139ffd8e462a38e36543fab0aa594c664dcb52c Mon Sep 17 00:00:00 2001 From: lyon Date: Sat, 20 Jun 2026 15:08:33 +0800 Subject: [PATCH] feat: normalize cloud api error diagnostics --- .../cloud/auth-canonical-principal.test.ts | 15 +- internal/cloud/error-envelope.ts | 141 ++++++++++++++++++ internal/cloud/provider-profile-management.ts | 10 +- internal/cloud/server-health.test.ts | 9 ++ internal/cloud/server-http-utils.ts | 6 +- internal/cloud/server.ts | 8 +- internal/cloud/web-performance.ts | 5 +- internal/dev-entrypoint/http.mjs | 37 ++++- 8 files changed, 223 insertions(+), 8 deletions(-) create mode 100644 internal/cloud/error-envelope.ts diff --git a/internal/cloud/auth-canonical-principal.test.ts b/internal/cloud/auth-canonical-principal.test.ts index 0d52e715..85eff577 100644 --- a/internal/cloud/auth-canonical-principal.test.ts +++ b/internal/cloud/auth-canonical-principal.test.ts @@ -8,8 +8,15 @@ test("/auth/session returns structured unauthenticated canonical result", async await listen(server); try { const { port } = server.address() as { port: number }; - const response = await fetch(`http://127.0.0.1:${port}/auth/session`); + const response = await fetch(`http://127.0.0.1:${port}/auth/session`, { + headers: { + traceparent: "00-22222222222222222222222222222222-3333333333333333-01", + "x-request-id": "req_auth_session_diagnostic" + } + }); assert.equal(response.status, 401); + assert.equal(response.headers.get("x-hwlab-otel-trace-id"), "22222222222222222222222222222222"); + assert.equal(response.headers.get("x-request-id"), "req_auth_session_diagnostic"); const body = await response.json(); assert.equal(body.authenticated, false); assert.equal(body.authMethod, null); @@ -18,6 +25,12 @@ test("/auth/session returns structured unauthenticated canonical result", async assert.equal(body.actor, null); assert.equal(body.valuesRedacted, true); assert.equal(body.error.code, "auth_required"); + assert.equal(body.error.diagnostic.traceId, "22222222222222222222222222222222"); + assert.equal(body.error.diagnostic.requestId, "req_auth_session_diagnostic"); + assert.equal(body.error.diagnostic.route, "/auth/session"); + assert.equal(body.error.diagnostic.layer, "api"); + assert.equal(body.error.diagnostic.category, "client"); + assert.equal(body.error.diagnostic.valuesPrinted, false); assert.equal(JSON.stringify(body).includes("hwlab_session="), false); } finally { await close(server); diff --git a/internal/cloud/error-envelope.ts b/internal/cloud/error-envelope.ts new file mode 100644 index 00000000..46dcea1a --- /dev/null +++ b/internal/cloud/error-envelope.ts @@ -0,0 +1,141 @@ +export const HWLAB_ERROR_DIAGNOSTIC_VERSION = "hwlab-error-diagnostic-v1"; + +export function decorateErrorPayloadForHttp(payload, options = {}) { + const statusCode = numericStatus(options.statusCode); + if (!statusCode || statusCode < 400 || !plainObject(payload) || !("error" in payload)) return payload; + + const context = options.context ?? options.response?.hwlabHttpRequestContext ?? options.request?.hwlabHttpRequestContext ?? {}; + const originalError = normalizeErrorObject(payload.error, payload); + if (!originalError) return payload; + + const rawCode = originalError.code ?? payload.code ?? payload.error ?? statusCodeToCode(statusCode); + const diagnosticCode = safeCode(rawCode); + const errorCode = typeof originalError.code === "number" ? originalError.code : diagnosticCode; + const message = firstText(originalError.message, payload.message, diagnosticCode); + const userMessage = firstText(originalError.userMessage, payload.userMessage, message); + const diagnostic = { + contractVersion: HWLAB_ERROR_DIAGNOSTIC_VERSION, + traceId: safeOtelTraceId(context.traceId) ?? safeOtelTraceId(originalError.diagnostic?.traceId) ?? safeOtelTraceId(originalError.otelTraceId) ?? safeOtelTraceId(payload.otelTraceId) ?? null, + requestId: safeRequestId(context.requestId) ?? safeRequestId(originalError.diagnostic?.requestId) ?? safeRequestId(originalError.requestId) ?? safeRequestId(payload.requestId) ?? null, + serviceId: safeToken(originalError.serviceId ?? payload.serviceId ?? options.serviceId, "hwlab-cloud-api"), + route: safeRoute(originalError.route ?? payload.route ?? options.route ?? context.route), + layer: safeToken(originalError.layer ?? options.layer, "api"), + category: safeToken(originalError.category ?? options.category, categoryForStatus(statusCode)), + code: diagnosticCode, + httpStatus: statusCode, + source: safeToken(options.source, "server"), + observedAt: new Date().toISOString(), + valuesPrinted: false + }; + const nextError = { + ...originalError, + code: errorCode, + message, + userMessage, + diagnostic: { + ...(plainObject(originalError.diagnostic) ? originalError.diagnostic : {}), + ...diagnostic + }, + valuesPrinted: originalError.valuesPrinted === true ? true : false + }; + return { ...payload, error: nextError }; +} + +export function logErrorEnvelope(payload, options = {}) { + const statusCode = numericStatus(options.statusCode); + if (!statusCode || statusCode < 400 || !plainObject(payload) || !plainObject(payload.error)) return; + const diagnostic = plainObject(payload.error.diagnostic) ? payload.error.diagnostic : null; + if (!diagnostic) return; + const record = { + event: "hwlab.error", + statusCode, + trace_id: diagnostic.traceId ?? null, + request_id: diagnostic.requestId ?? null, + route: diagnostic.route ?? null, + serviceId: diagnostic.serviceId ?? "hwlab-cloud-api", + error: { + code: payload.error.code ?? diagnostic.code ?? "unknown_error", + layer: payload.error.layer ?? diagnostic.layer ?? "api", + category: payload.error.category ?? diagnostic.category ?? categoryForStatus(statusCode), + retryable: payload.error.retryable ?? null + }, + valuesPrinted: false + }; + const logger = options.logger ?? console; + const line = JSON.stringify(record); + if (statusCode >= 500 && typeof logger.error === "function") { + logger.error(line); + } else if (typeof logger.warn === "function") { + logger.warn(line); + } +} + +function normalizeErrorObject(value, payload) { + if (plainObject(value)) return { ...value }; + const text = firstText(value, payload?.message, "unknown_error"); + return { + code: safeCode(text), + message: text, + userMessage: text + }; +} + +function numericStatus(value) { + const status = Number(value); + return Number.isInteger(status) && status >= 100 && status <= 599 ? status : null; +} + +function statusCodeToCode(statusCode) { + if (statusCode === 400) return "bad_request"; + if (statusCode === 401) return "auth_required"; + if (statusCode === 403) return "forbidden"; + if (statusCode === 404) return "not_found"; + if (statusCode === 409) return "conflict"; + if (statusCode === 429) return "rate_limited"; + if (statusCode >= 500) return "internal_error"; + return "error"; +} + +function categoryForStatus(statusCode) { + if (statusCode >= 500) return "server"; + if (statusCode >= 400) return "client"; + return "error"; +} + +function firstText(...values) { + for (const value of values) { + const text = String(value ?? "").trim(); + if (text) return text; + } + return "unknown_error"; +} + +function safeCode(value) { + const text = firstText(value).toLowerCase().replace(/[^a-z0-9_.:-]+/gu, "_").replace(/^_+|_+$/gu, ""); + return text.slice(0, 96) || "unknown_error"; +} + +function safeToken(value, fallback) { + const text = String(value ?? "").trim(); + return /^[A-Za-z0-9_.:-]{1,96}$/u.test(text) ? text : fallback; +} + +function safeRoute(value) { + const text = String(value ?? "").trim().split("?")[0] || "/"; + if (!text.startsWith("/")) return "/"; + return text.slice(0, 180); +} + +function safeRequestId(value) { + const text = String(value ?? "").trim(); + return /^[A-Za-z0-9_.:-]{3,180}$/u.test(text) ? text : null; +} + +function safeOtelTraceId(value) { + const text = String(value ?? "").trim().toLowerCase(); + return /^[0-9a-f]{32}$/u.test(text) && text !== "00000000000000000000000000000000" ? text : null; +} + +function plainObject(value) { + return Boolean(value && typeof value === "object" && !Array.isArray(value)); +} diff --git a/internal/cloud/provider-profile-management.ts b/internal/cloud/provider-profile-management.ts index 61e22583..d51e3391 100644 --- a/internal/cloud/provider-profile-management.ts +++ b/internal/cloud/provider-profile-management.ts @@ -1,5 +1,7 @@ import { randomUUID } from "node:crypto"; +import { decorateErrorPayloadForHttp, logErrorEnvelope } from "./error-envelope.ts"; + const CONTRACT_VERSION = "provider-profile-management-v1"; const DEFAULT_AGENTRUN_MGR_URL = "http://agentrun-mgr.agentrun-v01.svc.cluster.local:8080"; const AGENTRUN_MANAGER_HOST_PATTERN = /^agentrun-mgr\.[a-z0-9]([-a-z0-9]*[a-z0-9])?\.svc\.cluster\.local$/u; @@ -477,7 +479,7 @@ function looksSecret(value) { } function requestIdFor(request) { - return getHeader(request, "x-request-id") || `req_provider_${randomUUID().replace(/-/gu, "")}`; + return request.hwlabHttpRequestContext?.requestId || getHeader(request, "x-request-id") || `req_provider_${randomUUID().replace(/-/gu, "")}`; } function getHeader(request, name) { @@ -524,8 +526,10 @@ function sendError(response, statusCode, code, message, details = {}) { } function sendJson(response, statusCode, payload, options = {}) { - response.writeHead(statusCode, { "content-type": "application/json" }); - const body = options.allowConfigToml ? redactObjectAllowingConfigToml(payload) : redactObject(payload); + const decorated = decorateErrorPayloadForHttp(payload, { response, statusCode, layer: "api", category: "provider_profile" }); + logErrorEnvelope(decorated, { statusCode }); + response.writeHead(statusCode, { "content-type": "application/json", "cache-control": "no-store" }); + const body = options.allowConfigToml ? redactObjectAllowingConfigToml(decorated) : redactObject(decorated); response.end(`${JSON.stringify(body)}\n`); } diff --git a/internal/cloud/server-health.test.ts b/internal/cloud/server-health.test.ts index 342d78dd..e60e877d 100644 --- a/internal/cloud/server-health.test.ts +++ b/internal/cloud/server-health.test.ts @@ -53,6 +53,15 @@ test("cloud api attaches OTel trace and request headers to normal and REST error assert.match(missing.headers.get("traceparent") ?? "", /^00-[0-9a-f]{32}-[0-9a-f]{16}-01$/u); assert.match(missing.headers.get("x-hwlab-otel-trace-id") ?? "", /^[0-9a-f]{32}$/u); assert.match(missing.headers.get("x-request-id") ?? "", /^req_[0-9a-f-]{36}$/u); + const missingPayload = await missing.json(); + assert.equal(missingPayload.error.code, "not_found"); + assert.equal(missingPayload.error.userMessage, "Route is not part of the L1 cloud-api runtime"); + assert.equal(missingPayload.error.diagnostic.traceId, missing.headers.get("x-hwlab-otel-trace-id")); + assert.equal(missingPayload.error.diagnostic.requestId, missing.headers.get("x-request-id")); + assert.equal(missingPayload.error.diagnostic.route, "/route-not-implemented"); + assert.equal(missingPayload.error.diagnostic.layer, "api"); + assert.equal(missingPayload.error.diagnostic.category, "client"); + assert.equal(missingPayload.error.diagnostic.valuesPrinted, false); } finally { await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); } diff --git a/internal/cloud/server-http-utils.ts b/internal/cloud/server-http-utils.ts index 33236514..78101242 100644 --- a/internal/cloud/server-http-utils.ts +++ b/internal/cloud/server-http-utils.ts @@ -1,4 +1,6 @@ +import { decorateErrorPayloadForHttp, logErrorEnvelope } from "./error-envelope.ts"; + const DEFAULT_BODY_LIMIT_BYTES = 1024 * 1024; export function readBody(request, limitBytes = DEFAULT_BODY_LIMIT_BYTES) { @@ -22,11 +24,13 @@ export function readBody(request, limitBytes = DEFAULT_BODY_LIMIT_BYTES) { export function sendJson(response, statusCode, body) { if (response.headersSent || response.writableEnded) return false; + const payload = decorateErrorPayloadForHttp(body, { response, statusCode }); + logErrorEnvelope(payload, { statusCode }); response.writeHead(statusCode, { "content-type": "application/json; charset=utf-8", "cache-control": "no-store" }); - response.end(JSON.stringify(body) + "\n"); + response.end(JSON.stringify(payload) + "\n"); return true; } diff --git a/internal/cloud/server.ts b/internal/cloud/server.ts index ed1f11a3..892f9f21 100644 --- a/internal/cloud/server.ts +++ b/internal/cloud/server.ts @@ -74,6 +74,7 @@ import { startWorkbenchEmptySessionGc } from "./workbench-empty-session-gc.ts"; import { handleM3IoControlHttp } from "./server-m3-http.ts"; import { handleSkillsHttp } from "./server-skills-http.ts"; import { httpRequestOtelTraceContext } from "./otel-trace.ts"; +import { decorateErrorPayloadForHttp, logErrorEnvelope } from "./error-envelope.ts"; import { handleProviderProfileCatalogHttp, handleProviderProfilesHttp } from "./provider-profile-management.ts"; import { configureSkillRuntime, @@ -164,12 +165,15 @@ function buildHttpRequestContext(request) { parentSpanId: otelContext.parentSpanId, spanId: otelContext.spanId, traceparent: otelContext.traceparent, + method: String(request.method || "GET").toUpperCase(), + route: String(request.url || "/").split("?")[0] || "/", valuesPrinted: false }; } function attachHttpRequestContext(request, response, context) { request.hwlabHttpRequestContext = context; + response.hwlabHttpRequestContext = context; if (response.headersSent || response.writableEnded) return; response.setHeader("traceparent", context.traceparent); response.setHeader("x-hwlab-otel-trace-id", context.traceId); @@ -1858,11 +1862,13 @@ function readBody(request, limitBytes = DEFAULT_BODY_LIMIT_BYTES) { function sendJson(response, statusCode, body) { if (response.headersSent || response.writableEnded) return false; + const payload = decorateErrorPayloadForHttp(body, { response, statusCode }); + logErrorEnvelope(payload, { statusCode }); response.writeHead(statusCode, { "content-type": "application/json; charset=utf-8", "cache-control": "no-store" }); - response.end(`${JSON.stringify(body)}\n`); + response.end(`${JSON.stringify(payload)}\n`); return true; } diff --git a/internal/cloud/web-performance.ts b/internal/cloud/web-performance.ts index 931fff7c..d09cd038 100644 --- a/internal/cloud/web-performance.ts +++ b/internal/cloud/web-performance.ts @@ -1,6 +1,7 @@ // SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-19-p0 // Keeps browser-reported performance metrics low-cardinality before Prometheus export. +import { decorateErrorPayloadForHttp, logErrorEnvelope } from "./error-envelope.ts"; import { emitWorkbenchUiOtelSpan } from "./otel-trace.ts"; const DEFAULT_WEB_PERFORMANCE_BODY_LIMIT_BYTES = 64 * 1024; @@ -2082,9 +2083,11 @@ function readBody(request, limitBytes) { } function sendJson(response, statusCode, body) { + const payload = decorateErrorPayloadForHttp(body, { response, statusCode, layer: "api", category: "web_performance" }); + logErrorEnvelope(payload, { statusCode }); response.writeHead(statusCode, { "content-type": "application/json; charset=utf-8", "cache-control": "no-store" }); - response.end(`${JSON.stringify(body)}\n`); + response.end(`${JSON.stringify(payload)}\n`); } diff --git a/internal/dev-entrypoint/http.mjs b/internal/dev-entrypoint/http.mjs index 9faf5b9b..8e25877e 100644 --- a/internal/dev-entrypoint/http.mjs +++ b/internal/dev-entrypoint/http.mjs @@ -51,7 +51,7 @@ export function healthPayload({ serviceId, role, details = {} }) { } export function sendJson(response, statusCode, body) { - const payload = JSON.stringify(body, null, 2); + const payload = JSON.stringify(withDevErrorDiagnostic(body, statusCode), null, 2); response.writeHead(statusCode, { "content-type": "application/json; charset=utf-8", "content-length": Buffer.byteLength(payload) @@ -59,6 +59,41 @@ export function sendJson(response, statusCode, body) { response.end(payload); } +function withDevErrorDiagnostic(body, statusCode) { + if (!body || typeof body !== "object" || Array.isArray(body) || statusCode < 400 || !("error" in body)) return body; + const error = typeof body.error === "object" && body.error !== null && !Array.isArray(body.error) + ? { ...body.error } + : { code: String(body.error || "unknown_error"), message: String(body.message || body.error || "unknown_error") }; + const businessTraceId = typeof error.traceId === "string" && /^trc_[A-Za-z0-9_.:-]+$/u.test(error.traceId) + ? error.traceId + : typeof body.traceId === "string" && /^trc_[A-Za-z0-9_.:-]+$/u.test(body.traceId) + ? body.traceId + : null; + return { + ...body, + error: { + ...error, + userMessage: error.userMessage || error.message || body.message || error.code, + diagnostic: { + contractVersion: "hwlab-error-diagnostic-v1", + traceId: /^[0-9a-f]{32}$/u.test(String(error.traceId ?? body.traceId ?? "")) ? String(error.traceId ?? body.traceId).toLowerCase() : null, + businessTraceId, + requestId: null, + serviceId: body.serviceId || error.serviceId || "hwlab-dev-entrypoint", + route: String(error.route || body.path || "/").split("?")[0].slice(0, 180), + layer: error.layer || "dev-entrypoint", + category: error.category || (statusCode >= 500 ? "server" : "client"), + code: error.code || String(body.error || "unknown_error"), + httpStatus: statusCode, + source: "dev-entrypoint", + observedAt: new Date().toISOString(), + valuesPrinted: false + }, + valuesPrinted: error.valuesPrinted === true ? true : false + } + }; +} + export function createServiceServer({ serviceId, role, routes = new Map(), details = () => ({}) }) { return createServer(async (request, response) => { try {