From bf272f7e3e4885b0f94d7a539f68e165891bdc1a Mon Sep 17 00:00:00 2001 From: UniDesk Codex Date: Wed, 1 Jul 2026 10:53:05 +0800 Subject: [PATCH] fix: expose hwpod node ops otel trace diagnostics --- internal/cloud/hwpod-node-ops.test.ts | 52 ++++-- internal/cloud/server.ts | 171 +++++++++++++----- internal/project-management/source-adapter.ts | 23 ++- tools/hwpod-harness.test.ts | 32 ++++ tools/src/hwpod-harness-lib.ts | 82 ++++++++- 5 files changed, 294 insertions(+), 66 deletions(-) diff --git a/internal/cloud/hwpod-node-ops.test.ts b/internal/cloud/hwpod-node-ops.test.ts index 6a929367..912e8380 100644 --- a/internal/cloud/hwpod-node-ops.test.ts +++ b/internal/cloud/hwpod-node-ops.test.ts @@ -8,8 +8,23 @@ import { createCloudApiBunServer } from "./bun-server.ts"; import { createCloudApiServer } from "./server.ts"; import { connectHwpodNodeWs, createHwpodNodeServer } from "../../tools/src/hwpod-node-lib.ts"; +function createHwpodTestCloudApiServer(options: any = {}) { + return createCloudApiServer({ accessController: allowAllAccessController(), ...options }); +} + +function allowAllAccessController() { + return { + required: false, + configureCodeAgentWorkspaceContext() {}, + async requireNavAccess() { return true; }, + async authenticate() { + return { ok: true, status: 200, actor: { id: "usr_hwpod_test", role: "admin" }, session: { id: "uss_hwpod_test" } }; + } + }; +} + test("cloud-api exposes hwpod-node-ops contract on /v1", async () => { - const server = createCloudApiServer(); + const server = createHwpodTestCloudApiServer(); await listen(server); try { const response = await fetch(`${serverUrl(server)}/v1`); @@ -31,7 +46,7 @@ test("cloud-api exposes hwpod-node-ops contract on /v1", async () => { }); test("cloud-api exposes hwlab-node Python update metadata", async () => { - const server = createCloudApiServer({ + const server = createHwpodTestCloudApiServer({ env: { PATH: process.env.PATH, HWLAB_PUBLIC_ENDPOINT: "https://hwlab.pikapython.com", @@ -63,7 +78,7 @@ test("cloud-api exposes hwlab-node Python update metadata", async () => { }); test("cloud-api serves bundled hwlab-node Python updater artifact by default", async () => { - const server = createCloudApiServer({ env: { PATH: process.env.PATH, HWLAB_PUBLIC_ENDPOINT: "https://hwlab.pikapython.com" } }); + const server = createHwpodTestCloudApiServer({ env: { PATH: process.env.PATH, HWLAB_PUBLIC_ENDPOINT: "https://hwlab.pikapython.com" } }); await listen(server); try { const response = await fetch(`${serverUrl(server)}/v1/hwlab-node/update?current=0.1.0&channel=stable&platform=windows`); @@ -90,7 +105,7 @@ test("cloud-api discovers preinstalled workspace hwpod-specs without hardcoded d await mkdir(specDir, { recursive: true }); await writeFile(path.join(specDir, "hwpod-spec.yaml"), sampleSpecYaml(), "utf8"); await writeFile(path.join(specDir, "hwpod-spec.meta.json"), JSON.stringify({ source: { kind: "preinstalled-verified-spec", caseRepo: "pikasTech/hwlab-case-registry", caseId: "d601-f103-v2-compile" } }), "utf8"); - const server = createCloudApiServer({ env: { PATH: process.env.PATH, HWLAB_CODE_AGENT_WORKSPACE: root } }); + const server = createHwpodTestCloudApiServer({ env: { PATH: process.env.PATH, HWLAB_CODE_AGENT_WORKSPACE: root } }); await listen(server); try { const response = await fetch(`${serverUrl(server)}/v1/hwpod/specs`); @@ -118,7 +133,7 @@ test("cloud-api discovers configmap-mounted registry hwpod-spec symlinks", async const targetPath = path.join(dataDir, "constart-71freq-c.yaml"); await writeFile(targetPath, sampleConstartSpecYaml(), "utf8"); await symlink(targetPath, path.join(registryDir, "constart-71freq-c.yaml")); - const server = createCloudApiServer({ + const server = createHwpodTestCloudApiServer({ env: { PATH: process.env.PATH, HWLAB_CODE_AGENT_WORKSPACE: root, @@ -148,7 +163,7 @@ test("cloud-api probes discovered hwpod-spec availability through node-ops", asy await mkdir(specDir, { recursive: true }); await writeFile(path.join(specDir, "hwpod-spec.yaml"), sampleSpecYaml(), "utf8"); const seen: any[] = []; - const server = createCloudApiServer({ + const server = createHwpodTestCloudApiServer({ env: { PATH: process.env.PATH, HWLAB_CODE_AGENT_WORKSPACE: root }, hwpodNodeOpsHandler: async (plan: any) => { seen.push(plan); @@ -174,7 +189,7 @@ test("cloud-api probes discovered hwpod-spec availability through node-ops", asy test("cloud-api forwards valid hwpod-node-ops plans to injected node handler", async () => { const seen: any[] = []; - const server = createCloudApiServer({ + const server = createHwpodTestCloudApiServer({ hwpodNodeOpsHandler: async (plan: any, context: any) => { seen.push({ plan, context }); return { @@ -206,20 +221,27 @@ test("cloud-api forwards valid hwpod-node-ops plans to injected node handler", a }); test("cloud-api returns blocked payload when hwpod-node is not wired yet", async () => { - const server = createCloudApiServer(); + const server = createHwpodTestCloudApiServer(); + const otelTraceId = "0123456789abcdef0123456789abcdef"; await listen(server); try { const response = await fetch(`${serverUrl(server)}/v1/hwpod-node-ops`, { method: "POST", - headers: { "content-type": "application/json" }, + headers: { "content-type": "application/json", "x-hwlab-otel-trace-id": otelTraceId }, body: JSON.stringify(samplePlan()) }); const payload = await response.json(); assert.equal(response.status, 200); + assert.equal(response.headers.get("x-hwlab-otel-trace-id"), otelTraceId); assert.equal(payload.ok, false); assert.equal(payload.status, "blocked"); + assert.equal(payload.otelTraceId, otelTraceId); + assert.equal(payload.requestMeta.otelTraceId, otelTraceId); assert.equal(payload.blocker.code, "hwpod_node_unavailable"); + assert.equal(payload.blocker.otelTraceId, otelTraceId); + assert.equal(payload.blocker.diagnostic.traceLine, `OTel traceId: ${otelTraceId}`); assert.equal(payload.results[0].blocker.layer, "hwpod-node"); + assert.match(payload.results[0].blocker.userMessage, /OTel traceId:/u); } finally { await close(server); } @@ -228,7 +250,7 @@ test("cloud-api returns blocked payload when hwpod-node is not wired yet", async test("cloud-api forwards hwpod-node-ops to configured thin hwpod-node URL", async () => { const node = createHwpodNodeServer({ nodeId: "pc-host-1" }); await listen(node); - const server = createCloudApiServer({ + const server = createHwpodTestCloudApiServer({ env: { PATH: process.env.PATH, HWLAB_HWPOD_NODE_OPS_URL: `${serverUrl(node)}/v1/hwpod-node-ops` @@ -258,7 +280,7 @@ test("cloud-api forwards hwpod-node-ops to configured thin hwpod-node URL", asyn test("cloud-api blocks direct hwpod-node URL when target node id does not match", async () => { const node = createHwpodNodeServer({ nodeId: "g14-host-hwpod-node" }); await listen(node); - const server = createCloudApiServer({ + const server = createHwpodTestCloudApiServer({ env: { PATH: process.env.PATH, HWPOD_NODE_ID: "g14-host-hwpod-node", @@ -290,7 +312,7 @@ test("cloud-api blocks direct hwpod-node URL when target node id does not match" }); test("cloud-api blocks direct hwpod-node URL when node identity is not visible", async () => { - const target = createCloudApiServer({ + const target = createHwpodTestCloudApiServer({ hwpodNodeOpsHandler: async (plan: any) => ({ ok: true, status: "completed", @@ -298,7 +320,7 @@ test("cloud-api blocks direct hwpod-node URL when node identity is not visible", }) }); await listen(target); - const server = createCloudApiServer({ + const server = createHwpodTestCloudApiServer({ env: { PATH: process.env.PATH, HWLAB_HWPOD_NODE_OPS_URL: `${serverUrl(target)}/v1/hwpod-node-ops` @@ -325,7 +347,7 @@ test("cloud-api blocks direct hwpod-node URL when node identity is not visible", test("cloud-api dispatches hwpod-node-ops to outbound WebSocket hwpod-node by nodeId", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-hwpod-node-ws-")); - const runtime = await createCloudApiBunServer({ host: "127.0.0.1", port: 0, env: { PATH: process.env.PATH } }); + const runtime = await createCloudApiBunServer({ host: "127.0.0.1", port: 0, env: { PATH: process.env.PATH }, accessController: allowAllAccessController() }); const connector = connectHwpodNodeWs({ cloudUrl: runtime.url, nodeId: "pc-host-1", reconnect: false, heartbeatIntervalMs: 1000 }); try { await writeFile(path.join(root, "main.c"), "int main(void) { return 0; }\n", "utf8"); @@ -352,7 +374,7 @@ test("cloud-api dispatches hwpod-node-ops to outbound WebSocket hwpod-node by no }); test("cloud-api rejects unsupported hwpod-node ops before forwarding", async () => { - const server = createCloudApiServer({ + const server = createHwpodTestCloudApiServer({ hwpodNodeOpsHandler: async () => { throw new Error("must not be called"); } diff --git a/internal/cloud/server.ts b/internal/cloud/server.ts index c70a0dff..23b6ee4f 100644 --- a/internal/cloud/server.ts +++ b/internal/cloud/server.ts @@ -1016,20 +1016,30 @@ async function handleHwpodNodeOpsHttp(request, response, options) { } const plan = validation.plan; - const requestMeta = { - requestId: getHeader(request, "x-request-id") || `req_hwpod_${randomUUID()}`, - traceId: getHeader(request, "x-trace-id") || `trc_hwpod_${randomUUID()}`, - serviceId: getHeader(request, "x-source-service-id") || CLOUD_API_SERVICE_ID, - environment: runtimeEnvironment(options.env ?? process.env) - }; + const requestMeta = hwpodNodeOpsRequestMeta(request, options, "hwpod"); try { const handled = await dispatchHwpodNodeOpsPlan(plan, requestMeta, options, { request }); sendJson(response, handled.httpStatus, handled.payload); } catch (error) { - sendJson(response, 200, hwpodNodeOpsBlockedPayload(plan, requestMeta, error?.message ?? "hwpod-node-ops handler failed")); + const summary = error?.message ?? "hwpod-node-ops handler failed"; + recordHwpodNodeOpsBlocked(request, options, plan, requestMeta, "hwpod_node_handler_failed", summary, { dispatchMode: "handler-exception" }); + sendJson(response, 200, hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, { dispatchMode: "handler-exception" })); } } +function hwpodNodeOpsRequestMeta(request, options, label) { + const httpContext = request?.hwlabHttpRequestContext ?? {}; + return { + requestId: getHeader(request, "x-request-id") || httpContext.requestId || `req_${label}_${randomUUID()}`, + traceId: getHeader(request, "x-trace-id") || `trc_${label}_${randomUUID()}`, + serviceId: getHeader(request, "x-source-service-id") || CLOUD_API_SERVICE_ID, + environment: runtimeEnvironment(options.env ?? process.env), + otelTraceId: httpContext.traceId ?? null, + traceparent: httpContext.traceparent ?? null, + valuesPrinted: false + }; +} + function handleHwlabNodeUpdateHttp(request, response, url, options) { if (request.method !== "GET") { sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "GET required." } }); @@ -1162,12 +1172,7 @@ async function probeDiscoveredHwpodSpec(spec, { request, options, observedAt }) } }; } - const requestMeta = { - requestId: getHeader(request, "x-request-id") || `req_hwpod_spec_${randomUUID()}`, - traceId: getHeader(request, "x-trace-id") || `trc_hwpod_spec_${randomUUID()}`, - serviceId: getHeader(request, "x-source-service-id") || CLOUD_API_SERVICE_ID, - environment: runtimeEnvironment(options.env ?? process.env) - }; + const requestMeta = hwpodNodeOpsRequestMeta(request, options, "hwpod_spec"); const handled = await dispatchHwpodNodeOpsPlan(validation.plan, requestMeta, options, { request }); const payload = handled.payload; return { @@ -1190,9 +1195,17 @@ async function dispatchHwpodNodeOpsPlan(plan, requestMeta, options, context = {} const hwpodNodeWsRegistry = options.hwpodNodeWsRegistry; const hasWsNode = hwpodNodeWsRegistry?.hasNode?.(plan.nodeId); if (typeof options.hwpodNodeOpsHandler !== "function" && !hasWsNode && !hwpodNodeOpsUrl) { + const summary = "no outbound WebSocket hwpod-node is connected and HWLAB_HWPOD_NODE_OPS_URL is not configured; cloud-api is only validating the hwpod-node-ops contract"; + const details = { + dispatchMode: "none", + websocketRegistryConfigured: Boolean(hwpodNodeWsRegistry), + websocketConnected: false, + directUrlConfigured: false + }; + recordHwpodNodeOpsBlocked(context.request, options, plan, requestMeta, "hwpod_node_unavailable", summary, details); return { httpStatus: 200, - payload: hwpodNodeOpsBlockedPayload(plan, requestMeta, "no outbound WebSocket hwpod-node is connected and HWLAB_HWPOD_NODE_OPS_URL is not configured; cloud-api is only validating the hwpod-node-ops contract") + payload: hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, details) }; } const handled = typeof options.hwpodNodeOpsHandler === "function" @@ -1202,7 +1215,7 @@ async function dispatchHwpodNodeOpsPlan(plan, requestMeta, options, context = {} : await forwardHwpodNodeOpsPlan(hwpodNodeOpsUrl, plan, requestMeta, options); const results = Array.isArray(handled?.results) ? handled.results : []; const failed = results.some((item) => item?.ok === false); - const payload = { + const payload = attachHwpodNodeOpsDiagnostics({ ok: handled?.ok ?? !failed, status: handled?.status ?? (failed ? "failed" : "completed"), contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, @@ -1213,7 +1226,10 @@ async function dispatchHwpodNodeOpsPlan(plan, requestMeta, options, context = {} results, blocker: handled?.blocker ?? null, requestMeta - }; + }, requestMeta, { dispatchMode: hasWsNode ? "websocket" : hwpodNodeOpsUrl ? "direct-url" : "handler" }); + if (payload.ok === false && payload.status === "blocked" && payload.blocker?.code) { + recordHwpodNodeOpsBlocked(context.request, options, plan, requestMeta, payload.blocker.code, payload.blocker.summary ?? "hwpod-node-ops blocked", payload.blocker.details ?? {}); + } return { httpStatus: handled?.httpStatus ?? (failed ? 409 : 200), payload }; } @@ -1226,7 +1242,7 @@ async function forwardHwpodNodeOpsPlan(targetUrl, plan, requestMeta, options) { dispatchMode: "direct-url", targetStatus: target.status ?? null, error: target.error ?? null - }); + }, requestMeta); } if (target.ok && target.nodeId && target.nodeId !== plan.nodeId) { return directHwpodNodeBlocked(plan, "hwpod_node_id_mismatch", `HWLAB_HWPOD_NODE_OPS_URL points to ${target.nodeId}, not requested node ${plan.nodeId}`, { @@ -1234,7 +1250,7 @@ async function forwardHwpodNodeOpsPlan(targetUrl, plan, requestMeta, options) { targetNodeId: target.nodeId, targetUrl: redactNodeOpsUrl(targetUrl), dispatchMode: "direct-url" - }); + }, requestMeta); } const response = await fetch(targetUrl, { method: "POST", @@ -1242,25 +1258,20 @@ async function forwardHwpodNodeOpsPlan(targetUrl, plan, requestMeta, options) { "content-type": "application/json", "x-request-id": requestMeta.requestId, "x-trace-id": requestMeta.traceId, - "x-source-service-id": CLOUD_API_SERVICE_ID + "x-source-service-id": CLOUD_API_SERVICE_ID, + ...(requestMeta.otelTraceId ? { "x-hwlab-otel-trace-id": requestMeta.otelTraceId } : {}), + ...(requestMeta.traceparent ? { traceparent: requestMeta.traceparent } : {}) }, body: JSON.stringify(plan), signal: AbortSignal.timeout(parsePositiveInteger(options.env?.HWLAB_HWPOD_NODE_OPS_TIMEOUT_MS, 30000)) }); const body = await response.json().catch(() => null); if (!body || typeof body !== "object") { - return { - ok: false, - status: "blocked", - httpStatus: 200, - results: [], - blocker: { - code: "hwpod_node_response_invalid", - layer: "hwpod-node", - retryable: true, - summary: `hwpod-node returned HTTP ${response.status} without a JSON object payload` - } - }; + return directHwpodNodeBlocked(plan, "hwpod_node_response_invalid", `hwpod-node returned HTTP ${response.status} without a JSON object payload`, { + targetUrl: redactNodeOpsUrl(targetUrl), + dispatchMode: "direct-url", + targetStatus: response.status + }, requestMeta); } return { ok: body.ok, @@ -1271,14 +1282,16 @@ async function forwardHwpodNodeOpsPlan(targetUrl, plan, requestMeta, options) { }; } -function directHwpodNodeBlocked(plan, code, summary, details) { - const blocker = { code, layer: "hwpod-node", retryable: true, summary, details }; +function directHwpodNodeBlocked(plan, code, summary, details, requestMeta = {}) { + const blocker = hwpodNodeOpsBlocker({ code, layer: "hwpod-node", retryable: true, summary, details }, requestMeta, details); return { ok: false, status: "blocked", httpStatus: 200, results: plan.ops.map((op) => ({ opId: op.opId, op: op.op, ok: false, status: "blocked", blocker })), - blocker + blocker, + otelTraceId: blocker.otelTraceId ?? null, + diagnostic: blocker.diagnostic ?? null }; } @@ -1357,7 +1370,15 @@ function validateHwpodNodeOpsPlan(value) { }; } -function hwpodNodeOpsBlockedPayload(plan, requestMeta, summary) { +function hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, details = {}) { + const blocker = hwpodNodeOpsBlocker({ + code: "hwpod_node_unavailable", + layer: "hwpod-node", + retryable: true, + summary, + details, + userMessage: "hwpod-node 尚未接入执行面；cloud-api 已完成 hwpod-node-ops 合同校验。" + }, requestMeta, details); return { ok: false, status: "blocked", @@ -1371,24 +1392,78 @@ function hwpodNodeOpsBlockedPayload(plan, requestMeta, summary) { op: op.op, ok: false, status: "blocked", - blocker: { - code: "hwpod_node_unavailable", - layer: "hwpod-node", - retryable: true, - summary, - userMessage: "hwpod-node 尚未接入执行面；cloud-api 已完成 hwpod-node-ops 合同校验。" - } + blocker })), - blocker: { - code: "hwpod_node_unavailable", - layer: "hwpod-node", - retryable: true, - summary - }, + blocker, + otelTraceId: blocker.otelTraceId ?? null, + diagnostic: blocker.diagnostic ?? null, requestMeta }; } +function hwpodNodeOpsBlocker(blocker, requestMeta = {}, details = {}) { + const code = cleanText(blocker?.code) || "hwpod_node_ops_blocked"; + const summary = cleanText(blocker?.summary ?? blocker?.message) || "hwpod-node-ops blocked"; + const otelTraceId = cleanText(blocker?.otelTraceId ?? requestMeta?.otelTraceId) || null; + const diagnostic = { + code, + rootCauseCode: code, + summary, + otelTraceId, + traceLine: otelTraceId ? `OTel traceId: ${otelTraceId}` : null, + details: details && typeof details === "object" ? details : {}, + valuesPrinted: false + }; + const userMessage = cleanText(blocker?.userMessage); + return { + ...blocker, + code, + layer: blocker?.layer ?? "hwpod-node", + retryable: blocker?.retryable ?? true, + summary, + ...(otelTraceId ? { otelTraceId } : {}), + diagnostic, + ...(userMessage ? { userMessage: `${userMessage}\n${diagnostic.traceLine ?? "OTel traceId: unavailable"}` } : {}) + }; +} + +function attachHwpodNodeOpsDiagnostics(payload, requestMeta, details = {}) { + if (!payload || typeof payload !== "object" || payload.ok !== false) return payload; + const topBlocker = payload.blocker ? hwpodNodeOpsBlocker(payload.blocker, requestMeta, payload.blocker.details ?? details) : null; + const results = Array.isArray(payload.results) + ? payload.results.map((result) => result?.blocker ? { ...result, blocker: hwpodNodeOpsBlocker(result.blocker, requestMeta, result.blocker.details ?? details) } : result) + : payload.results; + const diagnostic = topBlocker?.diagnostic ?? results?.find?.((result) => result?.blocker?.diagnostic)?.blocker?.diagnostic ?? null; + return { + ...payload, + results, + blocker: topBlocker, + ...(diagnostic?.otelTraceId ? { otelTraceId: diagnostic.otelTraceId } : {}), + ...(diagnostic ? { diagnostic } : {}) + }; +} + +function recordHwpodNodeOpsBlocked(request, options, plan, requestMeta, code, summary, details = {}) { + const httpContext = request?.hwlabHttpRequestContext; + const attributes = { + "hwlab.hwpod.plan_id": cleanText(plan?.planId) || null, + "hwlab.hwpod.hwpod_id": cleanText(plan?.hwpodId) || null, + "hwlab.hwpod.node_id": cleanText(plan?.nodeId) || null, + "hwlab.hwpod.accepted_ops": Array.isArray(plan?.ops) ? plan.ops.length : 0, + "hwlab.hwpod.ops": Array.isArray(plan?.ops) ? plan.ops.map((op) => cleanText(op?.op)).filter(Boolean).join(",").slice(0, 240) : null, + "hwlab.hwpod.blocker.code": code, + "hwlab.hwpod.blocker.summary": String(summary ?? "").slice(0, 500), + "hwlab.hwpod.dispatch_mode": cleanText(details?.dispatchMode) || "unknown", + "hwlab.hwpod.websocket_connected": Boolean(details?.websocketConnected), + "hwlab.hwpod.direct_url_configured": Boolean(details?.directUrlConfigured), + "hwlab.hwpod.otel_trace_id": cleanText(requestMeta?.otelTraceId) || null, + traceId: cleanText(requestMeta?.traceId) || null + }; + if (httpContext) httpContext.otelAttributes = { ...(httpContext.otelAttributes ?? {}), ...attributes }; + const error = Object.assign(new Error(summary), { code }); + emitHttpRoutePhaseSpan(request, options, "hwpod-node-ops.blocked", Date.now(), Date.now(), "error", error, attributes); +} + async function codeAgentOptions(request, response, options, authOptions = {}) { const auth = await options.accessController.authenticate(request, { required: authOptions.required ?? options.accessController.required }); if (!auth.ok) { diff --git a/internal/project-management/source-adapter.ts b/internal/project-management/source-adapter.ts index c3139c0e..750bb2e7 100644 --- a/internal/project-management/source-adapter.ts +++ b/internal/project-management/source-adapter.ts @@ -180,11 +180,32 @@ async function runHwpodOp(hwpodNodeOpsHandler, source, op, args) { const result = Array.isArray(payload?.results) ? payload.results[0] : null; if (!payload?.ok || !result?.ok) { const blocker = result?.blocker ?? payload?.blocker ?? payload?.error ?? {}; - throw sourceError(blocker.code ?? "hwpod_node_ops_failed", blocker.summary ?? blocker.message ?? "HWPOD node-ops request failed", payload?.status === "blocked" ? 503 : 502, { hwpodStatus: payload?.status ?? null }); + const otelTraceId = hwpodNodeOpsPayloadOtelTraceId(payload, blocker); + const summary = String(blocker.summary ?? blocker.message ?? "HWPOD node-ops request failed"); + const traceLine = otelTraceId ? `OTel traceId: ${otelTraceId}` : null; + throw sourceError(blocker.code ?? "hwpod_node_ops_failed", traceLine ? `${summary}\n${traceLine}` : summary, payload?.status === "blocked" ? 503 : 502, { + hwpodStatus: payload?.status ?? null, + otelTraceId: otelTraceId ?? null, + traceLine, + diagnostic: blocker.diagnostic ?? payload?.diagnostic ?? null + }); } return result.output ?? {}; } +function hwpodNodeOpsPayloadOtelTraceId(payload, blocker) { + const resultBlocker = Array.isArray(payload?.results) ? payload.results.find((result) => result?.blocker?.otelTraceId || result?.blocker?.diagnostic?.otelTraceId)?.blocker : null; + const value = payload?.otelTraceId + ?? payload?.diagnostic?.otelTraceId + ?? payload?.requestMeta?.otelTraceId + ?? blocker?.otelTraceId + ?? blocker?.diagnostic?.otelTraceId + ?? resultBlocker?.otelTraceId + ?? resultBlocker?.diagnostic?.otelTraceId; + const text = String(value ?? "").trim(); + return text || null; +} + function fileContentPayload(source, relativePath, content) { const parsed = parseMdtodoDocument(content, { sourceId: source.sourceId, relativePath, projectId: source.projectId }); return { diff --git a/tools/hwpod-harness.test.ts b/tools/hwpod-harness.test.ts index b5b3e565..73e118cc 100644 --- a/tools/hwpod-harness.test.ts +++ b/tools/hwpod-harness.test.ts @@ -344,6 +344,38 @@ test("hwpod-cli submits compiled node ops to hwlab-api when not dry-run", async } }); +test("hwpod-cli surfaces hwpod-node-ops OTel trace id on blocked response", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-hwpod-cli-blocked-")); + const specPath = path.join(root, ".hwlab", "hwpod-spec.yaml"); + const otelTraceId = "fedcba98765432100123456789abcdef"; + try { + await runHwpodCtl(["spec", "init", "--spec", specPath, "--node", "pc-host-1"], { now: () => NOW }); + const result = await runHwpodCli(["workspace", "ls", "src", "--spec", specPath], { + env: { + HWLAB_RUNTIME_API_URL: "http://cloud.test", + HWLAB_RUNTIME_NAMESPACE: "hwlab-v02", + HWLAB_RUNTIME_ENDPOINT_LOCKED: "1", + HWLAB_API_KEY: "hwl_live_test" + }, + fetchImpl: async () => new Response(JSON.stringify({ + ok: false, + status: "blocked", + contractVersion: "hwpod-node-ops-v1", + blocker: { code: "hwpod_node_unavailable", layer: "hwpod-node", retryable: true, summary: "no outbound WebSocket hwpod-node is connected" }, + results: [] + }), { status: 200, headers: { "content-type": "application/json", "x-hwlab-otel-trace-id": otelTraceId } }), + now: () => NOW + }); + assert.equal(result.exitCode, 1); + assert.equal(result.payload.ok, false); + assert.equal(result.payload.otelTraceId, otelTraceId); + assert.equal(result.payload.diagnostic.traceLine, `OTel traceId: ${otelTraceId}`); + assert.equal(result.payload.body.blocker.otelTraceId, otelTraceId); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + test("hwpod-cli rejects API key aliases and requires HWLAB_API_KEY", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-hwpod-cli-api-key-alias-")); const specPath = path.join(root, ".hwlab", "hwpod-spec.yaml"); diff --git a/tools/src/hwpod-harness-lib.ts b/tools/src/hwpod-harness-lib.ts index b72f908f..c3933f4a 100644 --- a/tools/src/hwpod-harness-lib.ts +++ b/tools/src/hwpod-harness-lib.ts @@ -114,7 +114,21 @@ export async function runHwpodCli(argv: string[], options: { env?: EnvLike; fetc } const response = await submitHwpodNodeOpsPlan({ parsed, env, fetchImpl: options.fetchImpl, plan }); const exitCode = response.body?.ok === false || response.status >= 400 ? 1 : 0; - const payload = ok("hwpod-cli.invoke", { specPath: compiled.specPath, hwpodId: compiled.hwpodId, specAuthority: compiled.specAuthority, intent: plan.intent, contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, compilerInvocation: compiled.compilerInvocation, route: response.route, runtimeEndpoint: response.runtimeEndpoint, body: response.body, httpStatus: response.status }, response.body?.status ?? (exitCode === 0 ? "succeeded" : "failed")); + const diagnostic = hwpodNodeOpsCliDiagnostic(response); + const payload = ok("hwpod-cli.invoke", { + specPath: compiled.specPath, + hwpodId: compiled.hwpodId, + specAuthority: compiled.specAuthority, + intent: plan.intent, + contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, + compilerInvocation: compiled.compilerInvocation, + route: response.route, + runtimeEndpoint: response.runtimeEndpoint, + body: response.body, + httpStatus: response.status, + ...(response.otelTraceId ? { otelTraceId: response.otelTraceId } : {}), + ...(diagnostic ? { diagnostic } : {}) + }, response.body?.status ?? (exitCode === 0 ? "succeeded" : "failed")); if (exitCode !== 0) payload.ok = false; return result(exitCode, payload, now); } catch (error) { @@ -925,7 +939,69 @@ async function submitHwpodNodeOpsPlan({ parsed, env, fetchImpl, plan }: { parsed ? await fetchImpl(url, { method: route.method, headers, body: requestBody }) : await postJsonNative(url, { method: route.method, headers, body: requestBody, timeoutMs: numberValue(parsed.timeoutMs) ?? DEFAULT_TIMEOUT_MS }); const body = await response.json().catch(() => null); - return { status: response.status, body, route, runtimeEndpoint: runtimeEndpointVisibility(endpoint) }; + const otelTraceId = responseHeader(response, "x-hwlab-otel-trace-id") || hwpodNodeOpsOtelTraceId(body); + const traceparent = responseHeader(response, "traceparent") || text(body?.requestMeta?.traceparent); + return { status: response.status, body: enrichHwpodNodeOpsBody(body, { otelTraceId, traceparent }), route, runtimeEndpoint: runtimeEndpointVisibility(endpoint), otelTraceId, traceparent }; +} + +function hwpodNodeOpsCliDiagnostic(response: any) { + const body = response?.body; + if (!body || typeof body !== "object" || body.ok !== false) return null; + const blocker = body.blocker ?? body.results?.find?.((result: any) => result?.blocker)?.blocker ?? body.error ?? {}; + const otelTraceId = text(response?.otelTraceId ?? blocker?.otelTraceId ?? blocker?.diagnostic?.otelTraceId ?? body?.otelTraceId ?? body?.requestMeta?.otelTraceId); + return { + code: text(blocker?.code) || "hwpod_node_ops_failed", + summary: text(blocker?.summary ?? blocker?.message) || "hwpod-node-ops request failed", + otelTraceId: otelTraceId || null, + traceLine: otelTraceId ? `OTel traceId: ${otelTraceId}` : null, + route: response?.route, + httpStatus: response?.status, + valuesPrinted: false + }; +} + +function enrichHwpodNodeOpsBody(body: any, meta: { otelTraceId?: string; traceparent?: string } = {}) { + if (!body || typeof body !== "object") return body; + const otelTraceId = text(meta.otelTraceId) || hwpodNodeOpsOtelTraceId(body); + if (!otelTraceId) return body; + const enrichBlocker = (blocker: any) => { + if (!blocker || typeof blocker !== "object") return blocker; + const diagnostic = { + ...(blocker.diagnostic && typeof blocker.diagnostic === "object" ? blocker.diagnostic : {}), + code: text(blocker.code) || text(blocker.diagnostic?.code) || "hwpod_node_ops_failed", + summary: text(blocker.summary ?? blocker.message ?? blocker.diagnostic?.summary) || "hwpod-node-ops request failed", + otelTraceId, + traceLine: `OTel traceId: ${otelTraceId}`, + valuesPrinted: false + }; + return { ...blocker, otelTraceId, diagnostic }; + }; + const blocker = enrichBlocker(body.blocker); + const results = Array.isArray(body.results) + ? body.results.map((result: any) => result?.blocker ? { ...result, blocker: enrichBlocker(result.blocker) } : result) + : body.results; + return { + ...body, + blocker, + results, + otelTraceId, + diagnostic: body.diagnostic ?? blocker?.diagnostic ?? results?.find?.((result: any) => result?.blocker?.diagnostic)?.blocker?.diagnostic, + requestMeta: body.requestMeta && typeof body.requestMeta === "object" + ? { ...body.requestMeta, otelTraceId, ...(meta.traceparent ? { traceparent: meta.traceparent } : {}) } + : body.requestMeta + }; +} + +function hwpodNodeOpsOtelTraceId(body: any) { + return text(body?.otelTraceId ?? body?.diagnostic?.otelTraceId ?? body?.blocker?.otelTraceId ?? body?.blocker?.diagnostic?.otelTraceId ?? body?.requestMeta?.otelTraceId ?? body?.results?.find?.((result: any) => result?.blocker?.otelTraceId)?.blocker?.otelTraceId); +} + +function responseHeader(response: any, name: string) { + const headers = response?.headers; + if (!headers) return ""; + if (typeof headers.get === "function") return text(headers.get(name)); + const raw = headers[String(name).toLowerCase()] ?? headers[name]; + return text(Array.isArray(raw) ? raw[0] : raw); } function authHeaders(parsed: ParsedArgs, env: EnvLike) { @@ -965,6 +1041,7 @@ function requestJsonNative(urlValue: string, { method, headers, timeoutMs }: { m const textValue = Buffer.concat(chunks).toString("utf8"); resolve({ status: response.statusCode ?? 0, + headers: response.headers, json: async () => textValue ? JSON.parse(textValue) : null }); }); @@ -993,6 +1070,7 @@ function postJsonNative(urlValue: string, { method, headers, body, timeoutMs }: const textValue = Buffer.concat(chunks).toString("utf8"); resolve({ status: response.statusCode ?? 0, + headers: response.headers, json: async () => textValue ? JSON.parse(textValue) : null }); });