fix: expose hwpod node ops otel trace diagnostics

2026-07-01 10:53:05 +08:00
parent d1d30d9a7f
commit bf272f7e3e
5 changed files with 294 additions and 66 deletions
@@ -8,8 +8,23 @@ import { createCloudApiBunServer } from "./bun-server.ts";
 import { createCloudApiServer } from "./server.ts";
 import { connectHwpodNodeWs, createHwpodNodeServer } from "../../tools/src/hwpod-node-lib.ts";

+function createHwpodTestCloudApiServer(options: any = {}) {
+  return createCloudApiServer({ accessController: allowAllAccessController(), ...options });
+}
+
+function allowAllAccessController() {
+  return {
+    required: false,
+    configureCodeAgentWorkspaceContext() {},
+    async requireNavAccess() { return true; },
+    async authenticate() {
+      return { ok: true, status: 200, actor: { id: "usr_hwpod_test", role: "admin" }, session: { id: "uss_hwpod_test" } };
+    }
+  };
+}
+
 test("cloud-api exposes hwpod-node-ops contract on /v1", async () => {
-  const server = createCloudApiServer();
+  const server = createHwpodTestCloudApiServer();
  await listen(server);
  try {
    const response = await fetch(`${serverUrl(server)}/v1`);
@@ -31,7 +46,7 @@ test("cloud-api exposes hwpod-node-ops contract on /v1", async () => {
 });

 test("cloud-api exposes hwlab-node Python update metadata", async () => {
-  const server = createCloudApiServer({
+  const server = createHwpodTestCloudApiServer({
    env: {
      PATH: process.env.PATH,
      HWLAB_PUBLIC_ENDPOINT: "https://hwlab.pikapython.com",
@@ -63,7 +78,7 @@ test("cloud-api exposes hwlab-node Python update metadata", async () => {
 });

 test("cloud-api serves bundled hwlab-node Python updater artifact by default", async () => {
-  const server = createCloudApiServer({ env: { PATH: process.env.PATH, HWLAB_PUBLIC_ENDPOINT: "https://hwlab.pikapython.com" } });
+  const server = createHwpodTestCloudApiServer({ env: { PATH: process.env.PATH, HWLAB_PUBLIC_ENDPOINT: "https://hwlab.pikapython.com" } });
  await listen(server);
  try {
    const response = await fetch(`${serverUrl(server)}/v1/hwlab-node/update?current=0.1.0&channel=stable&platform=windows`);
@@ -90,7 +105,7 @@ test("cloud-api discovers preinstalled workspace hwpod-specs without hardcoded d
  await mkdir(specDir, { recursive: true });
  await writeFile(path.join(specDir, "hwpod-spec.yaml"), sampleSpecYaml(), "utf8");
  await writeFile(path.join(specDir, "hwpod-spec.meta.json"), JSON.stringify({ source: { kind: "preinstalled-verified-spec", caseRepo: "pikasTech/hwlab-case-registry", caseId: "d601-f103-v2-compile" } }), "utf8");
-  const server = createCloudApiServer({ env: { PATH: process.env.PATH, HWLAB_CODE_AGENT_WORKSPACE: root } });
+  const server = createHwpodTestCloudApiServer({ env: { PATH: process.env.PATH, HWLAB_CODE_AGENT_WORKSPACE: root } });
  await listen(server);
  try {
    const response = await fetch(`${serverUrl(server)}/v1/hwpod/specs`);
@@ -118,7 +133,7 @@ test("cloud-api discovers configmap-mounted registry hwpod-spec symlinks", async
  const targetPath = path.join(dataDir, "constart-71freq-c.yaml");
  await writeFile(targetPath, sampleConstartSpecYaml(), "utf8");
  await symlink(targetPath, path.join(registryDir, "constart-71freq-c.yaml"));
-  const server = createCloudApiServer({
+  const server = createHwpodTestCloudApiServer({
    env: {
      PATH: process.env.PATH,
      HWLAB_CODE_AGENT_WORKSPACE: root,
@@ -148,7 +163,7 @@ test("cloud-api probes discovered hwpod-spec availability through node-ops", asy
  await mkdir(specDir, { recursive: true });
  await writeFile(path.join(specDir, "hwpod-spec.yaml"), sampleSpecYaml(), "utf8");
  const seen: any[] = [];
-  const server = createCloudApiServer({
+  const server = createHwpodTestCloudApiServer({
    env: { PATH: process.env.PATH, HWLAB_CODE_AGENT_WORKSPACE: root },
    hwpodNodeOpsHandler: async (plan: any) => {
      seen.push(plan);
@@ -174,7 +189,7 @@ test("cloud-api probes discovered hwpod-spec availability through node-ops", asy

 test("cloud-api forwards valid hwpod-node-ops plans to injected node handler", async () => {
  const seen: any[] = [];
-  const server = createCloudApiServer({
+  const server = createHwpodTestCloudApiServer({
    hwpodNodeOpsHandler: async (plan: any, context: any) => {
      seen.push({ plan, context });
      return {
@@ -206,20 +221,27 @@ test("cloud-api forwards valid hwpod-node-ops plans to injected node handler", a
 });

 test("cloud-api returns blocked payload when hwpod-node is not wired yet", async () => {
-  const server = createCloudApiServer();
+  const server = createHwpodTestCloudApiServer();
+  const otelTraceId = "0123456789abcdef0123456789abcdef";
  await listen(server);
  try {
    const response = await fetch(`${serverUrl(server)}/v1/hwpod-node-ops`, {
      method: "POST",
-      headers: { "content-type": "application/json" },
+      headers: { "content-type": "application/json", "x-hwlab-otel-trace-id": otelTraceId },
      body: JSON.stringify(samplePlan())
    });
    const payload = await response.json();
    assert.equal(response.status, 200);
+    assert.equal(response.headers.get("x-hwlab-otel-trace-id"), otelTraceId);
    assert.equal(payload.ok, false);
    assert.equal(payload.status, "blocked");
+    assert.equal(payload.otelTraceId, otelTraceId);
+    assert.equal(payload.requestMeta.otelTraceId, otelTraceId);
    assert.equal(payload.blocker.code, "hwpod_node_unavailable");
+    assert.equal(payload.blocker.otelTraceId, otelTraceId);
+    assert.equal(payload.blocker.diagnostic.traceLine, `OTel traceId: ${otelTraceId}`);
    assert.equal(payload.results[0].blocker.layer, "hwpod-node");
+    assert.match(payload.results[0].blocker.userMessage, /OTel traceId:/u);
  } finally {
    await close(server);
  }
@@ -228,7 +250,7 @@ test("cloud-api returns blocked payload when hwpod-node is not wired yet", async
 test("cloud-api forwards hwpod-node-ops to configured thin hwpod-node URL", async () => {
  const node = createHwpodNodeServer({ nodeId: "pc-host-1" });
  await listen(node);
-  const server = createCloudApiServer({
+  const server = createHwpodTestCloudApiServer({
    env: {
      PATH: process.env.PATH,
      HWLAB_HWPOD_NODE_OPS_URL: `${serverUrl(node)}/v1/hwpod-node-ops`
@@ -258,7 +280,7 @@ test("cloud-api forwards hwpod-node-ops to configured thin hwpod-node URL", asyn
 test("cloud-api blocks direct hwpod-node URL when target node id does not match", async () => {
  const node = createHwpodNodeServer({ nodeId: "g14-host-hwpod-node" });
  await listen(node);
-  const server = createCloudApiServer({
+  const server = createHwpodTestCloudApiServer({
    env: {
      PATH: process.env.PATH,
      HWPOD_NODE_ID: "g14-host-hwpod-node",
@@ -290,7 +312,7 @@ test("cloud-api blocks direct hwpod-node URL when target node id does not match"
 });

 test("cloud-api blocks direct hwpod-node URL when node identity is not visible", async () => {
-  const target = createCloudApiServer({
+  const target = createHwpodTestCloudApiServer({
    hwpodNodeOpsHandler: async (plan: any) => ({
      ok: true,
      status: "completed",
@@ -298,7 +320,7 @@ test("cloud-api blocks direct hwpod-node URL when node identity is not visible",
    })
  });
  await listen(target);
-  const server = createCloudApiServer({
+  const server = createHwpodTestCloudApiServer({
    env: {
      PATH: process.env.PATH,
      HWLAB_HWPOD_NODE_OPS_URL: `${serverUrl(target)}/v1/hwpod-node-ops`
@@ -325,7 +347,7 @@ test("cloud-api blocks direct hwpod-node URL when node identity is not visible",

 test("cloud-api dispatches hwpod-node-ops to outbound WebSocket hwpod-node by nodeId", async () => {
  const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-hwpod-node-ws-"));
-  const runtime = await createCloudApiBunServer({ host: "127.0.0.1", port: 0, env: { PATH: process.env.PATH } });
+  const runtime = await createCloudApiBunServer({ host: "127.0.0.1", port: 0, env: { PATH: process.env.PATH }, accessController: allowAllAccessController() });
  const connector = connectHwpodNodeWs({ cloudUrl: runtime.url, nodeId: "pc-host-1", reconnect: false, heartbeatIntervalMs: 1000 });
  try {
    await writeFile(path.join(root, "main.c"), "int main(void) { return 0; }\n", "utf8");
@@ -352,7 +374,7 @@ test("cloud-api dispatches hwpod-node-ops to outbound WebSocket hwpod-node by no
 });

 test("cloud-api rejects unsupported hwpod-node ops before forwarding", async () => {
-  const server = createCloudApiServer({
+  const server = createHwpodTestCloudApiServer({
    hwpodNodeOpsHandler: async () => {
      throw new Error("must not be called");
    }
@@ -1016,20 +1016,30 @@ async function handleHwpodNodeOpsHttp(request, response, options) {
  }

  const plan = validation.plan;
-  const requestMeta = {
-    requestId: getHeader(request, "x-request-id") || `req_hwpod_${randomUUID()}`,
-    traceId: getHeader(request, "x-trace-id") || `trc_hwpod_${randomUUID()}`,
-    serviceId: getHeader(request, "x-source-service-id") || CLOUD_API_SERVICE_ID,
-    environment: runtimeEnvironment(options.env ?? process.env)
-  };
+  const requestMeta = hwpodNodeOpsRequestMeta(request, options, "hwpod");
  try {
    const handled = await dispatchHwpodNodeOpsPlan(plan, requestMeta, options, { request });
    sendJson(response, handled.httpStatus, handled.payload);
  } catch (error) {
-    sendJson(response, 200, hwpodNodeOpsBlockedPayload(plan, requestMeta, error?.message ?? "hwpod-node-ops handler failed"));
+    const summary = error?.message ?? "hwpod-node-ops handler failed";
+    recordHwpodNodeOpsBlocked(request, options, plan, requestMeta, "hwpod_node_handler_failed", summary, { dispatchMode: "handler-exception" });
+    sendJson(response, 200, hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, { dispatchMode: "handler-exception" }));
  }
 }

+function hwpodNodeOpsRequestMeta(request, options, label) {
+  const httpContext = request?.hwlabHttpRequestContext ?? {};
+  return {
+    requestId: getHeader(request, "x-request-id") || httpContext.requestId || `req_${label}_${randomUUID()}`,
+    traceId: getHeader(request, "x-trace-id") || `trc_${label}_${randomUUID()}`,
+    serviceId: getHeader(request, "x-source-service-id") || CLOUD_API_SERVICE_ID,
+    environment: runtimeEnvironment(options.env ?? process.env),
+    otelTraceId: httpContext.traceId ?? null,
+    traceparent: httpContext.traceparent ?? null,
+    valuesPrinted: false
+  };
+}
+
 function handleHwlabNodeUpdateHttp(request, response, url, options) {
  if (request.method !== "GET") {
    sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "GET required." } });
@@ -1162,12 +1172,7 @@ async function probeDiscoveredHwpodSpec(spec, { request, options, observedAt })
      }
    };
  }
-  const requestMeta = {
-    requestId: getHeader(request, "x-request-id") || `req_hwpod_spec_${randomUUID()}`,
-    traceId: getHeader(request, "x-trace-id") || `trc_hwpod_spec_${randomUUID()}`,
-    serviceId: getHeader(request, "x-source-service-id") || CLOUD_API_SERVICE_ID,
-    environment: runtimeEnvironment(options.env ?? process.env)
-  };
+  const requestMeta = hwpodNodeOpsRequestMeta(request, options, "hwpod_spec");
  const handled = await dispatchHwpodNodeOpsPlan(validation.plan, requestMeta, options, { request });
  const payload = handled.payload;
  return {
@@ -1190,9 +1195,17 @@ async function dispatchHwpodNodeOpsPlan(plan, requestMeta, options, context = {}
  const hwpodNodeWsRegistry = options.hwpodNodeWsRegistry;
  const hasWsNode = hwpodNodeWsRegistry?.hasNode?.(plan.nodeId);
  if (typeof options.hwpodNodeOpsHandler !== "function" && !hasWsNode && !hwpodNodeOpsUrl) {
+    const summary = "no outbound WebSocket hwpod-node is connected and HWLAB_HWPOD_NODE_OPS_URL is not configured; cloud-api is only validating the hwpod-node-ops contract";
+    const details = {
+      dispatchMode: "none",
+      websocketRegistryConfigured: Boolean(hwpodNodeWsRegistry),
+      websocketConnected: false,
+      directUrlConfigured: false
+    };
+    recordHwpodNodeOpsBlocked(context.request, options, plan, requestMeta, "hwpod_node_unavailable", summary, details);
    return {
      httpStatus: 200,
-      payload: hwpodNodeOpsBlockedPayload(plan, requestMeta, "no outbound WebSocket hwpod-node is connected and HWLAB_HWPOD_NODE_OPS_URL is not configured; cloud-api is only validating the hwpod-node-ops contract")
+      payload: hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, details)
    };
  }
  const handled = typeof options.hwpodNodeOpsHandler === "function"
@@ -1202,7 +1215,7 @@ async function dispatchHwpodNodeOpsPlan(plan, requestMeta, options, context = {}
      : await forwardHwpodNodeOpsPlan(hwpodNodeOpsUrl, plan, requestMeta, options);
  const results = Array.isArray(handled?.results) ? handled.results : [];
  const failed = results.some((item) => item?.ok === false);
-  const payload = {
+  const payload = attachHwpodNodeOpsDiagnostics({
    ok: handled?.ok ?? !failed,
    status: handled?.status ?? (failed ? "failed" : "completed"),
    contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION,
@@ -1213,7 +1226,10 @@ async function dispatchHwpodNodeOpsPlan(plan, requestMeta, options, context = {}
    results,
    blocker: handled?.blocker ?? null,
    requestMeta
-  };
+  }, requestMeta, { dispatchMode: hasWsNode ? "websocket" : hwpodNodeOpsUrl ? "direct-url" : "handler" });
+  if (payload.ok === false && payload.status === "blocked" && payload.blocker?.code) {
+    recordHwpodNodeOpsBlocked(context.request, options, plan, requestMeta, payload.blocker.code, payload.blocker.summary ?? "hwpod-node-ops blocked", payload.blocker.details ?? {});
+  }
  return { httpStatus: handled?.httpStatus ?? (failed ? 409 : 200), payload };
 }

@@ -1226,7 +1242,7 @@ async function forwardHwpodNodeOpsPlan(targetUrl, plan, requestMeta, options) {
      dispatchMode: "direct-url",
      targetStatus: target.status ?? null,
      error: target.error ?? null
-    });
+    }, requestMeta);
  }
  if (target.ok && target.nodeId && target.nodeId !== plan.nodeId) {
    return directHwpodNodeBlocked(plan, "hwpod_node_id_mismatch", `HWLAB_HWPOD_NODE_OPS_URL points to ${target.nodeId}, not requested node ${plan.nodeId}`, {
@@ -1234,7 +1250,7 @@ async function forwardHwpodNodeOpsPlan(targetUrl, plan, requestMeta, options) {
      targetNodeId: target.nodeId,
      targetUrl: redactNodeOpsUrl(targetUrl),
      dispatchMode: "direct-url"
-    });
+    }, requestMeta);
  }
  const response = await fetch(targetUrl, {
    method: "POST",
@@ -1242,25 +1258,20 @@ async function forwardHwpodNodeOpsPlan(targetUrl, plan, requestMeta, options) {
      "content-type": "application/json",
      "x-request-id": requestMeta.requestId,
      "x-trace-id": requestMeta.traceId,
-      "x-source-service-id": CLOUD_API_SERVICE_ID
+      "x-source-service-id": CLOUD_API_SERVICE_ID,
+      ...(requestMeta.otelTraceId ? { "x-hwlab-otel-trace-id": requestMeta.otelTraceId } : {}),
+      ...(requestMeta.traceparent ? { traceparent: requestMeta.traceparent } : {})
    },
    body: JSON.stringify(plan),
    signal: AbortSignal.timeout(parsePositiveInteger(options.env?.HWLAB_HWPOD_NODE_OPS_TIMEOUT_MS, 30000))
  });
  const body = await response.json().catch(() => null);
  if (!body || typeof body !== "object") {
-    return {
-      ok: false,
-      status: "blocked",
-      httpStatus: 200,
-      results: [],
-      blocker: {
-        code: "hwpod_node_response_invalid",
-        layer: "hwpod-node",
-        retryable: true,
-        summary: `hwpod-node returned HTTP ${response.status} without a JSON object payload`
-      }
-    };
+    return directHwpodNodeBlocked(plan, "hwpod_node_response_invalid", `hwpod-node returned HTTP ${response.status} without a JSON object payload`, {
+      targetUrl: redactNodeOpsUrl(targetUrl),
+      dispatchMode: "direct-url",
+      targetStatus: response.status
+    }, requestMeta);
  }
  return {
    ok: body.ok,
@@ -1271,14 +1282,16 @@ async function forwardHwpodNodeOpsPlan(targetUrl, plan, requestMeta, options) {
  };
 }

-function directHwpodNodeBlocked(plan, code, summary, details) {
-  const blocker = { code, layer: "hwpod-node", retryable: true, summary, details };
+function directHwpodNodeBlocked(plan, code, summary, details, requestMeta = {}) {
+  const blocker = hwpodNodeOpsBlocker({ code, layer: "hwpod-node", retryable: true, summary, details }, requestMeta, details);
  return {
    ok: false,
    status: "blocked",
    httpStatus: 200,
    results: plan.ops.map((op) => ({ opId: op.opId, op: op.op, ok: false, status: "blocked", blocker })),
-    blocker
+    blocker,
+    otelTraceId: blocker.otelTraceId ?? null,
+    diagnostic: blocker.diagnostic ?? null
  };
 }

@@ -1357,7 +1370,15 @@ function validateHwpodNodeOpsPlan(value) {
  };
 }

-function hwpodNodeOpsBlockedPayload(plan, requestMeta, summary) {
+function hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, details = {}) {
+  const blocker = hwpodNodeOpsBlocker({
+    code: "hwpod_node_unavailable",
+    layer: "hwpod-node",
+    retryable: true,
+    summary,
+    details,
+    userMessage: "hwpod-node 尚未接入执行面；cloud-api 已完成 hwpod-node-ops 合同校验。"
+  }, requestMeta, details);
  return {
    ok: false,
    status: "blocked",
@@ -1371,24 +1392,78 @@ function hwpodNodeOpsBlockedPayload(plan, requestMeta, summary) {
      op: op.op,
      ok: false,
      status: "blocked",
-      blocker: {
-        code: "hwpod_node_unavailable",
-        layer: "hwpod-node",
-        retryable: true,
-        summary,
-        userMessage: "hwpod-node 尚未接入执行面；cloud-api 已完成 hwpod-node-ops 合同校验。"
-      }
+      blocker
    })),
-    blocker: {
-      code: "hwpod_node_unavailable",
-      layer: "hwpod-node",
-      retryable: true,
-      summary
-    },
+    blocker,
+    otelTraceId: blocker.otelTraceId ?? null,
+    diagnostic: blocker.diagnostic ?? null,
    requestMeta
  };
 }

+function hwpodNodeOpsBlocker(blocker, requestMeta = {}, details = {}) {
+  const code = cleanText(blocker?.code) || "hwpod_node_ops_blocked";
+  const summary = cleanText(blocker?.summary ?? blocker?.message) || "hwpod-node-ops blocked";
+  const otelTraceId = cleanText(blocker?.otelTraceId ?? requestMeta?.otelTraceId) || null;
+  const diagnostic = {
+    code,
+    rootCauseCode: code,
+    summary,
+    otelTraceId,
+    traceLine: otelTraceId ? `OTel traceId: ${otelTraceId}` : null,
+    details: details && typeof details === "object" ? details : {},
+    valuesPrinted: false
+  };
+  const userMessage = cleanText(blocker?.userMessage);
+  return {
+    ...blocker,
+    code,
+    layer: blocker?.layer ?? "hwpod-node",
+    retryable: blocker?.retryable ?? true,
+    summary,
+    ...(otelTraceId ? { otelTraceId } : {}),
+    diagnostic,
+    ...(userMessage ? { userMessage: `${userMessage}\n${diagnostic.traceLine ?? "OTel traceId: unavailable"}` } : {})
+  };
+}
+
+function attachHwpodNodeOpsDiagnostics(payload, requestMeta, details = {}) {
+  if (!payload || typeof payload !== "object" || payload.ok !== false) return payload;
+  const topBlocker = payload.blocker ? hwpodNodeOpsBlocker(payload.blocker, requestMeta, payload.blocker.details ?? details) : null;
+  const results = Array.isArray(payload.results)
+    ? payload.results.map((result) => result?.blocker ? { ...result, blocker: hwpodNodeOpsBlocker(result.blocker, requestMeta, result.blocker.details ?? details) } : result)
+    : payload.results;
+  const diagnostic = topBlocker?.diagnostic ?? results?.find?.((result) => result?.blocker?.diagnostic)?.blocker?.diagnostic ?? null;
+  return {
+    ...payload,
+    results,
+    blocker: topBlocker,
+    ...(diagnostic?.otelTraceId ? { otelTraceId: diagnostic.otelTraceId } : {}),
+    ...(diagnostic ? { diagnostic } : {})
+  };
+}
+
+function recordHwpodNodeOpsBlocked(request, options, plan, requestMeta, code, summary, details = {}) {
+  const httpContext = request?.hwlabHttpRequestContext;
+  const attributes = {
+    "hwlab.hwpod.plan_id": cleanText(plan?.planId) || null,
+    "hwlab.hwpod.hwpod_id": cleanText(plan?.hwpodId) || null,
+    "hwlab.hwpod.node_id": cleanText(plan?.nodeId) || null,
+    "hwlab.hwpod.accepted_ops": Array.isArray(plan?.ops) ? plan.ops.length : 0,
+    "hwlab.hwpod.ops": Array.isArray(plan?.ops) ? plan.ops.map((op) => cleanText(op?.op)).filter(Boolean).join(",").slice(0, 240) : null,
+    "hwlab.hwpod.blocker.code": code,
+    "hwlab.hwpod.blocker.summary": String(summary ?? "").slice(0, 500),
+    "hwlab.hwpod.dispatch_mode": cleanText(details?.dispatchMode) || "unknown",
+    "hwlab.hwpod.websocket_connected": Boolean(details?.websocketConnected),
+    "hwlab.hwpod.direct_url_configured": Boolean(details?.directUrlConfigured),
+    "hwlab.hwpod.otel_trace_id": cleanText(requestMeta?.otelTraceId) || null,
+    traceId: cleanText(requestMeta?.traceId) || null
+  };
+  if (httpContext) httpContext.otelAttributes = { ...(httpContext.otelAttributes ?? {}), ...attributes };
+  const error = Object.assign(new Error(summary), { code });
+  emitHttpRoutePhaseSpan(request, options, "hwpod-node-ops.blocked", Date.now(), Date.now(), "error", error, attributes);
+}
+
 async function codeAgentOptions(request, response, options, authOptions = {}) {
  const auth = await options.accessController.authenticate(request, { required: authOptions.required ?? options.accessController.required });
  if (!auth.ok) {
@@ -180,11 +180,32 @@ async function runHwpodOp(hwpodNodeOpsHandler, source, op, args) {
  const result = Array.isArray(payload?.results) ? payload.results[0] : null;
  if (!payload?.ok || !result?.ok) {
    const blocker = result?.blocker ?? payload?.blocker ?? payload?.error ?? {};
-    throw sourceError(blocker.code ?? "hwpod_node_ops_failed", blocker.summary ?? blocker.message ?? "HWPOD node-ops request failed", payload?.status === "blocked" ? 503 : 502, { hwpodStatus: payload?.status ?? null });
+    const otelTraceId = hwpodNodeOpsPayloadOtelTraceId(payload, blocker);
+    const summary = String(blocker.summary ?? blocker.message ?? "HWPOD node-ops request failed");
+    const traceLine = otelTraceId ? `OTel traceId: ${otelTraceId}` : null;
+    throw sourceError(blocker.code ?? "hwpod_node_ops_failed", traceLine ? `${summary}\n${traceLine}` : summary, payload?.status === "blocked" ? 503 : 502, {
+      hwpodStatus: payload?.status ?? null,
+      otelTraceId: otelTraceId ?? null,
+      traceLine,
+      diagnostic: blocker.diagnostic ?? payload?.diagnostic ?? null
+    });
  }
  return result.output ?? {};
 }

+function hwpodNodeOpsPayloadOtelTraceId(payload, blocker) {
+  const resultBlocker = Array.isArray(payload?.results) ? payload.results.find((result) => result?.blocker?.otelTraceId || result?.blocker?.diagnostic?.otelTraceId)?.blocker : null;
+  const value = payload?.otelTraceId
+    ?? payload?.diagnostic?.otelTraceId
+    ?? payload?.requestMeta?.otelTraceId
+    ?? blocker?.otelTraceId
+    ?? blocker?.diagnostic?.otelTraceId
+    ?? resultBlocker?.otelTraceId
+    ?? resultBlocker?.diagnostic?.otelTraceId;
+  const text = String(value ?? "").trim();
+  return text || null;
+}
+
 function fileContentPayload(source, relativePath, content) {
  const parsed = parseMdtodoDocument(content, { sourceId: source.sourceId, relativePath, projectId: source.projectId });
  return {
@@ -344,6 +344,38 @@ test("hwpod-cli submits compiled node ops to hwlab-api when not dry-run", async
  }
 });

+test("hwpod-cli surfaces hwpod-node-ops OTel trace id on blocked response", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-hwpod-cli-blocked-"));
+  const specPath = path.join(root, ".hwlab", "hwpod-spec.yaml");
+  const otelTraceId = "fedcba98765432100123456789abcdef";
+  try {
+    await runHwpodCtl(["spec", "init", "--spec", specPath, "--node", "pc-host-1"], { now: () => NOW });
+    const result = await runHwpodCli(["workspace", "ls", "src", "--spec", specPath], {
+      env: {
+        HWLAB_RUNTIME_API_URL: "http://cloud.test",
+        HWLAB_RUNTIME_NAMESPACE: "hwlab-v02",
+        HWLAB_RUNTIME_ENDPOINT_LOCKED: "1",
+        HWLAB_API_KEY: "hwl_live_test"
+      },
+      fetchImpl: async () => new Response(JSON.stringify({
+        ok: false,
+        status: "blocked",
+        contractVersion: "hwpod-node-ops-v1",
+        blocker: { code: "hwpod_node_unavailable", layer: "hwpod-node", retryable: true, summary: "no outbound WebSocket hwpod-node is connected" },
+        results: []
+      }), { status: 200, headers: { "content-type": "application/json", "x-hwlab-otel-trace-id": otelTraceId } }),
+      now: () => NOW
+    });
+    assert.equal(result.exitCode, 1);
+    assert.equal(result.payload.ok, false);
+    assert.equal(result.payload.otelTraceId, otelTraceId);
+    assert.equal(result.payload.diagnostic.traceLine, `OTel traceId: ${otelTraceId}`);
+    assert.equal(result.payload.body.blocker.otelTraceId, otelTraceId);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
 test("hwpod-cli rejects API key aliases and requires HWLAB_API_KEY", async () => {
  const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-hwpod-cli-api-key-alias-"));
  const specPath = path.join(root, ".hwlab", "hwpod-spec.yaml");
@@ -114,7 +114,21 @@ export async function runHwpodCli(argv: string[], options: { env?: EnvLike; fetc
    }
    const response = await submitHwpodNodeOpsPlan({ parsed, env, fetchImpl: options.fetchImpl, plan });
    const exitCode = response.body?.ok === false || response.status >= 400 ? 1 : 0;
-    const payload = ok("hwpod-cli.invoke", { specPath: compiled.specPath, hwpodId: compiled.hwpodId, specAuthority: compiled.specAuthority, intent: plan.intent, contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, compilerInvocation: compiled.compilerInvocation, route: response.route, runtimeEndpoint: response.runtimeEndpoint, body: response.body, httpStatus: response.status }, response.body?.status ?? (exitCode === 0 ? "succeeded" : "failed"));
+    const diagnostic = hwpodNodeOpsCliDiagnostic(response);
+    const payload = ok("hwpod-cli.invoke", {
+      specPath: compiled.specPath,
+      hwpodId: compiled.hwpodId,
+      specAuthority: compiled.specAuthority,
+      intent: plan.intent,
+      contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION,
+      compilerInvocation: compiled.compilerInvocation,
+      route: response.route,
+      runtimeEndpoint: response.runtimeEndpoint,
+      body: response.body,
+      httpStatus: response.status,
+      ...(response.otelTraceId ? { otelTraceId: response.otelTraceId } : {}),
+      ...(diagnostic ? { diagnostic } : {})
+    }, response.body?.status ?? (exitCode === 0 ? "succeeded" : "failed"));
    if (exitCode !== 0) payload.ok = false;
    return result(exitCode, payload, now);
  } catch (error) {
@@ -925,7 +939,69 @@ async function submitHwpodNodeOpsPlan({ parsed, env, fetchImpl, plan }: { parsed
    ? await fetchImpl(url, { method: route.method, headers, body: requestBody })
    : await postJsonNative(url, { method: route.method, headers, body: requestBody, timeoutMs: numberValue(parsed.timeoutMs) ?? DEFAULT_TIMEOUT_MS });
  const body = await response.json().catch(() => null);
-  return { status: response.status, body, route, runtimeEndpoint: runtimeEndpointVisibility(endpoint) };
+  const otelTraceId = responseHeader(response, "x-hwlab-otel-trace-id") || hwpodNodeOpsOtelTraceId(body);
+  const traceparent = responseHeader(response, "traceparent") || text(body?.requestMeta?.traceparent);
+  return { status: response.status, body: enrichHwpodNodeOpsBody(body, { otelTraceId, traceparent }), route, runtimeEndpoint: runtimeEndpointVisibility(endpoint), otelTraceId, traceparent };
+}
+
+function hwpodNodeOpsCliDiagnostic(response: any) {
+  const body = response?.body;
+  if (!body || typeof body !== "object" || body.ok !== false) return null;
+  const blocker = body.blocker ?? body.results?.find?.((result: any) => result?.blocker)?.blocker ?? body.error ?? {};
+  const otelTraceId = text(response?.otelTraceId ?? blocker?.otelTraceId ?? blocker?.diagnostic?.otelTraceId ?? body?.otelTraceId ?? body?.requestMeta?.otelTraceId);
+  return {
+    code: text(blocker?.code) || "hwpod_node_ops_failed",
+    summary: text(blocker?.summary ?? blocker?.message) || "hwpod-node-ops request failed",
+    otelTraceId: otelTraceId || null,
+    traceLine: otelTraceId ? `OTel traceId: ${otelTraceId}` : null,
+    route: response?.route,
+    httpStatus: response?.status,
+    valuesPrinted: false
+  };
+}
+
+function enrichHwpodNodeOpsBody(body: any, meta: { otelTraceId?: string; traceparent?: string } = {}) {
+  if (!body || typeof body !== "object") return body;
+  const otelTraceId = text(meta.otelTraceId) || hwpodNodeOpsOtelTraceId(body);
+  if (!otelTraceId) return body;
+  const enrichBlocker = (blocker: any) => {
+    if (!blocker || typeof blocker !== "object") return blocker;
+    const diagnostic = {
+      ...(blocker.diagnostic && typeof blocker.diagnostic === "object" ? blocker.diagnostic : {}),
+      code: text(blocker.code) || text(blocker.diagnostic?.code) || "hwpod_node_ops_failed",
+      summary: text(blocker.summary ?? blocker.message ?? blocker.diagnostic?.summary) || "hwpod-node-ops request failed",
+      otelTraceId,
+      traceLine: `OTel traceId: ${otelTraceId}`,
+      valuesPrinted: false
+    };
+    return { ...blocker, otelTraceId, diagnostic };
+  };
+  const blocker = enrichBlocker(body.blocker);
+  const results = Array.isArray(body.results)
+    ? body.results.map((result: any) => result?.blocker ? { ...result, blocker: enrichBlocker(result.blocker) } : result)
+    : body.results;
+  return {
+    ...body,
+    blocker,
+    results,
+    otelTraceId,
+    diagnostic: body.diagnostic ?? blocker?.diagnostic ?? results?.find?.((result: any) => result?.blocker?.diagnostic)?.blocker?.diagnostic,
+    requestMeta: body.requestMeta && typeof body.requestMeta === "object"
+      ? { ...body.requestMeta, otelTraceId, ...(meta.traceparent ? { traceparent: meta.traceparent } : {}) }
+      : body.requestMeta
+  };
+}
+
+function hwpodNodeOpsOtelTraceId(body: any) {
+  return text(body?.otelTraceId ?? body?.diagnostic?.otelTraceId ?? body?.blocker?.otelTraceId ?? body?.blocker?.diagnostic?.otelTraceId ?? body?.requestMeta?.otelTraceId ?? body?.results?.find?.((result: any) => result?.blocker?.otelTraceId)?.blocker?.otelTraceId);
+}
+
+function responseHeader(response: any, name: string) {
+  const headers = response?.headers;
+  if (!headers) return "";
+  if (typeof headers.get === "function") return text(headers.get(name));
+  const raw = headers[String(name).toLowerCase()] ?? headers[name];
+  return text(Array.isArray(raw) ? raw[0] : raw);
 }

 function authHeaders(parsed: ParsedArgs, env: EnvLike) {
@@ -965,6 +1041,7 @@ function requestJsonNative(urlValue: string, { method, headers, timeoutMs }: { m
        const textValue = Buffer.concat(chunks).toString("utf8");
        resolve({
          status: response.statusCode ?? 0,
+          headers: response.headers,
          json: async () => textValue ? JSON.parse(textValue) : null
        });
      });
@@ -993,6 +1070,7 @@ function postJsonNative(urlValue: string, { method, headers, body, timeoutMs }:
        const textValue = Buffer.concat(chunks).toString("utf8");
        resolve({
          status: response.statusCode ?? 0,
+          headers: response.headers,
          json: async () => textValue ? JSON.parse(textValue) : null
        });
      });