From 65405d3f2db0ea1c564c10d03f69c910f52ae600 Mon Sep 17 00:00:00 2001 From: lyon Date: Thu, 25 Jun 2026 18:33:21 +0800 Subject: [PATCH] feat: trace workbench launch failures --- .../server-workbench-launch-http.test.ts | 111 +++++++++ .../cloud/server-workbench-launch-http.ts | 214 ++++++++++++++---- 2 files changed, 285 insertions(+), 40 deletions(-) create mode 100644 internal/cloud/server-workbench-launch-http.test.ts diff --git a/internal/cloud/server-workbench-launch-http.test.ts b/internal/cloud/server-workbench-launch-http.test.ts new file mode 100644 index 00000000..49767765 --- /dev/null +++ b/internal/cloud/server-workbench-launch-http.test.ts @@ -0,0 +1,111 @@ +import assert from "node:assert/strict"; +import { Readable } from "node:stream"; +import { test } from "bun:test"; + +import { handleWorkbenchLaunchHttp } from "./server-workbench-launch-http.ts"; + +test("workbench launch records OTel stage for Project Management link write 404", async () => { + const originalFetch = globalThis.fetch; + const request = jsonRequest({ + projectId: "project_hwlab_v03", + taskRef: "mdtodo:hwlab-v03-mdtodo:file_demo:t4_open", + sessionId: "ses_launch_otel", + conversationId: "cnv_launch_otel" + }); + const response = jsonResponse(); + const fetchCalls = []; + globalThis.fetch = async (url, init) => { + fetchCalls.push({ url: String(url), method: init?.method, body: init?.body }); + return new Response(JSON.stringify({ error: { code: "not_found", message: "Project link route was not found." } }), { + status: 404, + headers: { "content-type": "application/json" } + }); + }; + try { + await handleWorkbenchLaunchHttp(request, response, { + env: { HWLAB_PROJECT_MANAGEMENT_URL: "http://project-management.test" }, + accessController: { + async authenticate() { + return { ok: true, authMethod: "web-session", actor: { id: "usr_launch_otel", role: "admin" } }; + }, + async recordAgentSessionOwner(input) { + return { + id: input.sessionId, + ownerUserId: input.ownerUserId, + ownerRole: input.ownerRole, + projectId: input.projectId, + conversationId: input.conversationId, + threadId: input.threadId, + traceId: input.traceId, + status: input.status, + session: input.session, + createdAt: "2026-06-25T10:00:00.000Z", + updatedAt: "2026-06-25T10:00:00.000Z" + }; + } + }, + runtimeStore: { + async writeWorkbenchFacts() { + return { ok: true }; + } + } + }); + } finally { + globalThis.fetch = originalFetch; + } + + assert.equal(response.statusCode, 404); + assert.equal(fetchCalls.length, 1); + const body = JSON.parse(response.body); + assert.equal(body.error.code, "not_found"); + const context = request.hwlabHttpRequestContext; + assert.equal(context.route, "/v1/workbench/launches"); + assert.equal(context.lastHttpError.code, "not_found"); + assert.equal(context.lastHttpError.category, "workbench_launch"); + assert.equal(context.lastHttpError.layer, "workbench.launch.project_link_write"); + assert.equal(context.otelAttributes["workbench.launch.stage"], "project_link_write"); + assert.equal(context.otelAttributes["workbench.launch.result"], "failed"); + assert.equal(context.otelAttributes["workbench.launch.upstream_status"], 404); + assert.equal(context.otelAttributes["workbench.launch.session_id"], "ses_launch_otel"); + assert.equal(context.otelAttributes["workbench.launch.task_ref"], "mdtodo:hwlab-v03-mdtodo:file_demo:t4_open"); + assert.match(context.otelAttributes["workbench.launch.task_ref_hash"], /^[0-9a-f]{24}$/u); + assert.equal(context.otelAttributes["workbench.launch.values_redacted"], true); +}); + +function jsonRequest(body) { + const request = Readable.from([JSON.stringify(body)]); + request.method = "POST"; + request.url = "/v1/workbench/launches"; + request.headers = { "content-type": "application/json" }; + request.hwlabHttpRequestContext = { + requestId: "req_launch_otel", + startedAtMs: Date.now(), + traceId: "11111111111111111111111111111111", + parentSpanId: null, + spanId: "2222222222222222", + traceparent: "00-11111111111111111111111111111111-2222222222222222-01", + method: "POST", + route: "/v1/workbench/launches", + valuesPrinted: false + }; + return request; +} + +function jsonResponse() { + return { + headersSent: false, + writableEnded: false, + statusCode: 0, + headers: {}, + body: "", + writeHead(statusCode, headers) { + this.statusCode = statusCode; + this.headers = headers; + this.headersSent = true; + }, + end(chunk) { + this.body += String(chunk ?? ""); + this.writableEnded = true; + } + }; +} diff --git a/internal/cloud/server-workbench-launch-http.ts b/internal/cloud/server-workbench-launch-http.ts index d05f3886..66336a9b 100644 --- a/internal/cloud/server-workbench-launch-http.ts +++ b/internal/cloud/server-workbench-launch-http.ts @@ -5,23 +5,44 @@ import { createHash, randomUUID } from "node:crypto"; import { getHeader, readBody, safeConversationId, safeOpaqueId, safeSessionId, safeTraceId, sendJson } from "./server-http-utils.ts"; +import { emitHttpServerRequestSpan, newOtelSpanId } from "./otel-trace.ts"; import { writeWorkbenchSessionAdmissionFact } from "./workbench-projection-writer.ts"; const contractVersion = "workbench-launch-v1"; const defaultProjectId = "project_hwlab_v03"; export async function handleWorkbenchLaunchHttp(request, response, options = {}) { - if (request.method !== "POST") return sendJson(response, 405, launchError("method_not_allowed", "POST required.")); + const env = options.env ?? process.env; + attachWorkbenchLaunchOtel(request, { stage: "start", result: "started" }); + if (request.method !== "POST") { + attachWorkbenchLaunchOtel(request, { stage: "method", result: "failed", errorCode: "method_not_allowed", statusCode: 405 }); + emitWorkbenchLaunchOtelSpan(request, env, "method", { result: "failed", errorCode: "method_not_allowed", statusCode: 405 }, { status: "error" }); + return sendJson(response, 405, launchError("method_not_allowed", "POST required.")); + } const auth = await options.accessController?.authenticate?.(request, { required: true }); - if (!auth?.ok) return sendJson(response, auth?.status ?? 401, auth ?? launchError("auth_required", "Authentication is required.")); + if (!auth?.ok) { + const statusCode = auth?.status ?? 401; + const errorCode = auth?.error?.code ?? auth?.code ?? "auth_required"; + attachWorkbenchLaunchOtel(request, { stage: "auth", result: "failed", errorCode, statusCode }); + emitWorkbenchLaunchOtelSpan(request, env, "auth", { result: "failed", errorCode, statusCode }, { status: "error" }); + return sendJson(response, statusCode, auth ?? launchError("auth_required", "Authentication is required.")); + } const body = await readJsonObject(request, options.bodyLimitBytes); - if (!body.ok) return sendJson(response, 400, launchError(body.code, body.message, { reason: body.reason })); + if (!body.ok) { + attachWorkbenchLaunchOtel(request, { stage: "parse_body", result: "failed", errorCode: body.code, statusCode: 400 }); + emitWorkbenchLaunchOtelSpan(request, env, "parse_body", { result: "failed", errorCode: body.code, statusCode: 400 }, { status: "error" }); + return sendJson(response, 400, launchError(body.code, body.message, { reason: body.reason })); + } const params = body.value; const projectId = safeToken(params.projectId, defaultProjectId); const taskRef = safeProjectOpaque(params.taskRef); - if (!taskRef) return sendJson(response, 400, launchError("task_ref_required", "Workbench launch requires taskRef.")); + if (!taskRef) { + attachWorkbenchLaunchOtel(request, { stage: "validate", result: "failed", projectId, errorCode: "task_ref_required", statusCode: 400 }); + emitWorkbenchLaunchOtelSpan(request, env, "validate", { result: "failed", projectId, errorCode: "task_ref_required", statusCode: 400 }, { status: "error" }); + return sendJson(response, 400, launchError("task_ref_required", "Workbench launch requires taskRef.")); + } const now = new Date().toISOString(); const conversationId = safeConversationId(params.conversationId) || `cnv_${randomUUID()}`; @@ -30,61 +51,94 @@ export async function handleWorkbenchLaunchHttp(request, response, options = {}) const providerProfile = safeToken(params.providerProfile ?? params.codeAgentProviderProfile, null); const launchContext = normalizeLaunchContext(params.launchContext, { projectId, taskRef, now }); const linkId = safeToken(params.linkId, stableLinkId({ projectId, taskRef, sessionId })); + const baseOtel = { projectId, taskRef, sessionId, conversationId, traceId, linkId }; + attachWorkbenchLaunchOtel(request, { ...baseOtel, stage: "session_owner", result: "started" }); - const session = await options.accessController.recordAgentSessionOwner({ - ownerUserId: auth.actor?.id, - ownerRole: auth.actor?.role, - sessionId, - projectId, - agentId: "hwlab-code-agent", - status: "idle", - conversationId, - threadId: safeOpaqueId(params.threadId), - traceId, - session: { - source: "workbench-launch", - providerProfile, - sessionStatus: "idle", - launchContext, - createdAt: now, - valuesRedacted: true, - secretMaterialStored: false - } - }); - await writeWorkbenchSessionAdmissionFact({ - runtimeStore: options.runtimeStore, - session, - ownerUserId: auth.actor?.id, - ownerRole: auth.actor?.role, - projectId, - conversationId, - threadId: safeOpaqueId(params.threadId), - status: "idle", - now - }); + let session; + const sessionOwnerStartedAt = Date.now(); + try { + session = await options.accessController.recordAgentSessionOwner({ + ownerUserId: auth.actor?.id, + ownerRole: auth.actor?.role, + sessionId, + projectId, + agentId: "hwlab-code-agent", + status: "idle", + conversationId, + threadId: safeOpaqueId(params.threadId), + traceId, + session: { + source: "workbench-launch", + providerProfile, + sessionStatus: "idle", + launchContext, + createdAt: now, + valuesRedacted: true, + secretMaterialStored: false + } + }); + attachWorkbenchLaunchOtel(request, { ...baseOtel, stage: "session_owner", result: "ok", ownerUserId: auth.actor?.id ?? null, sessionId: session?.id ?? sessionId }); + emitWorkbenchLaunchOtelSpan(request, env, "session_owner", { ...baseOtel, sessionId: session?.id ?? sessionId, result: "ok", ownerUserId: auth.actor?.id ?? null, statusCode: 200 }, { startTimeMs: sessionOwnerStartedAt }); + } catch (error) { + attachWorkbenchLaunchOtel(request, { ...baseOtel, stage: "session_owner", result: "failed", errorCode: "workbench_launch_session_owner_failed", statusCode: 500 }); + emitWorkbenchLaunchOtelSpan(request, env, "session_owner", { ...baseOtel, result: "failed", errorCode: "workbench_launch_session_owner_failed", statusCode: 500 }, { startTimeMs: sessionOwnerStartedAt, status: "error", error }); + throw error; + } - const workbenchUrl = `/workbench/sessions/${encodeURIComponent(session?.id ?? sessionId)}`; + const admittedSessionId = session?.id ?? sessionId; + const admissionStartedAt = Date.now(); + attachWorkbenchLaunchOtel(request, { ...baseOtel, sessionId: admittedSessionId, stage: "admission_fact", result: "started" }); + try { + const admissionWrite = await writeWorkbenchSessionAdmissionFact({ + runtimeStore: options.runtimeStore, + session, + ownerUserId: auth.actor?.id, + ownerRole: auth.actor?.role, + projectId, + conversationId, + threadId: safeOpaqueId(params.threadId), + status: "idle", + now + }); + attachWorkbenchLaunchOtel(request, { ...baseOtel, sessionId: admittedSessionId, stage: "admission_fact", result: "ok", admissionFactWritten: Boolean(admissionWrite) }); + emitWorkbenchLaunchOtelSpan(request, env, "admission_fact", { ...baseOtel, sessionId: admittedSessionId, result: "ok", admissionFactWritten: Boolean(admissionWrite), statusCode: 200 }, { startTimeMs: admissionStartedAt }); + } catch (error) { + attachWorkbenchLaunchOtel(request, { ...baseOtel, sessionId: admittedSessionId, stage: "admission_fact", result: "failed", errorCode: "workbench_launch_admission_fact_failed", statusCode: 500 }); + emitWorkbenchLaunchOtelSpan(request, env, "admission_fact", { ...baseOtel, sessionId: admittedSessionId, result: "failed", errorCode: "workbench_launch_admission_fact_failed", statusCode: 500 }, { startTimeMs: admissionStartedAt, status: "error", error }); + throw error; + } + + const workbenchUrl = `/workbench/sessions/${encodeURIComponent(admittedSessionId)}`; + const linkWriteStartedAt = Date.now(); + attachWorkbenchLaunchOtel(request, { ...baseOtel, sessionId: admittedSessionId, stage: "project_link_write", result: "started" }); const linkWrite = await writeProjectManagementLink(request, { - env: options.env ?? process.env, + env, auth, link: { linkId, projectId, taskRef, - sessionId: session?.id ?? sessionId, + sessionId: admittedSessionId, traceId, role: "launch", createdBy: auth.actor?.id ?? null, link: { launchContext, workbenchUrl, valuesRedacted: true } } }); - if (!linkWrite.ok) return sendJson(response, linkWrite.status, launchError(linkWrite.code, linkWrite.message, { sessionId: session?.id ?? sessionId, retryable: true })); + if (!linkWrite.ok) { + attachWorkbenchLaunchOtel(request, { ...baseOtel, sessionId: admittedSessionId, stage: "project_link_write", result: "failed", errorCode: linkWrite.code, upstreamStatus: linkWrite.status, statusCode: linkWrite.status }); + emitWorkbenchLaunchOtelSpan(request, env, "project_link_write", { ...baseOtel, sessionId: admittedSessionId, result: "failed", errorCode: linkWrite.code, upstreamStatus: linkWrite.status, statusCode: linkWrite.status }, { startTimeMs: linkWriteStartedAt, status: "error", statusMessage: linkWrite.code }); + return sendJson(response, linkWrite.status, launchError(linkWrite.code, linkWrite.message, { sessionId: admittedSessionId, retryable: true })); + } + attachWorkbenchLaunchOtel(request, { ...baseOtel, sessionId: admittedSessionId, linkId: linkWrite.link?.linkId ?? linkId, stage: "completed", result: "created", upstreamStatus: linkWrite.status, statusCode: 201 }); + emitWorkbenchLaunchOtelSpan(request, env, "project_link_write", { ...baseOtel, sessionId: admittedSessionId, linkId: linkWrite.link?.linkId ?? linkId, result: "ok", upstreamStatus: linkWrite.status, statusCode: linkWrite.status }, { startTimeMs: linkWriteStartedAt }); + emitWorkbenchLaunchOtelSpan(request, env, "completed", { ...baseOtel, sessionId: admittedSessionId, linkId: linkWrite.link?.linkId ?? linkId, result: "created", upstreamStatus: linkWrite.status, statusCode: 201 }); return sendJson(response, 201, { ok: true, status: "created", contractVersion, - sessionId: session?.id ?? sessionId, + sessionId: admittedSessionId, conversationId, projectId, taskRef, @@ -97,6 +151,77 @@ export async function handleWorkbenchLaunchHttp(request, response, options = {}) }); } +function attachWorkbenchLaunchOtel(request, fields = {}) { + const context = request?.hwlabHttpRequestContext; + if (!context) return; + const errorCode = safeAttr(fields.errorCode, null); + context.route = "/v1/workbench/launches"; + if (errorCode) { + context.lastHttpError = { + code: errorCode, + category: "workbench_launch", + layer: `workbench.launch.${safeAttr(fields.stage, "unknown")}` + }; + } + context.otelAttributes = { + ...(context.otelAttributes ?? {}), + "workbench.route": "launches", + "workbench.launch.stage": safeAttr(fields.stage, null), + "workbench.launch.result": safeAttr(fields.result, null), + "workbench.launch.project_id": safeAttr(fields.projectId, null), + "workbench.launch.task_ref": safeAttr(fields.taskRef, null), + "workbench.launch.task_ref_hash": fields.taskRef ? stableHash(fields.taskRef) : null, + "workbench.launch.session_id": safeAttr(fields.sessionId, null), + "workbench.launch.conversation_id": safeAttr(fields.conversationId, null), + "workbench.launch.link_id": safeAttr(fields.linkId, null), + "workbench.launch.trace_id": safeAttr(fields.traceId, null), + "workbench.launch.owner_user_id": safeAttr(fields.ownerUserId, null), + "workbench.launch.admission_fact_written": typeof fields.admissionFactWritten === "boolean" ? fields.admissionFactWritten : null, + "workbench.launch.upstream_status": Number.isFinite(Number(fields.upstreamStatus)) ? Math.trunc(Number(fields.upstreamStatus)) : null, + "workbench.launch.values_redacted": true, + ...(errorCode ? { "error.code": errorCode, "error.category": "workbench_launch", "error.layer": `workbench.launch.${safeAttr(fields.stage, "unknown")}` } : {}) + }; +} + +function emitWorkbenchLaunchOtelSpan(request, env, stage, fields = {}, options = {}) { + const context = request?.hwlabHttpRequestContext; + if (!context?.traceId) return; + const statusCode = Number.isFinite(Number(fields.statusCode)) ? Math.trunc(Number(fields.statusCode)) : (options.status === "error" ? 500 : 200); + void emitHttpServerRequestSpan(context, env, { + name: `workbench.launch.${stage}`, + spanId: newOtelSpanId(), + parentSpanId: context.spanId, + startTimeMs: options.startTimeMs, + endTimeMs: Date.now(), + statusCode, + method: "POST", + route: "/v1/workbench/launches", + status: options.status ?? (statusCode >= 400 ? "error" : "ok"), + statusMessage: options.statusMessage ?? safeAttr(fields.errorCode, null), + error: options.error, + errorCode: safeAttr(fields.errorCode, null), + errorCategory: fields.errorCode ? "workbench_launch" : null, + errorLayer: fields.errorCode ? `workbench.launch.${stage}` : null, + attributes: { + ...(context.otelAttributes ?? {}), + "workbench.launch.span": stage, + "workbench.launch.stage": stage, + "workbench.launch.result": safeAttr(fields.result, null), + "workbench.launch.project_id": safeAttr(fields.projectId, null), + "workbench.launch.task_ref": safeAttr(fields.taskRef, null), + "workbench.launch.task_ref_hash": fields.taskRef ? stableHash(fields.taskRef) : null, + "workbench.launch.session_id": safeAttr(fields.sessionId, null), + "workbench.launch.conversation_id": safeAttr(fields.conversationId, null), + "workbench.launch.link_id": safeAttr(fields.linkId, null), + "workbench.launch.trace_id": safeAttr(fields.traceId, null), + "workbench.launch.owner_user_id": safeAttr(fields.ownerUserId, null), + "workbench.launch.admission_fact_written": typeof fields.admissionFactWritten === "boolean" ? fields.admissionFactWritten : null, + "workbench.launch.upstream_status": Number.isFinite(Number(fields.upstreamStatus)) ? Math.trunc(Number(fields.upstreamStatus)) : null, + "workbench.launch.values_redacted": true + } + }).catch(() => undefined); +} + async function readJsonObject(request, bodyLimitBytes) { const body = await readBody(request, bodyLimitBytes); try { @@ -160,6 +285,15 @@ function stableLinkId({ projectId, taskRef, sessionId }) { return `lnk_${createHash("sha256").update(`${projectId}\n${taskRef}\n${sessionId}`).digest("hex").slice(0, 24)}`; } +function stableHash(value) { + return createHash("sha256").update(String(value ?? "")).digest("hex").slice(0, 24); +} + +function safeAttr(value, fallback) { + const text = String(value ?? "").trim(); + return text ? text.slice(0, 360) : fallback; +} + function safeToken(value, fallback) { const text = String(value ?? "").trim(); return /^[A-Za-z0-9_.:-]{1,160}$/u.test(text) ? text : fallback;