feat: trace workbench launch failures
This commit is contained in:
lyon
@@ -0,0 +1,111 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { Readable } from "node:stream";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { handleWorkbenchLaunchHttp } from "./server-workbench-launch-http.ts";
|
||||
|
||||
test("workbench launch records OTel stage for Project Management link write 404", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
const request = jsonRequest({
|
||||
projectId: "project_hwlab_v03",
|
||||
taskRef: "mdtodo:hwlab-v03-mdtodo:file_demo:t4_open",
|
||||
sessionId: "ses_launch_otel",
|
||||
conversationId: "cnv_launch_otel"
|
||||
});
|
||||
const response = jsonResponse();
|
||||
const fetchCalls = [];
|
||||
globalThis.fetch = async (url, init) => {
|
||||
fetchCalls.push({ url: String(url), method: init?.method, body: init?.body });
|
||||
return new Response(JSON.stringify({ error: { code: "not_found", message: "Project link route was not found." } }), {
|
||||
status: 404,
|
||||
headers: { "content-type": "application/json" }
|
||||
});
|
||||
};
|
||||
try {
|
||||
await handleWorkbenchLaunchHttp(request, response, {
|
||||
env: { HWLAB_PROJECT_MANAGEMENT_URL: "http://project-management.test" },
|
||||
accessController: {
|
||||
async authenticate() {
|
||||
return { ok: true, authMethod: "web-session", actor: { id: "usr_launch_otel", role: "admin" } };
|
||||
},
|
||||
async recordAgentSessionOwner(input) {
|
||||
return {
|
||||
id: input.sessionId,
|
||||
ownerUserId: input.ownerUserId,
|
||||
ownerRole: input.ownerRole,
|
||||
projectId: input.projectId,
|
||||
conversationId: input.conversationId,
|
||||
threadId: input.threadId,
|
||||
traceId: input.traceId,
|
||||
status: input.status,
|
||||
session: input.session,
|
||||
createdAt: "2026-06-25T10:00:00.000Z",
|
||||
updatedAt: "2026-06-25T10:00:00.000Z"
|
||||
};
|
||||
}
|
||||
},
|
||||
runtimeStore: {
|
||||
async writeWorkbenchFacts() {
|
||||
return { ok: true };
|
||||
}
|
||||
}
|
||||
});
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
|
||||
assert.equal(response.statusCode, 404);
|
||||
assert.equal(fetchCalls.length, 1);
|
||||
const body = JSON.parse(response.body);
|
||||
assert.equal(body.error.code, "not_found");
|
||||
const context = request.hwlabHttpRequestContext;
|
||||
assert.equal(context.route, "/v1/workbench/launches");
|
||||
assert.equal(context.lastHttpError.code, "not_found");
|
||||
assert.equal(context.lastHttpError.category, "workbench_launch");
|
||||
assert.equal(context.lastHttpError.layer, "workbench.launch.project_link_write");
|
||||
assert.equal(context.otelAttributes["workbench.launch.stage"], "project_link_write");
|
||||
assert.equal(context.otelAttributes["workbench.launch.result"], "failed");
|
||||
assert.equal(context.otelAttributes["workbench.launch.upstream_status"], 404);
|
||||
assert.equal(context.otelAttributes["workbench.launch.session_id"], "ses_launch_otel");
|
||||
assert.equal(context.otelAttributes["workbench.launch.task_ref"], "mdtodo:hwlab-v03-mdtodo:file_demo:t4_open");
|
||||
assert.match(context.otelAttributes["workbench.launch.task_ref_hash"], /^[0-9a-f]{24}$/u);
|
||||
assert.equal(context.otelAttributes["workbench.launch.values_redacted"], true);
|
||||
});
|
||||
|
||||
function jsonRequest(body) {
|
||||
const request = Readable.from([JSON.stringify(body)]);
|
||||
request.method = "POST";
|
||||
request.url = "/v1/workbench/launches";
|
||||
request.headers = { "content-type": "application/json" };
|
||||
request.hwlabHttpRequestContext = {
|
||||
requestId: "req_launch_otel",
|
||||
startedAtMs: Date.now(),
|
||||
traceId: "11111111111111111111111111111111",
|
||||
parentSpanId: null,
|
||||
spanId: "2222222222222222",
|
||||
traceparent: "00-11111111111111111111111111111111-2222222222222222-01",
|
||||
method: "POST",
|
||||
route: "/v1/workbench/launches",
|
||||
valuesPrinted: false
|
||||
};
|
||||
return request;
|
||||
}
|
||||
|
||||
function jsonResponse() {
|
||||
return {
|
||||
headersSent: false,
|
||||
writableEnded: false,
|
||||
statusCode: 0,
|
||||
headers: {},
|
||||
body: "",
|
||||
writeHead(statusCode, headers) {
|
||||
this.statusCode = statusCode;
|
||||
this.headers = headers;
|
||||
this.headersSent = true;
|
||||
},
|
||||
end(chunk) {
|
||||
this.body += String(chunk ?? "");
|
||||
this.writableEnded = true;
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -5,23 +5,44 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
|
||||
import { getHeader, readBody, safeConversationId, safeOpaqueId, safeSessionId, safeTraceId, sendJson } from "./server-http-utils.ts";
|
||||
import { emitHttpServerRequestSpan, newOtelSpanId } from "./otel-trace.ts";
|
||||
import { writeWorkbenchSessionAdmissionFact } from "./workbench-projection-writer.ts";
|
||||
|
||||
const contractVersion = "workbench-launch-v1";
|
||||
const defaultProjectId = "project_hwlab_v03";
|
||||
|
||||
export async function handleWorkbenchLaunchHttp(request, response, options = {}) {
|
||||
if (request.method !== "POST") return sendJson(response, 405, launchError("method_not_allowed", "POST required."));
|
||||
const env = options.env ?? process.env;
|
||||
attachWorkbenchLaunchOtel(request, { stage: "start", result: "started" });
|
||||
if (request.method !== "POST") {
|
||||
attachWorkbenchLaunchOtel(request, { stage: "method", result: "failed", errorCode: "method_not_allowed", statusCode: 405 });
|
||||
emitWorkbenchLaunchOtelSpan(request, env, "method", { result: "failed", errorCode: "method_not_allowed", statusCode: 405 }, { status: "error" });
|
||||
return sendJson(response, 405, launchError("method_not_allowed", "POST required."));
|
||||
}
|
||||
const auth = await options.accessController?.authenticate?.(request, { required: true });
|
||||
if (!auth?.ok) return sendJson(response, auth?.status ?? 401, auth ?? launchError("auth_required", "Authentication is required."));
|
||||
if (!auth?.ok) {
|
||||
const statusCode = auth?.status ?? 401;
|
||||
const errorCode = auth?.error?.code ?? auth?.code ?? "auth_required";
|
||||
attachWorkbenchLaunchOtel(request, { stage: "auth", result: "failed", errorCode, statusCode });
|
||||
emitWorkbenchLaunchOtelSpan(request, env, "auth", { result: "failed", errorCode, statusCode }, { status: "error" });
|
||||
return sendJson(response, statusCode, auth ?? launchError("auth_required", "Authentication is required."));
|
||||
}
|
||||
|
||||
const body = await readJsonObject(request, options.bodyLimitBytes);
|
||||
if (!body.ok) return sendJson(response, 400, launchError(body.code, body.message, { reason: body.reason }));
|
||||
if (!body.ok) {
|
||||
attachWorkbenchLaunchOtel(request, { stage: "parse_body", result: "failed", errorCode: body.code, statusCode: 400 });
|
||||
emitWorkbenchLaunchOtelSpan(request, env, "parse_body", { result: "failed", errorCode: body.code, statusCode: 400 }, { status: "error" });
|
||||
return sendJson(response, 400, launchError(body.code, body.message, { reason: body.reason }));
|
||||
}
|
||||
|
||||
const params = body.value;
|
||||
const projectId = safeToken(params.projectId, defaultProjectId);
|
||||
const taskRef = safeProjectOpaque(params.taskRef);
|
||||
if (!taskRef) return sendJson(response, 400, launchError("task_ref_required", "Workbench launch requires taskRef."));
|
||||
if (!taskRef) {
|
||||
attachWorkbenchLaunchOtel(request, { stage: "validate", result: "failed", projectId, errorCode: "task_ref_required", statusCode: 400 });
|
||||
emitWorkbenchLaunchOtelSpan(request, env, "validate", { result: "failed", projectId, errorCode: "task_ref_required", statusCode: 400 }, { status: "error" });
|
||||
return sendJson(response, 400, launchError("task_ref_required", "Workbench launch requires taskRef."));
|
||||
}
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const conversationId = safeConversationId(params.conversationId) || `cnv_${randomUUID()}`;
|
||||
@@ -30,61 +51,94 @@ export async function handleWorkbenchLaunchHttp(request, response, options = {})
|
||||
const providerProfile = safeToken(params.providerProfile ?? params.codeAgentProviderProfile, null);
|
||||
const launchContext = normalizeLaunchContext(params.launchContext, { projectId, taskRef, now });
|
||||
const linkId = safeToken(params.linkId, stableLinkId({ projectId, taskRef, sessionId }));
|
||||
const baseOtel = { projectId, taskRef, sessionId, conversationId, traceId, linkId };
|
||||
attachWorkbenchLaunchOtel(request, { ...baseOtel, stage: "session_owner", result: "started" });
|
||||
|
||||
const session = await options.accessController.recordAgentSessionOwner({
|
||||
ownerUserId: auth.actor?.id,
|
||||
ownerRole: auth.actor?.role,
|
||||
sessionId,
|
||||
projectId,
|
||||
agentId: "hwlab-code-agent",
|
||||
status: "idle",
|
||||
conversationId,
|
||||
threadId: safeOpaqueId(params.threadId),
|
||||
traceId,
|
||||
session: {
|
||||
source: "workbench-launch",
|
||||
providerProfile,
|
||||
sessionStatus: "idle",
|
||||
launchContext,
|
||||
createdAt: now,
|
||||
valuesRedacted: true,
|
||||
secretMaterialStored: false
|
||||
}
|
||||
});
|
||||
await writeWorkbenchSessionAdmissionFact({
|
||||
runtimeStore: options.runtimeStore,
|
||||
session,
|
||||
ownerUserId: auth.actor?.id,
|
||||
ownerRole: auth.actor?.role,
|
||||
projectId,
|
||||
conversationId,
|
||||
threadId: safeOpaqueId(params.threadId),
|
||||
status: "idle",
|
||||
now
|
||||
});
|
||||
let session;
|
||||
const sessionOwnerStartedAt = Date.now();
|
||||
try {
|
||||
session = await options.accessController.recordAgentSessionOwner({
|
||||
ownerUserId: auth.actor?.id,
|
||||
ownerRole: auth.actor?.role,
|
||||
sessionId,
|
||||
projectId,
|
||||
agentId: "hwlab-code-agent",
|
||||
status: "idle",
|
||||
conversationId,
|
||||
threadId: safeOpaqueId(params.threadId),
|
||||
traceId,
|
||||
session: {
|
||||
source: "workbench-launch",
|
||||
providerProfile,
|
||||
sessionStatus: "idle",
|
||||
launchContext,
|
||||
createdAt: now,
|
||||
valuesRedacted: true,
|
||||
secretMaterialStored: false
|
||||
}
|
||||
});
|
||||
attachWorkbenchLaunchOtel(request, { ...baseOtel, stage: "session_owner", result: "ok", ownerUserId: auth.actor?.id ?? null, sessionId: session?.id ?? sessionId });
|
||||
emitWorkbenchLaunchOtelSpan(request, env, "session_owner", { ...baseOtel, sessionId: session?.id ?? sessionId, result: "ok", ownerUserId: auth.actor?.id ?? null, statusCode: 200 }, { startTimeMs: sessionOwnerStartedAt });
|
||||
} catch (error) {
|
||||
attachWorkbenchLaunchOtel(request, { ...baseOtel, stage: "session_owner", result: "failed", errorCode: "workbench_launch_session_owner_failed", statusCode: 500 });
|
||||
emitWorkbenchLaunchOtelSpan(request, env, "session_owner", { ...baseOtel, result: "failed", errorCode: "workbench_launch_session_owner_failed", statusCode: 500 }, { startTimeMs: sessionOwnerStartedAt, status: "error", error });
|
||||
throw error;
|
||||
}
|
||||
|
||||
const workbenchUrl = `/workbench/sessions/${encodeURIComponent(session?.id ?? sessionId)}`;
|
||||
const admittedSessionId = session?.id ?? sessionId;
|
||||
const admissionStartedAt = Date.now();
|
||||
attachWorkbenchLaunchOtel(request, { ...baseOtel, sessionId: admittedSessionId, stage: "admission_fact", result: "started" });
|
||||
try {
|
||||
const admissionWrite = await writeWorkbenchSessionAdmissionFact({
|
||||
runtimeStore: options.runtimeStore,
|
||||
session,
|
||||
ownerUserId: auth.actor?.id,
|
||||
ownerRole: auth.actor?.role,
|
||||
projectId,
|
||||
conversationId,
|
||||
threadId: safeOpaqueId(params.threadId),
|
||||
status: "idle",
|
||||
now
|
||||
});
|
||||
attachWorkbenchLaunchOtel(request, { ...baseOtel, sessionId: admittedSessionId, stage: "admission_fact", result: "ok", admissionFactWritten: Boolean(admissionWrite) });
|
||||
emitWorkbenchLaunchOtelSpan(request, env, "admission_fact", { ...baseOtel, sessionId: admittedSessionId, result: "ok", admissionFactWritten: Boolean(admissionWrite), statusCode: 200 }, { startTimeMs: admissionStartedAt });
|
||||
} catch (error) {
|
||||
attachWorkbenchLaunchOtel(request, { ...baseOtel, sessionId: admittedSessionId, stage: "admission_fact", result: "failed", errorCode: "workbench_launch_admission_fact_failed", statusCode: 500 });
|
||||
emitWorkbenchLaunchOtelSpan(request, env, "admission_fact", { ...baseOtel, sessionId: admittedSessionId, result: "failed", errorCode: "workbench_launch_admission_fact_failed", statusCode: 500 }, { startTimeMs: admissionStartedAt, status: "error", error });
|
||||
throw error;
|
||||
}
|
||||
|
||||
const workbenchUrl = `/workbench/sessions/${encodeURIComponent(admittedSessionId)}`;
|
||||
const linkWriteStartedAt = Date.now();
|
||||
attachWorkbenchLaunchOtel(request, { ...baseOtel, sessionId: admittedSessionId, stage: "project_link_write", result: "started" });
|
||||
const linkWrite = await writeProjectManagementLink(request, {
|
||||
env: options.env ?? process.env,
|
||||
env,
|
||||
auth,
|
||||
link: {
|
||||
linkId,
|
||||
projectId,
|
||||
taskRef,
|
||||
sessionId: session?.id ?? sessionId,
|
||||
sessionId: admittedSessionId,
|
||||
traceId,
|
||||
role: "launch",
|
||||
createdBy: auth.actor?.id ?? null,
|
||||
link: { launchContext, workbenchUrl, valuesRedacted: true }
|
||||
}
|
||||
});
|
||||
if (!linkWrite.ok) return sendJson(response, linkWrite.status, launchError(linkWrite.code, linkWrite.message, { sessionId: session?.id ?? sessionId, retryable: true }));
|
||||
if (!linkWrite.ok) {
|
||||
attachWorkbenchLaunchOtel(request, { ...baseOtel, sessionId: admittedSessionId, stage: "project_link_write", result: "failed", errorCode: linkWrite.code, upstreamStatus: linkWrite.status, statusCode: linkWrite.status });
|
||||
emitWorkbenchLaunchOtelSpan(request, env, "project_link_write", { ...baseOtel, sessionId: admittedSessionId, result: "failed", errorCode: linkWrite.code, upstreamStatus: linkWrite.status, statusCode: linkWrite.status }, { startTimeMs: linkWriteStartedAt, status: "error", statusMessage: linkWrite.code });
|
||||
return sendJson(response, linkWrite.status, launchError(linkWrite.code, linkWrite.message, { sessionId: admittedSessionId, retryable: true }));
|
||||
}
|
||||
attachWorkbenchLaunchOtel(request, { ...baseOtel, sessionId: admittedSessionId, linkId: linkWrite.link?.linkId ?? linkId, stage: "completed", result: "created", upstreamStatus: linkWrite.status, statusCode: 201 });
|
||||
emitWorkbenchLaunchOtelSpan(request, env, "project_link_write", { ...baseOtel, sessionId: admittedSessionId, linkId: linkWrite.link?.linkId ?? linkId, result: "ok", upstreamStatus: linkWrite.status, statusCode: linkWrite.status }, { startTimeMs: linkWriteStartedAt });
|
||||
emitWorkbenchLaunchOtelSpan(request, env, "completed", { ...baseOtel, sessionId: admittedSessionId, linkId: linkWrite.link?.linkId ?? linkId, result: "created", upstreamStatus: linkWrite.status, statusCode: 201 });
|
||||
|
||||
return sendJson(response, 201, {
|
||||
ok: true,
|
||||
status: "created",
|
||||
contractVersion,
|
||||
sessionId: session?.id ?? sessionId,
|
||||
sessionId: admittedSessionId,
|
||||
conversationId,
|
||||
projectId,
|
||||
taskRef,
|
||||
@@ -97,6 +151,77 @@ export async function handleWorkbenchLaunchHttp(request, response, options = {})
|
||||
});
|
||||
}
|
||||
|
||||
function attachWorkbenchLaunchOtel(request, fields = {}) {
|
||||
const context = request?.hwlabHttpRequestContext;
|
||||
if (!context) return;
|
||||
const errorCode = safeAttr(fields.errorCode, null);
|
||||
context.route = "/v1/workbench/launches";
|
||||
if (errorCode) {
|
||||
context.lastHttpError = {
|
||||
code: errorCode,
|
||||
category: "workbench_launch",
|
||||
layer: `workbench.launch.${safeAttr(fields.stage, "unknown")}`
|
||||
};
|
||||
}
|
||||
context.otelAttributes = {
|
||||
...(context.otelAttributes ?? {}),
|
||||
"workbench.route": "launches",
|
||||
"workbench.launch.stage": safeAttr(fields.stage, null),
|
||||
"workbench.launch.result": safeAttr(fields.result, null),
|
||||
"workbench.launch.project_id": safeAttr(fields.projectId, null),
|
||||
"workbench.launch.task_ref": safeAttr(fields.taskRef, null),
|
||||
"workbench.launch.task_ref_hash": fields.taskRef ? stableHash(fields.taskRef) : null,
|
||||
"workbench.launch.session_id": safeAttr(fields.sessionId, null),
|
||||
"workbench.launch.conversation_id": safeAttr(fields.conversationId, null),
|
||||
"workbench.launch.link_id": safeAttr(fields.linkId, null),
|
||||
"workbench.launch.trace_id": safeAttr(fields.traceId, null),
|
||||
"workbench.launch.owner_user_id": safeAttr(fields.ownerUserId, null),
|
||||
"workbench.launch.admission_fact_written": typeof fields.admissionFactWritten === "boolean" ? fields.admissionFactWritten : null,
|
||||
"workbench.launch.upstream_status": Number.isFinite(Number(fields.upstreamStatus)) ? Math.trunc(Number(fields.upstreamStatus)) : null,
|
||||
"workbench.launch.values_redacted": true,
|
||||
...(errorCode ? { "error.code": errorCode, "error.category": "workbench_launch", "error.layer": `workbench.launch.${safeAttr(fields.stage, "unknown")}` } : {})
|
||||
};
|
||||
}
|
||||
|
||||
function emitWorkbenchLaunchOtelSpan(request, env, stage, fields = {}, options = {}) {
|
||||
const context = request?.hwlabHttpRequestContext;
|
||||
if (!context?.traceId) return;
|
||||
const statusCode = Number.isFinite(Number(fields.statusCode)) ? Math.trunc(Number(fields.statusCode)) : (options.status === "error" ? 500 : 200);
|
||||
void emitHttpServerRequestSpan(context, env, {
|
||||
name: `workbench.launch.${stage}`,
|
||||
spanId: newOtelSpanId(),
|
||||
parentSpanId: context.spanId,
|
||||
startTimeMs: options.startTimeMs,
|
||||
endTimeMs: Date.now(),
|
||||
statusCode,
|
||||
method: "POST",
|
||||
route: "/v1/workbench/launches",
|
||||
status: options.status ?? (statusCode >= 400 ? "error" : "ok"),
|
||||
statusMessage: options.statusMessage ?? safeAttr(fields.errorCode, null),
|
||||
error: options.error,
|
||||
errorCode: safeAttr(fields.errorCode, null),
|
||||
errorCategory: fields.errorCode ? "workbench_launch" : null,
|
||||
errorLayer: fields.errorCode ? `workbench.launch.${stage}` : null,
|
||||
attributes: {
|
||||
...(context.otelAttributes ?? {}),
|
||||
"workbench.launch.span": stage,
|
||||
"workbench.launch.stage": stage,
|
||||
"workbench.launch.result": safeAttr(fields.result, null),
|
||||
"workbench.launch.project_id": safeAttr(fields.projectId, null),
|
||||
"workbench.launch.task_ref": safeAttr(fields.taskRef, null),
|
||||
"workbench.launch.task_ref_hash": fields.taskRef ? stableHash(fields.taskRef) : null,
|
||||
"workbench.launch.session_id": safeAttr(fields.sessionId, null),
|
||||
"workbench.launch.conversation_id": safeAttr(fields.conversationId, null),
|
||||
"workbench.launch.link_id": safeAttr(fields.linkId, null),
|
||||
"workbench.launch.trace_id": safeAttr(fields.traceId, null),
|
||||
"workbench.launch.owner_user_id": safeAttr(fields.ownerUserId, null),
|
||||
"workbench.launch.admission_fact_written": typeof fields.admissionFactWritten === "boolean" ? fields.admissionFactWritten : null,
|
||||
"workbench.launch.upstream_status": Number.isFinite(Number(fields.upstreamStatus)) ? Math.trunc(Number(fields.upstreamStatus)) : null,
|
||||
"workbench.launch.values_redacted": true
|
||||
}
|
||||
}).catch(() => undefined);
|
||||
}
|
||||
|
||||
async function readJsonObject(request, bodyLimitBytes) {
|
||||
const body = await readBody(request, bodyLimitBytes);
|
||||
try {
|
||||
@@ -160,6 +285,15 @@ function stableLinkId({ projectId, taskRef, sessionId }) {
|
||||
return `lnk_${createHash("sha256").update(`${projectId}\n${taskRef}\n${sessionId}`).digest("hex").slice(0, 24)}`;
|
||||
}
|
||||
|
||||
function stableHash(value) {
|
||||
return createHash("sha256").update(String(value ?? "")).digest("hex").slice(0, 24);
|
||||
}
|
||||
|
||||
function safeAttr(value, fallback) {
|
||||
const text = String(value ?? "").trim();
|
||||
return text ? text.slice(0, 360) : fallback;
|
||||
}
|
||||
|
||||
function safeToken(value, fallback) {
|
||||
const text = String(value ?? "").trim();
|
||||
return /^[A-Za-z0-9_.:-]{1,160}$/u.test(text) ? text : fallback;
|
||||
|
||||
Reference in New Issue
Block a user