From 7fc4844b9dc2b9a56b6858e28713c193d0e7ca47 Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Mon, 22 Jun 2026 15:06:51 +0800 Subject: [PATCH] fix: instrument slow readiness routes (#1896) --- internal/cloud/otel-trace.ts | 6 +-- internal/cloud/server-rest-payloads.ts | 25 +++++++++-- internal/cloud/server.ts | 60 +++++++++++++++++++++++--- 3 files changed, 80 insertions(+), 11 deletions(-) diff --git a/internal/cloud/otel-trace.ts b/internal/cloud/otel-trace.ts index 53ca3ed4..b78fbdf2 100644 --- a/internal/cloud/otel-trace.ts +++ b/internal/cloud/otel-trace.ts @@ -254,7 +254,7 @@ export async function emitHttpServerRequestSpan(context, env = process.env, opti const traceId = normalizeOtelTraceId(context?.traceId); if (!traceId) return { ok: false, skipped: true, reason: "trace-id-missing", valuesPrinted: false }; const spanId = normalizeOtelSpanId(options.spanId) ?? newOtelSpanId(); - const parentSpanId = normalizeOtelSpanId(context?.parentSpanId) ?? null; + const parentSpanId = normalizeOtelSpanId(options.parentSpanId) ?? normalizeOtelSpanId(context?.parentSpanId) ?? null; const now = Date.now(); const startedAtMs = epochUnixMs(options.startTimeMs, now); const endedAtMs = epochUnixMs(options.endTimeMs, startedAtMs); @@ -263,8 +263,8 @@ export async function emitHttpServerRequestSpan(context, env = process.env, opti const statusCode = Number.isInteger(Number(options.statusCode)) ? Number(options.statusCode) : 0; const statusClassValue = statusClassFor(statusCode); const errorCode = options.errorCode ? String(options.errorCode) : null; - const isError = statusCode >= 500; - const name = `${method} ${route}`; + const isError = options.status === "error" || Boolean(options.error) || statusCode >= 500; + const name = typeof options.name === "string" && options.name.trim() ? options.name.trim() : `${method} ${route}`; const attributes = { "otel.trace_id": traceId, "request.id": context?.requestId ?? null, diff --git a/internal/cloud/server-rest-payloads.ts b/internal/cloud/server-rest-payloads.ts index b8f7ec9d..3b753fcf 100644 --- a/internal/cloud/server-rest-payloads.ts +++ b/internal/cloud/server-rest-payloads.ts @@ -39,10 +39,22 @@ function liveBuildProfile(env = process.env) { export async function buildLiveBuildsPayload(options = {}, dependencies = {}) { const env = options.env ?? process.env; - const metadata = await loadLiveBuildMetadata(options, env); - const inventory = liveBuildServiceInventory(metadata, env); + const metadata = await observeLiveBuildPhase(options, "metadata", () => loadLiveBuildMetadata(options, env)); + const inventory = await observeLiveBuildPhase(options, "inventory", async () => liveBuildServiceInventory(metadata, env), { + "hwlab.live_builds.deploy_manifest_status": metadata.deployManifest?.status ?? null, + "hwlab.live_builds.artifact_catalog_status": metadata.artifactCatalog?.status ?? null + }); const services = await Promise.all( - inventory.map((service) => observeLiveBuildService(service, { ...options, env, buildHealthPayload: dependencies.buildHealthPayload })) + inventory.map((service) => observeLiveBuildPhase( + options, + "service_health", + () => observeLiveBuildService(service, { ...options, env, buildHealthPayload: dependencies.buildHealthPayload }), + { + "hwlab.live_builds.service_id": service.serviceId, + "hwlab.live_builds.service_kind": service.kind, + "hwlab.live_builds.external": Boolean(service.externalImage) + } + )) ); const hwlabServices = services.filter((service) => service.kind === "hwlab"); const latest = [...hwlabServices] @@ -76,6 +88,13 @@ export async function buildLiveBuildsPayload(options = {}, dependencies = {}) { }; } +async function observeLiveBuildPhase(options, phase, operation, attributes = {}) { + if (typeof options.observeLiveBuildPhase === "function") { + return options.observeLiveBuildPhase(phase, operation, attributes); + } + return operation(); +} + async function observeLiveBuildService(service, options) { if (service.externalImage) { return externalLiveBuildRecord(service, { diff --git a/internal/cloud/server.ts b/internal/cloud/server.ts index 28c56d0b..323a6ce1 100644 --- a/internal/cloud/server.ts +++ b/internal/cloud/server.ts @@ -73,7 +73,7 @@ import { handleWorkbenchReadModelHttp, handleWorkbenchRealtimeHttp } from "./ser import { startWorkbenchEmptySessionGc } from "./workbench-empty-session-gc.ts"; import { handleM3IoControlHttp } from "./server-m3-http.ts"; import { handleSkillsHttp } from "./server-skills-http.ts"; -import { emitHttpServerRequestSpan, httpRequestOtelTraceContext } from "./otel-trace.ts"; +import { emitHttpServerRequestSpan, httpRequestOtelTraceContext, newOtelSpanId } from "./otel-trace.ts"; import { decorateErrorPayloadForHttp, logErrorEnvelope } from "./error-envelope.ts"; import { handleProviderProfileCatalogHttp, handleProviderProfilesHttp } from "./provider-profile-management.ts"; import { @@ -96,6 +96,51 @@ function runtimeEnvironment(env = process.env) { return typeof value === "string" && value.trim() ? value.trim() : ENVIRONMENT_DEV; } +async function observeHttpRoutePhase(request, options, phase, operation, attributes = {}) { + const startedAtMs = Date.now(); + try { + const value = await operation(); + emitHttpRoutePhaseSpan(request, options, phase, startedAtMs, Date.now(), "ok", null, attributes); + return value; + } catch (error) { + emitHttpRoutePhaseSpan(request, options, phase, startedAtMs, Date.now(), "error", error, attributes); + throw error; + } +} + +function createHttpRoutePhaseObserver(request, options, namespace) { + return (phase, operation, attributes = {}) => observeHttpRoutePhase(request, options, `${namespace}.${phase}`, operation, attributes); +} + +function emitHttpRoutePhaseSpan(request, options, phase, startTimeMs, endTimeMs, outcome, error, attributes = {}) { + const context = request?.hwlabHttpRequestContext; + if (!context?.traceId) return; + const errorMessage = error ? String(error?.message ?? error).slice(0, 300) : null; + void emitHttpServerRequestSpan(context, options?.env ?? process.env, { + name: `${String(context.method ?? "GET").toUpperCase()} ${String(context.route ?? "/")} ${phase}`, + method: context.method, + route: context.route, + parentSpanId: context.spanId, + spanId: newOtelSpanId(), + startTimeMs, + endTimeMs, + statusCode: outcome === "error" ? 500 : 200, + status: outcome === "error" ? "error" : "ok", + statusMessage: errorMessage, + errorCode: error ? phaseErrorCode(error) : null, + attributes: { + "hwlab.http.stage": "http.server.phase", + "hwlab.http.phase": phase, + "hwlab.http.phase.outcome": outcome, + ...attributes + } + }).catch(() => undefined); +} + +function phaseErrorCode(error) { + return String(error?.code ?? error?.name ?? "phase_error").slice(0, 120); +} + export function createCloudApiServer(options = {}) { const env = options.env ?? process.env; ensureCodeAgentRuntimeBase(env); @@ -544,9 +589,11 @@ async function handleRestAdapter(request, response, url, options) { } if (request.method === "GET" && url.pathname === "/v1") { - const dbProbe = await buildDbRuntimeReadiness(options.env ?? process.env, options.dbProbe); - const codeAgent = await describeCodeAgentAvailability(options.env ?? process.env, options); - const runtime = await runtimeReadiness(options.runtimeStore); + const [dbProbe, codeAgent, runtime] = await Promise.all([ + observeHttpRoutePhase(request, options, "v1.db_readiness", () => buildDbRuntimeReadiness(options.env ?? process.env, options.dbProbe)), + observeHttpRoutePhase(request, options, "v1.code_agent_availability", () => describeCodeAgentAvailability(options.env ?? process.env, options)), + observeHttpRoutePhase(request, options, "v1.runtime_readiness", () => runtimeReadiness(options.runtimeStore)) + ]); const db = applyRuntimeDbReadinessLayers(dbProbe, runtime); const readiness = buildCloudApiReadiness({ db, codeAgent, runtime }); sendJson(response, 200, { @@ -677,7 +724,10 @@ async function handleRestAdapter(request, response, url, options) { } if (request.method === "GET" && url.pathname === "/v1/live-builds") { - sendJson(response, 200, await buildLiveBuildsPayload(options, { buildHealthPayload, runtimeEnvironment })); + sendJson(response, 200, await buildLiveBuildsPayload({ + ...options, + observeLiveBuildPhase: createHttpRoutePhaseObserver(request, options, "live_builds") + }, { buildHealthPayload, runtimeEnvironment })); return; }