fix: instrument slow readiness routes (#1896)

internal/cloud/server.ts

@@ -73,7 +73,7 @@ import { handleWorkbenchReadModelHttp, handleWorkbenchRealtimeHttp } from "./ser
 import { startWorkbenchEmptySessionGc } from "./workbench-empty-session-gc.ts";
 import { handleM3IoControlHttp } from "./server-m3-http.ts";
 import { handleSkillsHttp } from "./server-skills-http.ts";
-import { emitHttpServerRequestSpan, httpRequestOtelTraceContext } from "./otel-trace.ts";
+import { emitHttpServerRequestSpan, httpRequestOtelTraceContext, newOtelSpanId } from "./otel-trace.ts";
 import { decorateErrorPayloadForHttp, logErrorEnvelope } from "./error-envelope.ts";
 import { handleProviderProfileCatalogHttp, handleProviderProfilesHttp } from "./provider-profile-management.ts";
 import {
@@ -96,6 +96,51 @@ function runtimeEnvironment(env = process.env) {
   return typeof value === "string" && value.trim() ? value.trim() : ENVIRONMENT_DEV;
 }
 
+async function observeHttpRoutePhase(request, options, phase, operation, attributes = {}) {
+  const startedAtMs = Date.now();
+  try {
+    const value = await operation();
+    emitHttpRoutePhaseSpan(request, options, phase, startedAtMs, Date.now(), "ok", null, attributes);
+    return value;
+  } catch (error) {
+    emitHttpRoutePhaseSpan(request, options, phase, startedAtMs, Date.now(), "error", error, attributes);
+    throw error;
+  }
+}
+
+function createHttpRoutePhaseObserver(request, options, namespace) {
+  return (phase, operation, attributes = {}) => observeHttpRoutePhase(request, options, `${namespace}.${phase}`, operation, attributes);
+}
+
+function emitHttpRoutePhaseSpan(request, options, phase, startTimeMs, endTimeMs, outcome, error, attributes = {}) {
+  const context = request?.hwlabHttpRequestContext;
+  if (!context?.traceId) return;
+  const errorMessage = error ? String(error?.message ?? error).slice(0, 300) : null;
+  void emitHttpServerRequestSpan(context, options?.env ?? process.env, {
+    name: `${String(context.method ?? "GET").toUpperCase()} ${String(context.route ?? "/")} ${phase}`,
+    method: context.method,
+    route: context.route,
+    parentSpanId: context.spanId,
+    spanId: newOtelSpanId(),
+    startTimeMs,
+    endTimeMs,
+    statusCode: outcome === "error" ? 500 : 200,
+    status: outcome === "error" ? "error" : "ok",
+    statusMessage: errorMessage,
+    errorCode: error ? phaseErrorCode(error) : null,
+    attributes: {
+      "hwlab.http.stage": "http.server.phase",
+      "hwlab.http.phase": phase,
+      "hwlab.http.phase.outcome": outcome,
+      ...attributes
+    }
+  }).catch(() => undefined);
+}
+
+function phaseErrorCode(error) {
+  return String(error?.code ?? error?.name ?? "phase_error").slice(0, 120);
+}
+
 export function createCloudApiServer(options = {}) {
   const env = options.env ?? process.env;
   ensureCodeAgentRuntimeBase(env);
@@ -544,9 +589,11 @@ async function handleRestAdapter(request, response, url, options) {
   }
 
   if (request.method === "GET" && url.pathname === "/v1") {
-    const dbProbe = await buildDbRuntimeReadiness(options.env ?? process.env, options.dbProbe);
-    const codeAgent = await describeCodeAgentAvailability(options.env ?? process.env, options);
-    const runtime = await runtimeReadiness(options.runtimeStore);
+    const [dbProbe, codeAgent, runtime] = await Promise.all([
+      observeHttpRoutePhase(request, options, "v1.db_readiness", () => buildDbRuntimeReadiness(options.env ?? process.env, options.dbProbe)),
+      observeHttpRoutePhase(request, options, "v1.code_agent_availability", () => describeCodeAgentAvailability(options.env ?? process.env, options)),
+      observeHttpRoutePhase(request, options, "v1.runtime_readiness", () => runtimeReadiness(options.runtimeStore))
+    ]);
     const db = applyRuntimeDbReadinessLayers(dbProbe, runtime);
     const readiness = buildCloudApiReadiness({ db, codeAgent, runtime });
     sendJson(response, 200, {
@@ -677,7 +724,10 @@ async function handleRestAdapter(request, response, url, options) {
   }
 
   if (request.method === "GET" && url.pathname === "/v1/live-builds") {
-    sendJson(response, 200, await buildLiveBuildsPayload(options, { buildHealthPayload, runtimeEnvironment }));
+    sendJson(response, 200, await buildLiveBuildsPayload({
+      ...options,
+      observeLiveBuildPhase: createHttpRoutePhaseObserver(request, options, "live_builds")
+    }, { buildHealthPayload, runtimeEnvironment }));
     return;
   }