mirrors/pikasTech-HWLAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: instrument slow readiness routes (#1896)

Browse Source
This commit is contained in:
Lyon
2026-06-22 15:06:51 +08:00
committed by GitHub
parent 19b6db4254
commit 7fc4844b9d
3 changed files with 80 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/cloud/otel-trace.ts
+3 -3
View File
@@ -254,7 +254,7 @@ export async function emitHttpServerRequestSpan(context, env = process.env, opti
const traceId = normalizeOtelTraceId(context?.traceId);
if (!traceId) return { ok: false, skipped: true, reason: "trace-id-missing", valuesPrinted: false };
const spanId = normalizeOtelSpanId(options.spanId) ?? newOtelSpanId();
const parentSpanId = normalizeOtelSpanId(context?.parentSpanId) ?? null;
const parentSpanId = normalizeOtelSpanId(options.parentSpanId) ?? normalizeOtelSpanId(context?.parentSpanId) ?? null;
const now = Date.now();
const startedAtMs = epochUnixMs(options.startTimeMs, now);
const endedAtMs = epochUnixMs(options.endTimeMs, startedAtMs);
@@ -263,8 +263,8 @@ export async function emitHttpServerRequestSpan(context, env = process.env, opti
const statusCode = Number.isInteger(Number(options.statusCode)) ? Number(options.statusCode) : 0;
const statusClassValue = statusClassFor(statusCode);
const errorCode = options.errorCode ? String(options.errorCode) : null;
const isError = statusCode >= 500;
const name = `${method} ${route}`;
const isError = options.status === "error" || Boolean(options.error) || statusCode >= 500;
const name = typeof options.name === "string" && options.name.trim() ? options.name.trim() : `${method} ${route}`;
const attributes = {
"otel.trace_id": traceId,
"request.id": context?.requestId ?? null,
internal/cloud/server-rest-payloads.ts
+22 -3
View File
@@ -39,10 +39,22 @@ function liveBuildProfile(env = process.env) {
export async function buildLiveBuildsPayload(options = {}, dependencies = {}) {
const env = options.env ?? process.env;
const metadata = await loadLiveBuildMetadata(options, env);
const inventory = liveBuildServiceInventory(metadata, env);
const metadata = await observeLiveBuildPhase(options, "metadata", () => loadLiveBuildMetadata(options, env));
const inventory = await observeLiveBuildPhase(options, "inventory", async () => liveBuildServiceInventory(metadata, env), {
"hwlab.live_builds.deploy_manifest_status": metadata.deployManifest?.status ?? null,
"hwlab.live_builds.artifact_catalog_status": metadata.artifactCatalog?.status ?? null
});
const services = await Promise.all(
inventory.map((service) => observeLiveBuildService(service, { ...options, env, buildHealthPayload: dependencies.buildHealthPayload }))
inventory.map((service) => observeLiveBuildPhase(
options,
"service_health",
() => observeLiveBuildService(service, { ...options, env, buildHealthPayload: dependencies.buildHealthPayload }),
{
"hwlab.live_builds.service_id": service.serviceId,
"hwlab.live_builds.service_kind": service.kind,
"hwlab.live_builds.external": Boolean(service.externalImage)
}
))
);
const hwlabServices = services.filter((service) => service.kind === "hwlab");
const latest = [...hwlabServices]
@@ -76,6 +88,13 @@ export async function buildLiveBuildsPayload(options = {}, dependencies = {}) {
};
}
async function observeLiveBuildPhase(options, phase, operation, attributes = {}) {
if (typeof options.observeLiveBuildPhase === "function") {
return options.observeLiveBuildPhase(phase, operation, attributes);
}
return operation();
}
async function observeLiveBuildService(service, options) {
if (service.externalImage) {
return externalLiveBuildRecord(service, {
internal/cloud/server.ts
+55 -5
View File
@@ -73,7 +73,7 @@ import { handleWorkbenchReadModelHttp, handleWorkbenchRealtimeHttp } from "./ser
import { startWorkbenchEmptySessionGc } from "./workbench-empty-session-gc.ts";
import { handleM3IoControlHttp } from "./server-m3-http.ts";
import { handleSkillsHttp } from "./server-skills-http.ts";
import { emitHttpServerRequestSpan, httpRequestOtelTraceContext } from "./otel-trace.ts";
import { emitHttpServerRequestSpan, httpRequestOtelTraceContext, newOtelSpanId } from "./otel-trace.ts";
import { decorateErrorPayloadForHttp, logErrorEnvelope } from "./error-envelope.ts";
import { handleProviderProfileCatalogHttp, handleProviderProfilesHttp } from "./provider-profile-management.ts";
import {
@@ -96,6 +96,51 @@ function runtimeEnvironment(env = process.env) {
return typeof value === "string" && value.trim() ? value.trim() : ENVIRONMENT_DEV;
}
async function observeHttpRoutePhase(request, options, phase, operation, attributes = {}) {
const startedAtMs = Date.now();
try {
const value = await operation();
emitHttpRoutePhaseSpan(request, options, phase, startedAtMs, Date.now(), "ok", null, attributes);
return value;
} catch (error) {
emitHttpRoutePhaseSpan(request, options, phase, startedAtMs, Date.now(), "error", error, attributes);
throw error;
}
}
function createHttpRoutePhaseObserver(request, options, namespace) {
return (phase, operation, attributes = {}) => observeHttpRoutePhase(request, options, `${namespace}.${phase}`, operation, attributes);
}
function emitHttpRoutePhaseSpan(request, options, phase, startTimeMs, endTimeMs, outcome, error, attributes = {}) {
const context = request?.hwlabHttpRequestContext;
if (!context?.traceId) return;
const errorMessage = error ? String(error?.message ?? error).slice(0, 300) : null;
void emitHttpServerRequestSpan(context, options?.env ?? process.env, {
name: `${String(context.method ?? "GET").toUpperCase()} ${String(context.route ?? "/")} ${phase}`,
method: context.method,
route: context.route,
parentSpanId: context.spanId,
spanId: newOtelSpanId(),
startTimeMs,
endTimeMs,
statusCode: outcome === "error" ? 500 : 200,
status: outcome === "error" ? "error" : "ok",
statusMessage: errorMessage,
errorCode: error ? phaseErrorCode(error) : null,
attributes: {
"hwlab.http.stage": "http.server.phase",
"hwlab.http.phase": phase,
"hwlab.http.phase.outcome": outcome,
...attributes
}
}).catch(() => undefined);
}
function phaseErrorCode(error) {
return String(error?.code ?? error?.name ?? "phase_error").slice(0, 120);
}
export function createCloudApiServer(options = {}) {
const env = options.env ?? process.env;
ensureCodeAgentRuntimeBase(env);
@@ -544,9 +589,11 @@ async function handleRestAdapter(request, response, url, options) {
}
if (request.method === "GET" && url.pathname === "/v1") {
const dbProbe = await buildDbRuntimeReadiness(options.env ?? process.env, options.dbProbe);
const codeAgent = await describeCodeAgentAvailability(options.env ?? process.env, options);
const runtime = await runtimeReadiness(options.runtimeStore);
const [dbProbe, codeAgent, runtime] = await Promise.all([
observeHttpRoutePhase(request, options, "v1.db_readiness", () => buildDbRuntimeReadiness(options.env ?? process.env, options.dbProbe)),
observeHttpRoutePhase(request, options, "v1.code_agent_availability", () => describeCodeAgentAvailability(options.env ?? process.env, options)),
observeHttpRoutePhase(request, options, "v1.runtime_readiness", () => runtimeReadiness(options.runtimeStore))
]);
const db = applyRuntimeDbReadinessLayers(dbProbe, runtime);
const readiness = buildCloudApiReadiness({ db, codeAgent, runtime });
sendJson(response, 200, {
@@ -677,7 +724,10 @@ async function handleRestAdapter(request, response, url, options) {
}
if (request.method === "GET" && url.pathname === "/v1/live-builds") {
sendJson(response, 200, await buildLiveBuildsPayload(options, { buildHealthPayload, runtimeEnvironment }));
sendJson(response, 200, await buildLiveBuildsPayload({
...options,
observeLiveBuildPhase: createHttpRoutePhaseObserver(request, options, "live_builds")
}, { buildHealthPayload, runtimeEnvironment }));
return;
}