diff --git a/deploy/deploy.yaml b/deploy/deploy.yaml index 8acfa460..0a656ae6 100644 --- a/deploy/deploy.yaml +++ b/deploy/deploy.yaml @@ -182,8 +182,12 @@ lanes: - internal/dev-entrypoint/cloud-web-runtime.mjs - internal/dev-entrypoint/cloud-web-proxy.mjs - internal/dev-entrypoint/cloud-web-routes.mjs + - internal/dev-entrypoint/opencode-provider-proxy.mjs + - deploy/runtime/boot/opencode-provider-proxy.sh - tools/src/hwlab-cli/trace-renderer.ts - env: {} + env: + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces + OTEL_SERVICE_NAME: hwlab-cloud-web observable: true hwlab-gateway: runtimeKind: bun-command @@ -571,6 +575,8 @@ lanes: - internal/dev-entrypoint/cloud-web-runtime.mjs - internal/dev-entrypoint/cloud-web-proxy.mjs - internal/dev-entrypoint/cloud-web-routes.mjs + - internal/dev-entrypoint/opencode-provider-proxy.mjs + - deploy/runtime/boot/opencode-provider-proxy.sh - tools/src/hwlab-cli/trace-renderer.ts env: HWLAB_CLOUD_WEB_DISPLAY_TIME_ZONE: Asia/Shanghai @@ -579,6 +585,8 @@ lanes: HWLAB_CLOUD_WEB_OPENCODE_UPSTREAM_URL: http://opencode-server.hwlab-v03.svc.cluster.local:4096 HWLAB_CLOUD_WEB_OPENCODE_USERNAME: secretRef:hwlab-opencode-server-auth/username HWLAB_CLOUD_WEB_OPENCODE_PASSWORD: secretRef:hwlab-opencode-server-auth/password + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces + OTEL_SERVICE_NAME: hwlab-cloud-web observable: true hwlab-gateway: runtimeKind: bun-command diff --git a/deploy/runtime/boot/opencode-provider-proxy.sh b/deploy/runtime/boot/opencode-provider-proxy.sh new file mode 100644 index 00000000..0add5d44 --- /dev/null +++ b/deploy/runtime/boot/opencode-provider-proxy.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu + +export HWLAB_SERVICE_ID="${HWLAB_SERVICE_ID:-opencode-provider-proxy}" +export HWLAB_ENVIRONMENT="${HWLAB_ENVIRONMENT:-v03}" +export HWLAB_COMMIT_ID="${HWLAB_BOOT_COMMIT:-${HWLAB_COMMIT_ID:-unknown}}" +export HWLAB_REVISION="${HWLAB_BOOT_COMMIT:-${HWLAB_REVISION:-$HWLAB_COMMIT_ID}}" +export PORT="${PORT:-${HWLAB_OPENCODE_PROVIDER_PROXY_PORT:-4097}}" + +exec node internal/dev-entrypoint/opencode-provider-proxy.mjs diff --git a/internal/dev-entrypoint/cloud-web-proxy.mjs b/internal/dev-entrypoint/cloud-web-proxy.mjs index 1a435049..d3cdadf4 100644 --- a/internal/dev-entrypoint/cloud-web-proxy.mjs +++ b/internal/dev-entrypoint/cloud-web-proxy.mjs @@ -95,9 +95,10 @@ export function proxyCloudApiRequest({ headers: upstreamRequestHeaders(request, body) }, (upstreamResponse) => { + const upstreamStatusCode = upstreamResponse.statusCode || 502; const streaming = forceStream || isEventStreamResponse(upstreamResponse.headers); if (streaming) { - response.writeHead(upstreamResponse.statusCode || 502, mergeResponseHeaders(copyProxyResponseHeaders(upstreamResponse.headers, { streaming: true }), extraResponseHeaders)); + response.writeHead(upstreamStatusCode, mergeResponseHeaders(copyProxyResponseHeaders(upstreamResponse.headers, { streaming: true }), extraResponseHeaders)); response.flushHeaders?.(); upstreamResponse.on("data", (chunk) => { armTimeout(); @@ -105,19 +106,19 @@ export function proxyCloudApiRequest({ }); upstreamResponse.on("end", () => { response.end(); - settle(resolve); + settle(resolve, { statusCode: upstreamStatusCode, streaming: true }); }); upstreamResponse.on("error", (error) => { if (response.headersSent) { response.destroy(error); - settle(resolve); + settle(resolve, { statusCode: upstreamStatusCode, streaming: true, errorCode: error?.code || "upstream_response_error" }); return; } settle(reject, error); }); upstreamResponse.on("close", () => { if (!response.writableEnded) response.end(); - settle(resolve); + settle(resolve, { statusCode: upstreamStatusCode, streaming: true }); }); response.on("close", () => upstream.destroy()); return; @@ -130,12 +131,12 @@ export function proxyCloudApiRequest({ }); upstreamResponse.on("end", () => { const responseBody = Buffer.concat(chunks); - response.writeHead(upstreamResponse.statusCode || 502, { + response.writeHead(upstreamStatusCode, { ...mergeResponseHeaders(copyProxyResponseHeaders(upstreamResponse.headers), extraResponseHeaders), "content-length": responseBody.length }); response.end(responseBody); - settle(resolve); + settle(resolve, { statusCode: upstreamStatusCode, streaming: false, bodyBytes: responseBody.length }); }); upstreamResponse.on("error", (error) => settle(reject, error)); } @@ -146,7 +147,7 @@ export function proxyCloudApiRequest({ if (timedOut) error.timedOut = true; if (response.headersSent) { response.destroy(error); - settle(resolve); + settle(resolve, { statusCode: 0, streaming: true, errorCode: error?.code || "upstream_request_error" }); return; } settle(reject, error); diff --git a/internal/dev-entrypoint/cloud-web-runtime.mjs b/internal/dev-entrypoint/cloud-web-runtime.mjs index 4c7afad3..01f7f6c4 100644 --- a/internal/dev-entrypoint/cloud-web-runtime.mjs +++ b/internal/dev-entrypoint/cloud-web-runtime.mjs @@ -665,16 +665,20 @@ export async function proxyCloudApi({ request, response, url, cloudApiBaseUrl, c async function proxyOpencodeRequest({ request, response, url, cloudApiBaseUrl, cloudApiProxyTimeoutMs, opencodeUpstreamUrl, opencodeProxyTimeoutMs, opencodeUsername, opencodePassword, serviceId, sendJson }) { const traceContext = cloudWebTraceContext(request); + const startedAtMs = Date.now(); if (!cloudApiBaseUrl) { + emitOpencodeProxySpanAsync({ traceContext, serviceId, request, url, startedAtMs, statusCode: 503, errorCode: "cloud_api_base_url_missing" }); sendJsonWithTraceContext(response, 503, { error: "cloud_api_base_url_missing", serviceId, path: url.pathname, diagnostic: opencodeTraceDiagnostic(traceContext) }, traceContext, sendJson); return; } if (!opencodeUpstreamUrl) { + emitOpencodeProxySpanAsync({ traceContext, serviceId, request, url, startedAtMs, statusCode: 503, errorCode: "opencode_upstream_url_missing" }); sendJsonWithTraceContext(response, 503, { error: "opencode_upstream_url_missing", serviceId, path: url.pathname, diagnostic: opencodeTraceDiagnostic(traceContext) }, traceContext, sendJson); return; } const authorization = opencodeBasicAuthorization(opencodeUsername, opencodePassword); if (!authorization) { + emitOpencodeProxySpanAsync({ traceContext, serviceId, request, url, startedAtMs, statusCode: 503, errorCode: "opencode_credentials_missing" }); sendJsonWithTraceContext(response, 503, { error: "opencode_credentials_missing", serviceId, path: url.pathname, diagnostic: opencodeTraceDiagnostic(traceContext) }, traceContext, sendJson); return; } @@ -702,6 +706,17 @@ async function proxyOpencodeRequest({ request, response, url, cloudApiBaseUrl, c valuesPrinted: false } }); + emitOpencodeProxySpanAsync({ + traceContext, + serviceId, + request, + url, + startedAtMs, + statusCode: session.statusCode === 403 ? 403 : 401, + errorCode: "opencode_auth_required", + ticketAuth, + sessionStatusCode: session.statusCode || 0 + }); sendJsonWithTraceContext(response, session.statusCode === 403 ? 403 : 401, { authenticated: false, error: "opencode_auth_required", @@ -727,7 +742,7 @@ async function proxyOpencodeRequest({ request, response, url, cloudApiBaseUrl, c headers: opencodeUpstreamHeaders({ headers: { ...request.headers, ...cloudWebTraceHeaders(traceContext, serviceId) } }, body, authorization) }; try { - await proxyCloudApiRequest({ + const proxyResult = await proxyCloudApiRequest({ target, request: upstreamRequest, response, @@ -735,11 +750,36 @@ async function proxyOpencodeRequest({ request, response, url, cloudApiBaseUrl, c timeoutMs: opencodeProxyTimeoutMs, extraResponseHeaders }); + emitOpencodeProxySpanAsync({ + traceContext, + serviceId, + request, + url, + target, + startedAtMs, + statusCode: proxyResult?.statusCode || 0, + errorCode: proxyResult?.errorCode || "", + ticketAuth, + sessionStatusCode: session.statusCode || 0, + streaming: proxyResult?.streaming === true + }); } catch (error) { if (response.headersSent || response.writableEnded) { if (!response.writableEnded) response.destroy(error); return; } + emitOpencodeProxySpanAsync({ + traceContext, + serviceId, + request, + url, + target, + startedAtMs, + statusCode: 502, + errorCode: error?.timedOut ? "opencode_proxy_timeout" : "opencode_proxy_unavailable", + ticketAuth, + sessionStatusCode: session.statusCode || 0 + }); sendJsonWithTraceContext(response, 502, { status: "failed", error: "opencode_proxy_unavailable", @@ -876,6 +916,131 @@ function opencodeTraceDiagnostic(traceContext, extra = {}) { }; } +function emitOpencodeProxySpanAsync({ traceContext, serviceId, request, url, target = null, startedAtMs, statusCode = 0, errorCode = "", ticketAuth = null, sessionStatusCode = 0, streaming = false } = {}) { + const endpoint = otelTracesEndpoint(process.env); + if (!endpoint || !traceContext?.traceId || !traceContext?.spanId) return; + const endedAtMs = Date.now(); + const route = opencodeProxyRoute(url?.pathname || ""); + const span = { + traceId: traceContext.traceId, + spanId: traceContext.spanId, + parentSpanId: traceContext.parentSpanId || "", + name: "opencode.proxy.request", + kind: 2, + startTimeMs: startedAtMs || endedAtMs, + endTimeMs: endedAtMs, + statusCode, + errorCode, + attributes: { + "http.request.method": request?.method || "GET", + "http.route": route, + "http.response.status_code": statusCode || undefined, + "http.response.status_class": statusCode ? `${Math.floor(statusCode / 100)}xx` : "unknown", + "url.scheme": target?.protocol ? target.protocol.replace(":", "") : undefined, + "server.address": target?.hostname || undefined, + "server.port": target ? Number(target.port || (target.protocol === "https:" ? 443 : 80)) : undefined, + "opencode.proxy.duration_ms": Math.max(0, endedAtMs - (startedAtMs || endedAtMs)), + "opencode.proxy.ticket_present": ticketAuth?.ticketPresent === true, + "opencode.proxy.ticket_accepted": Boolean(ticketAuth?.ticket), + "opencode.proxy.session_status_code": sessionStatusCode || undefined, + "opencode.proxy.streaming": streaming === true, + valuesPrinted: false + } + }; + setTimeout(() => { + emitOtelSpan(endpoint, span, serviceId || "hwlab-cloud-web", "hwlab.cloud_web.opencode_proxy").catch((error) => { + process.stderr.write(JSON.stringify({ + event: "cloud-web-opencode-otel-emit-failed", + serviceId: serviceId || "hwlab-cloud-web", + traceId: traceContext.traceId, + requestId: traceContext.requestId, + error: error instanceof Error ? error.message : String(error), + valuesPrinted: false + }) + "\n"); + }); + }, 0); +} + +function opencodeProxyRoute(pathname) { + const path = String(pathname || "/"); + if (path === "/global/event") return "/global/event"; + if (path === "/project" || path === "/project/current" || path === "/project/git/init") return path; + if (path === "/provider" || path === "/path" || path === "/global/health") return path; + if (/^\/session\/[^/]+\/prompt_async$/u.test(path)) return "/session/:sessionID/prompt_async"; + if (/^\/session\/[^/]+\/message$/u.test(path)) return "/session/:sessionID/message"; + if (/^\/session\/[^/]+\/abort$/u.test(path)) return "/session/:sessionID/abort"; + if (/^\/session\/[^/]+(?:\/.*)?$/u.test(path)) return "/session/:sessionID/*"; + if (path.startsWith("/assets/")) return "/assets/*"; + return path; +} + +async function emitOtelSpan(endpoint, span, serviceId, scopeName) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 1500); + try { + await fetch(endpoint, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(otelPayload(span, serviceId, scopeName)), + signal: controller.signal + }); + } finally { + clearTimeout(timeout); + } +} + +function otelPayload(span, serviceId, scopeName) { + return { + resourceSpans: [{ + resource: { attributes: otelAttributes({ "service.name": serviceId, "deployment.environment": process.env.HWLAB_ENVIRONMENT || "v03" }) }, + scopeSpans: [{ + scope: { name: scopeName, version: "1" }, + spans: [{ + traceId: span.traceId, + spanId: span.spanId, + parentSpanId: span.parentSpanId || undefined, + name: span.name, + kind: span.kind, + startTimeUnixNano: unixNano(span.startTimeMs), + endTimeUnixNano: unixNano(span.endTimeMs), + attributes: otelAttributes({ ...span.attributes, "otel.trace_id": span.traceId }), + status: span.errorCode || (span.statusCode && span.statusCode >= 500) ? { code: 2, message: span.errorCode || `http_${span.statusCode}` } : { code: 1 } + }] + }] + }] + }; +} + +function otelAttributes(values) { + const result = []; + for (const [key, value] of Object.entries(values || {})) { + if (value === undefined || value === null || value === "") continue; + if (typeof value === "number" && Number.isFinite(value)) result.push({ key, value: Number.isInteger(value) ? { intValue: String(value) } : { doubleValue: value } }); + else if (typeof value === "boolean") result.push({ key, value: { boolValue: value } }); + else result.push({ key, value: { stringValue: String(value) } }); + } + return result; +} + +function unixNano(ms) { + return String(BigInt(Math.max(0, Math.trunc(ms))) * 1000000n); +} + +function otelTracesEndpoint(env) { + const explicit = firstNonEmpty(env.HWLAB_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT); + if (explicit) return explicit; + const base = firstNonEmpty(env.HWLAB_OTEL_EXPORTER_OTLP_ENDPOINT, env.OTEL_EXPORTER_OTLP_ENDPOINT); + return base ? `${base.replace(/\/+$/u, "")}/v1/traces` : ""; +} + +function firstNonEmpty(...values) { + for (const value of values) { + const text = String(value || "").trim(); + if (text) return text; + } + return ""; +} + function parseOtelTraceparent(value) { const match = firstHeaderValue(value).trim().match(OTEL_TRACEPARENT_PATTERN); if (!match) return null; diff --git a/internal/dev-entrypoint/cloud-web-runtime.test.mjs b/internal/dev-entrypoint/cloud-web-runtime.test.mjs index cdb2c8e9..4b50f194 100644 --- a/internal/dev-entrypoint/cloud-web-runtime.test.mjs +++ b/internal/dev-entrypoint/cloud-web-runtime.test.mjs @@ -705,6 +705,74 @@ test("cloud web OpenCode proxy injects upstream Basic Auth without forwarding HW } }); +test("cloud web OpenCode proxy emits bounded OTLP span", async () => { + const otelBodies = []; + const otel = createServer(async (request, response) => { + let body = ""; + for await (const chunk of request) body += chunk; + otelBodies.push(JSON.parse(body)); + response.writeHead(200, { "content-type": "application/json" }); + response.end("{}\n"); + }); + await listen(otel); + const restoreEnv = withEnv({ OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: `${serverUrl(otel)}/v1/traces` }); + + const cloudApi = createServer((request, response) => { + request.resume(); + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ authenticated: request.headers.cookie === "hwlab_session=session-a" })); + }); + const opencode = createServer((request, response) => { + request.resume(); + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ ok: true })); + }); + await listen(cloudApi); + await listen(opencode); + + const cloudWeb = createCloudWebServer({ + serviceId: "hwlab-cloud-web", + roots: [], + cloudApiBaseUrl: serverUrl(cloudApi), + cloudApiProxyTimeoutMs: 1000, + opencodeUpstreamUrl: serverUrl(opencode), + opencodeProxyHost: "127.0.0.1", + opencodeProxyTimeoutMs: 1000, + opencodeUsername: "oc_user", + opencodePassword: "oc_password", + healthPayload: () => ({ status: "ok" }), + sendJson(response, statusCode, body) { + const payload = JSON.stringify(body); + response.writeHead(statusCode, { "content-type": "application/json", "content-length": Buffer.byteLength(payload) }); + response.end(payload); + } + }); + await listen(cloudWeb); + + try { + const response = await fetch(`${serverUrl(cloudWeb)}/session/ses_otel/prompt_async`, { + method: "POST", + headers: { accept: "application/json", "content-type": "application/json", cookie: "hwlab_session=session-a" }, + body: JSON.stringify({ parts: [{ type: "text", text: "hi" }] }) + }); + assert.equal(response.status, 200); + await waitFor(() => otelBodies.length > 0, "expected OpenCode proxy OTel span"); + const spans = otelBodies.flatMap((body) => body.resourceSpans?.flatMap((resourceSpan) => resourceSpan.scopeSpans?.flatMap((scopeSpan) => scopeSpan.spans ?? []) ?? []) ?? []); + const span = spans.find((item) => item.name === "opencode.proxy.request"); + assert.ok(span, "expected opencode.proxy.request span"); + const attrs = new Map((span.attributes ?? []).map((entry) => [entry.key, entry.value?.stringValue ?? entry.value?.intValue ?? entry.value?.boolValue])); + assert.equal(attrs.get("http.route"), "/session/:sessionID/prompt_async"); + assert.equal(attrs.get("http.response.status_class"), "2xx"); + assert.equal(attrs.get("valuesPrinted"), false); + } finally { + restoreEnv(); + await close(cloudWeb); + await close(opencode); + await close(cloudApi); + await close(otel); + } +}); + test("cloud web OpenCode proxy accepts short-lived tickets minted by the shell", async () => { const restoreEnv = withEnv({ HWLAB_CLOUD_WEB_DISPLAY_TIME_ZONE: "Asia/Shanghai", @@ -829,6 +897,15 @@ function close(server) { }); } +async function waitFor(predicate, message, timeoutMs = 1000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate()) return; + await new Promise((resolve) => setTimeout(resolve, 20)); + } + assert.fail(message); +} + function disconnectDuringResponse(baseUrl, pathname) { const url = new URL(pathname, baseUrl); return new Promise((resolve, reject) => { diff --git a/internal/dev-entrypoint/opencode-provider-proxy.mjs b/internal/dev-entrypoint/opencode-provider-proxy.mjs new file mode 100644 index 00000000..1fd6b598 --- /dev/null +++ b/internal/dev-entrypoint/opencode-provider-proxy.mjs @@ -0,0 +1,318 @@ +#!/usr/bin/env node +import { randomBytes } from "node:crypto"; +import { createServer, request as httpRequest } from "node:http"; +import { request as httpsRequest } from "node:https"; + +const serviceId = process.env.OTEL_SERVICE_NAME || process.env.HWLAB_SERVICE_ID || "opencode-provider-proxy"; +const port = parsePositiveInteger(process.env.PORT || process.env.HWLAB_OPENCODE_PROVIDER_PROXY_PORT, 4097); +const upstreamBaseUrl = new URL(requiredEnv("HWLAB_OPENCODE_PROVIDER_PROXY_UPSTREAM_BASE_URL")); +const publicBasePath = normalizeBasePath(process.env.HWLAB_OPENCODE_PROVIDER_PROXY_PUBLIC_BASE_PATH || "/v1"); +const timeoutMs = parsePositiveInteger(process.env.HWLAB_OPENCODE_PROVIDER_PROXY_TIMEOUT_MS, 600000); +const otelEndpoint = otelTracesEndpoint(process.env); +const startedAt = new Date(); + +const hopByHopHeaders = new Set(["connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailer", "transfer-encoding", "upgrade"]); + +const server = createServer((clientReq, clientRes) => { + const url = new URL(clientReq.url || "/", "http://opencode-provider-proxy.local"); + if (url.pathname === "/health" || url.pathname === "/health/live" || url.pathname === "/health/readiness") { + sendJson(clientRes, 200, healthPayload()); + return; + } + proxyProviderRequest(clientReq, clientRes, url).catch((error) => { + if (!clientRes.headersSent) { + sendJson(clientRes, 502, { + error: "opencode_provider_proxy_failed", + message: "OpenCode provider proxy failed", + reason: error instanceof Error ? error.message : String(error) + }); + } else { + clientRes.destroy(error); + } + }); +}); + +server.listen(port, "0.0.0.0", () => { + process.stdout.write(JSON.stringify({ serviceId, status: "listening", port, upstreamHost: upstreamBaseUrl.host, publicBasePath, valuesPrinted: false }) + "\n"); +}); + +async function proxyProviderRequest(clientReq, clientRes, url) { + const trace = traceContextFromHeaders(clientReq.headers); + const startSpanId = randomOtelSpanId(); + const started = process.hrtime.bigint(); + const startTime = Date.now(); + const target = providerTargetUrl(url); + const route = providerRoute(url.pathname); + const attrs = { + "http.request.method": clientReq.method || "GET", + "http.route": route, + "url.scheme": target.protocol.replace(":", ""), + "server.address": target.hostname, + "server.port": Number(target.port || (target.protocol === "https:" ? 443 : 80)), + "opencode.provider.upstream_host": upstreamBaseUrl.host, + "opencode.provider.public_base_path": publicBasePath, + valuesPrinted: false + }; + + emitOtelSpanAsync({ + name: "opencode.provider.request.start", + traceId: trace.traceId, + spanId: startSpanId, + parentSpanId: trace.parentSpanId, + kind: 1, + startTimeMs: startTime, + endTimeMs: startTime, + attributes: { ...attrs, "opencode.provider.phase": "start" } + }); + + clientRes.setHeader("traceparent", traceparent(trace.traceId, startSpanId)); + clientRes.setHeader("x-hwlab-otel-trace-id", trace.traceId); + + await new Promise((resolve) => { + let settled = false; + let statusCode = 0; + let errorCode = ""; + let timeout = null; + const settle = () => { + if (settled) return; + settled = true; + if (timeout) clearTimeout(timeout); + const elapsedMs = Number((process.hrtime.bigint() - started) / 1000000n); + emitOtelSpanAsync({ + name: "opencode.provider.request", + traceId: trace.traceId, + spanId: randomOtelSpanId(), + parentSpanId: startSpanId, + kind: 3, + startTimeMs: startTime, + endTimeMs: startTime + Math.max(elapsedMs, 0), + statusCode, + errorCode, + attributes: { + ...attrs, + "http.response.status_code": statusCode || undefined, + "http.response.status_class": statusCode ? `${Math.floor(statusCode / 100)}xx` : "unknown", + "opencode.provider.elapsed_ms": elapsedMs, + "opencode.provider.timeout_ms": timeoutMs, + "opencode.provider.phase": errorCode ? "failed" : "complete" + } + }); + resolve(); + }; + + const requestImpl = target.protocol === "https:" ? httpsRequest : httpRequest; + const upstreamReq = requestImpl({ + protocol: target.protocol, + hostname: target.hostname, + port: target.port || (target.protocol === "https:" ? 443 : 80), + method: clientReq.method, + path: `${target.pathname}${target.search}`, + headers: providerRequestHeaders(clientReq.headers, target, trace, startSpanId) + }, (upstreamRes) => { + statusCode = upstreamRes.statusCode || 502; + clientRes.writeHead(statusCode, { + ...providerResponseHeaders(upstreamRes.headers), + traceparent: traceparent(trace.traceId, startSpanId), + "x-hwlab-otel-trace-id": trace.traceId + }); + upstreamRes.pipe(clientRes); + upstreamRes.on("end", settle); + upstreamRes.on("close", settle); + upstreamRes.on("error", (error) => { + errorCode = error?.code || "upstream_response_error"; + if (!clientRes.writableEnded) clientRes.destroy(error); + settle(); + }); + }); + + timeout = setTimeout(() => { + errorCode = "upstream_timeout"; + upstreamReq.destroy(new Error(`opencode provider proxy timed out after ${timeoutMs}ms`)); + }, timeoutMs); + + upstreamReq.on("error", (error) => { + if (!errorCode) errorCode = error?.code || "upstream_request_error"; + if (!clientRes.headersSent) { + statusCode = errorCode === "upstream_timeout" ? 504 : 502; + sendJson(clientRes, statusCode, { + error: "opencode_provider_upstream_error", + message: "OpenCode provider upstream request failed", + reason: error instanceof Error ? error.message : String(error), + traceId: trace.traceId, + valuesPrinted: false + }); + } else if (!clientRes.writableEnded) { + clientRes.destroy(error); + } + settle(); + }); + + clientReq.on("error", (error) => { + errorCode = error?.code || "client_request_error"; + upstreamReq.destroy(error); + }); + clientReq.pipe(upstreamReq); + }); +} + +function healthPayload() { + return { serviceId, status: "ok", port, upstreamHost: upstreamBaseUrl.host, publicBasePath, startedAt: startedAt.toISOString(), uptimeSeconds: Math.round((Date.now() - startedAt.getTime()) / 1000), otelEnabled: Boolean(otelEndpoint), valuesPrinted: false }; +} + +function providerTargetUrl(url) { + const suffix = stripBasePath(url.pathname, publicBasePath); + const basePath = upstreamBaseUrl.pathname.replace(/\/+$/u, ""); + const nextPath = suffix ? `${basePath}/${suffix.replace(/^\/+/, "")}` : basePath || "/"; + const target = new URL(upstreamBaseUrl.toString()); + target.pathname = nextPath; + target.search = url.search; + return target; +} + +function stripBasePath(pathname, basePath) { + if (!basePath || basePath === "/") return pathname; + if (pathname === basePath) return "/"; + if (pathname.startsWith(`${basePath}/`)) return pathname.slice(basePath.length) || "/"; + return pathname; +} + +function providerRoute(pathname) { + const stripped = stripBasePath(pathname, publicBasePath); + if (/^\/chat\/completions\/?$/u.test(stripped)) return `${publicBasePath}/chat/completions`; + if (/^\/models\/?$/u.test(stripped)) return `${publicBasePath}/models`; + return `${publicBasePath}${stripped.startsWith("/") ? stripped : `/${stripped}`}`; +} + +function providerRequestHeaders(headers, target, trace, parentSpanId) { + const result = {}; + for (const [name, value] of Object.entries(headers)) { + const lower = name.toLowerCase(); + if (hopByHopHeaders.has(lower) || lower === "host") continue; + if (value !== undefined) result[lower] = value; + } + result.host = target.host; + result.traceparent = traceparent(trace.traceId, parentSpanId); + result["x-hwlab-otel-trace-id"] = trace.traceId; + result["x-source-service-id"] = serviceId; + return result; +} + +function providerResponseHeaders(headers) { + const result = {}; + for (const [name, value] of Object.entries(headers)) { + const lower = name.toLowerCase(); + if (hopByHopHeaders.has(lower) || lower === "content-length") continue; + if (value !== undefined) result[name] = value; + } + return result; +} + +function sendJson(response, statusCode, body) { + const payload = JSON.stringify(body); + response.writeHead(statusCode, { "content-type": "application/json; charset=utf-8", "content-length": Buffer.byteLength(payload) }); + response.end(payload); +} + +function emitOtelSpanAsync(span) { + if (!otelEndpoint) return; + setTimeout(() => { + emitOtelSpan(span).catch((error) => { + process.stderr.write(JSON.stringify({ event: "opencode-provider-proxy-otel-emit-failed", serviceId, error: error instanceof Error ? error.message : String(error), valuesPrinted: false }) + "\n"); + }); + }, 0); +} + +async function emitOtelSpan(span) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 1500); + try { + await fetch(otelEndpoint, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(otelPayload(span)), signal: controller.signal }); + } finally { + clearTimeout(timeout); + } +} + +function otelPayload(span) { + return { resourceSpans: [{ resource: { attributes: otelAttributes({ "service.name": serviceId, "deployment.environment": process.env.HWLAB_ENVIRONMENT || "v03" }) }, scopeSpans: [{ scope: { name: "hwlab.opencode.provider_proxy", version: "1" }, spans: [{ traceId: span.traceId, spanId: span.spanId, parentSpanId: span.parentSpanId || undefined, name: span.name, kind: span.kind, startTimeUnixNano: unixNano(span.startTimeMs), endTimeUnixNano: unixNano(span.endTimeMs), attributes: otelAttributes({ ...span.attributes, "otel.trace_id": span.traceId }), status: span.errorCode || (span.statusCode && span.statusCode >= 500) ? { code: 2, message: span.errorCode || `http_${span.statusCode}` } : { code: 1 } }] }] }] }; +} + +function otelAttributes(values) { + const result = []; + for (const [key, value] of Object.entries(values || {})) { + if (value === undefined || value === null || value === "") continue; + if (typeof value === "number" && Number.isFinite(value)) result.push({ key, value: Number.isInteger(value) ? { intValue: String(value) } : { doubleValue: value } }); + else if (typeof value === "boolean") result.push({ key, value: { boolValue: value } }); + else result.push({ key, value: { stringValue: String(value) } }); + } + return result; +} + +function unixNano(ms) { + return String(BigInt(Math.max(0, Math.trunc(ms))) * 1000000n); +} + +function traceContextFromHeaders(headers) { + const parsed = parseTraceparent(headers.traceparent); + return { traceId: parsed?.traceId || normalizedTraceId(headers["x-hwlab-otel-trace-id"]) || randomOtelTraceId(), parentSpanId: parsed?.spanId || "" }; +} + +function parseTraceparent(value) { + const match = firstHeaderValue(value).trim().match(/^00-([0-9a-f]{32})-([0-9a-f]{16})-[0-9a-f]{2}$/u); + if (!match) return null; + const traceId = normalizedTraceId(match[1]); + const spanId = normalizedSpanId(match[2]); + return traceId && spanId ? { traceId, spanId } : null; +} + +function traceparent(traceId, spanId) { + return `00-${traceId}-${spanId}-01`; +} + +function randomOtelTraceId() { + const value = randomBytes(16).toString("hex"); + return value === "00000000000000000000000000000000" ? "00000000000000000000000000000001" : value; +} + +function randomOtelSpanId() { + const value = randomBytes(8).toString("hex"); + return value === "0000000000000000" ? "0000000000000001" : value; +} + +function normalizedTraceId(value) { + const text = firstHeaderValue(value).trim().toLowerCase(); + return /^[0-9a-f]{32}$/u.test(text) && text !== "00000000000000000000000000000000" ? text : ""; +} + +function normalizedSpanId(value) { + const text = firstHeaderValue(value).trim().toLowerCase(); + return /^[0-9a-f]{16}$/u.test(text) && text !== "0000000000000000" ? text : ""; +} + +function firstHeaderValue(value) { + if (Array.isArray(value)) return String(value[0] ?? ""); + return String(value ?? ""); +} + +function otelTracesEndpoint(env) { + const explicit = String(env.HWLAB_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT || env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT || "").trim(); + if (explicit) return explicit; + const base = String(env.HWLAB_OTEL_EXPORTER_OTLP_ENDPOINT || env.OTEL_EXPORTER_OTLP_ENDPOINT || "").trim(); + return base ? `${base.replace(/\/+$/u, "")}/v1/traces` : ""; +} + +function normalizeBasePath(value) { + const text = String(value || "/").trim(); + const normalized = `/${text.replace(/^\/+|\/+$/gu, "")}`; + return normalized === "/" ? "/" : normalized; +} + +function parsePositiveInteger(value, fallback) { + const parsed = Number.parseInt(String(value ?? ""), 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +function requiredEnv(name) { + const value = String(process.env[name] || "").trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} diff --git a/scripts/gitops-render.mjs b/scripts/gitops-render.mjs index a5401a56..dc471763 100644 --- a/scripts/gitops-render.mjs +++ b/scripts/gitops-render.mjs @@ -5645,13 +5645,20 @@ function opencodeRuntimeConfigForProfile(deploy, profile) { const apiKeyEnv = configString(config.apiKeyEnv) || opencodeApiKeyEnvName(providerId); const contextLimit = opencodePositiveInteger(config.contextLimit, 1000000, `deploy.lanes.${profile}.opencode.contextLimit`); const outputLimit = opencodePositiveInteger(config.outputLimit, 384000, `deploy.lanes.${profile}.opencode.outputLimit`); + const upstreamBaseURL = configString(config.upstreamBaseURL) || configString(config.baseURL) || "https://opencode.ai/zen/go/v1"; + const providerProxyPort = opencodePositiveInteger(config.providerProxyPort, 4097, `deploy.lanes.${profile}.opencode.providerProxyPort`); + const providerProxyBasePath = configString(config.providerProxyBasePath) || "/v1"; + const providerProxyBaseURL = configString(config.providerProxyBaseURL) || `http://127.0.0.1:${providerProxyPort}${providerProxyBasePath}`; return { providerProfile, providerId, displayName: configString(config.displayName) || `AgentRun ${providerProfile}`, model, smallModel, - baseURL: configString(config.baseURL) || "https://opencode.ai/zen/go/v1", + baseURL: providerProxyBaseURL, + upstreamBaseURL, + providerProxyPort, + providerProxyBasePath, npm: configString(config.npm) || "@ai-sdk/openai-compatible", apiKeyEnv, apiKeySecretRef: configString(config.apiKeySecretRef) || "hwlab-code-agent-provider/opencode-api-key", @@ -5710,10 +5717,19 @@ function opencodeConfigContent(config) { }); } -function opencodeServerManifest({ profile = "v03", source, deploy = null }) { +function opencodeServerManifest({ profile = "v03", source, deploy = null, catalog = null, registryPrefix = defaultRegistryPrefix, useDeployImages = false, sourceBranch = defaultBranch, gitReadUrl = defaultV02GitReadUrl, sourceRepo = defaultSourceRepo }) { assert.ok(isRuntimeLane(profile), `opencode-server profile must be a runtime lane, got ${profile}`); const namespace = namespaceNameForProfile(profile); const name = "opencode-server"; + const helperServiceId = "hwlab-cloud-web"; + const envReuseServiceIds = envReuseServiceIdsForLane(deploy, profile); + const helperImage = runtimeImageForService({ catalog, deploy, serviceId: helperServiceId, source, registryPrefix, useDeployImages, digestPin: true, envReuseServiceIds }); + const helperCommit = runtimeCommitForService({ catalog, deploy, serviceId: helperServiceId, source, registryPrefix, useDeployImages, digestPin: true, envReuseServiceIds }); + const helperImageTag = runtimeImageTagForService({ catalog, deploy, serviceId: helperServiceId, source, registryPrefix, useDeployImages, digestPin: true, envReuseServiceIds }); + const providerProxyBootSh = "deploy/runtime/boot/opencode-provider-proxy.sh"; + const providerProxyBootMetadata = v02EnvReuseEnabled(catalog, helperServiceId, envReuseServiceIds) + ? bootMetadataForService({ args: { sourceRepo: deploy?.lanes?.[profile]?.sourceRepo || sourceRepo, registryPrefix }, catalog, deployService: deployServiceForBoot(deploy, helperServiceId, profile), serviceId: helperServiceId, source }) + : null; const selector = { "app.kubernetes.io/name": name, "app.kubernetes.io/part-of": "hwlab", @@ -5736,12 +5752,23 @@ function opencodeServerManifest({ profile = "v03", source, deploy = null }) { "hwlab.pikastech.local/opencode-provider-profile": opencodeConfig.providerProfile, "hwlab.pikastech.local/opencode-provider-id": opencodeConfig.providerId, "hwlab.pikastech.local/opencode-model": opencodeConfig.model, + "hwlab.pikastech.local/opencode-provider-base-url": opencodeConfig.baseURL, + "hwlab.pikastech.local/opencode-provider-upstream-base-url": opencodeConfig.upstreamBaseURL, "hwlab.pikastech.local/opencode-config-sha256": configSha256, "hwlab.pikastech.local/opencode-api-key-secret-ref": `${apiKeySecretRef.name}/${apiKeySecretRef.key}`, "hwlab.pikastech.local/agentrun-provider-secret-ref": `${opencodeConfig.agentrunSecretNamespace}/${opencodeConfig.agentrunSecretName}`, "hwlab.pikastech.local/agentrun-provider-secret-keys": opencodeConfig.agentrunSecretKeys.join(","), "hwlab.pikastech.local/values-printed": "false" }; + if (providerProxyBootMetadata) { + Object.assign(annotations, { + "hwlab.pikastech.local/opencode-provider-proxy-runtime-mode": providerProxyBootMetadata.runtimeMode, + "hwlab.pikastech.local/opencode-provider-proxy-boot-repo": providerProxyBootMetadata.bootRepo, + "hwlab.pikastech.local/opencode-provider-proxy-boot-commit": providerProxyBootMetadata.bootCommit, + "hwlab.pikastech.local/opencode-provider-proxy-boot-sh": providerProxyBootSh, + "hwlab.pikastech.local/opencode-provider-proxy-environment-digest": providerProxyBootMetadata.environmentDigest ?? "not_published" + }); + } const authSecretName = secretNameForProfile("hwlab-opencode-server-auth", profile); return { apiVersion: "v1", @@ -5788,8 +5815,8 @@ function opencodeServerManifest({ profile = "v03", source, deploy = null }) { name, image: "ghcr.io/anomalyco/opencode:1.17.7", imagePullPolicy: "IfNotPresent", - command: ["opencode"], - args: ["serve", "--hostname", "0.0.0.0", "--port", "4096"], + command: ["/bin/sh", "-ec"], + args: ["if ! command -v git >/dev/null 2>&1; then apk add --no-cache git; fi\nexec opencode serve --hostname 0.0.0.0 --port 4096"], workingDir: "/workspace", env: [ { name: "HOME", value: "/workspace" }, @@ -5806,7 +5833,35 @@ function opencodeServerManifest({ profile = "v03", source, deploy = null }) { livenessProbe: { tcpSocket: { port: "http" }, periodSeconds: 20, failureThreshold: 3 }, resources: { requests: { cpu: "100m", memory: "256Mi" }, limits: { cpu: "1", memory: "1Gi" } }, volumeMounts: [{ name: "workspace", mountPath: "/workspace" }] - }], + }, (() => { + const container = { + name: "opencode-provider-proxy", + image: helperImage, + imagePullPolicy: "IfNotPresent", + command: ["node", "/app/internal/dev-entrypoint/opencode-provider-proxy.mjs"], + env: [ + { name: "HWLAB_SERVICE_ID", value: "opencode-provider-proxy" }, + { name: "HWLAB_ENVIRONMENT", value: profile }, + { name: "HWLAB_COMMIT_ID", value: helperCommit }, + { name: "HWLAB_IMAGE", value: helperImage }, + { name: "HWLAB_IMAGE_TAG", value: helperImageTag }, + { name: "PORT", value: String(opencodeConfig.providerProxyPort) }, + { name: "HWLAB_OPENCODE_PROVIDER_PROXY_UPSTREAM_BASE_URL", value: opencodeConfig.upstreamBaseURL }, + { name: "HWLAB_OPENCODE_PROVIDER_PROXY_PUBLIC_BASE_PATH", value: opencodeConfig.providerProxyBasePath }, + { name: "HWLAB_OPENCODE_PROVIDER_PROXY_TIMEOUT_MS", value: "600000" }, + { name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", value: "http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces" }, + { name: "OTEL_SERVICE_NAME", value: "opencode-provider-proxy" } + ], + ports: [{ name: "provider", containerPort: opencodeConfig.providerProxyPort }], + readinessProbe: { httpGet: { path: "/health/readiness", port: "provider" }, periodSeconds: 10, failureThreshold: 3 }, + livenessProbe: { httpGet: { path: "/health/live", port: "provider" }, periodSeconds: 20, failureThreshold: 3 }, + resources: { requests: { cpu: "25m", memory: "64Mi" }, limits: { cpu: "250m", memory: "256Mi" } } + }; + if (providerProxyBootMetadata) { + applyEnvReuseBootEnv(container, providerProxyBootMetadata, { sourceBranch, gitReadUrl, bootSh: providerProxyBootSh }); + } + return container; + })()], volumes: [{ name: "workspace", persistentVolumeClaim: { claimName: `${name}-data` } }] } } @@ -5991,7 +6046,7 @@ async function plannedFiles(args) { if (workbenchRedis) putJson(`${runtimePath}/workbench-redis.yaml`, workbenchRuntimeRedisManifest({ profile, config: workbenchRedis, source })); if (isRuntimeLane(profile)) putJson(`${runtimePath}/openfga.yaml`, v02OpenFgaManifest({ profile, source })); if (isRuntimeLane(profile)) putJson(`${runtimePath}/observability.yaml`, observabilityManifest({ deploy, profile, namespace, labels: profileLabels, annotations, metricsSidecarScript })); - if (isRuntimeLane(profile)) putJson(`${runtimePath}/opencode.yaml`, opencodeServerManifest({ profile, source, deploy })); + if (isRuntimeLane(profile)) putJson(`${runtimePath}/opencode.yaml`, opencodeServerManifest({ profile, source, deploy, catalog: artifactCatalog, registryPrefix: args.registryPrefix, useDeployImages: args.useDeployImages, sourceBranch: args.sourceBranch, gitReadUrl: args.gitReadUrl, sourceRepo: args.sourceRepo })); if (includeDeviceAgent) putJson(`${runtimePath}/device-agent-71-freq.yaml`, deviceAgent71FreqManifest({ profile, source, registryPrefix: args.registryPrefix, catalog: artifactCatalog, useDeployImages: args.useDeployImages })); putJson(`${runtimePath}/node-frpc.yaml`, nodeFrpcManifest({ profile, source })); } diff --git a/scripts/gitops-render.test.ts b/scripts/gitops-render.test.ts index b3a1ec04..3c1677b4 100644 --- a/scripts/gitops-render.test.ts +++ b/scripts/gitops-render.test.ts @@ -723,18 +723,34 @@ test("v03 render includes D518 secret-plane smoke on D518 gitops root", async () assert.ok(opencodeDeployment, "expected opencode-server deployment"); assert.equal(opencodeDeployment.metadata?.annotations?.["hwlab.pikastech.local/opencode-provider-profile"], "dsflash-go"); assert.equal(opencodeDeployment.metadata?.annotations?.["hwlab.pikastech.local/agentrun-provider-secret-ref"], "agentrun-v02/agentrun-v01-provider-dsflash-go"); + assert.equal(opencodeDeployment.metadata?.annotations?.["hwlab.pikastech.local/opencode-provider-base-url"], "http://127.0.0.1:4097/v1"); + assert.equal(opencodeDeployment.metadata?.annotations?.["hwlab.pikastech.local/opencode-provider-upstream-base-url"], "https://opencode.ai/zen/go/v1"); + assert.equal(opencodeDeployment.metadata?.annotations?.["hwlab.pikastech.local/opencode-provider-proxy-boot-sh"], "deploy/runtime/boot/opencode-provider-proxy.sh"); assert.equal(opencodeDeployment.metadata?.annotations?.["hwlab.pikastech.local/values-printed"], "false"); assert.match(opencodeDeployment.spec?.template?.metadata?.annotations?.["hwlab.pikastech.local/opencode-config-sha256"] ?? "", /^[a-f0-9]{64}$/u); const opencodeContainer = collectContainersFromItem(opencodeDeployment).find((container) => container.name === "opencode-server"); + assert.deepEqual(opencodeContainer?.command, ["/bin/sh", "-ec"]); + assert.match(opencodeContainer?.args?.[0] ?? "", /apk add --no-cache git/u); const opencodeEnvEntries = new Map((opencodeContainer?.env ?? []).map((entry) => [entry.name, entry])); assert.deepEqual(opencodeEnvEntries.get("OPENCODE_DSFLASH_GO_API_KEY")?.valueFrom?.secretKeyRef, { name: "hwlab-v03-code-agent-provider", key: "opencode-api-key", optional: true }); const opencodeConfig = JSON.parse(opencodeEnvEntries.get("OPENCODE_CONFIG_CONTENT")?.value ?? "{}"); assert.equal(opencodeConfig.model, "dsflash-go/deepseek-v4-flash"); assert.equal(opencodeConfig.small_model, "dsflash-go/deepseek-v4-flash"); assert.equal(opencodeConfig.provider["dsflash-go"].npm, "@ai-sdk/openai-compatible"); - assert.equal(opencodeConfig.provider["dsflash-go"].options.baseURL, "https://opencode.ai/zen/go/v1"); + assert.equal(opencodeConfig.provider["dsflash-go"].options.baseURL, "http://127.0.0.1:4097/v1"); assert.equal(opencodeConfig.provider["dsflash-go"].options.apiKey, "{env:OPENCODE_DSFLASH_GO_API_KEY}"); assert.deepEqual(opencodeConfig.provider["dsflash-go"].models["deepseek-v4-flash"].limit, { context: 1000000, output: 384000 }); + const providerProxy = collectContainersFromItem(opencodeDeployment).find((container) => container.name === "opencode-provider-proxy"); + assert.ok(providerProxy, "expected provider proxy sidecar"); + assert.equal(providerProxy.command, undefined); + const providerProxyEnv = new Map((providerProxy.env ?? []).map((entry) => [entry.name, entry.value])); + assert.equal(providerProxyEnv.get("HWLAB_BOOT_SH"), "deploy/runtime/boot/opencode-provider-proxy.sh"); + assert.equal(providerProxyEnv.get("HWLAB_BOOT_REF"), "v0.3"); + assert.match(providerProxyEnv.get("HWLAB_BOOT_READ_URL") ?? "", /pikasTech\/HWLAB/u); + assert.equal(providerProxyEnv.get("HWLAB_OPENCODE_PROVIDER_PROXY_UPSTREAM_BASE_URL"), "https://opencode.ai/zen/go/v1"); + assert.equal(providerProxyEnv.get("HWLAB_OPENCODE_PROVIDER_PROXY_PUBLIC_BASE_PATH"), "/v1"); + assert.equal(providerProxyEnv.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"), "http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces"); + assert.equal(providerProxyEnv.get("OTEL_SERVICE_NAME"), "opencode-provider-proxy"); } finally { await rm(outDir, { recursive: true, force: true }); await rm(scratchDir, { recursive: true, force: true }); diff --git a/scripts/src/check-plan.mjs b/scripts/src/check-plan.mjs index ff5640df..d865c742 100644 --- a/scripts/src/check-plan.mjs +++ b/scripts/src/check-plan.mjs @@ -37,6 +37,7 @@ export const checkProfiles = Object.freeze({ { id: "check-017-cloud-api-run-bun", group: "cloud-api", command: ["node","scripts/run-bun.mjs","build","internal/cloud/db-contract.ts","--target=bun","--packages=external","--outdir=/tmp/hwlab-ts-check"] }, { id: "check-018-repo-cloud-web-routes", group: "repo", command: ["node","--check","internal/dev-entrypoint/cloud-web-routes.mjs"] }, { id: "check-019-repo-cloud-web-proxy", group: "repo", command: ["node","--check","internal/dev-entrypoint/cloud-web-proxy.mjs"] }, + { id: "check-019-opencode-provider-proxy", group: "repo", command: ["node","--check","internal/dev-entrypoint/opencode-provider-proxy.mjs"] }, { id: "check-020-repo-cloud-web-proxy-test", group: "repo", command: ["node","--check","internal/dev-entrypoint/cloud-web-proxy.test.mjs"] }, { id: "check-021-repo-http", group: "repo", command: ["node","--check","internal/dev-entrypoint/http.mjs"] }, { id: "check-022-repo-http-test", group: "repo", command: ["node","--check","internal/dev-entrypoint/http.test.mjs"] },