diff --git a/deploy/runtime/boot/hwlab-cloud-web.sh b/deploy/runtime/boot/hwlab-cloud-web.sh index c7782936..1051e61e 100755 --- a/deploy/runtime/boot/hwlab-cloud-web.sh +++ b/deploy/runtime/boot/hwlab-cloud-web.sh @@ -35,6 +35,7 @@ import { serveCloudWeb } from "./internal/dev-entrypoint/cloud-web-runtime.mjs"; const port = Number.parseInt(process.env.PORT || process.env.HWLAB_PORT || "8080", 10); const serviceId = process.env.HWLAB_SERVICE_ID || "hwlab-cloud-web"; const environment = process.env.HWLAB_ENVIRONMENT || "v02"; +const processStartedAt = new Date(); const roots = [ `${process.cwd()}/web/hwlab-cloud-web/dist`, `${process.cwd()}/web/hwlab-cloud-web` @@ -89,6 +90,11 @@ function healthPayload() { build: { source: process.env.HWLAB_BUILD_SOURCE || "runtime-boot:web-build", metadataSource: "runtime-boot" + }, + process: { + pid: process.pid, + startedAt: processStartedAt.toISOString(), + uptimeSeconds: Math.round((Date.now() - processStartedAt.getTime()) / 1000) } }; } diff --git a/internal/dev-entrypoint/cloud-web-runtime.mjs b/internal/dev-entrypoint/cloud-web-runtime.mjs index 2ce17788..6894c3cf 100644 --- a/internal/dev-entrypoint/cloud-web-runtime.mjs +++ b/internal/dev-entrypoint/cloud-web-runtime.mjs @@ -18,8 +18,16 @@ const GZIP_MIN_BYTES = 1024; const DEFAULT_AUTH_USERNAME = "admin"; const DEFAULT_AUTH_PASSWORD = "hwlab2026"; const DEFAULT_WORKBENCH_PROJECT_ID = "prj_hwpod_workbench"; +const CLIENT_DISCONNECT_ERROR_CODES = new Set([ + "ECONNRESET", + "EPIPE", + "ERR_STREAM_PREMATURE_CLOSE" +]); +const CLIENT_DISCONNECT_ERROR_PATTERN = /\b(?:socket hang up|client disconnected|premature close)\b/iu; +const CLOUD_WEB_ERROR_HANDLER_INSTALLED = Symbol.for("hwlab.cloud-web.error-handler-installed"); export async function serveCloudWeb(options) { + installCloudWebProcessErrorHandlers({ serviceId: options.serviceId }); const server = createCloudWebServer(options); server.listen(options.port, "0.0.0.0", () => { process.stdout.write(JSON.stringify({ @@ -43,50 +51,130 @@ export function createCloudWebServer({ }) }) { return createServer(async (request, response) => { + attachRequestErrorHandlers({ request, response, serviceId }); const url = new URL(request.url || "/", "http://hwlab-cloud-web.local"); - if (url.pathname === "/health/live" || url.pathname === "/health") { - sendJson(response, 200, healthPayload()); - return; - } - if (await handleCloudWebAuth({ request, response, url, cloudApiBaseUrl, cloudApiProxyTimeoutMs, serviceId, sendJson })) { - return; - } - - const proxyPolicy = cloudWebProxyRoutePolicy(request.method, url.pathname); - if (proxyPolicy.proxy) { - await proxyCloudApi({ request, response, url, cloudApiBaseUrl, cloudApiProxyTimeoutMs, serviceId, sendJson }); - return; - } - - const routePath = url.pathname.replace(/\/+$/u, "") || "/"; - const relativePath = routePath === "/" || routePath === "/gate" || routePath === "/diagnostics/gate" || routePath === "/help" || routePath === "/skills" || routePath === "/access" - ? "index.html" - : url.pathname.slice(1); - for (const root of roots) { - const candidate = path.resolve(root, relativePath); - if (!candidate.startsWith(root)) continue; - try { - const info = await stat(candidate); - if (!info.isFile()) continue; - const body = await readFile(candidate); - const type = contentType(candidate); - const encoded = await encodeStaticBody(request, body, type); - response.writeHead(200, { - "content-type": type, - ...encoded.headers, - "content-length": encoded.body.length - }); - response.end(encoded.body); + try { + if (url.pathname === "/health/live" || url.pathname === "/health") { + sendJson(response, 200, healthPayload()); + return; + } + if (await handleCloudWebAuth({ request, response, url, cloudApiBaseUrl, cloudApiProxyTimeoutMs, serviceId, sendJson })) { return; - } catch { - // Try the next root. } - } - sendJson(response, 404, { error: "not_found", serviceId, path: url.pathname }); + const proxyPolicy = cloudWebProxyRoutePolicy(request.method, url.pathname); + if (proxyPolicy.proxy) { + await proxyCloudApi({ request, response, url, cloudApiBaseUrl, cloudApiProxyTimeoutMs, serviceId, sendJson }); + return; + } + + const routePath = url.pathname.replace(/\/+$/u, "") || "/"; + const relativePath = routePath === "/" || routePath === "/gate" || routePath === "/diagnostics/gate" || routePath === "/help" || routePath === "/skills" || routePath === "/access" + ? "index.html" + : url.pathname.slice(1); + for (const root of roots) { + const candidate = path.resolve(root, relativePath); + if (!candidate.startsWith(root)) continue; + try { + const info = await stat(candidate); + if (!info.isFile()) continue; + const body = await readFile(candidate); + const type = contentType(candidate); + const encoded = await encodeStaticBody(request, body, type); + response.writeHead(200, { + "content-type": type, + ...encoded.headers, + "content-length": encoded.body.length + }); + response.end(encoded.body); + return; + } catch (error) { + if (isClientDisconnectError(error)) { + logCloudWebRuntimeEvent("cloud-web-client-disconnect", { serviceId, request, error }); + return; + } + // Try the next root for ordinary filesystem misses. + } + } + + sendJson(response, 404, { error: "not_found", serviceId, path: url.pathname }); + } catch (error) { + if (isClientDisconnectError(error)) { + logCloudWebRuntimeEvent("cloud-web-client-disconnect", { serviceId, request, error }); + return; + } + if (!response.headersSent && !response.writableEnded) { + sendJson(response, 500, { + error: "internal_error", + serviceId, + message: error instanceof Error ? error.message : String(error) + }); + return; + } + response.destroy(error); + } }); } +export function installCloudWebProcessErrorHandlers({ serviceId } = {}) { + if (process[CLOUD_WEB_ERROR_HANDLER_INSTALLED]) return; + process[CLOUD_WEB_ERROR_HANDLER_INSTALLED] = true; + process.on("uncaughtException", (error) => { + if (isClientDisconnectError(error)) { + logCloudWebRuntimeEvent("cloud-web-client-disconnect", { serviceId, error, source: "uncaughtException" }); + return; + } + logCloudWebRuntimeEvent("cloud-web-runtime-fatal", { serviceId, error, source: "uncaughtException" }); + process.exit(1); + }); + process.on("unhandledRejection", (reason) => { + const error = reason instanceof Error ? reason : new Error(String(reason)); + if (isClientDisconnectError(error)) { + logCloudWebRuntimeEvent("cloud-web-client-disconnect", { serviceId, error, source: "unhandledRejection" }); + return; + } + logCloudWebRuntimeEvent("cloud-web-runtime-fatal", { serviceId, error, source: "unhandledRejection" }); + process.exit(1); + }); +} + +function attachRequestErrorHandlers({ request, response, serviceId }) { + const handle = (error) => { + if (isClientDisconnectError(error)) { + logCloudWebRuntimeEvent("cloud-web-client-disconnect", { serviceId, request, error }); + return; + } + throw error; + }; + request.on("error", handle); + response.on("error", handle); +} + +export function isClientDisconnectError(error) { + const candidate = error instanceof Error ? error : new Error(String(error)); + const code = typeof candidate.code === "string" ? candidate.code : ""; + return CLIENT_DISCONNECT_ERROR_CODES.has(code) || CLIENT_DISCONNECT_ERROR_PATTERN.test(candidate.message || ""); +} + +function logCloudWebRuntimeEvent(event, { serviceId, request = null, error = null, source = null } = {}) { + const payload = { + event, + serviceId: serviceId || "hwlab-cloud-web", + source, + method: request?.method || null, + path: request?.url || null, + error: error + ? { + name: error.name || "Error", + code: typeof error.code === "string" ? error.code : null, + message: error.message || String(error) + } + : null, + observedAt: new Date().toISOString() + }; + process.stderr.write(JSON.stringify(payload) + "\n"); +} + export async function handleCloudWebAuth(options) { const { request, response, url } = options; if (url.pathname === "/auth/workspace-bootstrap" && request.method === "GET") { diff --git a/internal/dev-entrypoint/cloud-web-runtime.test.mjs b/internal/dev-entrypoint/cloud-web-runtime.test.mjs index 35bef532..72dcdef4 100644 --- a/internal/dev-entrypoint/cloud-web-runtime.test.mjs +++ b/internal/dev-entrypoint/cloud-web-runtime.test.mjs @@ -1,11 +1,19 @@ import assert from "node:assert/strict"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { createServer } from "node:http"; +import { connect } from "node:net"; import os from "node:os"; import path from "node:path"; import test from "node:test"; -import { createCloudWebServer } from "./cloud-web-runtime.mjs"; +import { createCloudWebServer, isClientDisconnectError } from "./cloud-web-runtime.mjs"; + +test("cloud web classifies Bun client disconnect ECONNRESET as non-fatal", () => { + const error = new Error("aborted"); + error.code = "ECONNRESET"; + assert.equal(isClientDisconnectError(error), true); + assert.equal(isClientDisconnectError(new Error("database migration failed")), false); +}); test("cloud web auth requests are proxied once", async () => { let upstreamRequests = 0; @@ -263,6 +271,36 @@ test("cloud web forwards hwpod node-ops plans to cloud-api", async () => { } }); +test("cloud web remains healthy after a client disconnects during static response", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-cloud-web-runtime-")); + await writeFile(path.join(root, "large.txt"), "x".repeat(512 * 1024), "utf8"); + const cloudWeb = createCloudWebServer({ + serviceId: "hwlab-cloud-web", + roots: [root], + cloudApiBaseUrl: "http://127.0.0.1:1", + healthPayload: () => ({ status: "ok", process: { uptimeSeconds: 1 } }), + sendJson(response, statusCode, body) { + const payload = JSON.stringify(body); + response.writeHead(statusCode, { + "content-type": "application/json", + "content-length": Buffer.byteLength(payload) + }); + response.end(payload); + } + }); + await listen(cloudWeb); + + try { + await disconnectDuringResponse(serverUrl(cloudWeb), "/large.txt"); + const response = await fetch(`${serverUrl(cloudWeb)}/health/live`); + assert.equal(response.status, 200); + assert.equal((await response.json()).status, "ok"); + } finally { + await close(cloudWeb); + await rm(root, { recursive: true, force: true }); + } +}); + test("cloud web serves access deep link through the React shell", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-cloud-web-runtime-")); await writeFile(path.join(root, "index.html"), "
\n", "utf8"); @@ -306,6 +344,29 @@ function close(server) { }); } +function disconnectDuringResponse(baseUrl, pathname) { + const url = new URL(pathname, baseUrl); + return new Promise((resolve, reject) => { + const socket = connect(Number(url.port), url.hostname); + const timeout = setTimeout(() => { + socket.destroy(); + reject(new Error("client disconnect probe timed out")); + }, 2000); + socket.once("connect", () => { + socket.write(`GET ${url.pathname} HTTP/1.1\r\nHost: ${url.host}\r\nConnection: close\r\n\r\n`); + }); + socket.once("data", () => { + clearTimeout(timeout); + socket.destroy(); + setTimeout(resolve, 50); + }); + socket.once("error", (error) => { + clearTimeout(timeout); + reject(error); + }); + }); +} + function serverUrl(server) { const address = server.address(); return `http://127.0.0.1:${address.port}`; diff --git a/tools/src/hwlab-cli/trace-renderer.ts b/tools/src/hwlab-cli/trace-renderer.ts index d76bf99d..9ad0e242 100644 --- a/tools/src/hwlab-cli/trace-renderer.ts +++ b/tools/src/hwlab-cli/trace-renderer.ts @@ -228,16 +228,18 @@ function upsertAuthoritativeFinalResponseRow(rows: TraceEventRow[], assistantRow } if (matchedState) { const row = rows[matchedState.rowIndex]; - rows[matchedState.rowIndex] = { - ...row, - body: finalText, - terminal: true, - tone: "ok", - header: `${traceClock(completionEvent.createdAt)} 助手最终消息` - }; - matchedState.comparableText = comparableText; - matchedState.derivedSnapshotSuffix = false; - return matchedState.rowIndex; + if (row) { + rows[matchedState.rowIndex] = { + ...row, + body: finalText, + terminal: true, + tone: "ok", + header: `${traceClock(completionEvent.createdAt)} 助手最终消息` + }; + matchedState.comparableText = comparableText; + matchedState.derivedSnapshotSuffix = false; + return matchedState.rowIndex; + } } rows.push({ rowId: `trace-final-response:${completionEvent.seq ?? "completed"}`,