fix: keep Cloud Web alive on client disconnect
This commit is contained in:
@@ -35,6 +35,7 @@ import { serveCloudWeb } from "./internal/dev-entrypoint/cloud-web-runtime.mjs";
|
||||
const port = Number.parseInt(process.env.PORT || process.env.HWLAB_PORT || "8080", 10);
|
||||
const serviceId = process.env.HWLAB_SERVICE_ID || "hwlab-cloud-web";
|
||||
const environment = process.env.HWLAB_ENVIRONMENT || "v02";
|
||||
const processStartedAt = new Date();
|
||||
const roots = [
|
||||
`${process.cwd()}/web/hwlab-cloud-web/dist`,
|
||||
`${process.cwd()}/web/hwlab-cloud-web`
|
||||
@@ -89,6 +90,11 @@ function healthPayload() {
|
||||
build: {
|
||||
source: process.env.HWLAB_BUILD_SOURCE || "runtime-boot:web-build",
|
||||
metadataSource: "runtime-boot"
|
||||
},
|
||||
process: {
|
||||
pid: process.pid,
|
||||
startedAt: processStartedAt.toISOString(),
|
||||
uptimeSeconds: Math.round((Date.now() - processStartedAt.getTime()) / 1000)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -18,8 +18,16 @@ const GZIP_MIN_BYTES = 1024;
|
||||
const DEFAULT_AUTH_USERNAME = "admin";
|
||||
const DEFAULT_AUTH_PASSWORD = "hwlab2026";
|
||||
const DEFAULT_WORKBENCH_PROJECT_ID = "prj_hwpod_workbench";
|
||||
const CLIENT_DISCONNECT_ERROR_CODES = new Set([
|
||||
"ECONNRESET",
|
||||
"EPIPE",
|
||||
"ERR_STREAM_PREMATURE_CLOSE"
|
||||
]);
|
||||
const CLIENT_DISCONNECT_ERROR_PATTERN = /\b(?:socket hang up|client disconnected|premature close)\b/iu;
|
||||
const CLOUD_WEB_ERROR_HANDLER_INSTALLED = Symbol.for("hwlab.cloud-web.error-handler-installed");
|
||||
|
||||
export async function serveCloudWeb(options) {
|
||||
installCloudWebProcessErrorHandlers({ serviceId: options.serviceId });
|
||||
const server = createCloudWebServer(options);
|
||||
server.listen(options.port, "0.0.0.0", () => {
|
||||
process.stdout.write(JSON.stringify({
|
||||
@@ -43,50 +51,130 @@ export function createCloudWebServer({
|
||||
})
|
||||
}) {
|
||||
return createServer(async (request, response) => {
|
||||
attachRequestErrorHandlers({ request, response, serviceId });
|
||||
const url = new URL(request.url || "/", "http://hwlab-cloud-web.local");
|
||||
if (url.pathname === "/health/live" || url.pathname === "/health") {
|
||||
sendJson(response, 200, healthPayload());
|
||||
return;
|
||||
}
|
||||
if (await handleCloudWebAuth({ request, response, url, cloudApiBaseUrl, cloudApiProxyTimeoutMs, serviceId, sendJson })) {
|
||||
return;
|
||||
}
|
||||
|
||||
const proxyPolicy = cloudWebProxyRoutePolicy(request.method, url.pathname);
|
||||
if (proxyPolicy.proxy) {
|
||||
await proxyCloudApi({ request, response, url, cloudApiBaseUrl, cloudApiProxyTimeoutMs, serviceId, sendJson });
|
||||
return;
|
||||
}
|
||||
|
||||
const routePath = url.pathname.replace(/\/+$/u, "") || "/";
|
||||
const relativePath = routePath === "/" || routePath === "/gate" || routePath === "/diagnostics/gate" || routePath === "/help" || routePath === "/skills" || routePath === "/access"
|
||||
? "index.html"
|
||||
: url.pathname.slice(1);
|
||||
for (const root of roots) {
|
||||
const candidate = path.resolve(root, relativePath);
|
||||
if (!candidate.startsWith(root)) continue;
|
||||
try {
|
||||
const info = await stat(candidate);
|
||||
if (!info.isFile()) continue;
|
||||
const body = await readFile(candidate);
|
||||
const type = contentType(candidate);
|
||||
const encoded = await encodeStaticBody(request, body, type);
|
||||
response.writeHead(200, {
|
||||
"content-type": type,
|
||||
...encoded.headers,
|
||||
"content-length": encoded.body.length
|
||||
});
|
||||
response.end(encoded.body);
|
||||
try {
|
||||
if (url.pathname === "/health/live" || url.pathname === "/health") {
|
||||
sendJson(response, 200, healthPayload());
|
||||
return;
|
||||
}
|
||||
if (await handleCloudWebAuth({ request, response, url, cloudApiBaseUrl, cloudApiProxyTimeoutMs, serviceId, sendJson })) {
|
||||
return;
|
||||
} catch {
|
||||
// Try the next root.
|
||||
}
|
||||
}
|
||||
|
||||
sendJson(response, 404, { error: "not_found", serviceId, path: url.pathname });
|
||||
const proxyPolicy = cloudWebProxyRoutePolicy(request.method, url.pathname);
|
||||
if (proxyPolicy.proxy) {
|
||||
await proxyCloudApi({ request, response, url, cloudApiBaseUrl, cloudApiProxyTimeoutMs, serviceId, sendJson });
|
||||
return;
|
||||
}
|
||||
|
||||
const routePath = url.pathname.replace(/\/+$/u, "") || "/";
|
||||
const relativePath = routePath === "/" || routePath === "/gate" || routePath === "/diagnostics/gate" || routePath === "/help" || routePath === "/skills" || routePath === "/access"
|
||||
? "index.html"
|
||||
: url.pathname.slice(1);
|
||||
for (const root of roots) {
|
||||
const candidate = path.resolve(root, relativePath);
|
||||
if (!candidate.startsWith(root)) continue;
|
||||
try {
|
||||
const info = await stat(candidate);
|
||||
if (!info.isFile()) continue;
|
||||
const body = await readFile(candidate);
|
||||
const type = contentType(candidate);
|
||||
const encoded = await encodeStaticBody(request, body, type);
|
||||
response.writeHead(200, {
|
||||
"content-type": type,
|
||||
...encoded.headers,
|
||||
"content-length": encoded.body.length
|
||||
});
|
||||
response.end(encoded.body);
|
||||
return;
|
||||
} catch (error) {
|
||||
if (isClientDisconnectError(error)) {
|
||||
logCloudWebRuntimeEvent("cloud-web-client-disconnect", { serviceId, request, error });
|
||||
return;
|
||||
}
|
||||
// Try the next root for ordinary filesystem misses.
|
||||
}
|
||||
}
|
||||
|
||||
sendJson(response, 404, { error: "not_found", serviceId, path: url.pathname });
|
||||
} catch (error) {
|
||||
if (isClientDisconnectError(error)) {
|
||||
logCloudWebRuntimeEvent("cloud-web-client-disconnect", { serviceId, request, error });
|
||||
return;
|
||||
}
|
||||
if (!response.headersSent && !response.writableEnded) {
|
||||
sendJson(response, 500, {
|
||||
error: "internal_error",
|
||||
serviceId,
|
||||
message: error instanceof Error ? error.message : String(error)
|
||||
});
|
||||
return;
|
||||
}
|
||||
response.destroy(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export function installCloudWebProcessErrorHandlers({ serviceId } = {}) {
|
||||
if (process[CLOUD_WEB_ERROR_HANDLER_INSTALLED]) return;
|
||||
process[CLOUD_WEB_ERROR_HANDLER_INSTALLED] = true;
|
||||
process.on("uncaughtException", (error) => {
|
||||
if (isClientDisconnectError(error)) {
|
||||
logCloudWebRuntimeEvent("cloud-web-client-disconnect", { serviceId, error, source: "uncaughtException" });
|
||||
return;
|
||||
}
|
||||
logCloudWebRuntimeEvent("cloud-web-runtime-fatal", { serviceId, error, source: "uncaughtException" });
|
||||
process.exit(1);
|
||||
});
|
||||
process.on("unhandledRejection", (reason) => {
|
||||
const error = reason instanceof Error ? reason : new Error(String(reason));
|
||||
if (isClientDisconnectError(error)) {
|
||||
logCloudWebRuntimeEvent("cloud-web-client-disconnect", { serviceId, error, source: "unhandledRejection" });
|
||||
return;
|
||||
}
|
||||
logCloudWebRuntimeEvent("cloud-web-runtime-fatal", { serviceId, error, source: "unhandledRejection" });
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
function attachRequestErrorHandlers({ request, response, serviceId }) {
|
||||
const handle = (error) => {
|
||||
if (isClientDisconnectError(error)) {
|
||||
logCloudWebRuntimeEvent("cloud-web-client-disconnect", { serviceId, request, error });
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
};
|
||||
request.on("error", handle);
|
||||
response.on("error", handle);
|
||||
}
|
||||
|
||||
export function isClientDisconnectError(error) {
|
||||
const candidate = error instanceof Error ? error : new Error(String(error));
|
||||
const code = typeof candidate.code === "string" ? candidate.code : "";
|
||||
return CLIENT_DISCONNECT_ERROR_CODES.has(code) || CLIENT_DISCONNECT_ERROR_PATTERN.test(candidate.message || "");
|
||||
}
|
||||
|
||||
function logCloudWebRuntimeEvent(event, { serviceId, request = null, error = null, source = null } = {}) {
|
||||
const payload = {
|
||||
event,
|
||||
serviceId: serviceId || "hwlab-cloud-web",
|
||||
source,
|
||||
method: request?.method || null,
|
||||
path: request?.url || null,
|
||||
error: error
|
||||
? {
|
||||
name: error.name || "Error",
|
||||
code: typeof error.code === "string" ? error.code : null,
|
||||
message: error.message || String(error)
|
||||
}
|
||||
: null,
|
||||
observedAt: new Date().toISOString()
|
||||
};
|
||||
process.stderr.write(JSON.stringify(payload) + "\n");
|
||||
}
|
||||
|
||||
export async function handleCloudWebAuth(options) {
|
||||
const { request, response, url } = options;
|
||||
if (url.pathname === "/auth/workspace-bootstrap" && request.method === "GET") {
|
||||
|
||||
@@ -1,11 +1,19 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { createServer } from "node:http";
|
||||
import { connect } from "node:net";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
|
||||
import { createCloudWebServer } from "./cloud-web-runtime.mjs";
|
||||
import { createCloudWebServer, isClientDisconnectError } from "./cloud-web-runtime.mjs";
|
||||
|
||||
test("cloud web classifies Bun client disconnect ECONNRESET as non-fatal", () => {
|
||||
const error = new Error("aborted");
|
||||
error.code = "ECONNRESET";
|
||||
assert.equal(isClientDisconnectError(error), true);
|
||||
assert.equal(isClientDisconnectError(new Error("database migration failed")), false);
|
||||
});
|
||||
|
||||
test("cloud web auth requests are proxied once", async () => {
|
||||
let upstreamRequests = 0;
|
||||
@@ -263,6 +271,36 @@ test("cloud web forwards hwpod node-ops plans to cloud-api", async () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("cloud web remains healthy after a client disconnects during static response", async () => {
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-cloud-web-runtime-"));
|
||||
await writeFile(path.join(root, "large.txt"), "x".repeat(512 * 1024), "utf8");
|
||||
const cloudWeb = createCloudWebServer({
|
||||
serviceId: "hwlab-cloud-web",
|
||||
roots: [root],
|
||||
cloudApiBaseUrl: "http://127.0.0.1:1",
|
||||
healthPayload: () => ({ status: "ok", process: { uptimeSeconds: 1 } }),
|
||||
sendJson(response, statusCode, body) {
|
||||
const payload = JSON.stringify(body);
|
||||
response.writeHead(statusCode, {
|
||||
"content-type": "application/json",
|
||||
"content-length": Buffer.byteLength(payload)
|
||||
});
|
||||
response.end(payload);
|
||||
}
|
||||
});
|
||||
await listen(cloudWeb);
|
||||
|
||||
try {
|
||||
await disconnectDuringResponse(serverUrl(cloudWeb), "/large.txt");
|
||||
const response = await fetch(`${serverUrl(cloudWeb)}/health/live`);
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal((await response.json()).status, "ok");
|
||||
} finally {
|
||||
await close(cloudWeb);
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("cloud web serves access deep link through the React shell", async () => {
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-cloud-web-runtime-"));
|
||||
await writeFile(path.join(root, "index.html"), "<div id=\"root\"></div>\n", "utf8");
|
||||
@@ -306,6 +344,29 @@ function close(server) {
|
||||
});
|
||||
}
|
||||
|
||||
function disconnectDuringResponse(baseUrl, pathname) {
|
||||
const url = new URL(pathname, baseUrl);
|
||||
return new Promise((resolve, reject) => {
|
||||
const socket = connect(Number(url.port), url.hostname);
|
||||
const timeout = setTimeout(() => {
|
||||
socket.destroy();
|
||||
reject(new Error("client disconnect probe timed out"));
|
||||
}, 2000);
|
||||
socket.once("connect", () => {
|
||||
socket.write(`GET ${url.pathname} HTTP/1.1\r\nHost: ${url.host}\r\nConnection: close\r\n\r\n`);
|
||||
});
|
||||
socket.once("data", () => {
|
||||
clearTimeout(timeout);
|
||||
socket.destroy();
|
||||
setTimeout(resolve, 50);
|
||||
});
|
||||
socket.once("error", (error) => {
|
||||
clearTimeout(timeout);
|
||||
reject(error);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function serverUrl(server) {
|
||||
const address = server.address();
|
||||
return `http://127.0.0.1:${address.port}`;
|
||||
|
||||
@@ -228,16 +228,18 @@ function upsertAuthoritativeFinalResponseRow(rows: TraceEventRow[], assistantRow
|
||||
}
|
||||
if (matchedState) {
|
||||
const row = rows[matchedState.rowIndex];
|
||||
rows[matchedState.rowIndex] = {
|
||||
...row,
|
||||
body: finalText,
|
||||
terminal: true,
|
||||
tone: "ok",
|
||||
header: `${traceClock(completionEvent.createdAt)} 助手最终消息`
|
||||
};
|
||||
matchedState.comparableText = comparableText;
|
||||
matchedState.derivedSnapshotSuffix = false;
|
||||
return matchedState.rowIndex;
|
||||
if (row) {
|
||||
rows[matchedState.rowIndex] = {
|
||||
...row,
|
||||
body: finalText,
|
||||
terminal: true,
|
||||
tone: "ok",
|
||||
header: `${traceClock(completionEvent.createdAt)} 助手最终消息`
|
||||
};
|
||||
matchedState.comparableText = comparableText;
|
||||
matchedState.derivedSnapshotSuffix = false;
|
||||
return matchedState.rowIndex;
|
||||
}
|
||||
}
|
||||
rows.push({
|
||||
rowId: `trace-final-response:${completionEvent.seq ?? "completed"}`,
|
||||
|
||||
Reference in New Issue
Block a user