From 43497efdbf0483497798a2429bf45de17438676a Mon Sep 17 00:00:00 2001 From: root Date: Fri, 17 Jul 2026 08:05:23 +0200 Subject: [PATCH] fix: bound CaseRun native smoke operations --- scripts/harnessrl-native-l1-smoke.mjs | 136 ++++++++++++++++++++------ 1 file changed, 106 insertions(+), 30 deletions(-) diff --git a/scripts/harnessrl-native-l1-smoke.mjs b/scripts/harnessrl-native-l1-smoke.mjs index 86dce799..d66c9f15 100644 --- a/scripts/harnessrl-native-l1-smoke.mjs +++ b/scripts/harnessrl-native-l1-smoke.mjs @@ -17,7 +17,10 @@ const stateDir = path.join(experimentRoot, "state"); const databaseName = `hwlab_r45_${process.pid}_${Date.now() % 1000000}`; const databaseUrl = `postgresql://root@/${databaseName}?host=/var/run/postgresql`; const taskQueue = `hwlab-r45-${tag}`; -const nativeIterationCount = Number.parseInt(process.env.HARNESSRL_NATIVE_ITERATION_COUNT || "1", 10); +const defaultHttpTimeoutMs = 5000; +const defaultProcessTimeoutMs = 30000; +const processTerminationGraceMs = 2000; +const pollTimeoutMs = 30000; const children = new Map(); const runIds = {}; let databaseCreated = false; @@ -71,7 +74,7 @@ try { await submitHttp(baseUrl, runIds.workerRestart); runIds.cancel = `r45-cancel-${tag}`; await submitHttp(baseUrl, runIds.cancel); - await requestJson(`${baseUrl}/v1/caserun/runs/${encodeURIComponent(runIds.cancel)}/cancel`, { method: "POST" }, 202); + await requestJson("cancel-run", `${baseUrl}/v1/caserun/runs/${encodeURIComponent(runIds.cancel)}/cancel`, { method: "POST" }, 202); await startWorker(runtimeEnv, workerPort); const workerRestart = await pollHttp(baseUrl, runIds.workerRestart, "completed"); stage("worker-restart-recovery", identity(workerRestart)); @@ -122,16 +125,16 @@ try { await startChild("native-fixture", ["bun", "scripts/caserun-native-supervisor.ts"], { cwd: path.join(root, "web/hwlab-cloud-web"), env: webEnv }); - await waitForHttp(`${nativeFixtureUrl}/health`); + await waitForHttp("wait-native-fixture", `${nativeFixtureUrl}/health`); await run("native-fixture-smoke", "bun", ["scripts/caserun-native-smoke.ts"], { cwd: path.join(root, "web/hwlab-cloud-web"), env: webEnv }); await startChild("vite", ["bunx", "vite", "--host", "127.0.0.1", "--port", String(vitePort), "--strictPort"], { cwd: path.join(root, "web/hwlab-cloud-web"), env: webEnv }); - const viteHtml = await waitForHttp(`${viteUrl}/workbench`, { returnText: true }); + const viteHtml = await waitForHttp("wait-vite-workbench", `${viteUrl}/workbench`, { returnText: true }); requireCondition(viteHtml.includes("/@vite/client"), "Vite HMR client was not injected"); - const proxiedCases = await requestJson(`${viteUrl}/v1/caserun/cases`); + const proxiedCases = await requestJson("vite-caserun-cases", `${viteUrl}/v1/caserun/cases`); requireCondition(proxiedCases.mode === "native-test" && Array.isArray(proxiedCases.cases), "Vite native CaseRun proxy failed"); await run("native-reload-smoke", "bun", ["scripts/caserun-native-reload-smoke.ts"], { cwd: path.join(root, "web/hwlab-cloud-web"), env: webEnv @@ -146,10 +149,7 @@ try { runs: runIds, registry: databaseCounts, web: { nativeFixtureUrl, viteUrl, hmrClient: true }, - nativeIterationCount: Number.isSafeInteger(nativeIterationCount) && nativeIterationCount > 0 ? nativeIterationCount : 1, - nativeElapsedSeconds: Math.ceil((Date.now() - startedAt) / 1000), - deliveryRunCount: 0, - rolloutCount: 0 + nativeElapsedSeconds: Math.ceil((Date.now() - startedAt) / 1000) }; } catch (error) { const logs = await diagnosticLogs(); @@ -201,13 +201,13 @@ async function availablePort() { async function startApi(env, baseUrl) { await startChild("api", ["bun", "cmd/hwlab-harnessrl-api/main.ts"], { cwd: root, env }); - await waitForHttp(`${baseUrl}/readyz`); + await waitForHttp("wait-api-ready", `${baseUrl}/readyz`); stage("api-started", { pid: children.get("api").pid }); } async function startWorker(env, port) { await startChild("worker", ["bun", "cmd/hwlab-harnessrl-worker/main.ts"], { cwd: root, env }); - await waitForHttp(`http://127.0.0.1:${port}/readyz`, { timeoutMs: 20000 }); + await waitForHttp("wait-worker-ready", `http://127.0.0.1:${port}/readyz`, { timeoutMs: 20000 }); stage("worker-started", { pid: children.get("worker").pid }); } @@ -241,15 +241,34 @@ async function stopChild(name) { } async function run(label, command, args, options = {}) { + const timeoutMs = positiveTimeout(options.timeoutMs, defaultProcessTimeoutMs, "process timeout"); const completed = await new Promise((resolve, reject) => { const child = spawn(command, args, { cwd: options.cwd || root, env: options.env || process.env, stdio: ["ignore", "pipe", "pipe"] }); let stdout = ""; let stderr = ""; + let timedOut = false; + let terminationTimer; + const timeoutTimer = setTimeout(() => { + timedOut = true; + child.kill("SIGTERM"); + terminationTimer = setTimeout(() => { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + }, processTerminationGraceMs); + }, timeoutMs); child.stdout.on("data", (chunk) => { stdout += chunk; }); child.stderr.on("data", (chunk) => { stderr += chunk; }); - child.once("error", reject); - child.once("exit", (exitCode, signal) => resolve({ exitCode: exitCode ?? 1, signal, stdout, stderr })); + child.once("error", (error) => { + clearTimeout(timeoutTimer); + if (terminationTimer) clearTimeout(terminationTimer); + reject(error); + }); + child.once("exit", (exitCode, signal) => { + clearTimeout(timeoutTimer); + if (terminationTimer) clearTimeout(terminationTimer); + resolve({ exitCode: exitCode ?? 1, signal, stdout, stderr, timedOut, timeoutMs }); + }); }); + if (completed.timedOut) throw new Error(`process operation timed out operation=${clip(label, 160)} timeoutMs=${completed.timeoutMs}`); if (completed.exitCode !== 0 && !options.allowFailure) { throw new Error(`${label} failed exitCode=${completed.exitCode}: ${clip(completed.stderr || completed.stdout)}`); } @@ -257,13 +276,15 @@ async function run(label, command, args, options = {}) { } async function cli(env, args, options = {}) { - const completed = await run(`hwlab-cli ${args.join(" ")}`, "bun", ["tools/hwlab-cli/bin/hwlab-cli.ts", ...args], { env, allowFailure: options.allowFailure }); + const completed = await run(`hwlab-cli ${args.join(" ")}`, "bun", ["tools/hwlab-cli/bin/hwlab-cli.ts", ...args], { + env, allowFailure: options.allowFailure, timeoutMs: options.timeoutMs + }); const payload = parseJson(completed.stdout, "hwlab-cli output"); return { ...payload, exitCode: completed.exitCode }; } async function submitHttp(baseUrl, runId) { - const response = await requestJson(`${baseUrl}/v1/caserun/runs`, { + const response = await requestJson("submit-run", `${baseUrl}/v1/caserun/runs`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ caseId: "software-smoke", runId }) @@ -273,9 +294,12 @@ async function submitHttp(baseUrl, runId) { } async function pollHttp(baseUrl, runId, expected) { + const deadline = Date.now() + pollTimeoutMs; let previousStatus; - for (let attempt = 1; attempt <= 100; attempt += 1) { - const record = await requestJson(`${baseUrl}/v1/caserun/runs/${encodeURIComponent(runId)}`); + for (let attempt = 1; Date.now() < deadline; attempt += 1) { + const record = await requestJson("poll-run", `${baseUrl}/v1/caserun/runs/${encodeURIComponent(runId)}`, undefined, 200, { + timeoutMs: Math.min(defaultHttpTimeoutMs, Math.max(1, deadline - Date.now())) + }); if (record.status !== previousStatus || record.status === expected) stage("poll", { transport: "api", runId, attempt, status: record.status }); previousStatus = record.status; if (record.status === expected) return record; @@ -287,9 +311,12 @@ async function pollHttp(baseUrl, runId, expected) { async function pollCli(env, mode, runId, expected, baseUrl) { const overApi = mode === "api" ? ["--over-api", "--base-url", baseUrl] : []; + const deadline = Date.now() + pollTimeoutMs; let previousStatus; - for (let attempt = 1; attempt <= 100; attempt += 1) { - const record = await cli(env, ["case", "run", "status", runId, ...overApi]); + for (let attempt = 1; Date.now() < deadline; attempt += 1) { + const record = await cli(env, ["case", "run", "status", runId, ...overApi], { + timeoutMs: Math.min(defaultProcessTimeoutMs, Math.max(1, deadline - Date.now())) + }); if (record.status !== previousStatus || record.status === expected) stage("poll", { transport: mode, runId, attempt, status: record.status }); previousStatus = record.status; if (record.status === expected) return record; @@ -299,30 +326,56 @@ async function pollCli(env, mode, runId, expected, baseUrl) { throw new Error(`timeout waiting for ${mode}/${runId}=${expected}`); } -async function waitForHttp(url, options = {}) { - const deadline = Date.now() + (options.timeoutMs || 10000); +async function waitForHttp(operation, url, options = {}) { + const timeoutMs = positiveTimeout(options.timeoutMs, 10000, "HTTP wait timeout"); + const deadline = Date.now() + timeoutMs; let lastError; while (Date.now() < deadline) { try { - const response = await fetch(url); - if (response.ok) return options.returnText ? await response.text() : response; - lastError = new Error(`${url} status=${response.status}`); + const requestTimeoutMs = Math.min(defaultHttpTimeoutMs, Math.max(1, deadline - Date.now())); + const { response, text } = await requestText(operation, url, undefined, { timeoutMs: requestTimeoutMs }); + if (response.ok) return options.returnText ? text : response; + lastError = new Error(`HTTP status operation=${boundedOperation(operation)} url=${boundedUrl(url)} status=${response.status}`); } catch (error) { lastError = error; } await sleep(100); } - throw new Error(`timeout waiting for ${url}: ${lastError instanceof Error ? lastError.message : String(lastError)}`); + throw new Error(`HTTP wait timed out operation=${boundedOperation(operation)} url=${boundedUrl(url)} timeoutMs=${timeoutMs} lastError=${clip(lastError instanceof Error ? lastError.message : String(lastError), 400)}`); } -async function requestJson(url, init, expectedStatus = 200) { - const response = await fetch(url, init); - const text = await response.text(); - const payload = parseJson(text, url); - if (response.status !== expectedStatus) throw new Error(`${url} status=${response.status}: ${clip(text)}`); +async function requestJson(operation, url, init, expectedStatus = 200, options = {}) { + const { response, text, timeoutMs } = await requestText(operation, url, init, options); + let payload; + try { + payload = JSON.parse(text); + } catch { + throw new Error(`HTTP invalid JSON operation=${boundedOperation(operation)} url=${boundedUrl(url)} timeoutMs=${timeoutMs} body=${clip(text)}`); + } + if (response.status !== expectedStatus) { + throw new Error(`HTTP status operation=${boundedOperation(operation)} url=${boundedUrl(url)} timeoutMs=${timeoutMs} status=${response.status} body=${clip(text)}`); + } return payload; } +async function requestText(operation, url, init, options = {}) { + const timeoutMs = positiveTimeout(options.timeoutMs, defaultHttpTimeoutMs, "HTTP request timeout"); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(url, { ...init, signal: controller.signal }); + const text = await response.text(); + return { response, text, timeoutMs }; + } catch (error) { + if (controller.signal.aborted) { + throw new Error(`HTTP request timed out operation=${boundedOperation(operation)} url=${boundedUrl(url)} timeoutMs=${timeoutMs}`); + } + throw new Error(`HTTP request failed operation=${boundedOperation(operation)} url=${boundedUrl(url)} timeoutMs=${timeoutMs} error=${clip(error instanceof Error ? error.message : String(error), 300)}`); + } finally { + clearTimeout(timer); + } +} + async function queryDatabase(url, sql) { const completed = await run("database-query", "psql", [url, "-At", "-F", "|", "-c", sql]); const [runs, events] = completed.stdout.trim().split("|").map(Number); @@ -378,6 +431,29 @@ function clip(value, limit = 1200) { return text.length <= limit ? text : `${text.slice(0, limit)}…`; } +function positiveTimeout(value, fallback, label) { + const timeoutMs = value === undefined ? fallback : Number(value); + if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) throw new Error(`${label} must be a positive integer`); + return timeoutMs; +} + +function boundedOperation(value) { + return clip(String(value || "unknown-operation").replace(/[\r\n\t]/gu, " "), 120); +} + +function boundedUrl(value) { + try { + const url = new URL(String(value)); + url.username = ""; + url.password = ""; + url.search = ""; + url.hash = ""; + return clip(url.toString(), 300); + } catch { + return clip(String(value || "invalid-url").split(/[?#]/u, 1)[0], 300); + } +} + function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); }