fix: bound CaseRun native smoke operations

scripts/harnessrl-native-l1-smoke.mjs

@@ -17,7 +17,10 @@ const stateDir = path.join(experimentRoot, "state");
 const databaseName = `hwlab_r45_${process.pid}_${Date.now() % 1000000}`;
 const databaseUrl = `postgresql://root@/${databaseName}?host=/var/run/postgresql`;
 const taskQueue = `hwlab-r45-${tag}`;
-const nativeIterationCount = Number.parseInt(process.env.HARNESSRL_NATIVE_ITERATION_COUNT || "1", 10);
+const defaultHttpTimeoutMs = 5000;
+const defaultProcessTimeoutMs = 30000;
+const processTerminationGraceMs = 2000;
+const pollTimeoutMs = 30000;
 const children = new Map();
 const runIds = {};
 let databaseCreated = false;
@@ -71,7 +74,7 @@ try {
   await submitHttp(baseUrl, runIds.workerRestart);
   runIds.cancel = `r45-cancel-${tag}`;
   await submitHttp(baseUrl, runIds.cancel);
-  await requestJson(`${baseUrl}/v1/caserun/runs/${encodeURIComponent(runIds.cancel)}/cancel`, { method: "POST" }, 202);
+  await requestJson("cancel-run", `${baseUrl}/v1/caserun/runs/${encodeURIComponent(runIds.cancel)}/cancel`, { method: "POST" }, 202);
   await startWorker(runtimeEnv, workerPort);
   const workerRestart = await pollHttp(baseUrl, runIds.workerRestart, "completed");
   stage("worker-restart-recovery", identity(workerRestart));
@@ -122,16 +125,16 @@ try {
   await startChild("native-fixture", ["bun", "scripts/caserun-native-supervisor.ts"], {
     cwd: path.join(root, "web/hwlab-cloud-web"), env: webEnv
   });
-  await waitForHttp(`${nativeFixtureUrl}/health`);
+  await waitForHttp("wait-native-fixture", `${nativeFixtureUrl}/health`);
   await run("native-fixture-smoke", "bun", ["scripts/caserun-native-smoke.ts"], {
     cwd: path.join(root, "web/hwlab-cloud-web"), env: webEnv
   });
   await startChild("vite", ["bunx", "vite", "--host", "127.0.0.1", "--port", String(vitePort), "--strictPort"], {
     cwd: path.join(root, "web/hwlab-cloud-web"), env: webEnv
   });
-  const viteHtml = await waitForHttp(`${viteUrl}/workbench`, { returnText: true });
+  const viteHtml = await waitForHttp("wait-vite-workbench", `${viteUrl}/workbench`, { returnText: true });
   requireCondition(viteHtml.includes("/@vite/client"), "Vite HMR client was not injected");
-  const proxiedCases = await requestJson(`${viteUrl}/v1/caserun/cases`);
+  const proxiedCases = await requestJson("vite-caserun-cases", `${viteUrl}/v1/caserun/cases`);
   requireCondition(proxiedCases.mode === "native-test" && Array.isArray(proxiedCases.cases), "Vite native CaseRun proxy failed");
   await run("native-reload-smoke", "bun", ["scripts/caserun-native-reload-smoke.ts"], {
     cwd: path.join(root, "web/hwlab-cloud-web"), env: webEnv
@@ -146,10 +149,7 @@ try {
     runs: runIds,
     registry: databaseCounts,
     web: { nativeFixtureUrl, viteUrl, hmrClient: true },
-    nativeIterationCount: Number.isSafeInteger(nativeIterationCount) && nativeIterationCount > 0 ? nativeIterationCount : 1,
+    nativeElapsedSeconds: Math.ceil((Date.now() - startedAt) / 1000)
-    nativeElapsedSeconds: Math.ceil((Date.now() - startedAt) / 1000),
-    deliveryRunCount: 0,
-    rolloutCount: 0
   };
 } catch (error) {
   const logs = await diagnosticLogs();
@@ -201,13 +201,13 @@ async function availablePort() {
 
 async function startApi(env, baseUrl) {
   await startChild("api", ["bun", "cmd/hwlab-harnessrl-api/main.ts"], { cwd: root, env });
-  await waitForHttp(`${baseUrl}/readyz`);
+  await waitForHttp("wait-api-ready", `${baseUrl}/readyz`);
   stage("api-started", { pid: children.get("api").pid });
 }
 
 async function startWorker(env, port) {
   await startChild("worker", ["bun", "cmd/hwlab-harnessrl-worker/main.ts"], { cwd: root, env });
-  await waitForHttp(`http://127.0.0.1:${port}/readyz`, { timeoutMs: 20000 });
+  await waitForHttp("wait-worker-ready", `http://127.0.0.1:${port}/readyz`, { timeoutMs: 20000 });
   stage("worker-started", { pid: children.get("worker").pid });
 }
 
@@ -241,15 +241,34 @@ async function stopChild(name) {
 }
 
 async function run(label, command, args, options = {}) {
+  const timeoutMs = positiveTimeout(options.timeoutMs, defaultProcessTimeoutMs, "process timeout");
   const completed = await new Promise((resolve, reject) => {
     const child = spawn(command, args, { cwd: options.cwd || root, env: options.env || process.env, stdio: ["ignore", "pipe", "pipe"] });
     let stdout = "";
     let stderr = "";
+    let timedOut = false;
+    let terminationTimer;
+    const timeoutTimer = setTimeout(() => {
+      timedOut = true;
+      child.kill("SIGTERM");
+      terminationTimer = setTimeout(() => {
+        if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
+      }, processTerminationGraceMs);
+    }, timeoutMs);
     child.stdout.on("data", (chunk) => { stdout += chunk; });
     child.stderr.on("data", (chunk) => { stderr += chunk; });
-    child.once("error", reject);
+    child.once("error", (error) => {
-    child.once("exit", (exitCode, signal) => resolve({ exitCode: exitCode ?? 1, signal, stdout, stderr }));
+      clearTimeout(timeoutTimer);
+      if (terminationTimer) clearTimeout(terminationTimer);
+      reject(error);
+    });
+    child.once("exit", (exitCode, signal) => {
+      clearTimeout(timeoutTimer);
+      if (terminationTimer) clearTimeout(terminationTimer);
+      resolve({ exitCode: exitCode ?? 1, signal, stdout, stderr, timedOut, timeoutMs });
+    });
   });
+  if (completed.timedOut) throw new Error(`process operation timed out operation=${clip(label, 160)} timeoutMs=${completed.timeoutMs}`);
   if (completed.exitCode !== 0 && !options.allowFailure) {
     throw new Error(`${label} failed exitCode=${completed.exitCode}: ${clip(completed.stderr || completed.stdout)}`);
   }
@@ -257,13 +276,15 @@ async function run(label, command, args, options = {}) {
 }
 
 async function cli(env, args, options = {}) {
-  const completed = await run(`hwlab-cli ${args.join(" ")}`, "bun", ["tools/hwlab-cli/bin/hwlab-cli.ts", ...args], { env, allowFailure: options.allowFailure });
+  const completed = await run(`hwlab-cli ${args.join(" ")}`, "bun", ["tools/hwlab-cli/bin/hwlab-cli.ts", ...args], {
+    env, allowFailure: options.allowFailure, timeoutMs: options.timeoutMs
+  });
   const payload = parseJson(completed.stdout, "hwlab-cli output");
   return { ...payload, exitCode: completed.exitCode };
 }
 
 async function submitHttp(baseUrl, runId) {
-  const response = await requestJson(`${baseUrl}/v1/caserun/runs`, {
+  const response = await requestJson("submit-run", `${baseUrl}/v1/caserun/runs`, {
     method: "POST",
     headers: { "content-type": "application/json" },
     body: JSON.stringify({ caseId: "software-smoke", runId })
@@ -273,9 +294,12 @@ async function submitHttp(baseUrl, runId) {
 }
 
 async function pollHttp(baseUrl, runId, expected) {
+  const deadline = Date.now() + pollTimeoutMs;
   let previousStatus;
-  for (let attempt = 1; attempt <= 100; attempt += 1) {
+  for (let attempt = 1; Date.now() < deadline; attempt += 1) {
-    const record = await requestJson(`${baseUrl}/v1/caserun/runs/${encodeURIComponent(runId)}`);
+    const record = await requestJson("poll-run", `${baseUrl}/v1/caserun/runs/${encodeURIComponent(runId)}`, undefined, 200, {
+      timeoutMs: Math.min(defaultHttpTimeoutMs, Math.max(1, deadline - Date.now()))
+    });
     if (record.status !== previousStatus || record.status === expected) stage("poll", { transport: "api", runId, attempt, status: record.status });
     previousStatus = record.status;
     if (record.status === expected) return record;
@@ -287,9 +311,12 @@ async function pollHttp(baseUrl, runId, expected) {
 
 async function pollCli(env, mode, runId, expected, baseUrl) {
   const overApi = mode === "api" ? ["--over-api", "--base-url", baseUrl] : [];
+  const deadline = Date.now() + pollTimeoutMs;
   let previousStatus;
-  for (let attempt = 1; attempt <= 100; attempt += 1) {
+  for (let attempt = 1; Date.now() < deadline; attempt += 1) {
-    const record = await cli(env, ["case", "run", "status", runId, ...overApi]);
+    const record = await cli(env, ["case", "run", "status", runId, ...overApi], {
+      timeoutMs: Math.min(defaultProcessTimeoutMs, Math.max(1, deadline - Date.now()))
+    });
     if (record.status !== previousStatus || record.status === expected) stage("poll", { transport: mode, runId, attempt, status: record.status });
     previousStatus = record.status;
     if (record.status === expected) return record;
@@ -299,30 +326,56 @@ async function pollCli(env, mode, runId, expected, baseUrl) {
   throw new Error(`timeout waiting for ${mode}/${runId}=${expected}`);
 }
 
-async function waitForHttp(url, options = {}) {
+async function waitForHttp(operation, url, options = {}) {
-  const deadline = Date.now() + (options.timeoutMs || 10000);
+  const timeoutMs = positiveTimeout(options.timeoutMs, 10000, "HTTP wait timeout");
+  const deadline = Date.now() + timeoutMs;
   let lastError;
   while (Date.now() < deadline) {
     try {
-      const response = await fetch(url);
+      const requestTimeoutMs = Math.min(defaultHttpTimeoutMs, Math.max(1, deadline - Date.now()));
-      if (response.ok) return options.returnText ? await response.text() : response;
+      const { response, text } = await requestText(operation, url, undefined, { timeoutMs: requestTimeoutMs });
-      lastError = new Error(`${url} status=${response.status}`);
+      if (response.ok) return options.returnText ? text : response;
+      lastError = new Error(`HTTP status operation=${boundedOperation(operation)} url=${boundedUrl(url)} status=${response.status}`);
     } catch (error) {
       lastError = error;
     }
     await sleep(100);
   }
-  throw new Error(`timeout waiting for ${url}: ${lastError instanceof Error ? lastError.message : String(lastError)}`);
+  throw new Error(`HTTP wait timed out operation=${boundedOperation(operation)} url=${boundedUrl(url)} timeoutMs=${timeoutMs} lastError=${clip(lastError instanceof Error ? lastError.message : String(lastError), 400)}`);
 }
 
-async function requestJson(url, init, expectedStatus = 200) {
+async function requestJson(operation, url, init, expectedStatus = 200, options = {}) {
-  const response = await fetch(url, init);
+  const { response, text, timeoutMs } = await requestText(operation, url, init, options);
-  const text = await response.text();
+  let payload;
-  const payload = parseJson(text, url);
+  try {
-  if (response.status !== expectedStatus) throw new Error(`${url} status=${response.status}: ${clip(text)}`);
+    payload = JSON.parse(text);
+  } catch {
+    throw new Error(`HTTP invalid JSON operation=${boundedOperation(operation)} url=${boundedUrl(url)} timeoutMs=${timeoutMs} body=${clip(text)}`);
+  }
+  if (response.status !== expectedStatus) {
+    throw new Error(`HTTP status operation=${boundedOperation(operation)} url=${boundedUrl(url)} timeoutMs=${timeoutMs} status=${response.status} body=${clip(text)}`);
+  }
   return payload;
 }
 
+async function requestText(operation, url, init, options = {}) {
+  const timeoutMs = positiveTimeout(options.timeoutMs, defaultHttpTimeoutMs, "HTTP request timeout");
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const response = await fetch(url, { ...init, signal: controller.signal });
+    const text = await response.text();
+    return { response, text, timeoutMs };
+  } catch (error) {
+    if (controller.signal.aborted) {
+      throw new Error(`HTTP request timed out operation=${boundedOperation(operation)} url=${boundedUrl(url)} timeoutMs=${timeoutMs}`);
+    }
+    throw new Error(`HTTP request failed operation=${boundedOperation(operation)} url=${boundedUrl(url)} timeoutMs=${timeoutMs} error=${clip(error instanceof Error ? error.message : String(error), 300)}`);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
 async function queryDatabase(url, sql) {
   const completed = await run("database-query", "psql", [url, "-At", "-F", "|", "-c", sql]);
   const [runs, events] = completed.stdout.trim().split("|").map(Number);
@@ -378,6 +431,29 @@ function clip(value, limit = 1200) {
   return text.length <= limit ? text : `${text.slice(0, limit)}…`;
 }
 
+function positiveTimeout(value, fallback, label) {
+  const timeoutMs = value === undefined ? fallback : Number(value);
+  if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) throw new Error(`${label} must be a positive integer`);
+  return timeoutMs;
+}
+
+function boundedOperation(value) {
+  return clip(String(value || "unknown-operation").replace(/[\r\n\t]/gu, " "), 120);
+}
+
+function boundedUrl(value) {
+  try {
+    const url = new URL(String(value));
+    url.username = "";
+    url.password = "";
+    url.search = "";
+    url.hash = "";
+    return clip(url.toString(), 300);
+  } catch {
+    return clip(String(value || "invalid-url").split(/[?#]/u, 1)[0], 300);
+  }
+}
+
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }