fix: bound CaseRun native smoke operations
This commit is contained in:
root
@@ -17,7 +17,10 @@ const stateDir = path.join(experimentRoot, "state");
|
||||
const databaseName = `hwlab_r45_${process.pid}_${Date.now() % 1000000}`;
|
||||
const databaseUrl = `postgresql://root@/${databaseName}?host=/var/run/postgresql`;
|
||||
const taskQueue = `hwlab-r45-${tag}`;
|
||||
const nativeIterationCount = Number.parseInt(process.env.HARNESSRL_NATIVE_ITERATION_COUNT || "1", 10);
|
||||
const defaultHttpTimeoutMs = 5000;
|
||||
const defaultProcessTimeoutMs = 30000;
|
||||
const processTerminationGraceMs = 2000;
|
||||
const pollTimeoutMs = 30000;
|
||||
const children = new Map();
|
||||
const runIds = {};
|
||||
let databaseCreated = false;
|
||||
@@ -71,7 +74,7 @@ try {
|
||||
await submitHttp(baseUrl, runIds.workerRestart);
|
||||
runIds.cancel = `r45-cancel-${tag}`;
|
||||
await submitHttp(baseUrl, runIds.cancel);
|
||||
await requestJson(`${baseUrl}/v1/caserun/runs/${encodeURIComponent(runIds.cancel)}/cancel`, { method: "POST" }, 202);
|
||||
await requestJson("cancel-run", `${baseUrl}/v1/caserun/runs/${encodeURIComponent(runIds.cancel)}/cancel`, { method: "POST" }, 202);
|
||||
await startWorker(runtimeEnv, workerPort);
|
||||
const workerRestart = await pollHttp(baseUrl, runIds.workerRestart, "completed");
|
||||
stage("worker-restart-recovery", identity(workerRestart));
|
||||
@@ -122,16 +125,16 @@ try {
|
||||
await startChild("native-fixture", ["bun", "scripts/caserun-native-supervisor.ts"], {
|
||||
cwd: path.join(root, "web/hwlab-cloud-web"), env: webEnv
|
||||
});
|
||||
await waitForHttp(`${nativeFixtureUrl}/health`);
|
||||
await waitForHttp("wait-native-fixture", `${nativeFixtureUrl}/health`);
|
||||
await run("native-fixture-smoke", "bun", ["scripts/caserun-native-smoke.ts"], {
|
||||
cwd: path.join(root, "web/hwlab-cloud-web"), env: webEnv
|
||||
});
|
||||
await startChild("vite", ["bunx", "vite", "--host", "127.0.0.1", "--port", String(vitePort), "--strictPort"], {
|
||||
cwd: path.join(root, "web/hwlab-cloud-web"), env: webEnv
|
||||
});
|
||||
const viteHtml = await waitForHttp(`${viteUrl}/workbench`, { returnText: true });
|
||||
const viteHtml = await waitForHttp("wait-vite-workbench", `${viteUrl}/workbench`, { returnText: true });
|
||||
requireCondition(viteHtml.includes("/@vite/client"), "Vite HMR client was not injected");
|
||||
const proxiedCases = await requestJson(`${viteUrl}/v1/caserun/cases`);
|
||||
const proxiedCases = await requestJson("vite-caserun-cases", `${viteUrl}/v1/caserun/cases`);
|
||||
requireCondition(proxiedCases.mode === "native-test" && Array.isArray(proxiedCases.cases), "Vite native CaseRun proxy failed");
|
||||
await run("native-reload-smoke", "bun", ["scripts/caserun-native-reload-smoke.ts"], {
|
||||
cwd: path.join(root, "web/hwlab-cloud-web"), env: webEnv
|
||||
@@ -146,10 +149,7 @@ try {
|
||||
runs: runIds,
|
||||
registry: databaseCounts,
|
||||
web: { nativeFixtureUrl, viteUrl, hmrClient: true },
|
||||
nativeIterationCount: Number.isSafeInteger(nativeIterationCount) && nativeIterationCount > 0 ? nativeIterationCount : 1,
|
||||
nativeElapsedSeconds: Math.ceil((Date.now() - startedAt) / 1000),
|
||||
deliveryRunCount: 0,
|
||||
rolloutCount: 0
|
||||
nativeElapsedSeconds: Math.ceil((Date.now() - startedAt) / 1000)
|
||||
};
|
||||
} catch (error) {
|
||||
const logs = await diagnosticLogs();
|
||||
@@ -201,13 +201,13 @@ async function availablePort() {
|
||||
|
||||
async function startApi(env, baseUrl) {
|
||||
await startChild("api", ["bun", "cmd/hwlab-harnessrl-api/main.ts"], { cwd: root, env });
|
||||
await waitForHttp(`${baseUrl}/readyz`);
|
||||
await waitForHttp("wait-api-ready", `${baseUrl}/readyz`);
|
||||
stage("api-started", { pid: children.get("api").pid });
|
||||
}
|
||||
|
||||
async function startWorker(env, port) {
|
||||
await startChild("worker", ["bun", "cmd/hwlab-harnessrl-worker/main.ts"], { cwd: root, env });
|
||||
await waitForHttp(`http://127.0.0.1:${port}/readyz`, { timeoutMs: 20000 });
|
||||
await waitForHttp("wait-worker-ready", `http://127.0.0.1:${port}/readyz`, { timeoutMs: 20000 });
|
||||
stage("worker-started", { pid: children.get("worker").pid });
|
||||
}
|
||||
|
||||
@@ -241,15 +241,34 @@ async function stopChild(name) {
|
||||
}
|
||||
|
||||
async function run(label, command, args, options = {}) {
|
||||
const timeoutMs = positiveTimeout(options.timeoutMs, defaultProcessTimeoutMs, "process timeout");
|
||||
const completed = await new Promise((resolve, reject) => {
|
||||
const child = spawn(command, args, { cwd: options.cwd || root, env: options.env || process.env, stdio: ["ignore", "pipe", "pipe"] });
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let timedOut = false;
|
||||
let terminationTimer;
|
||||
const timeoutTimer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
child.kill("SIGTERM");
|
||||
terminationTimer = setTimeout(() => {
|
||||
if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
|
||||
}, processTerminationGraceMs);
|
||||
}, timeoutMs);
|
||||
child.stdout.on("data", (chunk) => { stdout += chunk; });
|
||||
child.stderr.on("data", (chunk) => { stderr += chunk; });
|
||||
child.once("error", reject);
|
||||
child.once("exit", (exitCode, signal) => resolve({ exitCode: exitCode ?? 1, signal, stdout, stderr }));
|
||||
child.once("error", (error) => {
|
||||
clearTimeout(timeoutTimer);
|
||||
if (terminationTimer) clearTimeout(terminationTimer);
|
||||
reject(error);
|
||||
});
|
||||
child.once("exit", (exitCode, signal) => {
|
||||
clearTimeout(timeoutTimer);
|
||||
if (terminationTimer) clearTimeout(terminationTimer);
|
||||
resolve({ exitCode: exitCode ?? 1, signal, stdout, stderr, timedOut, timeoutMs });
|
||||
});
|
||||
});
|
||||
if (completed.timedOut) throw new Error(`process operation timed out operation=${clip(label, 160)} timeoutMs=${completed.timeoutMs}`);
|
||||
if (completed.exitCode !== 0 && !options.allowFailure) {
|
||||
throw new Error(`${label} failed exitCode=${completed.exitCode}: ${clip(completed.stderr || completed.stdout)}`);
|
||||
}
|
||||
@@ -257,13 +276,15 @@ async function run(label, command, args, options = {}) {
|
||||
}
|
||||
|
||||
async function cli(env, args, options = {}) {
|
||||
const completed = await run(`hwlab-cli ${args.join(" ")}`, "bun", ["tools/hwlab-cli/bin/hwlab-cli.ts", ...args], { env, allowFailure: options.allowFailure });
|
||||
const completed = await run(`hwlab-cli ${args.join(" ")}`, "bun", ["tools/hwlab-cli/bin/hwlab-cli.ts", ...args], {
|
||||
env, allowFailure: options.allowFailure, timeoutMs: options.timeoutMs
|
||||
});
|
||||
const payload = parseJson(completed.stdout, "hwlab-cli output");
|
||||
return { ...payload, exitCode: completed.exitCode };
|
||||
}
|
||||
|
||||
async function submitHttp(baseUrl, runId) {
|
||||
const response = await requestJson(`${baseUrl}/v1/caserun/runs`, {
|
||||
const response = await requestJson("submit-run", `${baseUrl}/v1/caserun/runs`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ caseId: "software-smoke", runId })
|
||||
@@ -273,9 +294,12 @@ async function submitHttp(baseUrl, runId) {
|
||||
}
|
||||
|
||||
async function pollHttp(baseUrl, runId, expected) {
|
||||
const deadline = Date.now() + pollTimeoutMs;
|
||||
let previousStatus;
|
||||
for (let attempt = 1; attempt <= 100; attempt += 1) {
|
||||
const record = await requestJson(`${baseUrl}/v1/caserun/runs/${encodeURIComponent(runId)}`);
|
||||
for (let attempt = 1; Date.now() < deadline; attempt += 1) {
|
||||
const record = await requestJson("poll-run", `${baseUrl}/v1/caserun/runs/${encodeURIComponent(runId)}`, undefined, 200, {
|
||||
timeoutMs: Math.min(defaultHttpTimeoutMs, Math.max(1, deadline - Date.now()))
|
||||
});
|
||||
if (record.status !== previousStatus || record.status === expected) stage("poll", { transport: "api", runId, attempt, status: record.status });
|
||||
previousStatus = record.status;
|
||||
if (record.status === expected) return record;
|
||||
@@ -287,9 +311,12 @@ async function pollHttp(baseUrl, runId, expected) {
|
||||
|
||||
async function pollCli(env, mode, runId, expected, baseUrl) {
|
||||
const overApi = mode === "api" ? ["--over-api", "--base-url", baseUrl] : [];
|
||||
const deadline = Date.now() + pollTimeoutMs;
|
||||
let previousStatus;
|
||||
for (let attempt = 1; attempt <= 100; attempt += 1) {
|
||||
const record = await cli(env, ["case", "run", "status", runId, ...overApi]);
|
||||
for (let attempt = 1; Date.now() < deadline; attempt += 1) {
|
||||
const record = await cli(env, ["case", "run", "status", runId, ...overApi], {
|
||||
timeoutMs: Math.min(defaultProcessTimeoutMs, Math.max(1, deadline - Date.now()))
|
||||
});
|
||||
if (record.status !== previousStatus || record.status === expected) stage("poll", { transport: mode, runId, attempt, status: record.status });
|
||||
previousStatus = record.status;
|
||||
if (record.status === expected) return record;
|
||||
@@ -299,30 +326,56 @@ async function pollCli(env, mode, runId, expected, baseUrl) {
|
||||
throw new Error(`timeout waiting for ${mode}/${runId}=${expected}`);
|
||||
}
|
||||
|
||||
async function waitForHttp(url, options = {}) {
|
||||
const deadline = Date.now() + (options.timeoutMs || 10000);
|
||||
async function waitForHttp(operation, url, options = {}) {
|
||||
const timeoutMs = positiveTimeout(options.timeoutMs, 10000, "HTTP wait timeout");
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let lastError;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const response = await fetch(url);
|
||||
if (response.ok) return options.returnText ? await response.text() : response;
|
||||
lastError = new Error(`${url} status=${response.status}`);
|
||||
const requestTimeoutMs = Math.min(defaultHttpTimeoutMs, Math.max(1, deadline - Date.now()));
|
||||
const { response, text } = await requestText(operation, url, undefined, { timeoutMs: requestTimeoutMs });
|
||||
if (response.ok) return options.returnText ? text : response;
|
||||
lastError = new Error(`HTTP status operation=${boundedOperation(operation)} url=${boundedUrl(url)} status=${response.status}`);
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
}
|
||||
await sleep(100);
|
||||
}
|
||||
throw new Error(`timeout waiting for ${url}: ${lastError instanceof Error ? lastError.message : String(lastError)}`);
|
||||
throw new Error(`HTTP wait timed out operation=${boundedOperation(operation)} url=${boundedUrl(url)} timeoutMs=${timeoutMs} lastError=${clip(lastError instanceof Error ? lastError.message : String(lastError), 400)}`);
|
||||
}
|
||||
|
||||
async function requestJson(url, init, expectedStatus = 200) {
|
||||
const response = await fetch(url, init);
|
||||
const text = await response.text();
|
||||
const payload = parseJson(text, url);
|
||||
if (response.status !== expectedStatus) throw new Error(`${url} status=${response.status}: ${clip(text)}`);
|
||||
async function requestJson(operation, url, init, expectedStatus = 200, options = {}) {
|
||||
const { response, text, timeoutMs } = await requestText(operation, url, init, options);
|
||||
let payload;
|
||||
try {
|
||||
payload = JSON.parse(text);
|
||||
} catch {
|
||||
throw new Error(`HTTP invalid JSON operation=${boundedOperation(operation)} url=${boundedUrl(url)} timeoutMs=${timeoutMs} body=${clip(text)}`);
|
||||
}
|
||||
if (response.status !== expectedStatus) {
|
||||
throw new Error(`HTTP status operation=${boundedOperation(operation)} url=${boundedUrl(url)} timeoutMs=${timeoutMs} status=${response.status} body=${clip(text)}`);
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
async function requestText(operation, url, init, options = {}) {
|
||||
const timeoutMs = positiveTimeout(options.timeoutMs, defaultHttpTimeoutMs, "HTTP request timeout");
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const response = await fetch(url, { ...init, signal: controller.signal });
|
||||
const text = await response.text();
|
||||
return { response, text, timeoutMs };
|
||||
} catch (error) {
|
||||
if (controller.signal.aborted) {
|
||||
throw new Error(`HTTP request timed out operation=${boundedOperation(operation)} url=${boundedUrl(url)} timeoutMs=${timeoutMs}`);
|
||||
}
|
||||
throw new Error(`HTTP request failed operation=${boundedOperation(operation)} url=${boundedUrl(url)} timeoutMs=${timeoutMs} error=${clip(error instanceof Error ? error.message : String(error), 300)}`);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
async function queryDatabase(url, sql) {
|
||||
const completed = await run("database-query", "psql", [url, "-At", "-F", "|", "-c", sql]);
|
||||
const [runs, events] = completed.stdout.trim().split("|").map(Number);
|
||||
@@ -378,6 +431,29 @@ function clip(value, limit = 1200) {
|
||||
return text.length <= limit ? text : `${text.slice(0, limit)}…`;
|
||||
}
|
||||
|
||||
function positiveTimeout(value, fallback, label) {
|
||||
const timeoutMs = value === undefined ? fallback : Number(value);
|
||||
if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) throw new Error(`${label} must be a positive integer`);
|
||||
return timeoutMs;
|
||||
}
|
||||
|
||||
function boundedOperation(value) {
|
||||
return clip(String(value || "unknown-operation").replace(/[\r\n\t]/gu, " "), 120);
|
||||
}
|
||||
|
||||
function boundedUrl(value) {
|
||||
try {
|
||||
const url = new URL(String(value));
|
||||
url.username = "";
|
||||
url.password = "";
|
||||
url.search = "";
|
||||
url.hash = "";
|
||||
return clip(url.toString(), 300);
|
||||
} catch {
|
||||
return clip(String(value || "invalid-url").split(/[?#]/u, 1)[0], 300);
|
||||
}
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user