diff --git a/web/hwlab-cloud-web/package.json b/web/hwlab-cloud-web/package.json index 0ca7915e..bbed1ff6 100644 --- a/web/hwlab-cloud-web/package.json +++ b/web/hwlab-cloud-web/package.json @@ -13,6 +13,7 @@ "caserun:native:status": "bun run scripts/caserun-native-service.ts status", "caserun:native:logs": "bun run scripts/caserun-native-service.ts logs", "caserun:native:smoke": "bun run scripts/caserun-native-smoke.ts", + "caserun:native:reload-smoke": "bun run scripts/caserun-native-reload-smoke.ts", "check": "bun run deps --quiet && bun run scripts/tsc-check.ts && bun test scripts", "check:tsc": "bun run deps --quiet && bun run scripts/tsc-check.ts", "check:tsc-strict": "bun run deps --quiet && bun run scripts/tsc-check.ts --strict", diff --git a/web/hwlab-cloud-web/scripts/caserun-native-reload-smoke.ts b/web/hwlab-cloud-web/scripts/caserun-native-reload-smoke.ts new file mode 100644 index 00000000..1cb2fe85 --- /dev/null +++ b/web/hwlab-cloud-web/scripts/caserun-native-reload-smoke.ts @@ -0,0 +1,129 @@ +import { readFileSync, renameSync, rmSync, statSync, utimesSync } from "node:fs"; +import { resolve } from "node:path"; + +const root = process.cwd(); +const watchedPath = resolve(root, "scripts/caserun-native-fixtures.ts"); +const displacedPath = `${watchedPath}.reload-smoke`; +const originalStat = statSync(watchedPath); +const port = Number(process.env.HWLAB_CASERUN_RELOAD_SMOKE_PORT || 14316); +const baseUrl = `http://127.0.0.1:${port}`; +const reloadEvents: Array<{ pid?: number; reason?: string }> = []; +let output = ""; +let displaced = false; + +function requireCondition(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} + +async function waitFor(description: string, condition: () => boolean | Promise, timeoutMs = 5000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await condition()) return; + await Bun.sleep(50); + } + throw new Error(`timeout waiting for ${description}`); +} + +async function readOutput(stream: ReadableStream): Promise { + const reader = stream.getReader(); + const decoder = new TextDecoder(); + let pending = ""; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + const chunk = decoder.decode(value, { stream: true }); + output += chunk; + pending += chunk; + const lines = pending.split("\n"); + pending = lines.pop() || ""; + for (const line of lines) { + if (!line.includes('"event":"caserun-native-reload"')) continue; + const event = JSON.parse(line) as { pid?: number; reason?: string }; + if (!reloadEvents.some((existing) => existing.pid === event.pid)) reloadEvents.push(event); + } + } + const finalChunk = decoder.decode(); + output += finalChunk; + pending += finalChunk; + if (pending.includes('"event":"caserun-native-reload"')) { + const event = JSON.parse(pending) as { pid?: number; reason?: string }; + if (!reloadEvents.some((existing) => existing.pid === event.pid)) reloadEvents.push(event); + } +} + +async function healthAvailable(): Promise { + try { + const response = await fetch(`${baseUrl}/health`); + return response.ok; + } catch { + return false; + } +} + +function childPids(supervisorPid: number): number[] { + try { + return readFileSync(`/proc/${supervisorPid}/task/${supervisorPid}/children`, "utf8") + .trim() + .split(/\s+/) + .filter(Boolean) + .map(Number); + } catch { + return []; + } +} + +rmSync(displacedPath, { force: true }); +const supervisor = Bun.spawn([process.execPath, "scripts/caserun-native-supervisor.ts"], { + cwd: root, + env: { ...process.env, HWLAB_CASERUN_NATIVE_PORT: String(port) }, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe" +}); +const stdoutTask = readOutput(supervisor.stdout); +const stderrTask = readOutput(supervisor.stderr); + +try { + await waitFor("initial native health", healthAvailable); + + renameSync(watchedPath, displacedPath); + displaced = true; + await Bun.sleep(300); + requireCondition(supervisor.exitCode === null, `supervisor exited during atomic save window: ${supervisor.exitCode}`); + renameSync(displacedPath, watchedPath); + displaced = false; + const restoredAt = new Date(Date.now() + 1000); + utimesSync(watchedPath, restoredAt, restoredAt); + await waitFor("reload after atomic save", () => reloadEvents.length >= 2); + + for (let index = 0; index < 5; index += 1) { + const changedAt = new Date(Date.now() + (index + 1) * 1000); + utimesSync(watchedPath, changedAt, changedAt); + await Bun.sleep(120); + } + + await waitFor("final native health", healthAvailable); + await Bun.sleep(300); + const liveChildren = childPids(supervisor.pid); + requireCondition(liveChildren.length === 1, `expected one native server child, found ${liveChildren.join(",") || "none"}`); + requireCondition(reloadEvents.length >= 3, `expected repeated reloads, found ${reloadEvents.length}`); + requireCondition(!output.includes("EADDRINUSE"), "native reload logged EADDRINUSE"); + + console.log(JSON.stringify({ + ok: true, + mode: "native-test", + supervisorPid: supervisor.pid, + serverPid: liveChildren[0], + reloadCount: reloadEvents.length, + reloadPids: reloadEvents.map((event) => event.pid), + atomicSaveMissingFileTolerated: true, + eaddrinuse: false + })); +} finally { + if (displaced) renameSync(displacedPath, watchedPath); + rmSync(displacedPath, { force: true }); + utimesSync(watchedPath, originalStat.atime, originalStat.mtime); + if (supervisor.exitCode === null) supervisor.kill("SIGTERM"); + await supervisor.exited; + await Promise.all([stdoutTask, stderrTask]); +} diff --git a/web/hwlab-cloud-web/scripts/caserun-native-service.ts b/web/hwlab-cloud-web/scripts/caserun-native-service.ts index e017fbbb..91871105 100644 --- a/web/hwlab-cloud-web/scripts/caserun-native-service.ts +++ b/web/hwlab-cloud-web/scripts/caserun-native-service.ts @@ -26,6 +26,15 @@ function isRunning(pid: number | null): boolean { } } +async function waitForExit(pid: number): Promise { + const deadline = Date.now() + 5000; + while (Date.now() < deadline) { + if (!isRunning(pid)) return true; + await Bun.sleep(25); + } + return !isRunning(pid); +} + mkdirSync(stateDir, { recursive: true }); const pid = readPid(); @@ -46,8 +55,12 @@ if (action === "start") { console.log(JSON.stringify({ ok: true, action, status: "stopped" })); } else { process.kill(pid!, "SIGTERM"); + if (!await waitForExit(pid!)) { + console.error(JSON.stringify({ ok: false, action, status: "stop-timeout", pid })); + process.exit(1); + } rmSync(pidPath, { force: true }); - console.log(JSON.stringify({ ok: true, action, status: "stopping", pid })); + console.log(JSON.stringify({ ok: true, action, status: "stopped", pid })); } } else if (action === "status") { const running = isRunning(pid); diff --git a/web/hwlab-cloud-web/scripts/caserun-native-supervisor.ts b/web/hwlab-cloud-web/scripts/caserun-native-supervisor.ts index e470c17a..7c5603fc 100644 --- a/web/hwlab-cloud-web/scripts/caserun-native-supervisor.ts +++ b/web/hwlab-cloud-web/scripts/caserun-native-supervisor.ts @@ -10,40 +10,62 @@ const watchedFiles = [ let child: ReturnType | null = null; let signature = sourceSignature(); let stopping = false; +let restarting = false; -function sourceSignature(): string { - return watchedFiles.map((path) => `${path}:${statSync(path).mtimeMs}:${statSync(path).size}`).join("|"); +function sourceSignature(): string | null { + try { + return watchedFiles.map((path) => { + const source = statSync(path); + return `${path}:${source.mtimeMs}:${source.size}`; + }).join("|"); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") return null; + throw error; + } } function startChild(reason: string): void { - child = Bun.spawn([process.execPath, "scripts/caserun-native-server.ts"], { cwd: root, stdin: "ignore", stdout: "inherit", stderr: "inherit" }); - console.log(JSON.stringify({ event: "caserun-native-reload", mode: "native-test", reason, pid: child.pid })); - void child.exited.then((exitCode) => { - if (!stopping && child?.exitCode === exitCode) console.log(JSON.stringify({ event: "caserun-native-exit", exitCode })); + const spawned = Bun.spawn([process.execPath, "scripts/caserun-native-server.ts"], { cwd: root, stdin: "ignore", stdout: "inherit", stderr: "inherit" }); + child = spawned; + console.log(JSON.stringify({ event: "caserun-native-reload", mode: "native-test", reason, pid: spawned.pid })); + void spawned.exited.then((exitCode) => { + if (!stopping && child === spawned) console.log(JSON.stringify({ event: "caserun-native-exit", exitCode })); }); } -function stopChild(): void { - if (child && child.exitCode === null) child.kill("SIGTERM"); - child = null; +async function stopChild(): Promise { + const runningChild = child; + if (!runningChild) return; + if (runningChild.exitCode === null) runningChild.kill("SIGTERM"); + await runningChild.exited; + if (child === runningChild) child = null; } startChild("initial"); -const timer = setInterval(() => { +async function reloadIfChanged(): Promise { + if (stopping || restarting) return; const nextSignature = sourceSignature(); - if (nextSignature === signature) return; + if (nextSignature === null || nextSignature === signature) return; signature = nextSignature; - stopChild(); - startChild("source-change"); -}, 500); + restarting = true; + try { + await stopChild(); + if (!stopping) startChild("source-change"); + } finally { + restarting = false; + } +} -function shutdown(): void { +const timer = setInterval(() => void reloadIfChanged(), 100); + +async function shutdown(): Promise { + if (stopping) return; stopping = true; clearInterval(timer); - stopChild(); + await stopChild(); process.exit(0); } -process.on("SIGINT", shutdown); -process.on("SIGTERM", shutdown); +process.on("SIGINT", () => void shutdown()); +process.on("SIGTERM", () => void shutdown());