204 lines
7.3 KiB
JavaScript
204 lines
7.3 KiB
JavaScript
#!/usr/bin/env bun
|
|
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join, resolve } from "node:path";
|
|
|
|
import { runTaskTreeCli } from "../tools/src/tasktree-cli.ts";
|
|
|
|
const repoRoot = resolve(import.meta.dir, "..");
|
|
const databaseEnvFile = requiredEnv("TASKTREE_NATIVE_DATABASE_ENV_FILE");
|
|
const databaseUrl = readEnvValue(databaseEnvFile, "DATABASE_URL");
|
|
const temporalAddress = requiredEnv("TASKTREE_TEMPORAL_ADDRESS");
|
|
const stateDir = mkdtempSync(join(tmpdir(), "tasktree-backup-l1-"));
|
|
const apiPort = positiveInteger(process.env.TASKTREE_API_PORT, 6673);
|
|
const workerPort = positiveInteger(process.env.TASKTREE_WORKER_HEALTH_PORT, 6674);
|
|
const apiUrl = `http://127.0.0.1:${apiPort}`;
|
|
const commonEnv = {
|
|
...process.env,
|
|
TASKTREE_DATABASE_URL: databaseUrl,
|
|
TASKTREE_TEMPORAL_ADDRESS: temporalAddress,
|
|
TASKTREE_TEMPORAL_NAMESPACE: process.env.TASKTREE_TEMPORAL_NAMESPACE ?? "unidesk",
|
|
TASKTREE_TEMPORAL_TASK_QUEUE: process.env.TASKTREE_TEMPORAL_TASK_QUEUE ?? "hwlab-v03-tasktree",
|
|
TASKTREE_API_HOST: "127.0.0.1",
|
|
TASKTREE_API_PORT: String(apiPort),
|
|
TASKTREE_WORKER_HEALTH_HOST: "127.0.0.1",
|
|
TASKTREE_WORKER_HEALTH_PORT: String(workerPort),
|
|
TASKTREE_BACKUP_ENABLED: "true",
|
|
TASKTREE_BACKUP_WORKTREE: join(stateDir, "repo"),
|
|
};
|
|
const cliEnv = { ...commonEnv, HWLAB_TASKTREE_API_URL: apiUrl };
|
|
const children = [];
|
|
let smokeGroupId = "";
|
|
let childrenStopped = false;
|
|
let readiness = {};
|
|
|
|
try {
|
|
event("children-starting");
|
|
children.push(spawn("cmd/hwlab-tasktree-worker/main.ts"));
|
|
children.push(spawn("cmd/hwlab-tasktree-api/main.ts"));
|
|
await waitUntil(async () => {
|
|
const [api, worker] = await Promise.all([
|
|
health(`${apiUrl}/health/ready`),
|
|
health(`http://127.0.0.1:${workerPort}/health/ready`),
|
|
]);
|
|
readiness = { api, worker };
|
|
return api.ok && worker.ok;
|
|
}, 30_000, () => `TaskTree native API and worker did not become ready: ${JSON.stringify(readiness)}`);
|
|
event("children-ready");
|
|
|
|
const initialStatus = await waitForBackupSuccess();
|
|
event("initial-backup-ready");
|
|
const initialSha = initialStatus.state.contentSha256;
|
|
const initialCommit = initialStatus.state.gitCommit;
|
|
const verify = successful(await runTaskTreeCli(["backup", "verify", "--overapi"], cliEnv));
|
|
if (verify.verified !== true) throw new Error("TaskTree --overapi backup verify did not match PostgreSQL");
|
|
const restore = successful(await runTaskTreeCli(["backup", "restore", "--overapi"], cliEnv));
|
|
if (restore.dryRun !== true || restore.restored !== false) throw new Error("TaskTree restore did not default to dry-run");
|
|
event("overapi-verified");
|
|
|
|
const group = successful(await runTaskTreeCli(
|
|
["group", "create", "--name", `Backup L1 smoke ${Date.now()}`],
|
|
cliEnv,
|
|
));
|
|
smokeGroupId = group.id;
|
|
event("smoke-group-created");
|
|
const changedStatus = await waitUntil(async () => {
|
|
const status = successful(await runTaskTreeCli(["backup", "status", "--overapi"], cliEnv));
|
|
return status.state.status === "succeeded" && status.state.contentSha256 !== initialSha ? status : null;
|
|
}, 30_000, "automatic backup did not capture the smoke mutation");
|
|
const changedSha = changedStatus.state.contentSha256;
|
|
event("smoke-mutation-backed-up");
|
|
|
|
successful(await runTaskTreeCli(["group", "delete", "--group", smokeGroupId], cliEnv));
|
|
smokeGroupId = "";
|
|
const restoredStatus = await waitUntil(async () => {
|
|
const status = successful(await runTaskTreeCli(["backup", "status", "--overapi"], cliEnv));
|
|
return status.state.status === "succeeded" && status.state.contentSha256 === initialSha ? status : null;
|
|
}, 30_000, "automatic backup did not capture smoke cleanup");
|
|
event("smoke-cleanup-backed-up");
|
|
|
|
console.log(JSON.stringify({
|
|
ok: true,
|
|
action: "tasktree-backup-native-l1-smoke",
|
|
mutation: true,
|
|
l1: {
|
|
apiReady: true,
|
|
workerReady: true,
|
|
temporalConnected: true,
|
|
automaticBackup: true,
|
|
overapiVerify: true,
|
|
restoreDryRun: true,
|
|
},
|
|
initialSha,
|
|
initialCommit,
|
|
changedSha,
|
|
restoredSha: restoredStatus.state.contentSha256,
|
|
valuesPrinted: false,
|
|
}, null, 2));
|
|
} catch (error) {
|
|
await stopChildren();
|
|
const diagnostics = [];
|
|
for (const child of children) {
|
|
diagnostics.push({
|
|
pid: child.pid,
|
|
exitCode: await child.exited,
|
|
});
|
|
}
|
|
process.stderr.write(`${JSON.stringify({
|
|
ok: false,
|
|
action: "tasktree-backup-native-l1-smoke",
|
|
error: error instanceof Error ? error.message : String(error),
|
|
readiness,
|
|
diagnostics,
|
|
valuesPrinted: false,
|
|
}, null, 2)}\n`);
|
|
throw error;
|
|
} finally {
|
|
if (smokeGroupId) {
|
|
await runTaskTreeCli(["group", "delete", "--group", smokeGroupId], cliEnv).catch(() => undefined);
|
|
}
|
|
await stopChildren();
|
|
rmSync(stateDir, { recursive: true, force: true });
|
|
}
|
|
|
|
function spawn(entrypoint) {
|
|
const env = entrypoint.includes("api")
|
|
? { ...commonEnv, TASKTREE_BACKUP_WORKTREE: `${commonEnv.TASKTREE_BACKUP_WORKTREE}-api` }
|
|
: commonEnv;
|
|
return Bun.spawn([process.execPath, entrypoint], {
|
|
cwd: repoRoot,
|
|
env,
|
|
stdin: "ignore",
|
|
stdout: "inherit",
|
|
stderr: "inherit",
|
|
});
|
|
}
|
|
|
|
async function stopChildren() {
|
|
if (childrenStopped) return;
|
|
childrenStopped = true;
|
|
for (const child of children) child.kill("SIGTERM");
|
|
await Promise.allSettled(children.map((child) => child.exited));
|
|
}
|
|
|
|
async function waitForBackupSuccess() {
|
|
return await waitUntil(async () => {
|
|
const status = successful(await runTaskTreeCli(["backup", "status", "--overapi"], cliEnv));
|
|
return status.state.status === "succeeded" && status.state.contentSha256 ? status : null;
|
|
}, 40_000, "automatic TaskTree backup did not succeed");
|
|
}
|
|
|
|
async function waitUntil(check, timeoutMs, message) {
|
|
const deadline = Date.now() + timeoutMs;
|
|
while (Date.now() < deadline) {
|
|
const result = await check();
|
|
if (result) return result;
|
|
await Bun.sleep(500);
|
|
}
|
|
throw new Error(typeof message === "function" ? message() : message);
|
|
}
|
|
|
|
async function health(url) {
|
|
try {
|
|
const response = await fetch(url);
|
|
const body = await response.json().catch(() => null);
|
|
return { ok: response.ok && body?.ok === true, status: response.status, error: null };
|
|
} catch (error) {
|
|
return { ok: false, status: null, error: error instanceof Error ? error.message : String(error) };
|
|
}
|
|
}
|
|
|
|
function successful(result) {
|
|
if (result?.ok !== true) {
|
|
const error = new Error(result?.error?.message ?? "TaskTree command failed");
|
|
error.code = result?.error?.code ?? "tasktree_error";
|
|
throw error;
|
|
}
|
|
return result.data;
|
|
}
|
|
|
|
function readEnvValue(path, key) {
|
|
const line = readFileSync(path, "utf8")
|
|
.split(/\r?\n/u)
|
|
.find((candidate) => candidate.startsWith(`${key}=`));
|
|
if (!line) throw new Error(`${key} is missing from the configured env source`);
|
|
const raw = line.slice(key.length + 1).trim();
|
|
if ((raw.startsWith('"') && raw.endsWith('"')) || (raw.startsWith("'") && raw.endsWith("'"))) return raw.slice(1, -1);
|
|
return raw;
|
|
}
|
|
|
|
function requiredEnv(name) {
|
|
const value = process.env[name];
|
|
if (!value) throw new Error(`${name} is required`);
|
|
return value;
|
|
}
|
|
|
|
function positiveInteger(value, fallback) {
|
|
const parsed = Number(value);
|
|
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
|
}
|
|
|
|
function event(stage) {
|
|
process.stderr.write(`${JSON.stringify({ action: "tasktree-backup-native-l1-smoke", stage })}\n`);
|
|
}
|