feat(tasktree): add GitHub disaster recovery backups

2026-07-18 15:15:57 +02:00
parent 8b57d87ff7
commit 12ad602a63
16 changed files with 1342 additions and 9 deletions
@@ -0,0 +1,203 @@
+#!/usr/bin/env bun
+import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+
+import { runTaskTreeCli } from "../tools/src/tasktree-cli.ts";
+
+const repoRoot = resolve(import.meta.dir, "..");
+const databaseEnvFile = requiredEnv("TASKTREE_NATIVE_DATABASE_ENV_FILE");
+const databaseUrl = readEnvValue(databaseEnvFile, "DATABASE_URL");
+const temporalAddress = requiredEnv("TASKTREE_TEMPORAL_ADDRESS");
+const stateDir = mkdtempSync(join(tmpdir(), "tasktree-backup-l1-"));
+const apiPort = positiveInteger(process.env.TASKTREE_API_PORT, 6673);
+const workerPort = positiveInteger(process.env.TASKTREE_WORKER_HEALTH_PORT, 6674);
+const apiUrl = `http://127.0.0.1:${apiPort}`;
+const commonEnv = {
+  ...process.env,
+  TASKTREE_DATABASE_URL: databaseUrl,
+  TASKTREE_TEMPORAL_ADDRESS: temporalAddress,
+  TASKTREE_TEMPORAL_NAMESPACE: process.env.TASKTREE_TEMPORAL_NAMESPACE ?? "unidesk",
+  TASKTREE_TEMPORAL_TASK_QUEUE: process.env.TASKTREE_TEMPORAL_TASK_QUEUE ?? "hwlab-v03-tasktree",
+  TASKTREE_API_HOST: "127.0.0.1",
+  TASKTREE_API_PORT: String(apiPort),
+  TASKTREE_WORKER_HEALTH_HOST: "127.0.0.1",
+  TASKTREE_WORKER_HEALTH_PORT: String(workerPort),
+  TASKTREE_BACKUP_ENABLED: "true",
+  TASKTREE_BACKUP_WORKTREE: join(stateDir, "repo"),
+};
+const cliEnv = { ...commonEnv, HWLAB_TASKTREE_API_URL: apiUrl };
+const children = [];
+let smokeGroupId = "";
+let childrenStopped = false;
+let readiness = {};
+
+try {
+  event("children-starting");
+  children.push(spawn("cmd/hwlab-tasktree-worker/main.ts"));
+  children.push(spawn("cmd/hwlab-tasktree-api/main.ts"));
+  await waitUntil(async () => {
+    const [api, worker] = await Promise.all([
+      health(`${apiUrl}/health/ready`),
+      health(`http://127.0.0.1:${workerPort}/health/ready`),
+    ]);
+    readiness = { api, worker };
+    return api.ok && worker.ok;
+  }, 30_000, () => `TaskTree native API and worker did not become ready: ${JSON.stringify(readiness)}`);
+  event("children-ready");
+
+  const initialStatus = await waitForBackupSuccess();
+  event("initial-backup-ready");
+  const initialSha = initialStatus.state.contentSha256;
+  const initialCommit = initialStatus.state.gitCommit;
+  const verify = successful(await runTaskTreeCli(["backup", "verify", "--overapi"], cliEnv));
+  if (verify.verified !== true) throw new Error("TaskTree --overapi backup verify did not match PostgreSQL");
+  const restore = successful(await runTaskTreeCli(["backup", "restore", "--overapi"], cliEnv));
+  if (restore.dryRun !== true || restore.restored !== false) throw new Error("TaskTree restore did not default to dry-run");
+  event("overapi-verified");
+
+  const group = successful(await runTaskTreeCli(
+    ["group", "create", "--name", `Backup L1 smoke ${Date.now()}`],
+    cliEnv,
+  ));
+  smokeGroupId = group.id;
+  event("smoke-group-created");
+  const changedStatus = await waitUntil(async () => {
+    const status = successful(await runTaskTreeCli(["backup", "status", "--overapi"], cliEnv));
+    return status.state.status === "succeeded" && status.state.contentSha256 !== initialSha ? status : null;
+  }, 30_000, "automatic backup did not capture the smoke mutation");
+  const changedSha = changedStatus.state.contentSha256;
+  event("smoke-mutation-backed-up");
+
+  successful(await runTaskTreeCli(["group", "delete", "--group", smokeGroupId], cliEnv));
+  smokeGroupId = "";
+  const restoredStatus = await waitUntil(async () => {
+    const status = successful(await runTaskTreeCli(["backup", "status", "--overapi"], cliEnv));
+    return status.state.status === "succeeded" && status.state.contentSha256 === initialSha ? status : null;
+  }, 30_000, "automatic backup did not capture smoke cleanup");
+  event("smoke-cleanup-backed-up");
+
+  console.log(JSON.stringify({
+    ok: true,
+    action: "tasktree-backup-native-l1-smoke",
+    mutation: true,
+    l1: {
+      apiReady: true,
+      workerReady: true,
+      temporalConnected: true,
+      automaticBackup: true,
+      overapiVerify: true,
+      restoreDryRun: true,
+    },
+    initialSha,
+    initialCommit,
+    changedSha,
+    restoredSha: restoredStatus.state.contentSha256,
+    valuesPrinted: false,
+  }, null, 2));
+} catch (error) {
+  await stopChildren();
+  const diagnostics = [];
+  for (const child of children) {
+    diagnostics.push({
+      pid: child.pid,
+      exitCode: await child.exited,
+    });
+  }
+  process.stderr.write(`${JSON.stringify({
+    ok: false,
+    action: "tasktree-backup-native-l1-smoke",
+    error: error instanceof Error ? error.message : String(error),
+    readiness,
+    diagnostics,
+    valuesPrinted: false,
+  }, null, 2)}\n`);
+  throw error;
+} finally {
+  if (smokeGroupId) {
+    await runTaskTreeCli(["group", "delete", "--group", smokeGroupId], cliEnv).catch(() => undefined);
+  }
+  await stopChildren();
+  rmSync(stateDir, { recursive: true, force: true });
+}
+
+function spawn(entrypoint) {
+  const env = entrypoint.includes("api")
+    ? { ...commonEnv, TASKTREE_BACKUP_WORKTREE: `${commonEnv.TASKTREE_BACKUP_WORKTREE}-api` }
+    : commonEnv;
+  return Bun.spawn([process.execPath, entrypoint], {
+    cwd: repoRoot,
+    env,
+    stdin: "ignore",
+    stdout: "inherit",
+    stderr: "inherit",
+  });
+}
+
+async function stopChildren() {
+  if (childrenStopped) return;
+  childrenStopped = true;
+  for (const child of children) child.kill("SIGTERM");
+  await Promise.allSettled(children.map((child) => child.exited));
+}
+
+async function waitForBackupSuccess() {
+  return await waitUntil(async () => {
+    const status = successful(await runTaskTreeCli(["backup", "status", "--overapi"], cliEnv));
+    return status.state.status === "succeeded" && status.state.contentSha256 ? status : null;
+  }, 40_000, "automatic TaskTree backup did not succeed");
+}
+
+async function waitUntil(check, timeoutMs, message) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    const result = await check();
+    if (result) return result;
+    await Bun.sleep(500);
+  }
+  throw new Error(typeof message === "function" ? message() : message);
+}
+
+async function health(url) {
+  try {
+    const response = await fetch(url);
+    const body = await response.json().catch(() => null);
+    return { ok: response.ok && body?.ok === true, status: response.status, error: null };
+  } catch (error) {
+    return { ok: false, status: null, error: error instanceof Error ? error.message : String(error) };
+  }
+}
+
+function successful(result) {
+  if (result?.ok !== true) {
+    const error = new Error(result?.error?.message ?? "TaskTree command failed");
+    error.code = result?.error?.code ?? "tasktree_error";
+    throw error;
+  }
+  return result.data;
+}
+
+function readEnvValue(path, key) {
+  const line = readFileSync(path, "utf8")
+    .split(/\r?\n/u)
+    .find((candidate) => candidate.startsWith(`${key}=`));
+  if (!line) throw new Error(`${key} is missing from the configured env source`);
+  const raw = line.slice(key.length + 1).trim();
+  if ((raw.startsWith('"') && raw.endsWith('"')) || (raw.startsWith("'") && raw.endsWith("'"))) return raw.slice(1, -1);
+  return raw;
+}
+
+function requiredEnv(name) {
+  const value = process.env[name];
+  if (!value) throw new Error(`${name} is required`);
+  return value;
+}
+
+function positiveInteger(value, fallback) {
+  const parsed = Number(value);
+  return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+function event(stage) {
+  process.stderr.write(`${JSON.stringify({ action: "tasktree-backup-native-l1-smoke", stage })}\n`);
+}
@@ -0,0 +1,113 @@
+#!/usr/bin/env bun
+import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import pg from "pg";
+
+import { canonicalSnapshot } from "../internal/tasktree/backup-contracts.ts";
+import { TaskTreeBackupService } from "../internal/tasktree/backup.ts";
+import { TaskTreeStore } from "../internal/tasktree/store.ts";
+
+const databaseUrl = readEnvValue(requiredEnv("TASKTREE_NATIVE_DATABASE_ENV_FILE"), "DATABASE_URL");
+const schema = `tasktree_backup_restore_${Date.now()}_${Math.random().toString(16).slice(2, 10)}`;
+const control = new pg.Pool({
+  connectionString: databaseUrl,
+  max: 1,
+  connectionTimeoutMillis: 5000,
+  ssl: { rejectUnauthorized: false },
+});
+const root = mkdtempSync(join(tmpdir(), "tasktree-backup-restore-l1-"));
+let store;
+
+try {
+  await control.query(`CREATE SCHEMA "${schema}" AUTHORIZATION CURRENT_USER`);
+  const isolatedUrl = new URL(databaseUrl);
+  isolatedUrl.searchParams.set("options", `-c search_path=${schema}`);
+  store = new TaskTreeStore(isolatedUrl.toString());
+  const backup = new TaskTreeBackupService(store, {
+    enabled: true,
+    repoSshUrl: join(root, "remote.git"),
+    branch: "main",
+    basePath: "tasktree/restore-smoke",
+    worktreePath: join(root, "repo"),
+    target: "restore-smoke",
+    intervalMs: 60_000,
+    sshCommand: "",
+    authorName: "TaskTree Restore Smoke",
+    authorEmail: "tasktree-restore@example.invalid",
+    commitMessagePrefix: "tasktree:",
+  });
+  const init = Bun.spawnSync(["git", "init", "--bare", backup.config.repoSshUrl], {
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  if (init.exitCode !== 0) throw new Error("failed to create disposable Git remote");
+
+  const group = await store.createGroup("Restore smoke", "Disposable L1 schema");
+  const task = await store.createTask({ groupId: group.id, title: "Stable identity" });
+  await store.writeReport({ taskId: task.id, title: "Execution", body: "Original report" });
+  const original = await store.exportSnapshot();
+  const published = await backup.create({ reason: "restore smoke baseline" });
+
+  await store.updateTask(task.id, { title: "Changed after backup" });
+  await store.createTask({ groupId: group.id, title: "Created after backup" });
+  const plan = await backup.restore({ confirm: false });
+  if (plan.dryRun !== true || plan.diff.tasks.create !== 0 || plan.diff.tasks.update !== 1 || plan.diff.tasks.delete !== 1) {
+    throw new Error("restore dry-run diff did not identify the isolated mutations");
+  }
+  await assertRejectsCode(
+    () => backup.restore({ confirm: true, expectedSha256: "0".repeat(64) }),
+    "backup_restore_sha_required",
+  );
+  const restored = await backup.restore({ confirm: true, expectedSha256: published.contentSha256 });
+  if (restored.restored !== true) throw new Error("confirmed restore did not report success");
+  const after = await store.exportSnapshot();
+  if (canonicalSnapshot(after) !== canonicalSnapshot(original)) {
+    throw new Error("confirmed restore did not preserve the canonical snapshot");
+  }
+  await Bun.write(Bun.stdout, `${JSON.stringify({
+    ok: true,
+    action: "tasktree-backup-restore-l1-smoke",
+    mutation: true,
+    storage: "native-postgresql-disposable-schema",
+    publishedSha: published.contentSha256,
+    stableGroupId: group.id,
+    stableTaskId: task.id,
+    dryRunDiff: plan.diff,
+    wrongShaRejected: true,
+    restored: true,
+    canonicalMatch: true,
+    valuesPrinted: false,
+  }, null, 2)}\n`);
+} finally {
+  await store?.close().catch(() => undefined);
+  await control.query(`DROP SCHEMA IF EXISTS "${schema}" CASCADE`).catch(() => undefined);
+  await control.end().catch(() => undefined);
+  rmSync(root, { recursive: true, force: true });
+}
+
+async function assertRejectsCode(run, code) {
+  try {
+    await run();
+  } catch (error) {
+    if (error?.code === code) return;
+    throw error;
+  }
+  throw new Error(`expected ${code}`);
+}
+
+function readEnvValue(path, key) {
+  const line = readFileSync(path, "utf8")
+    .split(/\r?\n/u)
+    .find((candidate) => candidate.startsWith(`${key}=`));
+  if (!line) throw new Error(`${key} is missing from the configured env source`);
+  const raw = line.slice(key.length + 1).trim();
+  if ((raw.startsWith('"') && raw.endsWith('"')) || (raw.startsWith("'") && raw.endsWith("'"))) return raw.slice(1, -1);
+  return raw;
+}
+
+function requiredEnv(name) {
+  const value = process.env[name];
+  if (!value) throw new Error(`${name} is required`);
+  return value;
+}