From 12ad602a638b4e1b0e9d28126f1cbe28515f8126 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 18 Jul 2026 15:15:57 +0200 Subject: [PATCH] feat(tasktree): add GitHub disaster recovery backups --- cmd/hwlab-tasktree-api/main.ts | 5 +- cmd/hwlab-tasktree-worker/main.ts | 20 +- internal/tasktree/backup-contracts.test.ts | 166 ++++++++++++ internal/tasktree/backup-contracts.ts | 241 ++++++++++++++++++ internal/tasktree/backup-git.ts | 173 +++++++++++++ internal/tasktree/backup.ts | 172 +++++++++++++ internal/tasktree/contracts.ts | 4 + internal/tasktree/dispatcher.ts | 21 +- internal/tasktree/runtime.ts | 3 + internal/tasktree/store.ts | 165 +++++++++++- package.json | 2 + scripts/tasktree-backup-native-l1-smoke.mjs | 203 +++++++++++++++ scripts/tasktree-backup-restore-l1-smoke.mjs | 113 ++++++++ tools/src/tasktree-cli.test.ts | 21 +- tools/src/tasktree-cli.ts | 28 +- .../scripts/tasktree-native-supervisor.ts | 14 +- 16 files changed, 1342 insertions(+), 9 deletions(-) create mode 100644 internal/tasktree/backup-contracts.test.ts create mode 100644 internal/tasktree/backup-contracts.ts create mode 100644 internal/tasktree/backup-git.ts create mode 100644 internal/tasktree/backup.ts create mode 100644 scripts/tasktree-backup-native-l1-smoke.mjs create mode 100644 scripts/tasktree-backup-restore-l1-smoke.mjs diff --git a/cmd/hwlab-tasktree-api/main.ts b/cmd/hwlab-tasktree-api/main.ts index 1b6468ab..c26c8dcd 100644 --- a/cmd/hwlab-tasktree-api/main.ts +++ b/cmd/hwlab-tasktree-api/main.ts @@ -3,7 +3,10 @@ import { createTaskTreeDispatcher } from "../../internal/tasktree/dispatcher.ts" import { createTaskTreeHttpApp } from "../../internal/tasktree/http.ts"; import { taskTreeRuntime } from "../../internal/tasktree/runtime.ts"; -const runtime = taskTreeRuntime(); +const runtime = taskTreeRuntime({ + ...process.env, + TASKTREE_BACKUP_WORKTREE: process.env.TASKTREE_BACKUP_WORKTREE ?? ".state/tasktree/backup-repo-api", +}); const dispatch = createTaskTreeDispatcher(runtime); const app = createTaskTreeHttpApp({ dispatch, close: () => runtime.store.close() }); const host = process.env.TASKTREE_API_HOST || "0.0.0.0"; diff --git a/cmd/hwlab-tasktree-worker/main.ts b/cmd/hwlab-tasktree-worker/main.ts index 9aa81aed..e7d02818 100644 --- a/cmd/hwlab-tasktree-worker/main.ts +++ b/cmd/hwlab-tasktree-worker/main.ts @@ -3,9 +3,13 @@ import { NativeConnection, Worker } from "@temporalio/worker"; import path from "node:path"; import { createTaskTreeActivities } from "../../internal/tasktree/activities.ts"; +import { startTaskTreeBackupScheduler } from "../../internal/tasktree/backup.ts"; import { taskTreeRuntime } from "../../internal/tasktree/runtime.ts"; -const runtime = taskTreeRuntime(); +const runtime = taskTreeRuntime({ + ...process.env, + TASKTREE_BACKUP_WORKTREE: process.env.TASKTREE_BACKUP_WORKTREE ?? ".state/tasktree/backup-repo-worker", +}); if (!runtime.temporalAddress) throw new Error("TASKTREE_TEMPORAL_ADDRESS is required"); await runtime.store.ensureSchema(); const connection = await NativeConnection.connect({ address: runtime.temporalAddress }); @@ -17,13 +21,22 @@ const worker = await Worker.create({ activities: createTaskTreeActivities(runtime.store) }); const healthPort = Number.parseInt(process.env.TASKTREE_WORKER_HEALTH_PORT || "6674", 10); +const backupScheduler = startTaskTreeBackupScheduler(runtime.backup, (result) => { + process.stdout.write(`${JSON.stringify({ serviceId: "hwlab-tasktree-worker", ...result })}\n`); +}); const health = Bun.serve({ hostname: process.env.TASKTREE_WORKER_HEALTH_HOST || "0.0.0.0", port: healthPort, - fetch(request) { + async fetch(request) { const pathname = new URL(request.url).pathname; if (pathname !== "/health/live" && pathname !== "/health/ready") return Response.json({ ok: false, error: "not_found" }, { status: 404 }); - return Response.json({ ok: true, serviceId: "hwlab-tasktree-worker", namespace: runtime.temporalNamespace, taskQueue: runtime.taskQueue }); + return Response.json({ + ok: true, + serviceId: "hwlab-tasktree-worker", + namespace: runtime.temporalNamespace, + taskQueue: runtime.taskQueue, + backup: await runtime.backup.status(), + }); } }); @@ -32,6 +45,7 @@ for (const signal of ["SIGINT", "SIGTERM"] as const) process.once(signal, () => try { await worker.run(); } finally { + await backupScheduler.stop(); health.stop(true); await connection.close(); await runtime.store.close(); diff --git a/internal/tasktree/backup-contracts.test.ts b/internal/tasktree/backup-contracts.test.ts new file mode 100644 index 00000000..a310d272 --- /dev/null +++ b/internal/tasktree/backup-contracts.test.ts @@ -0,0 +1,166 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "bun:test"; + +import { + canonicalSnapshot, + parseManifest, + parseSnapshot, + restoreDiff, + snapshotManifest, + snapshotSha256, + verifySnapshotManifest, + type TaskTreeSnapshot, +} from "./backup-contracts.ts"; +import { TaskTreeGitBackup } from "./backup-git.ts"; +import { startTaskTreeBackupScheduler, type TaskTreeBackupService } from "./backup.ts"; + +const snapshot: TaskTreeSnapshot = { + schemaVersion: 1, + groups: [ + { + id: "tg_1", + name: "Backup", + description: "", + createdAt: "2026-07-18T00:00:00.000Z", + updatedAt: "2026-07-18T00:00:00.000Z", + }, + ], + tasks: [ + { + id: "tt_2", + groupId: "tg_1", + parentId: "tt_1", + kind: "subtask", + title: "Child", + description: "", + status: "pending", + startAt: null, + dueAt: null, + sortOrder: 1, + createdAt: "2026-07-18T00:00:00.000Z", + updatedAt: "2026-07-18T00:00:00.000Z", + }, + { + id: "tt_1", + groupId: "tg_1", + parentId: null, + kind: "task", + title: "Root", + description: "", + status: "in_progress", + startAt: "2026-07-18T00:00:00.000Z", + dueAt: "2026-07-19T00:00:00.000Z", + sortOrder: 0, + createdAt: "2026-07-18T00:00:00.000Z", + updatedAt: "2026-07-18T00:00:00.000Z", + }, + ], + milestones: [], + reports: [ + { + id: "tr_1", + taskId: "tt_1", + title: "Execution", + body: "Passed.", + status: "succeeded", + createdAt: "2026-07-18T00:00:00.000Z", + }, + ], +}; + +test("TaskTree backup canonical JSON and SHA ignore entity input order", () => { + const reversed = { ...snapshot, tasks: [...snapshot.tasks].reverse() }; + assert.equal(canonicalSnapshot(snapshot), canonicalSnapshot(reversed)); + assert.equal(snapshotSha256(snapshot), snapshotSha256(reversed)); + assert.deepEqual(parseSnapshot(canonicalSnapshot(snapshot)), reversed); +}); + +test("TaskTree backup manifest rejects changed snapshot content", () => { + const manifest = snapshotManifest(snapshot, { + target: "development", + generatedAt: "2026-07-18T01:00:00.000Z", + }); + assert.doesNotThrow(() => verifySnapshotManifest(snapshot, parseManifest(JSON.stringify(manifest)))); + const changed = { + ...snapshot, + tasks: snapshot.tasks.map((task) => task.id === "tt_1" ? { ...task, title: "Changed" } : task), + }; + assert.throws( + () => verifySnapshotManifest(changed, manifest), + (error: any) => error?.code === "backup_content_sha_mismatch", + ); +}); + +test("TaskTree restore diff reports create, update, and delete independently", () => { + const incoming: TaskTreeSnapshot = { + ...snapshot, + tasks: [ + { ...snapshot.tasks[0], title: "Updated child" }, + { + ...snapshot.tasks[1], + id: "tt_3", + parentId: null, + title: "New root", + }, + ], + }; + assert.deepEqual(restoreDiff(snapshot, incoming).tasks, { create: 1, update: 1, delete: 1 }); +}); + +test("TaskTree Git backup publishes once and deduplicates identical content", async () => { + const root = mkdtempSync(join(tmpdir(), "tasktree-backup-test-")); + const remote = join(root, "remote.git"); + const worktree = join(root, "worktree"); + try { + const init = Bun.spawnSync(["git", "init", "--bare", remote], { stdout: "pipe", stderr: "pipe" }); + assert.equal(init.exitCode, 0); + const backup = new TaskTreeGitBackup({ + repoSshUrl: remote, + branch: "main", + basePath: "tasktree/test", + worktreePath: worktree, + target: "test", + sshCommand: "", + authorName: "TaskTree Test", + authorEmail: "tasktree-test@example.invalid", + commitMessagePrefix: "tasktree:", + }); + const first = await backup.publish(snapshot, "test snapshot"); + const second = await backup.publish(snapshot, "duplicate snapshot"); + assert.equal(first.changed, true); + assert.equal(second.changed, false); + assert.equal(second.gitCommit, first.gitCommit); + assert.equal(backup.readCurrent()?.manifest.contentSha256, snapshotSha256(snapshot)); + assert.equal(JSON.parse(readFileSync(join(worktree, "tasktree/test/manifest.json"), "utf8")).target, "test"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("TaskTree backup scheduler waits for an in-flight snapshot before stopping", async () => { + let finish!: () => void; + const createFinished = new Promise((resolve) => { + finish = resolve; + }); + const service = { + config: { enabled: true, intervalMs: 60_000 }, + async create() { + await createFinished; + return { changed: false }; + }, + } as unknown as TaskTreeBackupService; + const scheduler = startTaskTreeBackupScheduler(service); + await Bun.sleep(10); + let stopped = false; + const stopping = scheduler.stop().then(() => { + stopped = true; + }); + await Bun.sleep(10); + assert.equal(stopped, false); + finish(); + await stopping; + assert.equal(stopped, true); +}); diff --git a/internal/tasktree/backup-contracts.ts b/internal/tasktree/backup-contracts.ts new file mode 100644 index 00000000..37f5cbbf --- /dev/null +++ b/internal/tasktree/backup-contracts.ts @@ -0,0 +1,241 @@ +import { createHash } from "node:crypto"; + +import type { ExecutionReport, Milestone, TaskGroup, TaskItem } from "./contracts.ts"; + +export const TASKTREE_BACKUP_SCHEMA_VERSION = 1; + +export type TaskTreeSnapshot = { + schemaVersion: typeof TASKTREE_BACKUP_SCHEMA_VERSION; + groups: TaskGroup[]; + tasks: TaskItem[]; + milestones: Milestone[]; + reports: ExecutionReport[]; +}; + +export type TaskTreeBackupManifest = { + schemaVersion: typeof TASKTREE_BACKUP_SCHEMA_VERSION; + target: string; + generatedAt: string; + contentSha256: string; + writer: string; + counts: { + groups: number; + tasks: number; + milestones: number; + reports: number; + }; +}; + +export type TaskTreeBackupState = { + target: string; + pending: boolean; + requestedAt: string | null; + requestReason: string; + lastAttemptAt: string | null; + lastSuccessAt: string | null; + contentSha256: string; + gitCommit: string; + status: "never" | "pending" | "succeeded" | "failed"; + errorCode: string; + errorMessage: string; + updatedAt: string; +}; + +export type TaskTreeRestoreDiff = Record< + "groups" | "tasks" | "milestones" | "reports", + { create: number; update: number; delete: number } +>; + +export function canonicalSnapshot(value: TaskTreeSnapshot): string { + return `${JSON.stringify(normalizeSnapshot(value), null, 2)}\n`; +} + +export function snapshotSha256(value: TaskTreeSnapshot): string { + return createHash("sha256").update(canonicalSnapshot(value)).digest("hex"); +} + +export function snapshotManifest( + snapshot: TaskTreeSnapshot, + options: { target: string; generatedAt?: string; writer?: string }, +): TaskTreeBackupManifest { + return { + schemaVersion: TASKTREE_BACKUP_SCHEMA_VERSION, + target: requiredText(options.target, "target"), + generatedAt: new Date(options.generatedAt ?? Date.now()).toISOString(), + contentSha256: snapshotSha256(snapshot), + writer: options.writer ?? "hwlab-tasktree-worker", + counts: snapshotCounts(snapshot), + }; +} + +export function parseSnapshot(text: string): TaskTreeSnapshot { + let value: unknown; + try { + value = JSON.parse(text); + } catch { + throw codedError("backup_snapshot_invalid_json", "backup snapshot is not valid JSON"); + } + return validateSnapshot(value); +} + +export function parseManifest(text: string): TaskTreeBackupManifest { + let value: unknown; + try { + value = JSON.parse(text); + } catch { + throw codedError("backup_manifest_invalid_json", "backup manifest is not valid JSON"); + } + if (!isRecord(value) || value.schemaVersion !== TASKTREE_BACKUP_SCHEMA_VERSION) { + throw codedError("backup_manifest_schema_unsupported", "backup manifest schema version is unsupported"); + } + const counts = value.counts; + if (!isRecord(counts)) throw codedError("backup_manifest_invalid", "backup manifest counts are required"); + return { + schemaVersion: TASKTREE_BACKUP_SCHEMA_VERSION, + target: requiredText(value.target, "manifest.target"), + generatedAt: validIso(value.generatedAt, "manifest.generatedAt"), + contentSha256: validSha(value.contentSha256, "manifest.contentSha256"), + writer: requiredText(value.writer, "manifest.writer"), + counts: { + groups: validCount(counts.groups, "manifest.counts.groups"), + tasks: validCount(counts.tasks, "manifest.counts.tasks"), + milestones: validCount(counts.milestones, "manifest.counts.milestones"), + reports: validCount(counts.reports, "manifest.counts.reports"), + }, + }; +} + +export function verifySnapshotManifest(snapshot: TaskTreeSnapshot, manifest: TaskTreeBackupManifest): void { + if (snapshotSha256(snapshot) !== manifest.contentSha256) { + throw codedError("backup_content_sha_mismatch", "backup snapshot does not match the manifest SHA"); + } + if (JSON.stringify(snapshotCounts(snapshot)) !== JSON.stringify(manifest.counts)) { + throw codedError("backup_count_mismatch", "backup snapshot counts do not match the manifest"); + } +} + +export function restoreDiff(current: TaskTreeSnapshot, incoming: TaskTreeSnapshot): TaskTreeRestoreDiff { + return { + groups: entityDiff(current.groups, incoming.groups), + tasks: entityDiff(current.tasks, incoming.tasks), + milestones: entityDiff(current.milestones, incoming.milestones), + reports: entityDiff(current.reports, incoming.reports), + }; +} + +export function validateSnapshot(value: unknown): TaskTreeSnapshot { + if (!isRecord(value) || value.schemaVersion !== TASKTREE_BACKUP_SCHEMA_VERSION) { + throw codedError("backup_snapshot_schema_unsupported", "backup snapshot schema version is unsupported"); + } + if (!Array.isArray(value.groups) || !Array.isArray(value.tasks) || !Array.isArray(value.milestones) || !Array.isArray(value.reports)) { + throw codedError("backup_snapshot_invalid", "backup snapshot entity arrays are required"); + } + const snapshot = normalizeSnapshot(value as TaskTreeSnapshot); + uniqueIds(snapshot.groups, "group"); + uniqueIds(snapshot.tasks, "task"); + uniqueIds(snapshot.milestones, "milestone"); + uniqueIds(snapshot.reports, "report"); + const groupIds = new Set(snapshot.groups.map((item) => item.id)); + const taskIds = new Set(snapshot.tasks.map((item) => item.id)); + for (const task of snapshot.tasks) { + if (!groupIds.has(task.groupId)) throw codedError("backup_reference_invalid", `task ${task.id} references a missing taskgroup`); + if (task.parentId !== null && !taskIds.has(task.parentId)) throw codedError("backup_reference_invalid", `task ${task.id} references a missing parent`); + } + for (const milestone of snapshot.milestones) { + if (!groupIds.has(milestone.groupId)) throw codedError("backup_reference_invalid", `milestone ${milestone.id} references a missing taskgroup`); + if (milestone.taskId !== null && !taskIds.has(milestone.taskId)) throw codedError("backup_reference_invalid", `milestone ${milestone.id} references a missing task`); + } + for (const report of snapshot.reports) { + if (!taskIds.has(report.taskId)) throw codedError("backup_reference_invalid", `report ${report.id} references a missing task`); + } + assertAcyclic(snapshot.tasks); + return snapshot; +} + +function normalizeSnapshot(value: TaskTreeSnapshot): TaskTreeSnapshot { + return { + schemaVersion: TASKTREE_BACKUP_SCHEMA_VERSION, + groups: [...value.groups].sort(byId), + tasks: [...value.tasks].sort(byId), + milestones: [...value.milestones].sort(byId), + reports: [...value.reports].sort(byId), + }; +} + +function snapshotCounts(snapshot: TaskTreeSnapshot) { + return { + groups: snapshot.groups.length, + tasks: snapshot.tasks.length, + milestones: snapshot.milestones.length, + reports: snapshot.reports.length, + }; +} + +function entityDiff(current: T[], incoming: T[]) { + const before = new Map(current.map((item) => [item.id, JSON.stringify(item)])); + const after = new Map(incoming.map((item) => [item.id, JSON.stringify(item)])); + let create = 0; + let update = 0; + let remove = 0; + for (const [id, value] of after) { + if (!before.has(id)) create += 1; + else if (before.get(id) !== value) update += 1; + } + for (const id of before.keys()) if (!after.has(id)) remove += 1; + return { create, update, delete: remove }; +} + +function uniqueIds(items: Array<{ id: string }>, kind: string): void { + const ids = new Set(); + for (const item of items) { + if (!item.id || ids.has(item.id)) throw codedError("backup_snapshot_invalid", `${kind} IDs must be non-empty and unique`); + ids.add(item.id); + } +} + +function assertAcyclic(tasks: TaskItem[]): void { + const parents = new Map(tasks.map((item) => [item.id, item.parentId])); + for (const task of tasks) { + const seen = new Set(); + let cursor: string | null = task.id; + while (cursor !== null) { + if (seen.has(cursor)) throw codedError("backup_reference_cycle", `task ${task.id} has a cyclic parent chain`); + seen.add(cursor); + cursor = parents.get(cursor) ?? null; + } + } +} + +function byId(left: { id: string }, right: { id: string }): number { + return left.id.localeCompare(right.id); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function requiredText(value: unknown, field: string): string { + if (typeof value !== "string" || value.trim().length === 0) throw codedError("backup_contract_invalid", `${field} is required`); + return value; +} + +function validIso(value: unknown, field: string): string { + const date = new Date(String(value ?? "")); + if (Number.isNaN(date.valueOf())) throw codedError("backup_contract_invalid", `${field} must be an ISO timestamp`); + return date.toISOString(); +} + +function validSha(value: unknown, field: string): string { + const text = requiredText(value, field); + if (!/^[0-9a-f]{64}$/u.test(text)) throw codedError("backup_contract_invalid", `${field} must be a SHA-256 hex digest`); + return text; +} + +function validCount(value: unknown, field: string): number { + if (!Number.isInteger(value) || Number(value) < 0) throw codedError("backup_contract_invalid", `${field} must be a non-negative integer`); + return Number(value); +} + +function codedError(code: string, message: string): Error { + return Object.assign(new Error(message), { code }); +} diff --git a/internal/tasktree/backup-git.ts b/internal/tasktree/backup-git.ts new file mode 100644 index 00000000..18ba3f91 --- /dev/null +++ b/internal/tasktree/backup-git.ts @@ -0,0 +1,173 @@ +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; + +import { + canonicalSnapshot, + parseManifest, + parseSnapshot, + snapshotManifest, + verifySnapshotManifest, + type TaskTreeBackupManifest, + type TaskTreeSnapshot, +} from "./backup-contracts.ts"; + +export type TaskTreeGitBackupConfig = { + repoSshUrl: string; + branch: string; + basePath: string; + worktreePath: string; + target: string; + sshCommand: string; + authorName: string; + authorEmail: string; + commitMessagePrefix: string; +}; + +export type PublishedTaskTreeBackup = { + changed: boolean; + contentSha256: string; + gitCommit: string; + pushAttempts: number; + manifest: TaskTreeBackupManifest; +}; + +let gitQueue: Promise = Promise.resolve(); + +export class TaskTreeGitBackup { + constructor(readonly config: TaskTreeGitBackupConfig) {} + + async publish(snapshot: TaskTreeSnapshot, reason: string): Promise { + return enqueueGit(async () => { + this.ensureWorktree(); + const manifest = snapshotManifest(snapshot, { target: this.config.target }); + const current = this.readCurrent(false); + if (current?.manifest.contentSha256 === manifest.contentSha256) { + return { + changed: false, + contentSha256: manifest.contentSha256, + gitCommit: this.head(), + pushAttempts: 0, + manifest: current.manifest, + }; + } + const base = this.basePath(); + const snapshotPath = join(this.config.worktreePath, base, "snapshot.json"); + const manifestPath = join(this.config.worktreePath, base, "manifest.json"); + mkdirSync(dirname(snapshotPath), { recursive: true }); + writeFileSync(snapshotPath, canonicalSnapshot(snapshot), "utf8"); + writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8"); + this.git(["add", base]); + const diff = this.git(["diff", "--cached", "--quiet"], true); + if (diff.exitCode === 0) { + return { + changed: false, + contentSha256: manifest.contentSha256, + gitCommit: this.head(), + pushAttempts: 0, + manifest, + }; + } + const env = { + GIT_AUTHOR_NAME: this.config.authorName, + GIT_AUTHOR_EMAIL: this.config.authorEmail, + GIT_COMMITTER_NAME: this.config.authorName, + GIT_COMMITTER_EMAIL: this.config.authorEmail, + }; + this.git(["commit", "-m", `${this.config.commitMessagePrefix} ${reason}`], false, env); + let pushAttempts = 0; + for (let attempt = 1; attempt <= 3; attempt += 1) { + pushAttempts = attempt; + if (this.git(["push", "origin", this.config.branch], true).ok) { + return { + changed: true, + contentSha256: manifest.contentSha256, + gitCommit: this.head(), + pushAttempts, + manifest, + }; + } + if (!this.git(["fetch", "origin", this.config.branch], true).ok) continue; + const rebase = this.git(["rebase", `origin/${this.config.branch}`], true); + if (!rebase.ok) { + this.git(["rebase", "--abort"], true); + throw codedError("backup_git_rebase_failed", "TaskTree backup Git rebase failed"); + } + } + throw codedError("backup_git_push_failed", "TaskTree backup Git push failed after retries"); + }); + } + + readCurrent(sync = true): { snapshot: TaskTreeSnapshot; manifest: TaskTreeBackupManifest; gitCommit: string } | null { + if (sync) this.ensureWorktree(); + const base = this.basePath(); + const snapshotPath = join(this.config.worktreePath, base, "snapshot.json"); + const manifestPath = join(this.config.worktreePath, base, "manifest.json"); + if (!existsSync(snapshotPath) || !existsSync(manifestPath)) return null; + const snapshot = parseSnapshot(readFileSync(snapshotPath, "utf8")); + const manifest = parseManifest(readFileSync(manifestPath, "utf8")); + if (manifest.target !== this.config.target) throw codedError("backup_target_mismatch", "backup manifest target does not match runtime target"); + verifySnapshotManifest(snapshot, manifest); + return { snapshot, manifest, gitCommit: this.head() }; + } + + private ensureWorktree(): void { + if (!this.config.repoSshUrl) throw codedError("backup_repo_missing", "TaskTree backup Git repository is not configured"); + mkdirSync(dirname(this.config.worktreePath), { recursive: true }); + if (!existsSync(join(this.config.worktreePath, ".git"))) { + if (existsSync(this.config.worktreePath)) rmSync(this.config.worktreePath, { recursive: true, force: true }); + this.git(["clone", this.config.repoSshUrl, this.config.worktreePath], false, {}, dirname(this.config.worktreePath)); + } + this.git(["remote", "set-url", "origin", this.config.repoSshUrl]); + const fetched = this.git(["fetch", "origin", this.config.branch], true); + if (fetched.ok) { + this.git(["checkout", "-B", this.config.branch, `origin/${this.config.branch}`]); + this.git(["pull", "--ff-only", "origin", this.config.branch]); + return; + } + if (this.git(["rev-parse", "--verify", "HEAD"], true).ok) this.git(["checkout", "-B", this.config.branch]); + else this.git(["symbolic-ref", "HEAD", `refs/heads/${this.config.branch}`]); + } + + private basePath(): string { + const raw = this.config.basePath.trim().replace(/^\/+/u, "").replace(/\/+$/u, ""); + if (!raw || raw === "." || raw.split("/").some((part) => part === "..")) { + throw codedError("backup_base_path_invalid", "TaskTree backup base path must be a non-root relative path"); + } + return raw; + } + + private head(): string { + return this.git(["rev-parse", "HEAD"]).stdout.trim(); + } + + private git( + args: string[], + allowFailure = false, + extraEnv: Record = {}, + cwd = this.config.worktreePath, + ): { ok: boolean; exitCode: number; stdout: string; stderr: string } { + const env = { ...process.env, ...extraEnv }; + if (this.config.sshCommand) env.GIT_SSH_COMMAND = this.config.sshCommand; + const result = Bun.spawnSync(["git", ...args], { cwd, env, stdout: "pipe", stderr: "pipe" }); + const output = { + ok: result.exitCode === 0, + exitCode: result.exitCode, + stdout: new TextDecoder().decode(result.stdout), + stderr: new TextDecoder().decode(result.stderr), + }; + if (!output.ok && !allowFailure) { + throw codedError("backup_git_command_failed", `TaskTree backup Git command failed: git ${args.join(" ")}: ${output.stderr.slice(-1000)}`); + } + return output; + } +} + +function enqueueGit(operation: () => Promise): Promise { + const next = gitQueue.then(operation, operation); + gitQueue = next.catch(() => undefined); + return next; +} + +function codedError(code: string, message: string): Error { + return Object.assign(new Error(message), { code }); +} diff --git a/internal/tasktree/backup.ts b/internal/tasktree/backup.ts new file mode 100644 index 00000000..3901bdb0 --- /dev/null +++ b/internal/tasktree/backup.ts @@ -0,0 +1,172 @@ +import { + restoreDiff, + snapshotManifest, + snapshotSha256, + type TaskTreeBackupState, +} from "./backup-contracts.ts"; +import { TaskTreeGitBackup, type TaskTreeGitBackupConfig } from "./backup-git.ts"; +import { TaskTreeStore } from "./store.ts"; + +export type TaskTreeBackupConfig = TaskTreeGitBackupConfig & { + enabled: boolean; + intervalMs: number; +}; + +export class TaskTreeBackupService { + private readonly git: TaskTreeGitBackup; + + constructor( + private readonly store: TaskTreeStore, + readonly config: TaskTreeBackupConfig, + ) { + this.git = new TaskTreeGitBackup(config); + } + + async request(reason: string): Promise { + await this.store.requestBackup(this.config.target, reason); + } + + async create(options: { dryRun?: boolean; reason?: string } = {}) { + const snapshot = await this.store.exportSnapshot(); + const manifest = snapshotManifest(snapshot, { target: this.config.target }); + if (options.dryRun) return { dryRun: true, changed: false, manifest }; + this.requireEnabled(); + await this.store.beginBackupAttempt(this.config.target); + try { + const published = await this.git.publish(snapshot, options.reason ?? "manual backup"); + await this.store.finishBackupSuccess(this.config.target, published.contentSha256, published.gitCommit); + return { dryRun: false, ...published }; + } catch (error: any) { + await this.store.finishBackupFailure( + this.config.target, + error?.code ?? "backup_failed", + error?.message ?? String(error), + ); + throw error; + } + } + + async status(): Promise<{ enabled: boolean; configured: boolean; state: TaskTreeBackupState }> { + return { + enabled: this.config.enabled, + configured: Boolean(this.config.repoSshUrl), + state: await this.store.backupState(this.config.target), + }; + } + + async verify() { + this.requireEnabled(); + const current = await this.store.exportSnapshot(); + const stored = this.git.readCurrent(); + if (stored === null) throw codedError("backup_snapshot_missing", "TaskTree backup snapshot does not exist"); + const databaseSha256 = snapshotSha256(current); + return { + verified: databaseSha256 === stored.manifest.contentSha256, + target: this.config.target, + databaseSha256, + backupSha256: stored.manifest.contentSha256, + gitCommit: stored.gitCommit, + counts: stored.manifest.counts, + }; + } + + async restore(options: { confirm: boolean; expectedSha256?: string }) { + this.requireEnabled(); + const stored = this.git.readCurrent(); + if (stored === null) throw codedError("backup_snapshot_missing", "TaskTree backup snapshot does not exist"); + const current = await this.store.exportSnapshot(); + const diff = restoreDiff(current, stored.snapshot); + if (!options.confirm) { + return { + dryRun: true, + restored: false, + target: this.config.target, + contentSha256: stored.manifest.contentSha256, + gitCommit: stored.gitCommit, + diff, + }; + } + if (!options.expectedSha256 || options.expectedSha256 !== stored.manifest.contentSha256) { + throw codedError("backup_restore_sha_required", "confirmed restore requires the exact current backup SHA"); + } + await this.store.restoreSnapshot(stored.snapshot); + await this.request("post-restore verification backup"); + return { + dryRun: false, + restored: true, + target: this.config.target, + contentSha256: stored.manifest.contentSha256, + gitCommit: stored.gitCommit, + diff, + }; + } + + private requireEnabled(): void { + if (!this.config.enabled) throw codedError("backup_disabled", "TaskTree GitHub backup is disabled"); + if (!this.config.repoSshUrl) throw codedError("backup_repo_missing", "TaskTree backup Git repository is not configured"); + } +} + +export function taskTreeBackupConfigFromEnv( + env: Record = process.env, +): TaskTreeBackupConfig { + return { + enabled: env.TASKTREE_BACKUP_ENABLED === "true", + repoSshUrl: env.TASKTREE_BACKUP_REPO_SSH_URL ?? "", + branch: env.TASKTREE_BACKUP_BRANCH ?? "main", + basePath: env.TASKTREE_BACKUP_BASE_PATH ?? "tasktree/development", + worktreePath: env.TASKTREE_BACKUP_WORKTREE ?? ".state/tasktree/backup-repo", + target: env.TASKTREE_BACKUP_TARGET ?? "development", + intervalMs: positiveInteger(env.TASKTREE_BACKUP_INTERVAL_MS, 300_000), + sshCommand: env.TASKTREE_BACKUP_GIT_SSH_COMMAND ?? "", + authorName: env.TASKTREE_BACKUP_GIT_AUTHOR_NAME ?? "HWLAB TaskTree", + authorEmail: env.TASKTREE_BACKUP_GIT_AUTHOR_EMAIL ?? "tasktree@hwlab.local", + commitMessagePrefix: env.TASKTREE_BACKUP_GIT_COMMIT_PREFIX ?? "tasktree:", + }; +} + +export function startTaskTreeBackupScheduler( + service: TaskTreeBackupService, + onResult: (result: Record) => void = () => {}, +): { stop: () => Promise; runNow: () => Promise } { + let current: Promise | null = null; + const runNow = () => { + if (!service.config.enabled) return Promise.resolve(); + if (current) return current; + current = (async () => { + try { + const result = await service.create({ reason: "automatic worker snapshot" }); + onResult({ ok: true, event: "tasktree_backup_succeeded", ...result }); + } catch (error: any) { + onResult({ + ok: false, + event: "tasktree_backup_failed", + code: error?.code ?? "backup_failed", + message: error?.message ?? String(error), + }); + } + })().finally(() => { + current = null; + }); + return current; + }; + const timer = setInterval(() => void runNow(), service.config.intervalMs); + timer.unref(); + void runNow(); + return { + async stop() { + clearInterval(timer); + await current; + }, + runNow, + }; +} + +function positiveInteger(value: string | undefined, fallback: number): number { + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +} + +function codedError(code: string, message: string): Error { + return Object.assign(new Error(message), { code }); +} diff --git a/internal/tasktree/contracts.ts b/internal/tasktree/contracts.ts index e7d16887..70c0fe1f 100644 --- a/internal/tasktree/contracts.ts +++ b/internal/tasktree/contracts.ts @@ -82,6 +82,10 @@ export type TaskTreeCommand = | { operation: "report.write"; taskId: string; title: string; body: string; status?: string } | { operation: "report.create"; taskId: string; title: string; body: string; status?: string } | { operation: "timeline.get"; groupId: string } + | { operation: "backup.create"; dryRun?: boolean; reason?: string } + | { operation: "backup.status" } + | { operation: "backup.verify" } + | { operation: "backup.restore"; confirm?: boolean; expectedSha256?: string } | { operation: "workflow.start"; taskId: string }; export type TaskTreeResult = { diff --git a/internal/tasktree/dispatcher.ts b/internal/tasktree/dispatcher.ts index 25bc7e17..579a49fe 100644 --- a/internal/tasktree/dispatcher.ts +++ b/internal/tasktree/dispatcher.ts @@ -1,11 +1,13 @@ import { Client, Connection } from "@temporalio/client"; import type { TaskItem, TaskTreeCommand, TaskTreeResult } from "./contracts.ts"; +import type { TaskTreeBackupService } from "./backup.ts"; import { parseMdtodoImport } from "./mdtodo-import.ts"; import { TaskTreeStore } from "./store.ts"; export type TaskTreeDispatcherOptions = { store: TaskTreeStore; + backup?: TaskTreeBackupService; temporalAddress?: string; temporalNamespace?: string; taskQueue?: string; @@ -17,7 +19,10 @@ export function createTaskTreeDispatcher(options: TaskTreeDispatcherOptions) { try { const operation = command.operation; let data: unknown; - if (operation === "health") data = await store.health(); + if (operation === "health") data = { + ...await store.health(), + ...(options.backup ? { backup: await options.backup.status() } : {}), + }; else if (operation === "group.list") data = { groups: await store.listGroups() }; else if (operation === "group.overview") data = { groups: await store.groupOverview() }; else if (operation === "group.get") data = required(await store.getGroup(command.groupId), "group_not_found", "taskgroup was not found"); @@ -92,6 +97,16 @@ export function createTaskTreeDispatcher(options: TaskTreeDispatcherOptions) { else if (operation === "report.write") data = await store.writeReport({ ...command, title: requiredText(command.title, "title"), body: requiredText(command.body, "body") }); else if (operation === "report.create") data = await store.createReport({ ...command, title: requiredText(command.title, "title"), body: requiredText(command.body, "body") }); else if (operation === "timeline.get") data = required(await store.timeline(command.groupId), "group_not_found", "taskgroup was not found"); + else if (operation === "backup.create") data = await requiredBackup(options.backup).create({ + dryRun: command.dryRun, + reason: command.reason, + }); + else if (operation === "backup.status") data = await requiredBackup(options.backup).status(); + else if (operation === "backup.verify") data = await requiredBackup(options.backup).verify(); + else if (operation === "backup.restore") data = await requiredBackup(options.backup).restore({ + confirm: command.confirm === true, + expectedSha256: command.expectedSha256, + }); else if (operation === "workflow.start") data = await startTaskWorkflow(options, command.taskId); else throw codedError("unsupported_operation", `unsupported operation: ${String(operation)}`); return { ok: true, operation, data }; @@ -115,6 +130,10 @@ async function startTaskWorkflow(options: TaskTreeDispatcherOptions, taskId: str } function required(value: T | null, code: string, message: string): T { if (value === null) throw codedError(code, message); return value; } +function requiredBackup(value: TaskTreeBackupService | undefined): TaskTreeBackupService { + if (!value) throw codedError("backup_unavailable", "TaskTree backup service is unavailable"); + return value; +} function requiredMutation(value: boolean, code: string, message: string): true { if (!value) throw codedError(code, message); return true; } function requiredText(value: unknown, field: string): string { const text = String(value ?? "").trim(); if (!text) throw codedError("invalid_input", `${field} is required`); return text; } function validTitle(value: unknown): string { diff --git a/internal/tasktree/runtime.ts b/internal/tasktree/runtime.ts index 744081f9..07eb0e0d 100644 --- a/internal/tasktree/runtime.ts +++ b/internal/tasktree/runtime.ts @@ -1,11 +1,14 @@ import { TaskTreeStore } from "./store.ts"; +import { TaskTreeBackupService, taskTreeBackupConfigFromEnv } from "./backup.ts"; export function taskTreeRuntime(env: Record = process.env) { const databaseUrl = env.TASKTREE_DATABASE_URL || env.DATABASE_URL || ""; if (!databaseUrl) throw Object.assign(new Error("TASKTREE_DATABASE_URL or DATABASE_URL is required"), { code: "missing_database_url" }); const store = new TaskTreeStore(databaseUrl); + const backup = new TaskTreeBackupService(store, taskTreeBackupConfigFromEnv(env)); return { store, + backup, temporalAddress: env.TASKTREE_TEMPORAL_ADDRESS || env.TEMPORAL_ADDRESS || "", temporalNamespace: env.TASKTREE_TEMPORAL_NAMESPACE || "unidesk", taskQueue: env.TASKTREE_TEMPORAL_TASK_QUEUE || "hwlab-v03-tasktree" diff --git a/internal/tasktree/store.ts b/internal/tasktree/store.ts index 7c6222c1..81dce39c 100644 --- a/internal/tasktree/store.ts +++ b/internal/tasktree/store.ts @@ -2,6 +2,11 @@ import { randomUUID } from "node:crypto"; import pg from "pg"; import type { ExecutionReport, Milestone, TaskGroup, TaskGroupOverview, TaskItem, TaskStatus, Timeline } from "./contracts.ts"; +import { + TASKTREE_BACKUP_SCHEMA_VERSION, + type TaskTreeBackupState, + type TaskTreeSnapshot, +} from "./backup-contracts.ts"; import type { MdtodoImportPlan } from "./mdtodo-import.ts"; const { Pool } = pg; @@ -30,6 +35,14 @@ const schema = [ `CREATE INDEX IF NOT EXISTS idx_tasktree_tasks_group_order ON tasktree_tasks(group_id, sort_order, created_at)`, `CREATE INDEX IF NOT EXISTS idx_tasktree_tasks_parent ON tasktree_tasks(parent_id)`, `CREATE INDEX IF NOT EXISTS idx_tasktree_reports_task ON tasktree_execution_reports(task_id, created_at DESC)` + ,`CREATE TABLE IF NOT EXISTS tasktree_backup_state ( + target TEXT PRIMARY KEY, pending BOOLEAN NOT NULL DEFAULT false, + requested_at TIMESTAMPTZ, request_reason TEXT NOT NULL DEFAULT '', + last_attempt_at TIMESTAMPTZ, last_success_at TIMESTAMPTZ, + content_sha256 TEXT NOT NULL DEFAULT '', git_commit TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'never' CHECK (status IN ('never','pending','succeeded','failed')), + error_code TEXT NOT NULL DEFAULT '', error_message TEXT NOT NULL DEFAULT '', + updated_at TIMESTAMPTZ NOT NULL DEFAULT now())` ,`ALTER TABLE tasktree_tasks DROP CONSTRAINT IF EXISTS tasktree_tasks_kind_check` ,`ALTER TABLE tasktree_tasks ADD CONSTRAINT tasktree_tasks_kind_check CHECK (kind IN ('task','subtask','subsubtask'))` ]; @@ -49,7 +62,7 @@ export class TaskTreeStore { try { await client.query("BEGIN"); for (const statement of schema) await client.query(statement); - await client.query("INSERT INTO tasktree_schema_migrations (migration_id) VALUES ($1),($2) ON CONFLICT DO NOTHING", ["tasktree-20260716-v1", "tasktree-20260717-v2-subsubtask"]); + await client.query("INSERT INTO tasktree_schema_migrations (migration_id) VALUES ($1),($2),($3) ON CONFLICT DO NOTHING", ["tasktree-20260716-v1", "tasktree-20260717-v2-subsubtask", "tasktree-20260718-v3-github-backup"]); await client.query("COMMIT"); this.ready = true; } catch (error) { @@ -366,6 +379,140 @@ export class TaskTreeStore { return { group, tasks: tasks.rows.map(taskRow), milestones: milestones.rows.map(milestoneRow), reports: reports.rows.map(reportRow) }; } + async exportSnapshot(): Promise { + await this.ensureSchema(); + const client = await this.pool.connect(); + try { + await client.query("BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY"); + const groups = await client.query("SELECT * FROM tasktree_groups ORDER BY id"); + const tasks = await client.query("SELECT * FROM tasktree_tasks ORDER BY id"); + const milestones = await client.query("SELECT * FROM tasktree_milestones ORDER BY id"); + const reports = await client.query("SELECT * FROM tasktree_execution_reports ORDER BY id"); + await client.query("COMMIT"); + return { + schemaVersion: TASKTREE_BACKUP_SCHEMA_VERSION, + groups: groups.rows.map(groupRow), + tasks: tasks.rows.map(taskRow), + milestones: milestones.rows.map(milestoneRow), + reports: reports.rows.map(reportRow), + }; + } catch (error) { + await client.query("ROLLBACK").catch(() => {}); + throw error; + } finally { + client.release(); + } + } + + async restoreSnapshot(snapshot: TaskTreeSnapshot): Promise { + await this.ensureSchema(); + const client = await this.pool.connect(); + try { + await client.query("BEGIN"); + await client.query("DELETE FROM tasktree_groups"); + for (const group of snapshot.groups) { + await client.query( + "INSERT INTO tasktree_groups (id,name,description,created_at,updated_at) VALUES ($1,$2,$3,$4,$5)", + [group.id, group.name, group.description, group.createdAt, group.updatedAt], + ); + } + const pending = new Map(snapshot.tasks.map((task) => [task.id, task])); + while (pending.size > 0) { + let inserted = 0; + for (const [id, task] of pending) { + if (task.parentId !== null && pending.has(task.parentId)) continue; + await client.query( + `INSERT INTO tasktree_tasks + (id,group_id,parent_id,kind,title,description,status,start_at,due_at,sort_order,created_at,updated_at) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)`, + [task.id, task.groupId, task.parentId, task.kind, task.title, task.description, task.status, + task.startAt, task.dueAt, task.sortOrder, task.createdAt, task.updatedAt], + ); + pending.delete(id); + inserted += 1; + } + if (inserted === 0) throw domainError("backup_reference_cycle", "backup snapshot contains a cyclic task hierarchy"); + } + for (const milestone of snapshot.milestones) { + await client.query( + "INSERT INTO tasktree_milestones (id,group_id,task_id,title,occurs_at,created_at) VALUES ($1,$2,$3,$4,$5,$6)", + [milestone.id, milestone.groupId, milestone.taskId, milestone.title, milestone.occursAt, milestone.createdAt], + ); + } + for (const report of snapshot.reports) { + await client.query( + "INSERT INTO tasktree_execution_reports (id,task_id,title,body,status,created_at) VALUES ($1,$2,$3,$4,$5,$6)", + [report.id, report.taskId, report.title, report.body, report.status, report.createdAt], + ); + } + await client.query("COMMIT"); + } catch (error) { + await client.query("ROLLBACK").catch(() => {}); + throw error; + } finally { + client.release(); + } + } + + async requestBackup(target: string, reason: string): Promise { + await this.ensureSchema(); + await this.pool.query( + `INSERT INTO tasktree_backup_state (target,pending,requested_at,request_reason,status) + VALUES ($1,true,now(),$2,'pending') + ON CONFLICT (target) DO UPDATE SET + pending=true,requested_at=now(),request_reason=excluded.request_reason,status='pending',updated_at=now()`, + [target, reason], + ); + } + + async beginBackupAttempt(target: string): Promise { + await this.ensureSchema(); + await this.pool.query( + `INSERT INTO tasktree_backup_state (target,last_attempt_at,status) + VALUES ($1,now(),'pending') + ON CONFLICT (target) DO UPDATE SET last_attempt_at=now(),status='pending',updated_at=now()`, + [target], + ); + } + + async finishBackupSuccess(target: string, contentSha256: string, gitCommit: string): Promise { + await this.ensureSchema(); + await this.pool.query( + `INSERT INTO tasktree_backup_state + (target,pending,last_attempt_at,last_success_at,content_sha256,git_commit,status,error_code,error_message) + VALUES ($1,false,now(),now(),$2,$3,'succeeded','','') + ON CONFLICT (target) DO UPDATE SET + pending=false,last_success_at=now(),content_sha256=excluded.content_sha256, + git_commit=excluded.git_commit,status='succeeded',error_code='',error_message='',updated_at=now()`, + [target, contentSha256, gitCommit], + ); + } + + async finishBackupFailure(target: string, code: string, message: string): Promise { + await this.ensureSchema(); + await this.pool.query( + `INSERT INTO tasktree_backup_state + (target,pending,last_attempt_at,status,error_code,error_message) + VALUES ($1,true,now(),'failed',$2,$3) + ON CONFLICT (target) DO UPDATE SET + pending=true,status='failed',error_code=excluded.error_code,error_message=excluded.error_message,updated_at=now()`, + [target, code, message.slice(0, 2000)], + ); + } + + async backupState(target: string): Promise { + await this.ensureSchema(); + const result = await this.pool.query("SELECT * FROM tasktree_backup_state WHERE target=$1", [target]); + if (!result.rows[0]) { + const created = await this.pool.query( + "INSERT INTO tasktree_backup_state (target) VALUES ($1) ON CONFLICT (target) DO UPDATE SET target=excluded.target RETURNING *", + [target], + ); + return backupStateRow(created.rows[0]); + } + return backupStateRow(result.rows[0]); + } + async close() { await this.pool.end(); } } @@ -374,6 +521,22 @@ function overviewRow(row: any): TaskGroupOverview { return { group: groupRow(row function taskRow(row: any): TaskItem { return { id: row.id, groupId: row.group_id, parentId: row.parent_id, kind: row.kind, title: row.title, description: row.description, status: row.status, startAt: nullableIso(row.start_at), dueAt: nullableIso(row.due_at), sortOrder: row.sort_order, createdAt: iso(row.created_at), updatedAt: iso(row.updated_at) }; } function milestoneRow(row: any): Milestone { return { id: row.id, groupId: row.group_id, taskId: row.task_id, title: row.title, occursAt: iso(row.occurs_at), createdAt: iso(row.created_at) }; } function reportRow(row: any): ExecutionReport { return { id: row.id, taskId: row.task_id, title: row.title, body: row.body, status: row.status, createdAt: iso(row.created_at) }; } +function backupStateRow(row: any): TaskTreeBackupState { + return { + target: row.target, + pending: row.pending, + requestedAt: nullableIso(row.requested_at), + requestReason: row.request_reason, + lastAttemptAt: nullableIso(row.last_attempt_at), + lastSuccessAt: nullableIso(row.last_success_at), + contentSha256: row.content_sha256, + gitCommit: row.git_commit, + status: row.status, + errorCode: row.error_code, + errorMessage: row.error_message, + updatedAt: iso(row.updated_at), + }; +} function iso(value: unknown): string { return new Date(value as any).toISOString(); } function nullableIso(value: unknown): string | null { return value == null ? null : iso(value); } function domainError(code: string, message: string) { return Object.assign(new Error(message), { code }); } diff --git a/package.json b/package.json index 83485908..333d08c4 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,8 @@ "harnessrl:worker:dev": "bun --watch cmd/hwlab-harnessrl-worker/main.ts", "harnessrl:native:smoke": "bun test internal/harnessrl/harnessrl.test.ts internal/cloud/server-caserun-http.test.ts", "harnessrl:native:l1-smoke": "node scripts/harnessrl-native-l1-smoke.mjs", + "tasktree:backup:l1-smoke": "bun scripts/tasktree-backup-native-l1-smoke.mjs", + "tasktree:backup:restore:l1-smoke": "bun scripts/tasktree-backup-restore-l1-smoke.mjs", "workbench:api:dev": "bun --watch cmd/hwlab-workbench-api/main.ts", "workbench:worker:dev": "bun --watch cmd/hwlab-workbench-worker/main.ts", "workbench:web:dev": "cd web/hwlab-cloud-web && WORKBENCH_VITE_USE_POLLING=1 bun run dev", diff --git a/scripts/tasktree-backup-native-l1-smoke.mjs b/scripts/tasktree-backup-native-l1-smoke.mjs new file mode 100644 index 00000000..95cb352f --- /dev/null +++ b/scripts/tasktree-backup-native-l1-smoke.mjs @@ -0,0 +1,203 @@ +#!/usr/bin/env bun +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +import { runTaskTreeCli } from "../tools/src/tasktree-cli.ts"; + +const repoRoot = resolve(import.meta.dir, ".."); +const databaseEnvFile = requiredEnv("TASKTREE_NATIVE_DATABASE_ENV_FILE"); +const databaseUrl = readEnvValue(databaseEnvFile, "DATABASE_URL"); +const temporalAddress = requiredEnv("TASKTREE_TEMPORAL_ADDRESS"); +const stateDir = mkdtempSync(join(tmpdir(), "tasktree-backup-l1-")); +const apiPort = positiveInteger(process.env.TASKTREE_API_PORT, 6673); +const workerPort = positiveInteger(process.env.TASKTREE_WORKER_HEALTH_PORT, 6674); +const apiUrl = `http://127.0.0.1:${apiPort}`; +const commonEnv = { + ...process.env, + TASKTREE_DATABASE_URL: databaseUrl, + TASKTREE_TEMPORAL_ADDRESS: temporalAddress, + TASKTREE_TEMPORAL_NAMESPACE: process.env.TASKTREE_TEMPORAL_NAMESPACE ?? "unidesk", + TASKTREE_TEMPORAL_TASK_QUEUE: process.env.TASKTREE_TEMPORAL_TASK_QUEUE ?? "hwlab-v03-tasktree", + TASKTREE_API_HOST: "127.0.0.1", + TASKTREE_API_PORT: String(apiPort), + TASKTREE_WORKER_HEALTH_HOST: "127.0.0.1", + TASKTREE_WORKER_HEALTH_PORT: String(workerPort), + TASKTREE_BACKUP_ENABLED: "true", + TASKTREE_BACKUP_WORKTREE: join(stateDir, "repo"), +}; +const cliEnv = { ...commonEnv, HWLAB_TASKTREE_API_URL: apiUrl }; +const children = []; +let smokeGroupId = ""; +let childrenStopped = false; +let readiness = {}; + +try { + event("children-starting"); + children.push(spawn("cmd/hwlab-tasktree-worker/main.ts")); + children.push(spawn("cmd/hwlab-tasktree-api/main.ts")); + await waitUntil(async () => { + const [api, worker] = await Promise.all([ + health(`${apiUrl}/health/ready`), + health(`http://127.0.0.1:${workerPort}/health/ready`), + ]); + readiness = { api, worker }; + return api.ok && worker.ok; + }, 30_000, () => `TaskTree native API and worker did not become ready: ${JSON.stringify(readiness)}`); + event("children-ready"); + + const initialStatus = await waitForBackupSuccess(); + event("initial-backup-ready"); + const initialSha = initialStatus.state.contentSha256; + const initialCommit = initialStatus.state.gitCommit; + const verify = successful(await runTaskTreeCli(["backup", "verify", "--overapi"], cliEnv)); + if (verify.verified !== true) throw new Error("TaskTree --overapi backup verify did not match PostgreSQL"); + const restore = successful(await runTaskTreeCli(["backup", "restore", "--overapi"], cliEnv)); + if (restore.dryRun !== true || restore.restored !== false) throw new Error("TaskTree restore did not default to dry-run"); + event("overapi-verified"); + + const group = successful(await runTaskTreeCli( + ["group", "create", "--name", `Backup L1 smoke ${Date.now()}`], + cliEnv, + )); + smokeGroupId = group.id; + event("smoke-group-created"); + const changedStatus = await waitUntil(async () => { + const status = successful(await runTaskTreeCli(["backup", "status", "--overapi"], cliEnv)); + return status.state.status === "succeeded" && status.state.contentSha256 !== initialSha ? status : null; + }, 30_000, "automatic backup did not capture the smoke mutation"); + const changedSha = changedStatus.state.contentSha256; + event("smoke-mutation-backed-up"); + + successful(await runTaskTreeCli(["group", "delete", "--group", smokeGroupId], cliEnv)); + smokeGroupId = ""; + const restoredStatus = await waitUntil(async () => { + const status = successful(await runTaskTreeCli(["backup", "status", "--overapi"], cliEnv)); + return status.state.status === "succeeded" && status.state.contentSha256 === initialSha ? status : null; + }, 30_000, "automatic backup did not capture smoke cleanup"); + event("smoke-cleanup-backed-up"); + + console.log(JSON.stringify({ + ok: true, + action: "tasktree-backup-native-l1-smoke", + mutation: true, + l1: { + apiReady: true, + workerReady: true, + temporalConnected: true, + automaticBackup: true, + overapiVerify: true, + restoreDryRun: true, + }, + initialSha, + initialCommit, + changedSha, + restoredSha: restoredStatus.state.contentSha256, + valuesPrinted: false, + }, null, 2)); +} catch (error) { + await stopChildren(); + const diagnostics = []; + for (const child of children) { + diagnostics.push({ + pid: child.pid, + exitCode: await child.exited, + }); + } + process.stderr.write(`${JSON.stringify({ + ok: false, + action: "tasktree-backup-native-l1-smoke", + error: error instanceof Error ? error.message : String(error), + readiness, + diagnostics, + valuesPrinted: false, + }, null, 2)}\n`); + throw error; +} finally { + if (smokeGroupId) { + await runTaskTreeCli(["group", "delete", "--group", smokeGroupId], cliEnv).catch(() => undefined); + } + await stopChildren(); + rmSync(stateDir, { recursive: true, force: true }); +} + +function spawn(entrypoint) { + const env = entrypoint.includes("api") + ? { ...commonEnv, TASKTREE_BACKUP_WORKTREE: `${commonEnv.TASKTREE_BACKUP_WORKTREE}-api` } + : commonEnv; + return Bun.spawn([process.execPath, entrypoint], { + cwd: repoRoot, + env, + stdin: "ignore", + stdout: "inherit", + stderr: "inherit", + }); +} + +async function stopChildren() { + if (childrenStopped) return; + childrenStopped = true; + for (const child of children) child.kill("SIGTERM"); + await Promise.allSettled(children.map((child) => child.exited)); +} + +async function waitForBackupSuccess() { + return await waitUntil(async () => { + const status = successful(await runTaskTreeCli(["backup", "status", "--overapi"], cliEnv)); + return status.state.status === "succeeded" && status.state.contentSha256 ? status : null; + }, 40_000, "automatic TaskTree backup did not succeed"); +} + +async function waitUntil(check, timeoutMs, message) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const result = await check(); + if (result) return result; + await Bun.sleep(500); + } + throw new Error(typeof message === "function" ? message() : message); +} + +async function health(url) { + try { + const response = await fetch(url); + const body = await response.json().catch(() => null); + return { ok: response.ok && body?.ok === true, status: response.status, error: null }; + } catch (error) { + return { ok: false, status: null, error: error instanceof Error ? error.message : String(error) }; + } +} + +function successful(result) { + if (result?.ok !== true) { + const error = new Error(result?.error?.message ?? "TaskTree command failed"); + error.code = result?.error?.code ?? "tasktree_error"; + throw error; + } + return result.data; +} + +function readEnvValue(path, key) { + const line = readFileSync(path, "utf8") + .split(/\r?\n/u) + .find((candidate) => candidate.startsWith(`${key}=`)); + if (!line) throw new Error(`${key} is missing from the configured env source`); + const raw = line.slice(key.length + 1).trim(); + if ((raw.startsWith('"') && raw.endsWith('"')) || (raw.startsWith("'") && raw.endsWith("'"))) return raw.slice(1, -1); + return raw; +} + +function requiredEnv(name) { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} + +function positiveInteger(value, fallback) { + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +} + +function event(stage) { + process.stderr.write(`${JSON.stringify({ action: "tasktree-backup-native-l1-smoke", stage })}\n`); +} diff --git a/scripts/tasktree-backup-restore-l1-smoke.mjs b/scripts/tasktree-backup-restore-l1-smoke.mjs new file mode 100644 index 00000000..d3579658 --- /dev/null +++ b/scripts/tasktree-backup-restore-l1-smoke.mjs @@ -0,0 +1,113 @@ +#!/usr/bin/env bun +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import pg from "pg"; + +import { canonicalSnapshot } from "../internal/tasktree/backup-contracts.ts"; +import { TaskTreeBackupService } from "../internal/tasktree/backup.ts"; +import { TaskTreeStore } from "../internal/tasktree/store.ts"; + +const databaseUrl = readEnvValue(requiredEnv("TASKTREE_NATIVE_DATABASE_ENV_FILE"), "DATABASE_URL"); +const schema = `tasktree_backup_restore_${Date.now()}_${Math.random().toString(16).slice(2, 10)}`; +const control = new pg.Pool({ + connectionString: databaseUrl, + max: 1, + connectionTimeoutMillis: 5000, + ssl: { rejectUnauthorized: false }, +}); +const root = mkdtempSync(join(tmpdir(), "tasktree-backup-restore-l1-")); +let store; + +try { + await control.query(`CREATE SCHEMA "${schema}" AUTHORIZATION CURRENT_USER`); + const isolatedUrl = new URL(databaseUrl); + isolatedUrl.searchParams.set("options", `-c search_path=${schema}`); + store = new TaskTreeStore(isolatedUrl.toString()); + const backup = new TaskTreeBackupService(store, { + enabled: true, + repoSshUrl: join(root, "remote.git"), + branch: "main", + basePath: "tasktree/restore-smoke", + worktreePath: join(root, "repo"), + target: "restore-smoke", + intervalMs: 60_000, + sshCommand: "", + authorName: "TaskTree Restore Smoke", + authorEmail: "tasktree-restore@example.invalid", + commitMessagePrefix: "tasktree:", + }); + const init = Bun.spawnSync(["git", "init", "--bare", backup.config.repoSshUrl], { + stdout: "pipe", + stderr: "pipe", + }); + if (init.exitCode !== 0) throw new Error("failed to create disposable Git remote"); + + const group = await store.createGroup("Restore smoke", "Disposable L1 schema"); + const task = await store.createTask({ groupId: group.id, title: "Stable identity" }); + await store.writeReport({ taskId: task.id, title: "Execution", body: "Original report" }); + const original = await store.exportSnapshot(); + const published = await backup.create({ reason: "restore smoke baseline" }); + + await store.updateTask(task.id, { title: "Changed after backup" }); + await store.createTask({ groupId: group.id, title: "Created after backup" }); + const plan = await backup.restore({ confirm: false }); + if (plan.dryRun !== true || plan.diff.tasks.create !== 0 || plan.diff.tasks.update !== 1 || plan.diff.tasks.delete !== 1) { + throw new Error("restore dry-run diff did not identify the isolated mutations"); + } + await assertRejectsCode( + () => backup.restore({ confirm: true, expectedSha256: "0".repeat(64) }), + "backup_restore_sha_required", + ); + const restored = await backup.restore({ confirm: true, expectedSha256: published.contentSha256 }); + if (restored.restored !== true) throw new Error("confirmed restore did not report success"); + const after = await store.exportSnapshot(); + if (canonicalSnapshot(after) !== canonicalSnapshot(original)) { + throw new Error("confirmed restore did not preserve the canonical snapshot"); + } + await Bun.write(Bun.stdout, `${JSON.stringify({ + ok: true, + action: "tasktree-backup-restore-l1-smoke", + mutation: true, + storage: "native-postgresql-disposable-schema", + publishedSha: published.contentSha256, + stableGroupId: group.id, + stableTaskId: task.id, + dryRunDiff: plan.diff, + wrongShaRejected: true, + restored: true, + canonicalMatch: true, + valuesPrinted: false, + }, null, 2)}\n`); +} finally { + await store?.close().catch(() => undefined); + await control.query(`DROP SCHEMA IF EXISTS "${schema}" CASCADE`).catch(() => undefined); + await control.end().catch(() => undefined); + rmSync(root, { recursive: true, force: true }); +} + +async function assertRejectsCode(run, code) { + try { + await run(); + } catch (error) { + if (error?.code === code) return; + throw error; + } + throw new Error(`expected ${code}`); +} + +function readEnvValue(path, key) { + const line = readFileSync(path, "utf8") + .split(/\r?\n/u) + .find((candidate) => candidate.startsWith(`${key}=`)); + if (!line) throw new Error(`${key} is missing from the configured env source`); + const raw = line.slice(key.length + 1).trim(); + if ((raw.startsWith('"') && raw.endsWith('"')) || (raw.startsWith("'") && raw.endsWith("'"))) return raw.slice(1, -1); + return raw; +} + +function requiredEnv(name) { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} diff --git a/tools/src/tasktree-cli.test.ts b/tools/src/tasktree-cli.test.ts index e564a158..4df95e7d 100644 --- a/tools/src/tasktree-cli.test.ts +++ b/tools/src/tasktree-cli.test.ts @@ -54,7 +54,18 @@ test("TaskTree command surface covers hierarchy, lifecycle, batch, stats, and re [["report", "get", "--report", "tr_1", "--overapi"], { operation: "report.get", reportId: "tr_1" }], [["report", "write", "--task", "tt_1", "--title", "Execution", "--stdin", "--overapi"], { operation: "report.write", taskId: "tt_1", title: "Execution", body: "# Result\n\nPassed.\n" - }, "# Result\n\nPassed.\n"] + }, "# Result\n\nPassed.\n"], + [["backup", "create", "--dry-run", "--reason", "smoke", "--overapi"], { + operation: "backup.create", dryRun: true, reason: "smoke" + }], + [["backup", "status", "--overapi"], { operation: "backup.status" }], + [["backup", "verify", "--overapi"], { operation: "backup.verify" }], + [["backup", "restore", "--overapi"], { + operation: "backup.restore", confirm: false + }], + [["backup", "restore", "--confirm", "--expected-sha", "a".repeat(64), "--overapi"], { + operation: "backup.restore", confirm: true, expectedSha256: "a".repeat(64) + }] ]; for (const [argv, expected, stdin] of cases) { await runTaskTreeCli(argv, env, stdin); @@ -83,6 +94,14 @@ test("TaskTree rejects ambiguous title input and empty update patches before tra runTaskTreeCli(["task", "update", "--task", "tt_1", "--overapi"], { HWLAB_TASKTREE_API_URL: "http://tasktree.test" }), (error: any) => error?.code === "update_patch_required" ); + await assert.rejects( + runTaskTreeCli(["backup", "restore", "--confirm", "--overapi"], { HWLAB_TASKTREE_API_URL: "http://tasktree.test" }), + (error: any) => error?.code === "restore_confirmation_invalid" + ); + await assert.rejects( + runTaskTreeCli(["backup", "restore", "--expected-sha", "a".repeat(64), "--overapi"], { HWLAB_TASKTREE_API_URL: "http://tasktree.test" }), + (error: any) => error?.code === "restore_confirmation_invalid" + ); }); test("TaskTree rejects unknown, duplicate, hidden, and extra CLI arguments", async () => { diff --git a/tools/src/tasktree-cli.ts b/tools/src/tasktree-cli.ts index 39a76c75..5d7ffa07 100644 --- a/tools/src/tasktree-cli.ts +++ b/tools/src/tasktree-cli.ts @@ -40,6 +40,9 @@ function help() { "hwlab-cli tasktree report list|get --task ID|--report ID", "hwlab-cli tasktree report write --task ID --title TEXT (--body TEXT|--body-file PATH|--stdin) [--status STATUS]", "hwlab-cli tasktree timeline --group ID [--overapi]", + "hwlab-cli tasktree backup create [--dry-run] [--reason TEXT]", + "hwlab-cli tasktree backup status|verify", + "hwlab-cli tasktree backup restore [--confirm --expected-sha SHA]", "hwlab-cli tasktree workflow start --task ID" ], transportContract: "--overapi only changes transport; commands and options are identical", @@ -105,6 +108,18 @@ async function commandFrom(parsed: Parsed, stdin?: string): Promise = { "report list": { positionals: 2, options: ["task"] }, "report get": { positionals: 2, options: ["report"] }, "report write": { positionals: 2, options: ["task", "title", "body", "body-file", "status"], flags: ["stdin"] }, + "backup create": { positionals: 2, options: ["reason"], flags: ["dry-run"] }, + "backup status": { positionals: 2 }, + "backup verify": { positionals: 2 }, + "backup restore": { positionals: 2, options: ["expected-sha"], flags: ["confirm"] }, "workflow start": { positionals: 2, options: ["task"] } }; @@ -236,4 +255,11 @@ function validateArguments(parsed: Parsed) { for (const flag of parsed.flags) { if (!allowedFlags.has(flag)) throw codedError("invalid_option", `--${flag} is not valid for ${key}`); } + if (key === "backup restore") { + const confirm = parsed.flags.has("confirm"); + const expectedSha = parsed.values["expected-sha"]; + if (confirm !== (expectedSha !== undefined)) { + throw codedError("restore_confirmation_invalid", "--confirm and --expected-sha must be provided together"); + } + } } diff --git a/web/hwlab-cloud-web/scripts/tasktree-native-supervisor.ts b/web/hwlab-cloud-web/scripts/tasktree-native-supervisor.ts index 22393237..b167acab 100644 --- a/web/hwlab-cloud-web/scripts/tasktree-native-supervisor.ts +++ b/web/hwlab-cloud-web/scripts/tasktree-native-supervisor.ts @@ -30,10 +30,22 @@ const commonEnv = { TASKTREE_TEMPORAL_NAMESPACE: process.env.TASKTREE_TEMPORAL_NAMESPACE ?? "unidesk", TASKTREE_TEMPORAL_TASK_QUEUE: process.env.TASKTREE_TEMPORAL_TASK_QUEUE ?? "hwlab-v03-tasktree" }; +const backupWorktree = process.env.TASKTREE_BACKUP_WORKTREE; const children = [ Bun.spawn([process.execPath, "cmd/hwlab-tasktree-worker/main.ts"], { cwd: repoRoot, env: commonEnv, stdin: "ignore", stdout: logFd, stderr: logFd }), - Bun.spawn([process.execPath, "cmd/hwlab-tasktree-api/main.ts"], { cwd: repoRoot, env: { ...commonEnv, TASKTREE_API_HOST: "127.0.0.1", TASKTREE_API_PORT: "6673" }, stdin: "ignore", stdout: logFd, stderr: logFd }), + Bun.spawn([process.execPath, "cmd/hwlab-tasktree-api/main.ts"], { + cwd: repoRoot, + env: { + ...commonEnv, + ...(backupWorktree ? { TASKTREE_BACKUP_WORKTREE: `${backupWorktree}-api` } : {}), + TASKTREE_API_HOST: "127.0.0.1", + TASKTREE_API_PORT: "6673" + }, + stdin: "ignore", + stdout: logFd, + stderr: logFd + }), Bun.spawn([process.execPath, "x", "vite", "--config", "scripts/tasktree-native-vite.config.ts"], { cwd: webRoot, env: { ...process.env, HWLAB_TASKTREE_NATIVE_API_URL: "http://127.0.0.1:6673" }, stdin: "ignore", stdout: logFd, stderr: logFd }) ];