root
325 lines
20 KiB
TypeScript
325 lines
20 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { mkdtemp, mkdir, readFile, writeFile } from "node:fs/promises";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import { test } from "node:test";
|
|
|
|
import { createHarnessRLActivities } from "./activities.ts";
|
|
import type { ActivityResult, CaseRunEvent, CaseRunRecord, CaseRunStatus, HarnessRLRegistry, HarnessRLTemporalGateway, StartCaseRunInput } from "./contracts.ts";
|
|
import { terminalStatus } from "./contracts.ts";
|
|
import { createHarnessRLHttpApp } from "./http.ts";
|
|
import { createHarnessRLService } from "./service.ts";
|
|
import { recoverAgentTaskStage } from "../../tools/src/hwlab-caserun-runtime.ts";
|
|
|
|
test("real AgentRun recovery helper maps only result 404 to typed not-found", async () => {
|
|
const run: any = { caseId: "case-1", runId: "run-1", runDir: "/tmp/run-1", agentTask: { providerProfile: "default", projectId: "project-1", timeoutMs: 1000, pollIntervalMs: 250 } };
|
|
const context = (status: number): any => ({
|
|
parsed: { _: [], baseUrl: "https://hwlab.example.test", traceId: "trace-authority" },
|
|
env: { HWLAB_API_KEY: "hwl_live_test" },
|
|
fetchImpl: async () => new Response(JSON.stringify({ ok: false, status }), { status, headers: { "content-type": "application/json" } }),
|
|
cwd: "/tmp", now: () => "2026-07-17T03:00:00.000Z", sleep: async () => {}, rest: []
|
|
});
|
|
await assert.rejects(recoverAgentTaskStage(context(404), run), (error: any) => error?.code === "agentrun_task_not_found" && error?.status === 404);
|
|
const failed = await recoverAgentTaskStage(context(503), run);
|
|
assert.equal(failed.agent.stageStatus, "result_request_failed");
|
|
assert.equal(failed.agent.lastHttpStatus, 503);
|
|
});
|
|
|
|
test("API restart preserves PostgreSQL-shaped registry read model", async () => {
|
|
const fixture = await softwareSmokeFixture();
|
|
const registry = new MemoryRegistry();
|
|
const temporal = new FakeTemporal();
|
|
const first = createHarnessRLService({ registry, temporal, cwd: fixture.root, caseRepo: fixture.root, stateRoot: fixture.stateRoot });
|
|
const app = createHarnessRLHttpApp({ service: first });
|
|
const response = await app.fetch(new Request("http://harnessrl.test/v1/caserun/runs", {
|
|
method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ caseId: fixture.caseId, runId: "restart-run" })
|
|
}));
|
|
assert.equal(response.status, 202);
|
|
assert.equal(temporal.starts, 1);
|
|
|
|
const restarted = createHarnessRLService({ registry, temporal: new FakeTemporal(), cwd: fixture.root, caseRepo: fixture.root, stateRoot: fixture.stateRoot });
|
|
const record = await restarted.getRun("restart-run");
|
|
assert.equal(record.status, "queued");
|
|
assert.equal(record.workflowRunId, "temporal-run-1");
|
|
});
|
|
|
|
test("run history lists shared registry results newest first with an opaque cursor", async () => {
|
|
const fixture = await softwareSmokeFixture();
|
|
const registry = new MemoryRegistry();
|
|
const service = createHarnessRLService({ registry, temporal: new FakeTemporal(), cwd: fixture.root, caseRepo: fixture.root, stateRoot: fixture.stateRoot });
|
|
const app = createHarnessRLHttpApp({ service });
|
|
await service.submit({ caseId: fixture.caseId, runId: "cli-smoke-older", runtimeApiUrl: "https://public.example.test" });
|
|
await service.submit({ caseId: fixture.caseId, runId: "cli-smoke-newer", runtimeApiUrl: "https://public.example.test" });
|
|
const older = registry.runs.get("cli-smoke-older")!;
|
|
const newer = registry.runs.get("cli-smoke-newer")!;
|
|
older.createdAt = "2026-07-17T10:00:00.000Z";
|
|
newer.createdAt = "2026-07-18T10:00:00.000Z";
|
|
await registry.updateRun("cli-smoke-older", { status: "completed", stage: "completed" });
|
|
await registry.updateRun("cli-smoke-newer", {
|
|
status: "completed",
|
|
stage: "completed",
|
|
result: {
|
|
artifactManifestSha256: "d".repeat(64),
|
|
summary: { artifactCount: 1, artifacts: [{ path: "software-smoke.json" }] },
|
|
evidence: { status: "recorded" }
|
|
}
|
|
});
|
|
|
|
const firstResponse = await app.fetch(new Request("http://harnessrl.test/v1/caserun/runs?status=completed&limit=1"));
|
|
assert.equal(firstResponse.status, 200);
|
|
const first = await firstResponse.json() as any;
|
|
assert.equal(first.count, 1);
|
|
assert.equal(first.runs[0].runId, "cli-smoke-newer");
|
|
assert.equal(first.runs[0].artifactCount, 1);
|
|
assert.equal(first.runs[0].evidenceStatus, "recorded");
|
|
assert.equal(first.runs[0].artifactManifestSha256, "d".repeat(64));
|
|
assert.equal("runDir" in first.runs[0], false);
|
|
assert.equal(typeof first.nextCursor, "string");
|
|
|
|
const secondResponse = await app.fetch(new Request(`http://harnessrl.test/v1/caserun/runs?limit=1&cursor=${encodeURIComponent(first.nextCursor)}`));
|
|
const second = await secondResponse.json() as any;
|
|
assert.equal(second.runs[0].runId, "cli-smoke-older");
|
|
assert.equal(second.nextCursor, null);
|
|
|
|
const invalidResponse = await app.fetch(new Request("http://harnessrl.test/v1/caserun/runs?status=done"));
|
|
assert.equal(invalidResponse.status, 400);
|
|
assert.equal((await invalidResponse.json() as any).error.code, "invalid_status");
|
|
});
|
|
|
|
test("worker restart reuses idempotent prepare/build/collect activity results", async () => {
|
|
const fixture = await softwareSmokeFixture();
|
|
const registry = new MemoryRegistry();
|
|
await registry.createRun({ runId: "worker-restart", caseId: fixture.caseId, workflowId: "workflow-worker-restart", caseRepo: fixture.root, runDir: path.join(fixture.stateRoot, "runs", "worker-restart"), runtimeApiUrl: "http://127.0.0.1:6667" });
|
|
const firstWorker = createHarnessRLActivities({ registry, env: {} });
|
|
await firstWorker.markRunning({ runId: "worker-restart" });
|
|
const prepared = await firstWorker.prepare({ runId: "worker-restart", identity: "worker-restart:prepare:v1" });
|
|
await firstWorker.agent({ runId: "worker-restart", identity: "worker-restart:agent:v1" });
|
|
await firstWorker.trace({ runId: "worker-restart", identity: "worker-restart:trace:v1" });
|
|
await firstWorker.diff({ runId: "worker-restart", identity: "worker-restart:diff:v1" });
|
|
const built = await firstWorker.build({ runId: "worker-restart", identity: "worker-restart:build:v1" });
|
|
|
|
const restartedWorker = createHarnessRLActivities({ registry, env: {} });
|
|
assert.deepEqual(await restartedWorker.prepare({ runId: "worker-restart", identity: "worker-restart:prepare:v1" }), prepared);
|
|
await restartedWorker.agent({ runId: "worker-restart", identity: "worker-restart:agent:v1" });
|
|
await restartedWorker.trace({ runId: "worker-restart", identity: "worker-restart:trace:v1" });
|
|
await restartedWorker.diff({ runId: "worker-restart", identity: "worker-restart:diff:v1" });
|
|
const rebuilt = await restartedWorker.build({ runId: "worker-restart", identity: "worker-restart:build:v1" });
|
|
assert.equal(rebuilt.mode, built.mode);
|
|
await restartedWorker.collect({ runId: "worker-restart", identity: "worker-restart:collect:v1" });
|
|
await restartedWorker.markCompleted({ runId: "worker-restart" });
|
|
|
|
const record = await registry.getRun("worker-restart");
|
|
assert.equal(record?.status, "completed");
|
|
assert.equal((record?.result as any)?.summary?.mode, "software-smoke");
|
|
const artifact = JSON.parse(await readFile(path.join(record!.runDir, "software-smoke.json"), "utf8"));
|
|
assert.equal(artifact.checks.temporalWorkerRunning, true);
|
|
assert.equal(registry.activityResults.size, 6);
|
|
});
|
|
|
|
test("worker uses configured cloud-api authority and resubmits the same identity after typed not-found", async () => {
|
|
const fixture = await softwareSmokeFixture();
|
|
const registry = new MemoryRegistry();
|
|
const runId = "authority-retry";
|
|
const identity = `${runId}:agent:v1`;
|
|
await registry.createRun({ runId, caseId: fixture.caseId, workflowId: `workflow-${runId}`, caseRepo: fixture.root, runDir: path.join(fixture.stateRoot, "runs", runId), runtimeApiUrl: "https://public.example.test" });
|
|
await registry.saveActivityResult({ runId, activityName: "prepare", identity: `${runId}:prepare:v1`, output: { state: "completed", result: { mode: "hardware", prepared: { run: { runId } } } } });
|
|
await registry.saveActivityResult({ runId, activityName: "agent", identity, output: { state: "started", authorityIdentity: identity } });
|
|
let submitCount = 0;
|
|
let recoverCount = 0;
|
|
const activities = createHarnessRLActivities({
|
|
registry,
|
|
env: { HWLAB_CASERUN_INTERNAL_API_URL: "http://hwlab-cloud-api.hwlab-v03.svc.cluster.local:6667" },
|
|
stages: {
|
|
agent: async (context: any, run: any) => { submitCount += 1; assert.equal(context.parsed.apiUrl, "http://hwlab-cloud-api.hwlab-v03.svc.cluster.local:6667"); assert.equal(context.parsed.operationIdentity, identity); return { run, agent: { traceId: "trace-authority-retry" } }; },
|
|
recoverAgent: async () => { recoverCount += 1; throw Object.assign(new Error("not found"), { code: "agentrun_task_not_found" }); }
|
|
}
|
|
});
|
|
const result = await activities.agent({ runId, identity });
|
|
assert.equal(result.agent.traceId, "trace-authority-retry");
|
|
assert.equal(recoverCount, 1);
|
|
assert.equal(submitCount, 1);
|
|
const deploy = await readFile(path.join(process.cwd(), "deploy", "deploy.yaml"), "utf8");
|
|
assert.match(deploy, /HWLAB_CASERUN_INTERNAL_API_URL: http:\/\/hwlab-cloud-api\.hwlab-v03\.svc\.cluster\.local:6667/u);
|
|
});
|
|
|
|
test("AgentRun and HWPOD crash gaps reuse stable authoritative operation identities", async () => {
|
|
const fixture = await hardwareFixture();
|
|
const registry = new MemoryRegistry();
|
|
await registry.createRun({ runId: "authority-gap", caseId: fixture.caseId, workflowId: "workflow-authority-gap", caseRepo: fixture.root, runDir: path.join(fixture.stateRoot, "runs", "authority-gap"), runtimeApiUrl: "http://127.0.0.1:6667" });
|
|
let agentDispatchCount = 0;
|
|
let hwpodDispatchCount = 0;
|
|
let agentLookupCount = 0;
|
|
let hwpodLookupCount = 0;
|
|
let agentCrash = true;
|
|
let hwpodCrash = true;
|
|
const stages = {
|
|
async prepare(context: any) { return { run: { runId: "authority-gap", caseId: fixture.caseId, runDir: context.parsed.runDir } }; },
|
|
async agent(context: any, run: any) {
|
|
agentDispatchCount += 1;
|
|
if (agentCrash) { agentCrash = false; throw new Error("worker exited after AgentRun accepted"); }
|
|
return { run: { ...run, agent: { traceId: context.parsed.traceId, sessionId: "ses-authoritative", resultUrl: "/v1/agentrun/result" } }, agent: { traceId: context.parsed.traceId, sessionId: "ses-authoritative", resultUrl: "/v1/agentrun/result" } };
|
|
},
|
|
async recoverAgent(context: any, run: any) {
|
|
agentLookupCount += 1;
|
|
return { run: { ...run, agent: { traceId: context.parsed.traceId, sessionId: "ses-authoritative", resultUrl: "/v1/agentrun/result" } }, agent: { traceId: context.parsed.traceId, sessionId: "ses-authoritative", resultUrl: "/v1/agentrun/result" } };
|
|
},
|
|
async trace(_context: any, run: any) { return { run, trace: { traceId: run.agent.traceId, status: "completed" } }; },
|
|
async diff(_context: any, run: any) { return { run, diff: { diffPatchSha256: "d".repeat(64) } }; },
|
|
async build(context: any, run: any) {
|
|
hwpodDispatchCount += 1;
|
|
if (hwpodCrash) { hwpodCrash = false; throw new Error("worker exited after HWPOD accepted"); }
|
|
return { run, summary: { jobId: "job-authoritative" }, evidence: { operation: { operationId: context.parsed.operationIdentity, jobId: "job-authoritative" } } };
|
|
},
|
|
async recoverBuild(context: any, run: any) {
|
|
hwpodLookupCount += 1;
|
|
return { run, summary: { jobId: "job-authoritative" }, evidence: { operation: { operationId: context.parsed.operationIdentity, jobId: "job-authoritative" } } };
|
|
},
|
|
async collect(_context: any, run: any, evidence: any) {
|
|
const manifestPath = path.join(run.runDir, "artifact-manifest.json");
|
|
await mkdir(run.runDir, { recursive: true });
|
|
await writeFile(manifestPath, `${JSON.stringify({ aggregate: { path: "aggregate.md", sha256: "a".repeat(64) } })}\n`);
|
|
return { run, status: "completed", summary: { artifactManifestPath: manifestPath }, evidence };
|
|
}
|
|
};
|
|
const worker = createHarnessRLActivities({ registry, env: {}, stages });
|
|
await worker.prepare({ runId: "authority-gap", identity: "authority-gap:prepare:v1" });
|
|
await assert.rejects(worker.agent({ runId: "authority-gap", identity: "authority-gap:agent:v1" }), /AgentRun accepted/);
|
|
await Promise.all([
|
|
worker.agent({ runId: "authority-gap", identity: "authority-gap:agent:v1" }),
|
|
worker.agent({ runId: "authority-gap", identity: "authority-gap:agent:v1" })
|
|
]);
|
|
await worker.trace({ runId: "authority-gap", identity: "authority-gap:trace:v1" });
|
|
await worker.diff({ runId: "authority-gap", identity: "authority-gap:diff:v1" });
|
|
await assert.rejects(worker.build({ runId: "authority-gap", identity: "authority-gap:build:v1" }), /HWPOD accepted/);
|
|
await Promise.all([
|
|
worker.build({ runId: "authority-gap", identity: "authority-gap:build:v1" }),
|
|
worker.build({ runId: "authority-gap", identity: "authority-gap:build:v1" })
|
|
]);
|
|
const result = await worker.collect({ runId: "authority-gap", identity: "authority-gap:collect:v1" });
|
|
assert.equal(agentDispatchCount, 1);
|
|
assert.equal(hwpodDispatchCount, 1);
|
|
assert.equal(agentLookupCount, 2);
|
|
assert.equal(hwpodLookupCount, 2);
|
|
assert.equal((result as any).authority.agentRun.sessionId, "ses-authoritative");
|
|
assert.equal((result as any).authority.hwpod.jobId, "job-authoritative");
|
|
assert.equal(typeof (result as any).artifactManifestSha256, "string");
|
|
assert.equal((result as any).aggregate.sha256, "a".repeat(64));
|
|
});
|
|
|
|
test("Temporal start crash gap reuses the existing workflow identity", async () => {
|
|
const fixture = await softwareSmokeFixture();
|
|
const registry = new MemoryRegistry();
|
|
const temporal = new CrashGapTemporal();
|
|
const service = createHarnessRLService({ registry, temporal, cwd: fixture.root, caseRepo: fixture.root, stateRoot: fixture.stateRoot });
|
|
await assert.rejects(service.submit({ caseId: fixture.caseId, runId: "start-gap", runtimeApiUrl: "http://harnessrl.test" }), /API exited/);
|
|
const recovered = await service.submit({ caseId: fixture.caseId, runId: "start-gap", runtimeApiUrl: "http://harnessrl.test" });
|
|
assert.equal(recovered?.workflowRunId, "temporal-start-gap");
|
|
assert.equal(temporal.accepted.size, 1);
|
|
});
|
|
|
|
test("cancel converges to durable canceled through workflow signal", async () => {
|
|
const fixture = await softwareSmokeFixture();
|
|
const registry = new MemoryRegistry();
|
|
const temporal = new FakeTemporal(async (workflowId) => {
|
|
const record = [...registry.runs.values()].find((item) => item.workflowId === workflowId)!;
|
|
await registry.updateRun(record.runId, { status: "canceled", stage: "canceled" });
|
|
await registry.appendEvent(record.runId, "canceled", "canceled", { source: "temporal-signal" });
|
|
});
|
|
const service = createHarnessRLService({ registry, temporal, cwd: fixture.root, caseRepo: fixture.root, stateRoot: fixture.stateRoot });
|
|
await service.submit({ caseId: fixture.caseId, runId: "cancel-run", runtimeApiUrl: "http://harnessrl.test" });
|
|
const canceled = await service.cancel("cancel-run");
|
|
assert.equal(canceled?.status, "canceled");
|
|
assert.equal(canceled?.terminal, true);
|
|
assert.deepEqual((await registry.listEvents("cancel-run")).map((event) => event.status), ["queued", "cancel_requested", "canceled"]);
|
|
});
|
|
|
|
test("read model keeps workflow identity separate from result authority", async () => {
|
|
const fixture = await softwareSmokeFixture();
|
|
const registry = new MemoryRegistry();
|
|
const service = createHarnessRLService({ registry, temporal: new FakeTemporal(), cwd: fixture.root, caseRepo: fixture.root, stateRoot: fixture.stateRoot });
|
|
await service.submit({ caseId: fixture.caseId, runId: "read-model", runtimeApiUrl: "http://harnessrl.test" });
|
|
await registry.updateRun("read-model", { status: "completed", stage: "completed", result: { artifactManifestSha256: "a".repeat(64), sourceAuthority: "hwpod" } });
|
|
const record = await service.getRun("read-model");
|
|
assert.equal(record.workflowId, "harnessrl-caserun-read-model");
|
|
assert.equal(record.status, "completed");
|
|
assert.equal(record.result?.sourceAuthority, "hwpod");
|
|
});
|
|
|
|
async function softwareSmokeFixture() {
|
|
const root = await mkdtemp(path.join(os.tmpdir(), "harnessrl-native-"));
|
|
const caseId = "software-smoke";
|
|
await mkdir(path.join(root, "cases", caseId), { recursive: true });
|
|
await writeFile(path.join(root, "cases", caseId, "case.json"), `${JSON.stringify({ caseId, title: "Native software smoke", mode: "software-smoke" }, null, 2)}\n`);
|
|
return { root, caseId, stateRoot: path.join(root, ".state", "harnessrl") };
|
|
}
|
|
|
|
async function hardwareFixture() {
|
|
const root = await mkdtemp(path.join(os.tmpdir(), "harnessrl-authority-"));
|
|
const caseId = "hardware-authority";
|
|
await mkdir(path.join(root, "cases", caseId), { recursive: true });
|
|
await writeFile(path.join(root, "cases", caseId, "case.json"), `${JSON.stringify({ caseId, title: "Authoritative fixture", mode: "compile-only" }, null, 2)}\n`);
|
|
return { root, caseId, stateRoot: path.join(root, ".state", "harnessrl") };
|
|
}
|
|
|
|
class FakeTemporal implements HarnessRLTemporalGateway {
|
|
starts = 0;
|
|
constructor(private readonly onCancel?: (workflowId: string) => Promise<void>) {}
|
|
async start() { this.starts += 1; return { workflowRunId: `temporal-run-${this.starts}` }; }
|
|
async requestCancel(workflowId: string) { await this.onCancel?.(workflowId); }
|
|
async close() {}
|
|
}
|
|
|
|
class CrashGapTemporal implements HarnessRLTemporalGateway {
|
|
accepted = new Map<string, string>();
|
|
crashed = false;
|
|
async start(input: { workflowId: string }) {
|
|
const workflowRunId = this.accepted.get(input.workflowId) ?? "temporal-start-gap";
|
|
this.accepted.set(input.workflowId, workflowRunId);
|
|
if (!this.crashed) { this.crashed = true; throw new Error("API exited after Temporal accepted"); }
|
|
return { workflowRunId, reused: true };
|
|
}
|
|
async requestCancel() {}
|
|
async close() {}
|
|
}
|
|
|
|
class MemoryRegistry implements HarnessRLRegistry {
|
|
runs = new Map<string, CaseRunRecord>();
|
|
events = new Map<string, CaseRunEvent[]>();
|
|
activityResults = new Map<string, ActivityResult>();
|
|
async ensureSchema() {}
|
|
async createRun(input: StartCaseRunInput) {
|
|
const existing = this.runs.get(input.runId);
|
|
if (existing) return structuredClone(existing);
|
|
const now = new Date().toISOString();
|
|
const record: CaseRunRecord = { ...input, workflowRunId: null, status: "queued", stage: "queued", terminal: false, result: null, error: null, blocker: null, createdAt: now, updatedAt: now };
|
|
this.runs.set(input.runId, record);
|
|
return structuredClone(record);
|
|
}
|
|
async getRun(runId: string) { const value = this.runs.get(runId); return value ? structuredClone(value) : null; }
|
|
async listRuns(input: { limit: number; status?: CaseRunStatus; before?: { createdAt: string; runId: string } }) {
|
|
return [...this.runs.values()]
|
|
.filter((record) => !input.status || record.status === input.status)
|
|
.filter((record) => !input.before || record.createdAt < input.before.createdAt || (record.createdAt === input.before.createdAt && record.runId < input.before.runId))
|
|
.sort((left, right) => right.createdAt.localeCompare(left.createdAt) || right.runId.localeCompare(left.runId))
|
|
.slice(0, input.limit)
|
|
.map((record) => structuredClone(record));
|
|
}
|
|
async updateRun(runId: string, patch: any) {
|
|
const current = this.runs.get(runId);
|
|
if (!current) throw new Error("missing run");
|
|
const next = { ...current, ...patch, terminal: terminalStatus((patch.status ?? current.status) as CaseRunStatus), updatedAt: new Date().toISOString() };
|
|
this.runs.set(runId, next);
|
|
return structuredClone(next);
|
|
}
|
|
async appendEvent(runId: string, status: CaseRunStatus, stage: string, payload: Record<string, unknown> = {}) {
|
|
const events = this.events.get(runId) ?? [];
|
|
const event = { id: events.length + 1, runId, status, stage, payload, createdAt: new Date().toISOString() };
|
|
events.push(event); this.events.set(runId, events); return structuredClone(event);
|
|
}
|
|
async listEvents(runId: string) { return structuredClone(this.events.get(runId) ?? []); }
|
|
async getActivityResult(runId: string, activityName: string, identity: string) { return structuredClone(this.activityResults.get(`${runId}:${activityName}:${identity}`) ?? null); }
|
|
async saveActivityResult(result: ActivityResult) { const key = `${result.runId}:${result.activityName}:${result.identity}`; this.activityResults.set(key, structuredClone(result)); return structuredClone(this.activityResults.get(key)!); }
|
|
async health() { return { ok: true as const }; }
|
|
async close() {}
|
|
}
|