root
221 lines
15 KiB
TypeScript
221 lines
15 KiB
TypeScript
import { heartbeat } from "@temporalio/activity";
|
|
import { createHash } from "node:crypto";
|
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
|
|
import { buildCaseRun, collectAgentDiff, collectAgentTraceEvidence, collectCaseRun, prepareCaseRun, recoverAgentTaskStage, recoverBuildCaseRun, runAgentTaskStage } from "../../tools/src/hwlab-caserun-lib.ts";
|
|
import type { HarnessRLRegistry } from "./contracts.ts";
|
|
import { caseRunInternalFetch } from "../cloud/server-caserun-http.ts";
|
|
|
|
export function createHarnessRLActivities(options: {
|
|
registry: HarnessRLRegistry;
|
|
env?: Record<string, string | undefined>;
|
|
stages?: Partial<{
|
|
prepare(context: any, action: string): Promise<any>;
|
|
agent(context: any, run: any): Promise<any>;
|
|
recoverAgent(context: any, run: any): Promise<any>;
|
|
trace(context: any, run: any): Promise<any>;
|
|
diff(context: any, run: any): Promise<any>;
|
|
build(context: any, run: any): Promise<any>;
|
|
recoverBuild(context: any, run: any): Promise<any>;
|
|
collect(context: any, run: any, evidence: any): Promise<any>;
|
|
}>;
|
|
}) {
|
|
const env = options.env ?? process.env;
|
|
const registry = options.registry;
|
|
const stages = {
|
|
prepare: options.stages?.prepare ?? prepareCaseRun,
|
|
agent: options.stages?.agent ?? runAgentTaskStage,
|
|
recoverAgent: options.stages?.recoverAgent ?? recoverAgentTaskStage,
|
|
trace: options.stages?.trace ?? collectAgentTraceEvidence,
|
|
diff: options.stages?.diff ?? collectAgentDiff,
|
|
build: options.stages?.build ?? buildCaseRun,
|
|
recoverBuild: options.stages?.recoverBuild ?? recoverBuildCaseRun,
|
|
collect: options.stages?.collect ?? collectCaseRun
|
|
};
|
|
return {
|
|
async markRunning({ runId }: { runId: string }) {
|
|
await transition(registry, runId, "running", "prepare", { worker: "hwlab-harnessrl-worker" });
|
|
},
|
|
async prepare(input: { runId: string; identity: string }) {
|
|
return idempotent(registry, input, async () => {
|
|
const record = required(await registry.getRun(input.runId));
|
|
const definition = await loadDefinition(record.caseRepo, record.caseId);
|
|
activityHeartbeat({ runId: input.runId, stage: "prepare" });
|
|
if (definition.mode === "software-smoke") {
|
|
await mkdir(record.runDir, { recursive: true });
|
|
const output = { mode: "software-smoke", run: { runId: record.runId, caseId: record.caseId, mode: "software-smoke", hardwareRequired: false } };
|
|
await transition(registry, input.runId, "running", "prepared", { mode: "software-smoke" });
|
|
return output;
|
|
}
|
|
const context = contextForRecord(record, env, input.identity);
|
|
const prepared = await stages.prepare(context, "harnessrl-worker");
|
|
const output = { mode: definition.mode, prepared } as Record<string, unknown>;
|
|
await transition(registry, input.runId, "running", "prepared", { runDir: prepared.runDir, specPath: prepared.specPath });
|
|
return output;
|
|
});
|
|
},
|
|
async agent(input: { runId: string; identity: string }) {
|
|
return idempotent(registry, input, async () => {
|
|
const record = required(await registry.getRun(input.runId));
|
|
const prepared = activityOutput(required(await registry.getActivityResult(input.runId, "prepare", `${input.runId}:prepare:v1`))) as any;
|
|
if (prepared.mode === "software-smoke") return { mode: prepared.mode, run: prepared.run, authority: { agentRun: "not-applicable" } };
|
|
const stage = await stages.agent(contextForRecord(record, env, input.identity), prepared.prepared.run);
|
|
await transition(registry, input.runId, "running", "agent-completed", { traceId: stage.agent?.traceId ?? null, sessionId: stage.agent?.sessionId ?? null });
|
|
return { mode: prepared.mode, run: stage.run, agent: stage.agent };
|
|
}, async () => {
|
|
const record = required(await registry.getRun(input.runId));
|
|
const prepared = activityOutput(required(await registry.getActivityResult(input.runId, "prepare", `${input.runId}:prepare:v1`))) as any;
|
|
const stage = await stages.recoverAgent(contextForRecord(record, env, input.identity), prepared.prepared.run);
|
|
await transition(registry, input.runId, "running", "agent-completed", { traceId: stage.agent?.traceId ?? null, sessionId: stage.agent?.sessionId ?? null, recovered: true });
|
|
return { mode: prepared.mode, run: stage.run, agent: stage.agent };
|
|
});
|
|
},
|
|
async trace(input: { runId: string; identity: string }) {
|
|
return idempotent(registry, input, async () => {
|
|
const record = required(await registry.getRun(input.runId));
|
|
const agent = activityOutput(required(await registry.getActivityResult(input.runId, "agent", `${input.runId}:agent:v1`))) as any;
|
|
if (agent.mode === "software-smoke") return agent;
|
|
const stage = await stages.trace(contextForRecord(record, env, input.identity), agent.run);
|
|
await transition(registry, input.runId, "running", "trace-completed", { traceId: stage.trace?.traceId ?? null });
|
|
return { ...agent, run: stage.run, trace: stage.trace };
|
|
});
|
|
},
|
|
async diff(input: { runId: string; identity: string }) {
|
|
return idempotent(registry, input, async () => {
|
|
const record = required(await registry.getRun(input.runId));
|
|
const traced = activityOutput(required(await registry.getActivityResult(input.runId, "trace", `${input.runId}:trace:v1`))) as any;
|
|
if (traced.mode === "software-smoke") return traced;
|
|
const stage = await stages.diff(contextForRecord(record, env, input.identity), traced.run);
|
|
await transition(registry, input.runId, "running", "diff-completed", { diffSha256: stage.diff?.diffPatchSha256 ?? null });
|
|
return { ...traced, run: stage.run, diff: stage.diff };
|
|
});
|
|
},
|
|
async build(input: { runId: string; identity: string }) {
|
|
return idempotent(registry, input, async () => {
|
|
const record = required(await registry.getRun(input.runId));
|
|
const prepared = activityOutput(required(await registry.getActivityResult(input.runId, "diff", `${input.runId}:diff:v1`))) as any;
|
|
activityHeartbeat({ runId: input.runId, stage: "build" });
|
|
if (prepared.mode === "software-smoke") {
|
|
const artifact = {
|
|
contractVersion: "hwlab-caserun-software-smoke-v1", caseId: record.caseId, runId: record.runId,
|
|
mode: "software-smoke", status: "completed", hardwareRequired: false,
|
|
checks: { caseDefinitionLoaded: true, runDirectoryWritable: true, temporalWorkerRunning: true }
|
|
};
|
|
const text = `${JSON.stringify(artifact, null, 2)}\n`;
|
|
await writeFile(path.join(record.runDir, "software-smoke.json"), text, "utf8");
|
|
const artifactRef = { path: "software-smoke.json", bytes: Buffer.byteLength(text), sha256: createHash("sha256").update(text).digest("hex") };
|
|
const output = { mode: "software-smoke", run: prepared.run, summary: { status: "completed", mode: "software-smoke", artifactCount: 1, artifacts: [artifactRef] }, evidence: { status: "recorded", mode: "software-smoke", artifacts: [artifactRef] } };
|
|
await transition(registry, input.runId, "running", "build-completed", { artifactSha256: artifactRef.sha256 });
|
|
return output;
|
|
}
|
|
const context = contextForRecord(record, env, input.identity);
|
|
const build = await stages.build(context, prepared.run);
|
|
await transition(registry, input.runId, "running", "build-completed", { jobId: build.summary?.jobId ?? build.evidence?.keilJob?.jobId ?? null });
|
|
return { mode: prepared.mode, run: build.run ?? prepared.run, build, agent: prepared.agent, trace: prepared.trace, diff: prepared.diff };
|
|
}, async () => {
|
|
const record = required(await registry.getRun(input.runId));
|
|
const prepared = activityOutput(required(await registry.getActivityResult(input.runId, "diff", `${input.runId}:diff:v1`))) as any;
|
|
const build = await stages.recoverBuild(contextForRecord(record, env, input.identity), prepared.run);
|
|
await transition(registry, input.runId, "running", "build-completed", { jobId: build.summary?.jobId ?? build.evidence?.keilJob?.jobId ?? null, recovered: true });
|
|
return { mode: prepared.mode, run: build.run ?? prepared.run, build, agent: prepared.agent, trace: prepared.trace, diff: prepared.diff };
|
|
});
|
|
},
|
|
async collect(input: { runId: string; identity: string }) {
|
|
return idempotent(registry, input, async () => {
|
|
const record = required(await registry.getRun(input.runId));
|
|
const built = activityOutput(required(await registry.getActivityResult(input.runId, "build", `${input.runId}:build:v1`))) as any;
|
|
activityHeartbeat({ runId: input.runId, stage: "collect" });
|
|
let result: Record<string, unknown>;
|
|
if (built.mode === "software-smoke") result = { summary: built.summary, evidence: built.evidence, run: built.run };
|
|
else {
|
|
const context = contextForRecord(record, env, input.identity);
|
|
const collected = await stages.collect(context, built.run, built.build.evidence);
|
|
const manifest = await manifestRefs(collected.summary);
|
|
result = {
|
|
summary: built.build.summary, evidence: built.build.evidence, run: collected.run ?? built.run,
|
|
authority: { agentRun: authorityRefs(built), hwpod: built.build.evidence?.operation ?? built.build.evidence?.validation ?? null },
|
|
artifactManifestPath: manifest.path, artifactManifestSha256: manifest.sha256,
|
|
aggregate: manifest.aggregate,
|
|
collected: { status: collected.status, summary: collected.summary ?? null }
|
|
};
|
|
}
|
|
await registry.updateRun(input.runId, { result });
|
|
await transition(registry, input.runId, "running", "collect-completed", {});
|
|
return result;
|
|
});
|
|
},
|
|
async markCompleted({ runId }: { runId: string }) { await transition(registry, runId, "completed", "completed", {}); },
|
|
async markCanceled({ runId }: { runId: string }) { await transition(registry, runId, "canceled", "canceled", {}); },
|
|
async markFailed(input: { runId: string; code: string; message: string }) {
|
|
await registry.updateRun(input.runId, { status: "failed", stage: "failed", error: { code: input.code, message: input.message } });
|
|
await registry.appendEvent(input.runId, "failed", "failed", { code: input.code, message: input.message.slice(0, 500) });
|
|
}
|
|
};
|
|
}
|
|
|
|
async function idempotent(registry: HarnessRLRegistry, input: { runId: string; identity: string }, execute: () => Promise<Record<string, unknown>>, recover?: () => Promise<Record<string, unknown>>) {
|
|
const activityName = input.identity.split(":").at(-2) ?? "activity";
|
|
const existing = await registry.getActivityResult(input.runId, activityName, input.identity);
|
|
if ((existing?.output as any)?.state === "completed") return (existing!.output as any).result;
|
|
if (["started", "accepted"].includes((existing?.output as any)?.state) && recover) {
|
|
let output: Record<string, unknown>;
|
|
try {
|
|
output = await recover();
|
|
} catch (error: any) {
|
|
if (!authorityNotFound(error)) throw error;
|
|
output = await execute();
|
|
}
|
|
const saved = await registry.saveActivityResult({ runId: input.runId, activityName, identity: input.identity, output: { state: "completed", authorityIdentity: input.identity, result: output } });
|
|
return activityOutput(saved);
|
|
}
|
|
if (!existing) await registry.saveActivityResult({ runId: input.runId, activityName, identity: input.identity, output: { state: "started", authorityIdentity: input.identity } });
|
|
const output = await execute();
|
|
const saved = await registry.saveActivityResult({ runId: input.runId, activityName, identity: input.identity, output: { state: "completed", authorityIdentity: input.identity, result: output } });
|
|
return activityOutput(saved);
|
|
}
|
|
|
|
async function transition(registry: HarnessRLRegistry, runId: string, status: any, stage: string, payload: Record<string, unknown>) {
|
|
await registry.updateRun(runId, { status, stage });
|
|
await registry.appendEvent(runId, status, stage, payload);
|
|
}
|
|
|
|
function contextForRecord(record: any, env: Record<string, string | undefined>, identity: string) {
|
|
const apiUrl = String(env.HWLAB_CASERUN_INTERNAL_API_URL ?? env.HWLAB_RUNTIME_INTERNAL_API_URL ?? record.runtimeApiUrl).replace(/\/+$/u, "");
|
|
const traceId = `trc_harnessrl_${createHash("sha256").update(identity).digest("hex").slice(0, 24)}`;
|
|
return {
|
|
parsed: { _: [], caseId: record.caseId, runId: record.runId, runDir: record.runDir, caseRepo: record.caseRepo, apiUrl, traceId, operationIdentity: identity, noCaseRepoRecord: false, stateDir: path.join(path.dirname(record.runDir), "cli") },
|
|
env: { ...process.env, ...env, HWLAB_CASE_REPO: record.caseRepo, HWLAB_RUNTIME_API_URL: apiUrl, HWLAB_CASERUN_PUBLIC_RUNTIME_API_URL: record.runtimeApiUrl, HWLAB_RUNTIME_ENDPOINT_LOCKED: "1", HWLAB_HWPOD_OPERATION_IDENTITY: identity },
|
|
fetchImpl: caseRunInternalFetch(globalThis.fetch, apiUrl, env), cwd: record.caseRepo,
|
|
now: () => new Date().toISOString(), sleep: (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)), argv: [], rest: ["worker", record.caseId]
|
|
};
|
|
}
|
|
|
|
function activityOutput(result: { output: Record<string, unknown> }) {
|
|
return (result.output as any)?.state === "completed" ? (result.output as any).result : result.output;
|
|
}
|
|
|
|
function authorityNotFound(error: any) {
|
|
const code = String(error?.code ?? "");
|
|
return code === "hwpod_operation_not_found" || code.endsWith("_not_found");
|
|
}
|
|
|
|
async function manifestRefs(summary: any) {
|
|
const manifestPath = String(summary?.artifactManifestPath ?? "");
|
|
if (!manifestPath) return { path: null, sha256: null, aggregate: null };
|
|
const body = await readFile(manifestPath);
|
|
const manifest = JSON.parse(body.toString("utf8"));
|
|
return { path: manifestPath, sha256: createHash("sha256").update(body).digest("hex"), aggregate: manifest.aggregate ?? null };
|
|
}
|
|
|
|
function authorityRefs(built: any) {
|
|
const agent = built.agent ?? built.run?.agent ?? {};
|
|
return { conversationId: agent.conversationId ?? null, sessionId: agent.sessionId ?? null, threadId: agent.threadId ?? null, traceId: agent.traceId ?? null, resultUrl: agent.resultUrl ?? null, commandStatus: agent.commandStatus ?? null, final: agent.finalResponse ?? agent.result ?? null, diffSha256: built.diff?.diffPatchSha256 ?? built.run?.agentDiff?.diffPatchSha256 ?? null };
|
|
}
|
|
|
|
async function loadDefinition(caseRepo: string, caseId: string) {
|
|
return JSON.parse(await readFile(path.join(caseRepo, "cases", caseId, "case.json"), "utf8"));
|
|
}
|
|
function required<T>(value: T | null): T { if (value === null) throw Object.assign(new Error("required registry record was not found"), { code: "registry_record_not_found" }); return value; }
|
|
function activityHeartbeat(details: Record<string, unknown>) { try { heartbeat(details); } catch {} }
|