fix: preserve caserun terminal semantics
This commit is contained in:
@@ -116,13 +116,13 @@ export function createHarnessRLActivities(options: {
|
||||
const context = contextForRecord(record, env, input.identity);
|
||||
const build = await stages.build(context, prepared.run);
|
||||
await transition(registry, input.runId, "running", "build-completed", { jobId: build.summary?.jobId ?? build.evidence?.keilJob?.jobId ?? null });
|
||||
return { mode: prepared.mode, run: build.run ?? prepared.run, build, agent: prepared.agent, trace: prepared.trace, diff: prepared.diff };
|
||||
return { mode: prepared.mode, run: build.run ?? prepared.run, build, agent: prepared.agent, trace: prepared.trace, diff: prepared.diff, validation: { status: build.evidence?.status ?? null, blocker: build.evidence?.validation?.blocker ?? null } };
|
||||
}, async () => {
|
||||
const record = required(await registry.getRun(input.runId));
|
||||
const prepared = activityOutput(required(await registry.getActivityResult(input.runId, "diff", `${input.runId}:diff:v1`))) as any;
|
||||
const build = await stages.recoverBuild(contextForRecord(record, env, input.identity), prepared.run);
|
||||
await transition(registry, input.runId, "running", "build-completed", { jobId: build.summary?.jobId ?? build.evidence?.keilJob?.jobId ?? null, recovered: true });
|
||||
return { mode: prepared.mode, run: build.run ?? prepared.run, build, agent: prepared.agent, trace: prepared.trace, diff: prepared.diff };
|
||||
return { mode: prepared.mode, run: build.run ?? prepared.run, build, agent: prepared.agent, trace: prepared.trace, diff: prepared.diff, validation: { status: build.evidence?.status ?? null, blocker: build.evidence?.validation?.blocker ?? null } };
|
||||
});
|
||||
},
|
||||
async collect(input: { runId: string; identity: string }) {
|
||||
|
||||
@@ -5,7 +5,7 @@ import path from "node:path";
|
||||
import { test } from "node:test";
|
||||
|
||||
import { createHarnessRLActivities } from "./activities.ts";
|
||||
import { caseRunWorkflowFailure } from "./workflows.ts";
|
||||
import { caseRunAgentFailure, caseRunBuildFailure, caseRunWorkflowFailure } from "./workflows.ts";
|
||||
import type { ActivityResult, CaseRunEvent, CaseRunRecord, CaseRunStatus, HarnessRLRegistry, HarnessRLTemporalGateway, StartCaseRunInput } from "./contracts.ts";
|
||||
import { terminalStatus } from "./contracts.ts";
|
||||
import { createHarnessRLHttpApp } from "./http.ts";
|
||||
@@ -192,6 +192,21 @@ test("Temporal activity wrappers preserve the underlying typed CaseRun failure",
|
||||
});
|
||||
});
|
||||
|
||||
test("HarnessRL does not promote AgentRun timeout or blocked HWPOD validation to completed", () => {
|
||||
assert.equal(caseRunAgentFailure({ agent: { stageStatus: "completed", timedOut: false } }), null);
|
||||
assert.deepEqual(caseRunAgentFailure({ agent: { stageStatus: "timeout", timedOut: true, traceId: "trace-timeout" } }), {
|
||||
code: "agentrun_task_timeout",
|
||||
message: "AgentRun task did not reach a terminal result within the CaseRun timeout",
|
||||
details: { traceId: "trace-timeout", status: "timeout" },
|
||||
});
|
||||
assert.equal(caseRunBuildFailure({ validation: { status: "recorded", blocker: null } }), null);
|
||||
assert.deepEqual(caseRunBuildFailure({ validation: { status: "blocked", blocker: { code: "hwpod_build_failed", summary: "build failed", details: { returnCode: 1 } } } }), {
|
||||
code: "hwpod_build_failed",
|
||||
message: "build failed",
|
||||
details: { returnCode: 1 },
|
||||
});
|
||||
});
|
||||
|
||||
test("activity errors become retryable ApplicationFailure values with typed details", async () => {
|
||||
const fixture = await hardwareFixture();
|
||||
const registry = new MemoryRegistry();
|
||||
|
||||
@@ -33,16 +33,26 @@ export async function caseRunWorkflow(input: { runId: string; activity: { startT
|
||||
if (await canceled()) return { status: "canceled", runId: input.runId };
|
||||
await activities.prepare({ runId: input.runId, identity: `${input.runId}:prepare:v1` });
|
||||
if (await canceled()) return { status: "canceled", runId: input.runId };
|
||||
await activities.agent({ runId: input.runId, identity: `${input.runId}:agent:v1` });
|
||||
const agent = await activities.agent({ runId: input.runId, identity: `${input.runId}:agent:v1` });
|
||||
const agentFailure = caseRunAgentFailure(agent);
|
||||
if (agentFailure) {
|
||||
await activities.markFailed({ runId: input.runId, ...agentFailure });
|
||||
return { status: "failed", runId: input.runId };
|
||||
}
|
||||
if (await canceled()) return { status: "canceled", runId: input.runId };
|
||||
await activities.trace({ runId: input.runId, identity: `${input.runId}:trace:v1` });
|
||||
if (await canceled()) return { status: "canceled", runId: input.runId };
|
||||
await activities.diff({ runId: input.runId, identity: `${input.runId}:diff:v1` });
|
||||
if (await canceled()) return { status: "canceled", runId: input.runId };
|
||||
await activities.build({ runId: input.runId, identity: `${input.runId}:build:v1` });
|
||||
const built = await activities.build({ runId: input.runId, identity: `${input.runId}:build:v1` });
|
||||
if (await canceled()) return { status: "canceled", runId: input.runId };
|
||||
await activities.collect({ runId: input.runId, identity: `${input.runId}:collect:v1` });
|
||||
if (await canceled()) return { status: "canceled", runId: input.runId };
|
||||
const buildFailure = caseRunBuildFailure(built);
|
||||
if (buildFailure) {
|
||||
await activities.markFailed({ runId: input.runId, ...buildFailure });
|
||||
return { status: "failed", runId: input.runId };
|
||||
}
|
||||
await activities.markCompleted({ runId: input.runId });
|
||||
return { status: "completed", runId: input.runId };
|
||||
} catch (error: any) {
|
||||
@@ -51,6 +61,21 @@ export async function caseRunWorkflow(input: { runId: string; activity: { startT
|
||||
}
|
||||
}
|
||||
|
||||
export function caseRunAgentFailure(result: any) {
|
||||
const agent = result?.agent ?? {};
|
||||
const status = String(agent.stageStatus ?? "").toLowerCase();
|
||||
if (agent.timedOut === true || status === "timeout" || status === "timed_out") return { code: "agentrun_task_timeout", message: "AgentRun task did not reach a terminal result within the CaseRun timeout", details: { traceId: agent.traceId ?? null, status } };
|
||||
if (agent.error || ["failed", "cancelled", "canceled"].includes(status)) return { code: String(agent.error?.code ?? "agentrun_task_failed"), message: String(agent.error?.message ?? `AgentRun task reached ${status || "failed"}`), details: { traceId: agent.traceId ?? null, status, error: agent.error ?? null } };
|
||||
return null;
|
||||
}
|
||||
|
||||
export function caseRunBuildFailure(result: any) {
|
||||
const validation = result?.validation ?? {};
|
||||
if (validation.status !== "blocked" && !validation.blocker) return null;
|
||||
const blocker = validation.blocker ?? {};
|
||||
return { code: String(blocker.code ?? "hwpod_validation_blocked"), message: String(blocker.summary ?? "HWPOD post-agent validation was blocked"), details: blocker.details ?? blocker };
|
||||
}
|
||||
|
||||
export function caseRunWorkflowFailure(error: any) {
|
||||
let current = error;
|
||||
let fallback = { code: "caserun_failed", message: String(error), details: undefined as unknown };
|
||||
|
||||
@@ -5,7 +5,7 @@ import path from "node:path";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { runHwlabCli } from "../src/hwlab-cli-lib.ts";
|
||||
import { artifactsFromKeilStatusForTest, buildWaitBlockerForTest, extractKeilJobId, jobStatusCommandForTest, summarizeHwpodOperationForTest } from "../src/hwlab-caserun-lib.ts";
|
||||
import { agentResultTerminalStatus, artifactsFromKeilStatusForTest, buildWaitBlockerForTest, extractKeilJobId, jobStatusCommandForTest, summarizeHwpodOperationForTest } from "../src/hwlab-caserun-lib.ts";
|
||||
|
||||
const SUBJECT_COMMIT_ID = "df7a4e6e551fa90d64bde5537cc000f89d63dd20";
|
||||
const SUBJECT_REPO_LOCAL_PATH = "F:\\Work\\HWLAB-CASE-F103";
|
||||
@@ -13,6 +13,16 @@ const TEST_RUNTIME_API_URL = "http://api.test";
|
||||
const TEST_RUNTIME_WEB_URL = "http://web.test";
|
||||
const TEST_RUNTIME_ENV = { HWLAB_RUNTIME_API_URL: TEST_RUNTIME_API_URL, HWLAB_RUNTIME_WEB_URL: TEST_RUNTIME_WEB_URL };
|
||||
|
||||
test("CaseRun waits through admitted AgentRun states and recognizes only real terminal states", () => {
|
||||
const noTerminal: any = { terminal: false };
|
||||
assert.equal(agentResultTerminalStatus("admitted", noTerminal), "");
|
||||
assert.equal(agentResultTerminalStatus("queued", noTerminal), "");
|
||||
assert.equal(agentResultTerminalStatus("running", noTerminal), "");
|
||||
assert.equal(agentResultTerminalStatus("completed", noTerminal), "completed");
|
||||
assert.equal(agentResultTerminalStatus("failed", noTerminal), "failed");
|
||||
assert.equal(agentResultTerminalStatus("", { terminal: true, terminalStatus: "natural_end" } as any), "natural_end");
|
||||
});
|
||||
|
||||
test("hwlab-cli case prepare copies a case hwpod-spec into isolated run state", async () => {
|
||||
const root = await mkdtempCaseRoot();
|
||||
const caseRepo = path.join(root, "hwlab-case-registry");
|
||||
|
||||
@@ -569,7 +569,8 @@ async function buildCaseRunWithAuthorityMode(context: CaseContext, prepared: Pre
|
||||
applyCaseRunEvidenceRelationships(evidence, run);
|
||||
const evidencePath = path.join(run.runDir, "evidence.json");
|
||||
await writeJson(evidencePath, evidence);
|
||||
const completed = await writeRunControl(context, { ...run, status: "completed", stage: "completed", evidencePath }, { status: "completed", stage: "completed", completedAt: context.now(), exitCode: invoked.exitCode, evidencePath });
|
||||
const terminalStatus = validationBlocker ? "blocked" : "completed";
|
||||
const completed = await writeRunControl(context, { ...run, status: terminalStatus, stage: terminalStatus, evidencePath }, { status: terminalStatus, stage: terminalStatus, completedAt: context.now(), exitCode: invoked.exitCode, evidencePath });
|
||||
return ok("case.build", {
|
||||
caseId: run.caseId,
|
||||
runId: run.runId,
|
||||
@@ -579,7 +580,7 @@ async function buildCaseRunWithAuthorityMode(context: CaseContext, prepared: Pre
|
||||
summary: buildSummary(evidence),
|
||||
evidence,
|
||||
run: completed.run
|
||||
}, "completed");
|
||||
}, terminalStatus);
|
||||
}
|
||||
|
||||
export function summarizeHwpodOperationForTest(payload: any, document: any, expectedOp = "") {
|
||||
@@ -989,15 +990,21 @@ async function pollAgentResult(context: CaseContext, input: { baseUrl: string; t
|
||||
await updateRun(context, input.run, { stage: "agent-running", agent: runningAgent });
|
||||
}
|
||||
const terminal = agentTerminalEvidence(lastBody);
|
||||
const status = text(lastBody?.status);
|
||||
if ((status && status !== "running") || terminal.terminal) {
|
||||
return { stageStatus: status && status !== "running" ? status : terminal.terminalStatus || "completed", body: lastBody, polls, timedOut: false, lastPollAt, lastHttpStatus: lastStatus, error: null };
|
||||
const terminalStatus = agentResultTerminalStatus(lastBody?.status, terminal);
|
||||
if (terminalStatus) {
|
||||
return { stageStatus: terminalStatus, body: lastBody, polls, timedOut: false, lastPollAt, lastHttpStatus: lastStatus, error: null };
|
||||
}
|
||||
await context.sleep(pollIntervalMs);
|
||||
}
|
||||
return { stageStatus: "timeout", body: lastBody, polls, timedOut: true, lastPollAt, lastHttpStatus: lastStatus, error: { code: "agent_task_timeout", traceId: input.traceId, timeoutMs, lastHttpStatus: lastStatus, lastBody: compactObject(lastBody) } };
|
||||
}
|
||||
|
||||
export function agentResultTerminalStatus(statusValue: unknown, terminal: ReturnType<typeof agentTerminalEvidence>) {
|
||||
const status = text(statusValue).toLowerCase();
|
||||
if (["completed", "succeeded", "failed", "cancelled", "canceled", "timed_out", "timeout", "natural_end"].includes(status)) return status;
|
||||
return terminal.terminal ? terminal.terminalStatus || terminal.commandStatus || terminal.agentRunStatus || "completed" : "";
|
||||
}
|
||||
|
||||
async function recoverHwpodInvocation(context: CaseContext, run: PreparedCaseRun) {
|
||||
const planId = text(context.env.HWLAB_HWPOD_OPERATION_IDENTITY);
|
||||
if (!planId) throw Object.assign(new Error("HWPOD recovery requires the stable operation identity"), { code: "hwpod_recovery_identity_required" });
|
||||
|
||||
Reference in New Issue
Block a user