Merge pull request #388 from pikasTech/fix/nonblocking-frozen-provenance-v2
Pipelines as Code CI / agentrun-nc01-v02-ci-2d2139d4eea00d606ac6e1c680a0fb9d4e700a34 Success
Pipelines as Code CI / agentrun-nc01-v02-ci-2d2139d4eea00d606ac6e1c680a0fb9d4e700a34 Success
fix: 将续跑 provenance 漂移降级为告警
This commit is contained in:
@@ -56,6 +56,24 @@ interface ResourceSkillWarning extends JsonRecord {
|
||||
valuesPrinted: false;
|
||||
}
|
||||
|
||||
interface PrimaryWorkspaceContractWarning extends JsonRecord {
|
||||
code:
|
||||
| "primary-workspace-declared-source-drift"
|
||||
| "primary-workspace-assembly-contract-drift"
|
||||
| "primary-workspace-resolution-authority-drift"
|
||||
| "primary-workspace-declared-commit-drift"
|
||||
| "primary-workspace-provenance-hash-drift"
|
||||
| "primary-workspace-provenance-schema-drift"
|
||||
| "primary-workspace-frozen-tree-drift"
|
||||
| "primary-workspace-origin-drift";
|
||||
severity: "warning";
|
||||
scope: "resource-bundle.primaryWorkspace.provenance";
|
||||
message: string;
|
||||
action: "continue-with-frozen-provenance";
|
||||
blocking: false;
|
||||
valuesPrinted: false;
|
||||
}
|
||||
|
||||
interface GitCheckout {
|
||||
repoUrl: string;
|
||||
fetchRepoUrl: string;
|
||||
@@ -136,7 +154,7 @@ interface PrimaryWorkspaceProvenance extends JsonRecord {
|
||||
|
||||
type PrimaryWorkspaceState =
|
||||
| { kind: "uninitialized" }
|
||||
| { kind: "continuation"; provenance: PrimaryWorkspaceProvenance };
|
||||
| { kind: "continuation"; provenance: PrimaryWorkspaceProvenance; warnings: PrimaryWorkspaceContractWarning[] };
|
||||
|
||||
const primaryWorkspaceProvenanceFile = "agentrun-primary-workspace.json";
|
||||
|
||||
@@ -161,9 +179,11 @@ export async function materializeResourceBundle(resourceBundleRef: ResourceBundl
|
||||
let materializedBundles: MaterializedGitBundle[];
|
||||
let checkoutPath: string | null = null;
|
||||
let workspaceLifecycle: "first-materialization" | "continuation-reuse";
|
||||
let workspaceWarnings: PrimaryWorkspaceContractWarning[] = [];
|
||||
|
||||
if (initialState.kind === "continuation") {
|
||||
provenance = initialState.provenance;
|
||||
workspaceWarnings = initialState.warnings;
|
||||
materializedBundles = provenance.bundleProvenance;
|
||||
workspaceLifecycle = "continuation-reuse";
|
||||
} else {
|
||||
@@ -197,6 +217,7 @@ export async function materializeResourceBundle(resourceBundleRef: ResourceBundl
|
||||
throw new AgentRunError("infra-failed", "primary workspace installation did not produce durable provenance", { httpStatus: 500, details: { workspacePath: pathSummary(workspacePath), valuesPrinted: false } });
|
||||
}
|
||||
provenance = installedState.provenance;
|
||||
workspaceWarnings = installedState.warnings;
|
||||
materializedBundles = provenance.bundleProvenance;
|
||||
workspaceLifecycle = installed ? "first-materialization" : "continuation-reuse";
|
||||
if (!installed) checkoutPath = null;
|
||||
@@ -205,7 +226,7 @@ export async function materializeResourceBundle(resourceBundleRef: ResourceBundl
|
||||
}
|
||||
}
|
||||
|
||||
const primaryWorkspace = await verifyPrimaryWorkspace(workspacePath, workspaceRef, provenance, workspaceLifecycle, false);
|
||||
const primaryWorkspace = await verifyPrimaryWorkspace(workspacePath, workspaceRef, provenance, workspaceLifecycle, false, workspaceWarnings);
|
||||
const managedOverlayRoot = await verifyManagedOverlay(workspacePath, materializedBundles);
|
||||
const tools = await prepareGitBundleTools(managedOverlayRoot, env);
|
||||
const skills = await discoverGitBundleSkills(managedOverlayRoot, materializedBundles);
|
||||
@@ -253,7 +274,11 @@ export async function materializeResourceBundle(resourceBundleRef: ResourceBundl
|
||||
tools: tools.event,
|
||||
skillDirs: skills.event,
|
||||
requiredSkills: requiredSkills.event,
|
||||
warnings: requiredSkills.warnings,
|
||||
warnings: {
|
||||
count: workspaceWarnings.length + Number(requiredSkills.warnings.count ?? 0),
|
||||
items: [...workspaceWarnings, ...((requiredSkills.warnings.items as JsonRecord[] | undefined) ?? [])],
|
||||
valuesPrinted: false,
|
||||
},
|
||||
promptRefs: prompts.event,
|
||||
initialPrompt: initialPrompt?.summary ?? { available: false, bytes: 0, sha256: null, promptRefCount: prompts.items.length, skillCount: skills.items.length, valuesPrinted: false },
|
||||
valuesPrinted: false,
|
||||
@@ -447,12 +472,13 @@ async function inspectPrimaryWorkspace(workspacePath: string, declaredSource: Pr
|
||||
throw new AgentRunError("infra-failed", "primary workspace directory cannot be read", { httpStatus: 500, details: { workspacePath: pathSummary(workspacePath), error: fileErrorSummary(error), valuesPrinted: false } });
|
||||
}
|
||||
if (entries.length === 0) return { kind: "uninitialized" };
|
||||
const provenance = await readPrimaryWorkspaceProvenance(workspacePath);
|
||||
assertPrimaryWorkspaceProvenanceContract(provenance, declaredSource, assemblyContractHash, bundleContractHash);
|
||||
return { kind: "continuation", provenance };
|
||||
const parsed = await readPrimaryWorkspaceProvenance(workspacePath);
|
||||
const provenance = parsed.provenance;
|
||||
const warnings = [...parsed.warnings, ...primaryWorkspaceProvenanceContractWarnings(provenance, declaredSource, assemblyContractHash, bundleContractHash)];
|
||||
return { kind: "continuation", provenance, warnings };
|
||||
}
|
||||
|
||||
async function readPrimaryWorkspaceProvenance(workspacePath: string): Promise<PrimaryWorkspaceProvenance> {
|
||||
async function readPrimaryWorkspaceProvenance(workspacePath: string): Promise<{ provenance: PrimaryWorkspaceProvenance; warnings: PrimaryWorkspaceContractWarning[] }> {
|
||||
const file = path.join(workspacePath, ".git", primaryWorkspaceProvenanceFile);
|
||||
let text: string;
|
||||
try {
|
||||
@@ -475,11 +501,19 @@ async function readPrimaryWorkspaceProvenance(workspacePath: string): Promise<Pr
|
||||
const provenanceHash = requiredProvenanceString(parsed, "provenanceHash");
|
||||
const hashBody = { ...parsed };
|
||||
delete hashBody.provenanceHash;
|
||||
const warnings: PrimaryWorkspaceContractWarning[] = [];
|
||||
if (stableHash(hashBody) !== provenanceHash) {
|
||||
throw new AgentRunError("schema-invalid", "primary workspace provenance hash does not match its content", { httpStatus: 400, details: { provenancePath: pathSummary(file), valuesPrinted: false } });
|
||||
warnings.push(primaryWorkspaceWarning("primary-workspace-provenance-hash-drift", "primary workspace provenance hash differs from its content", {
|
||||
provenancePath: pathSummary(file),
|
||||
declaredProvenanceHash: provenanceHash,
|
||||
actualProvenanceHash: stableHash(hashBody),
|
||||
}));
|
||||
}
|
||||
if (parsed.schemaVersion !== 1) {
|
||||
throw new AgentRunError("schema-invalid", "primary workspace provenance schemaVersion is unsupported", { httpStatus: 400, details: { schemaVersion: parsed.schemaVersion ?? null, valuesPrinted: false } });
|
||||
warnings.push(primaryWorkspaceWarning("primary-workspace-provenance-schema-drift", "primary workspace provenance schemaVersion differs from the current reader", {
|
||||
declaredSchemaVersion: parsed.schemaVersion ?? null,
|
||||
supportedSchemaVersion: 1,
|
||||
}));
|
||||
}
|
||||
const declaredSourceRecord = requiredProvenanceObject(parsed, "declaredSource");
|
||||
const resolvedSourceRecord = requiredProvenanceObject(parsed, "resolvedSource");
|
||||
@@ -511,23 +545,55 @@ async function readPrimaryWorkspaceProvenance(workspacePath: string): Promise<Pr
|
||||
bundleProvenance: parsed.bundleProvenance.map((item, index) => parseMaterializedBundleProvenance(item, index)),
|
||||
provenanceHash,
|
||||
};
|
||||
return provenance;
|
||||
return { provenance, warnings };
|
||||
}
|
||||
|
||||
function assertPrimaryWorkspaceProvenanceContract(provenance: PrimaryWorkspaceProvenance, declaredSource: PrimarySourceContract, assemblyContractHash: string, bundleContractHash: string): void {
|
||||
function primaryWorkspaceWarning(code: PrimaryWorkspaceContractWarning["code"], message: string, details: JsonRecord): PrimaryWorkspaceContractWarning {
|
||||
return {
|
||||
code,
|
||||
severity: "warning",
|
||||
scope: "resource-bundle.primaryWorkspace.provenance",
|
||||
message,
|
||||
action: "continue-with-frozen-provenance",
|
||||
blocking: false,
|
||||
...details,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function primaryWorkspaceProvenanceContractWarnings(provenance: PrimaryWorkspaceProvenance, declaredSource: PrimarySourceContract, assemblyContractHash: string, bundleContractHash: string): PrimaryWorkspaceContractWarning[] {
|
||||
const warnings: PrimaryWorkspaceContractWarning[] = [];
|
||||
const warning = (code: PrimaryWorkspaceContractWarning["code"], message: string, details: JsonRecord): void => {
|
||||
warnings.push(primaryWorkspaceWarning(code, message, details));
|
||||
};
|
||||
if (stableHash(provenance.declaredSource) !== stableHash(declaredSource)) {
|
||||
throw new AgentRunError("schema-invalid", "primary workspace declared source conflicts with frozen provenance", { httpStatus: 400, details: { expectedSourceHash: stableHash(declaredSource), actualSourceHash: stableHash(provenance.declaredSource), valuesPrinted: false } });
|
||||
warning("primary-workspace-declared-source-drift", "primary workspace declared source differs from frozen provenance", {
|
||||
declaredSourceHash: stableHash(declaredSource),
|
||||
frozenSourceHash: stableHash(provenance.declaredSource),
|
||||
});
|
||||
}
|
||||
if (provenance.assemblyContractHash !== assemblyContractHash || provenance.bundleContractHash !== bundleContractHash) {
|
||||
throw new AgentRunError("schema-invalid", "primary workspace assembly contract conflicts with frozen provenance", { httpStatus: 400, details: { expectedAssemblyContractHash: assemblyContractHash, actualAssemblyContractHash: provenance.assemblyContractHash, expectedBundleContractHash: bundleContractHash, actualBundleContractHash: provenance.bundleContractHash, valuesPrinted: false } });
|
||||
warning("primary-workspace-assembly-contract-drift", "primary workspace assembly contract differs from frozen provenance", {
|
||||
declaredAssemblyContractHash: assemblyContractHash,
|
||||
frozenAssemblyContractHash: provenance.assemblyContractHash,
|
||||
declaredBundleContractHash: bundleContractHash,
|
||||
frozenBundleContractHash: provenance.bundleContractHash,
|
||||
});
|
||||
}
|
||||
const expectedAuthority = declaredSource.requestedCommitId ? "declared-commit" : "runner-first-materialization";
|
||||
if (provenance.resolutionAuthority !== expectedAuthority) {
|
||||
throw new AgentRunError("schema-invalid", "primary workspace resolution authority conflicts with declared source", { httpStatus: 400, details: { expectedAuthority, actualAuthority: provenance.resolutionAuthority, valuesPrinted: false } });
|
||||
warning("primary-workspace-resolution-authority-drift", "primary workspace resolution authority differs from frozen provenance", {
|
||||
declaredAuthority: expectedAuthority,
|
||||
frozenAuthority: provenance.resolutionAuthority,
|
||||
});
|
||||
}
|
||||
if (declaredSource.requestedCommitId && provenance.resolvedSource.commitId !== declaredSource.requestedCommitId) {
|
||||
throw new AgentRunError("schema-invalid", "primary workspace frozen commit conflicts with declared commit", { httpStatus: 400, details: { declaredCommit: declaredSource.requestedCommitId, frozenCommit: provenance.resolvedSource.commitId, valuesPrinted: false } });
|
||||
warning("primary-workspace-declared-commit-drift", "primary workspace declared commit differs from frozen provenance", {
|
||||
declaredCommit: declaredSource.requestedCommitId,
|
||||
frozenCommit: provenance.resolvedSource.commitId,
|
||||
});
|
||||
}
|
||||
return warnings;
|
||||
}
|
||||
|
||||
function parseMaterializedBundleProvenance(value: unknown, index: number): MaterializedGitBundle {
|
||||
@@ -683,7 +749,7 @@ async function assertCleanPrimarySourceView(workspacePath: string): Promise<void
|
||||
}
|
||||
}
|
||||
|
||||
async function verifyPrimaryWorkspace(workspacePath: string, workspaceRef: WorkspaceRef, provenance: PrimaryWorkspaceProvenance, workspaceLifecycle: "first-materialization" | "continuation-reuse", requireExactSource: boolean): Promise<JsonRecord> {
|
||||
async function verifyPrimaryWorkspace(workspacePath: string, workspaceRef: WorkspaceRef, provenance: PrimaryWorkspaceProvenance, workspaceLifecycle: "first-materialization" | "continuation-reuse", requireExactSource: boolean, warnings: PrimaryWorkspaceContractWarning[] = []): Promise<JsonRecord> {
|
||||
let rootStat;
|
||||
let gitStat;
|
||||
try {
|
||||
@@ -711,12 +777,19 @@ async function verifyPrimaryWorkspace(workspacePath: string, workspaceRef: Works
|
||||
}
|
||||
const frozenTreeId = (await git(["rev-parse", `${frozenCommit}^{tree}`], workspacePath)).stdout.trim();
|
||||
if (frozenTreeId !== provenance.resolvedSource.treeId) {
|
||||
throw new AgentRunError("schema-invalid", "primary workspace frozen source tree conflicts with provenance", { httpStatus: 400, details: { frozenCommit, expectedTreeId: provenance.resolvedSource.treeId, actualTreeId: frozenTreeId, valuesPrinted: false } });
|
||||
warnings.push(primaryWorkspaceWarning("primary-workspace-frozen-tree-drift", "primary workspace frozen source tree differs from provenance", {
|
||||
frozenCommit,
|
||||
declaredTreeId: provenance.resolvedSource.treeId,
|
||||
actualTreeId: frozenTreeId,
|
||||
}));
|
||||
}
|
||||
const branch = (await git(["branch", "--show-current"], workspacePath)).stdout.trim() || null;
|
||||
const origin = (await git(["remote", "get-url", "origin"], workspacePath)).stdout.trim();
|
||||
if (origin !== provenance.declaredSource.repoUrl) {
|
||||
throw new AgentRunError("schema-invalid", "primary workspace origin conflicts with declared source authority", { httpStatus: 400, details: { declared: gitRemoteSummary(provenance.declaredSource.repoUrl), actual: gitRemoteSummary(origin), valuesPrinted: false } });
|
||||
warnings.push(primaryWorkspaceWarning("primary-workspace-origin-drift", "primary workspace origin differs from provenance", {
|
||||
declared: gitRemoteSummary(provenance.declaredSource.repoUrl),
|
||||
actual: gitRemoteSummary(origin),
|
||||
}));
|
||||
}
|
||||
const probe = path.join(workspacePath, `.agentrun-write-probe-${stableHash({ workspacePath, actualCommit }).slice(0, 12)}`);
|
||||
try {
|
||||
|
||||
@@ -5,7 +5,6 @@ import { access, mkdir, readFile, rename, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { AgentRunError } from "../../common/errors.js";
|
||||
import type { JsonRecord, ResourceBundleRef, WorkspaceRef } from "../../common/types.js";
|
||||
import { stableHash } from "../../common/validation.js";
|
||||
import { ManagerClient } from "../../mgr/client.js";
|
||||
import { startManagerServer } from "../../mgr/server.js";
|
||||
import { MemoryAgentRunStore } from "../../mgr/store.js";
|
||||
@@ -115,24 +114,22 @@ const selfTest: SelfTestCase = async (context) => {
|
||||
assert.match(replayStatus, / M tools\/source-tool/u);
|
||||
assert.match(replayStatus, /\?\? dirty-from-previous-attempt\.txt/u);
|
||||
|
||||
await assert.rejects(
|
||||
() => materializeResourceBundle({ ...resourceBundleRef, ref: "changed-source-ref" }, workspaceRef, { ...env, AGENTRUN_RUN_ID: "run_changed_source_contract" }),
|
||||
(error) => error instanceof AgentRunError && error.failureKind === "schema-invalid" && /declared source conflicts/u.test(error.message),
|
||||
);
|
||||
await assert.rejects(
|
||||
() => materializeResourceBundle({ ...resourceBundleRef, bundles: [{ ...resourceBundleRef.bundles[0]!, targetPath: "alternate-tools" }] }, workspaceRef, { ...env, AGENTRUN_RUN_ID: "run_changed_assembly_contract" }),
|
||||
(error) => error instanceof AgentRunError && error.failureKind === "schema-invalid" && /assembly contract conflicts/u.test(error.message),
|
||||
);
|
||||
const sourceDrift = await materializeResourceBundle({ ...resourceBundleRef, ref: "changed-source-ref" }, workspaceRef, { ...env, AGENTRUN_RUN_ID: "run_changed_source_contract" });
|
||||
assert.ok(sourceDrift);
|
||||
assert.equal((sourceDrift.event.primaryWorkspace as JsonRecord).sourceCommit, primary.commitId);
|
||||
assert.equal((sourceDrift.event.primaryWorkspace as JsonRecord).workspaceLifecycle, "continuation-reuse");
|
||||
assert.ok((((sourceDrift.event.warnings as JsonRecord).items as JsonRecord[])).some((item) => item.code === "primary-workspace-declared-source-drift" && item.blocking === false));
|
||||
const assemblyDrift = await materializeResourceBundle({ ...resourceBundleRef, bundles: [{ ...resourceBundleRef.bundles[0]!, targetPath: "alternate-tools" }] }, workspaceRef, { ...env, AGENTRUN_RUN_ID: "run_changed_assembly_contract" });
|
||||
assert.ok(assemblyDrift);
|
||||
assert.equal((assemblyDrift.event.primaryWorkspace as JsonRecord).sourceCommit, primary.commitId);
|
||||
assert.ok((((assemblyDrift.event.warnings as JsonRecord).items as JsonRecord[])).some((item) => item.code === "primary-workspace-assembly-contract-drift" && item.blocking === false));
|
||||
const tamperedProvenance = JSON.parse(await readFile(provenancePath, "utf8")) as JsonRecord;
|
||||
(tamperedProvenance.resolvedSource as JsonRecord).treeId = "0000000000000000000000000000000000000000";
|
||||
const tamperedBody = { ...tamperedProvenance };
|
||||
delete tamperedBody.provenanceHash;
|
||||
tamperedProvenance.provenanceHash = stableHash(tamperedBody);
|
||||
await writeFile(provenancePath, `${JSON.stringify(tamperedProvenance, null, 2)}\n`, "utf8");
|
||||
await assert.rejects(
|
||||
() => materializeResourceBundle(resourceBundleRef, workspaceRef, { ...env, AGENTRUN_RUN_ID: "run_tampered_frozen_tree" }),
|
||||
(error) => error instanceof AgentRunError && error.failureKind === "schema-invalid" && /frozen source tree conflicts/u.test(error.message),
|
||||
);
|
||||
const provenanceDrift = await materializeResourceBundle(resourceBundleRef, workspaceRef, { ...env, AGENTRUN_RUN_ID: "run_tampered_frozen_tree" });
|
||||
assert.ok(provenanceDrift);
|
||||
assert.ok((((provenanceDrift.event.warnings as JsonRecord).items as JsonRecord[])).some((item) => item.code === "primary-workspace-provenance-hash-drift" && item.blocking === false));
|
||||
assert.ok((((provenanceDrift.event.warnings as JsonRecord).items as JsonRecord[])).some((item) => item.code === "primary-workspace-frozen-tree-drift" && item.blocking === false));
|
||||
|
||||
const missingCredentialWorkspace = path.join(context.tmp, "primary-workspace-missing-credential");
|
||||
await assert.rejects(
|
||||
@@ -270,9 +267,9 @@ const selfTest: SelfTestCase = async (context) => {
|
||||
"primary-workspace-continuation-does-not-fetch",
|
||||
"primary-workspace-continuation-preserves-successor-commit",
|
||||
"primary-workspace-continuation-reuses-bundle-provenance",
|
||||
"primary-workspace-source-contract-conflict-fails-closed",
|
||||
"primary-workspace-assembly-contract-conflict-fails-closed",
|
||||
"primary-workspace-frozen-tree-provenance-verified",
|
||||
"primary-workspace-source-contract-drift-warns-and-continues",
|
||||
"primary-workspace-assembly-contract-drift-warns-and-continues",
|
||||
"primary-workspace-provenance-drift-warns-and-continues",
|
||||
"primary-workspace-nonempty-unowned-root-fails-closed",
|
||||
"primary-workspace-credential-missing-typed-blocker",
|
||||
"primary-workspace-ref-not-found-first-error",
|
||||
|
||||
Reference in New Issue
Block a user