From 83d98ebf074aa56e9585474c2f4448a05e8ed38d Mon Sep 17 00:00:00 2001 From: Code Queue Review Date: Sat, 23 May 2026 12:51:02 +0000 Subject: [PATCH] fix: resolve dev cd artifact promotion boundary --- docs/reference/deployment-publish.md | 23 +- scripts/src/dev-cd-apply.mjs | 338 +++++++++++++++++++++++--- scripts/src/dev-cd-apply.test.mjs | 344 ++++++++++++++++++++++++++- 3 files changed, 667 insertions(+), 38 deletions(-) diff --git a/docs/reference/deployment-publish.md b/docs/reference/deployment-publish.md index 5c6068e9..01f53897 100644 --- a/docs/reference/deployment-publish.md +++ b/docs/reference/deployment-publish.md @@ -52,6 +52,19 @@ latest-main 发布时它可以等于当前 target ref；artifact merge commit commander 可以始终从最新 `origin/main` 控制入口执行正式 DEV CD，而不需要 checkout promotion parent 或手工拼装发布环境。 +状态面和事务报告会显式展示这条边界：`target.controlRef` 是最新 +`origin/main` 控制入口，`target.promotionCommit` 是从 `deploy/deploy.json` +解析出的真实 promotion commit，`target.artifactBoundary` 记录最近一次触碰 +`deploy/deploy.json`、`deploy/artifact-catalog.dev.json`、 +`deploy/k8s/base/workloads.yaml`、`reports/dev-gate/dev-artifacts.json` 的 +first-parent artifact merge commit，以及该 merge commit 的 promotion parent +和 artifact/report parent。`artifactCatalog`、`artifactReport`、`deployJson` +分别记录 hash、commit、digest 计数和 publish source。若 deploy/catalog/report +或 artifact merge parent 不能同时指向同一个 promotion commit，状态降级为 +`degraded`，真实 `--apply` 在获取 Lease 前以 artifact-boundary blocker 停止。 +这保证 `deploy/deploy.json` 仍是唯一发布控制源；Lease 只串行化事务，不保存第 +二套 desired-state。 + Lease 只是发布平面的互斥锁，不是 desired-state 来源。Lease annotations 至少记录 `promotionCommit`、`deployJsonHash`、`ownerTaskId`、 `transactionId`、`phase`、`startedAt`、`updatedAt`、`ttlSeconds`、 @@ -69,10 +82,12 @@ previous holder 保留在审计 annotations/report 字段中。未过期 Lease `scripts/dev-cd-apply.mjs` 同时提供只读状态面。无参数、`--status`、或 `--dry-run` 都返回紧凑 JSON，覆盖 target ref/commit、 `deploy/deploy.json` commit/hash、当前 Lease 状态摘要、以及公开 -`16666/16667` live health identity 摘要。这些模式 host commander 和 -runner 都可以执行，但必须保持只读：不获取或替换 Lease，不 publish -artifact，不执行 `kubectl apply/rollout`，不写 report，不修改 -`deploy.json`/catalog，也不读取或打印 Secret value。kubectl 或 kubeconfig +`16666/16667` live health identity 摘要。状态面还包含 `liveDelta`，用于对比 +当前公开 live commit 与 `target.promotionCommit`；这只是 rollout 口径，不会在 +status/dry-run 中执行 rollout。这些模式 host commander 和 runner 都可以执行， +但必须保持只读：不获取或替换 Lease，不 publish artifact，不执行 +`kubectl apply/rollout`，不写 report，不修改 `deploy.json`/catalog，也不读取或 +打印 Secret value。kubectl 或 kubeconfig 不可用时，状态面应返回 `lock.status=unavailable` 和脱敏原因，而不是把 只读观测失败升级成写路径 blocker。 diff --git a/scripts/src/dev-cd-apply.mjs b/scripts/src/dev-cd-apply.mjs index 74a4ac72..d7960172 100644 --- a/scripts/src/dev-cd-apply.mjs +++ b/scripts/src/dev-cd-apply.mjs @@ -18,13 +18,21 @@ const defaultNamespace = "hwlab-dev"; const defaultLockName = "hwlab-dev-cd-lock"; const defaultTtlSeconds = 3600; const defaultReportPath = "reports/dev-gate/dev-cd-apply.json"; +const artifactCatalogPath = "deploy/artifact-catalog.dev.json"; const artifactReportPath = "reports/dev-gate/dev-artifacts.json"; +const artifactDesiredStatePaths = [ + "deploy/artifact-catalog.dev.json", + "deploy/deploy.json", + "deploy/k8s/base/workloads.yaml", + "reports/dev-gate/dev-artifacts.json" +]; const runtimeProvisioningReportPath = "reports/dev-gate/dev-runtime-provisioning-report.json"; const runtimeMigrationReportPath = "reports/dev-gate/dev-runtime-migration-report.json"; const runtimePostflightReportPath = "reports/dev-gate/dev-runtime-postflight-report.json"; const runtimeJobReportPrefix = "/tmp/hwlab-dev-runtime-report"; const browserLiveUrl = "http://74.48.78.17:16666/health/live"; const apiLiveUrl = `${DEV_ENDPOINT}/health/live`; +const digestPattern = /^sha256:[a-f0-9]{64}$/u; const lockAnnotationPrefix = "hwlab.pikastech.local"; const lockAnnotationFields = { promotionCommit: `${lockAnnotationPrefix}/promotionCommit`, @@ -291,6 +299,87 @@ async function readDeployJson(repoRoot) { }; } +async function readJsonFileSummary(repoRoot, relativePath, summarize) { + try { + const raw = await readFile(path.join(repoRoot, relativePath), "utf8"); + const json = JSON.parse(raw); + return { + path: relativePath, + hash: sha256(raw), + exists: true, + ...summarize(json) + }; + } catch (error) { + if (error?.code === "ENOENT") { + return { + path: relativePath, + hash: null, + exists: false, + error: "missing" + }; + } + throw error; + } +} + +function servicesArray(value) { + if (Array.isArray(value)) return value; + if (value && typeof value === "object") return Object.values(value); + return []; +} + +function summarizeArtifactCatalog(catalog) { + const services = servicesArray(catalog?.services); + const digestCounts = { sha256: 0, notPublished: 0, invalid: 0 }; + for (const service of services) { + if (digestPattern.test(service?.digest ?? "")) digestCounts.sha256 += 1; + else if (service?.digest === "not_published") digestCounts.notPublished += 1; + else digestCounts.invalid += 1; + } + return { + kind: catalog?.kind ?? null, + commitId: catalog?.commitId ?? null, + artifactState: catalog?.artifactState ?? null, + ciPublished: catalog?.publish?.ciPublished ?? null, + registryVerified: catalog?.publish?.registryVerified ?? null, + provenance: catalog?.publish?.provenance ?? null, + serviceCount: services.length, + digestCounts + }; +} + +function summarizeArtifactReport(report) { + const publish = report?.artifactPublish ?? {}; + const services = servicesArray(publish.services); + const requiredServices = services.filter((service) => service?.artifactRequired !== false); + const publishedRequiredServices = requiredServices.filter((service) => service?.status === "published"); + const digestCounts = { sha256: 0, notPublished: 0, invalid: 0 }; + for (const service of services) { + if (digestPattern.test(service?.digest ?? "")) digestCounts.sha256 += 1; + else if (service?.digest === "not_published") digestCounts.notPublished += 1; + else digestCounts.invalid += 1; + } + return { + commitId: report?.commitId ?? null, + status: report?.status ?? null, + sourceCommitId: publish.sourceCommitId ?? null, + registryPrefix: publish.registryPrefix ?? null, + serviceCount: publish.serviceCount ?? services.length, + requiredServiceCount: publish.requiredServiceCount ?? requiredServices.length, + publishedCount: publish.publishedCount ?? publishedRequiredServices.length, + digestCounts, + generatedAt: publish.generatedAt ?? null + }; +} + +async function readArtifactEvidence(repoRoot) { + const [catalog, report] = await Promise.all([ + readJsonFileSummary(repoRoot, artifactCatalogPath, summarizeArtifactCatalog), + readJsonFileSummary(repoRoot, artifactReportPath, summarizeArtifactReport) + ]); + return { catalog, report }; +} + async function resolveTargetRef(ctx, targetRef) { const target = await ctx.runCommand("git", ["rev-parse", "--verify", `${targetRef}^{commit}`], { cwd: ctx.repoRoot, @@ -325,6 +414,130 @@ async function resolveTargetRef(ctx, targetRef) { }; } +async function gitOutput(ctx, args, { allowFailure = false, timeoutMs = 10000 } = {}) { + const result = await ctx.runCommand("git", args, { + cwd: ctx.repoRoot, + timeoutMs + }); + if (result.code !== 0) { + if (allowFailure) return null; + throw new DevCdApplyError("git command failed while resolving DEV CD provenance", { + code: "git-provenance-failed", + command: shellCommand("git", args), + stderr: redactSensitiveText(result.stderr || result.stdout) + }); + } + return result.stdout.trim(); +} + +async function commitParents(ctx, commitId) { + const line = await gitOutput(ctx, ["rev-list", "--parents", "-n", "1", commitId], { allowFailure: true }); + if (!line) return []; + return line.split(/\s+/u).slice(1); +} + +async function shortGitCommit(ctx, commitId) { + const value = await gitOutput(ctx, ["rev-parse", "--short=7", commitId], { allowFailure: true }); + return value || shortCommit(commitId); +} + +async function filesChangedBetween(ctx, left, right) { + const output = await gitOutput(ctx, ["diff-tree", "--no-commit-id", "--name-only", "-r", left, right], { + allowFailure: true + }); + if (!output) return []; + return output.split(/\r?\n/u).map((item) => item.trim()).filter(Boolean); +} + +async function latestArtifactMergeCommit(ctx, controlCommit) { + const output = await gitOutput(ctx, [ + "log", + "--first-parent", + "--format=%H", + "-n", + "1", + controlCommit, + "--", + ...artifactDesiredStatePaths + ], { allowFailure: true }); + return output?.split(/\r?\n/u).find(Boolean) ?? null; +} + +async function resolveArtifactBoundary(ctx, control, promotion, deployBefore, artifactEvidence) { + const mergeCommitId = await latestArtifactMergeCommit(ctx, control.commitId); + if (!mergeCommitId) { + return { + status: "unavailable", + reason: "No first-parent commit touching artifact desired-state files was found.", + desiredStatePaths: artifactDesiredStatePaths + }; + } + + const parents = await commitParents(ctx, mergeCommitId); + const mergeShortCommitId = await shortGitCommit(ctx, mergeCommitId); + const firstParent = parents[0] ?? null; + const secondParent = parents[1] ?? null; + const promotionParentMatches = firstParent ? commitMatches(firstParent, promotion.commitId) : false; + const changedPaths = firstParent ? await filesChangedBetween(ctx, firstParent, mergeCommitId) : []; + const touchedDesiredStatePaths = artifactDesiredStatePaths.filter((entry) => changedPaths.includes(entry)); + const artifactCommitId = secondParent || mergeCommitId; + const artifactShortCommitId = await shortGitCommit(ctx, artifactCommitId); + const artifactParents = secondParent ? await commitParents(ctx, secondParent) : []; + const artifactParentMatches = artifactParents.some((parent) => commitMatches(parent, promotion.commitId)); + const catalogCommitMatches = commitMatches(artifactEvidence.catalog.commitId, promotion.commitId); + const reportSource = artifactEvidence.report.sourceCommitId ?? artifactEvidence.report.commitId; + const reportCommitMatches = commitMatches(reportSource, promotion.commitId); + const deployCommitMatches = commitMatches(deployBefore.commitId, promotion.commitId); + const isMergeCommit = parents.length >= 2; + const status = ( + deployCommitMatches && + catalogCommitMatches && + reportCommitMatches && + ( + promotionParentMatches || + artifactParentMatches || + commitMatches(mergeCommitId, promotion.commitId) + ) + ) ? "pass" : "degraded"; + + return { + status, + controlCommit: { + commitId: control.commitId, + shortCommitId: control.shortCommitId + }, + artifactMergeCommit: { + commitId: mergeCommitId, + shortCommitId: mergeShortCommitId, + isMergeCommit, + parents, + promotionParent: firstParent, + artifactParent: secondParent, + promotionParentMatches, + changedPaths: touchedDesiredStatePaths + }, + artifactCommit: { + commitId: artifactCommitId, + shortCommitId: artifactShortCommitId, + parents: artifactParents, + promotionParentMatches: artifactParentMatches + }, + desiredState: { + deployCommitId: deployBefore.commitId, + deployJsonHash: deployBefore.hash, + catalogCommitId: artifactEvidence.catalog.commitId, + catalogHash: artifactEvidence.catalog.hash, + reportCommitId: artifactEvidence.report.commitId, + reportSourceCommitId: artifactEvidence.report.sourceCommitId, + reportHash: artifactEvidence.report.hash, + deployCommitMatches, + catalogCommitMatches, + reportCommitMatches, + desiredStatePaths: artifactDesiredStatePaths + } + }; +} + async function resolveCommitRef(ctx, ref) { const result = await ctx.runCommand("git", ["rev-parse", "--verify", `${ref}^{commit}`], { cwd: ctx.repoRoot, @@ -376,7 +589,7 @@ async function checkPromotionDesiredState(ctx, promotionCommit) { return compactDesiredStateCheck(result); } -async function resolveEffectiveTarget(ctx, args, deployBefore) { +async function resolveEffectiveTarget(ctx, args, deployBefore, artifactEvidence = null) { const control = await resolveTargetRef(ctx, args.targetRef); if (deployJsonMatchesTarget(deployBefore, control)) { return { @@ -388,7 +601,10 @@ async function resolveEffectiveTarget(ctx, args, deployBefore) { }, promotionSource: "target-ref", publishRequired: true, - desiredStateCheck: null + desiredStateCheck: null, + artifactBoundary: artifactEvidence + ? await resolveArtifactBoundary(ctx, control, control, deployBefore, artifactEvidence) + : null }; } @@ -404,7 +620,10 @@ async function resolveEffectiveTarget(ctx, args, deployBefore) { }, promotionSource: "target-ref", publishRequired: true, - desiredStateCheck + desiredStateCheck, + artifactBoundary: artifactEvidence + ? await resolveArtifactBoundary(ctx, control, promotion, deployBefore, artifactEvidence) + : null }; } @@ -414,7 +633,7 @@ async function resolveEffectiveTarget(ctx, args, deployBefore) { shortCommitId: promotion.shortCommitId, headCommitId: control.headCommitId, headShortCommitId: control.headShortCommitId, - headMatchesTarget: true, + headMatchesTarget: control.headMatchesTarget, controlRef: { ref: control.ref, commitId: control.commitId, @@ -422,7 +641,10 @@ async function resolveEffectiveTarget(ctx, args, deployBefore) { }, promotionSource: "deploy-json", publishRequired: false, - desiredStateCheck + desiredStateCheck, + artifactBoundary: artifactEvidence + ? await resolveArtifactBoundary(ctx, control, promotion, deployBefore, artifactEvidence) + : null }; } @@ -1441,6 +1663,24 @@ function compactLiveReport(live) { }; } +function buildLiveDelta(live, target) { + const expectedCommit = target?.shortCommitId ?? target?.commitId ?? null; + const endpoints = (live?.endpoints ?? []).map((endpoint) => ({ + id: endpoint.id, + observedCommit: endpoint.observedCommit ?? null, + expectedCommit, + matchesPromotion: expectedCommit ? commitMatches(endpoint.observedCommit, expectedCommit) : null, + status: endpoint.status + })); + const mismatched = endpoints.filter((endpoint) => endpoint.matchesPromotion === false); + return { + status: expectedCommit && mismatched.length === 0 && endpoints.length > 0 ? "pass" : expectedCommit ? "mismatch" : "unknown", + expectedCommit, + mismatched: mismatched.length, + endpoints + }; +} + async function readDeployLockStatus({ ctx, args, kubectlContext }) { const get = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "get", "lease", args.lockName, "-o", "json"], { timeoutMs: 15000 @@ -1516,7 +1756,8 @@ function buildStatusNextActions({ target, deployBefore, lock, liveVerify }) { actions.push("deploy/deploy.json does not match the target ref and the deploy-json promotion desired-state check failed; refresh artifacts/catalog or choose an explicit published promotion."); } if (!target.headMatchesTarget) { - actions.push(`Checkout or fast-forward to ${target.ref} ${target.shortCommitId} before apply.`); + const controlShortCommit = target.controlRef?.shortCommitId ?? target.shortCommitId; + actions.push(`Checkout or fast-forward to ${target.ref} ${controlShortCommit} before apply.`); } if (deployBefore.commitId !== target.shortCommitId && deployBefore.commitId !== target.commitId) { actions.push(`deploy/deploy.json commitId is ${deployBefore.commitId}; target ref is ${target.shortCommitId}.`); @@ -1531,6 +1772,10 @@ function buildStatusNextActions({ target, deployBefore, lock, liveVerify }) { if (liveVerify.status !== "pass") { actions.push("Inspect live endpoint identity/health before treating DEV as converged."); } + const liveDelta = buildLiveDelta(liveVerify, target); + if (liveDelta.status === "mismatch") { + actions.push(`Live DEV is still serving ${liveDelta.mismatched} endpoint(s) that do not match promotion ${target.shortCommitId}; host commander should run the controlled DEV rollout after the apply report is ready.`); + } if (actions.length === 0) { actions.push(target.publishRequired === false ? "Run --apply --confirm-dev --confirmed-non-production --write-report when ready to apply the published deploy.json promotion without republishing HEAD." @@ -1542,7 +1787,8 @@ function buildStatusNextActions({ target, deployBefore, lock, liveVerify }) { async function runDevCdStatus(args, ctx, stdout) { const generatedAt = ctx.now().toISOString(); const deployBefore = await readDeployJson(ctx.repoRoot); - const target = await resolveEffectiveTarget(ctx, args, deployBefore); + const artifactEvidence = await readArtifactEvidence(ctx.repoRoot); + const target = await resolveEffectiveTarget(ctx, args, deployBefore, artifactEvidence); const kubectlContext = await resolveKubectlContext(args, ctx.env, ctx.runCommand); const lockRead = await readDeployLockStatus({ ctx, args, kubectlContext }); const lock = summarizeLockStatus(lockRead, ctx.now()); @@ -1550,11 +1796,15 @@ async function runDevCdStatus(args, ctx, stdout) { ? { status: "not_run", reason: "--skip-live-verify", endpoints: [], summary: { checked: 0 } } : await verifyDevLive(ctx, { expectedCommit: null }); const deployMatchesTarget = deployJsonMatchesTarget(deployBefore, target); + const liveDelta = buildLiveDelta(liveVerify, target); const status = ( lock.status === "unavailable" || liveVerify.status !== "pass" || + liveDelta.status === "mismatch" || !target.headMatchesTarget || - !deployMatchesTarget + !deployMatchesTarget || + target.publishRequired === false && + target.artifactBoundary?.status === "degraded" ) ? "degraded" : "pass"; const report = { ok: true, @@ -1574,12 +1824,15 @@ async function runDevCdStatus(args, ctx, stdout) { headCommitId: target.headCommitId, headMatchesTarget: target.headMatchesTarget, desiredStateCheck: target.desiredStateCheck, + artifactBoundary: target.artifactBoundary, namespace: args.targetNamespace }, deployJson: { ...summarizeDeployJson(deployBefore), matchesTarget: deployMatchesTarget }, + artifactCatalog: artifactEvidence.catalog, + artifactReport: artifactEvidence.report, kubectl: { kubeconfigSource: kubectlContext.kubeconfigSource, kubeconfig: kubectlContext.kubeconfig ? "" : "", @@ -1587,6 +1840,7 @@ async function runDevCdStatus(args, ctx, stdout) { }, lock, live: compactLiveReport(liveVerify), + liveDelta, nextActions: buildStatusNextActions({ target, deployBefore, lock, liveVerify }), fullOutputHint: "Use --apply --confirm-dev --confirmed-non-production --write-report for the full transaction report; use --full-output only when stdout needs the full JSON." }; @@ -1743,6 +1997,7 @@ function buildReport({ headCommitId: target.headCommitId, headMatchesTarget: target.headMatchesTarget, desiredStateCheck: target.desiredStateCheck, + artifactBoundary: target.artifactBoundary, namespace: args.targetNamespace, apiEndpoint: DEV_ENDPOINT, browserEndpoint: "http://74.48.78.17:16666/" @@ -1759,14 +2014,14 @@ function buildReport({ steps, liveBefore, liveVerify, - reportPaths: { - transaction: args.writeReport ? args.reportPath : null, - artifacts: artifactReportPath, - deployApply: "reports/dev-gate/dev-deploy-report.json", - runtimeProvisioning: runtimeProvisioningReportPath, - runtimeMigration: runtimeMigrationReportPath, - runtimePostflight: runtimePostflightReportPath - }, + reportPaths: { + transaction: args.writeReport ? args.reportPath : null, + artifacts: artifactReportPath, + deployApply: "reports/dev-gate/dev-deploy-report.json", + runtimeProvisioning: runtimeProvisioningReportPath, + runtimeMigration: runtimeMigrationReportPath, + runtimePostflight: runtimePostflightReportPath + }, safety: { prodTouched: false, secretValuesRead: false, @@ -1865,7 +2120,8 @@ export async function runDevCdApply(argv, io = {}) { try { deployBefore = await readDeployJson(ctx.repoRoot); - target = await resolveEffectiveTarget(ctx, args, deployBefore); + const artifactEvidence = await readArtifactEvidence(ctx.repoRoot); + target = await resolveEffectiveTarget(ctx, args, deployBefore, artifactEvidence); if (target.desiredStateCheck?.status === "blocked") { throw new DevCdApplyError("deploy desired-state does not match the target ref or a published deploy.json promotion", { code: "desired-state-promotion-check-failed", @@ -1877,6 +2133,31 @@ export async function runDevCdApply(argv, io = {}) { }] }); } + if (target.publishRequired === false && target.artifactBoundary?.status === "degraded") { + blockers.push({ + type: "contract_blocker", + scope: "artifact-boundary", + status: "open", + summary: "deploy/deploy.json, deploy/artifact-catalog.dev.json, reports/dev-gate/dev-artifacts.json, and the artifact merge parent must resolve to the same promotion commit before apply." + }); + throw new DevCdApplyError("artifact desired-state provenance does not match the deploy.json promotion", { + code: "artifact-boundary-provenance-mismatch", + suppressCatchBlocker: true + }); + } + if (!target.headMatchesTarget) { + const controlShortCommit = target.controlRef?.shortCommitId ?? target.shortCommitId; + blockers.push({ + type: "contract_blocker", + scope: "target-ref-head", + status: "open", + summary: `HEAD ${target.headShortCommitId} must match target ref ${target.ref} ${controlShortCommit} before DEV CD apply.` + }); + throw new DevCdApplyError("checkout is behind the requested DEV CD control ref", { + code: "target-ref-head-mismatch", + suppressCatchBlocker: true + }); + } const kubectlContext = await resolveKubectlContext(args, env, ctx.runCommand); transaction.kubectlContext = kubectlContext; liveBefore = { @@ -1905,15 +2186,6 @@ export async function runDevCdApply(argv, io = {}) { }); transaction.phases.push({ phase: "publishing", status: "entered", at: ctx.now().toISOString() }); - if (!target.headMatchesTarget) { - blockers.push({ - type: "contract_blocker", - scope: "target-ref-head", - status: "open", - summary: `HEAD ${target.headShortCommitId} must match target ref ${target.ref} ${target.shortCommitId} before DEV CD publish.` - }); - } - const publishSteps = target.publishRequired === false ? [ { @@ -2123,12 +2395,14 @@ export async function runDevCdApply(argv, io = {}) { stdout.write(`${JSON.stringify(error.lockFailure, null, 2)}\n`); return 2; } - blockers.push({ - type: "runtime_blocker", - scope: error.code ?? "dev-cd-apply", - status: "open", - summary: oneLine(error.message) - }); + if (!error?.suppressCatchBlocker) { + blockers.push({ + type: "runtime_blocker", + scope: error.code ?? "dev-cd-apply", + status: "open", + summary: oneLine(error.message) + }); + } status = "blocked"; if (lockState?.acquired) { const kubectlContext = await resolveKubectlContext(args, env, ctx.runCommand); diff --git a/scripts/src/dev-cd-apply.test.mjs b/scripts/src/dev-cd-apply.test.mjs index d77511c7..41c19585 100644 --- a/scripts/src/dev-cd-apply.test.mjs +++ b/scripts/src/dev-cd-apply.test.mjs @@ -50,7 +50,8 @@ function parsePatch(args) { async function makeRepo(options = {}) { const commitId = options.commitId ?? "abc1234"; const repoRoot = await fsTempDir(); - await mkdir(path.join(repoRoot, "deploy"), { recursive: true }); + await mkdir(path.join(repoRoot, "deploy/k8s/base"), { recursive: true }); + await mkdir(path.join(repoRoot, "reports/dev-gate"), { recursive: true }); await writeFile( path.join(repoRoot, "deploy/deploy.json"), `${JSON.stringify({ @@ -58,9 +59,103 @@ async function makeRepo(options = {}) { environment: "dev", namespace: "hwlab-dev", endpoint: "http://74.48.78.17:16667", - commitId + commitId, + services: [ + { + serviceId: "hwlab-cloud-api", + image: `127.0.0.1:5000/hwlab/hwlab-cloud-api:${commitId.slice(0, 7)}`, + namespace: "hwlab-dev", + profile: "dev", + env: { HWLAB_COMMIT_ID: commitId.slice(0, 7) } + }, + { + serviceId: "hwlab-cloud-web", + image: `127.0.0.1:5000/hwlab/hwlab-cloud-web:${commitId.slice(0, 7)}`, + namespace: "hwlab-dev", + profile: "dev", + env: { HWLAB_COMMIT_ID: commitId.slice(0, 7) } + } + ] }, null, 2)}\n` ); + await writeFile( + path.join(repoRoot, "deploy/artifact-catalog.dev.json"), + `${JSON.stringify({ + catalogVersion: "v1", + kind: "hwlab-artifact-catalog", + environment: "dev", + profile: "dev", + namespace: "hwlab-dev", + endpoint: "http://74.48.78.17:16667", + commitId, + artifactState: "published", + publish: { + ciPublished: true, + registryVerified: true, + provenance: "reports/dev-gate/dev-artifacts.json" + }, + services: [ + { + serviceId: "hwlab-cloud-api", + commitId, + image: `127.0.0.1:5000/hwlab/hwlab-cloud-api:${commitId.slice(0, 7)}`, + imageTag: commitId.slice(0, 7), + digest: "sha256:b3713aa07164a9ff79a436bd583552427de42922835a5baff50484f53d98f296", + publishState: "published", + profile: "dev", + namespace: "hwlab-dev", + artifactRequired: true + }, + { + serviceId: "hwlab-cloud-web", + commitId, + image: `127.0.0.1:5000/hwlab/hwlab-cloud-web:${commitId.slice(0, 7)}`, + imageTag: commitId.slice(0, 7), + digest: "sha256:3be80fcfa769d68b0a7ec44e9215b93546d01bf23411533c5cb4888438f2bd84", + publishState: "published", + profile: "dev", + namespace: "hwlab-dev", + artifactRequired: true + } + ] + }, null, 2)}\n` + ); + await writeFile( + path.join(repoRoot, "reports/dev-gate/dev-artifacts.json"), + `${JSON.stringify({ + reportVersion: "v1", + taskId: "dev-artifact-publish", + status: "published", + commitId, + artifactPublish: { + status: "published", + mode: "publish", + sourceCommitId: commitId, + registryPrefix: "127.0.0.1:5000/hwlab", + serviceCount: 2, + requiredServiceCount: 2, + publishedCount: 2, + generatedAt: iso(), + services: [ + { + serviceId: "hwlab-cloud-api", + status: "published", + artifactRequired: true, + image: `127.0.0.1:5000/hwlab/hwlab-cloud-api:${commitId.slice(0, 7)}`, + digest: "sha256:b3713aa07164a9ff79a436bd583552427de42922835a5baff50484f53d98f296" + }, + { + serviceId: "hwlab-cloud-web", + status: "published", + artifactRequired: true, + image: `127.0.0.1:5000/hwlab/hwlab-cloud-web:${commitId.slice(0, 7)}`, + digest: "sha256:3be80fcfa769d68b0a7ec44e9215b93546d01bf23411533c5cb4888438f2bd84" + } + ] + } + }, null, 2)}\n` + ); + await writeFile(path.join(repoRoot, "deploy/k8s/base/workloads.yaml"), `# fixture ${commitId}\n`); return repoRoot; } @@ -101,6 +196,8 @@ function makeHttpGetJson() { const publishedCommit = "abc1234abc1234abc1234abc1234abc1234abc1"; const laterControlCommit = "def5678def5678def5678def5678def5678def5"; +const artifactMergeCommit = "fedcba9fedcba9fedcba9fedcba9fedcba9fedc"; +const artifactReportCommit = "a77efa1a77efa1a77efa1a77efa1a77efa1a77e"; function makeRunCommand({ heldLock = null, @@ -108,6 +205,14 @@ function makeRunCommand({ targetCommitId = publishedCommit, headCommitId = targetCommitId, extraGitRefs = {}, + commitParents = {}, + latestArtifactMergeCommit = null, + changedPaths = [ + "deploy/artifact-catalog.dev.json", + "deploy/deploy.json", + "deploy/k8s/base/workloads.yaml", + "reports/dev-gate/dev-artifacts.json" + ], desiredStatePromotionStatus = "pass", runtimeJobLogSuffix = "" } = {}) { @@ -120,6 +225,11 @@ function makeRunCommand({ if (command === "git" && args[0] === "rev-parse" && args.includes("origin/main^{commit}")) { return { code: 0, stdout: `${targetCommitId}\n`, stderr: "" }; } + if (command === "git" && args[0] === "rev-parse" && args.includes("--short=7")) { + const ref = String(args.at(-1) ?? ""); + const commitId = extraGitRefs[ref] ?? ref; + return { code: 0, stdout: `${commitId.slice(0, 7)}\n`, stderr: "" }; + } if (command === "git" && args[0] === "rev-parse" && args.includes("HEAD^{commit}")) { return { code: 0, stdout: `${headCommitId}\n`, stderr: "" }; } @@ -129,6 +239,19 @@ function makeRunCommand({ if (commitId) return { code: 0, stdout: `${commitId}\n`, stderr: "" }; return { code: 1, stdout: "", stderr: `fatal: ambiguous argument '${ref}'` }; } + if (command === "git" && args[0] === "log" && args.includes("--first-parent")) { + return latestArtifactMergeCommit + ? { code: 0, stdout: `${latestArtifactMergeCommit}\n`, stderr: "" } + : { code: 1, stdout: "", stderr: "no matching commits" }; + } + if (command === "git" && args[0] === "rev-list" && args.includes("--parents")) { + const ref = String(args.at(-1) ?? ""); + const parents = commitParents[ref] ?? []; + return { code: 0, stdout: `${[ref, ...parents].join(" ")}\n`, stderr: "" }; + } + if (command === "git" && args[0] === "diff-tree") { + return { code: 0, stdout: `${changedPaths.join("\n")}\n`, stderr: "" }; + } if (command === "which" && args[0] === "kubectl") { return { code: 0, stdout: "/usr/local/bin/kubectl\n", stderr: "" }; } @@ -333,6 +456,11 @@ test("dev-cd status accepts a published deploy-json promotion under a later cont headCommitId: laterControlCommit, extraGitRefs: { abc1234: publishedCommit + }, + latestArtifactMergeCommit: artifactMergeCommit, + commitParents: { + [artifactMergeCommit]: [publishedCommit, artifactReportCommit], + [artifactReportCommit]: [publishedCommit] } }), httpGetJson: makeHttpGetJson(), @@ -348,6 +476,16 @@ test("dev-cd status accepts a published deploy-json promotion under a later cont assert.equal(status.target.promotionSource, "deploy-json"); assert.equal(status.target.publishRequired, false); assert.equal(status.target.controlRef.shortCommitId, "def5678"); + assert.equal(status.target.artifactBoundary.status, "pass"); + assert.equal(status.target.artifactBoundary.artifactMergeCommit.shortCommitId, "fedcba9"); + assert.equal(status.target.artifactBoundary.artifactMergeCommit.promotionParentMatches, true); + assert.equal(status.target.artifactBoundary.artifactCommit.shortCommitId, "a77efa1"); + assert.equal(status.target.artifactBoundary.artifactCommit.promotionParentMatches, true); + assert.equal(status.artifactCatalog.commitId, "abc1234"); + assert.equal(status.artifactCatalog.digestCounts.sha256, 2); + assert.equal(status.artifactReport.sourceCommitId, "abc1234"); + assert.equal(status.liveDelta.status, "pass"); + assert.equal(status.liveDelta.expectedCommit, "abc1234"); assert.equal(status.target.headMatchesTarget, true); assert.equal(status.target.desiredStateCheck.status, "pass"); assert.equal(status.deployJson.matchesTarget, true); @@ -362,6 +500,188 @@ test("dev-cd status accepts a published deploy-json promotion under a later cont assertNoReadOnlySideEffects(commandLog); }); +test("dev-cd status exposes moving origin/main when checkout is behind a deploy-json promotion control ref", async () => { + const repoRoot = await makeRepo({ commitId: "abc1234" }); + const commandLog = []; + let output = ""; + const code = await runDevCdApply(["--status", "--kubeconfig", "/tmp/kubeconfig"], { + repoRoot, + env: {}, + runCommand: makeRunCommand({ + commandLog, + targetCommitId: laterControlCommit, + headCommitId: publishedCommit, + extraGitRefs: { + abc1234: publishedCommit + }, + latestArtifactMergeCommit: artifactMergeCommit, + commitParents: { + [artifactMergeCommit]: [publishedCommit, artifactReportCommit], + [artifactReportCommit]: [publishedCommit] + } + }), + httpGetJson: makeHttpGetJson(), + now: () => new Date(iso(100000)), + stdout: { write: (chunk) => { output += chunk; } } + }); + + const status = JSON.parse(output); + assert.equal(code, 0); + assert.equal(status.status, "degraded"); + assert.equal(status.target.promotionCommit, publishedCommit); + assert.equal(status.target.controlRef.shortCommitId, "def5678"); + assert.equal(status.target.headCommitId, publishedCommit); + assert.equal(status.target.headMatchesTarget, false); + assert.match(status.nextActions.join("\n"), /Checkout or fast-forward/u); + assertNoReadOnlySideEffects(commandLog); +}); + +test("apply stops before lock acquisition when checkout is behind deploy-json promotion control ref", async () => { + const repoRoot = await makeRepo({ commitId: "abc1234" }); + const commandLog = []; + let output = ""; + const code = await runDevCdApply([ + "--apply", + "--confirm-dev", + "--confirmed-non-production", + "--owner-task-id", + "task-moving-main", + "--kubeconfig", + "/tmp/kubeconfig" + ], { + repoRoot, + env: {}, + runCommand: makeRunCommand({ + commandLog, + targetCommitId: laterControlCommit, + headCommitId: publishedCommit, + extraGitRefs: { + abc1234: publishedCommit + }, + latestArtifactMergeCommit: artifactMergeCommit, + commitParents: { + [artifactMergeCommit]: [publishedCommit, artifactReportCommit], + [artifactReportCommit]: [publishedCommit] + } + }), + httpGetJson: makeHttpGetJson(), + now: () => new Date(iso(100000)), + stdout: { write: (chunk) => { output += chunk; } } + }); + + const summary = JSON.parse(output); + assert.equal(code, 2); + assert.equal(summary.status, "blocked"); + assert.equal(summary.target.controlRef.shortCommitId, "def5678"); + assert.equal(summary.blockers.some((blocker) => blocker.scope === "target-ref-head"), true); + assert.equal(commandLog.some((entry) => entry.command.includes("kubectl")), false); + assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")), false); + assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")), false); +}); + +test("dev-cd status degrades artifact merge provenance when catalog/report do not match deploy-json promotion", async () => { + const repoRoot = await makeRepo({ commitId: "abc1234" }); + await writeFile( + path.join(repoRoot, "deploy/artifact-catalog.dev.json"), + `${JSON.stringify({ + kind: "hwlab-artifact-catalog", + commitId: "9999999", + artifactState: "published", + publish: { ciPublished: true, registryVerified: true, provenance: "reports/dev-gate/dev-artifacts.json" }, + services: [] + }, null, 2)}\n` + ); + const commandLog = []; + let output = ""; + const code = await runDevCdApply(["--status", "--kubeconfig", "/tmp/kubeconfig"], { + repoRoot, + env: {}, + runCommand: makeRunCommand({ + commandLog, + targetCommitId: laterControlCommit, + headCommitId: laterControlCommit, + extraGitRefs: { + abc1234: publishedCommit + }, + latestArtifactMergeCommit: artifactMergeCommit, + commitParents: { + [artifactMergeCommit]: [publishedCommit, artifactReportCommit], + [artifactReportCommit]: [publishedCommit] + } + }), + httpGetJson: makeHttpGetJson(), + now: () => new Date(iso(100000)), + stdout: { write: (chunk) => { output += chunk; } } + }); + + const status = JSON.parse(output); + assert.equal(code, 0); + assert.equal(status.status, "degraded"); + assert.equal(status.target.promotionCommit, publishedCommit); + assert.equal(status.target.artifactBoundary.status, "degraded"); + assert.equal(status.target.artifactBoundary.desiredState.deployCommitMatches, true); + assert.equal(status.target.artifactBoundary.desiredState.catalogCommitMatches, false); + assert.equal(status.target.artifactBoundary.desiredState.reportCommitMatches, true); + assertNoReadOnlySideEffects(commandLog); +}); + +test("apply stops before lock acquisition when artifact merge provenance mismatches deploy-json promotion", async () => { + const repoRoot = await makeRepo({ commitId: "abc1234" }); + await writeFile( + path.join(repoRoot, "reports/dev-gate/dev-artifacts.json"), + `${JSON.stringify({ + status: "published", + commitId: "9999999", + artifactPublish: { + status: "published", + sourceCommitId: "9999999", + serviceCount: 0, + requiredServiceCount: 0, + publishedCount: 0, + services: [] + } + }, null, 2)}\n` + ); + const commandLog = []; + let output = ""; + const code = await runDevCdApply([ + "--apply", + "--confirm-dev", + "--confirmed-non-production", + "--owner-task-id", + "task-provenance", + "--kubeconfig", + "/tmp/kubeconfig" + ], { + repoRoot, + env: {}, + runCommand: makeRunCommand({ + commandLog, + targetCommitId: laterControlCommit, + headCommitId: laterControlCommit, + extraGitRefs: { + abc1234: publishedCommit + }, + latestArtifactMergeCommit: artifactMergeCommit, + commitParents: { + [artifactMergeCommit]: [publishedCommit, artifactReportCommit], + [artifactReportCommit]: [publishedCommit] + } + }), + httpGetJson: makeHttpGetJson(), + now: () => new Date(iso(100000)), + stdout: { write: (chunk) => { output += chunk; } } + }); + + const summary = JSON.parse(output); + assert.equal(code, 2); + assert.equal(summary.status, "blocked"); + assert.equal(summary.blockers.some((blocker) => blocker.scope === "artifact-boundary"), true); + assert.equal(commandLog.some((entry) => entry.command.includes("kubectl")), false); + assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")), false); + assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")), false); +}); + test("dev-cd status summarizes held, released, and stale locks without unsafe takeover hints", async () => { const cases = [ { @@ -444,9 +764,22 @@ test("dev-cd status reports unavailable kubectl with redacted reason", async () if (command === "git" && args[0] === "rev-parse" && args.includes("origin/main^{commit}")) { return { code: 0, stdout: "abc1234abc1234abc1234abc1234abc1234abc1\n", stderr: "" }; } + if (command === "git" && args[0] === "rev-parse" && args.includes("--short=7")) { + return { code: 0, stdout: `${String(args.at(-1)).slice(0, 7)}\n`, stderr: "" }; + } if (command === "git" && args[0] === "rev-parse" && args.includes("HEAD^{commit}")) { return { code: 0, stdout: "abc1234abc1234abc1234abc1234abc1234abc1\n", stderr: "" }; } + if (command === "git" && args[0] === "log" && args.includes("--first-parent")) { + return { code: 1, stdout: "", stderr: "no matching commits" }; + } + if (command === "git" && args[0] === "rev-list" && args.includes("--parents")) { + const ref = String(args.at(-1) ?? ""); + return { code: 0, stdout: `${ref}\n`, stderr: "" }; + } + if (command === "git" && args[0] === "diff-tree") { + return { code: 0, stdout: "", stderr: "" }; + } if (command === "which" && args[0] === "kubectl") { return { code: 1, stdout: "", stderr: "kubectl missing" }; } @@ -770,6 +1103,11 @@ test("transaction can apply an already published deploy-json promotion without r headCommitId: laterControlCommit, extraGitRefs: { abc1234: publishedCommit + }, + latestArtifactMergeCommit: artifactMergeCommit, + commitParents: { + [artifactMergeCommit]: [publishedCommit, artifactReportCommit], + [artifactReportCommit]: [publishedCommit] } }), httpGetJson: makeHttpGetJson(), @@ -783,6 +1121,8 @@ test("transaction can apply an already published deploy-json promotion without r assert.equal(report.devCdApply.target.promotionCommit, publishedCommit); assert.equal(report.devCdApply.target.promotionSource, "deploy-json"); assert.equal(report.devCdApply.target.publishRequired, false); + assert.equal(report.devCdApply.target.artifactBoundary.status, "pass"); + assert.equal(report.devCdApply.target.artifactBoundary.artifactMergeCommit.shortCommitId, "fedcba9"); assert.deepEqual( report.devCdApply.steps.map((step) => step.id), [