fix: rebuild missing v03 env reuse images
This commit is contained in:
root
+3
-1
@@ -21,6 +21,8 @@ function parseArgs(argv) {
|
||||
else if (arg === "--artifact-catalog") parsed.artifactCatalogPath = readOption(argv, ++index, arg);
|
||||
else if (arg === "--lane") parsed.lane = readOption(argv, ++index, arg);
|
||||
else if (arg === "--registry-prefix") parsed.registryPrefix = readOption(argv, ++index, arg);
|
||||
else if (arg === "--verify-reuse-registry") parsed.verifyReuseRegistry = true;
|
||||
else if (arg === "--no-verify-reuse-registry") parsed.verifyReuseRegistry = false;
|
||||
else if (arg === "--base-image") parsed.baseImage = readOption(argv, ++index, arg);
|
||||
else if (arg === "--services") parsed.services = readOption(argv, ++index, arg).split(",").map((item) => item.trim()).filter(Boolean);
|
||||
else if (arg === "--pretty") parsed.pretty = true;
|
||||
@@ -38,7 +40,7 @@ function readOption(argv, index, name) {
|
||||
|
||||
function usage() {
|
||||
return [
|
||||
"usage: node scripts/ci-plan.mjs [--lane v02|v03] [--base-ref REF] [--target-ref REF] [--pretty]",
|
||||
"usage: node scripts/ci-plan.mjs [--lane v02|v03] [--base-ref REF] [--target-ref REF] [--verify-reuse-registry] [--pretty]",
|
||||
"",
|
||||
"Plan node monorepo image builds without mutating CI/CD state.",
|
||||
"For runtime lane env reuse, service component boundaries and env recipe must come from deploy/deploy.yaml.",
|
||||
|
||||
@@ -132,6 +132,75 @@ test("v02 planner rebuilds env image when an env reuse catalog digest is missing
|
||||
assert.deepEqual(cloudApi.reason, ["environment-input-changed"]);
|
||||
});
|
||||
|
||||
test("v03 planner rebuilds env image when catalog env input hash is stale", async () => {
|
||||
const repo = await createFixtureRepo();
|
||||
const initialSha = (await git(repo, ["rev-parse", "HEAD"])).stdout.trim();
|
||||
const initialPlan = await createCiPlan({
|
||||
repoRoot: repo,
|
||||
lane: "v03",
|
||||
baseRef: "HEAD",
|
||||
targetRef: initialSha,
|
||||
artifactCatalogPath: "deploy/artifact-catalog.v03.json",
|
||||
services: ["hwlab-cloud-api"]
|
||||
});
|
||||
const catalog = createCatalogFixture("ready", catalogOptionsFromPlan(initialPlan, { sourceCommitId: initialSha }));
|
||||
catalog.services.find((service) => service.serviceId === "hwlab-cloud-api").environmentInputHash = "stale-environment-input";
|
||||
await writeFile(path.join(repo, "deploy/artifact-catalog.v03.json"), JSON.stringify(catalog, null, 2));
|
||||
await git(repo, ["add", "deploy/artifact-catalog.v03.json"]);
|
||||
await git(repo, ["commit", "-m", "seed stale v03 env catalog"]);
|
||||
|
||||
const plan = await createCiPlan({
|
||||
repoRoot: repo,
|
||||
lane: "v03",
|
||||
baseRef: "HEAD~1",
|
||||
targetRef: "HEAD",
|
||||
artifactCatalogPath: "deploy/artifact-catalog.v03.json",
|
||||
services: ["hwlab-cloud-api"]
|
||||
});
|
||||
const cloudApi = plan.services.find((service) => service.serviceId === "hwlab-cloud-api");
|
||||
assert.equal(cloudApi.buildRequired, true);
|
||||
assert.equal(cloudApi.environmentInputChanged, true);
|
||||
assert.deepEqual(cloudApi.reason, ["environment-input-mismatch"]);
|
||||
assert.deepEqual(plan.buildServices, ["hwlab-cloud-api"]);
|
||||
});
|
||||
|
||||
test("v03 planner rebuilds env image when target registry lacks reused digest", async () => {
|
||||
const repo = await createFixtureRepo();
|
||||
const initialSha = (await git(repo, ["rev-parse", "HEAD"])).stdout.trim();
|
||||
const initialPlan = await createCiPlan({
|
||||
repoRoot: repo,
|
||||
lane: "v03",
|
||||
baseRef: "HEAD",
|
||||
targetRef: initialSha,
|
||||
artifactCatalogPath: "deploy/artifact-catalog.v03.json",
|
||||
services: ["hwlab-cloud-api"]
|
||||
});
|
||||
await writeFile(path.join(repo, "deploy/artifact-catalog.v03.json"), JSON.stringify(createCatalogFixture("ready", catalogOptionsFromPlan(initialPlan, { sourceCommitId: initialSha })), null, 2));
|
||||
await git(repo, ["add", "deploy/artifact-catalog.v03.json"]);
|
||||
await git(repo, ["commit", "-m", "seed fresh v03 catalog"]);
|
||||
|
||||
await mkdir(path.join(repo, "deploy/gitops/node/d601/runtime-v03"), { recursive: true });
|
||||
await writeFile(path.join(repo, "deploy/gitops/node/d601/runtime-v03/workloads.yaml"), "kind: List\nitems: []\n");
|
||||
await git(repo, ["add", "."]);
|
||||
await git(repo, ["commit", "-m", "change generated gitops after fresh catalog"]);
|
||||
|
||||
const plan = await createCiPlan({
|
||||
repoRoot: repo,
|
||||
lane: "v03",
|
||||
baseRef: "HEAD~1",
|
||||
targetRef: "HEAD",
|
||||
artifactCatalogPath: "deploy/artifact-catalog.v03.json",
|
||||
services: ["hwlab-cloud-api"],
|
||||
verifyReuseRegistry: true,
|
||||
reuseRegistryProbe: async () => ({ status: "missing", method: "HEAD", statusCode: 404 })
|
||||
});
|
||||
const cloudApi = plan.services.find((service) => service.serviceId === "hwlab-cloud-api");
|
||||
assert.equal(cloudApi.buildRequired, true);
|
||||
assert.equal(cloudApi.reuseRegistry.status, "missing");
|
||||
assert.deepEqual(cloudApi.reason, ["reuse-registry-missing"]);
|
||||
assert.deepEqual(plan.buildServices, ["hwlab-cloud-api"]);
|
||||
});
|
||||
|
||||
test("artifact reuse paths preserve catalog provenance instead of current planner provenance", async () => {
|
||||
const artifactPublish = await readFile("scripts/artifact-publish.mjs", "utf8");
|
||||
assert.match(artifactPublish, /downloadDependencyScript/u);
|
||||
|
||||
@@ -1646,7 +1646,7 @@ echo '{"event":"ci-base-image","phase":"plan-artifacts","image":"${ciToolsRunner
|
||||
for tool in node git; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"plan-artifacts","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done
|
||||
cd /workspace/source/repo
|
||||
test "$(git rev-parse HEAD)" = "$(params.revision)"
|
||||
node scripts/ci-plan.mjs --lane "$(params.lane)" --target-ref HEAD --deploy-config deploy/deploy.yaml --artifact-catalog "$(params.catalog-path)" --registry-prefix "$(params.registry-prefix)" --services "$(params.services)" > /workspace/source/ci-plan.json
|
||||
node scripts/ci-plan.mjs --lane "$(params.lane)" --target-ref HEAD --deploy-config deploy/deploy.yaml --artifact-catalog "$(params.catalog-path)" --registry-prefix "$(params.registry-prefix)" --services "$(params.services)" --verify-reuse-registry > /workspace/source/ci-plan.json
|
||||
node - <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const plan = JSON.parse(fs.readFileSync("/workspace/source/ci-plan.json", "utf8"));
|
||||
@@ -1789,7 +1789,7 @@ if [ "$buildkit_ready" != "1" ]; then
|
||||
echo '{"event":"buildkit-sidecar-not-ready","serviceId":"'"$(params.service-id)"'","addr":"'"$HWLAB_BUILDKIT_ADDR"'"}' >&2
|
||||
exit 1
|
||||
fi
|
||||
node scripts/artifact-publish.mjs --publish --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --registry-prefix "$(params.registry-prefix)" --report "/workspace/source/service-results/$(params.service-id).json" --tekton-results-dir /tekton/results --quiet-build --concurrency 1 --services "$(params.service-id)" --buildkit-command /workspace/buildkit-bin/buildctl --buildkit-addr "$HWLAB_BUILDKIT_ADDR"
|
||||
HWLAB_CI_PLAN_VERIFY_REUSE_REGISTRY=1 node scripts/artifact-publish.mjs --publish --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --registry-prefix "$(params.registry-prefix)" --report "/workspace/source/service-results/$(params.service-id).json" --tekton-results-dir /tekton/results --quiet-build --concurrency 1 --services "$(params.service-id)" --buildkit-command /workspace/buildkit-bin/buildctl --buildkit-addr "$HWLAB_BUILDKIT_ADDR"
|
||||
`;
|
||||
}
|
||||
|
||||
@@ -1809,7 +1809,7 @@ export HWLAB_ARTIFACT_LANE="$(params.lane)"
|
||||
export HWLAB_ARTIFACT_CATALOG_PATH="$(params.catalog-path)"
|
||||
export HWLAB_DEPLOY_MANIFEST_PATH="deploy/deploy.yaml"
|
||||
export HWLAB_ARTIFACT_IMAGE_TAG_MODE="$(params.image-tag-mode)"
|
||||
node scripts/artifact-publish.mjs --publish --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --registry-prefix "$(params.registry-prefix)" --report /workspace/source/dev-artifacts.json --quiet-build --concurrency 1 --services "$(params.services)" --external-service-report-dir /workspace/source/service-results
|
||||
HWLAB_CI_PLAN_VERIFY_REUSE_REGISTRY=1 node scripts/artifact-publish.mjs --publish --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --registry-prefix "$(params.registry-prefix)" --report /workspace/source/dev-artifacts.json --quiet-build --concurrency 1 --services "$(params.services)" --external-service-report-dir /workspace/source/service-results
|
||||
node scripts/refresh-artifact-catalog.mjs --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --target-ref HEAD --publish-report /workspace/source/dev-artifacts.json --no-write
|
||||
`;
|
||||
}
|
||||
|
||||
+107
-5
@@ -1,5 +1,7 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { execFile } from "node:child_process";
|
||||
import * as http from "node:http";
|
||||
import * as https from "node:https";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
@@ -54,6 +56,9 @@ export async function createCiPlan(options = {}) {
|
||||
const lane = options.lane ?? "v02";
|
||||
const registryPrefix = options.registryPrefix ?? process.env.HWLAB_DEV_REGISTRY_PREFIX ?? process.env.HWLAB_DEV_REGISTRY ?? "127.0.0.1:5000/hwlab";
|
||||
const baseImage = options.baseImage ?? process.env.HWLAB_DEV_BASE_IMAGE ?? "127.0.0.1:5000/hwlab/hwlab-node20-base:20-bookworm-slim";
|
||||
const verifyReuseRegistry = options.verifyReuseRegistry ?? booleanEnv(process.env.HWLAB_CI_PLAN_VERIFY_REUSE_REGISTRY);
|
||||
const reuseRegistryProbe = options.reuseRegistryProbe ?? (verifyReuseRegistry ? probeRegistryManifest : null);
|
||||
const registryProbeTimeoutMs = Number.parseInt(String(options.registryProbeTimeoutMs ?? process.env.HWLAB_CI_PLAN_REGISTRY_PROBE_TIMEOUT_MS ?? 3000), 10);
|
||||
|
||||
const deployJson = await readStructuredFileIfPresent(repoRoot, deployConfigPath, {});
|
||||
const laneConfig = laneDeployConfig(deployJson, lane);
|
||||
@@ -118,7 +123,18 @@ export async function createCiPlan(options = {}) {
|
||||
const environmentImage = envReuse ? environmentImageFromCatalog(model.serviceId, catalogRecord) : null;
|
||||
const environmentDigest = envReuse ? environmentDigestFromCatalog(catalogRecord) : null;
|
||||
const environmentReady = envReuse && /^sha256:[a-f0-9]{64}$/u.test(environmentDigest ?? "") && Boolean(environmentImage);
|
||||
const envChanged = envReuse ? relevantEnvMatches.length > 0 || !environmentReady : null;
|
||||
const environmentInputChanged = envReuse ? catalogRecord?.environmentInputHash !== environmentInputHash : null;
|
||||
const reuseRegistry = envReuse && reuseRegistryProbe && environmentReady
|
||||
? await reuseRegistryProbe({
|
||||
serviceId: model.serviceId,
|
||||
image: environmentImage,
|
||||
digest: environmentDigest,
|
||||
registryPrefix,
|
||||
timeoutMs: Number.isFinite(registryProbeTimeoutMs) ? registryProbeTimeoutMs : 3000
|
||||
})
|
||||
: null;
|
||||
const reuseRegistryUnavailable = Boolean(reuseRegistry && reuseRegistry.status !== "present");
|
||||
const envChanged = envReuse ? Boolean(relevantEnvMatches.length > 0 || !environmentReady || environmentInputChanged || reuseRegistryUnavailable) : null;
|
||||
const codeChanged = envReuse ? relevantCodeMatches.length > 0 || catalogRecord?.codeInputHash !== codeInputHash : null;
|
||||
const componentInputPaths = uniqueSorted([
|
||||
...model.componentPaths,
|
||||
@@ -197,7 +213,9 @@ export async function createCiPlan(options = {}) {
|
||||
runtimeMode: envReuse ? ENV_REUSE_RUNTIME_MODE : "service-image",
|
||||
envReuse,
|
||||
envChanged,
|
||||
environmentInputChanged,
|
||||
codeChanged,
|
||||
reuseRegistry,
|
||||
environmentInputHash,
|
||||
codeInputHash,
|
||||
bootRepo: envReuse ? canonicalBootRepo(deployJson, lane) : null,
|
||||
@@ -208,7 +226,7 @@ export async function createCiPlan(options = {}) {
|
||||
envChangedPaths: relevantEnvMatches,
|
||||
codeChangedPaths: relevantCodeMatches,
|
||||
reason: affected
|
||||
? reasonForService({ directMatches, sharedMatches, runtimeDepMatches, buildSystemMatches, catalogReuse, envReuse, envChanged, codeChanged })
|
||||
? reasonForService({ directMatches, sharedMatches, runtimeDepMatches, buildSystemMatches, catalogReuse, envReuse, envChanged, environmentInputChanged, codeChanged, reuseRegistry })
|
||||
: reasonForUnchanged(globalChange),
|
||||
reuse: catalogReuse
|
||||
});
|
||||
@@ -243,7 +261,8 @@ export async function createCiPlan(options = {}) {
|
||||
inputs: {
|
||||
registryPrefix,
|
||||
baseImage,
|
||||
serviceCount: services.length
|
||||
serviceCount: services.length,
|
||||
verifyReuseRegistry
|
||||
},
|
||||
changedPaths: normalizedChangedPaths,
|
||||
changedPathSummary: globalChange,
|
||||
@@ -648,9 +667,11 @@ async function gitValue(repoRoot, args) {
|
||||
return result.stdout.trim();
|
||||
}
|
||||
|
||||
function reasonForService({ directMatches, sharedMatches, runtimeDepMatches, buildSystemMatches, catalogReuse = null, envReuse = false, envChanged = null, codeChanged = null }) {
|
||||
function reasonForService({ directMatches, sharedMatches, runtimeDepMatches, buildSystemMatches, catalogReuse = null, envReuse = false, envChanged = null, environmentInputChanged = null, codeChanged = null, reuseRegistry = null }) {
|
||||
const reasons = [];
|
||||
if (envReuse && envChanged) reasons.push("environment-input-changed");
|
||||
if (envReuse && environmentInputChanged) reasons.push("environment-input-mismatch");
|
||||
if (envReuse && reuseRegistry && reuseRegistry.status !== "present") reasons.push("reuse-registry-missing");
|
||||
if (envReuse && envChanged && reasons.length === 0) reasons.push("environment-input-changed");
|
||||
if (envReuse && codeChanged) reasons.push("code-input-changed");
|
||||
if (directMatches.some((item) => !isTestOnlyPath(item) && !isDocsOnlyPath(item))) reasons.push("component-path-changed");
|
||||
if (sharedMatches.some((item) => !isTestOnlyPath(item) && !isDocsOnlyPath(item))) reasons.push("shared-runtime-changed");
|
||||
@@ -780,6 +801,87 @@ function digestFromImageReference(image) {
|
||||
return match ? match[1] : null;
|
||||
}
|
||||
|
||||
function booleanEnv(value) {
|
||||
return ["1", "true", "yes", "on"].includes(String(value ?? "").trim().toLowerCase());
|
||||
}
|
||||
|
||||
async function probeRegistryManifest({ image, digest, timeoutMs = 3000 }) {
|
||||
const request = registryManifestRequest(image, digest);
|
||||
if (!request) return { status: "invalid-reference", image, digest, reason: "registry-reference-unparseable" };
|
||||
const result = await registryManifestHttpProbe(request, "HEAD", timeoutMs);
|
||||
if (result.status === "present" || result.status === "missing") return { ...result, image, digest };
|
||||
const fallback = await registryManifestHttpProbe(request, "GET", timeoutMs);
|
||||
return { ...fallback, image, digest };
|
||||
}
|
||||
|
||||
function registryManifestRequest(image, digest) {
|
||||
const parsed = parseTaggedImage(image);
|
||||
if (!parsed || !/^sha256:[a-f0-9]{64}$/u.test(digest ?? "")) return null;
|
||||
const protocol = parsed.host === "127.0.0.1" || parsed.host.startsWith("127.") || parsed.host === "localhost" || parsed.host.includes(":5000")
|
||||
? "http:"
|
||||
: "https:";
|
||||
return {
|
||||
protocol,
|
||||
host: parsed.host,
|
||||
path: `/v2/${parsed.repository}/manifests/${digest}`
|
||||
};
|
||||
}
|
||||
|
||||
function parseTaggedImage(image) {
|
||||
const text = String(image ?? "").trim();
|
||||
const slash = text.indexOf("/");
|
||||
if (slash <= 0) return null;
|
||||
const host = text.slice(0, slash);
|
||||
const rest = text.slice(slash + 1);
|
||||
const tagSeparator = rest.lastIndexOf(":");
|
||||
if (!host || tagSeparator <= 0 || rest.includes("@")) return null;
|
||||
return {
|
||||
host,
|
||||
repository: rest.slice(0, tagSeparator)
|
||||
};
|
||||
}
|
||||
|
||||
function registryManifestHttpProbe(request, method, timeoutMs) {
|
||||
const client = request.protocol === "http:" ? http : https;
|
||||
return new Promise((resolve) => {
|
||||
const req = client.request({
|
||||
protocol: request.protocol,
|
||||
host: request.host.includes(":") ? request.host.slice(0, request.host.lastIndexOf(":")) : request.host,
|
||||
port: request.host.includes(":") ? request.host.slice(request.host.lastIndexOf(":") + 1) : undefined,
|
||||
method,
|
||||
path: request.path,
|
||||
timeout: timeoutMs,
|
||||
headers: {
|
||||
Accept: [
|
||||
"application/vnd.oci.image.manifest.v1+json",
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
"application/vnd.oci.image.index.v1+json",
|
||||
"application/vnd.docker.distribution.manifest.list.v2+json"
|
||||
].join(", ")
|
||||
}
|
||||
}, (res) => {
|
||||
res.resume();
|
||||
res.on("end", () => {
|
||||
if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) {
|
||||
resolve({ status: "present", method, statusCode: res.statusCode });
|
||||
} else if (res.statusCode === 404) {
|
||||
resolve({ status: "missing", method, statusCode: res.statusCode });
|
||||
} else {
|
||||
resolve({ status: "unknown", method, statusCode: res.statusCode ?? 0, reason: "unexpected-status" });
|
||||
}
|
||||
});
|
||||
});
|
||||
req.on("timeout", () => {
|
||||
req.destroy();
|
||||
resolve({ status: "unknown", method, statusCode: 0, reason: "timeout" });
|
||||
});
|
||||
req.on("error", (error) => {
|
||||
resolve({ status: "unknown", method, statusCode: 0, reason: error.message });
|
||||
});
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
function stableHash(value) {
|
||||
return createHash("sha256").update(stableJson(value)).digest("hex");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user