Generalize artifact catalog refresh lanes
This commit is contained in:
Codex Agent
@@ -23,19 +23,28 @@ const defaultPublishReportPath = tempReportPath("dev-artifacts.json");
|
||||
const defaultRegistryPrefix = process.env.HWLAB_DEV_REGISTRY_PREFIX || process.env.HWLAB_DEV_REGISTRY || "127.0.0.1:5000/hwlab";
|
||||
const digestPattern = /^sha256:[a-f0-9]{64}$/;
|
||||
const commitPattern = /^[a-f0-9]{7,40}$/;
|
||||
const runtimeArtifactLanePattern = /^v\d{2,}$/u;
|
||||
|
||||
const publishReadyStatuses = new Set(["published", "reused"]);
|
||||
const v02EnvReuseRuntimeMode = "env-reuse-git-mirror-checkout";
|
||||
|
||||
function serviceIdsForLane(lane, deploy = null) {
|
||||
return lane === "v02" ? v02ServiceIdsFromDeploy(deploy) : SERVICE_IDS;
|
||||
function isRuntimeArtifactLane(lane) {
|
||||
return runtimeArtifactLanePattern.test(String(lane ?? ""));
|
||||
}
|
||||
|
||||
function v02ServiceIdsFromDeploy(deploy) {
|
||||
const serviceIds = uniquePreserveOrder((Array.isArray(deploy?.lanes?.v02?.envReuseServices) ? deploy.lanes.v02.envReuseServices : [])
|
||||
function laneDeployConfig(deploy, lane) {
|
||||
return deploy?.lanes?.[lane] ?? null;
|
||||
}
|
||||
|
||||
function serviceIdsForLane(lane, deploy = null) {
|
||||
return isRuntimeArtifactLane(lane) ? runtimeServiceIdsFromDeploy(deploy, lane) : SERVICE_IDS;
|
||||
}
|
||||
|
||||
function runtimeServiceIdsFromDeploy(deploy, lane) {
|
||||
const serviceIds = uniquePreserveOrder((Array.isArray(deploy?.lanes?.[lane]?.envReuseServices) ? deploy.lanes[lane].envReuseServices : [])
|
||||
.map((item) => String(item ?? "").trim())
|
||||
.filter(Boolean));
|
||||
assert.ok(serviceIds.length > 0, "deploy.lanes.v02.envReuseServices must not be empty");
|
||||
assert.ok(serviceIds.length > 0, `deploy.lanes.${lane}.envReuseServices must not be empty`);
|
||||
return serviceIds;
|
||||
}
|
||||
|
||||
@@ -90,7 +99,7 @@ function parseArgs(argv) {
|
||||
}
|
||||
|
||||
if (!args.help) {
|
||||
assert.ok(["g14", "v02"].includes(args.lane), `unknown artifact lane ${args.lane}`);
|
||||
assert.ok(args.lane === "g14" || isRuntimeArtifactLane(args.lane), `unknown artifact lane ${args.lane}`);
|
||||
assert.ok(["short", "full"].includes(args.imageTagMode), `unknown image tag mode ${args.imageTagMode}`);
|
||||
assert.notEqual(args.blocked && Boolean(args.publishReportPath), true, "--blocked and --publish-report are mutually exclusive");
|
||||
assert.ok(args.blocked || args.publishReportPath, `choose --blocked or --publish-report ${defaultPublishReportPath}`);
|
||||
@@ -118,7 +127,7 @@ function usage() {
|
||||
"examples:",
|
||||
" node scripts/refresh-artifact-catalog.mjs --target-ref HEAD --blocked --no-write",
|
||||
` node scripts/refresh-artifact-catalog.mjs --target-ref HEAD --publish-report ${defaultPublishReportPath} --no-write`,
|
||||
` node scripts/refresh-artifact-catalog.mjs --lane v02 --catalog-path deploy/artifact-catalog.v02.json --image-tag-mode full --target-ref HEAD --publish-report ${defaultPublishReportPath} --write`
|
||||
` node scripts/refresh-artifact-catalog.mjs --lane v03 --catalog-path deploy/artifact-catalog.v03.json --image-tag-mode full --target-ref HEAD --publish-report ${defaultPublishReportPath} --write`
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@@ -173,11 +182,14 @@ function targetImage(serviceId, shortCommitId, registryPrefix = defaultRegistryP
|
||||
}
|
||||
|
||||
function artifactEnvironment(args) {
|
||||
return args.lane === "v02" ? "v02" : ENVIRONMENT_DEV;
|
||||
return isRuntimeArtifactLane(args.lane) ? args.lane : ENVIRONMENT_DEV;
|
||||
}
|
||||
|
||||
function artifactNamespace(args) {
|
||||
return args.lane === "v02" ? "hwlab-v02" : "hwlab-dev";
|
||||
function artifactNamespace(args, deploy) {
|
||||
if (!isRuntimeArtifactLane(args.lane)) return "hwlab-dev";
|
||||
const namespace = laneDeployConfig(deploy, args.lane)?.namespace;
|
||||
assert.ok(String(namespace ?? "").trim(), `deploy.lanes.${args.lane}.namespace must not be empty`);
|
||||
return namespace;
|
||||
}
|
||||
|
||||
function isHttpEndpoint(value) {
|
||||
@@ -191,15 +203,15 @@ function isHttpEndpoint(value) {
|
||||
}
|
||||
|
||||
function artifactEndpoint(args, deploy) {
|
||||
if (args.lane !== "v02") return DEV_ENDPOINT;
|
||||
const endpoint = deploy?.lanes?.v02?.endpoint;
|
||||
assert.ok(isHttpEndpoint(endpoint), "deploy.lanes.v02.endpoint must be a non-empty http(s) URL");
|
||||
if (!isRuntimeArtifactLane(args.lane)) return DEV_ENDPOINT;
|
||||
const endpoint = laneDeployConfig(deploy, args.lane)?.endpoint;
|
||||
assert.ok(isHttpEndpoint(endpoint), `deploy.lanes.${args.lane}.endpoint must be a non-empty http(s) URL`);
|
||||
return endpoint;
|
||||
}
|
||||
|
||||
function artifactCatalogSkeleton({ args, deploy, target, registryPrefix = defaultRegistryPrefix }) {
|
||||
const environment = artifactEnvironment(args);
|
||||
const namespace = artifactNamespace(args);
|
||||
const namespace = artifactNamespace(args, deploy);
|
||||
return {
|
||||
catalogVersion: "v1",
|
||||
kind: "hwlab-artifact-catalog",
|
||||
@@ -222,7 +234,7 @@ function artifactCatalogSkeleton({ args, deploy, target, registryPrefix = defaul
|
||||
requiredFields: ["serviceId", "environment", "status"]
|
||||
},
|
||||
allowedProfiles: [environment],
|
||||
forbiddenProfiles: args.lane === "v02" ? ["dev", "prod"] : ["prod"],
|
||||
forbiddenProfiles: isRuntimeArtifactLane(args.lane) ? ["dev", "prod"] : ["prod"],
|
||||
services: serviceIdsForLane(args.lane, deploy).map((serviceId) => catalogSkeletonService({ args, serviceId, target, registryPrefix, environment, namespace }))
|
||||
};
|
||||
}
|
||||
@@ -297,26 +309,26 @@ function assertDevOnlyDeployAndCatalog(deploy, catalog) {
|
||||
assert.deepEqual((catalog.services ?? []).map((service) => service.serviceId), SERVICE_IDS, "catalog services must cover frozen service IDs in order");
|
||||
}
|
||||
|
||||
function assertV02DeployAndCatalog(deploy, catalog) {
|
||||
const lane = deploy?.lanes?.v02;
|
||||
assert.ok(lane && typeof lane === "object", "deploy.lanes.v02 must exist for v02 catalog refresh");
|
||||
assert.equal(lane.namespace, "hwlab-v02", "deploy.lanes.v02.namespace must be hwlab-v02");
|
||||
assert.ok(isHttpEndpoint(lane.endpoint), "deploy.lanes.v02.endpoint must be a non-empty http(s) URL");
|
||||
const serviceIds = serviceIdsForLane("v02", deploy);
|
||||
assert.ok(serviceIds.every((serviceId) => lane.serviceDeclarations?.[serviceId]), "deploy.lanes.v02.serviceDeclarations must cover envReuseServices");
|
||||
assert.ok(serviceIds.every((serviceId) => lane.bootScripts?.[serviceId]), "deploy.lanes.v02.bootScripts must cover envReuseServices");
|
||||
function assertRuntimeLaneDeployAndCatalog(deploy, catalog, laneName) {
|
||||
const lane = laneDeployConfig(deploy, laneName);
|
||||
assert.ok(lane && typeof lane === "object", `deploy.lanes.${laneName} must exist for ${laneName} catalog refresh`);
|
||||
assert.ok(String(lane.namespace ?? "").trim(), `deploy.lanes.${laneName}.namespace must not be empty`);
|
||||
assert.ok(isHttpEndpoint(lane.endpoint), `deploy.lanes.${laneName}.endpoint must be a non-empty http(s) URL`);
|
||||
const serviceIds = serviceIdsForLane(laneName, deploy);
|
||||
assert.ok(serviceIds.every((serviceId) => lane.serviceDeclarations?.[serviceId]), `deploy.lanes.${laneName}.serviceDeclarations must cover envReuseServices`);
|
||||
assert.ok(serviceIds.every((serviceId) => lane.bootScripts?.[serviceId]), `deploy.lanes.${laneName}.bootScripts must cover envReuseServices`);
|
||||
|
||||
assert.equal(catalog.environment, "v02", "catalog environment must be v02");
|
||||
assert.equal(catalog.profile, "v02", "catalog profile must be v02");
|
||||
assert.equal(catalog.namespace, "hwlab-v02", "catalog namespace must be hwlab-v02");
|
||||
assert.equal(catalog.environment, laneName, `catalog environment must be ${laneName}`);
|
||||
assert.equal(catalog.profile, laneName, `catalog profile must be ${laneName}`);
|
||||
assert.equal(catalog.namespace, lane.namespace, `catalog namespace must be ${lane.namespace}`);
|
||||
if (catalog.endpoint != null) assert.ok(isHttpEndpoint(catalog.endpoint), "catalog endpoint must be a non-empty http(s) URL");
|
||||
assert.deepEqual(catalog.allowedProfiles, ["v02"], "catalog allowedProfiles must only allow v02");
|
||||
assert.deepEqual(catalog.allowedProfiles, [laneName], `catalog allowedProfiles must only allow ${laneName}`);
|
||||
assert.deepEqual(catalog.forbiddenProfiles, ["dev", "prod"], "catalog forbiddenProfiles must forbid dev/prod");
|
||||
assert.deepEqual((catalog.services ?? []).map((service) => service.serviceId), serviceIds, "catalog services must cover deploy.lanes.v02.envReuseServices in order");
|
||||
assert.deepEqual((catalog.services ?? []).map((service) => service.serviceId), serviceIds, `catalog services must cover deploy.lanes.${laneName}.envReuseServices in order`);
|
||||
}
|
||||
|
||||
function normalizeCatalogForLane(catalog, lane, deploy) {
|
||||
if (!catalog || lane !== "v02") return catalog;
|
||||
if (!catalog || !isRuntimeArtifactLane(lane)) return catalog;
|
||||
const allowed = new Set(serviceIdsForLane(lane, deploy));
|
||||
return {
|
||||
...catalog,
|
||||
@@ -325,7 +337,7 @@ function normalizeCatalogForLane(catalog, lane, deploy) {
|
||||
}
|
||||
|
||||
function assertLaneDeployAndCatalog(args, deploy, catalog) {
|
||||
if (args.lane === "v02") return assertV02DeployAndCatalog(deploy, catalog);
|
||||
if (isRuntimeArtifactLane(args.lane)) return assertRuntimeLaneDeployAndCatalog(deploy, catalog, args.lane);
|
||||
return assertDevOnlyDeployAndCatalog(deploy, catalog);
|
||||
}
|
||||
|
||||
@@ -414,7 +426,7 @@ function publishRecordsFromReport(report, target, serviceIds = SERVICE_IDS) {
|
||||
return { records, requiredServiceIds, disabledServiceIds };
|
||||
}
|
||||
|
||||
function refreshDocuments({ deploy, catalog, target, publishRecords, serviceInventory, provenancePath, registryPrefix, serviceIds = SERVICE_IDS }) {
|
||||
function refreshDocuments({ deploy, catalog, target, publishRecords, serviceInventory, provenancePath, registryPrefix, lane = "g14", serviceIds = SERVICE_IDS }) {
|
||||
const published = Boolean(publishRecords);
|
||||
const records = publishRecords?.records ?? null;
|
||||
const requiredIds = new Set(publishRecords?.requiredServiceIds ?? serviceInventory.requiredServiceIds);
|
||||
@@ -425,7 +437,7 @@ function refreshDocuments({ deploy, catalog, target, publishRecords, serviceInve
|
||||
const nextCatalogServices = [];
|
||||
|
||||
catalog.commitId = target.imageTag;
|
||||
if (catalog.profile === "v02") catalog.endpoint = artifactEndpoint({ lane: "v02" }, deploy);
|
||||
if (isRuntimeArtifactLane(lane)) catalog.endpoint = artifactEndpoint({ lane }, deploy);
|
||||
catalog.artifactState = published ? "published" : "contract-skeleton";
|
||||
catalog.serviceInventory = serviceInventory;
|
||||
catalog.publish = {
|
||||
@@ -546,7 +558,7 @@ async function main() {
|
||||
args.publishReportPath ? readConfig(args.publishReportPath) : Promise.resolve(null)
|
||||
]);
|
||||
let catalog = await readConfigIfPresent(args.catalogPath, null);
|
||||
if (!catalog && args.lane === "v02") catalog = artifactCatalogSkeleton({ args, deploy, target, registryPrefix: publishReport?.artifactPublish?.registryPrefix ?? defaultRegistryPrefix });
|
||||
if (!catalog && isRuntimeArtifactLane(args.lane)) catalog = artifactCatalogSkeleton({ args, deploy, target, registryPrefix: publishReport?.artifactPublish?.registryPrefix ?? defaultRegistryPrefix });
|
||||
catalog = normalizeCatalogForLane(catalog, args.lane, deploy);
|
||||
assert.ok(catalog, `${args.catalogPath} is required for ${args.lane} catalog refresh`);
|
||||
assertLaneDeployAndCatalog(args, deploy, catalog);
|
||||
@@ -564,6 +576,7 @@ async function main() {
|
||||
serviceInventory,
|
||||
provenancePath: publishReportProvenance(publishReport, args.publishReportPath ?? defaultPublishReportPath),
|
||||
registryPrefix: publishReport?.artifactPublish?.registryPrefix ?? defaultRegistryPrefix,
|
||||
lane: args.lane,
|
||||
serviceIds
|
||||
});
|
||||
|
||||
|
||||
@@ -32,6 +32,10 @@ function digestFor(index) {
|
||||
return `sha256:${String(index + 1).padStart(64, "0")}`;
|
||||
}
|
||||
|
||||
function envInputHashFor(index) {
|
||||
return String(index + 1).padStart(64, "a");
|
||||
}
|
||||
|
||||
test("refresh accepts an absolute publish report path", async () => {
|
||||
const commitId = await git(["rev-parse", "HEAD"]);
|
||||
const shortCommitId = await git(["rev-parse", "--short=7", "HEAD"]);
|
||||
@@ -323,6 +327,136 @@ test("v02 refresh accepts current runtime service full image tags", async () =>
|
||||
}
|
||||
});
|
||||
|
||||
test("v03 refresh accepts runtime lane config and env-reuse image tags", async () => {
|
||||
const commitId = await git(["rev-parse", "HEAD"]);
|
||||
const tempDir = await mkdtemp(path.join(os.tmpdir(), "hwlab-refresh-v03-runtime-services-"));
|
||||
const deployPath = path.join(tempDir, "deploy.yaml");
|
||||
const catalogPath = path.join(tempDir, "artifact-catalog.v03.json");
|
||||
const reportPath = path.join(tempDir, "v03-artifacts.json");
|
||||
const serviceIds = V02_SERVICE_IDS;
|
||||
await writeStructuredFile(tempDir, deployPath, {
|
||||
environment: "dev",
|
||||
namespace: "hwlab-dev",
|
||||
endpoint: "http://74.48.78.17:16667",
|
||||
profiles: { dev: { enabled: true }, prod: { enabled: false } },
|
||||
lanes: {
|
||||
v03: {
|
||||
namespace: "hwlab-v03",
|
||||
endpoint: "https://hwlab-v03.74-48-78-17.nip.io",
|
||||
envReuseServices: serviceIds,
|
||||
serviceDeclarations: Object.fromEntries(serviceIds.map((serviceId) => [serviceId, {
|
||||
runtimeKind: "node-command",
|
||||
entrypoint: `cmd/${serviceId}/main.ts`,
|
||||
componentPaths: [`cmd/${serviceId}/`]
|
||||
}])),
|
||||
bootScripts: Object.fromEntries(serviceIds.map((serviceId) => [serviceId, `deploy/runtime/boot/${serviceId}.sh`]))
|
||||
}
|
||||
},
|
||||
services: SERVICE_IDS.map((serviceId) => ({ serviceId }))
|
||||
});
|
||||
await writeFile(reportPath, `${JSON.stringify({
|
||||
reportVersion: "v1",
|
||||
taskId: "v03-artifact-publish",
|
||||
commitId,
|
||||
artifactPublish: {
|
||||
status: "published",
|
||||
mode: "publish",
|
||||
sourceCommitId: commitId,
|
||||
registryPrefix: "127.0.0.1:5000/hwlab",
|
||||
serviceCount: serviceIds.length,
|
||||
requiredServiceCount: serviceIds.length,
|
||||
disabledServiceCount: 0,
|
||||
publishedCount: serviceIds.length,
|
||||
reusedCount: 0,
|
||||
serviceInventory: {
|
||||
version: "v2",
|
||||
serviceCount: serviceIds.length,
|
||||
requiredServiceCount: serviceIds.length,
|
||||
disabledServiceCount: 0,
|
||||
requiredServiceIds: serviceIds,
|
||||
disabledServiceIds: [],
|
||||
services: serviceIds.map((serviceId) => ({
|
||||
serviceId,
|
||||
publishEnabled: true,
|
||||
artifactRequired: true,
|
||||
artifactScope: "required",
|
||||
runtimeKind: "node-command",
|
||||
implementationState: "repo-entrypoint",
|
||||
sourceState: "source-present",
|
||||
entrypoint: `cmd/${serviceId}/main.ts`,
|
||||
disabledReason: null
|
||||
}))
|
||||
},
|
||||
publishPlan: {
|
||||
version: "v2",
|
||||
services: serviceIds.map((serviceId, index) => {
|
||||
const envTag = `env-${envInputHashFor(index).slice(0, 12)}`;
|
||||
return {
|
||||
serviceId,
|
||||
required: true,
|
||||
digest: digestFor(index),
|
||||
image: `127.0.0.1:5000/hwlab/${serviceId}-env:${envTag}`,
|
||||
imageTag: envTag
|
||||
};
|
||||
})
|
||||
},
|
||||
services: serviceIds.map((serviceId, index) => {
|
||||
const environmentInputHash = envInputHashFor(index);
|
||||
const envTag = `env-${environmentInputHash.slice(0, 12)}`;
|
||||
const image = `127.0.0.1:5000/hwlab/${serviceId}-env:${envTag}`;
|
||||
return {
|
||||
serviceId,
|
||||
status: "published",
|
||||
artifactRequired: true,
|
||||
runtimeMode: "env-reuse-git-mirror-checkout",
|
||||
envReuse: true,
|
||||
commitId,
|
||||
sourceCommitId: commitId,
|
||||
image,
|
||||
imageTag: envTag,
|
||||
digest: digestFor(index),
|
||||
repositoryDigest: `127.0.0.1:5000/hwlab/${serviceId}-env@${digestFor(index)}`,
|
||||
environmentImage: image,
|
||||
environmentDigest: digestFor(index),
|
||||
environmentInputHash,
|
||||
codeInputHash: envInputHashFor(index + serviceIds.length),
|
||||
bootRepo: "git@github.com:pikasTech/HWLAB.git",
|
||||
bootCommit: commitId,
|
||||
bootSh: `deploy/runtime/boot/${serviceId}.sh`
|
||||
};
|
||||
})
|
||||
}
|
||||
}, null, 2)}\n`);
|
||||
|
||||
const result = await execFileAsync(process.execPath, [
|
||||
"scripts/refresh-artifact-catalog.mjs",
|
||||
"--lane",
|
||||
"v03",
|
||||
"--image-tag-mode",
|
||||
"full",
|
||||
"--target-ref",
|
||||
"HEAD",
|
||||
"--deploy-config",
|
||||
deployPath,
|
||||
"--catalog-path",
|
||||
catalogPath,
|
||||
"--publish-report",
|
||||
reportPath,
|
||||
"--no-write"
|
||||
], {
|
||||
cwd: repoRoot,
|
||||
timeout: 15000,
|
||||
maxBuffer: 10 * 1024 * 1024
|
||||
});
|
||||
const payload = JSON.parse(result.stdout);
|
||||
assert.equal(payload.status, "published");
|
||||
assert.equal(payload.endpoint, "https://hwlab-v03.74-48-78-17.nip.io");
|
||||
assert.equal(payload.requiredServiceCount, serviceIds.length);
|
||||
assert.deepEqual(payload.services.map((service) => service.serviceId), serviceIds);
|
||||
assert.equal(payload.services.every((service) => service.runtimeMode === "env-reuse-git-mirror-checkout"), true);
|
||||
assert.equal(payload.services.every((service) => service.imageTag.startsWith("env-")), true);
|
||||
});
|
||||
|
||||
test("v02 refresh follows the current deploy endpoint instead of the retired 19667 freeze", async () => {
|
||||
const commitId = await git(["rev-parse", "HEAD"]);
|
||||
const tempDir = await mkdtemp(path.join(os.tmpdir(), "hwlab-refresh-v02-https-endpoint-"));
|
||||
|
||||
Reference in New Issue
Block a user