diff --git a/cmd/hwlab-device-pod/main.ts b/cmd/hwlab-device-pod/main.ts index f22b4a7b..c9315a2b 100644 --- a/cmd/hwlab-device-pod/main.ts +++ b/cmd/hwlab-device-pod/main.ts @@ -2,6 +2,7 @@ import { createServer } from "node:http"; import { randomUUID } from "node:crypto"; +import { runtimeIdentityFromEnv } from "../../internal/build-metadata.mjs"; import { jsonResponse, listen, parsePort, readJson } from "../../internal/sim/http.mjs"; const SERVICE_ID = "hwlab-device-pod"; @@ -78,6 +79,7 @@ function healthPayload() { acceptsUserAuthority: false, fake: false, gatewayAdapter: gatewayAdapterState(), + runtime: runtimeIdentityFromEnv(process.env), source: sourcePayload(), note: "Profile, grant, lease, and user-facing job authority live in hwlab-cloud-api; this service only exposes the internal executor boundary." }; @@ -351,7 +353,14 @@ function isInternalCaller(request) { } function sourcePayload() { - return { kind: "INTERNAL_EXECUTOR", serviceId: SERVICE_ID, authority: "hwlab-cloud-api", fake: false, devLiveEvidence: false }; + return { + kind: "INTERNAL_EXECUTOR", + serviceId: SERVICE_ID, + authority: "hwlab-cloud-api", + fake: false, + devLiveEvidence: false, + runtime: runtimeIdentityFromEnv(process.env) + }; } function gatewayAdapterBlocker(summary) { diff --git a/deploy/deploy.json b/deploy/deploy.json index 430899d7..295798c5 100644 --- a/deploy/deploy.json +++ b/deploy/deploy.json @@ -141,6 +141,12 @@ "artifactCatalog": "deploy/artifact-catalog.v02.json", "runtimePath": "deploy/gitops/g14/runtime-v02", "imageTagMode": "full", + "envReuseServices": [ + "hwlab-device-pod" + ], + "bootScripts": { + "hwlab-device-pod": "deploy/runtime/boot/hwlab-device-pod.sh" + }, "services": [ { "serviceId": "hwlab-cloud-api", diff --git a/deploy/runtime/boot/hwlab-device-pod.sh b/deploy/runtime/boot/hwlab-device-pod.sh new file mode 100644 index 00000000..7eb37bf0 --- /dev/null +++ b/deploy/runtime/boot/hwlab-device-pod.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu + +export HWLAB_SERVICE_ID="hwlab-device-pod" +export HWLAB_RUNTIME_MODE="${HWLAB_RUNTIME_MODE:-env-reuse-git-mirror-checkout}" +export HWLAB_COMMIT_ID="${HWLAB_BOOT_COMMIT:?HWLAB_BOOT_COMMIT is required}" +export HWLAB_REVISION="${HWLAB_BOOT_COMMIT}" +export HWLAB_IMAGE="${HWLAB_ENVIRONMENT_IMAGE:-${HWLAB_IMAGE:-}}" +export HWLAB_IMAGE_DIGEST="${HWLAB_ENVIRONMENT_DIGEST:-${HWLAB_IMAGE_DIGEST:-unknown}}" + +exec bun cmd/hwlab-device-pod/main.ts diff --git a/deploy/runtime/launcher/hwlab-env-reuse-launcher.ts b/deploy/runtime/launcher/hwlab-env-reuse-launcher.ts new file mode 100644 index 00000000..36b28689 --- /dev/null +++ b/deploy/runtime/launcher/hwlab-env-reuse-launcher.ts @@ -0,0 +1,61 @@ +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, rmSync, symlinkSync } from "node:fs"; +import path from "node:path"; + +const bootRepo = requiredEnv("HWLAB_BOOT_REPO"); +const bootCommit = requiredEnv("HWLAB_BOOT_COMMIT"); +const bootSh = requiredEnv("HWLAB_BOOT_SH"); +const checkoutDir = process.env.HWLAB_BOOT_CHECKOUT_DIR || "/workspace/hwlab-boot/repo"; +const runtimeNodeModules = process.env.HWLAB_RUNTIME_NODE_MODULES || "/opt/hwlab-env/node_modules"; + +if (!/^[a-f0-9]{40}$/u.test(bootCommit)) fail("HWLAB_BOOT_COMMIT must be a full 40-char SHA"); +if (path.isAbsolute(bootSh) || bootSh.split("/").includes("..")) fail("HWLAB_BOOT_SH must be repo-relative"); + +const readUrl = process.env.HWLAB_BOOT_READ_URL || mirrorReadUrl(bootRepo); +mkdirSync(path.dirname(checkoutDir), { recursive: true }); +rmSync(checkoutDir, { recursive: true, force: true }); +run("git", ["clone", "--no-checkout", readUrl, checkoutDir], "/"); +run("git", ["fetch", "--depth", "1", "origin", bootCommit], checkoutDir); +run("git", ["checkout", "--detach", bootCommit], checkoutDir); +linkNodeModules(); + +const scriptPath = path.join(checkoutDir, bootSh); +if (!existsSync(scriptPath)) fail(`boot script not found: ${bootSh}`); +run("sh", [scriptPath], checkoutDir, { replace: true }); + +function requiredEnv(name: string): string { + const value = process.env[name]; + if (!value) fail(`${name} is required`); + return value; +} + +function mirrorReadUrl(repo: string): string { + if (/github\.com[:/]pikasTech\/HWLAB(?:\.git)?$/u.test(repo)) { + return "http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git"; + } + fail("no mirror resolver for HWLAB_BOOT_REPO; refusing direct runtime GitHub fallback"); +} + +function linkNodeModules(): void { + const target = path.join(checkoutDir, "node_modules"); + if (existsSync(target) || !existsSync(runtimeNodeModules)) return; + symlinkSync(runtimeNodeModules, target, "dir"); +} + +function run(command: string, args: string[], cwd: string, options: { replace?: boolean } = {}): void { + const result = spawnSync(command, args, { + cwd, + env: process.env, + stdio: "inherit" + }); + if (result.error) fail(`${command} failed to start: ${result.error.message}`); + if (result.signal) process.kill(process.pid, result.signal); + const code = result.status ?? 0; + if (options.replace) process.exit(code); + if (code !== 0) process.exit(code); +} + +function fail(message: string): never { + process.stderr.write(`${JSON.stringify({ status: "failed", error: "hwlab_env_reuse_launcher_failed", message })}\n`); + process.exit(1); +} diff --git a/internal/build-metadata.mjs b/internal/build-metadata.mjs index b76a6639..e06e425b 100644 --- a/internal/build-metadata.mjs +++ b/internal/build-metadata.mjs @@ -55,6 +55,31 @@ export function buildMetadataFromEnv(env = process.env, { serviceId = "hwlab-unk provenance: firstNonEmpty(env.HWLAB_BUILD_PROVENANCE, null), metadataSource: buildCreatedAt ? "runtime-env:HWLAB_BUILD_CREATED_AT" : "unavailable", unavailableReason: buildCreatedAt ? null : "HWLAB_BUILD_CREATED_AT missing from runtime environment" + }, + runtime: runtimeIdentityFromEnv(env) + }; +} + +export function runtimeIdentityFromEnv(env = process.env) { + const runtimeMode = firstNonEmpty(env.HWLAB_RUNTIME_MODE, null); + const bootRepo = firstNonEmpty(env.HWLAB_BOOT_REPO, null); + const bootCommit = firstNonEmpty(env.HWLAB_BOOT_COMMIT, null); + const bootSh = firstNonEmpty(env.HWLAB_BOOT_SH, null); + const environmentImage = firstNonEmpty(env.HWLAB_ENVIRONMENT_IMAGE, null); + const environmentDigest = firstNonEmpty(env.HWLAB_ENVIRONMENT_DIGEST, env.HWLAB_IMAGE_DIGEST, null); + const environmentInputHash = firstNonEmpty(env.HWLAB_ENVIRONMENT_INPUT_HASH, null); + const codeInputHash = firstNonEmpty(env.HWLAB_CODE_INPUT_HASH, null); + if (!runtimeMode && !bootRepo && !bootCommit && !bootSh && !environmentDigest) return null; + return { + mode: runtimeMode, + environmentImage, + environmentDigest, + environmentInputHash, + codeInputHash, + boot: { + repo: bootRepo, + commit: bootCommit, + sh: bootSh } }; } diff --git a/internal/cloud/server.ts b/internal/cloud/server.ts index 48c06ec7..b1eedb39 100644 --- a/internal/cloud/server.ts +++ b/internal/cloud/server.ts @@ -165,6 +165,7 @@ export async function buildHealthPayload(options = {}) { commit: metadata.commit, image: metadata.image, build: metadata.build, + runtime: metadata.runtime, endpoint: env.HWLAB_PUBLIC_ENDPOINT || DEV_ENDPOINT, observedAt: new Date().toISOString(), db, diff --git a/internal/dev-entrypoint/http.mjs b/internal/dev-entrypoint/http.mjs index 6b1c6213..8a5df80f 100644 --- a/internal/dev-entrypoint/http.mjs +++ b/internal/dev-entrypoint/http.mjs @@ -41,6 +41,7 @@ export function healthPayload({ serviceId, role, details = {} }) { commit: metadata.commit, image: metadata.image, build: metadata.build, + runtime: metadata.runtime, endpoint: process.env.HWLAB_PUBLIC_ENDPOINT || DEV_ENDPOINT, observedAt: new Date().toISOString(), details diff --git a/scripts/artifact-publish.mjs b/scripts/artifact-publish.mjs index d02e66ab..6b108687 100644 --- a/scripts/artifact-publish.mjs +++ b/scripts/artifact-publish.mjs @@ -31,6 +31,7 @@ const defaultRegistryPrefix = const defaultBaseImage = process.env.HWLAB_DEV_BASE_IMAGE || null; const defaultReportPath = tempReportPath("dev-artifacts.json"); const buildBackend = "buildkit"; +const v02EnvReuseRuntimeMode = "env-reuse-git-mirror-checkout"; const defaultBuildkitCommand = process.env.HWLAB_BUILDKIT_BUILDCTL || "buildctl-daemonless.sh"; const defaultBuildkitAddr = process.env.HWLAB_BUILDKIT_ADDR || null; const defaultCatalogPath = process.env.HWLAB_ARTIFACT_CATALOG_PATH || "deploy/artifact-catalog.dev.json"; @@ -466,6 +467,15 @@ function buildkitCacheRef(registryPrefix, serviceId) { return `${registryPrefix}/cache/${serviceId}`; } +function envReuseImageRef(registryPrefix, serviceId, environmentInputHash) { + const tag = String(environmentInputHash || "unknown").slice(0, 12); + return `${registryPrefix}/${serviceId}-env:env-${tag}`; +} + +function envReuseCacheRef(registryPrefix, serviceId) { + return `${registryPrefix}/cache/${serviceId}-env`; +} + function buildkitRegistryOption(registryPrefix) { const parsed = parseRegistryPrefix(registryPrefix); return parsed.host === "127.0.0.1" || parsed.host === "localhost" @@ -627,23 +637,44 @@ function artifactCatalogSkeleton({ args, commitId }) { }, allowedProfiles: [artifactEnvironment(args)], forbiddenProfiles: args.lane === "v02" ? ["dev", "prod"] : ["prod"], - services: serviceIdsForLane(args.lane).map((serviceId) => ({ - serviceId, - commitId: tag, - sourceCommitId: commitId, - image: imageRef(args.registryPrefix, serviceId, tag), - imageTag: tag, - digest: "not_published", - publishState: "skeleton-only", - profile: artifactEnvironment(args), - namespace: args.lane === "v02" ? "hwlab-v02" : "hwlab-dev", - healthPath: "/health/live", - sourceState: "source-present", - publishEnabled: true, - artifactRequired: true, - artifactScope: "required", - notPublishedReason: "publish_not_run" - })) + services: serviceIdsForLane(args.lane).map((serviceId) => artifactCatalogSkeletonService({ args, serviceId, tag, commitId })) + }; +} + +function artifactCatalogSkeletonService({ args, serviceId, tag, commitId }) { + const base = { + serviceId, + commitId: tag, + sourceCommitId: commitId, + image: imageRef(args.registryPrefix, serviceId, tag), + imageTag: tag, + digest: "not_published", + publishState: "skeleton-only", + profile: artifactEnvironment(args), + namespace: args.lane === "v02" ? "hwlab-v02" : "hwlab-dev", + healthPath: "/health/live", + sourceState: "source-present", + publishEnabled: true, + artifactRequired: true, + artifactScope: "required", + notPublishedReason: "publish_not_run" + }; + if (args.lane !== "v02" || serviceId !== "hwlab-device-pod") return base; + const environmentImage = envReuseImageRef(args.registryPrefix, serviceId, commitId); + return { + ...base, + runtimeMode: v02EnvReuseRuntimeMode, + envReuse: true, + image: environmentImage, + imageTag: imageTagFromReference(environmentImage), + environmentImage, + environmentDigest: "not_published", + environmentInputHash: null, + codeInputHash: null, + bootRepo: "git@github.com:pikasTech/HWLAB.git", + bootCommit: commitId, + bootSh: `deploy/runtime/boot/${serviceId}.sh`, + bootEnvEvidence: bootEnvEvidence({ service: { serviceId, bootCommit: commitId, bootSh: `deploy/runtime/boot/${serviceId}.sh` }, commitId, environmentImage, environmentDigest: "not_published" }) }; } @@ -1268,6 +1299,25 @@ function dockerfile(baseImage, port, labels = []) { ].join("\n"); } +function envReuseDockerfile(baseImage, labels = []) { + const dependencyScript = "set -eu; if [ -f package-lock.json ]; then npm ci --omit=dev --include=optional --ignore-scripts; else npm install --omit=dev --include=optional --ignore-scripts; fi; if ! command -v bun >/dev/null 2>&1; then bun_pkg=\"\"; case \"$(uname -m)\" in x86_64|amd64) bun_pkg=\"@oven/bun-linux-x64@1.3.13\" ;; aarch64|arm64) bun_pkg=\"@oven/bun-linux-aarch64@1.3.13\" ;; *) echo \"unsupported bun architecture: $(uname -m)\" >&2; false ;; esac; npm install --no-save --omit=dev --include=optional --ignore-scripts \"$bun_pkg\"; test -x \"node_modules/${bun_pkg%@*}/bin/bun\"; ln -sf \"/opt/hwlab-env/node_modules/${bun_pkg%@*}/bin/bun\" /usr/local/bin/bun; fi"; + const readinessScript = "set -eu; command -v node >/dev/null; command -v npm >/dev/null; command -v git >/dev/null; command -v sh >/dev/null; command -v bun >/dev/null; test -d /opt/hwlab-env/node_modules"; + return [ + `FROM ${baseImage}`, + "WORKDIR /opt/hwlab-env", + ...labels.map(([name, value]) => `LABEL ${name}=${dockerfileQuote(value)}`), + "COPY package.json ./package.json", + "COPY package-lock.json* ./", + `RUN ${dependencyScript}`, + "COPY deploy/runtime/launcher/hwlab-env-reuse-launcher.ts /usr/local/bin/hwlab-env-reuse-launcher.ts", + `RUN ${readinessScript}`, + "ENV HWLAB_RUNTIME_MODE=env-reuse-git-mirror-checkout", + "ENV HWLAB_RUNTIME_NODE_MODULES=/opt/hwlab-env/node_modules", + "CMD [\"bun\", \"/usr/local/bin/hwlab-env-reuse-launcher.ts\"]", + "" + ].join("\n"); +} + function imageRef(registryPrefix, serviceId, tag) { return `${registryPrefix}/${serviceId}:${tag}`; } @@ -1398,6 +1448,7 @@ function hasFatalBlocker(blockers) { } async function buildService({ args, repo, commitId, shortCommit, service, buildCreatedAt }) { + if (service.envReuse) return buildEnvReuseService({ args, repo, commitId, service, buildCreatedAt }); const tag = shortCommit; const ref = imageRef(args.registryPrefix, service.serviceId, tag); const buildSource = `${repo}@${commitId}`; @@ -1495,6 +1546,166 @@ async function buildService({ args, repo, commitId, shortCommit, service, buildC return buildServiceWithBuildkit({ args, service, ref, tag, buildCreatedAt, buildSource, port, labels, envs }); } +async function buildEnvReuseService({ args, repo, commitId, service, buildCreatedAt }) { + const ref = envReuseImageRef(args.registryPrefix, service.serviceId, service.environmentInputHash); + const buildSource = `${repo}@env:${service.environmentInputHash}`; + const labels = [ + ["org.opencontainers.image.source", "https://github.com/pikasTech/HWLAB"], + ["org.opencontainers.image.revision", commitId], + ["org.opencontainers.image.created", buildCreatedAt], + ["org.opencontainers.image.title", `${service.serviceId}-env`], + ["hwlab.pikastech.local/runtime-mode", v02EnvReuseRuntimeMode], + ["hwlab.pikastech.local/service-id", service.serviceId], + ["hwlab.pikastech.local/environment-input-hash", service.environmentInputHash], + ["hwlab.pikastech.local/build-created-at", buildCreatedAt] + ]; + const artifact = await buildEnvReuseServiceWithBuildkit({ args, service, ref, buildCreatedAt, buildSource, labels }); + return withEnvReuseArtifactFields({ artifact, service, commitId, buildCreatedAt, buildSource }); +} + +async function buildEnvReuseServiceWithBuildkit({ args, service, ref, buildCreatedAt, buildSource, labels }) { + const tmpDir = await mkdtemp(path.join(os.tmpdir(), `hwlab-env-reuse-${service.serviceId}-`)); + const metadataPath = path.join(tmpDir, "metadata.json"); + const dockerfilePath = path.join(tmpDir, "Dockerfile"); + const cacheRef = envReuseCacheRef(args.registryPrefix, service.serviceId); + const registryOption = buildkitRegistryOption(args.registryPrefix); + const buildkitAddrArgs = args.buildkitAddr ? ["--addr", args.buildkitAddr] : []; + const startedAt = Date.now(); + try { + await writeFile(dockerfilePath, envReuseDockerfile(args.baseImage, labels)); + const argsList = [ + ...buildkitAddrArgs, + "build", + "--frontend", "dockerfile.v0", + "--local", `context=${repoRoot}`, + "--local", `dockerfile=${tmpDir}`, + "--opt", "filename=Dockerfile", + "--metadata-file", metadataPath, + "--import-cache", `type=registry,ref=${cacheRef}${registryOption}`, + "--export-cache", `type=registry,ref=${cacheRef},mode=max${registryOption}`, + "--output", `type=image,name=${ref},push=true${registryOption}` + ]; + if (!args.quietBuild || ciTimingEnabled()) argsList.push("--progress", "plain"); + const buildEnv = { ...process.env }; + if (!args.buildkitAddr) { + buildEnv.BUILDKITD_FLAGS = process.env.BUILDKITD_FLAGS || "--oci-worker-no-process-sandbox"; + buildEnv.XDG_RUNTIME_DIR = process.env.XDG_RUNTIME_DIR || path.join(os.tmpdir(), "hwlab-buildkit-runtime"); + await mkdir(buildEnv.XDG_RUNTIME_DIR, { recursive: true }); + } + const result = await run(args.buildkitCommand, argsList, { env: buildEnv }); + emitBuildkitTimingEvents({ service, result, timingEnabled: ciTimingEnabled(buildEnv) }); + if (result.code !== 0) { + return { + ...service, + image: ref, + imageTag: imageTagFromReference(ref), + buildCreatedAt, + buildSource, + status: "build_failed", + digest: "not_published", + blocker: blocker({ + type: "environment_blocker", + scope: service.serviceId, + summary: `BuildKit env image build failed for ${service.serviceId}`, + next: "Inspect the env image BuildKit output and base image git/bun availability." + }), + logTail: tailText(`${result.stdout}\n${result.stderr}`), + buildBackend: "buildkit-env-reuse", + buildkitCacheRef: cacheRef, + buildkitBuildDurationMs: result.durationMs + }; + } + let metadata = {}; + try { + metadata = JSON.parse(await readFile(metadataPath, "utf8")); + } catch {} + const digest = buildkitDigestFromMetadata(metadata); + if (!shaDigestPattern.test(digest ?? "")) { + return { + ...service, + image: ref, + imageTag: imageTagFromReference(ref), + buildCreatedAt, + buildSource, + status: "published_unverified_digest", + digest: "not_published", + blocker: blocker({ + type: "environment_blocker", + scope: service.serviceId, + summary: `BuildKit completed env image for ${service.serviceId} but did not expose an immutable digest`, + next: "Inspect BuildKit metadata output before updating the v02 env catalog." + }), + pushLogTail: tailText(`${result.stdout}\n${result.stderr}`, 1200), + buildBackend: "buildkit-env-reuse", + buildkitCacheRef: cacheRef, + buildkitBuildDurationMs: Date.now() - startedAt + }; + } + return { + ...service, + image: ref, + imageTag: imageTagFromReference(ref), + buildCreatedAt, + buildSource, + status: args.mode === "publish" ? "published" : "built", + digest, + repositoryDigest: `${repositoryFromImageRef(ref)}@${digest}`, + buildBackend: "buildkit-env-reuse", + buildkitCacheRef: cacheRef, + buildkitBuildDurationMs: Date.now() - startedAt, + publishDurationMs: args.mode === "publish" ? Date.now() - startedAt : null, + pushLogTail: tailText(`${result.stdout}\n${result.stderr}`, 1200) + }; + } finally { + await rm(tmpDir, { recursive: true, force: true }); + } +} + +function withEnvReuseArtifactFields({ artifact, service, commitId, buildCreatedAt = null, buildSource = null }) { + const environmentImage = artifact.image ?? service.environmentImage ?? null; + const environmentDigest = artifact.digest ?? service.environmentDigest ?? "not_published"; + return { + ...artifact, + runtimeMode: v02EnvReuseRuntimeMode, + envReuse: true, + envChanged: service.envChanged ?? null, + codeChanged: service.codeChanged ?? null, + commitId, + sourceCommitId: commitId, + environmentImage, + environmentDigest, + environmentInputHash: service.environmentInputHash ?? null, + codeInputHash: service.codeInputHash ?? null, + bootRepo: service.bootRepo ?? "git@github.com:pikasTech/HWLAB.git", + bootCommit: service.bootCommit ?? commitId, + bootSh: service.bootSh ?? `deploy/runtime/boot/${service.serviceId}.sh`, + image: environmentImage, + imageTag: imageTagFromReference(environmentImage), + digest: environmentDigest, + buildCreatedAt: artifact.buildCreatedAt ?? buildCreatedAt, + buildSource: artifact.buildSource ?? buildSource, + bootEnvEvidence: bootEnvEvidence({ service, commitId, environmentImage, environmentDigest }) + }; +} + +function bootEnvEvidence({ service, commitId, environmentImage, environmentDigest }) { + return { + runtimeMode: v02EnvReuseRuntimeMode, + environmentDigest: environmentDigest ?? null, + environmentImage: environmentImage ?? null, + bootRepo: service.bootRepo ?? "git@github.com:pikasTech/HWLAB.git", + bootCommit: service.bootCommit ?? commitId, + bootSh: service.bootSh ?? `deploy/runtime/boot/${service.serviceId}.sh`, + env: { + HWLAB_BOOT_REPO: service.bootRepo ?? "git@github.com:pikasTech/HWLAB.git", + HWLAB_BOOT_COMMIT: service.bootCommit ?? commitId, + HWLAB_BOOT_SH: service.bootSh ?? `deploy/runtime/boot/${service.serviceId}.sh`, + HWLAB_ENVIRONMENT_IMAGE: environmentImage ?? "", + HWLAB_ENVIRONMENT_DIGEST: environmentDigest ?? "" + } + }; +} + async function buildServiceWithBuildkit({ args, service, ref, tag, buildCreatedAt, buildSource, port, labels, envs }) { const tmpDir = await mkdtemp(path.join(os.tmpdir(), `hwlab-buildkit-${service.serviceId}-`)); const metadataPath = path.join(tmpDir, "metadata.json"); @@ -1711,6 +1922,12 @@ async function writeTektonResultFiles(dir, report) { "repository-digest": service.repositoryDigest ?? repositoryDigestFor(service.image, service.digest) ?? "", "source-commit-id": service.sourceCommitId, "component-input-hash": service.componentInputHash ?? "", + "environment-input-hash": service.environmentInputHash ?? "", + "code-input-hash": service.codeInputHash ?? "", + "runtime-mode": service.runtimeMode ?? "service-image", + "boot-repo": service.bootRepo ?? "", + "boot-commit": service.bootCommit ?? "", + "boot-sh": service.bootSh ?? "", "build-created-at": service.buildCreatedAt ?? "", "build-backend": report.artifactPublish.buildBackend ?? "unknown", "reused-from": service.reusedFrom ?? "" @@ -1720,6 +1937,28 @@ async function writeTektonResultFiles(dir, report) { } function artifactRecord({ args, service, commitId, shortCommit, buildCreatedAt = null, buildSource = null, status, digest = "not_published" }) { + if (service.envReuse) { + const environmentImage = service.environmentImage ?? envReuseImageRef(args.registryPrefix, service.serviceId, service.environmentInputHash ?? commitId); + const environmentDigest = service.environmentDigest ?? digest; + return withEnvReuseArtifactFields({ + artifact: { + ...service, + commitId, + sourceCommitId: commitId, + status, + image: environmentImage, + imageTag: imageTagFromReference(environmentImage), + buildCreatedAt, + buildSource, + digest: environmentDigest, + repositoryDigest: repositoryDigestFor(environmentImage, environmentDigest) + }, + service, + commitId, + buildCreatedAt, + buildSource + }); + } return { ...service, commitId: shortCommit, @@ -1735,6 +1974,9 @@ function artifactRecord({ args, service, commitId, shortCommit, buildCreatedAt = } function artifactRecordForCatalogReuse({ service, catalogService, commitId }) { + if (service.envReuse) { + return artifactRecordForEnvReuseCatalogReuse({ service, catalogService, commitId }); + } const image = catalogService?.image ?? null; const digest = catalogService?.digest ?? "not_published"; const imageTag = catalogService?.imageTag ?? imageTagFromReference(image); @@ -1771,6 +2013,49 @@ function artifactRecordForCatalogReuse({ service, catalogService, commitId }) { }; } +function artifactRecordForEnvReuseCatalogReuse({ service, catalogService, commitId }) { + const environmentImage = catalogService?.environmentImage ?? service.environmentImage ?? null; + const environmentDigest = catalogService?.environmentDigest ?? service.environmentDigest ?? "not_published"; + const imageTag = imageTagFromReference(environmentImage); + const base = { + ...service, + runtimeMode: v02EnvReuseRuntimeMode, + envReuse: true, + envChanged: service.envChanged ?? null, + codeChanged: service.codeChanged ?? null, + commitId, + sourceCommitId: commitId, + image: environmentImage, + imageTag, + digest: environmentDigest, + environmentImage, + environmentDigest, + environmentInputHash: service.environmentInputHash ?? catalogService?.environmentInputHash ?? null, + codeInputHash: service.codeInputHash ?? null, + bootRepo: service.bootRepo ?? catalogService?.bootRepo ?? "git@github.com:pikasTech/HWLAB.git", + bootCommit: service.bootCommit ?? commitId, + bootSh: service.bootSh ?? catalogService?.bootSh ?? `deploy/runtime/boot/${service.serviceId}.sh`, + buildCreatedAt: catalogService?.buildCreatedAt ?? null, + buildSource: catalogService?.buildSource ?? null, + repositoryDigest: catalogService?.repositoryDigest ?? repositoryDigestFor(environmentImage, environmentDigest), + bootEnvEvidence: bootEnvEvidence({ service, commitId, environmentImage, environmentDigest }) + }; + if (!environmentImage || !imageTag || !shaDigestPattern.test(environmentDigest)) { + return { + ...base, + status: "blocked_reuse_unavailable", + reusedFrom: null, + notPublishedReason: "environment_reuse_digest_unavailable" + }; + } + return { + ...base, + status: "reused", + reusedFrom: service.reuse?.reusedFrom ?? catalogService?.environmentInputHash ?? catalogService?.componentInputHash ?? imageTag, + notPublishedReason: null + }; +} + function externalServiceReportPath(dir, serviceId) { const base = path.isAbsolute(dir) ? dir : path.join(repoRoot, dir); return path.join(base, `${serviceId}.json`); @@ -1857,7 +2142,7 @@ function artifactRecordFromServiceResultsEnv({ service, env }) { const image = serviceResultValue(env, service.serviceId, "IMAGE"); const digest = serviceResultValue(env, service.serviceId, "DIGEST") ?? "not_published"; const imageTag = serviceResultValue(env, service.serviceId, "IMAGE_TAG") ?? imageTagFromReference(image); - return { + const artifact = { ...service, serviceId: service.serviceId, status, @@ -1869,10 +2154,20 @@ function artifactRecordFromServiceResultsEnv({ service, env }) { commitId: imageTag, buildCreatedAt: serviceResultValue(env, service.serviceId, "BUILD_CREATED_AT"), componentInputHash: serviceResultValue(env, service.serviceId, "COMPONENT_INPUT_HASH") ?? service.componentInputHash ?? null, + environmentInputHash: serviceResultValue(env, service.serviceId, "ENVIRONMENT_INPUT_HASH") ?? service.environmentInputHash ?? null, + codeInputHash: serviceResultValue(env, service.serviceId, "CODE_INPUT_HASH") ?? service.codeInputHash ?? null, + runtimeMode: serviceResultValue(env, service.serviceId, "RUNTIME_MODE") ?? service.runtimeMode ?? "service-image", + bootRepo: serviceResultValue(env, service.serviceId, "BOOT_REPO") ?? service.bootRepo ?? null, + bootCommit: serviceResultValue(env, service.serviceId, "BOOT_COMMIT") ?? service.bootCommit ?? null, + bootSh: serviceResultValue(env, service.serviceId, "BOOT_SH") ?? service.bootSh ?? null, reusedFrom: serviceResultValue(env, service.serviceId, "REUSED_FROM"), buildBackend: serviceResultValue(env, service.serviceId, "BUILD_BACKEND"), notPublishedReason: null }; + if (service.envReuse) { + return withEnvReuseArtifactFields({ artifact, service, commitId: artifact.sourceCommitId || service.bootCommit || "" }); + } + return artifact; } async function readExternalServiceArtifactsFromEnv({ services, commitId, env = process.env }) { @@ -1948,6 +2243,18 @@ function publishPlanFor({ args, commitId, shortCommit, mode, artifacts, fatalBlo buildArgsHash: artifact.buildArgsHash ?? null, ciAffected: artifact.ciAffected ?? null, ciReason: artifact.ciReason ?? [], + runtimeMode: artifact.runtimeMode ?? "service-image", + envReuse: artifact.envReuse === true, + envChanged: artifact.envChanged ?? null, + codeChanged: artifact.codeChanged ?? null, + environmentImage: artifact.environmentImage ?? null, + environmentDigest: artifact.environmentDigest ?? null, + environmentInputHash: artifact.environmentInputHash ?? null, + codeInputHash: artifact.codeInputHash ?? null, + bootRepo: artifact.bootRepo ?? null, + bootCommit: artifact.bootCommit ?? null, + bootSh: artifact.bootSh ?? null, + bootEnvEvidence: artifact.bootEnvEvidence ?? null, reuse: artifact.reuse ?? null, reusedFrom: artifact.reusedFrom ?? null, notPublishedReason: artifactNotPublishedReason(artifact, mode, fatalBlocked) @@ -1961,6 +2268,17 @@ function plannerFieldsFor(servicePlan) { return { componentCommitId: servicePlan.componentCommitId ?? null, componentInputHash: servicePlan.componentInputHash ?? null, + runtimeMode: servicePlan.runtimeMode ?? "service-image", + envReuse: servicePlan.envReuse === true, + envChanged: servicePlan.envChanged ?? null, + codeChanged: servicePlan.codeChanged ?? null, + environmentInputHash: servicePlan.environmentInputHash ?? null, + codeInputHash: servicePlan.codeInputHash ?? null, + bootRepo: servicePlan.bootRepo ?? null, + bootCommit: servicePlan.bootCommit ?? null, + bootSh: servicePlan.bootSh ?? null, + environmentImage: servicePlan.environmentImage ?? null, + environmentDigest: servicePlan.environmentDigest ?? null, dockerfileHash: servicePlan.dockerfileHash ?? null, baseImageReference: servicePlan.baseImageReference ?? null, baseImageDigest: servicePlan.baseImageDigest ?? null, @@ -2170,6 +2488,18 @@ function createReport({ args, repo, commitId, shortCommit, mode, services, artif buildArgsHash: artifact.buildArgsHash ?? null, ciAffected: artifact.ciAffected ?? null, ciReason: artifact.ciReason ?? [], + runtimeMode: artifact.runtimeMode ?? "service-image", + envReuse: artifact.envReuse === true, + envChanged: artifact.envChanged ?? null, + codeChanged: artifact.codeChanged ?? null, + environmentImage: artifact.environmentImage ?? null, + environmentDigest: artifact.environmentDigest ?? null, + environmentInputHash: artifact.environmentInputHash ?? null, + codeInputHash: artifact.codeInputHash ?? null, + bootRepo: artifact.bootRepo ?? null, + bootCommit: artifact.bootCommit ?? null, + bootSh: artifact.bootSh ?? null, + bootEnvEvidence: artifact.bootEnvEvidence ?? null, reuse: artifact.reuse ?? null, reusedFrom: artifact.reusedFrom ?? null, logTail: artifact.logTail ?? null, @@ -2247,7 +2577,9 @@ async function main() { const catalogByServiceId = new Map((catalog?.services ?? []).map((service) => [service.serviceId, service])); const affectedServiceIds = new Set(ciPlan?.affectedServices ?? []); const buildServiceIds = new Set( - services.filter((service) => service.artifactRequired && affectedServiceIds.has(service.serviceId)).map((service) => service.serviceId) + services + .filter((service) => service.artifactRequired && (service.envReuse ? service.envChanged === true : affectedServiceIds.has(service.serviceId))) + .map((service) => service.serviceId) ); const preflightBuildServiceIds = args.externalServiceReportDir || args.externalServiceResultsEnv ? new Set() : buildServiceIds; diff --git a/scripts/g14-ci-plan.mjs b/scripts/g14-ci-plan.mjs index 4d970e0b..79694b0b 100644 --- a/scripts/g14-ci-plan.mjs +++ b/scripts/g14-ci-plan.mjs @@ -19,6 +19,7 @@ function parseArgs(argv) { else if (arg === "--target-ref") parsed.targetRef = readOption(argv, ++index, arg); else if (arg === "--deploy-json") parsed.deployJsonPath = readOption(argv, ++index, arg); else if (arg === "--artifact-catalog") parsed.artifactCatalogPath = readOption(argv, ++index, arg); + else if (arg === "--lane") parsed.lane = readOption(argv, ++index, arg); else if (arg === "--registry-prefix") parsed.registryPrefix = readOption(argv, ++index, arg); else if (arg === "--base-image") parsed.baseImage = readOption(argv, ++index, arg); else if (arg === "--services") parsed.services = readOption(argv, ++index, arg).split(",").map((item) => item.trim()).filter(Boolean); diff --git a/scripts/g14-ci-plan.test.mjs b/scripts/g14-ci-plan.test.mjs index 56e699a4..8020839b 100644 --- a/scripts/g14-ci-plan.test.mjs +++ b/scripts/g14-ci-plan.test.mjs @@ -16,7 +16,7 @@ import { const execFileAsync = promisify(execFile); test("G14 CI planner maps bridge changes to cloud-api only", async () => { - const repo = await createFixtureRepo(); + const repo = await createFixtureRepo({ includeDevicePod: true }); await writeFile(path.join(repo, "cmd/hwlab-deepseek-responses-bridge/main.ts"), "console.log('bridge v2');\n"); await git(repo, ["add", "."]); await git(repo, ["commit", "-m", "change bridge"]); @@ -88,18 +88,62 @@ test("global change classifier distinguishes gitops-only and test-only", () => { assert.equal(classifyGlobalChange(["cmd/hwlab-cloud-api/runtime-options.test.ts"]).testOnly, true); }); +test("v02 planner marks device-pod code-only change as env reuse rollout", async () => { + const repo = await createFixtureRepo({ includeDevicePod: true }); + const initialPlan = await createG14CiPlan({ + repoRoot: repo, + lane: "v02", + baseRef: "HEAD", + targetRef: "HEAD", + artifactCatalogPath: "deploy/artifact-catalog.v02.json", + services: ["hwlab-device-pod"] + }); + const initialDevicePod = initialPlan.services.find((service) => service.serviceId === "hwlab-device-pod"); + await writeFile(path.join(repo, "deploy/artifact-catalog.v02.json"), JSON.stringify(createCatalogFixture("ready", { + devicePodEnvironmentInputHash: initialDevicePod.environmentInputHash, + devicePodCodeInputHash: initialDevicePod.codeInputHash, + devicePodBootCommit: initialPlan.sourceCommitId + }), null, 2)); + await git(repo, ["add", "deploy/artifact-catalog.v02.json"]); + await git(repo, ["commit", "-m", "seed v02 catalog"]); + + await writeFile(path.join(repo, "internal/device-pod/executor.ts"), "export const version = 2;\n"); + await git(repo, ["add", "."]); + await git(repo, ["commit", "-m", "change device pod code"]); + + const plan = await createG14CiPlan({ + repoRoot: repo, + lane: "v02", + baseRef: "HEAD~1", + targetRef: "HEAD", + artifactCatalogPath: "deploy/artifact-catalog.v02.json", + services: ["hwlab-device-pod"] + }); + const devicePod = plan.services.find((service) => service.serviceId === "hwlab-device-pod"); + assert.equal(devicePod.runtimeMode, "env-reuse-git-mirror-checkout"); + assert.equal(devicePod.envChanged, false); + assert.equal(devicePod.codeChanged, true); + assert.equal(devicePod.bootSh, "deploy/runtime/boot/hwlab-device-pod.sh"); + assert.deepEqual(plan.affectedServices, ["hwlab-device-pod"]); +}); + async function createFixtureRepo(options = {}) { const deployServices = options.deployServices !== false; const k3sServiceMappings = options.k3sServiceMappings === true; + const includeDevicePod = options.includeDevicePod === true; const repo = await mkdtemp(path.join(os.tmpdir(), "hwlab-g14-ci-plan-")); await mkdir(path.join(repo, "cmd/hwlab-cloud-api"), { recursive: true }); await mkdir(path.join(repo, "cmd/hwlab-deepseek-responses-bridge"), { recursive: true }); await mkdir(path.join(repo, "web/hwlab-cloud-web"), { recursive: true }); + await mkdir(path.join(repo, "internal/device-pod"), { recursive: true }); + await mkdir(path.join(repo, "deploy/runtime/boot"), { recursive: true }); + await mkdir(path.join(repo, "deploy/runtime/launcher"), { recursive: true }); await mkdir(path.join(repo, "deploy"), { recursive: true }); - await writeFile(path.join(repo, "deploy/deploy.json"), JSON.stringify(createDeployFixture({ deployServices, k3sServiceMappings }), null, 2)); + await writeFile(path.join(repo, "deploy/deploy.json"), JSON.stringify(createDeployFixture({ deployServices, k3sServiceMappings, includeDevicePod }), null, 2)); const catalogMode = options.catalog ?? "ready"; if (catalogMode !== false) { await writeFile(path.join(repo, "deploy/artifact-catalog.dev.json"), JSON.stringify(createCatalogFixture(catalogMode), null, 2)); + await writeFile(path.join(repo, "deploy/artifact-catalog.v02.json"), JSON.stringify(createCatalogFixture(catalogMode), null, 2)); } await writeFile(path.join(repo, "package.json"), JSON.stringify({ type: "module" }, null, 2)); await writeFile(path.join(repo, "package-lock.json"), JSON.stringify({ lockfileVersion: 3 }, null, 2)); @@ -107,40 +151,64 @@ async function createFixtureRepo(options = {}) { await writeFile(path.join(repo, "cmd/hwlab-deepseek-responses-bridge/main.ts"), "console.log('bridge');\n"); await writeFile(path.join(repo, "cmd/hwlab-deepseek-responses-bridge/main.test.ts"), "console.log('test');\n"); await writeFile(path.join(repo, "web/hwlab-cloud-web/index.html"), "\n"); + await writeFile(path.join(repo, "internal/device-pod/executor.ts"), "export const version = 1;\n"); + await writeFile(path.join(repo, "deploy/runtime/boot/hwlab-device-pod.sh"), "#!/bin/sh\nexec bun cmd/hwlab-device-pod/main.ts\n"); + await writeFile(path.join(repo, "deploy/runtime/launcher/hwlab-env-reuse-launcher.ts"), "console.log('launcher');\n"); await git(repo, ["init"]); await git(repo, ["add", "."]); await git(repo, ["commit", "-m", "initial"]); return repo; } -function createDeployFixture({ deployServices, k3sServiceMappings }) { - const deploy = {}; +function createDeployFixture({ deployServices, k3sServiceMappings, includeDevicePod }) { + const deploy = { + lanes: { + v02: { + sourceRepo: "git@github.com:pikasTech/HWLAB.git", + envReuseServices: ["hwlab-device-pod"], + bootScripts: { "hwlab-device-pod": "deploy/runtime/boot/hwlab-device-pod.sh" } + } + } + }; if (deployServices) { deploy.services = [ { serviceId: "hwlab-cloud-api" }, - { serviceId: "hwlab-cloud-web" } + { serviceId: "hwlab-cloud-web" }, + ...(includeDevicePod ? [{ serviceId: "hwlab-device-pod" }] : []) ]; } if (k3sServiceMappings) { deploy.k3s = { serviceMappings: [ { serviceId: "hwlab-cloud-api", name: "hwlab-cloud-api" }, - { serviceId: "hwlab-cloud-web", name: "hwlab-cloud-web" } + { serviceId: "hwlab-cloud-web", name: "hwlab-cloud-web" }, + ...(includeDevicePod ? [{ serviceId: "hwlab-device-pod", name: "hwlab-device-pod" }] : []) ] }; } return deploy; } -function createCatalogFixture(mode) { +function createCatalogFixture(mode, options = {}) { const digest = mode === "ready" ? `sha256:${"1".repeat(64)}` : "not_published"; return { - services: ["hwlab-cloud-api", "hwlab-cloud-web"].map((serviceId) => ({ + services: ["hwlab-cloud-api", "hwlab-cloud-web", "hwlab-device-pod"].map((serviceId) => ({ serviceId, commitId: "abc1234", image: `127.0.0.1:5000/hwlab/${serviceId}:abc1234`, imageTag: "abc1234", - digest + digest, + ...(serviceId === "hwlab-device-pod" ? { + runtimeMode: "env-reuse-git-mirror-checkout", + envReuse: true, + environmentImage: "127.0.0.1:5000/hwlab/hwlab-device-pod-env:env-abc1234", + environmentDigest: digest, + environmentInputHash: options.devicePodEnvironmentInputHash ?? "e".repeat(64), + codeInputHash: options.devicePodCodeInputHash ?? "c".repeat(64), + bootRepo: "git@github.com:pikasTech/HWLAB.git", + bootCommit: options.devicePodBootCommit ?? "abc1234", + bootSh: "deploy/runtime/boot/hwlab-device-pod.sh" + } : {}) })) }; } diff --git a/scripts/g14-gitops-render.mjs b/scripts/g14-gitops-render.mjs index 0fe8e509..d537b39b 100644 --- a/scripts/g14-gitops-render.mjs +++ b/scripts/g14-gitops-render.mjs @@ -76,6 +76,7 @@ const codexApiProfileBaseUrl = process.env.HWLAB_G14_CODEX_API_PROFILE_BASE_URL const codexApiForwarderPort = process.env.HWLAB_G14_CODEX_API_FORWARDER_PORT || "49280"; const codexApiForwarderUpstreamBaseUrl = process.env.HWLAB_G14_CODEX_API_UPSTREAM_BASE_URL || "https://hyueapi.com"; const sourceCommitPattern = /^[a-f0-9]{7,40}$/u; +const v02EnvReuseRuntimeMode = "env-reuse-git-mirror-checkout"; const primitiveValidationTasks = Object.freeze([ { name: "repo-reports-guard", @@ -407,8 +408,47 @@ function catalogServiceById(catalog, serviceId) { return (catalog?.services ?? []).find((service) => service?.serviceId === serviceId) ?? null; } +function v02EnvReuseEnabled(catalog, serviceId) { + const service = catalogServiceById(catalog, serviceId); + return service?.runtimeMode === v02EnvReuseRuntimeMode || service?.envReuse === true; +} + +function bootShForService(deployService, serviceId) { + const value = deployService?.bootSh || deployService?.bootScript || `deploy/runtime/boot/${serviceId}.sh`; + const text = String(value).replaceAll("\\", "/").replace(/^\.\//u, ""); + assert.ok(text && !text.startsWith("/") && !text.split("/").includes(".."), `${serviceId} boot script must be repo-relative`); + return text; +} + +function bootMetadataForService({ args, catalog, deployService, serviceId, source }) { + const catalogService = catalogServiceById(catalog, serviceId); + const environmentImage = catalogService?.environmentImage ?? imageFor(args.registryPrefix, `${serviceId}-env`, `env-${String(catalogService?.environmentInputHash ?? source.full).slice(0, 12)}`); + const environmentDigest = catalogService?.environmentDigest ?? catalogService?.digest ?? null; + return { + runtimeMode: v02EnvReuseRuntimeMode, + environmentImage, + environmentDigest, + environmentInputHash: catalogService?.environmentInputHash ?? null, + codeInputHash: catalogService?.codeInputHash ?? null, + bootRepo: catalogService?.bootRepo ?? args.sourceRepo, + bootCommit: catalogService?.bootCommit ?? source.full, + bootSh: catalogService?.bootSh ?? bootShForService(deployService, serviceId) + }; +} + function runtimeArtifactForService({ catalog, serviceId, source, registryPrefix, useDeployImages = false, digestPin = false }) { const catalogService = catalogServiceById(catalog, serviceId); + if (catalogService?.runtimeMode === v02EnvReuseRuntimeMode || catalogService?.envReuse === true) { + const environmentImage = catalogService.environmentImage ?? imageFor(registryPrefix, `${serviceId}-env`, `env-${String(catalogService.environmentInputHash ?? source.full).slice(0, 12)}`); + const runtimeImage = digestPin && useDeployImages ? digestPinnedImage(environmentImage, catalogService.environmentDigest ?? catalogService.digest) : environmentImage; + return { + image: runtimeImage, + imageTag: imageTagFromReference(environmentImage) ?? source.imageTag ?? source.short, + commit: catalogService.bootCommit ?? source.full, + runtimeMode: v02EnvReuseRuntimeMode, + environmentDigest: catalogService.environmentDigest ?? catalogService.digest ?? null + }; + } const fallbackImageTag = source.imageTag ?? source.short; const image = useDeployImages && catalogService?.image ? catalogService.image @@ -480,17 +520,44 @@ function artifactCatalogSkeleton({ args, source }) { }, allowedProfiles: [environment], forbiddenProfiles: args.lane === "v02" ? ["dev", "prod"] : ["prod"], - services: serviceIdsForLane(args.lane).map((serviceId) => ({ - serviceId, - profile: environment, - namespace, - commitId: tag, - sourceCommitId: source.full, - image: imageFor(args.registryPrefix, serviceId, tag), - imageTag: tag, - digest: null, - buildBackend: "contract-skeleton" - })) + services: serviceIdsForLane(args.lane).map((serviceId) => artifactCatalogSkeletonService({ args, source, serviceId, environment, namespace, tag })) + }; +} + +function artifactCatalogSkeletonService({ args, source, serviceId, environment, namespace, tag }) { + const base = { + serviceId, + profile: environment, + namespace, + commitId: tag, + sourceCommitId: source.full, + image: imageFor(args.registryPrefix, serviceId, tag), + imageTag: tag, + digest: null, + buildBackend: "contract-skeleton" + }; + if (args.lane !== "v02" || serviceId !== "hwlab-device-pod") return base; + const environmentImage = imageFor(args.registryPrefix, `${serviceId}-env`, `env-${String(source.full).slice(0, 12)}`); + return { + ...base, + image: environmentImage, + imageTag: imageTagFromReference(environmentImage), + runtimeMode: v02EnvReuseRuntimeMode, + envReuse: true, + environmentImage, + environmentDigest: null, + environmentInputHash: null, + bootRepo: args.sourceRepo, + bootCommit: source.full, + bootSh: `deploy/runtime/boot/${serviceId}.sh`, + codeInputHash: null, + bootEnvEvidence: { + env: { + HWLAB_BOOT_REPO: args.sourceRepo, + HWLAB_BOOT_COMMIT: source.full, + HWLAB_BOOT_SH: `deploy/runtime/boot/${serviceId}.sh` + } + } }; } @@ -718,6 +785,8 @@ function transformWorkloads({ workloads, deploy, catalog, source, registryPrefix const deployService = deployServices.get(serviceId); if (!deployService) continue; const artifact = runtimeArtifactForService({ catalog, serviceId, source, registryPrefix, useDeployImages, digestPin }); + const envReuse = profile === "v02" && v02EnvReuseEnabled(catalog, serviceId); + const bootMetadata = envReuse ? bootMetadataForService({ args: { sourceRepo: deploy?.lanes?.v02?.sourceRepo || defaultSourceRepo, registryPrefix }, catalog, deployService, serviceId, source }) : null; const image = artifact.image; const runtimeCommit = artifact.commit; const runtimeImageTag = artifact.imageTag; @@ -739,6 +808,31 @@ function transformWorkloads({ workloads, deploy, catalog, source, registryPrefix upsertEnv(container.env, "HWLAB_GITOPS_TARGET", gitopsTarget); upsertEnv(container.env, "HWLAB_GITOPS_PROFILE", profileLabel); upsertEnv(container.env, "HWLAB_GITOPS_SOURCE_COMMIT", source.full); + if (bootMetadata) { + upsertEnv(container.env, "HWLAB_RUNTIME_MODE", bootMetadata.runtimeMode); + upsertEnv(container.env, "HWLAB_BOOT_REPO", bootMetadata.bootRepo); + upsertEnv(container.env, "HWLAB_BOOT_COMMIT", bootMetadata.bootCommit); + upsertEnv(container.env, "HWLAB_BOOT_SH", bootMetadata.bootSh); + upsertEnv(container.env, "HWLAB_BOOT_READ_URL", defaultV02GitReadUrl); + upsertEnv(container.env, "HWLAB_ENVIRONMENT_IMAGE", bootMetadata.environmentImage ?? ""); + upsertEnv(container.env, "HWLAB_ENVIRONMENT_DIGEST", bootMetadata.environmentDigest ?? ""); + upsertEnv(container.env, "HWLAB_ENVIRONMENT_INPUT_HASH", bootMetadata.environmentInputHash ?? ""); + upsertEnv(container.env, "HWLAB_CODE_INPUT_HASH", bootMetadata.codeInputHash ?? ""); + upsertEnv(container.env, "HWLAB_IMAGE_DIGEST", bootMetadata.environmentDigest ?? "unknown"); + annotate(podTemplate.metadata, { + "hwlab.pikastech.local/runtime-mode": bootMetadata.runtimeMode, + "hwlab.pikastech.local/boot-repo": bootMetadata.bootRepo, + "hwlab.pikastech.local/boot-commit": bootMetadata.bootCommit, + "hwlab.pikastech.local/boot-sh": bootMetadata.bootSh, + "hwlab.pikastech.local/environment-digest": bootMetadata.environmentDigest ?? "not_published", + "hwlab.pikastech.local/environment-input-hash": bootMetadata.environmentInputHash ?? "unknown", + "hwlab.pikastech.local/code-input-hash": bootMetadata.codeInputHash ?? "unknown" + }); + label(podTemplate.metadata, { + "hwlab.pikastech.local/runtime-mode": bootMetadata.runtimeMode, + "hwlab.pikastech.local/boot-commit": bootMetadata.bootCommit + }); + } if (serviceId === "hwlab-cloud-api" || serviceId === "hwlab-edge-proxy") { upsertEnv(container.env, "HWLAB_PUBLIC_ENDPOINT", runtimeEndpoint); } @@ -1033,6 +1127,7 @@ function prepareSourceScript() { "git remote set-url origin \"$git_read_url\"", "git checkout \"$(params.revision)\"", "test \"$(git rev-parse HEAD)\" = \"$(params.revision)\"", + "git merge-base --is-ancestor \"$(params.revision)\" \"origin/$(params.source-branch)\" || { echo '{\"event\":\"source-ancestry\",\"status\":\"failed\",\"revision\":\"'\"$(params.revision)\"'\",\"sourceBranch\":\"'\"$(params.source-branch)\"'\"}'; exit 32; }", "ci_timing_emit source-clone succeeded \"$source_clone_started_ms\"", "catalog_path=\"$(params.catalog-path)\"", "mkdir -p \"$(dirname \"$catalog_path\")\"", @@ -1064,7 +1159,14 @@ function prepareSourceScript() { "const namespace = 'hwlab-v02';", "const selected = (selectedServices || '').split(',').filter(Boolean);", "const serviceIds = selected.length ? selected : (deploy.services || []).map((service) => service.serviceId);", - "const services = serviceIds.map((serviceId) => ({ serviceId, profile: 'v02', namespace, commitId: tag, sourceCommitId: revision, image: `${registryPrefix}/${serviceId}:${tag}`, imageTag: tag, digest: null, buildBackend: 'contract-skeleton' }));", + "const services = serviceIds.map((serviceId) => {", + " const service = { serviceId, profile: 'v02', namespace, commitId: tag, sourceCommitId: revision, image: `${registryPrefix}/${serviceId}:${tag}`, imageTag: tag, digest: null, buildBackend: 'contract-skeleton' };", + " if (serviceId === 'hwlab-device-pod') {", + " const environmentImage = `${registryPrefix}/${serviceId}-env:env-${revision.slice(0, 12)}`;", + " Object.assign(service, { image: environmentImage, imageTag: environmentImage.slice(environmentImage.lastIndexOf(':') + 1), runtimeMode: 'env-reuse-git-mirror-checkout', envReuse: true, environmentImage, environmentDigest: null, environmentInputHash: null, codeInputHash: null, bootRepo: 'git@github.com:pikasTech/HWLAB.git', bootCommit: revision, bootSh: 'deploy/runtime/boot/hwlab-device-pod.sh', bootEnvEvidence: { env: { HWLAB_BOOT_REPO: 'git@github.com:pikasTech/HWLAB.git', HWLAB_BOOT_COMMIT: revision, HWLAB_BOOT_SH: 'deploy/runtime/boot/hwlab-device-pod.sh', HWLAB_ENVIRONMENT_IMAGE: environmentImage, HWLAB_ENVIRONMENT_DIGEST: '' } } });", + " }", + " return service;", + "});", "fs.mkdirSync(path.dirname(catalogPath), { recursive: true });", "fs.writeFileSync(catalogPath, JSON.stringify({ catalogVersion: 'v1', kind: 'hwlab-artifact-catalog', environment: 'v02', profile: 'v02', namespace, endpoint: 'http://74.48.78.17:19667', commitId: tag, artifactState: 'contract-skeleton', publish: { registryPrefix, sourceCommitId: revision, imageTag: tag, publishedAt: null }, allowedProfiles: ['v02'], forbiddenProfiles: ['dev', 'prod'], services }, null, 2) + '\\n');", "NODE", @@ -1130,7 +1232,7 @@ echo '{"event":"ci-base-image","phase":"plan-artifacts","image":"${ciToolsRunner for tool in node git; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"plan-artifacts","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done cd /workspace/source/repo test "$(git rev-parse HEAD)" = "$(params.revision)" -node scripts/g14-ci-plan.mjs --target-ref HEAD --deploy-json deploy/deploy.json --artifact-catalog "$(params.catalog-path)" --registry-prefix "$(params.registry-prefix)" --services "$(params.services)" > /workspace/source/ci-plan.json +node scripts/g14-ci-plan.mjs --lane "$(params.lane)" --target-ref HEAD --deploy-json deploy/deploy.json --artifact-catalog "$(params.catalog-path)" --registry-prefix "$(params.registry-prefix)" --services "$(params.services)" > /workspace/source/ci-plan.json node - <<'NODE' const fs = require("node:fs"); const plan = JSON.parse(fs.readFileSync("/workspace/source/ci-plan.json", "utf8")); @@ -1138,7 +1240,23 @@ const selected = String(process.env.HWLAB_SELECTED_SERVICES || "").split(",").ma const allServices = selected.length > 0 ? selected : ${JSON.stringify(defaultServiceIds)}; const affected = new Set(plan.affectedServices || []); const selectedSet = new Set(selected); -const entries = allServices.map((serviceId) => ({ serviceId, selected: selectedSet.has(serviceId), affected: selectedSet.has(serviceId) && affected.has(serviceId) })); +const byService = new Map((plan.services || []).map((service) => [service.serviceId, service])); +const entries = allServices.map((serviceId) => { + const service = byService.get(serviceId) || {}; + const serviceSelected = selectedSet.has(serviceId); + const rolloutAffected = serviceSelected && affected.has(serviceId); + const envReuse = service.runtimeMode === "env-reuse-git-mirror-checkout" || service.envReuse === true; + const buildRequired = envReuse ? service.envChanged === true : rolloutAffected; + return { + serviceId, + selected: serviceSelected, + affected: buildRequired, + rolloutAffected, + runtimeMode: service.runtimeMode || "service-image", + envChanged: service.envChanged ?? null, + codeChanged: service.codeChanged ?? null + }; +}); for (const entry of entries) { fs.writeFileSync("/tekton/results/affected-" + entry.serviceId, entry.affected ? "true" : "false"); } @@ -1147,6 +1265,7 @@ fs.writeFileSync("/workspace/source/affected-services.json", JSON.stringify({ affectedServices: plan.affectedServices || [], reusedServices: plan.reusedServices || [], buildSkippedCount: plan.buildSkippedCount || 0, + services: plan.services || [], entries }, null, 2) + String.fromCharCode(10)); console.log(JSON.stringify({ event: "g14-ci-plan", sourceCommitId: plan.sourceCommitId, affectedServices: plan.affectedServices || [], reusedServices: plan.reusedServices || [], buildSkippedCount: plan.buildSkippedCount || 0 })); @@ -1178,11 +1297,15 @@ if [ -s /workspace/source/affected-services.json ] && ! node -e 'const fs=requir const fs = require("node:fs"); const serviceId = process.argv[2]; const catalog = JSON.parse(fs.readFileSync(process.env.HWLAB_ARTIFACT_CATALOG_PATH, "utf8")); +const plan = fs.existsSync("/workspace/source/affected-services.json") ? JSON.parse(fs.readFileSync("/workspace/source/affected-services.json", "utf8")) : {}; const service = (catalog.services || []).find((item) => item.serviceId === serviceId) || {}; -const image = service.image || ""; -const digest = service.digest || "not_published"; +const planned = (plan.services || []).find((item) => item.serviceId === serviceId) || {}; +const envReuse = planned.runtimeMode === "env-reuse-git-mirror-checkout" || service.runtimeMode === "env-reuse-git-mirror-checkout" || planned.envReuse === true || service.envReuse === true; +const image = envReuse ? (service.environmentImage || service.image || "") : (service.image || ""); +const digest = envReuse ? (service.environmentDigest || service.digest || "not_published") : (service.digest || "not_published"); const imageTag = service.imageTag || (image.includes(":") ? image.slice(image.lastIndexOf(":") + 1) : ""); const repository = image.includes(":") ? image.slice(0, image.lastIndexOf(":")) : image; +const revision = process.env.HWLAB_SOURCE_REVISION || planned.bootCommit || service.bootCommit || service.sourceCommitId || service.commitId || imageTag; const values = { "service-id": serviceId, status: /^sha256:[a-f0-9]{64}$/.test(digest) ? "reused" : "blocked_reuse_unavailable", @@ -1190,11 +1313,17 @@ const values = { "image-tag": imageTag, digest, "repository-digest": /^sha256:[a-f0-9]{64}$/.test(digest) && repository ? repository + "@" + digest : "", - "source-commit-id": service.sourceCommitId || service.commitId || imageTag, - "component-input-hash": service.componentInputHash || "", + "source-commit-id": envReuse ? revision : (service.sourceCommitId || service.commitId || imageTag), + "component-input-hash": planned.componentInputHash || service.componentInputHash || "", + "environment-input-hash": planned.environmentInputHash || service.environmentInputHash || "", + "code-input-hash": planned.codeInputHash || service.codeInputHash || "", + "runtime-mode": envReuse ? "env-reuse-git-mirror-checkout" : "service-image", + "boot-repo": planned.bootRepo || service.bootRepo || "", + "boot-commit": envReuse ? revision : (service.bootCommit || ""), + "boot-sh": planned.bootSh || service.bootSh || "", "build-created-at": service.buildCreatedAt || "", - "build-backend": "reused-catalog", - "reused-from": service.componentInputHash || service.commitId || imageTag || "catalog" + "build-backend": envReuse ? "reused-env-catalog" : "reused-catalog", + "reused-from": (envReuse ? service.environmentInputHash : service.componentInputHash) || service.commitId || imageTag || "catalog" }; for (const [name, value] of Object.entries(values)) fs.writeFileSync("/tekton/results/" + name, String(value || "")); NODE @@ -1895,6 +2024,12 @@ const serviceResultFields = Object.freeze([ "repository-digest", "source-commit-id", "component-input-hash", + "environment-input-hash", + "code-input-hash", + "runtime-mode", + "boot-repo", + "boot-commit", + "boot-sh", "build-created-at", "build-backend", "reused-from" diff --git a/scripts/g14-gitops-render.test.ts b/scripts/g14-gitops-render.test.ts index 28bd69bd..ad10caa8 100644 --- a/scripts/g14-gitops-render.test.ts +++ b/scripts/g14-gitops-render.test.ts @@ -129,6 +129,15 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots assert.ok(devicePod, "expected v02 hwlab-device-pod workload"); const devicePodEnv = devicePod.spec.template.spec.containers[0].env; assert.ok(devicePodEnv.some((entry) => entry.name === "HWLAB_CLOUD_API_INTERNAL_URL" && entry.value === "http://hwlab-cloud-api.hwlab-v02.svc.cluster.local:6667")); + assert.ok(devicePodEnv.some((entry) => entry.name === "HWLAB_RUNTIME_MODE" && entry.value === "env-reuse-git-mirror-checkout")); + assert.ok(devicePodEnv.some((entry) => entry.name === "HWLAB_BOOT_REPO" && entry.value === "git@github.com:pikasTech/HWLAB.git")); + assert.ok(devicePodEnv.some((entry) => entry.name === "HWLAB_BOOT_COMMIT" && entry.value === sourceRevision)); + assert.ok(devicePodEnv.some((entry) => entry.name === "HWLAB_BOOT_SH" && entry.value === "deploy/runtime/boot/hwlab-device-pod.sh")); + assert.equal(devicePod.spec.template.metadata.annotations["hwlab.pikastech.local/runtime-mode"], "env-reuse-git-mirror-checkout"); + assert.equal(devicePod.spec.template.metadata.annotations["hwlab.pikastech.local/boot-commit"], sourceRevision); + const agentWorker = workloadsJson.items.find((item) => item.metadata?.name === "hwlab-agent-worker"); + const agentWorkerEnv = agentWorker?.spec?.template?.spec?.containers?.[0]?.env ?? []; + assert.equal(agentWorkerEnv.some((entry) => entry.name === "HWLAB_BOOT_COMMIT"), false, "agent-worker must not be switched to env reuse by device-pod work"); const workloadTemplates = workloadsJson.items.filter((item) => item.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/source-commit"]); assert.ok(workloadTemplates.length > 0, "expected rendered pod-template source labels"); for (const item of workloadTemplates) { diff --git a/scripts/refresh-artifact-catalog.mjs b/scripts/refresh-artifact-catalog.mjs index cfc50c9a..4b16c4aa 100644 --- a/scripts/refresh-artifact-catalog.mjs +++ b/scripts/refresh-artifact-catalog.mjs @@ -24,6 +24,7 @@ const digestPattern = /^sha256:[a-f0-9]{64}$/; const commitPattern = /^[a-f0-9]{7,40}$/; const publishReadyStatuses = new Set(["published", "reused"]); +const v02EnvReuseRuntimeMode = "env-reuse-git-mirror-checkout"; const v02RuntimeServiceIds = Object.freeze([ "hwlab-cloud-api", @@ -202,23 +203,50 @@ function artifactCatalogSkeleton({ args, target, registryPrefix = defaultRegistr }, allowedProfiles: [environment], forbiddenProfiles: args.lane === "v02" ? ["dev", "prod"] : ["prod"], - services: serviceIdsForLane(args.lane).map((serviceId) => ({ - serviceId, - commitId: target.imageTag, - sourceCommitId: target.commitId, - image: targetImage(serviceId, target.imageTag, registryPrefix), - imageTag: target.imageTag, - digest: "not_published", - publishState: "skeleton-only", - profile: environment, - namespace, - healthPath: "/health/live", - sourceState: "source-present", - publishEnabled: true, - artifactRequired: true, - artifactScope: "required", - notPublishedReason: "publish_not_run" - })) + services: serviceIdsForLane(args.lane).map((serviceId) => catalogSkeletonService({ args, serviceId, target, registryPrefix, environment, namespace })) + }; +} + +function catalogSkeletonService({ args, serviceId, target, registryPrefix, environment, namespace }) { + const base = { + serviceId, + commitId: target.imageTag, + sourceCommitId: target.commitId, + image: targetImage(serviceId, target.imageTag, registryPrefix), + imageTag: target.imageTag, + digest: "not_published", + publishState: "skeleton-only", + profile: environment, + namespace, + healthPath: "/health/live", + sourceState: "source-present", + publishEnabled: true, + artifactRequired: true, + artifactScope: "required", + notPublishedReason: "publish_not_run" + }; + if (args.lane !== "v02" || serviceId !== "hwlab-device-pod") return base; + const environmentImage = targetImage(`${serviceId}-env`, `env-${String(target.commitId).slice(0, 12)}`, registryPrefix); + return { + ...base, + image: environmentImage, + imageTag: parseTaggedImage(environmentImage, `${serviceId} environment image`).tag, + runtimeMode: v02EnvReuseRuntimeMode, + envReuse: true, + environmentImage, + environmentDigest: "not_published", + environmentInputHash: null, + bootRepo: "git@github.com:pikasTech/HWLAB.git", + bootCommit: target.commitId, + bootSh: `deploy/runtime/boot/${serviceId}.sh`, + codeInputHash: null, + bootEnvEvidence: { + env: { + HWLAB_BOOT_REPO: "git@github.com:pikasTech/HWLAB.git", + HWLAB_BOOT_COMMIT: target.commitId, + HWLAB_BOOT_SH: `deploy/runtime/boot/${serviceId}.sh` + } + } }; } @@ -343,6 +371,18 @@ function publishRecordsFromReport(report, target, serviceIds = SERVICE_IDS) { repositoryDigest: service.repositoryDigest ?? `${image.repository}@${service.digest}`, componentCommitId: service.componentCommitId ?? null, componentInputHash: service.componentInputHash ?? null, + runtimeMode: service.runtimeMode ?? "service-image", + envReuse: service.envReuse === true, + envChanged: service.envChanged ?? null, + codeChanged: service.codeChanged ?? null, + environmentImage: service.environmentImage ?? null, + environmentDigest: service.environmentDigest ?? null, + environmentInputHash: service.environmentInputHash ?? null, + codeInputHash: service.codeInputHash ?? null, + bootRepo: service.bootRepo ?? null, + bootCommit: service.bootCommit ?? null, + bootSh: service.bootSh ?? null, + bootEnvEvidence: service.bootEnvEvidence ?? null, dockerfileHash: service.dockerfileHash ?? null, baseImageReference: service.baseImageReference ?? null, baseImageDigest: service.baseImageDigest ?? null, @@ -390,8 +430,14 @@ function refreshDocuments({ deploy, catalog, target, publishRecords, serviceInve const inventory = inventoryByService.get(serviceId); const required = requiredIds.has(serviceId); const publishRecord = records?.get(serviceId) ?? null; - const image = publishRecord?.image ?? targetImage(serviceId, target.imageTag, registryPrefix); - const imageTag = publishRecord?.imageTag ?? target.imageTag; + const envReuse = publishRecord?.runtimeMode === v02EnvReuseRuntimeMode + || publishRecord?.envReuse === true + || catalogService.runtimeMode === v02EnvReuseRuntimeMode + || catalogService.envReuse === true; + const fallbackEnvTag = `env-${String(publishRecord?.environmentInputHash ?? catalogService.environmentInputHash ?? target.commitId).slice(0, 12)}`; + const fallbackImage = envReuse ? targetImage(`${serviceId}-env`, fallbackEnvTag, registryPrefix) : targetImage(serviceId, target.imageTag, registryPrefix); + const image = publishRecord?.image ?? publishRecord?.environmentImage ?? fallbackImage; + const imageTag = publishRecord?.imageTag ?? parseTaggedImage(image, `${serviceId} catalog image`).tag; const serviceCommitId = shortCommitForRecord(publishRecord, target); catalogService.commitId = serviceCommitId; @@ -402,6 +448,18 @@ function refreshDocuments({ deploy, catalog, target, publishRecords, serviceInve catalogService.digest = publishRecord?.digest ?? "not_published"; catalogService.componentCommitId = publishRecord?.componentCommitId ?? catalogService.componentCommitId ?? null; catalogService.componentInputHash = publishRecord?.componentInputHash ?? catalogService.componentInputHash ?? null; + catalogService.runtimeMode = publishRecord?.runtimeMode ?? catalogService.runtimeMode ?? "service-image"; + catalogService.envReuse = envReuse; + catalogService.envChanged = publishRecord?.envChanged ?? catalogService.envChanged ?? null; + catalogService.codeChanged = publishRecord?.codeChanged ?? catalogService.codeChanged ?? null; + catalogService.environmentImage = publishRecord?.environmentImage ?? (envReuse ? image : catalogService.environmentImage ?? null); + catalogService.environmentDigest = publishRecord?.environmentDigest ?? catalogService.environmentDigest ?? null; + catalogService.environmentInputHash = publishRecord?.environmentInputHash ?? catalogService.environmentInputHash ?? null; + catalogService.codeInputHash = publishRecord?.codeInputHash ?? catalogService.codeInputHash ?? null; + catalogService.bootRepo = publishRecord?.bootRepo ?? catalogService.bootRepo ?? null; + catalogService.bootCommit = publishRecord?.bootCommit ?? target.commitId; + catalogService.bootSh = publishRecord?.bootSh ?? catalogService.bootSh ?? null; + catalogService.bootEnvEvidence = publishRecord?.bootEnvEvidence ?? catalogService.bootEnvEvidence ?? null; catalogService.dockerfileHash = publishRecord?.dockerfileHash ?? catalogService.dockerfileHash ?? null; catalogService.baseImageReference = publishRecord?.baseImageReference ?? catalogService.baseImageReference ?? null; catalogService.baseImageDigest = publishRecord?.baseImageDigest ?? catalogService.baseImageDigest ?? null; @@ -425,6 +483,12 @@ function refreshDocuments({ deploy, catalog, target, publishRecords, serviceInve digest: catalogService.digest, componentCommitId: catalogService.componentCommitId, componentInputHash: catalogService.componentInputHash, + runtimeMode: catalogService.runtimeMode, + environmentDigest: catalogService.environmentDigest ?? null, + environmentInputHash: catalogService.environmentInputHash ?? null, + codeInputHash: catalogService.codeInputHash ?? null, + bootCommit: catalogService.bootCommit ?? null, + bootSh: catalogService.bootSh ?? null, dockerfileHash: catalogService.dockerfileHash, ciAffected: catalogService.ciAffected, ciReason: catalogService.ciReason, diff --git a/scripts/src/g14-ci-plan-lib.mjs b/scripts/src/g14-ci-plan-lib.mjs index 4724123f..18651784 100644 --- a/scripts/src/g14-ci-plan-lib.mjs +++ b/scripts/src/g14-ci-plan-lib.mjs @@ -9,6 +9,7 @@ import { SERVICE_IDS } from "../../internal/protocol/index.mjs"; const execFileAsync = promisify(execFile); export const G14_CI_PLAN_VERSION = "v1"; +export const V02_ENV_REUSE_RUNTIME_MODE = "env-reuse-git-mirror-checkout"; export const DEFAULT_BUILD_SYSTEM_PATHS = Object.freeze([ "scripts/artifact-publish.mjs", @@ -16,6 +17,14 @@ export const DEFAULT_BUILD_SYSTEM_PATHS = Object.freeze([ "scripts/src/dev-artifact-services.mjs" ]); +export const DEFAULT_ENV_REUSE_LAUNCHER_PATHS = Object.freeze([ + "deploy/runtime/launcher/", + "scripts/artifact-publish.mjs", + "scripts/g14-artifact-publish.mjs", + "scripts/src/dev-artifact-services.mjs", + "scripts/src/g14-ci-plan-lib.mjs" +]); + export const DEFAULT_RUNTIME_DEP_PATHS = Object.freeze([ "package.json", "package-lock.json" @@ -73,9 +82,10 @@ const bunCommandServices = new Set([ export async function createG14CiPlan(options = {}) { const repoRoot = options.repoRoot ?? process.cwd(); const deployJsonPath = options.deployJsonPath ?? "deploy/deploy.json"; - const artifactCatalogPath = options.artifactCatalogPath ?? "deploy/artifact-catalog.dev.json"; const targetRef = options.targetRef ?? "HEAD"; const baseRef = options.baseRef ?? await defaultBaseRef(repoRoot, targetRef); + const lane = options.lane ?? (options.artifactCatalogPath?.includes(".v02.") ? "v02" : "g14"); + const artifactCatalogPath = options.artifactCatalogPath ?? (lane === "v02" ? "deploy/artifact-catalog.v02.json" : "deploy/artifact-catalog.dev.json"); const registryPrefix = options.registryPrefix ?? process.env.HWLAB_DEV_REGISTRY_PREFIX ?? process.env.HWLAB_DEV_REGISTRY ?? "127.0.0.1:5000/hwlab"; const baseImage = options.baseImage ?? process.env.HWLAB_DEV_BASE_IMAGE ?? "127.0.0.1:5000/hwlab/hwlab-node20-base:20-bookworm-slim"; @@ -89,6 +99,7 @@ export async function createG14CiPlan(options = {}) { const normalizedChangedPaths = changedPaths.map(normalizeRepoPath).filter(Boolean); const serviceIdResolution = resolveServiceIds({ deployJson, options }); const componentModels = componentModelsForServices(serviceIdResolution.serviceIds); + const envReuseServices = enabledEnvReuseServices(deployJson, lane, serviceIdResolution.serviceIds); const globalChange = classifyGlobalChange(normalizedChangedPaths); const dockerfileHash = await hashGitPaths(repoRoot, targetRef, [ ...DEFAULT_BUILD_SYSTEM_PATHS, @@ -102,6 +113,21 @@ export async function createG14CiPlan(options = {}) { const sharedMatches = matchingPaths(normalizedChangedPaths, model.sharedPaths); const runtimeDepMatches = matchingPaths(normalizedChangedPaths, model.runtimeDeps); const buildSystemMatches = matchingPaths(normalizedChangedPaths, model.buildSystemPaths); + const envReuse = lane === "v02" && envReuseServices.has(model.serviceId); + const bootSh = envReuse ? bootShForService(deployJson, model.serviceId) : null; + const envInputPaths = envReuse ? uniqueSorted([ + ...model.runtimeDeps, + ...DEFAULT_ENV_REUSE_LAUNCHER_PATHS.map(normalizeRepoPath) + ]) : []; + const codeInputPaths = envReuse ? uniqueSorted([ + ...model.componentPaths, + ...model.sharedPaths, + bootSh + ]) : []; + const envMatches = envReuse ? matchingPaths(normalizedChangedPaths, envInputPaths) : []; + const codeMatches = envReuse ? matchingPaths(normalizedChangedPaths, codeInputPaths) : []; + const relevantEnvMatches = envMatches.filter((item) => !isTestOnlyPath(item) && !isDocsOnlyPath(item)); + const relevantCodeMatches = codeMatches.filter((item) => !isTestOnlyPath(item) && !isDocsOnlyPath(item)); const imageRelevantChangedPaths = uniqueSorted([ ...directMatches, ...sharedMatches, @@ -109,8 +135,21 @@ export async function createG14CiPlan(options = {}) { ...buildSystemMatches ].filter((item) => !isTestOnlyPath(item) && !isDocsOnlyPath(item))); const catalogRecord = catalogByServiceId.get(model.serviceId) ?? null; - const catalogReuse = reuseCandidate(catalogRecord, imageRelevantChangedPaths.length > 0); - const affected = imageRelevantChangedPaths.length > 0 || catalogReuse.status === "candidate-no-catalog" || catalogReuse.status === "candidate-unverified-digest"; + const environmentInputHash = envReuse ? await hashGitPaths(repoRoot, targetRef, envInputPaths, { + skipPath: (filePath) => isTestOnlyPath(filePath) || isDocsOnlyPath(filePath) + }) : null; + const codeInputHash = envReuse ? await hashGitPaths(repoRoot, targetRef, codeInputPaths, { + skipPath: (filePath) => isTestOnlyPath(filePath) || isDocsOnlyPath(filePath) + }) : null; + const environmentImage = envReuse ? environmentImageFromCatalog(model.serviceId, catalogRecord) : null; + const environmentDigest = envReuse ? environmentDigestFromCatalog(catalogRecord) : null; + const environmentReady = envReuse && /^sha256:[a-f0-9]{64}$/u.test(environmentDigest ?? "") && Boolean(environmentImage); + const envChanged = envReuse ? relevantEnvMatches.length > 0 || !environmentReady || catalogRecord?.environmentInputHash !== environmentInputHash : null; + const codeChanged = envReuse ? relevantCodeMatches.length > 0 || catalogRecord?.codeInputHash !== codeInputHash || catalogRecord?.bootCommit !== sourceCommitId : null; + const catalogReuse = envReuse ? envReuseCandidate(catalogRecord, envChanged) : reuseCandidate(catalogRecord, imageRelevantChangedPaths.length > 0); + const affected = envReuse + ? Boolean(envChanged || codeChanged) + : imageRelevantChangedPaths.length > 0 || catalogReuse.status === "candidate-no-catalog" || catalogReuse.status === "candidate-unverified-digest"; const componentInputPaths = uniqueSorted([ ...model.componentPaths, ...model.sharedPaths, @@ -144,10 +183,23 @@ export async function createG14CiPlan(options = {}) { buildArgsHash, changedPaths: imageRelevantChangedPaths, affected, + runtimeMode: envReuse ? V02_ENV_REUSE_RUNTIME_MODE : "service-image", + envReuse, + envChanged, + codeChanged, + environmentInputHash, + codeInputHash, + bootRepo: envReuse ? canonicalBootRepo(deployJson) : null, + bootCommit: envReuse ? sourceCommitId : null, + bootSh, + environmentImage, + environmentDigest, + envChangedPaths: relevantEnvMatches, + codeChangedPaths: relevantCodeMatches, reason: affected - ? reasonForService({ directMatches, sharedMatches, runtimeDepMatches, buildSystemMatches, catalogReuse }) + ? reasonForService({ directMatches, sharedMatches, runtimeDepMatches, buildSystemMatches, catalogReuse, envReuse, envChanged, codeChanged }) : reasonForUnchanged(globalChange), - reuse: reuseCandidate(catalogRecord, affected) + reuse: catalogReuse }); } @@ -162,11 +214,13 @@ export async function createG14CiPlan(options = {}) { compatibility: { deployJson: deployJsonPath, artifactCatalog: artifactCatalog ? artifactCatalogPath : null, + lane, ciContractSource: "scripts/g14-gitops-render.mjs", mode: "advisory-read-only", currentRenderCompatible: true, plannerMutatesDeployJson: false, serviceIdSource: serviceIdResolution.source, + v02EnvReuseServiceIds: [...envReuseServices], defaultComponentLazyBuild: true, fullBuildFallback: false }, @@ -205,6 +259,27 @@ export function componentModelsForServices(serviceIds) { }); } +export function enabledEnvReuseServices(deployJson, lane, serviceIds) { + if (lane !== "v02") return new Set(); + const configured = deployJson?.lanes?.v02?.envReuseServices; + const values = Array.isArray(configured) ? configured : ["hwlab-device-pod"]; + const allowed = new Set(serviceIds); + return new Set(values.filter((serviceId) => allowed.has(serviceId))); +} + +export function bootShForService(deployJson, serviceId) { + const configured = deployJson?.lanes?.v02?.bootScripts?.[serviceId]; + const bootSh = normalizeRepoPath(configured || `deploy/runtime/boot/${serviceId}.sh`); + if (!bootSh || bootSh.startsWith("/") || bootSh.split("/").includes("..")) { + throw new Error(`invalid v02 boot script for ${serviceId}: ${configured}`); + } + return bootSh; +} + +function canonicalBootRepo(deployJson) { + return deployJson?.lanes?.v02?.sourceRepo || "git@github.com:pikasTech/HWLAB.git"; +} + export function classifyGlobalChange(changedPaths) { const relevant = changedPaths.filter(Boolean); const testOnly = relevant.length > 0 && relevant.every(isTestOnlyPath); @@ -325,8 +400,10 @@ async function gitValue(repoRoot, args) { return result.stdout.trim(); } -function reasonForService({ directMatches, sharedMatches, runtimeDepMatches, buildSystemMatches, catalogReuse = null }) { +function reasonForService({ directMatches, sharedMatches, runtimeDepMatches, buildSystemMatches, catalogReuse = null, envReuse = false, envChanged = null, codeChanged = null }) { const reasons = []; + if (envReuse && envChanged) reasons.push("environment-input-changed"); + if (envReuse && codeChanged) reasons.push("code-input-changed"); if (directMatches.some((item) => !isTestOnlyPath(item) && !isDocsOnlyPath(item))) reasons.push("component-path-changed"); if (sharedMatches.some((item) => !isTestOnlyPath(item) && !isDocsOnlyPath(item))) reasons.push("shared-runtime-changed"); if (runtimeDepMatches.some((item) => !isTestOnlyPath(item) && !isDocsOnlyPath(item))) reasons.push("runtime-deps-changed"); @@ -358,6 +435,33 @@ function reuseCandidate(catalogRecord, affected) { }; } +function envReuseCandidate(catalogRecord, envChanged) { + if (envChanged) return { status: "not-reused", reason: "environment-affected" }; + if (!catalogRecord) return { status: "candidate-no-catalog", reason: "no-previous-artifact-record" }; + const image = environmentImageFromCatalog(catalogRecord.serviceId, catalogRecord); + const digest = environmentDigestFromCatalog(catalogRecord); + if (!/^sha256:[a-f0-9]{64}$/u.test(digest ?? "")) { + return { status: "candidate-unverified-digest", image, digest }; + } + return { + status: catalogRecord.environmentInputHash ? "ready" : "candidate-without-environment-metadata", + image, + digest, + reusedFrom: catalogRecord.environmentInputHash ?? catalogRecord.commitId ?? catalogRecord.imageTag ?? null + }; +} + +function environmentImageFromCatalog(serviceId, catalogRecord) { + const image = catalogRecord?.environmentImage ?? null; + if (image) return image; + const fallback = catalogRecord?.image ?? null; + return typeof fallback === "string" && fallback.includes(`/${serviceId}-env:`) ? fallback : null; +} + +function environmentDigestFromCatalog(catalogRecord) { + return catalogRecord?.environmentDigest ?? (catalogRecord?.environmentImage ? catalogRecord?.digest : null) ?? null; +} + function runtimeKindForService(serviceId) { if (bunCommandServices.has(serviceId)) return "bun-command"; if (serviceId === "hwlab-cloud-web") return "cloud-web";