From 1657c846976470d798218587e4885c8e0ed79dfc Mon Sep 17 00:00:00 2001 From: Codex Agent Date: Mon, 8 Jun 2026 14:46:20 +0800 Subject: [PATCH] Generalize runtime lane git mirror refs --- scripts/g14-gitops-render.mjs | 204 +++++++++++++++++++----------- scripts/g14-gitops-render.test.ts | 11 +- scripts/src/g14-gitops-lane.ts | 25 ++++ 3 files changed, 162 insertions(+), 78 deletions(-) diff --git a/scripts/g14-gitops-render.mjs b/scripts/g14-gitops-render.mjs index f11566b3..1d53e033 100644 --- a/scripts/g14-gitops-render.mjs +++ b/scripts/g14-gitops-render.mjs @@ -12,6 +12,7 @@ import { defaultEndpointForRuntimeLane, defaultPortForRuntimeLane, isRuntimeLane, + runtimeLaneMirrorSpecs, runtimeLaneConfig, versionNameForRuntimeLane } from "./src/g14-gitops-lane.ts"; @@ -3424,7 +3425,27 @@ function registryManifest() { }; } -function devopsInfraGitMirrorManifest() { +function uniqueStrings(values) { + return Array.from(new Set(values.filter((value) => typeof value === "string" && value.length > 0))); +} + +function devopsInfraGitMirrorManifest(_args, deploy) { + const mirrorSpecs = runtimeLaneMirrorSpecs(deploy); + assert.ok(mirrorSpecs.length > 0, "runtime lane git mirror requires at least one deploy.lanes entry"); + const sourceBranches = uniqueStrings(mirrorSpecs.map((spec) => spec.sourceBranch)); + const gitopsBranches = uniqueStrings(mirrorSpecs.map((spec) => spec.gitopsBranch)); + const mirrorBranches = uniqueStrings([...sourceBranches, ...gitopsBranches]); + const fetchConfigCommands = mirrorBranches + .map((branch) => `git -C "$repo_path" config --add remote.origin.fetch '+refs/heads/${branch}:refs/mirror-stage/heads/${branch}'`) + .join("\n"); + const requiredRefArgs = mirrorBranches.map((branch) => shellSingleQuote(`refs/mirror-stage/heads/${branch}`)).join(" "); + const sourceBranchArgs = sourceBranches.map((branch) => shellSingleQuote(branch)).join(" "); + const gitopsBranchArgs = gitopsBranches.map((branch) => shellSingleQuote(branch)).join(" "); + const mirrorBranchesJson = JSON.stringify(mirrorBranches); + const sourceBranchesJson = JSON.stringify(sourceBranches); + const gitopsBranchesJson = JSON.stringify(gitopsBranches); + const preReceiveAllowedRefs = gitopsBranches.map((branch) => `refs/heads/${branch}`).map(shellSingleQuote).join("|"); + const preReceivePathCase = mirrorSpecs.map((spec) => ` refs/heads/${spec.gitopsBranch})\n case "$path" in\n ${spec.artifactCatalog}|${spec.runtimePath}/*) ;;\n "") ;;\n *) echo "git mirror write rejected: path $path is outside ${spec.lane} GitOps outputs" >&2; exit 64 ;;\n esac\n ;;`).join("\n"); const labels = { "app.kubernetes.io/name": "git-mirror", "app.kubernetes.io/part-of": "devops-infra" @@ -3512,10 +3533,7 @@ git -C "$repo_path" config uploadpack.hideRefs refs/mirror-stage git -C "$repo_path" config uploadpack.allowReachableSHA1InWant true git -C "$repo_path" config uploadpack.allowAnySHA1InWant true git -C "$repo_path" config --unset-all remote.origin.fetch 2>/dev/null || true -git -C "$repo_path" config --add remote.origin.fetch '+refs/heads/v0.2:refs/mirror-stage/heads/v0.2' -git -C "$repo_path" config --add remote.origin.fetch '+refs/heads/v0.2-gitops:refs/mirror-stage/heads/v0.2-gitops' -git -C "$repo_path" config --add remote.origin.fetch '+refs/heads/G14:refs/mirror-stage/heads/G14' -git -C "$repo_path" config --add remote.origin.fetch '+refs/heads/G14-gitops:refs/mirror-stage/heads/G14-gitops' +${fetchConfigCommands} fetch_started_ms=$(now_ms) timeout 180 git -C "$repo_path" fetch --quiet --prune origin emit_timing fetch succeeded "$fetch_started_ms" @@ -3527,7 +3545,7 @@ if [ ! -s /tmp/hwlab-stage-heads ]; then echo "git mirror sync fetched no branch refs" >&2 exit 41 fi -for required_ref in refs/mirror-stage/heads/v0.2 refs/mirror-stage/heads/v0.2-gitops refs/mirror-stage/heads/G14 refs/mirror-stage/heads/G14-gitops; do +for required_ref in ${requiredRefArgs}; do git -C "$repo_path" show-ref --verify --quiet "$required_ref" || { echo "git mirror sync missing required ref $required_ref" >&2; exit 43; } done validate_started_ms=$(now_ms) @@ -3545,15 +3563,19 @@ publish_started_ms=$(now_ms) while read -r sha ref; do [ -n "$sha" ] || continue name=$(printf '%s\n' "$ref" | sed 's#^refs/mirror-stage/heads/##') - if [ "$name" = "v0.2-gitops" ]; then - local_sha=$(git -C "$repo_path" rev-parse refs/heads/v0.2-gitops 2>/dev/null || true) + is_gitops_branch=false + for gitops_branch in ${gitopsBranchArgs}; do + if [ "$name" = "$gitops_branch" ]; then is_gitops_branch=true; break; fi + done + if [ "$is_gitops_branch" = true ]; then + local_sha=$(git -C "$repo_path" rev-parse "refs/heads/$name" 2>/dev/null || true) if [ -n "$local_sha" ] && [ "$local_sha" != "$sha" ]; then if git -C "$repo_path" merge-base --is-ancestor "$sha" "$local_sha"; then - echo "git mirror sync keeps local v0.2-gitops ahead of GitHub" + echo "git mirror sync keeps local $name ahead of GitHub" continue fi if ! git -C "$repo_path" merge-base --is-ancestor "$local_sha" "$sha"; then - echo "git mirror sync found divergent v0.2-gitops local=$local_sha github=$sha" >&2 + echo "git mirror sync found divergent $name local=$local_sha github=$sha" >&2 exit 44 fi fi @@ -3582,46 +3604,51 @@ while read -r ref; do fi done < /tmp/hwlab-public-tags emit_timing publish succeeded "$publish_started_ms" -if git -C "$repo_path" show-ref --verify --quiet refs/heads/v0.2; then - git -C "$repo_path" symbolic-ref HEAD refs/heads/v0.2 -elif git -C "$repo_path" show-ref --verify --quiet refs/heads/G14; then - git -C "$repo_path" symbolic-ref HEAD refs/heads/G14 -fi +for head_branch in ${sourceBranchArgs}; do + if git -C "$repo_path" show-ref --verify --quiet "refs/heads/$head_branch"; then + git -C "$repo_path" symbolic-ref HEAD "refs/heads/$head_branch" + break + fi +done fsck_started_ms=$(now_ms) git -C "$repo_path" fsck --connectivity-only --no-dangling >/tmp/hwlab-git-fsck.out emit_timing fsck succeeded "$fsck_started_ms" git -C "$repo_path" update-server-info -v02_head=$(git -C "$repo_path" rev-parse refs/heads/v0.2 2>/dev/null || true) -v02_github_head=$(git -C "$repo_path" rev-parse refs/mirror-stage/heads/v0.2 2>/dev/null || true) -g14_head=$(git -C "$repo_path" rev-parse refs/heads/G14 2>/dev/null || true) -g14_github_head=$(git -C "$repo_path" rev-parse refs/mirror-stage/heads/G14 2>/dev/null || true) -v02_gitops_head=$(git -C "$repo_path" rev-parse refs/heads/v0.2-gitops 2>/dev/null || true) -v02_gitops_github_head=$(git -C "$repo_path" rev-parse refs/mirror-stage/heads/v0.2-gitops 2>/dev/null || true) published_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) -printf '%s\n' "$v02_head" | tee /cache/HWLAB-v0.2.head +for source_branch in ${sourceBranchArgs}; do + git -C "$repo_path" rev-parse "refs/heads/$source_branch" 2>/dev/null | tee "/cache/HWLAB-$source_branch.head" >/dev/null || true +done printf '%s\n' "$published_at" > /cache/HWLAB.last-sync -export v02_head v02_github_head g14_head g14_github_head v02_gitops_head v02_gitops_github_head published_at +export published_at +export mirror_branches_json=${shellSingleQuote(mirrorBranchesJson)} +export source_branches_json=${shellSingleQuote(sourceBranchesJson)} +export gitops_branches_json=${shellSingleQuote(gitopsBranchesJson)} node - <<'NODE' | tee /cache/HWLAB.last-sync.json -const v02Head = process.env.v02_head || null; -const v02GithubHead = process.env.v02_github_head || null; -const gitopsHead = process.env.v02_gitops_head || null; -const gitopsGithubHead = process.env.v02_gitops_github_head || null; +const { execFileSync } = require("node:child_process"); +const repo = "/cache/pikasTech/HWLAB.git"; +const mirrorBranches = JSON.parse(process.env.mirror_branches_json || "[]"); +const sourceBranches = JSON.parse(process.env.source_branches_json || "[]"); +const gitopsBranches = JSON.parse(process.env.gitops_branches_json || "[]"); +function rev(ref) { + try { return execFileSync("git", ["-C", repo, "rev-parse", ref], { encoding: "utf8" }).trim() || null; } catch { return null; } +} +const refs = {}; +for (const branch of mirrorBranches) { + refs["refs/heads/" + branch] = rev("refs/heads/" + branch); + refs["refs/mirror-stage/heads/" + branch] = rev("refs/mirror-stage/heads/" + branch); +} +const sourceInSync = sourceBranches.every((branch) => refs["refs/heads/" + branch] && refs["refs/heads/" + branch] === refs["refs/mirror-stage/heads/" + branch]); +const gitopsInSync = gitopsBranches.every((branch) => refs["refs/heads/" + branch] && refs["refs/heads/" + branch] === refs["refs/mirror-stage/heads/" + branch]); +const pendingFlush = gitopsBranches.some((branch) => refs["refs/heads/" + branch] && refs["refs/mirror-stage/heads/" + branch] && refs["refs/heads/" + branch] !== refs["refs/mirror-stage/heads/" + branch]); const payload = { event: "git-mirror-sync", status: "published", repository: "pikasTech/HWLAB", publishedAt: process.env.published_at, - pendingFlush: Boolean(gitopsHead && gitopsGithubHead && gitopsHead !== gitopsGithubHead), - sourceInSync: Boolean(v02Head && v02GithubHead && v02Head === v02GithubHead), - gitopsInSync: Boolean(gitopsHead && gitopsGithubHead && gitopsHead === gitopsGithubHead), - refs: { - "refs/heads/v0.2": v02Head, - "refs/mirror-stage/heads/v0.2": v02GithubHead, - "refs/heads/G14": process.env.g14_head || null, - "refs/mirror-stage/heads/G14": process.env.g14_github_head || null, - "refs/heads/v0.2-gitops": gitopsHead, - "refs/mirror-stage/heads/v0.2-gitops": gitopsGithubHead - } + pendingFlush, + sourceInSync, + gitopsInSync, + refs }; console.log(JSON.stringify(payload)); NODE @@ -3637,11 +3664,14 @@ set -eu zero="0000000000000000000000000000000000000000" status=0 while read -r old new ref; do - if [ "$ref" != "refs/heads/v0.2-gitops" ]; then + case "$ref" in + ${preReceiveAllowedRefs}) ;; + *) echo "git mirror write rejected: ref $ref is not allowlisted" >&2 status=1 continue - fi + ;; + esac if [ "$new" = "$zero" ]; then echo "git mirror write rejected: deleting $ref is forbidden" >&2 status=1 @@ -3664,10 +3694,9 @@ while read -r old new ref; do else git diff --name-only "$old" "$new" fi | while read -r path; do - case "$path" in - deploy/artifact-catalog.v02.json|deploy/gitops/g14/runtime-v02/*) ;; - "") ;; - *) echo "git mirror write rejected: path $path is outside v0.2 GitOps outputs" >&2; exit 64 ;; + case "$ref" in +${preReceivePathCase} + *) echo "git mirror write rejected: ref $ref has no path allowlist" >&2; exit 64 ;; esac done || status=1 done @@ -3678,23 +3707,29 @@ cat > "$repo_path/hooks/post-receive" <<'HOOK' set -eu repo_path=$(git rev-parse --git-dir) git -C "$repo_path" update-server-info -local_head=$(git -C "$repo_path" rev-parse refs/heads/v0.2-gitops 2>/dev/null || true) -github_head=$(git -C "$repo_path" rev-parse refs/mirror-stage/heads/v0.2-gitops 2>/dev/null || true) written_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) -export local_head github_head written_at +export repo_path written_at +export gitops_branches_json=${shellSingleQuote(gitopsBranchesJson)} node - <<'NODE' > /cache/HWLAB.last-write.json -const localHead = process.env.local_head || null; -const githubHead = process.env.github_head || null; +const { execFileSync } = require("node:child_process"); +const repo = process.env.repo_path || "/cache/pikasTech/HWLAB.git"; +const gitopsBranches = JSON.parse(process.env.gitops_branches_json || "[]"); +function rev(ref) { + try { return execFileSync("git", ["-C", repo, "rev-parse", ref], { encoding: "utf8" }).trim() || null; } catch { return null; } +} +const refs = {}; +for (const branch of gitopsBranches) { + refs["refs/heads/" + branch] = rev("refs/heads/" + branch); + refs["refs/mirror-stage/heads/" + branch] = rev("refs/mirror-stage/heads/" + branch); +} +const pendingFlush = gitopsBranches.some((branch) => refs["refs/heads/" + branch] && refs["refs/heads/" + branch] !== refs["refs/mirror-stage/heads/" + branch]); console.log(JSON.stringify({ event: "git-mirror-write", status: "received", repository: "pikasTech/HWLAB", writtenAt: process.env.written_at, - pendingFlush: Boolean(localHead && localHead !== githubHead), - refs: { - "refs/heads/v0.2-gitops": localHead, - "refs/mirror-stage/heads/v0.2-gitops": githubHead - } + pendingFlush, + refs })); NODE HOOK @@ -3759,34 +3794,53 @@ NODE_PROXY chmod 0700 /tmp/hwlab-github-proxy-connect.mjs export GIT_SSH_COMMAND="ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=10 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ProxyCommand='node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808 %h %p'" /script/install-hooks.sh -local_head=$(git -C "$repo_path" rev-parse refs/heads/v0.2-gitops 2>/dev/null || true) -if [ -z "$local_head" ]; then - echo "git mirror flush missing local refs/heads/v0.2-gitops" >&2 - exit 51 -fi -fetch_started=$(now_ms) -timeout 90 git -C "$repo_path" fetch --quiet "$repo_url" refs/heads/v0.2-gitops:refs/mirror-stage/heads/v0.2-gitops -emit_timing fetch succeeded "$fetch_started" -github_head=$(git -C "$repo_path" rev-parse refs/mirror-stage/heads/v0.2-gitops 2>/dev/null || true) -if [ -n "$github_head" ] && [ "$github_head" != "$local_head" ] && ! git -C "$repo_path" merge-base --is-ancestor "$github_head" "$local_head"; then - echo "git mirror flush rejected divergent v0.2-gitops local=$local_head github=$github_head" >&2 - exit 52 -fi -push_started=$(now_ms) -timeout 120 git -C "$repo_path" push "$repo_url" "$local_head:refs/heads/v0.2-gitops" -emit_timing push succeeded "$push_started" -git -C "$repo_path" update-ref refs/mirror-stage/heads/v0.2-gitops "$local_head" +for gitops_branch in ${gitopsBranchArgs}; do + local_head=$(git -C "$repo_path" rev-parse "refs/heads/$gitops_branch" 2>/dev/null || true) + if [ -z "$local_head" ]; then + echo "git mirror flush skips missing local refs/heads/$gitops_branch" + continue + fi + fetch_started=$(now_ms) + timeout 90 git -C "$repo_path" fetch --quiet "$repo_url" "refs/heads/$gitops_branch:refs/mirror-stage/heads/$gitops_branch" + emit_timing "fetch-$gitops_branch" succeeded "$fetch_started" + github_head=$(git -C "$repo_path" rev-parse "refs/mirror-stage/heads/$gitops_branch" 2>/dev/null || true) + if [ -n "$github_head" ] && [ "$github_head" != "$local_head" ] && ! git -C "$repo_path" merge-base --is-ancestor "$github_head" "$local_head"; then + echo "git mirror flush rejected divergent $gitops_branch local=$local_head github=$github_head" >&2 + exit 52 + fi + if [ "$local_head" != "$github_head" ]; then + push_started=$(now_ms) + timeout 120 git -C "$repo_path" push "$repo_url" "$local_head:refs/heads/$gitops_branch" + emit_timing "push-$gitops_branch" succeeded "$push_started" + else + echo "git mirror flush $gitops_branch already matches GitHub" + fi + git -C "$repo_path" update-ref "refs/mirror-stage/heads/$gitops_branch" "$local_head" +done git -C "$repo_path" update-server-info flushed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) -export local_head flushed_at +export flushed_at +export gitops_branches_json=${shellSingleQuote(gitopsBranchesJson)} node - <<'NODE' | tee /cache/HWLAB.last-flush.json +const { execFileSync } = require("node:child_process"); +const repo = "/cache/pikasTech/HWLAB.git"; +const gitopsBranches = JSON.parse(process.env.gitops_branches_json || "[]"); +function rev(ref) { + try { return execFileSync("git", ["-C", repo, "rev-parse", ref], { encoding: "utf8" }).trim() || null; } catch { return null; } +} +const refs = {}; +for (const branch of gitopsBranches) { + refs["refs/heads/" + branch] = rev("refs/heads/" + branch); + refs["refs/mirror-stage/heads/" + branch] = rev("refs/mirror-stage/heads/" + branch); +} +const pendingFlush = gitopsBranches.some((branch) => refs["refs/heads/" + branch] && refs["refs/mirror-stage/heads/" + branch] && refs["refs/heads/" + branch] !== refs["refs/mirror-stage/heads/" + branch]); console.log(JSON.stringify({ event: "git-mirror-flush", status: "flushed", repository: "pikasTech/HWLAB", flushedAt: process.env.flushed_at, - pendingFlush: false, - refs: { "refs/heads/v0.2-gitops": process.env.local_head || null } + pendingFlush, + refs })); NODE emit_timing total succeeded "$started_ms" @@ -4913,7 +4967,7 @@ async function plannedFiles(args) { putJson("source.json", sourceDescriptor); putText("README.md", readme({ source, args })); putJson("registry/registry.yaml", registryManifest()); - if (isRuntimeLane(args.lane)) putJson("devops-infra/git-mirror.yaml", devopsInfraGitMirrorManifest(args)); + if (isRuntimeLane(args.lane)) putJson("devops-infra/git-mirror.yaml", devopsInfraGitMirrorManifest(args, deploy)); putJson(`${settings.tektonDir}/rbac.yaml`, tektonRbac(args)); putJson(`${settings.tektonDir}/pipeline.yaml`, tektonPipeline(args, deploy)); if (!isRuntimeLane(args.lane)) { diff --git a/scripts/g14-gitops-render.test.ts b/scripts/g14-gitops-render.test.ts index d7b9a5a6..7e174c3a 100644 --- a/scripts/g14-gitops-render.test.ts +++ b/scripts/g14-gitops-render.test.ts @@ -216,8 +216,10 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots assert.match(gitMirrorScript, /uploadpack\.allowAnySHA1InWant true/u); assert.match(gitMirrorScript, /refs\/heads\/v0\.2:refs\/mirror-stage\/heads\/v0\.2/u); assert.match(gitMirrorScript, /refs\/heads\/v0\.2-gitops:refs\/mirror-stage\/heads\/v0\.2-gitops/u); - assert.match(gitMirrorScript, /refs\/heads\/G14:refs\/mirror-stage\/heads\/G14/u); - assert.match(gitMirrorScript, /refs\/heads\/G14-gitops:refs\/mirror-stage\/heads\/G14-gitops/u); + assert.match(gitMirrorScript, /refs\/heads\/v0\.3:refs\/mirror-stage\/heads\/v0\.3/u); + assert.match(gitMirrorScript, /refs\/heads\/v0\.3-gitops:refs\/mirror-stage\/heads\/v0\.3-gitops/u); + assert.doesNotMatch(gitMirrorScript, /refs\/heads\/G14:refs\/mirror-stage\/heads\/G14/u); + assert.doesNotMatch(gitMirrorScript, /refs\/heads\/G14-gitops:refs\/mirror-stage\/heads\/G14-gitops/u); assert.doesNotMatch(gitMirrorScript, /refs\/heads\/\*:refs\/mirror-stage\/heads\/\*/u); assert.match(gitMirrorScript, /git -C "\$repo_path" update-ref "refs\/heads\/\$name" "\$sha"/u); assert.match(gitMirrorScript, /git-mirror-sync-timing/u); @@ -225,7 +227,10 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots assert.match(gitMirrorScript, /emit_timing total succeeded/u); assert.match(gitMirrorScript, /sourceInSync/u); assert.match(gitMirrorScript, /refs\/mirror-stage\/heads\/v0\.2/u); - assert.match(gitMirrorScript, /keeps local v0\.2-gitops ahead of GitHub/u); + assert.match(gitMirrorScript, /refs\/mirror-stage\/heads\/v0\.3/u); + assert.match(gitMirrorScript, /for gitops_branch in 'v0\.2-gitops' 'v0\.3-gitops'/u); + assert.match(gitMirrorScript, /keeps local \$name ahead of GitHub/u); + assert.match(gitMirror, /deploy\/artifact-catalog\.v03\.json\|deploy\/gitops\/g14\/runtime-v03\/\*/u); assert.ok((gitMirrorJson.items || []).every((item) => item.kind !== "Secret")); assert.ok((gitMirrorJson.items || []).every((item) => item.kind !== "CronJob")); assert.ok((gitMirrorJson.items || []).every((item) => !item.data?.["ssh-privatekey"])); diff --git a/scripts/src/g14-gitops-lane.ts b/scripts/src/g14-gitops-lane.ts index e40865da..b4098089 100644 --- a/scripts/src/g14-gitops-lane.ts +++ b/scripts/src/g14-gitops-lane.ts @@ -34,6 +34,14 @@ export type RuntimeLaneConfig = { sourceRepo?: string; }; +export type RuntimeLaneMirrorSpec = { + lane: string; + sourceBranch: string; + gitopsBranch: string; + artifactCatalog: string; + runtimePath: string; +}; + export type DeployConfig = { lanes?: Record; }; @@ -111,6 +119,23 @@ export function applyRuntimeLaneDeployConfig(args: GitOpsRenderArgs, deploy: Dep return result; } +export function runtimeLaneMirrorSpecs(deploy: DeployConfig): RuntimeLaneMirrorSpec[] { + return Object.entries(deploy.lanes ?? {}) + .filter(([lane, config]) => isRuntimeLane(lane) && isObject(config)) + .sort(([left], [right]) => runtimeLaneMinor(left) - runtimeLaneMinor(right)) + .map(([lane, config]) => { + const laneConfig = config as RuntimeLaneConfig; + const versionName = versionNameForRuntimeLane(lane); + return { + lane, + sourceBranch: laneConfig.sourceBranch ?? versionName, + gitopsBranch: laneConfig.gitopsBranch ?? `${versionName}-gitops`, + artifactCatalog: laneConfig.artifactCatalog ?? `deploy/artifact-catalog.${lane}.json`, + runtimePath: `deploy/gitops/g14/${runtimePathForRuntimeLane(lane)}`, + }; + }); +} + function isObject(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); }