Generalize runtime lane git mirror refs

This commit is contained in:
Codex Agent
2026-06-08 14:46:20 +08:00
parent 6c5166d816
commit 1657c84697
3 changed files with 162 additions and 78 deletions
+129 -75
View File
@@ -12,6 +12,7 @@ import {
defaultEndpointForRuntimeLane,
defaultPortForRuntimeLane,
isRuntimeLane,
runtimeLaneMirrorSpecs,
runtimeLaneConfig,
versionNameForRuntimeLane
} from "./src/g14-gitops-lane.ts";
@@ -3424,7 +3425,27 @@ function registryManifest() {
};
}
function devopsInfraGitMirrorManifest() {
function uniqueStrings(values) {
return Array.from(new Set(values.filter((value) => typeof value === "string" && value.length > 0)));
}
function devopsInfraGitMirrorManifest(_args, deploy) {
const mirrorSpecs = runtimeLaneMirrorSpecs(deploy);
assert.ok(mirrorSpecs.length > 0, "runtime lane git mirror requires at least one deploy.lanes entry");
const sourceBranches = uniqueStrings(mirrorSpecs.map((spec) => spec.sourceBranch));
const gitopsBranches = uniqueStrings(mirrorSpecs.map((spec) => spec.gitopsBranch));
const mirrorBranches = uniqueStrings([...sourceBranches, ...gitopsBranches]);
const fetchConfigCommands = mirrorBranches
.map((branch) => `git -C "$repo_path" config --add remote.origin.fetch '+refs/heads/${branch}:refs/mirror-stage/heads/${branch}'`)
.join("\n");
const requiredRefArgs = mirrorBranches.map((branch) => shellSingleQuote(`refs/mirror-stage/heads/${branch}`)).join(" ");
const sourceBranchArgs = sourceBranches.map((branch) => shellSingleQuote(branch)).join(" ");
const gitopsBranchArgs = gitopsBranches.map((branch) => shellSingleQuote(branch)).join(" ");
const mirrorBranchesJson = JSON.stringify(mirrorBranches);
const sourceBranchesJson = JSON.stringify(sourceBranches);
const gitopsBranchesJson = JSON.stringify(gitopsBranches);
const preReceiveAllowedRefs = gitopsBranches.map((branch) => `refs/heads/${branch}`).map(shellSingleQuote).join("|");
const preReceivePathCase = mirrorSpecs.map((spec) => ` refs/heads/${spec.gitopsBranch})\n case "$path" in\n ${spec.artifactCatalog}|${spec.runtimePath}/*) ;;\n "") ;;\n *) echo "git mirror write rejected: path $path is outside ${spec.lane} GitOps outputs" >&2; exit 64 ;;\n esac\n ;;`).join("\n");
const labels = {
"app.kubernetes.io/name": "git-mirror",
"app.kubernetes.io/part-of": "devops-infra"
@@ -3512,10 +3533,7 @@ git -C "$repo_path" config uploadpack.hideRefs refs/mirror-stage
git -C "$repo_path" config uploadpack.allowReachableSHA1InWant true
git -C "$repo_path" config uploadpack.allowAnySHA1InWant true
git -C "$repo_path" config --unset-all remote.origin.fetch 2>/dev/null || true
git -C "$repo_path" config --add remote.origin.fetch '+refs/heads/v0.2:refs/mirror-stage/heads/v0.2'
git -C "$repo_path" config --add remote.origin.fetch '+refs/heads/v0.2-gitops:refs/mirror-stage/heads/v0.2-gitops'
git -C "$repo_path" config --add remote.origin.fetch '+refs/heads/G14:refs/mirror-stage/heads/G14'
git -C "$repo_path" config --add remote.origin.fetch '+refs/heads/G14-gitops:refs/mirror-stage/heads/G14-gitops'
${fetchConfigCommands}
fetch_started_ms=$(now_ms)
timeout 180 git -C "$repo_path" fetch --quiet --prune origin
emit_timing fetch succeeded "$fetch_started_ms"
@@ -3527,7 +3545,7 @@ if [ ! -s /tmp/hwlab-stage-heads ]; then
echo "git mirror sync fetched no branch refs" >&2
exit 41
fi
for required_ref in refs/mirror-stage/heads/v0.2 refs/mirror-stage/heads/v0.2-gitops refs/mirror-stage/heads/G14 refs/mirror-stage/heads/G14-gitops; do
for required_ref in ${requiredRefArgs}; do
git -C "$repo_path" show-ref --verify --quiet "$required_ref" || { echo "git mirror sync missing required ref $required_ref" >&2; exit 43; }
done
validate_started_ms=$(now_ms)
@@ -3545,15 +3563,19 @@ publish_started_ms=$(now_ms)
while read -r sha ref; do
[ -n "$sha" ] || continue
name=$(printf '%s\n' "$ref" | sed 's#^refs/mirror-stage/heads/##')
if [ "$name" = "v0.2-gitops" ]; then
local_sha=$(git -C "$repo_path" rev-parse refs/heads/v0.2-gitops 2>/dev/null || true)
is_gitops_branch=false
for gitops_branch in ${gitopsBranchArgs}; do
if [ "$name" = "$gitops_branch" ]; then is_gitops_branch=true; break; fi
done
if [ "$is_gitops_branch" = true ]; then
local_sha=$(git -C "$repo_path" rev-parse "refs/heads/$name" 2>/dev/null || true)
if [ -n "$local_sha" ] && [ "$local_sha" != "$sha" ]; then
if git -C "$repo_path" merge-base --is-ancestor "$sha" "$local_sha"; then
echo "git mirror sync keeps local v0.2-gitops ahead of GitHub"
echo "git mirror sync keeps local $name ahead of GitHub"
continue
fi
if ! git -C "$repo_path" merge-base --is-ancestor "$local_sha" "$sha"; then
echo "git mirror sync found divergent v0.2-gitops local=$local_sha github=$sha" >&2
echo "git mirror sync found divergent $name local=$local_sha github=$sha" >&2
exit 44
fi
fi
@@ -3582,46 +3604,51 @@ while read -r ref; do
fi
done < /tmp/hwlab-public-tags
emit_timing publish succeeded "$publish_started_ms"
if git -C "$repo_path" show-ref --verify --quiet refs/heads/v0.2; then
git -C "$repo_path" symbolic-ref HEAD refs/heads/v0.2
elif git -C "$repo_path" show-ref --verify --quiet refs/heads/G14; then
git -C "$repo_path" symbolic-ref HEAD refs/heads/G14
fi
for head_branch in ${sourceBranchArgs}; do
if git -C "$repo_path" show-ref --verify --quiet "refs/heads/$head_branch"; then
git -C "$repo_path" symbolic-ref HEAD "refs/heads/$head_branch"
break
fi
done
fsck_started_ms=$(now_ms)
git -C "$repo_path" fsck --connectivity-only --no-dangling >/tmp/hwlab-git-fsck.out
emit_timing fsck succeeded "$fsck_started_ms"
git -C "$repo_path" update-server-info
v02_head=$(git -C "$repo_path" rev-parse refs/heads/v0.2 2>/dev/null || true)
v02_github_head=$(git -C "$repo_path" rev-parse refs/mirror-stage/heads/v0.2 2>/dev/null || true)
g14_head=$(git -C "$repo_path" rev-parse refs/heads/G14 2>/dev/null || true)
g14_github_head=$(git -C "$repo_path" rev-parse refs/mirror-stage/heads/G14 2>/dev/null || true)
v02_gitops_head=$(git -C "$repo_path" rev-parse refs/heads/v0.2-gitops 2>/dev/null || true)
v02_gitops_github_head=$(git -C "$repo_path" rev-parse refs/mirror-stage/heads/v0.2-gitops 2>/dev/null || true)
published_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
printf '%s\n' "$v02_head" | tee /cache/HWLAB-v0.2.head
for source_branch in ${sourceBranchArgs}; do
git -C "$repo_path" rev-parse "refs/heads/$source_branch" 2>/dev/null | tee "/cache/HWLAB-$source_branch.head" >/dev/null || true
done
printf '%s\n' "$published_at" > /cache/HWLAB.last-sync
export v02_head v02_github_head g14_head g14_github_head v02_gitops_head v02_gitops_github_head published_at
export published_at
export mirror_branches_json=${shellSingleQuote(mirrorBranchesJson)}
export source_branches_json=${shellSingleQuote(sourceBranchesJson)}
export gitops_branches_json=${shellSingleQuote(gitopsBranchesJson)}
node - <<'NODE' | tee /cache/HWLAB.last-sync.json
const v02Head = process.env.v02_head || null;
const v02GithubHead = process.env.v02_github_head || null;
const gitopsHead = process.env.v02_gitops_head || null;
const gitopsGithubHead = process.env.v02_gitops_github_head || null;
const { execFileSync } = require("node:child_process");
const repo = "/cache/pikasTech/HWLAB.git";
const mirrorBranches = JSON.parse(process.env.mirror_branches_json || "[]");
const sourceBranches = JSON.parse(process.env.source_branches_json || "[]");
const gitopsBranches = JSON.parse(process.env.gitops_branches_json || "[]");
function rev(ref) {
try { return execFileSync("git", ["-C", repo, "rev-parse", ref], { encoding: "utf8" }).trim() || null; } catch { return null; }
}
const refs = {};
for (const branch of mirrorBranches) {
refs["refs/heads/" + branch] = rev("refs/heads/" + branch);
refs["refs/mirror-stage/heads/" + branch] = rev("refs/mirror-stage/heads/" + branch);
}
const sourceInSync = sourceBranches.every((branch) => refs["refs/heads/" + branch] && refs["refs/heads/" + branch] === refs["refs/mirror-stage/heads/" + branch]);
const gitopsInSync = gitopsBranches.every((branch) => refs["refs/heads/" + branch] && refs["refs/heads/" + branch] === refs["refs/mirror-stage/heads/" + branch]);
const pendingFlush = gitopsBranches.some((branch) => refs["refs/heads/" + branch] && refs["refs/mirror-stage/heads/" + branch] && refs["refs/heads/" + branch] !== refs["refs/mirror-stage/heads/" + branch]);
const payload = {
event: "git-mirror-sync",
status: "published",
repository: "pikasTech/HWLAB",
publishedAt: process.env.published_at,
pendingFlush: Boolean(gitopsHead && gitopsGithubHead && gitopsHead !== gitopsGithubHead),
sourceInSync: Boolean(v02Head && v02GithubHead && v02Head === v02GithubHead),
gitopsInSync: Boolean(gitopsHead && gitopsGithubHead && gitopsHead === gitopsGithubHead),
refs: {
"refs/heads/v0.2": v02Head,
"refs/mirror-stage/heads/v0.2": v02GithubHead,
"refs/heads/G14": process.env.g14_head || null,
"refs/mirror-stage/heads/G14": process.env.g14_github_head || null,
"refs/heads/v0.2-gitops": gitopsHead,
"refs/mirror-stage/heads/v0.2-gitops": gitopsGithubHead
}
pendingFlush,
sourceInSync,
gitopsInSync,
refs
};
console.log(JSON.stringify(payload));
NODE
@@ -3637,11 +3664,14 @@ set -eu
zero="0000000000000000000000000000000000000000"
status=0
while read -r old new ref; do
if [ "$ref" != "refs/heads/v0.2-gitops" ]; then
case "$ref" in
${preReceiveAllowedRefs}) ;;
*)
echo "git mirror write rejected: ref $ref is not allowlisted" >&2
status=1
continue
fi
;;
esac
if [ "$new" = "$zero" ]; then
echo "git mirror write rejected: deleting $ref is forbidden" >&2
status=1
@@ -3664,10 +3694,9 @@ while read -r old new ref; do
else
git diff --name-only "$old" "$new"
fi | while read -r path; do
case "$path" in
deploy/artifact-catalog.v02.json|deploy/gitops/g14/runtime-v02/*) ;;
"") ;;
*) echo "git mirror write rejected: path $path is outside v0.2 GitOps outputs" >&2; exit 64 ;;
case "$ref" in
${preReceivePathCase}
*) echo "git mirror write rejected: ref $ref has no path allowlist" >&2; exit 64 ;;
esac
done || status=1
done
@@ -3678,23 +3707,29 @@ cat > "$repo_path/hooks/post-receive" <<'HOOK'
set -eu
repo_path=$(git rev-parse --git-dir)
git -C "$repo_path" update-server-info
local_head=$(git -C "$repo_path" rev-parse refs/heads/v0.2-gitops 2>/dev/null || true)
github_head=$(git -C "$repo_path" rev-parse refs/mirror-stage/heads/v0.2-gitops 2>/dev/null || true)
written_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
export local_head github_head written_at
export repo_path written_at
export gitops_branches_json=${shellSingleQuote(gitopsBranchesJson)}
node - <<'NODE' > /cache/HWLAB.last-write.json
const localHead = process.env.local_head || null;
const githubHead = process.env.github_head || null;
const { execFileSync } = require("node:child_process");
const repo = process.env.repo_path || "/cache/pikasTech/HWLAB.git";
const gitopsBranches = JSON.parse(process.env.gitops_branches_json || "[]");
function rev(ref) {
try { return execFileSync("git", ["-C", repo, "rev-parse", ref], { encoding: "utf8" }).trim() || null; } catch { return null; }
}
const refs = {};
for (const branch of gitopsBranches) {
refs["refs/heads/" + branch] = rev("refs/heads/" + branch);
refs["refs/mirror-stage/heads/" + branch] = rev("refs/mirror-stage/heads/" + branch);
}
const pendingFlush = gitopsBranches.some((branch) => refs["refs/heads/" + branch] && refs["refs/heads/" + branch] !== refs["refs/mirror-stage/heads/" + branch]);
console.log(JSON.stringify({
event: "git-mirror-write",
status: "received",
repository: "pikasTech/HWLAB",
writtenAt: process.env.written_at,
pendingFlush: Boolean(localHead && localHead !== githubHead),
refs: {
"refs/heads/v0.2-gitops": localHead,
"refs/mirror-stage/heads/v0.2-gitops": githubHead
}
pendingFlush,
refs
}));
NODE
HOOK
@@ -3759,34 +3794,53 @@ NODE_PROXY
chmod 0700 /tmp/hwlab-github-proxy-connect.mjs
export GIT_SSH_COMMAND="ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=10 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ProxyCommand='node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808 %h %p'"
/script/install-hooks.sh
local_head=$(git -C "$repo_path" rev-parse refs/heads/v0.2-gitops 2>/dev/null || true)
if [ -z "$local_head" ]; then
echo "git mirror flush missing local refs/heads/v0.2-gitops" >&2
exit 51
fi
fetch_started=$(now_ms)
timeout 90 git -C "$repo_path" fetch --quiet "$repo_url" refs/heads/v0.2-gitops:refs/mirror-stage/heads/v0.2-gitops
emit_timing fetch succeeded "$fetch_started"
github_head=$(git -C "$repo_path" rev-parse refs/mirror-stage/heads/v0.2-gitops 2>/dev/null || true)
if [ -n "$github_head" ] && [ "$github_head" != "$local_head" ] && ! git -C "$repo_path" merge-base --is-ancestor "$github_head" "$local_head"; then
echo "git mirror flush rejected divergent v0.2-gitops local=$local_head github=$github_head" >&2
exit 52
fi
push_started=$(now_ms)
timeout 120 git -C "$repo_path" push "$repo_url" "$local_head:refs/heads/v0.2-gitops"
emit_timing push succeeded "$push_started"
git -C "$repo_path" update-ref refs/mirror-stage/heads/v0.2-gitops "$local_head"
for gitops_branch in ${gitopsBranchArgs}; do
local_head=$(git -C "$repo_path" rev-parse "refs/heads/$gitops_branch" 2>/dev/null || true)
if [ -z "$local_head" ]; then
echo "git mirror flush skips missing local refs/heads/$gitops_branch"
continue
fi
fetch_started=$(now_ms)
timeout 90 git -C "$repo_path" fetch --quiet "$repo_url" "refs/heads/$gitops_branch:refs/mirror-stage/heads/$gitops_branch"
emit_timing "fetch-$gitops_branch" succeeded "$fetch_started"
github_head=$(git -C "$repo_path" rev-parse "refs/mirror-stage/heads/$gitops_branch" 2>/dev/null || true)
if [ -n "$github_head" ] && [ "$github_head" != "$local_head" ] && ! git -C "$repo_path" merge-base --is-ancestor "$github_head" "$local_head"; then
echo "git mirror flush rejected divergent $gitops_branch local=$local_head github=$github_head" >&2
exit 52
fi
if [ "$local_head" != "$github_head" ]; then
push_started=$(now_ms)
timeout 120 git -C "$repo_path" push "$repo_url" "$local_head:refs/heads/$gitops_branch"
emit_timing "push-$gitops_branch" succeeded "$push_started"
else
echo "git mirror flush $gitops_branch already matches GitHub"
fi
git -C "$repo_path" update-ref "refs/mirror-stage/heads/$gitops_branch" "$local_head"
done
git -C "$repo_path" update-server-info
flushed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
export local_head flushed_at
export flushed_at
export gitops_branches_json=${shellSingleQuote(gitopsBranchesJson)}
node - <<'NODE' | tee /cache/HWLAB.last-flush.json
const { execFileSync } = require("node:child_process");
const repo = "/cache/pikasTech/HWLAB.git";
const gitopsBranches = JSON.parse(process.env.gitops_branches_json || "[]");
function rev(ref) {
try { return execFileSync("git", ["-C", repo, "rev-parse", ref], { encoding: "utf8" }).trim() || null; } catch { return null; }
}
const refs = {};
for (const branch of gitopsBranches) {
refs["refs/heads/" + branch] = rev("refs/heads/" + branch);
refs["refs/mirror-stage/heads/" + branch] = rev("refs/mirror-stage/heads/" + branch);
}
const pendingFlush = gitopsBranches.some((branch) => refs["refs/heads/" + branch] && refs["refs/mirror-stage/heads/" + branch] && refs["refs/heads/" + branch] !== refs["refs/mirror-stage/heads/" + branch]);
console.log(JSON.stringify({
event: "git-mirror-flush",
status: "flushed",
repository: "pikasTech/HWLAB",
flushedAt: process.env.flushed_at,
pendingFlush: false,
refs: { "refs/heads/v0.2-gitops": process.env.local_head || null }
pendingFlush,
refs
}));
NODE
emit_timing total succeeded "$started_ms"
@@ -4913,7 +4967,7 @@ async function plannedFiles(args) {
putJson("source.json", sourceDescriptor);
putText("README.md", readme({ source, args }));
putJson("registry/registry.yaml", registryManifest());
if (isRuntimeLane(args.lane)) putJson("devops-infra/git-mirror.yaml", devopsInfraGitMirrorManifest(args));
if (isRuntimeLane(args.lane)) putJson("devops-infra/git-mirror.yaml", devopsInfraGitMirrorManifest(args, deploy));
putJson(`${settings.tektonDir}/rbac.yaml`, tektonRbac(args));
putJson(`${settings.tektonDir}/pipeline.yaml`, tektonPipeline(args, deploy));
if (!isRuntimeLane(args.lane)) {
+8 -3
View File
@@ -216,8 +216,10 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots
assert.match(gitMirrorScript, /uploadpack\.allowAnySHA1InWant true/u);
assert.match(gitMirrorScript, /refs\/heads\/v0\.2:refs\/mirror-stage\/heads\/v0\.2/u);
assert.match(gitMirrorScript, /refs\/heads\/v0\.2-gitops:refs\/mirror-stage\/heads\/v0\.2-gitops/u);
assert.match(gitMirrorScript, /refs\/heads\/G14:refs\/mirror-stage\/heads\/G14/u);
assert.match(gitMirrorScript, /refs\/heads\/G14-gitops:refs\/mirror-stage\/heads\/G14-gitops/u);
assert.match(gitMirrorScript, /refs\/heads\/v0\.3:refs\/mirror-stage\/heads\/v0\.3/u);
assert.match(gitMirrorScript, /refs\/heads\/v0\.3-gitops:refs\/mirror-stage\/heads\/v0\.3-gitops/u);
assert.doesNotMatch(gitMirrorScript, /refs\/heads\/G14:refs\/mirror-stage\/heads\/G14/u);
assert.doesNotMatch(gitMirrorScript, /refs\/heads\/G14-gitops:refs\/mirror-stage\/heads\/G14-gitops/u);
assert.doesNotMatch(gitMirrorScript, /refs\/heads\/\*:refs\/mirror-stage\/heads\/\*/u);
assert.match(gitMirrorScript, /git -C "\$repo_path" update-ref "refs\/heads\/\$name" "\$sha"/u);
assert.match(gitMirrorScript, /git-mirror-sync-timing/u);
@@ -225,7 +227,10 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots
assert.match(gitMirrorScript, /emit_timing total succeeded/u);
assert.match(gitMirrorScript, /sourceInSync/u);
assert.match(gitMirrorScript, /refs\/mirror-stage\/heads\/v0\.2/u);
assert.match(gitMirrorScript, /keeps local v0\.2-gitops ahead of GitHub/u);
assert.match(gitMirrorScript, /refs\/mirror-stage\/heads\/v0\.3/u);
assert.match(gitMirrorScript, /for gitops_branch in 'v0\.2-gitops' 'v0\.3-gitops'/u);
assert.match(gitMirrorScript, /keeps local \$name ahead of GitHub/u);
assert.match(gitMirror, /deploy\/artifact-catalog\.v03\.json\|deploy\/gitops\/g14\/runtime-v03\/\*/u);
assert.ok((gitMirrorJson.items || []).every((item) => item.kind !== "Secret"));
assert.ok((gitMirrorJson.items || []).every((item) => item.kind !== "CronJob"));
assert.ok((gitMirrorJson.items || []).every((item) => !item.data?.["ssh-privatekey"]));
+25
View File
@@ -34,6 +34,14 @@ export type RuntimeLaneConfig = {
sourceRepo?: string;
};
export type RuntimeLaneMirrorSpec = {
lane: string;
sourceBranch: string;
gitopsBranch: string;
artifactCatalog: string;
runtimePath: string;
};
export type DeployConfig = {
lanes?: Record<string, RuntimeLaneConfig | undefined>;
};
@@ -111,6 +119,23 @@ export function applyRuntimeLaneDeployConfig(args: GitOpsRenderArgs, deploy: Dep
return result;
}
export function runtimeLaneMirrorSpecs(deploy: DeployConfig): RuntimeLaneMirrorSpec[] {
return Object.entries(deploy.lanes ?? {})
.filter(([lane, config]) => isRuntimeLane(lane) && isObject(config))
.sort(([left], [right]) => runtimeLaneMinor(left) - runtimeLaneMinor(right))
.map(([lane, config]) => {
const laneConfig = config as RuntimeLaneConfig;
const versionName = versionNameForRuntimeLane(lane);
return {
lane,
sourceBranch: laneConfig.sourceBranch ?? versionName,
gitopsBranch: laneConfig.gitopsBranch ?? `${versionName}-gitops`,
artifactCatalog: laneConfig.artifactCatalog ?? `deploy/artifact-catalog.${lane}.json`,
runtimePath: `deploy/gitops/g14/${runtimePathForRuntimeLane(lane)}`,
};
});
}
function isObject(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}