Merge pull request #596 from pikasTech/fix/v02-git-mirror-manual-sync

fix: make v02 git mirror sync manual and atomic
2026-05-30 02:56:24 +08:00
parent a509d74f8f e46f2c28ea
commit 836bfbe95e
3 changed files with 96 additions and 54 deletions
@@ -50,7 +50,7 @@ CI/CD 内部由 UniDesk 手动触发入口、PipelineRun、component planner、B
 | FRP Deployment | `hwlab-v02/hwlab-v02-frpc` |
 | Web entry | `http://74.48.78.17:19666/` |
 | API/live entry | `http://74.48.78.17:19667/health/live` |
-| Git mirror/cache | 独立 `devops-infra` 集群只读服务 |
+| Git mirror/cache | 独立 `devops-infra` 集群只读服务；不设 CronJob，按需由 UniDesk CLI 手动同步 |
 | Registry | 维持当前 G14 `hwlab-ci/hwlab-registry` |

 `scripts/g14-gitops-render.mjs --lane v02` 是当前 `v0.2` GitOps render 入口。该 lane 必须把默认 source branch 改为 `v0.2`、GitOps branch 改为 `v0.2-gitops`、catalog 改为 `deploy/artifact-catalog.v02.json`、runtime endpoint 改为 `19667`、web endpoint 改为 `19666`，并使用完整 source commit 作为 image tag。
@@ -107,6 +107,8 @@ CI/CD 内部由 UniDesk 手动触发入口、PipelineRun、component planner、B

 `v0.2` 不设自动轮询发布。历史上若存在 `hwlab-v02-branch-poller`、`hwlab-v02-control-plane-reconciler` 或等价 CronJob，均视为迁移残留，应由 UniDesk control-plane apply 清理；后续发布、重跑、暂停和恢复都通过手动 CLI 触发或停止创建新的 PipelineRun 完成，不新增替代 CronJob。

+`devops-infra` git mirror/cache 同样不设周期 CronJob。标准操作是先按需执行 `bun scripts/cli.ts hwlab g14 git-mirror sync --confirm`，再执行 `bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm`。`git-mirror apply` 只维护 mirror 的 PVC、只读 HTTP 服务、同步脚本和旧 CronJob 清理；`git-mirror sync` 创建一次性 Job，先把 GitHub refs fetch 到隐藏 staging refs，校验 commit/tree/object closure，再用 `update-ref` 发布到公开 refs。这样 mirror read path 仍是本地磁盘速度，同时避免 CI 看到 ref 已更新但对象还不可 checkout 的半发布窗口。
+
 ## Env 容器复用与三变量启动

 `v0.2` code-only fast lane 的 runtime desired state 由三类输入组成：可复用 env image digest、自动推导的 code boot metadata、以及既有 service runtime config。开发者仍按当前 DEV/OPS 流程提交 `v0.2` source commit 和维护 `deploy/deploy.json`；发布入口不得要求人工填写 repo、commitId 或 boot script 路径。
@@ -238,7 +240,7 @@ GitOps branch 已更新、source branch render 通过、PipelineRun 名称存在
 | FRP `19666/19667` 入口 | 已实现 | 由 `hwlab-v02-frpc` 与 master frps allowlist 共同提供。 |
 | SecretRef 独立与 provider 验收 | 已实现/持续约束 | SecretRef 已独立；验收必须做真实短连接聊天。 |
 | env 容器复用三变量启动 | 已实现/持续约束 | device-pod fast lane 已由 CI/CD 自动推导 `HWLAB_BOOT_REPO`、`HWLAB_BOOT_COMMIT`、`HWLAB_BOOT_SH`；code-only rollout 复用 env image digest，只更新代码身份。 |
-| `devops-infra` git mirror/cache 加速 | 已实现/持续约束 | runtime checkout 读路径使用独立基础设施集群只读 mirror/cache；runtime namespace 不持有 GitHub deploy key，registry 保持 G14 `hwlab-ci/hwlab-registry`。 |
+| `devops-infra` git mirror/cache 加速 | 已实现/持续约束 | runtime checkout 读路径使用独立基础设施集群只读 mirror/cache；runtime namespace 不持有 GitHub deploy key，registry 保持 G14 `hwlab-ci/hwlab-registry`；mirror 同步由 UniDesk CLI 手动触发，不设置 CronJob。 |
 | 自动 registry GC | 未实现 | 初期不启用自动 GC，后续需 lane/profile 保护集。 |

 ## 平行 lane 运维边界
@@ -2809,24 +2809,99 @@ repo_url="ssh://git@ssh.github.com:443/pikasTech/HWLAB.git"
 repo_path="/cache/pikasTech/HWLAB.git"
 lock_dir="/cache/.hwlab-sync.lock"
 mkdir -p /cache/pikasTech /root/.ssh
-if ! mkdir "\${lock_dir}" 2>/dev/null; then
+if ! mkdir "$lock_dir" 2>/dev/null; then
  echo "git mirror sync already running"
  exit 0
 fi
-trap 'rmdir "\${lock_dir}" 2>/dev/null || true' EXIT INT TERM
+trap 'rmdir "$lock_dir" 2>/dev/null || true' EXIT INT TERM
 cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa
 chmod 0400 /root/.ssh/id_rsa
 export GIT_SSH_COMMAND="ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=10 -o ServerAliveInterval=5 -o ServerAliveCountMax=1"
-if [ -d "\${repo_path}/objects" ] && [ -f "\${repo_path}/HEAD" ]; then
-  git -C "\${repo_path}" remote set-url origin "\${repo_url}"
-  timeout 180 git -C "\${repo_path}" remote update --prune
+if [ -d "$repo_path/objects" ] && [ -f "$repo_path/HEAD" ]; then
+  git -C "$repo_path" remote set-url origin "$repo_url" || git -C "$repo_path" remote add origin "$repo_url"
 else
-  rm -rf "\${repo_path}"
-  timeout 240 git clone --mirror "\${repo_url}" "\${repo_path}"
+  rm -rf "$repo_path"
+  git init --bare "$repo_path"
+  git -C "$repo_path" remote add origin "$repo_url"
 fi
-git -C "\${repo_path}" update-server-info
-git -C "\${repo_path}" rev-parse refs/heads/v0.2 | tee /cache/HWLAB-v0.2.head
-date -u +%Y-%m-%dT%H:%M:%SZ > /cache/HWLAB.last-sync
+git -C "$repo_path" config uploadpack.hideRefs refs/mirror-stage
+git -C "$repo_path" config --unset-all remote.origin.fetch 2>/dev/null || true
+git -C "$repo_path" config --add remote.origin.fetch '+refs/heads/*:refs/mirror-stage/heads/*'
+git -C "$repo_path" config --add remote.origin.fetch '+refs/tags/*:refs/mirror-stage/tags/*'
+git -C "$repo_path" for-each-ref --format='delete %(refname)' refs/mirror-stage | git -C "$repo_path" update-ref --stdin || true
+timeout 180 git -C "$repo_path" fetch --prune origin
+git -C "$repo_path" for-each-ref --format='%(objectname) %(refname)' refs/mirror-stage/heads > /tmp/hwlab-stage-heads
+git -C "$repo_path" for-each-ref --format='%(refname)' refs/mirror-stage/heads > /tmp/hwlab-stage-head-refs
+git -C "$repo_path" for-each-ref --format='%(objectname) %(refname)' refs/mirror-stage/tags > /tmp/hwlab-stage-tags
+git -C "$repo_path" for-each-ref --format='%(refname)' refs/mirror-stage/tags > /tmp/hwlab-stage-tag-refs
+if [ ! -s /tmp/hwlab-stage-heads ]; then
+  echo "git mirror sync fetched no branch refs" >&2
+  exit 41
+fi
+while read -r sha ref; do
+  [ -n "$sha" ] || continue
+  git -C "$repo_path" cat-file -e "$sha^{commit}"
+  git -C "$repo_path" cat-file -e "$sha^{tree}"
+  if git -C "$repo_path" rev-list --objects --missing=print "$sha" | grep -q '^?'; then
+    echo "git mirror sync found missing objects for $ref $sha" >&2
+    exit 42
+  fi
+done < /tmp/hwlab-stage-heads
+while read -r sha ref; do
+  [ -n "$sha" ] || continue
+  name=$(printf '%s\n' "$ref" | sed 's#^refs/mirror-stage/heads/##')
+  git -C "$repo_path" update-ref "refs/heads/$name" "$sha"
+done < /tmp/hwlab-stage-heads
+while read -r sha ref; do
+  [ -n "$sha" ] || continue
+  name=$(printf '%s\n' "$ref" | sed 's#^refs/mirror-stage/tags/##')
+  git -C "$repo_path" update-ref "refs/tags/$name" "$sha"
+done < /tmp/hwlab-stage-tags
+git -C "$repo_path" for-each-ref --format='%(refname)' refs/heads > /tmp/hwlab-public-heads
+while read -r ref; do
+  [ -n "$ref" ] || continue
+  name=$(printf '%s\n' "$ref" | sed 's#^refs/heads/##')
+  if ! grep -Fxq "refs/mirror-stage/heads/$name" /tmp/hwlab-stage-head-refs; then
+    git -C "$repo_path" update-ref -d "$ref"
+  fi
+done < /tmp/hwlab-public-heads
+git -C "$repo_path" for-each-ref --format='%(refname)' refs/tags > /tmp/hwlab-public-tags
+while read -r ref; do
+  [ -n "$ref" ] || continue
+  name=$(printf '%s\n' "$ref" | sed 's#^refs/tags/##')
+  if ! grep -Fxq "refs/mirror-stage/tags/$name" /tmp/hwlab-stage-tag-refs; then
+    git -C "$repo_path" update-ref -d "$ref"
+  fi
+done < /tmp/hwlab-public-tags
+git -C "$repo_path" for-each-ref --format='delete %(refname)' refs/mirror-stage | git -C "$repo_path" update-ref --stdin || true
+if git -C "$repo_path" show-ref --verify --quiet refs/heads/v0.2; then
+  git -C "$repo_path" symbolic-ref HEAD refs/heads/v0.2
+elif git -C "$repo_path" show-ref --verify --quiet refs/heads/G14; then
+  git -C "$repo_path" symbolic-ref HEAD refs/heads/G14
+fi
+git -C "$repo_path" fsck --connectivity-only --no-dangling >/tmp/hwlab-git-fsck.out
+git -C "$repo_path" update-server-info
+v02_head=$(git -C "$repo_path" rev-parse refs/heads/v0.2 2>/dev/null || true)
+g14_head=$(git -C "$repo_path" rev-parse refs/heads/G14 2>/dev/null || true)
+v02_gitops_head=$(git -C "$repo_path" rev-parse refs/heads/v0.2-gitops 2>/dev/null || true)
+published_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+printf '%s\n' "$v02_head" | tee /cache/HWLAB-v0.2.head
+printf '%s\n' "$published_at" > /cache/HWLAB.last-sync
+export v02_head g14_head v02_gitops_head published_at
+node - <<'NODE' | tee /cache/HWLAB.last-sync.json
+const payload = {
+  event: "git-mirror-sync",
+  status: "published",
+  repository: "pikasTech/HWLAB",
+  publishedAt: process.env.published_at,
+  refs: {
+    "refs/heads/v0.2": process.env.v02_head || null,
+    "refs/heads/G14": process.env.g14_head || null,
+    "refs/heads/v0.2-gitops": process.env.v02_gitops_head || null
+  }
+};
+console.log(JSON.stringify(payload));
+NODE
 `,
          "http.mjs": `import { createServer } from "node:http";
 import { spawn } from "node:child_process";
@@ -3007,47 +3082,6 @@ function contentType(filePath) {
        kind: "Service",
        metadata: { name: "git-mirror-http", namespace: "devops-infra", labels: { ...labels, "app.kubernetes.io/component": "read-only-http" } },
        spec: { selector: { "app.kubernetes.io/name": "git-mirror", "app.kubernetes.io/component": "read-only-http" }, ports: [{ name: "http", port: 80, targetPort: "http" }] }
-      },
-      {
-        apiVersion: "batch/v1",
-        kind: "CronJob",
-        metadata: { name: "git-mirror-hwlab-sync", namespace: "devops-infra", labels: { ...labels, "app.kubernetes.io/component": "sync-controller" } },
-        spec: {
-          schedule: "*/2 * * * *",
-          concurrencyPolicy: "Forbid",
-          successfulJobsHistoryLimit: 3,
-          failedJobsHistoryLimit: 3,
-          jobTemplate: {
-            spec: {
-              backoffLimit: 1,
-              activeDeadlineSeconds: 300,
-              template: {
-                metadata: { labels: { ...labels, "app.kubernetes.io/component": "sync-controller" } },
-                spec: {
-                  restartPolicy: "Never",
-                  hostNetwork: true,
-                  dnsPolicy: "ClusterFirstWithHostNet",
-                  volumes: [
-                    { name: "cache", persistentVolumeClaim: { claimName: "git-mirror-cache" } },
-                    { name: "git-ssh", secret: { secretName: "git-mirror-github-ssh", defaultMode: 0o400 } },
-                    { name: "script", configMap: { name: "git-mirror-sync-script", defaultMode: 0o755 } }
-                  ],
-                  containers: [{
-                    name: "sync",
-                    image: ciToolsRunnerImage,
-                    imagePullPolicy: "IfNotPresent",
-                    command: ["/script/sync.sh"],
-                    volumeMounts: [
-                      { name: "cache", mountPath: "/cache" },
-                      { name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
-                      { name: "script", mountPath: "/script", readOnly: true }
-                    ]
-                  }]
-                }
-              }
-            }
-          }
-        }
      }
    ]
  };
@@ -81,10 +81,16 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots
    const gitMirror = await readFile(path.join(outDir, "devops-infra", "git-mirror.yaml"), "utf8");
    assert.match(gitMirror, /"name": "devops-infra"/u);
    assert.match(gitMirror, /"name": "git-mirror-http"/u);
-    assert.match(gitMirror, /"name": "git-mirror-hwlab-sync"/u);
+    assert.match(gitMirror, /"name": "git-mirror-sync-script"/u);
    assert.match(gitMirror, /ssh:\/\/git@ssh\.github\.com:443\/pikasTech\/HWLAB\.git/u);
+    assert.doesNotMatch(gitMirror, /"kind": "CronJob"/u);
+    assert.doesNotMatch(gitMirror, /"name": "git-mirror-hwlab-sync"/u);
    const gitMirrorJson = JSON.parse(gitMirror);
+    const gitMirrorScript = gitMirrorJson.items.find((item) => item.kind === "ConfigMap" && item.metadata?.name === "git-mirror-sync-script")?.data?.["sync.sh"] ?? "";
+    assert.match(gitMirrorScript, /refs\/mirror-stage\/heads\/\*/u);
+    assert.match(gitMirrorScript, /git -C "\$repo_path" update-ref "refs\/heads\/\$name" "\$sha"/u);
    assert.ok((gitMirrorJson.items || []).every((item) => item.kind !== "Secret"));
+    assert.ok((gitMirrorJson.items || []).every((item) => item.kind !== "CronJob"));
    assert.ok((gitMirrorJson.items || []).every((item) => !item.data?.["ssh-privatekey"]));

    const removedServiceIds = [