From 59883f41687f8a4ff64dee7aac2b97cdc192e468 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 30 May 2026 08:38:41 +0800 Subject: [PATCH] feat: add v02 git write mirror relay --- scripts/g14-gitops-render.mjs | 275 +++++++++++++++++++++++++++--- scripts/g14-gitops-render.test.ts | 31 +++- 2 files changed, 286 insertions(+), 20 deletions(-) diff --git a/scripts/g14-gitops-render.mjs b/scripts/g14-gitops-render.mjs index 359c8712..0a2cef35 100644 --- a/scripts/g14-gitops-render.mjs +++ b/scripts/g14-gitops-render.mjs @@ -14,6 +14,7 @@ const defaultRegistryPrefix = process.env.HWLAB_G14_REGISTRY_PREFIX || "127.0.0. const defaultSourceRepo = process.env.HWLAB_G14_GIT_URL || "git@github.com:pikasTech/HWLAB.git"; const defaultGitReadUrl = process.env.HWLAB_G14_GIT_READ_URL || null; const defaultV02GitReadUrl = process.env.HWLAB_V02_GIT_READ_URL || "http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git"; +const defaultV02GitWriteUrl = process.env.HWLAB_V02_GIT_WRITE_URL || "http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/HWLAB.git"; const defaultBranch = process.env.HWLAB_G14_BRANCH || "G14"; const defaultGitopsBranch = process.env.HWLAB_G14_GITOPS_BRANCH || "G14-gitops"; const defaultCatalogPath = process.env.HWLAB_ARTIFACT_CATALOG_PATH || "deploy/artifact-catalog.dev.json"; @@ -25,7 +26,7 @@ const defaultV02RuntimeEndpoint = process.env.HWLAB_V02_RUNTIME_ENDPOINT || "htt const defaultV02WebEndpoint = process.env.HWLAB_V02_WEB_ENDPOINT || "http://74.48.78.17:19666"; const defaultProxyUrl = process.env.HWLAB_G14_PROXY_URL || "http://127.0.0.1:10808"; const defaultAllProxyUrl = process.env.HWLAB_G14_ALL_PROXY_URL || "socks5h://127.0.0.1:10808"; -const defaultNoProxy = process.env.HWLAB_G14_NO_PROXY || "hyueapi.com,.hyueapi.com,127.0.0.1,localhost,::1,10.0.0.0/8,10.42.0.0/16,10.43.0.0/16,192.168.0.0/16,.svc,.cluster.local,kubernetes.default.svc,registry.hwlab-ci.svc,hwlab-registry.hwlab-ci.svc,git-mirror-http,git-mirror-http.devops-infra,git-mirror-http.devops-infra.svc,git-mirror-http.devops-infra.svc.cluster.local"; +const defaultNoProxy = process.env.HWLAB_G14_NO_PROXY || "hyueapi.com,.hyueapi.com,127.0.0.1,localhost,::1,10.0.0.0/8,10.42.0.0/16,10.43.0.0/16,192.168.0.0/16,.svc,.cluster.local,kubernetes.default.svc,registry.hwlab-ci.svc,hwlab-registry.hwlab-ci.svc,git-mirror-http,git-mirror-http.devops-infra,git-mirror-http.devops-infra.svc,git-mirror-http.devops-infra.svc.cluster.local,git-mirror-write,git-mirror-write.devops-infra,git-mirror-write.devops-infra.svc,git-mirror-write.devops-infra.svc.cluster.local"; const ciToolsRunnerImage = process.env.HWLAB_G14_CI_TOOLS_IMAGE || "127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1"; const buildkitRunnerImage = process.env.HWLAB_G14_BUILDKIT_IMAGE || "moby/buildkit:rootless"; const defaultDevBaseImage = process.env.HWLAB_G14_DEV_BASE_IMAGE || "127.0.0.1:5000/hwlab/hwlab-node20-base:20-bookworm-slim"; @@ -187,6 +188,7 @@ function parseArgs(argv) { gitopsBranch: defaultGitopsBranch, sourceRepo: defaultSourceRepo, gitReadUrl: defaultGitReadUrl, + gitWriteUrl: null, runtimeEndpoint: defaultRuntimeEndpoint, webEndpoint: defaultWebEndpoint, prodRuntimeEndpoint: defaultProdRuntimeEndpoint, @@ -208,6 +210,7 @@ function parseArgs(argv) { else if (arg === "--gitops-branch") args.gitopsBranch = readOption(argv, ++index, arg); else if (arg === "--source-repo") args.sourceRepo = readOption(argv, ++index, arg); else if (arg === "--git-read-url") args.gitReadUrl = readOption(argv, ++index, arg); + else if (arg === "--git-write-url") args.gitWriteUrl = readOption(argv, ++index, arg); else if (arg === "--runtime-endpoint") args.runtimeEndpoint = readOption(argv, ++index, arg); else if (arg === "--web-endpoint") args.webEndpoint = readOption(argv, ++index, arg); else if (arg === "--prod-runtime-endpoint") args.prodRuntimeEndpoint = readOption(argv, ++index, arg); @@ -228,8 +231,10 @@ function parseArgs(argv) { if (args.webEndpoint === defaultWebEndpoint) args.webEndpoint = defaultV02WebEndpoint; if (args.imageTagMode === "short") args.imageTagMode = "full"; if (!args.gitReadUrl) args.gitReadUrl = defaultV02GitReadUrl; + if (!args.gitWriteUrl) args.gitWriteUrl = defaultV02GitWriteUrl; } if (!args.gitReadUrl) args.gitReadUrl = args.sourceRepo; + if (!args.gitWriteUrl) args.gitWriteUrl = args.sourceRepo; assert.ok(["g14", "v02"].includes(args.lane), `unknown lane ${args.lane}`); assert.ok(["short", "full"].includes(args.imageTagMode), `unknown image tag mode ${args.imageTagMode}`); if (args.lane === "v02") { @@ -258,6 +263,7 @@ function usage() { " --image-tag-mode MODE short or full; v02 uses full source commit IDs", ` --source-repo URL default: ${defaultSourceRepo}`, " --git-read-url URL read-only source URL; v02 defaults to devops-infra git mirror", + " --git-write-url URL GitOps write URL; v02 defaults to devops-infra git mirror write relay", ` --source-branch BRANCH default: ${defaultBranch}`, ` --gitops-branch BRANCH default: ${defaultGitopsBranch}`, " --source-revision SHA source commit used for image tags and runtime annotations; default: git rev-parse HEAD", @@ -1414,8 +1420,12 @@ test "$(git rev-parse HEAD)" = "$(params.revision)" check_source_head() { phase="$1" expected="\${2:-$(params.revision)}" + source_head_url="$(params.git-url)" + if [ "$(params.lane)" = "v02" ]; then + source_head_url="$(params.git-read-url)" + fi latest_file="$(mktemp)" - if ! git_timed "source-head-$phase" 45 git ls-remote "$(params.git-url)" "refs/heads/$(params.source-branch)" > "$latest_file"; then + if ! git_timed "source-head-$phase" 45 git ls-remote "$source_head_url" "refs/heads/$(params.source-branch)" > "$latest_file"; then echo '{"status":"failed","reason":"source-head-check-failed","phase":"'"$phase"'","sourceBranch":"'"$(params.source-branch)"'","sourceRevision":"'"$(params.revision)"'"}' exit 1 fi @@ -1528,9 +1538,15 @@ node scripts/g14-gitops-render.mjs --lane "$lane" --catalog-path "$catalog_path" ci_timing_emit gitops-render succeeded "$gitops_render_started_ms" check_source_head before-push workdir="$(mktemp -d)" +gitops_read_url="$(params.git-url)" +gitops_write_url="$(params.git-write-url)" +if [ "$lane" = "v02" ]; then + gitops_read_url="$(params.git-read-url)" +fi gitops_clone_started_ms="$(ci_now_ms)" -git_timed gitops-clone 180 git clone --no-checkout "$(params.git-url)" "$workdir/gitops" +git_timed gitops-clone 180 git clone --no-checkout "$gitops_read_url" "$workdir/gitops" cd "$workdir/gitops" +git remote set-url origin "$gitops_write_url" if git_timed gitops-branch-ls 45 git ls-remote --exit-code --heads origin "$(params.gitops-branch)" >/dev/null; then git_timed gitops-fetch 90 git fetch origin "$(params.gitops-branch)" git checkout -B "$(params.gitops-branch)" "origin/$(params.gitops-branch)" @@ -1566,7 +1582,7 @@ gitops_push_started_ms="$(ci_now_ms)" git_timed gitops-push 120 git push origin "HEAD:$(params.gitops-branch)" ci_timing_emit gitops-push succeeded "$gitops_push_started_ms" argo_hard_refresh_v02 -echo '{"status":"pushed","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}' +echo '{"status":"pushed","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'","gitopsWriteUrl":"'"$gitops_write_url"'"}' `; } @@ -2454,6 +2470,7 @@ function tektonPipeline(args = { lane: "g14" }) { params: [ { name: "git-url", type: "string", default: defaultSourceRepo }, { name: "git-read-url", type: "string", default: args.gitReadUrl }, + { name: "git-write-url", type: "string", default: args.gitWriteUrl }, { name: "source-branch", type: "string", default: args.sourceBranch }, { name: "gitops-branch", type: "string", default: args.gitopsBranch }, { name: "lane", type: "string", default: settings.lane }, @@ -2517,6 +2534,8 @@ function tektonPipeline(args = { lane: "g14" }) { taskSpec: { params: [ { name: "git-url" }, + { name: "git-read-url" }, + { name: "git-write-url" }, { name: "source-branch" }, { name: "gitops-branch" }, { name: "lane" }, @@ -2531,6 +2550,8 @@ function tektonPipeline(args = { lane: "g14" }) { }, params: [ { name: "git-url", value: "$(params.git-url)" }, + { name: "git-read-url", value: "$(params.git-read-url)" }, + { name: "git-write-url", value: "$(params.git-write-url)" }, { name: "source-branch", value: "$(params.source-branch)" }, { name: "gitops-branch", value: "$(params.gitops-branch)" }, { name: "lane", value: "$(params.lane)" }, @@ -2575,6 +2596,7 @@ function tektonPipelineRunTemplate({ source, args }) { params: [ { name: "git-url", value: args.sourceRepo }, { name: "git-read-url", value: args.gitReadUrl }, + { name: "git-write-url", value: args.gitWriteUrl }, { name: "source-branch", value: args.sourceBranch }, { name: "gitops-branch", value: args.gitopsBranch }, { name: "lane", value: settings.lane }, @@ -2834,6 +2856,7 @@ else git init --bare "$repo_path" git -C "$repo_path" remote add origin "$repo_url" fi +/script/install-hooks.sh git -C "$repo_path" config uploadpack.hideRefs refs/mirror-stage git -C "$repo_path" config --unset-all remote.origin.fetch 2>/dev/null || true git -C "$repo_path" config --add remote.origin.fetch '+refs/heads/v0.2:refs/mirror-stage/heads/v0.2' @@ -2869,6 +2892,19 @@ publish_started_ms=$(now_ms) while read -r sha ref; do [ -n "$sha" ] || continue name=$(printf '%s\n' "$ref" | sed 's#^refs/mirror-stage/heads/##') + if [ "$name" = "v0.2-gitops" ]; then + local_sha=$(git -C "$repo_path" rev-parse refs/heads/v0.2-gitops 2>/dev/null || true) + if [ -n "$local_sha" ] && [ "$local_sha" != "$sha" ]; then + if git -C "$repo_path" merge-base --is-ancestor "$sha" "$local_sha"; then + echo "git mirror sync keeps local v0.2-gitops ahead of GitHub" + continue + fi + if ! git -C "$repo_path" merge-base --is-ancestor "$local_sha" "$sha"; then + echo "git mirror sync found divergent v0.2-gitops local=$local_sha github=$sha" >&2 + exit 44 + fi + fi + fi git -C "$repo_path" update-ref "refs/heads/$name" "$sha" done < /tmp/hwlab-stage-heads while read -r sha ref; do @@ -2905,25 +2941,161 @@ git -C "$repo_path" update-server-info v02_head=$(git -C "$repo_path" rev-parse refs/heads/v0.2 2>/dev/null || true) g14_head=$(git -C "$repo_path" rev-parse refs/heads/G14 2>/dev/null || true) v02_gitops_head=$(git -C "$repo_path" rev-parse refs/heads/v0.2-gitops 2>/dev/null || true) +v02_gitops_github_head=$(git -C "$repo_path" rev-parse refs/mirror-stage/heads/v0.2-gitops 2>/dev/null || true) published_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) printf '%s\n' "$v02_head" | tee /cache/HWLAB-v0.2.head printf '%s\n' "$published_at" > /cache/HWLAB.last-sync -export v02_head g14_head v02_gitops_head published_at +export v02_head g14_head v02_gitops_head v02_gitops_github_head published_at node - <<'NODE' | tee /cache/HWLAB.last-sync.json const payload = { event: "git-mirror-sync", status: "published", repository: "pikasTech/HWLAB", publishedAt: process.env.published_at, + pendingFlush: Boolean(process.env.v02_gitops_head && process.env.v02_gitops_github_head && process.env.v02_gitops_head !== process.env.v02_gitops_github_head), refs: { "refs/heads/v0.2": process.env.v02_head || null, "refs/heads/G14": process.env.g14_head || null, - "refs/heads/v0.2-gitops": process.env.v02_gitops_head || null + "refs/heads/v0.2-gitops": process.env.v02_gitops_head || null, + "refs/mirror-stage/heads/v0.2-gitops": process.env.v02_gitops_github_head || null } }; console.log(JSON.stringify(payload)); NODE emit_timing total succeeded "$sync_started_ms" +`, + "install-hooks.sh": `#!/bin/sh +set -eu +repo_path="/cache/pikasTech/HWLAB.git" +mkdir -p "$repo_path/hooks" +cat > "$repo_path/hooks/pre-receive" <<'HOOK' +#!/bin/sh +set -eu +zero="0000000000000000000000000000000000000000" +status=0 +while read -r old new ref; do + if [ "$ref" != "refs/heads/v0.2-gitops" ]; then + echo "git mirror write rejected: ref $ref is not allowlisted" >&2 + status=1 + continue + fi + if [ "$new" = "$zero" ]; then + echo "git mirror write rejected: deleting $ref is forbidden" >&2 + status=1 + continue + fi + git cat-file -e "$new^{commit}" || status=1 + git cat-file -e "$new^{tree}" || status=1 + if git rev-list --objects --missing=print "$new" | grep -q '^?'; then + echo "git mirror write rejected: missing objects for $new" >&2 + status=1 + continue + fi + if [ "$old" != "$zero" ] && ! git merge-base --is-ancestor "$old" "$new"; then + echo "git mirror write rejected: non-fast-forward $ref" >&2 + status=1 + continue + fi + if [ "$old" = "$zero" ]; then + git diff-tree --no-commit-id --name-only -r "$new" + else + git diff --name-only "$old" "$new" + fi | while read -r path; do + case "$path" in + deploy/artifact-catalog.v02.json|deploy/gitops/g14/runtime-v02/*) ;; + "") ;; + *) echo "git mirror write rejected: path $path is outside v0.2 GitOps outputs" >&2; exit 64 ;; + esac + done || status=1 +done +exit "$status" +HOOK +cat > "$repo_path/hooks/post-receive" <<'HOOK' +#!/bin/sh +set -eu +repo_path=$(git rev-parse --git-dir) +git -C "$repo_path" update-server-info +local_head=$(git -C "$repo_path" rev-parse refs/heads/v0.2-gitops 2>/dev/null || true) +github_head=$(git -C "$repo_path" rev-parse refs/mirror-stage/heads/v0.2-gitops 2>/dev/null || true) +written_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +export local_head github_head written_at +node - <<'NODE' > /cache/HWLAB.last-write.json +const localHead = process.env.local_head || null; +const githubHead = process.env.github_head || null; +console.log(JSON.stringify({ + event: "git-mirror-write", + status: "received", + repository: "pikasTech/HWLAB", + writtenAt: process.env.written_at, + pendingFlush: Boolean(localHead && localHead !== githubHead), + refs: { + "refs/heads/v0.2-gitops": localHead, + "refs/mirror-stage/heads/v0.2-gitops": githubHead + } +})); +NODE +HOOK +chmod 0755 "$repo_path/hooks/pre-receive" "$repo_path/hooks/post-receive" +git -C "$repo_path" config http.receivepack true +git -C "$repo_path" config receive.denyDeletes true +git -C "$repo_path" config receive.denyNonFastForwards true +`, + "flush.sh": `#!/bin/sh +set -eu +repo_url="ssh://git@ssh.github.com:443/pikasTech/HWLAB.git" +repo_path="/cache/pikasTech/HWLAB.git" +lock_dir="/cache/.hwlab-flush.lock" +started_ms=$(node -e 'console.log(Date.now())') +now_ms() { node -e 'console.log(Date.now())'; } +emit_timing() { + phase="$1" + status="$2" + started="$3" + finished=$(now_ms) + duration=$((finished - started)) + printf '{"event":"git-mirror-flush-timing","phase":"%s","status":"%s","durationMs":%s,"repository":"pikasTech/HWLAB"}\n' "$phase" "$status" "$duration" +} +if ! mkdir "$lock_dir" 2>/dev/null; then + echo "git mirror flush already running" + exit 0 +fi +trap 'rmdir "$lock_dir" 2>/dev/null || true' EXIT INT TERM +mkdir -p /root/.ssh +cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa +chmod 0400 /root/.ssh/id_rsa +export GIT_SSH_COMMAND="ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=10 -o ServerAliveInterval=5 -o ServerAliveCountMax=1" +/script/install-hooks.sh +local_head=$(git -C "$repo_path" rev-parse refs/heads/v0.2-gitops 2>/dev/null || true) +if [ -z "$local_head" ]; then + echo "git mirror flush missing local refs/heads/v0.2-gitops" >&2 + exit 51 +fi +fetch_started=$(now_ms) +timeout 90 git -C "$repo_path" fetch --quiet "$repo_url" refs/heads/v0.2-gitops:refs/mirror-stage/heads/v0.2-gitops +emit_timing fetch succeeded "$fetch_started" +github_head=$(git -C "$repo_path" rev-parse refs/mirror-stage/heads/v0.2-gitops 2>/dev/null || true) +if [ -n "$github_head" ] && [ "$github_head" != "$local_head" ] && ! git -C "$repo_path" merge-base --is-ancestor "$github_head" "$local_head"; then + echo "git mirror flush rejected divergent v0.2-gitops local=$local_head github=$github_head" >&2 + exit 52 +fi +push_started=$(now_ms) +timeout 120 git -C "$repo_path" push "$repo_url" "$local_head:refs/heads/v0.2-gitops" +emit_timing push succeeded "$push_started" +git -C "$repo_path" update-ref refs/mirror-stage/heads/v0.2-gitops "$local_head" +git -C "$repo_path" update-server-info +flushed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +export local_head flushed_at +node - <<'NODE' | tee /cache/HWLAB.last-flush.json +console.log(JSON.stringify({ + event: "git-mirror-flush", + status: "flushed", + repository: "pikasTech/HWLAB", + flushedAt: process.env.flushed_at, + pendingFlush: false, + refs: { "refs/heads/v0.2-gitops": process.env.local_head || null } +})); +NODE +emit_timing total succeeded "$started_ms" `, "http.mjs": `import { createServer } from "node:http"; import { spawn } from "node:child_process"; @@ -2945,9 +3117,10 @@ createServer((req, res) => { async function handle(req, res) { const url = new URL(req.url || "/", "http://git-mirror-http"); + const writeHost = isWriteHost(req.headers.host || ""); if (req.method === "GET" && (url.pathname === "/" || url.pathname === "/health")) { res.writeHead(200, { "content-type": "application/json", "cache-control": "no-cache" }); - res.end(JSON.stringify({ ok: true, service: "git-mirror-smart-http" })); + res.end(JSON.stringify({ ok: true, service: "git-mirror-smart-http", writeHost })); return; } if (req.method === "GET" && url.pathname.endsWith("/info/refs") && url.searchParams.get("service") === "git-upload-pack") { @@ -2958,12 +3131,32 @@ async function handle(req, res) { await runUploadPackRpc(repoPathFromGitServicePath(url.pathname, "/git-upload-pack"), req, res); return; } + if (req.method === "GET" && url.pathname.endsWith("/info/refs") && url.searchParams.get("service") === "git-receive-pack") { + if (!writeHost) return rejectReadOnly(res); + await runReceivePackAdvertisement(repoPathFromGitServicePath(url.pathname, "/info/refs"), res); + return; + } + if (req.method === "POST" && url.pathname.endsWith("/git-receive-pack")) { + if (!writeHost) return rejectReadOnly(res); + await runReceivePackRpc(repoPathFromGitServicePath(url.pathname, "/git-receive-pack"), req, res); + return; + } if (req.method === "GET" || req.method === "HEAD") { serveStaticFile(url.pathname, req, res); return; } res.writeHead(405, { "content-type": "text/plain" }); - res.end("method not allowed\n"); + res.end("method not allowed\\n"); +} + +function isWriteHost(hostHeader) { + const host = String(hostHeader || "").split(":", 1)[0].toLowerCase(); + return host === "git-mirror-write" || host.startsWith("git-mirror-write."); +} + +function rejectReadOnly(res) { + res.writeHead(403, { "content-type": "application/json", "cache-control": "no-cache" }); + res.end(JSON.stringify({ ok: false, error: "git mirror receive-pack is only available through git-mirror-write" })); } function repoPathFromGitServicePath(urlPath, suffix) { @@ -2973,7 +3166,7 @@ function repoPathFromGitServicePath(urlPath, suffix) { } function safePath(urlPath) { - const decoded = decodeURIComponent(urlPath.replace(/^\/+/, "")); + const decoded = decodeURIComponent(urlPath.replace(/^[/]+/u, "")); const fullPath = path.resolve(cacheRoot, decoded); const normalizedRoot = path.resolve(cacheRoot); if (fullPath !== normalizedRoot && !fullPath.startsWith(` + "`" + `\${normalizedRoot}\${path.sep}` + "`" + `)) throw new Error("path escapes cache root"); @@ -3013,7 +3206,7 @@ function runUploadPackAdvertisement(repoPath, res) { else resolve(); }); res.writeHead(200, smartHeaders("application/x-git-upload-pack-advertisement")); - res.write(pktLine("# service=git-upload-pack\n")); + res.write(pktLine("# service=git-upload-pack\\n")); res.write("0000"); child.stdout.pipe(res, { end: false }); child.stdout.on("end", () => res.end()); @@ -3038,17 +3231,53 @@ function runUploadPackRpc(repoPath, req, res) { }); } +function runReceivePackAdvertisement(repoPath, res) { + return new Promise((resolve, reject) => { + const child = spawn("git-receive-pack", ["--stateless-rpc", "--advertise-refs", repoPath], { stdio: ["ignore", "pipe", "pipe"] }); + let stderr = ""; + child.stderr.on("data", (chunk) => { stderr += chunk; process.stderr.write(chunk); }); + child.on("error", reject); + child.on("close", (code) => { + if (code !== 0 && !res.headersSent) reject(new Error(` + "`" + `git-receive-pack advertise failed: \${stderr.trim()}` + "`" + `)); + else resolve(); + }); + res.writeHead(200, smartHeaders("application/x-git-receive-pack-advertisement")); + res.write(pktLine("# service=git-receive-pack\\n")); + res.write("0000"); + child.stdout.pipe(res, { end: false }); + child.stdout.on("end", () => res.end()); + }); +} + +function runReceivePackRpc(repoPath, req, res) { + return new Promise((resolve, reject) => { + const child = spawn("git-receive-pack", ["--stateless-rpc", repoPath], { stdio: ["pipe", "pipe", "pipe"] }); + let stderr = ""; + child.stderr.on("data", (chunk) => { stderr += chunk; process.stderr.write(chunk); }); + child.on("error", reject); + child.on("close", (code) => { + if (code !== 0 && !res.headersSent) reject(new Error(` + "`" + `git-receive-pack rpc failed: \${stderr.trim()}` + "`" + `)); + else resolve(); + }); + res.writeHead(200, smartHeaders("application/x-git-receive-pack-result")); + const body = requestBodyStream(req); + body.on("error", (error) => child.stdin.destroy(error)); + body.pipe(child.stdin); + child.stdout.pipe(res); + }); +} + function serveStaticFile(urlPath, req, res) { const fullPath = safePath(urlPath); let stat; try { stat = statSync(fullPath); } catch { res.writeHead(404, { "content-type": "text/plain" }); - res.end("not found\n"); + res.end("not found\\n"); return; } if (!stat.isFile()) { res.writeHead(404, { "content-type": "text/plain" }); - res.end("not found\n"); + res.end("not found\\n"); return; } res.writeHead(200, { "content-length": stat.size, "content-type": contentType(fullPath), "cache-control": "no-cache" }); @@ -3087,12 +3316,12 @@ function contentType(filePath) { readinessProbe: { httpGet: { path: "/pikasTech/HWLAB.git/HEAD", port: "http" }, initialDelaySeconds: 5, periodSeconds: 10 }, livenessProbe: { httpGet: { path: "/", port: "http" }, initialDelaySeconds: 10, periodSeconds: 30 }, volumeMounts: [ - { name: "cache", mountPath: "/cache", readOnly: true }, + { name: "cache", mountPath: "/cache" }, { name: "script", mountPath: "/script", readOnly: true } ] }], volumes: [ - { name: "cache", persistentVolumeClaim: { claimName: "git-mirror-cache", readOnly: true } }, + { name: "cache", persistentVolumeClaim: { claimName: "git-mirror-cache" } }, { name: "script", configMap: { name: "git-mirror-sync-script", defaultMode: 0o555 } } ] } @@ -3104,6 +3333,12 @@ function contentType(filePath) { kind: "Service", metadata: { name: "git-mirror-http", namespace: "devops-infra", labels: { ...labels, "app.kubernetes.io/component": "read-only-http" } }, spec: { selector: { "app.kubernetes.io/name": "git-mirror", "app.kubernetes.io/component": "read-only-http" }, ports: [{ name: "http", port: 80, targetPort: "http" }] } + }, + { + apiVersion: "v1", + kind: "Service", + metadata: { name: "git-mirror-write", namespace: "devops-infra", labels: { ...labels, "app.kubernetes.io/component": "write-relay" } }, + spec: { selector: { "app.kubernetes.io/name": "git-mirror", "app.kubernetes.io/component": "read-only-http" }, ports: [{ name: "http", port: 80, targetPort: "http" }] } } ] }; @@ -3117,7 +3352,7 @@ function argoProject(args = { lane: "g14" }) { metadata: { name: "hwlab-v02", namespace: "argocd" }, spec: { description: "HWLAB v0.2 additive GitOps project on G14; DEV/PROD remain outside this lane.", - sourceRepos: [defaultSourceRepo], + sourceRepos: [args.gitReadUrl], destinations: [{ server: "https://kubernetes.default.svc", namespace: "hwlab-v02" }], clusterResourceWhitelist: [{ group: "", kind: "Namespace" }], namespaceResourceWhitelist: [{ group: "*", kind: "*" }] @@ -3146,6 +3381,7 @@ function argoApplication(args, profile = "dev") { const runtimePath = runtimePathForProfile(profile); const gitopsTarget = gitopsTargetForProfile(profile); const project = profile === "v02" ? "hwlab-v02" : "hwlab-g14"; + const repoURL = profile === "v02" ? args.gitReadUrl : args.sourceRepo; return { apiVersion: "argoproj.io/v1alpha1", kind: "Application", @@ -3156,7 +3392,7 @@ function argoApplication(args, profile = "dev") { }, spec: { project, - source: { repoURL: args.sourceRepo, targetRevision: args.gitopsBranch, path: `deploy/gitops/g14/${runtimePath}` }, + source: { repoURL, targetRevision: args.gitopsBranch, path: `deploy/gitops/g14/${runtimePath}` }, destination: { server: "https://kubernetes.default.svc", namespace }, syncPolicy: { automated: { prune: profile === "v02", selfHeal: true }, syncOptions: ["CreateNamespace=true", "ApplyOutOfSyncOnly=true"] } } @@ -3718,11 +3954,11 @@ This directory is generated by \`scripts/g14-gitops-render.mjs --lane v02\` from - Target: additive G14 v0.2 lane only. - CI: Tekton Pipeline \`hwlab-ci/hwlab-v02-ci-image-publish\`. -- Source reads: v0.2 CI source clone uses devops-infra read-only mirror \`${args.gitReadUrl}\`; canonical repo \`${args.sourceRepo}\` remains the identity and GitOps write path. +- Git mirror/relay: v0.2 CI reads source/catalog from \`${args.gitReadUrl}\` and writes GitOps promotion to \`${args.gitWriteUrl}\`; canonical repo \`${args.sourceRepo}\` remains the source identity and manual flush upstream. - Artifact contract: runtime images come from \`${args.catalogPath}\`; source branch \`v0.2\` does not track this generated catalog. - Manual trigger: run \`bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm\` from UniDesk to create commit-pinned PipelineRun \`hwlab-v02-ci-poll-\` for the current \`origin/v0.2\` commit. -- Branch split: source branch \`${args.sourceBranch}\` remains human-authored source; generated desired state is promoted to \`${args.gitopsBranch}\`. -- CD: Argo CD Application \`argocd/hwlab-g14-v02\` consumes \`${args.gitopsBranch}:deploy/gitops/g14/runtime-v02\` and deploys only to namespace \`hwlab-v02\`. +- Branch split: source branch \`${args.sourceBranch}\` remains human-authored source; generated desired state is promoted to local mirror branch \`${args.gitopsBranch}\`. +- CD: Argo CD Application \`argocd/hwlab-g14-v02\` consumes \`${args.gitReadUrl}:${args.gitopsBranch}:deploy/gitops/g14/runtime-v02\` and deploys only to namespace \`hwlab-v02\`. - Public preview: FRP exposes v0.2 web \`${args.webEndpoint}\` / edge \`${args.runtimeEndpoint}\` through \`hwlab-v02/hwlab-v02-frpc\`. - G14 DEV/PROD boundary: this lane does not modify \`G14\`, \`G14-gitops\`, \`hwlab-dev\`, \`hwlab-prod\`, 17666/17667 or 18666/18667. - CronJob policy: v0.2 does not generate \`hwlab-v02-branch-poller\` or \`hwlab-v02-control-plane-reconciler\`; obsolete live CronJobs should be deleted by the UniDesk control-plane apply command. @@ -3785,6 +4021,7 @@ async function plannedFiles(args) { lane: settings.lane, sourceRepo: args.sourceRepo, gitReadUrl: args.gitReadUrl, + gitWriteUrl: args.gitWriteUrl, registryPrefix: args.registryPrefix, runtimePaths: profiles.map((profile) => `deploy/gitops/g14/${runtimePathForProfile(profile)}`), publicEndpoints, diff --git a/scripts/g14-gitops-render.test.ts b/scripts/g14-gitops-render.test.ts index 24fc2f8e..8ac24a9a 100644 --- a/scripts/g14-gitops-render.test.ts +++ b/scripts/g14-gitops-render.test.ts @@ -61,6 +61,9 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots assert.match(gitopsPromoteScript, /BatchMode=yes/u); assert.match(gitopsPromoteScript, /ConnectTimeout=10/u); assert.match(gitopsPromoteScript, /git_timed "source-head-\$phase" 45 git ls-remote/u); + assert.match(gitopsPromoteScript, /source_head_url="\$\(params\.git-read-url\)"/u); + assert.match(gitopsPromoteScript, /gitops_write_url="\$\(params\.git-write-url\)"/u); + assert.match(gitopsPromoteScript, /git clone --no-checkout "\$gitops_read_url"/u); assert.match(gitopsPromoteScript, /git_timed gitops-push 120 git push origin/u); const prepareSource = taskByName(pipelineJson, "prepare-source"); @@ -74,19 +77,28 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots const pipelineGitReadParam = pipelineJson.spec.params.find((param) => param.name === "git-read-url"); assert.equal(pipelineGitReadParam?.default, "http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git"); + const pipelineGitWriteParam = pipelineJson.spec.params.find((param) => param.name === "git-write-url"); + assert.equal(pipelineGitWriteParam?.default, "http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/HWLAB.git"); const sourceDescriptor = JSON.parse(await readFile(path.join(outDir, "source.json"), "utf8")); assert.equal(sourceDescriptor.gitReadUrl, "http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git"); + assert.equal(sourceDescriptor.gitWriteUrl, "http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/HWLAB.git"); const gitMirror = await readFile(path.join(outDir, "devops-infra", "git-mirror.yaml"), "utf8"); assert.match(gitMirror, /"name": "devops-infra"/u); assert.match(gitMirror, /"name": "git-mirror-http"/u); + assert.match(gitMirror, /"name": "git-mirror-write"/u); assert.match(gitMirror, /"name": "git-mirror-sync-script"/u); + assert.match(gitMirror, /git-receive-pack/u); + assert.match(gitMirror, /last-flush\.json/u); + assert.match(gitMirror, /last-write\.json/u); assert.match(gitMirror, /ssh:\/\/git@ssh\.github\.com:443\/pikasTech\/HWLAB\.git/u); assert.doesNotMatch(gitMirror, /"kind": "CronJob"/u); assert.doesNotMatch(gitMirror, /"name": "git-mirror-hwlab-sync"/u); const gitMirrorJson = JSON.parse(gitMirror); - const gitMirrorScript = gitMirrorJson.items.find((item) => item.kind === "ConfigMap" && item.metadata?.name === "git-mirror-sync-script")?.data?.["sync.sh"] ?? ""; + const gitMirrorConfigMap = gitMirrorJson.items.find((item) => item.kind === "ConfigMap" && item.metadata?.name === "git-mirror-sync-script"); + const gitMirrorScripts = gitMirrorConfigMap?.data ?? {}; + const gitMirrorScript = gitMirrorScripts["sync.sh"] ?? ""; assert.match(gitMirrorScript, /refs\/heads\/v0\.2:refs\/mirror-stage\/heads\/v0\.2/u); assert.match(gitMirrorScript, /refs\/heads\/v0\.2-gitops:refs\/mirror-stage\/heads\/v0\.2-gitops/u); assert.match(gitMirrorScript, /refs\/heads\/G14:refs\/mirror-stage\/heads\/G14/u); @@ -96,10 +108,24 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots assert.match(gitMirrorScript, /git-mirror-sync-timing/u); assert.match(gitMirrorScript, /emit_timing fetch succeeded/u); assert.match(gitMirrorScript, /emit_timing total succeeded/u); + assert.match(gitMirrorScript, /keeps local v0\.2-gitops ahead of GitHub/u); assert.ok((gitMirrorJson.items || []).every((item) => item.kind !== "Secret")); assert.ok((gitMirrorJson.items || []).every((item) => item.kind !== "CronJob")); assert.ok((gitMirrorJson.items || []).every((item) => !item.data?.["ssh-privatekey"])); + const scriptCheckDir = path.join(outDir, "script-check"); + await mkdir(scriptCheckDir, { recursive: true }); + for (const scriptName of ["http.mjs", "sync.sh", "install-hooks.sh", "flush.sh"]) { + assert.ok(gitMirrorScripts[scriptName], `missing git mirror script ${scriptName}`); + await writeFile(path.join(scriptCheckDir, scriptName), gitMirrorScripts[scriptName]); + } + const httpCheck = spawnSync("node", ["--check", path.join(scriptCheckDir, "http.mjs")], { encoding: "utf8" }); + assert.equal(httpCheck.status, 0, httpCheck.stderr || httpCheck.stdout); + for (const scriptName of ["sync.sh", "install-hooks.sh", "flush.sh"]) { + const shellCheck = spawnSync("sh", ["-n", path.join(scriptCheckDir, scriptName)], { encoding: "utf8" }); + assert.equal(shellCheck.status, 0, `${scriptName}: ${shellCheck.stderr || shellCheck.stdout}`); + } + const removedServiceIds = [ "hwlab-router", "hwlab-tunnel-client", @@ -177,6 +203,8 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots const application = await readFile(path.join(outDir, "argocd", "application-v02.yaml"), "utf8"); assert.match(application, /"prune": true/u); + assert.match(application, /git-mirror-http\.devops-infra\.svc\.cluster\.local/u); + assert.doesNotMatch(application, /git@github\.com:pikasTech\/HWLAB\.git/u); for (const generatedFile of ["deepseek-proxy.yaml", "postgres.yaml", "g14-frpc.yaml"]) { const content = await readFile(path.join(outDir, "runtime-v02", generatedFile), "utf8"); @@ -188,6 +216,7 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots const pipelineRunSample = await readFile(path.join(outDir, "tekton-v02", "pipelinerun.sample.yaml"), "utf8"); assert.match(pipelineRunSample, /"name": "git-read-url"[\s\S]{0,140}git-mirror-http\.devops-infra\.svc\.cluster\.local/u); + assert.match(pipelineRunSample, /"name": "git-write-url"[\s\S]{0,140}git-mirror-write\.devops-infra\.svc\.cluster\.local/u); assert.match(pipelineRunSample, /"name": "revision"[\s\S]{0,80}"value": "[0-9a-f]{40}"/u); assert.match(pipelineRunSample, /"name": "source-branch"[\s\S]{0,80}"value": "v0\.2"/u); } finally {