From b97465bbd7eb077e8fa20221c8518fe863c6c47f Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 29 May 2026 21:07:34 +0800 Subject: [PATCH] fix: bound v02 ci git operations --- scripts/g14-gitops-render.mjs | 99 ++++++++++++++++++++----------- scripts/g14-gitops-render.test.ts | 9 +++ 2 files changed, 75 insertions(+), 33 deletions(-) diff --git a/scripts/g14-gitops-render.mjs b/scripts/g14-gitops-render.mjs index 1bae54e9..b69a3318 100644 --- a/scripts/g14-gitops-render.mjs +++ b/scripts/g14-gitops-render.mjs @@ -958,6 +958,45 @@ curl_dependency_probe() { }`; } +function gitSshShellFunction() { + return `git_ssh_setup() { + mkdir -p /root/.ssh + cp /workspace/git-ssh/ssh-privatekey /root/.ssh/id_rsa + chmod 600 /root/.ssh/id_rsa + timeout 10 ssh-keyscan github.com >> /root/.ssh/known_hosts 2>/dev/null || true + export GIT_SSH_COMMAND="ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ServerAliveInterval=10 -o ServerAliveCountMax=2" +} + +git_now_ms() { + node -e 'console.log(Date.now())' 2>/dev/null || date +%s000 +} + +git_timed() { + phase="$1" + timeout_seconds="$2" + shift 2 + started_ms="$(git_now_ms)" + echo '{"event":"git-operation","phase":"'"$phase"'","status":"started","timeoutSeconds":'"$timeout_seconds"'}' >&2 + set +e + timeout "$timeout_seconds" "$@" + status="$?" + set -e + finished_ms="$(git_now_ms)" + duration_ms="$((finished_ms - started_ms))" + if [ "$status" -eq 0 ]; then + echo '{"event":"git-operation","phase":"'"$phase"'","status":"succeeded","durationMs":'"$duration_ms"'}' >&2 + return 0 + fi + if [ "$status" -eq 124 ] || [ "$status" -eq 137 ]; then + echo '{"event":"git-operation","phase":"'"$phase"'","status":"failed","reason":"timeout","durationMs":'"$duration_ms"',"timeoutSeconds":'"$timeout_seconds"'}' >&2 + else + echo '{"event":"git-operation","phase":"'"$phase"'","status":"failed","exitCode":'"$status"',"durationMs":'"$duration_ms"'}' >&2 + fi + return "$status" +} +`; +} + function prepareSourceScript() { const lines = [ "#!/bin/sh", @@ -970,16 +1009,14 @@ function prepareSourceScript() { dependencyProxyProbeScript("prepare-source-proxy-preflight", "http://deb.debian.org/debian/dists/bookworm/InRelease"), curlDownloadProbeFunction(), `echo '{"event":"ci-base-image","phase":"prepare-source","image":"${ciToolsRunnerImage}","policy":"no-runtime-apk"}'`, - "for tool in node npm git python3 ssh curl; do command -v \"$tool\" >/dev/null 2>&1 || { echo '{\"event\":\"ci-base-image\",\"ok\":false,\"reason\":\"missing-tool\",\"tool\":\"'\"$tool\"'\"}'; exit 31; }; done", + "for tool in node npm git python3 ssh curl timeout; do command -v \"$tool\" >/dev/null 2>&1 || { echo '{\"event\":\"ci-base-image\",\"ok\":false,\"reason\":\"missing-tool\",\"tool\":\"'\"$tool\"'\"}'; exit 31; }; done", "node -e 'console.log(JSON.stringify({event:\"ci-base-image\",phase:\"prepare-source\",ok:true,node:process.version}))'", "curl_dependency_probe \"prepare-source-pre-npm\" \"https://registry.npmjs.org/npm/latest\"", - "mkdir -p /root/.ssh", - "cp /workspace/git-ssh/ssh-privatekey /root/.ssh/id_rsa", - "chmod 600 /root/.ssh/id_rsa", - "ssh-keyscan github.com >> /root/.ssh/known_hosts 2>/dev/null", + gitSshShellFunction(), + "git_ssh_setup", "rm -rf /workspace/source/repo", "source_clone_started_ms=\"$(ci_now_ms)\"", - "git clone --branch \"$(params.source-branch)\" \"$(params.git-url)\" /workspace/source/repo", + "git_timed source-clone 180 git clone --branch \"$(params.source-branch)\" \"$(params.git-url)\" /workspace/source/repo", "cd /workspace/source/repo", "git config --global --add safe.directory /workspace/source/repo", "git checkout \"$(params.revision)\"", @@ -988,8 +1025,8 @@ function prepareSourceScript() { "catalog_path=\"$(params.catalog-path)\"", "mkdir -p \"$(dirname \"$catalog_path\")\"", "catalog_fetch_started_ms=\"$(ci_now_ms)\"", - "if git ls-remote --exit-code --heads origin \"$(params.gitops-branch)\" >/dev/null 2>&1; then", - " git fetch --depth 1 origin \"$(params.gitops-branch)\" >/dev/null 2>&1 || true", + "if git_timed catalog-ls-remote 45 git ls-remote --exit-code --heads origin \"$(params.gitops-branch)\" >/dev/null; then", + " git_timed catalog-fetch 90 git fetch --depth 1 origin \"$(params.gitops-branch)\" >/dev/null || true", " if git cat-file -e FETCH_HEAD:\"$catalog_path\" 2>/dev/null; then", " git show FETCH_HEAD:\"$catalog_path\" > /tmp/hwlab-gitops-artifact-catalog.json", " if node -e 'const fs=require(\"fs\"); const ids=(doc)=>(doc.services||[]).map((s)=>s.serviceId); const selected=(process.argv[3]||\"\").split(\",\").filter(Boolean); const gitops=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const deploy=JSON.parse(fs.readFileSync(process.argv[2],\"utf8\")); const expected=selected.length ? selected : ids(deploy); if (JSON.stringify(ids(gitops)) !== JSON.stringify(expected)) process.exit(42);' /tmp/hwlab-gitops-artifact-catalog.json deploy/deploy.json \"$(params.services)\"; then", @@ -1222,19 +1259,21 @@ export HWLAB_TEKTON_TASK="gitops-promote" export HWLAB_SOURCE_REVISION="$(params.revision)" ${dependencyProxyProbeScript("gitops-promote-proxy-preflight", "http://deb.debian.org/debian/dists/bookworm/InRelease")} echo '{"event":"ci-base-image","phase":"gitops-promote","image":"${ciToolsRunnerImage}","policy":"no-runtime-apt"}' -for tool in node npm git ssh curl; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"gitops-promote","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done -mkdir -p /root/.ssh -cp /workspace/git-ssh/ssh-privatekey /root/.ssh/id_rsa -chmod 600 /root/.ssh/id_rsa -ssh-keyscan github.com >> /root/.ssh/known_hosts 2>/dev/null || true -export GIT_SSH_COMMAND="ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new" +for tool in node npm git ssh curl timeout; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"gitops-promote","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done +${gitSshShellFunction()} +git_ssh_setup cd /workspace/source/repo test "$(git rev-parse HEAD)" = "$(params.revision)" check_source_head() { phase="$1" expected="\${2:-$(params.revision)}" - latest="$(git ls-remote "$(params.git-url)" "refs/heads/$(params.source-branch)" | cut -f1)" + latest_file="$(mktemp)" + if ! git_timed "source-head-$phase" 45 git ls-remote "$(params.git-url)" "refs/heads/$(params.source-branch)" > "$latest_file"; then + echo '{"status":"failed","reason":"source-head-check-failed","phase":"'"$phase"'","sourceBranch":"'"$(params.source-branch)"'","sourceRevision":"'"$(params.revision)"'"}' + exit 1 + fi + latest="$(cut -f1 "$latest_file" | head -n 1)" if [ -z "$latest" ]; then echo '{"status":"failed","reason":"source-head-unresolved","phase":"'"$phase"'","sourceBranch":"'"$(params.source-branch)"'","sourceRevision":"'"$(params.revision)"'"}' exit 1 @@ -1344,10 +1383,10 @@ ci_timing_emit gitops-render succeeded "$gitops_render_started_ms" check_source_head before-push workdir="$(mktemp -d)" gitops_clone_started_ms="$(ci_now_ms)" -git clone --no-checkout "$(params.git-url)" "$workdir/gitops" +git_timed gitops-clone 180 git clone --no-checkout "$(params.git-url)" "$workdir/gitops" cd "$workdir/gitops" -if git ls-remote --exit-code --heads origin "$(params.gitops-branch)" >/dev/null 2>&1; then - git fetch origin "$(params.gitops-branch)" +if git_timed gitops-branch-ls 45 git ls-remote --exit-code --heads origin "$(params.gitops-branch)" >/dev/null; then + git_timed gitops-fetch 90 git fetch origin "$(params.gitops-branch)" git checkout -B "$(params.gitops-branch)" "origin/$(params.gitops-branch)" if [ "$lane" = "v02" ]; then rm -rf "$runtime_path" "$catalog_path"; else rm -rf deploy/gitops/g14 "$catalog_path"; fi else @@ -1378,7 +1417,7 @@ gitops_commit_started_ms="$(ci_now_ms)" git commit -m "chore: promote G14 GitOps source $short" ci_timing_emit gitops-commit succeeded "$gitops_commit_started_ms" gitops_push_started_ms="$(ci_now_ms)" -git push origin "HEAD:$(params.gitops-branch)" +git_timed gitops-push 120 git push origin "HEAD:$(params.gitops-branch)" ci_timing_emit gitops-push succeeded "$gitops_push_started_ms" argo_hard_refresh_v02 echo '{"status":"pushed","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}' @@ -1390,14 +1429,11 @@ function controlPlaneReconcileScript() { set -eu ${dependencyProxyProbeScript("control-plane-proxy-preflight", "http://deb.debian.org/debian/dists/bookworm/InRelease")} echo '{"event":"ci-base-image","phase":"control-plane-reconciler","image":"${ciToolsRunnerImage}","policy":"no-runtime-apk"}' -for tool in node npm git python3 ssh curl; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"control-plane-reconciler","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done -mkdir -p /root/.ssh -cp /workspace/git-ssh/ssh-privatekey /root/.ssh/id_rsa -chmod 600 /root/.ssh/id_rsa -ssh-keyscan github.com >> /root/.ssh/known_hosts 2>/dev/null || true -export GIT_SSH_COMMAND="ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new" +for tool in node npm git python3 ssh curl timeout; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"control-plane-reconciler","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done +${gitSshShellFunction()} +git_ssh_setup workdir="$(mktemp -d)" -git clone --depth 1 --branch "$SOURCE_BRANCH" "$GIT_URL" "$workdir/repo" +git_timed control-plane-clone 180 git clone --depth 1 --branch "$SOURCE_BRANCH" "$GIT_URL" "$workdir/repo" cd "$workdir/repo" revision="$(git rev-parse HEAD)" : "\${LANE:?LANE is required}" @@ -1529,14 +1565,11 @@ function pollerScript() { set -eu ${dependencyProxyProbeScript("poller-proxy-preflight", "http://deb.debian.org/debian/dists/bookworm/InRelease")} echo '{"event":"ci-base-image","phase":"poller","image":"${ciToolsRunnerImage}","policy":"no-runtime-apk"}' -for tool in node git ssh curl; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"poller","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done -mkdir -p /root/.ssh -cp /workspace/git-ssh/ssh-privatekey /root/.ssh/id_rsa -chmod 600 /root/.ssh/id_rsa -ssh-keyscan github.com >> /root/.ssh/known_hosts 2>/dev/null || true -export GIT_SSH_COMMAND="ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new" +for tool in node git ssh curl timeout; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"poller","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done +${gitSshShellFunction()} +git_ssh_setup workdir="$(mktemp -d)" -git clone --depth 1 --branch "$SOURCE_BRANCH" "$GIT_URL" "$workdir/repo" +git_timed poller-clone 180 git clone --depth 1 --branch "$SOURCE_BRANCH" "$GIT_URL" "$workdir/repo" cd "$workdir/repo" revision="$(git rev-parse HEAD)" subject="$(git log -1 --pretty=%s)" diff --git a/scripts/g14-gitops-render.test.ts b/scripts/g14-gitops-render.test.ts index 795490f8..114295df 100644 --- a/scripts/g14-gitops-render.test.ts +++ b/scripts/g14-gitops-render.test.ts @@ -58,6 +58,15 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots assert.match(gitopsPromoteScript, /argo_hard_refresh_v02\(\) \{/u); assert.match(gitopsPromoteScript, /argocd\.argoproj\.io\/refresh": "hard"/u); assert.match(gitopsPromoteScript, /applications\/" \+ encodeURIComponent\(application\) \+ "\?fieldManager=hwlab-v02-gitops-promote/u); + assert.match(gitopsPromoteScript, /BatchMode=yes/u); + assert.match(gitopsPromoteScript, /ConnectTimeout=10/u); + assert.match(gitopsPromoteScript, /git_timed "source-head-\$phase" 45 git ls-remote/u); + assert.match(gitopsPromoteScript, /git_timed gitops-push 120 git push origin/u); + + const prepareSource = taskByName(pipelineJson, "prepare-source"); + const prepareSourceScript = prepareSource.taskSpec.steps[0].script; + assert.match(prepareSourceScript, /git_timed source-clone 180 git clone/u); + assert.match(prepareSourceScript, /git_timed catalog-ls-remote 45 git ls-remote/u); const removedServiceIds = [ "hwlab-router",