fix: read v02 ci source from git mirror

This commit is contained in:
Codex
2026-05-29 21:45:24 +08:00
parent 9deffc415d
commit 9e4fd3f758
2 changed files with 183 additions and 8 deletions
+157 -6
View File
@@ -12,6 +12,8 @@ const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."
const defaultOutDir = "deploy/gitops/g14";
const defaultRegistryPrefix = process.env.HWLAB_G14_REGISTRY_PREFIX || "127.0.0.1:5000/hwlab";
const defaultSourceRepo = process.env.HWLAB_G14_GIT_URL || "git@github.com:pikasTech/HWLAB.git";
const defaultGitReadUrl = process.env.HWLAB_G14_GIT_READ_URL || null;
const defaultV02GitReadUrl = process.env.HWLAB_V02_GIT_READ_URL || "http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git";
const defaultBranch = process.env.HWLAB_G14_BRANCH || "G14";
const defaultGitopsBranch = process.env.HWLAB_G14_GITOPS_BRANCH || "G14-gitops";
const defaultCatalogPath = process.env.HWLAB_ARTIFACT_CATALOG_PATH || "deploy/artifact-catalog.dev.json";
@@ -23,7 +25,7 @@ const defaultV02RuntimeEndpoint = process.env.HWLAB_V02_RUNTIME_ENDPOINT || "htt
const defaultV02WebEndpoint = process.env.HWLAB_V02_WEB_ENDPOINT || "http://74.48.78.17:19666";
const defaultProxyUrl = process.env.HWLAB_G14_PROXY_URL || "http://127.0.0.1:10808";
const defaultAllProxyUrl = process.env.HWLAB_G14_ALL_PROXY_URL || "socks5h://127.0.0.1:10808";
const defaultNoProxy = process.env.HWLAB_G14_NO_PROXY || "hyueapi.com,.hyueapi.com,127.0.0.1,localhost,::1,10.0.0.0/8,10.42.0.0/16,10.43.0.0/16,192.168.0.0/16,.svc,.cluster.local,kubernetes.default.svc,registry.hwlab-ci.svc,hwlab-registry.hwlab-ci.svc";
const defaultNoProxy = process.env.HWLAB_G14_NO_PROXY || "hyueapi.com,.hyueapi.com,127.0.0.1,localhost,::1,10.0.0.0/8,10.42.0.0/16,10.43.0.0/16,192.168.0.0/16,.svc,.cluster.local,kubernetes.default.svc,registry.hwlab-ci.svc,hwlab-registry.hwlab-ci.svc,git-mirror-http,git-mirror-http.devops-infra,git-mirror-http.devops-infra.svc,git-mirror-http.devops-infra.svc.cluster.local";
const ciToolsRunnerImage = process.env.HWLAB_G14_CI_TOOLS_IMAGE || "127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1";
const buildkitRunnerImage = process.env.HWLAB_G14_BUILDKIT_IMAGE || "moby/buildkit:rootless";
const defaultDevBaseImage = process.env.HWLAB_G14_DEV_BASE_IMAGE || "127.0.0.1:5000/hwlab/hwlab-node20-base:20-bookworm-slim";
@@ -183,6 +185,7 @@ function parseArgs(argv) {
sourceBranch: defaultBranch,
gitopsBranch: defaultGitopsBranch,
sourceRepo: defaultSourceRepo,
gitReadUrl: defaultGitReadUrl,
runtimeEndpoint: defaultRuntimeEndpoint,
webEndpoint: defaultWebEndpoint,
prodRuntimeEndpoint: defaultProdRuntimeEndpoint,
@@ -203,6 +206,7 @@ function parseArgs(argv) {
else if (arg === "--source-branch") args.sourceBranch = readOption(argv, ++index, arg);
else if (arg === "--gitops-branch") args.gitopsBranch = readOption(argv, ++index, arg);
else if (arg === "--source-repo") args.sourceRepo = readOption(argv, ++index, arg);
else if (arg === "--git-read-url") args.gitReadUrl = readOption(argv, ++index, arg);
else if (arg === "--runtime-endpoint") args.runtimeEndpoint = readOption(argv, ++index, arg);
else if (arg === "--web-endpoint") args.webEndpoint = readOption(argv, ++index, arg);
else if (arg === "--prod-runtime-endpoint") args.prodRuntimeEndpoint = readOption(argv, ++index, arg);
@@ -222,7 +226,9 @@ function parseArgs(argv) {
if (args.runtimeEndpoint === defaultRuntimeEndpoint) args.runtimeEndpoint = defaultV02RuntimeEndpoint;
if (args.webEndpoint === defaultWebEndpoint) args.webEndpoint = defaultV02WebEndpoint;
if (args.imageTagMode === "short") args.imageTagMode = "full";
if (!args.gitReadUrl) args.gitReadUrl = defaultV02GitReadUrl;
}
if (!args.gitReadUrl) args.gitReadUrl = args.sourceRepo;
assert.ok(["g14", "v02"].includes(args.lane), `unknown lane ${args.lane}`);
assert.ok(["short", "full"].includes(args.imageTagMode), `unknown image tag mode ${args.imageTagMode}`);
if (args.lane === "v02") {
@@ -251,6 +257,7 @@ function usage() {
` --catalog-path PATH default: ${defaultCatalogPath}`,
" --image-tag-mode MODE short or full; v02 uses full source commit IDs",
` --source-repo URL default: ${defaultSourceRepo}`,
" --git-read-url URL read-only source URL; v02 defaults to devops-infra git mirror",
` --source-branch BRANCH default: ${defaultBranch}`,
` --gitops-branch BRANCH default: ${defaultGitopsBranch}`,
" --source-revision SHA source commit used for image tags and runtime annotations; default: git rev-parse HEAD",
@@ -964,7 +971,10 @@ function gitSshShellFunction() {
cp /workspace/git-ssh/ssh-privatekey /root/.ssh/id_rsa
chmod 600 /root/.ssh/id_rsa
timeout 10 ssh-keyscan github.com >> /root/.ssh/known_hosts 2>/dev/null || true
timeout 10 ssh-keyscan -p 443 ssh.github.com >> /root/.ssh/known_hosts 2>/dev/null || true
export GIT_SSH_COMMAND="ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ServerAliveInterval=10 -o ServerAliveCountMax=2"
git config --global url."ssh://git@ssh.github.com:443/".insteadOf "git@github.com:"
git config --global url."ssh://git@ssh.github.com:443/".insteadOf "ssh://git@github.com/"
}
git_now_ms() {
@@ -1014,19 +1024,22 @@ function prepareSourceScript() {
"curl_dependency_probe \"prepare-source-pre-npm\" \"https://registry.npmjs.org/npm/latest\"",
gitSshShellFunction(),
"git_ssh_setup",
"git_read_url=\"$(params.git-read-url)\"",
"rm -rf /workspace/source/repo",
"source_clone_started_ms=\"$(ci_now_ms)\"",
"git_timed source-clone 180 git clone --branch \"$(params.source-branch)\" \"$(params.git-url)\" /workspace/source/repo",
"git_timed source-clone 180 git clone --branch \"$(params.source-branch)\" \"$git_read_url\" /workspace/source/repo",
"cd /workspace/source/repo",
"git config --global --add safe.directory /workspace/source/repo",
"git remote set-url origin \"$git_read_url\"",
"git checkout \"$(params.revision)\"",
"test \"$(git rev-parse HEAD)\" = \"$(params.revision)\"",
"ci_timing_emit source-clone succeeded \"$source_clone_started_ms\"",
"catalog_path=\"$(params.catalog-path)\"",
"mkdir -p \"$(dirname \"$catalog_path\")\"",
"catalog_fetch_started_ms=\"$(ci_now_ms)\"",
"if git_timed catalog-ls-remote 45 git ls-remote --exit-code --heads origin \"$(params.gitops-branch)\" >/dev/null; then",
" git_timed catalog-fetch 90 git fetch --depth 1 origin \"$(params.gitops-branch)\" >/dev/null || true",
"if git_timed catalog-ls-remote 45 git ls-remote --exit-code --heads \"$(params.git-url)\" \"$(params.gitops-branch)\" >/dev/null; then",
" git remote set-url gitops-catalog \"$(params.git-url)\" 2>/dev/null || git remote add gitops-catalog \"$(params.git-url)\"",
" git_timed catalog-fetch 90 git fetch --depth 1 gitops-catalog \"$(params.gitops-branch)\" >/dev/null || true",
" if git cat-file -e FETCH_HEAD:\"$catalog_path\" 2>/dev/null; then",
" git show FETCH_HEAD:\"$catalog_path\" > /tmp/hwlab-gitops-artifact-catalog.json",
" if node -e 'const fs=require(\"fs\"); const ids=(doc)=>(doc.services||[]).map((s)=>s.serviceId); const selected=(process.argv[3]||\"\").split(\",\").filter(Boolean); const gitops=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const deploy=JSON.parse(fs.readFileSync(process.argv[2],\"utf8\")); const expected=selected.length ? selected : ids(deploy); if (JSON.stringify(ids(gitops)) !== JSON.stringify(expected)) process.exit(42);' /tmp/hwlab-gitops-artifact-catalog.json deploy/deploy.json \"$(params.services)\"; then",
@@ -1433,8 +1446,9 @@ for tool in node npm git python3 ssh curl timeout; do command -v "$tool" >/dev/n
${gitSshShellFunction()}
git_ssh_setup
workdir="$(mktemp -d)"
git_timed control-plane-clone 180 git clone --depth 1 --branch "$SOURCE_BRANCH" "$GIT_URL" "$workdir/repo"
git_timed control-plane-clone 180 git clone --depth 1 --branch "$SOURCE_BRANCH" "$GIT_READ_URL" "$workdir/repo"
cd "$workdir/repo"
git remote set-url origin "$GIT_READ_URL"
revision="$(git rev-parse HEAD)"
: "\${LANE:?LANE is required}"
: "\${CATALOG_PATH:?CATALOG_PATH is required}"
@@ -1569,8 +1583,9 @@ for tool in node git ssh curl timeout; do command -v "$tool" >/dev/null 2>&1 ||
${gitSshShellFunction()}
git_ssh_setup
workdir="$(mktemp -d)"
git_timed poller-clone 180 git clone --depth 1 --branch "$SOURCE_BRANCH" "$GIT_URL" "$workdir/repo"
git_timed poller-clone 180 git clone --depth 1 --branch "$SOURCE_BRANCH" "$GIT_READ_URL" "$workdir/repo"
cd "$workdir/repo"
git remote set-url origin "$GIT_READ_URL"
revision="$(git rev-parse HEAD)"
subject="$(git log -1 --pretty=%s)"
case "$subject" in
@@ -1660,6 +1675,7 @@ const pipelineRun = {
},
params: [
{ name: "git-url", value: requiredEnv("GIT_URL") },
{ name: "git-read-url", value: requiredEnv("GIT_READ_URL") },
{ name: "source-branch", value: requiredEnv("SOURCE_BRANCH") },
{ name: "gitops-branch", value: requiredEnv("GITOPS_BRANCH") },
{ name: "lane", value: requiredEnv("LANE") },
@@ -2297,6 +2313,7 @@ function tektonPipeline(args = { lane: "g14" }) {
spec: {
params: [
{ name: "git-url", type: "string", default: defaultSourceRepo },
{ name: "git-read-url", type: "string", default: args.gitReadUrl },
{ name: "source-branch", type: "string", default: args.sourceBranch },
{ name: "gitops-branch", type: "string", default: args.gitopsBranch },
{ name: "lane", type: "string", default: settings.lane },
@@ -2322,6 +2339,7 @@ function tektonPipeline(args = { lane: "g14" }) {
taskSpec: {
params: [
{ name: "git-url" },
{ name: "git-read-url" },
{ name: "source-branch" },
{ name: "gitops-branch" },
{ name: "lane" },
@@ -2336,6 +2354,7 @@ function tektonPipeline(args = { lane: "g14" }) {
},
params: [
{ name: "git-url", value: "$(params.git-url)" },
{ name: "git-read-url", value: "$(params.git-read-url)" },
{ name: "source-branch", value: "$(params.source-branch)" },
{ name: "gitops-branch", value: "$(params.gitops-branch)" },
{ name: "lane", value: "$(params.lane)" },
@@ -2415,6 +2434,7 @@ function tektonPipelineRunTemplate({ source, args }) {
},
params: [
{ name: "git-url", value: args.sourceRepo },
{ name: "git-read-url", value: args.gitReadUrl },
{ name: "source-branch", value: args.sourceBranch },
{ name: "gitops-branch", value: args.gitopsBranch },
{ name: "lane", value: settings.lane },
@@ -2486,6 +2506,7 @@ function tektonPollerCronJob(args) {
{ name: "IMAGE_TAG_MODE", value: args.imageTagMode || settings.imageTagMode },
{ name: "RUNTIME_PATH", value: `deploy/gitops/g14/${runtimePathForProfile(settings.profile)}` },
{ name: "GIT_URL", value: args.sourceRepo },
{ name: "GIT_READ_URL", value: args.gitReadUrl },
{ name: "SOURCE_BRANCH", value: args.sourceBranch },
{ name: "GITOPS_BRANCH", value: args.gitopsBranch },
{ name: "REGISTRY_PREFIX", value: args.registryPrefix },
@@ -2558,6 +2579,7 @@ function tektonControlPlaneReconcilerCronJob(args) {
{ name: "FIELD_MANAGER", value: settings.controlPlaneReconcilerName },
{ name: "RECONCILE_EVENT", value: `${settings.gitopsTarget}-control-plane-reconciled` },
{ name: "GIT_URL", value: args.sourceRepo },
{ name: "GIT_READ_URL", value: args.gitReadUrl },
{ name: "SOURCE_BRANCH", value: args.sourceBranch },
{ name: "GITOPS_BRANCH", value: args.gitopsBranch },
{ name: "REGISTRY_PREFIX", value: args.registryPrefix }
@@ -2618,6 +2640,132 @@ function registryManifest() {
};
}
function devopsInfraGitMirrorManifest() {
const labels = {
"app.kubernetes.io/name": "git-mirror",
"app.kubernetes.io/part-of": "devops-infra"
};
return {
apiVersion: "v1",
kind: "List",
items: [
{ apiVersion: "v1", kind: "Namespace", metadata: { name: "devops-infra" } },
{
apiVersion: "v1",
kind: "PersistentVolumeClaim",
metadata: { name: "git-mirror-cache", namespace: "devops-infra", labels },
spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "20Gi" } } }
},
{
apiVersion: "v1",
kind: "ConfigMap",
metadata: { name: "git-mirror-sync-script", namespace: "devops-infra", labels },
data: {
"sync.sh": `#!/bin/sh
set -eu
repo_url="ssh://git@ssh.github.com:443/pikasTech/HWLAB.git"
repo_path="/cache/pikasTech/HWLAB.git"
lock_dir="/cache/.hwlab-sync.lock"
mkdir -p /cache/pikasTech /root/.ssh
if ! mkdir "\${lock_dir}" 2>/dev/null; then
echo "git mirror sync already running"
exit 0
fi
trap 'rmdir "\${lock_dir}" 2>/dev/null || true' EXIT INT TERM
cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa
chmod 0400 /root/.ssh/id_rsa
export GIT_SSH_COMMAND="ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=10 -o ServerAliveInterval=5 -o ServerAliveCountMax=1"
if [ -d "\${repo_path}/objects" ] && [ -f "\${repo_path}/HEAD" ]; then
git -C "\${repo_path}" remote set-url origin "\${repo_url}"
timeout 180 git -C "\${repo_path}" remote update --prune
else
rm -rf "\${repo_path}"
timeout 240 git clone --mirror "\${repo_url}" "\${repo_path}"
fi
git -C "\${repo_path}" update-server-info
git -C "\${repo_path}" rev-parse refs/heads/v0.2 | tee /cache/HWLAB-v0.2.head
date -u +%Y-%m-%dT%H:%M:%SZ > /cache/HWLAB.last-sync
`
}
},
{
apiVersion: "apps/v1",
kind: "Deployment",
metadata: { name: "git-mirror-http", namespace: "devops-infra", labels: { ...labels, "app.kubernetes.io/component": "read-only-http" } },
spec: {
replicas: 1,
selector: { matchLabels: { "app.kubernetes.io/name": "git-mirror", "app.kubernetes.io/component": "read-only-http" } },
template: {
metadata: { labels: { ...labels, "app.kubernetes.io/component": "read-only-http" } },
spec: {
containers: [{
name: "http",
image: ciToolsRunnerImage,
imagePullPolicy: "IfNotPresent",
workingDir: "/cache",
command: ["python3"],
args: ["-m", "http.server", "8080", "--bind", "0.0.0.0"],
ports: [{ name: "http", containerPort: 8080 }],
readinessProbe: { httpGet: { path: "/pikasTech/HWLAB.git/HEAD", port: "http" }, initialDelaySeconds: 5, periodSeconds: 10 },
livenessProbe: { httpGet: { path: "/", port: "http" }, initialDelaySeconds: 10, periodSeconds: 30 },
volumeMounts: [{ name: "cache", mountPath: "/cache", readOnly: true }]
}],
volumes: [{ name: "cache", persistentVolumeClaim: { claimName: "git-mirror-cache", readOnly: true } }]
}
}
}
},
{
apiVersion: "v1",
kind: "Service",
metadata: { name: "git-mirror-http", namespace: "devops-infra", labels: { ...labels, "app.kubernetes.io/component": "read-only-http" } },
spec: { selector: { "app.kubernetes.io/name": "git-mirror", "app.kubernetes.io/component": "read-only-http" }, ports: [{ name: "http", port: 80, targetPort: "http" }] }
},
{
apiVersion: "batch/v1",
kind: "CronJob",
metadata: { name: "git-mirror-hwlab-sync", namespace: "devops-infra", labels: { ...labels, "app.kubernetes.io/component": "sync-controller" } },
spec: {
schedule: "*/2 * * * *",
concurrencyPolicy: "Forbid",
successfulJobsHistoryLimit: 3,
failedJobsHistoryLimit: 3,
jobTemplate: {
spec: {
backoffLimit: 1,
activeDeadlineSeconds: 300,
template: {
metadata: { labels: { ...labels, "app.kubernetes.io/component": "sync-controller" } },
spec: {
restartPolicy: "Never",
hostNetwork: true,
dnsPolicy: "ClusterFirstWithHostNet",
volumes: [
{ name: "cache", persistentVolumeClaim: { claimName: "git-mirror-cache" } },
{ name: "git-ssh", secret: { secretName: "git-mirror-github-ssh", defaultMode: 0o400 } },
{ name: "script", configMap: { name: "git-mirror-sync-script", defaultMode: 0o755 } }
],
containers: [{
name: "sync",
image: ciToolsRunnerImage,
imagePullPolicy: "IfNotPresent",
command: ["/script/sync.sh"],
volumeMounts: [
{ name: "cache", mountPath: "/cache" },
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
{ name: "script", mountPath: "/script", readOnly: true }
]
}]
}
}
}
}
}
}
]
};
}
function argoProject(args = { lane: "g14" }) {
if (args.lane === "v02") {
return {
@@ -3227,6 +3375,7 @@ This directory is generated by \`scripts/g14-gitops-render.mjs --lane v02\` from
- Target: additive G14 v0.2 lane only.
- CI: Tekton Pipeline \`hwlab-ci/hwlab-v02-ci-image-publish\`.
- Source reads: v0.2 CI source clone uses devops-infra read-only mirror \`${args.gitReadUrl}\`; canonical repo \`${args.sourceRepo}\` remains the identity and GitOps write path.
- Artifact contract: runtime images come from \`${args.catalogPath}\`; source branch \`v0.2\` does not track this generated catalog.
- Branch split: source branch \`${args.sourceBranch}\` remains human-authored source; generated desired state is promoted to \`${args.gitopsBranch}\`.
- CD: Argo CD Application \`argocd/hwlab-g14-v02\` consumes \`${args.gitopsBranch}:deploy/gitops/g14/runtime-v02\` and deploys only to namespace \`hwlab-v02\`.
@@ -3290,6 +3439,7 @@ async function plannedFiles(args) {
gitopsBranch: args.gitopsBranch,
lane: settings.lane,
sourceRepo: args.sourceRepo,
gitReadUrl: args.gitReadUrl,
registryPrefix: args.registryPrefix,
runtimePaths: profiles.map((profile) => `deploy/gitops/g14/${runtimePathForProfile(profile)}`),
publicEndpoints,
@@ -3307,6 +3457,7 @@ async function plannedFiles(args) {
putJson("source.json", sourceDescriptor);
putText("README.md", readme({ source, args }));
putJson("registry/registry.yaml", registryManifest());
if (args.lane === "v02") putJson("devops-infra/git-mirror.yaml", devopsInfraGitMirrorManifest());
putJson(`${settings.tektonDir}/rbac.yaml`, tektonRbac(args));
putJson(`${settings.tektonDir}/pipeline.yaml`, tektonPipeline(args));
putJson(`${settings.tektonDir}/poller.yaml`, tektonPollerCronJob(args));