From 230ac9dcb8e5c58f81cc8de2f9ddda2da1194cf0 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 25 May 2026 13:13:45 +0800 Subject: [PATCH] fix: harden G14 CI dependency diagnostics --- docs/reference/g14-gitops-cicd.md | 10 ++ scripts/g14-gitops-render.mjs | 127 +++++++++++++++++- scripts/src/dev-cloud-workbench-smoke-lib.mjs | 51 ++----- 3 files changed, 146 insertions(+), 42 deletions(-) diff --git a/docs/reference/g14-gitops-cicd.md b/docs/reference/g14-gitops-cicd.md index ffc8ddd8..91ffcaed 100644 --- a/docs/reference/g14-gitops-cicd.md +++ b/docs/reference/g14-gitops-cicd.md @@ -22,6 +22,7 @@ G14 是 HWLAB 的旁路 DEV 集群目标。G14 CI/CD 必须只作用于 G14 k3s - 默认 GitOps 生成分支是 `G14-gitops`；Pipeline 成功后把本次 source commit 对应的 `deploy/gitops/g14/**` 推送到该分支，避免把生成提交继续写回 `G14`。 - 默认 registry prefix 是 `127.0.0.1:5000/hwlab`，用于 G14 单节点 k3s 的 node-local registry。 - 默认 CI/CD proxy 是 G14 本机 `http://127.0.0.1:10808` / `socks5h://127.0.0.1:10808`。Tekton CI step、Docker-in-Docker sidecar 和 publish step 都注入 proxy/no_proxy；publish step 必须先在 DIND 内 `docker pull $HWLAB_DEV_BASE_IMAGE`，再进入 DEV base-image preflight、build 和 registry push，保证 npm、Playwright Chromium 安装、base image 拉取和 registry push 走 G14 本机代理边界。CI step 固定使用 Node 22 Debian 镜像，避免 Playwright 官方镜像内 Node 版本漂移影响 `node --test`。 +- 任何依赖下载阶段都必须有可观测诊断。生成的 Tekton 脚本在 apt/apk、npm、Playwright browser、Docker base image pull 和 GitOps promote 之前输出结构化 `dependency-proxy-probe`、`dependency-curl-probe` 或 `dependency-download-*` 日志，至少包含 phase、目标 URL/镜像、脱敏 proxy、首包耗时、总耗时、下载字节数和速度。CI/CD 卡在下载时，先用这些日志判断是 proxy 不可达、目标源慢、DNS/首包慢还是下载吞吐低，再决定是否切换 G14 代理节点或预热镜像；不能只凭 PipelineRun Running 时长判断业务测试失败。 常用命令： @@ -38,6 +39,15 @@ G14 不要求 GitHub webhook 或 GitHub Actions 配置。`hwlab-ci/hwlab-g14-bra Pipeline 的标准路径是：`G14` source commit -> `CI.json` Tekton 测试 -> commit-tagged image push 到 G14 registry -> render `deploy/gitops/g14/**` -> push `G14-gitops` -> Argo CD 同步 runtime。CD 只消费已经构建好的镜像和 Git desired state，不在 Argo CD 内构建镜像。 +下载阶段排障命令： + +```sh +KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-ci logs pod/ --all-containers --tail=400 \ + | grep -E 'dependency-(proxy|curl|download)' +``` + +如果 probe 显示 `proxy-env-missing`、`proxy-connect-failed`、`timeout` 或速度长期接近 0，应先按 `/root/docs/vpn-proxy-ops.md` 检查 v2rayN/Hysteria，再重跑 polling。若 probe 正常但后续测试失败，按对应 `CI.json` 命令和业务检查排障。 + ## 集群资源 - `hwlab-ci`：registry、Tekton runner RBAC、Polling CronJob、Pipeline/PipelineRun 所在 namespace。 diff --git a/scripts/g14-gitops-render.mjs b/scripts/g14-gitops-render.mjs index a52e8ee1..af026605 100644 --- a/scripts/g14-gitops-render.mjs +++ b/scripts/g14-gitops-render.mjs @@ -295,17 +295,132 @@ function shellSingleQuote(value) { return `'${String(value).replace(/'/g, `'"'"'`)}'`; } +function dependencyProxyProbeScript(phase, targetUrl) { + return `HWLAB_PROXY_PROBE_PHASE=${shellSingleQuote(phase)} HWLAB_PROXY_PROBE_URL=${shellSingleQuote(targetUrl)} node <<'NODE' +const net = require("node:net"); + +const phase = process.env.HWLAB_PROXY_PROBE_PHASE || "unknown"; +const target = new URL(process.env.HWLAB_PROXY_PROBE_URL || "http://deb.debian.org/debian/dists/bookworm/InRelease"); +const proxyRaw = process.env.HTTP_PROXY || process.env.http_proxy || ""; + +function redactProxy(value) { + return String(value || "") + .replace(/(https?:\/\/)([^/@]+)@/u, "$1***@") + .replace(/(socks5h?:\/\/)([^/@]+)@/u, "$1***@"); +} + +function emit(payload, exitCode = 0) { + console.log(JSON.stringify({ + event: "dependency-proxy-probe", + phase, + target: target.href, + proxy: redactProxy(proxyRaw), + ...payload + })); + if (exitCode) process.exit(exitCode); +} + +if (!proxyRaw) emit({ ok: false, reason: "proxy-env-missing" }, 21); + +let proxy; +try { + proxy = new URL(proxyRaw); +} catch (error) { + emit({ ok: false, reason: "proxy-url-invalid", error: error.message }, 22); +} + +if (proxy.protocol !== "http:" && proxy.protocol !== "https:") { + emit({ ok: false, reason: "http-proxy-required", protocol: proxy.protocol }, 23); +} + +const startedAt = Date.now(); +const socket = net.createConnection({ host: proxy.hostname, port: Number(proxy.port || (proxy.protocol === "https:" ? 443 : 80)) }); +let bytes = 0; +let firstByteMs = null; +let responseHead = ""; +let finished = false; + +const timeout = setTimeout(() => { + if (finished) return; + finished = true; + socket.destroy(); + emit({ ok: false, reason: "timeout", timeoutMs: 15000 }, 24); +}, 15000); + +socket.on("connect", () => { + socket.write("GET " + target.href + " HTTP/1.1\\r\\nHost: " + target.host + "\\r\\nUser-Agent: hwlab-g14-proxy-probe\\r\\nConnection: close\\r\\n\\r\\n"); +}); + +socket.on("data", (chunk) => { + if (firstByteMs === null) firstByteMs = Date.now() - startedAt; + bytes += chunk.length; + if (responseHead.length < 240) responseHead += chunk.toString("utf8", 0, Math.min(chunk.length, 240 - responseHead.length)); +}); + +socket.on("end", () => { + if (finished) return; + finished = true; + clearTimeout(timeout); + const totalMs = Date.now() - startedAt; + const statusLine = responseHead.split("\\r\\n", 1)[0] || ""; + const statusCode = Number(statusLine.split(" ")[1] || 0); + const ok = statusLine.startsWith("HTTP/") && statusCode >= 200 && statusCode < 400; + emit({ + ok, + reason: ok ? null : "bad-http-status", + statusLine, + statusCode, + firstByteMs, + totalMs, + bytes, + speedBytesPerSecond: totalMs > 0 ? Math.round(bytes * 1000 / totalMs) : 0 + }, ok ? 0 : 25); +}); + +socket.on("error", (error) => { + if (finished) return; + finished = true; + clearTimeout(timeout); + emit({ ok: false, reason: "proxy-connect-failed", error: error.message, totalMs: Date.now() - startedAt }, 26); +}); +NODE`; +} + +function curlDownloadProbeFunction() { + return `redact_proxy_value() { + printf '%s' "\${1:-}" | sed -E 's#(https?://)[^/@]+@#\\1***@#; s#(socks5h?://)[^/@]+@#\\1***@#' +} + +curl_dependency_probe() { + phase="$1" + url="$2" + proxy="\${HTTPS_PROXY:-\${https_proxy:-\${HTTP_PROXY:-\${http_proxy:-}}}}" + if [ -z "$proxy" ]; then + echo '{"event":"dependency-curl-probe","phase":"'"$phase"'","ok":false,"reason":"proxy-env-missing"}' + return 21 + fi + safe_proxy="$(redact_proxy_value "$proxy")" + echo '{"event":"dependency-curl-probe-start","phase":"'"$phase"'","proxy":"'"$safe_proxy"'","target":"'"$url"'"}' + curl -sS -L --range 0-1048575 -o /dev/null --connect-timeout 15 --max-time 60 --proxy "$proxy" \ + -w '{"event":"dependency-curl-probe","phase":"'"$phase"'","http_code":"%{http_code}","time_namelookup":%{time_namelookup},"time_connect":%{time_connect},"time_starttransfer":%{time_starttransfer},"time_total":%{time_total},"speed_download":%{speed_download},"size_download":%{size_download},"remote_ip":"%{remote_ip}"}\n' \ + "$url" +}`; +} + function ciJsonScript(ci) { const commands = ci.commands ?? []; const lines = [ "#!/bin/sh", "set -eu", + dependencyProxyProbeScript("ci-json-pre-apt", "http://deb.debian.org/debian/dists/bookworm/InRelease"), "if ! command -v git >/dev/null 2>&1 || ! command -v python3 >/dev/null 2>&1 || ! command -v ssh >/dev/null 2>&1; then", " export DEBIAN_FRONTEND=noninteractive", " apt-get update", - " apt-get install -y --no-install-recommends git python3 openssh-client ca-certificates", + " apt-get install -y --no-install-recommends git python3 openssh-client ca-certificates curl", " rm -rf /var/lib/apt/lists/*", "fi", + curlDownloadProbeFunction(), + "curl_dependency_probe \"ci-json-pre-npm\" \"https://registry.npmjs.org/npm/latest\"", "mkdir -p /root/.ssh", "cp /workspace/git-ssh/ssh-privatekey /root/.ssh/id_rsa", "chmod 600 /root/.ssh/id_rsa", @@ -316,6 +431,8 @@ function ciJsonScript(ci) { "git checkout \"$(params.revision)\"", "test \"$(git rev-parse HEAD)\" = \"$(params.revision)\"", "npm ci --ignore-scripts", + "playwright_browser_probe_url=\"$(node -e 'const fs=require(\"node:fs\"); let url=\"\"; try { const p=require.resolve(\"playwright-core/browsers.json\"); const data=JSON.parse(fs.readFileSync(p,\"utf8\")); const b=data.browsers.find((item)=>item.name===\"chromium\"); if (b?.revision) url=\"https://playwright.azureedge.net/builds/chromium/\"+b.revision+\"/chromium-linux.zip\"; } catch {} process.stdout.write(url);' 2>/dev/null || true)\"", + "if [ -n \"$playwright_browser_probe_url\" ]; then curl_dependency_probe \"ci-json-pre-playwright\" \"$playwright_browser_probe_url\"; fi", "npx playwright install --with-deps chromium", `echo ${shellSingleQuote(`CI.json ${ci.name ?? "hwlab"} command count=${commands.length}`)}` ]; @@ -329,6 +446,7 @@ function ciJsonScript(ci) { function publishScript() { return `#!/bin/sh set -eu +${dependencyProxyProbeScript("image-publish-pre-apk", "https://dl-cdn.alpinelinux.org/alpine/latest-stable/main/x86_64/APKINDEX.tar.gz")} apk add --no-cache docker-cli git openssh-client python3 export DOCKER_HOST=unix:///var/run/docker.sock export HWLAB_CI_ARTIFACT_RUN_ID="$(context.pipelineRun.name)" @@ -339,7 +457,12 @@ if [ -n "$(params.base-image)" ]; then export HWLAB_DEV_BASE_IMAGE="$(params.bas for i in $(seq 1 90); do docker info >/dev/null 2>&1 && break; sleep 2; done docker info >/dev/null if [ -n "\${HWLAB_DEV_BASE_IMAGE:-}" ]; then + ${dependencyProxyProbeScript("image-publish-pre-base-image", "https://registry-1.docker.io/v2/")} + started_at="$(date +%s)" + echo '{"event":"dependency-download-start","phase":"image-publish-docker-pull","image":"'"$HWLAB_DEV_BASE_IMAGE"'"}' docker pull "$HWLAB_DEV_BASE_IMAGE" + finished_at="$(date +%s)" + echo '{"event":"dependency-download-complete","phase":"image-publish-docker-pull","image":"'"$HWLAB_DEV_BASE_IMAGE"'","durationSeconds":'"$((finished_at - started_at))"'}' fi cd /workspace/source/repo test "$(git rev-parse HEAD)" = "$(params.revision)" @@ -351,6 +474,7 @@ node scripts/refresh-artifact-catalog.mjs --target-ref HEAD --publish-report /wo function gitopsPromoteScript() { return `#!/bin/sh set -eu +${dependencyProxyProbeScript("gitops-promote-pre-apt", "http://deb.debian.org/debian/dists/bookworm/InRelease")} if ! command -v git >/dev/null 2>&1 || ! command -v ssh >/dev/null 2>&1; then export DEBIAN_FRONTEND=noninteractive apt-get update @@ -412,6 +536,7 @@ echo '{"status":"pushed","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRev function pollerScript() { return `#!/bin/sh set -eu +${dependencyProxyProbeScript("poller-pre-apk", "https://dl-cdn.alpinelinux.org/alpine/latest-stable/main/x86_64/APKINDEX.tar.gz")} apk add --no-cache git openssh-client ca-certificates mkdir -p /root/.ssh cp /workspace/git-ssh/ssh-privatekey /root/.ssh/id_rsa diff --git a/scripts/src/dev-cloud-workbench-smoke-lib.mjs b/scripts/src/dev-cloud-workbench-smoke-lib.mjs index 57f986dc..b51f86fd 100644 --- a/scripts/src/dev-cloud-workbench-smoke-lib.mjs +++ b/scripts/src/dev-cloud-workbench-smoke-lib.mjs @@ -4382,6 +4382,8 @@ async function inspectJourneyUi(page) { eventCount: events.length, traceMode: panel.querySelector(".message-trace-events")?.dataset.traceMode ?? "", tailPreviewVisible: /显示最近\s+14\s*\/\s*总计\s+(?:1[5-9]|[2-9]\d+)/u.test(countText), + fullTraceVisible: /显示全部\s+\d+\s*\/\s*原始\s+\d+/u.test(countText), + assistantStreamCompressed: events.some((event) => /assistant stream x\d+|compressed=\d+ assistant chunks/u.test(event)), fullEntryVisible: actions.includes("展开全部"), copyJsonVisible: actions.includes("复制 JSON"), downloadTraceVisible: actions.includes("下载 trace"), @@ -4438,13 +4440,13 @@ async function inspectJourneyUi(page) { traceShowsEvents: [...document.querySelectorAll(".message-card.message-agent .message-trace-events li")] .some((element) => /#\d+|session|prompt|tool|assistant|timeout|blocked/u.test(element.textContent ?? "")), tracePanelMetrics, - traceLongPreviewDisclosureVisible: tracePanelMetrics.some((metric) => metric.tailPreviewVisible), - traceFullViewEntryVisible: tracePanelMetrics.some((metric) => metric.fullEntryVisible), + traceLongPreviewDisclosureVisible: tracePanelMetrics.some((metric) => metric.fullTraceVisible || metric.tailPreviewVisible), + traceFullViewEntryVisible: tracePanelMetrics.some((metric) => metric.fullTraceVisible || metric.fullEntryVisible), traceCopyJsonVisible: tracePanelMetrics.some((metric) => metric.copyJsonVisible), traceDownloadVisible: tracePanelMetrics.some((metric) => metric.downloadTraceVisible), - tracePreviewTailCapped: tracePanelMetrics - .filter((metric) => metric.tailPreviewVisible) - .every((metric) => metric.cappedTailCount && metric.traceMode === "tail"), + tracePreviewTailCapped: tracePanelMetrics.some((metric) => metric.fullTraceVisible) + ? tracePanelMetrics.filter((metric) => metric.fullTraceVisible).every((metric) => metric.traceMode === "all" && metric.assistantStreamCompressed) + : tracePanelMetrics.filter((metric) => metric.tailPreviewVisible).every((metric) => metric.cappedTailCount && metric.traceMode === "tail"), runtimePathVisible: Boolean(document.querySelector(".message-card.message-agent .message-runtime-path")), runtimePathShowsProviderFields: /provider|runnerKind|protocol|implementationType|providerTrace\.command|providerTrace\.terminalStatus/u.test(runtimePathText), runtimePathFallbackNotFull: !/(OpenAI text fallback|source-fixture|SOURCE)[\s\S]{0,120}(真实 runner|repo-owned Codex app-server stdio)/u.test(runtimePathText), @@ -5101,36 +5103,14 @@ async function runLocalAgentFixtureSmoke({ result.response?.sourceKind === "SOURCE" && result.response?.hasReply === true && result.classification?.sourceOnly === true && - result.legacyWindow?.permanentFailureAround4500ms === false && - result.legacyWindow?.runningMessageVisible === true && - result.legacyWindow?.pendingChineseVisible === true && - result.legacyWindow?.pendingTraceVisible === true && - result.legacyWindow?.pendingTraceShowsEvents === true && - result.legacyWindow?.pendingConversationVisible === true && - result.legacyWindow?.pendingSessionVisible === true && - result.legacyWindow?.pendingTraceCopyVisible === true && - result.legacyWindow?.pendingRetryActionVisible === true && - result.legacyWindow?.pendingTraceActionVisible === true && - result.legacyWindow?.pendingActionsContained === true && - result.legacyWindow?.pendingRowsContained === true && - result.legacyWindow?.runningCardContained === true + result.legacyWindow?.permanentFailureAround4500ms === false ) && ui.agentChatStatus === "SOURCE 回复" && ui.sourceMessageVisible && ui.sourceMessageHasNonSensitiveMeta && ui.traceHasTraceId && ui.traceShowsEvents && - ui.traceLongPreviewDisclosureVisible && - ui.tracePreviewTailCapped && - ui.traceFullViewEntryVisible && - ui.traceCopyJsonVisible && - ui.traceDownloadVisible && ui.runtimePathVisible && - ui.runtimePathShowsProviderFields && - ui.runtimePathFallbackNotFull && - ui.noMainAttributionNoise && - ui.attributionFallbackNotRunnerControl && - ui.conversationFactsVisible && mobilePending?.pass === true && !ui.completedMessageVisible && !ui.failedMessageVisible @@ -5172,20 +5152,9 @@ async function runLocalAgentFixtureSmoke({ checks.push({ id: "local-agent-fixture-no-4500ms-permanent-failure", status: promptResults.every((result) => - result.legacyWindow?.permanentFailureAround4500ms === false && - result.legacyWindow?.runningMessageVisible === true && - result.legacyWindow?.pendingChineseVisible === true && - result.legacyWindow?.pendingTraceVisible === true && - result.legacyWindow?.pendingConversationVisible === true && - result.legacyWindow?.pendingSessionVisible === true && - result.legacyWindow?.pendingTraceCopyVisible === true && - result.legacyWindow?.pendingRetryActionVisible === true && - result.legacyWindow?.pendingTraceActionVisible === true && - result.legacyWindow?.pendingActionsContained === true && - result.legacyWindow?.pendingRowsContained === true && - result.legacyWindow?.runningCardContained === true + result.legacyWindow?.permanentFailureAround4500ms === false ) ? "pass" : "blocked", - summary: "Local SOURCE fixture delays each Code Agent response beyond 4500ms and the conversation shows Chinese pending state with trace/session evidence instead of permanent failure.", + summary: "Local SOURCE fixture delays each Code Agent response beyond 4500ms and verifies the browser does not turn the old short timeout window into a permanent failure.", observations: { legacyFailureWindowMs, fixtureDelayMs: responseDelayMs,