From f4b6c12488c72e65625abe4da8a5eb22d3bf0e38 Mon Sep 17 00:00:00 2001 From: Codex Agent Date: Mon, 8 Jun 2026 15:39:17 +0800 Subject: [PATCH] Configure runtime lane download stack --- deploy/deploy.yaml | 28 +++++++++++++ scripts/artifact-publish.mjs | 73 +++++++++++++++++++++++++++++++-- scripts/g14-ci-plan.test.mjs | 18 ++++++++ scripts/src/g14-ci-plan-lib.mjs | 52 ++++++++++++++++++++++- 4 files changed, 166 insertions(+), 5 deletions(-) diff --git a/deploy/deploy.yaml b/deploy/deploy.yaml index d359ba03..6a2bb37f 100644 --- a/deploy/deploy.yaml +++ b/deploy/deploy.yaml @@ -178,6 +178,20 @@ lanes: - ca-certificates - git bunVersion: 1.3.13 + downloadStack: + httpProxy: http://127.0.0.1:10808 + httpsProxy: http://127.0.0.1:10808 + noProxy: + - 127.0.0.1 + - localhost + - ::1 + - hyueapi.com + - .hyueapi.com + npmRegistry: https://registry.npmjs.org/ + npmFetchTimeoutMs: 300000 + bunPackages: + x64: '@oven/bun-linux-x64' + arm64: '@oven/bun-linux-aarch64' launcherPath: deploy/runtime/launcher/hwlab-env-reuse-launcher.ts runtimeNodeModulesPath: /opt/hwlab-env/node_modules additionalEnvPaths: @@ -314,6 +328,20 @@ lanes: - ca-certificates - git bunVersion: 1.3.13 + downloadStack: + httpProxy: http://127.0.0.1:10808 + httpsProxy: http://127.0.0.1:10808 + noProxy: + - 127.0.0.1 + - localhost + - ::1 + - hyueapi.com + - .hyueapi.com + npmRegistry: https://registry.npmjs.org/ + npmFetchTimeoutMs: 300000 + bunPackages: + x64: '@oven/bun-linux-x64' + arm64: '@oven/bun-linux-aarch64' launcherPath: deploy/runtime/launcher/hwlab-env-reuse-launcher.ts runtimeNodeModulesPath: /opt/hwlab-env/node_modules additionalEnvPaths: diff --git a/scripts/artifact-publish.mjs b/scripts/artifact-publish.mjs index 0f8a833f..85039fb3 100644 --- a/scripts/artifact-publish.mjs +++ b/scripts/artifact-publish.mjs @@ -865,18 +865,22 @@ function dockerfile(baseImage, port, labels = []) { function envReuseDockerfile(baseImage, labels = [], recipe) { const normalizedRecipe = normalizeEnvReuseRecipe(recipe); + const runtimeRoot = normalizedRecipe.runtimeNodeModulesPath.replace(/\/node_modules$/u, ""); + const downloadEnvPrefix = shellEnvPrefix(downloadEnvPairs(normalizedRecipe.downloadStack)); + const npmConfigScript = npmDownloadConfigScript(normalizedRecipe.downloadStack); + const downloadDependencyScript = `set -eu; ${npmConfigScript}; if [ -f package-lock.json ]; then npm ci --omit=dev --include=optional --ignore-scripts; else npm install --omit=dev --include=optional --ignore-scripts; fi; if command -v bun >/dev/null 2>&1; then bun_path="$(command -v bun)"; if [ "$bun_path" != "/usr/local/bin/bun" ]; then ln -sf "$bun_path" /usr/local/bin/bun; fi; fi; if [ ! -x /usr/local/bin/bun ]; then bun_pkg=""; case "$(uname -m)" in x86_64|amd64) bun_pkg="${normalizedRecipe.downloadStack.bunPackages.x64}@${normalizedRecipe.bunVersion}" ;; aarch64|arm64) bun_pkg="${normalizedRecipe.downloadStack.bunPackages.arm64}@${normalizedRecipe.bunVersion}" ;; *) echo "unsupported bun architecture: $(uname -m)" >&2; false ;; esac; npm install --no-save --omit=dev --include=optional --ignore-scripts "$bun_pkg"; bun_bin="node_modules/\${bun_pkg%@*}/bin/bun"; test -x "$bun_bin"; ln -sf "${runtimeRoot}/$bun_bin" /usr/local/bin/bun; fi`; const osPackageScript = `set -eu; apt-get update; apt-get install -y --no-install-recommends ${normalizedRecipe.osPackages.map(shellQuote).join(" ")}; rm -rf /var/lib/apt/lists/*`; const dependencyScript = `set -eu; if [ -f package-lock.json ]; then npm ci --omit=dev --include=optional --ignore-scripts; else npm install --omit=dev --include=optional --ignore-scripts; fi; if command -v bun >/dev/null 2>&1; then bun_path=\"$(command -v bun)\"; if [ \"$bun_path\" != \"/usr/local/bin/bun\" ]; then ln -sf \"$bun_path\" /usr/local/bin/bun; fi; fi; if [ ! -x /usr/local/bin/bun ]; then bun_pkg=\"\"; case \"$(uname -m)\" in x86_64|amd64) bun_pkg=\"@oven/bun-linux-x64@${normalizedRecipe.bunVersion}\" ;; aarch64|arm64) bun_pkg=\"@oven/bun-linux-aarch64@${normalizedRecipe.bunVersion}\" ;; *) echo \"unsupported bun architecture: $(uname -m)\" >&2; false ;; esac; npm install --no-save --omit=dev --include=optional --ignore-scripts \"$bun_pkg\"; bun_bin=\"node_modules/\${bun_pkg%@*}/bin/bun\"; test -x \"$bun_bin\"; ln -sf \"${normalizedRecipe.runtimeNodeModulesPath}/$bun_bin\" /usr/local/bin/bun; fi`; const aliasScript = normalizedRecipe.hwpodAliases.map((alias) => `printf '%s\\n' '#!/usr/bin/env sh' 'exec node /workspace/hwlab-boot/repo/scripts/run-bun.mjs /workspace/hwlab-boot/repo/${alias.script} \"$@\"' > /usr/local/bin/${alias.command}; chmod 755 /usr/local/bin/${alias.command}`).join("; "); const readinessScript = `set -eu; command -v node >/dev/null; command -v npm >/dev/null; command -v git >/dev/null; command -v sh >/dev/null; test -d ${shellQuote(normalizedRecipe.runtimeNodeModulesPath)}; test -x /usr/local/bin/bun; /usr/local/bin/bun --version >/tmp/hwlab-env-bun-version.txt; ${aliasScript}`; return [ `FROM ${baseImage}`, - `WORKDIR ${normalizedRecipe.runtimeNodeModulesPath.replace(/\/node_modules$/u, "")}`, + `WORKDIR ${runtimeRoot}`, ...labels.map(([name, value]) => `LABEL ${name}=${dockerfileQuote(value)}`), "COPY package.json ./package.json", "COPY package-lock.json* ./", - `RUN ${osPackageScript}`, - `RUN ${dependencyScript}`, + `RUN ${downloadEnvPrefix}${osPackageScript}`, + `RUN ${downloadEnvPrefix}${downloadDependencyScript}`, `COPY ${normalizedRecipe.launcherPath} /usr/local/bin/hwlab-env-reuse-launcher.ts`, `RUN ${readinessScript}`, "ENV HWLAB_RUNTIME_MODE=env-reuse-git-mirror-checkout", @@ -888,12 +892,13 @@ function envReuseDockerfile(baseImage, labels = [], recipe) { } function normalizeEnvReuseRecipe(recipe) { - assert.ok(recipe && typeof recipe === "object", "envRecipe must be provided by deploy.lanes.v02.envRecipe"); + assert.ok(recipe && typeof recipe === "object", "envRecipe must be provided by deploy.lanes..envRecipe"); const normalized = { osPackages: normalizeStringList(recipe.osPackages), bunVersion: String(recipe.bunVersion ?? "").trim(), launcherPath: normalizeRepoPath(recipe.launcherPath), runtimeNodeModulesPath: normalizeAbsolutePath(recipe.runtimeNodeModulesPath), + downloadStack: normalizeDownloadStack(recipe.downloadStack), hwpodAliases: normalizeHwpodAliases(recipe.hwpodAliases) }; assert.ok(normalized.osPackages.length > 0, "envRecipe.osPackages must not be empty"); @@ -904,6 +909,66 @@ function normalizeEnvReuseRecipe(recipe) { return normalized; } +function normalizeDownloadStack(value) { + assert.ok(value && typeof value === "object", "envRecipe.downloadStack must be configured in deploy/deploy.yaml"); + const normalized = { + httpProxy: normalizeOptionalHttpUrl(value.httpProxy), + httpsProxy: normalizeOptionalHttpUrl(value.httpsProxy), + noProxy: normalizeStringList(value.noProxy), + npmRegistry: normalizeOptionalHttpUrl(value.npmRegistry), + npmFetchTimeoutMs: Number(value.npmFetchTimeoutMs), + bunPackages: { + x64: normalizePackageName(value.bunPackages?.x64), + arm64: normalizePackageName(value.bunPackages?.arm64) + } + }; + assert.ok(normalized.npmRegistry, "envRecipe.downloadStack.npmRegistry must be an http(s) URL"); + assert.ok(Number.isInteger(normalized.npmFetchTimeoutMs) && normalized.npmFetchTimeoutMs > 0, "envRecipe.downloadStack.npmFetchTimeoutMs must be a positive integer"); + assert.ok(normalized.bunPackages.x64 && normalized.bunPackages.arm64, "envRecipe.downloadStack.bunPackages.x64/arm64 are required"); + return normalized; +} + +function normalizeOptionalHttpUrl(value) { + const text = String(value ?? "").trim(); + if (!text) return null; + try { + const url = new URL(text); + if (url.protocol === "http:" || url.protocol === "https:") return text; + } catch {} + return null; +} + +function normalizePackageName(value) { + const text = String(value ?? "").trim(); + return text && !text.includes(" ") ? text : null; +} + +function downloadEnvPairs(stack) { + return [ + ["HTTP_PROXY", stack.httpProxy], + ["http_proxy", stack.httpProxy], + ["HTTPS_PROXY", stack.httpsProxy], + ["https_proxy", stack.httpsProxy], + ["NO_PROXY", stack.noProxy.join(",")], + ["no_proxy", stack.noProxy.join(",")] + ].filter(([, value]) => value); +} + +function shellEnvPrefix(pairs) { + return pairs.length === 0 ? "" : `${pairs.map(([name, value]) => `${name}=${shellQuote(value)}`).join(" ")} `; +} + +function npmDownloadConfigScript(stack) { + const commands = [ + `npm config set registry ${shellQuote(stack.npmRegistry)}`, + `npm config set fetch-timeout ${Number(stack.npmFetchTimeoutMs)}` + ]; + if (stack.httpProxy) commands.push(`npm config set proxy ${shellQuote(stack.httpProxy)}`); + if (stack.httpsProxy) commands.push(`npm config set https-proxy ${shellQuote(stack.httpsProxy)}`); + if (stack.noProxy.length > 0) commands.push(`npm config set noproxy ${shellQuote(stack.noProxy.join(","))}`); + return commands.join("; "); +} + function normalizeStringList(value) { const source = Array.isArray(value) ? value : []; return source.map((item) => String(item ?? "").trim()).filter(Boolean); diff --git a/scripts/g14-ci-plan.test.mjs b/scripts/g14-ci-plan.test.mjs index 572514e2..98f21ab5 100644 --- a/scripts/g14-ci-plan.test.mjs +++ b/scripts/g14-ci-plan.test.mjs @@ -134,6 +134,10 @@ test("v02 planner rebuilds env image when an env reuse catalog digest is missing test("artifact reuse paths preserve catalog provenance instead of current planner provenance", async () => { const artifactPublish = await readFile("scripts/artifact-publish.mjs", "utf8"); + assert.match(artifactPublish, /downloadDependencyScript/u); + assert.match(artifactPublish, /RUN \$\{downloadEnvPrefix\}\$\{downloadDependencyScript\}/u); + assert.match(artifactPublish, /ln -sf "\$\{runtimeRoot\}\/\$bun_bin" \/usr\/local\/bin\/bun/u); + assert.doesNotMatch(artifactPublish, /RUN \$\{dependencyScript\}/u); assert.match(artifactPublish, /const catalogProvenance = serviceImageCatalogProvenance\(catalogService\);/u); assert.match(artifactPublish, /\.\.\.catalogProvenance,[\s\S]{0,80}buildBackend: "reused-catalog"/u); assert.match(artifactPublish, /envReuseRecipe: servicePlan\.envReuseRecipe \?\? null/u); @@ -606,6 +610,8 @@ test("v02 planner reads env reuse declarations and recipe from deploy config", a const recipe = envReuseRecipeForLane(deploy, "v02"); assert.equal(recipe.bunVersion, "1.3.14"); + assert.equal(recipe.downloadStack.npmRegistry, "https://registry.npmjs.org/"); + assert.equal(recipe.downloadStack.bunPackages.x64, "@oven/bun-linux-x64"); assert.equal(recipe.launcherInputPaths.includes("deploy/runtime/launcher/hwlab-env-reuse-launcher.ts"), true); assert.equal(recipe.launcherInputPaths.includes("custom/env-tool.mjs"), true); }); @@ -627,6 +633,7 @@ test("v03 planner reads env reuse declarations and catalog from v03 lane config" assert.equal(plan.compatibility.serviceDeclarationSource, "deploy.lanes.v03.serviceDeclarations"); assert.equal(plan.compatibility.envRecipeSource, "deploy.lanes.v03.envRecipe"); assert.deepEqual(plan.compatibility.envReuseServiceIds, ["hwlab-cloud-api", "hwlab-cloud-web"]); + assert.equal(plan.services.every((service) => service.envReuseRecipe.downloadStack.npmFetchTimeoutMs === 300000), true); for (const service of plan.services) { assert.equal(service.runtimeMode, "env-reuse-git-mirror-checkout", service.serviceId); assert.equal(service.envReuse, true, service.serviceId); @@ -830,6 +837,17 @@ function createDeployFixture({ serviceIds = V02_RUNTIME_SERVICE_IDS } = {}) { envRecipe: { osPackages: ["ca-certificates", "git"], bunVersion: "1.3.13", + downloadStack: { + httpProxy: "http://127.0.0.1:10808", + httpsProxy: "http://127.0.0.1:10808", + noProxy: ["127.0.0.1", "localhost", "::1", "hyueapi.com", ".hyueapi.com"], + npmRegistry: "https://registry.npmjs.org/", + npmFetchTimeoutMs: 300000, + bunPackages: { + x64: "@oven/bun-linux-x64", + arm64: "@oven/bun-linux-aarch64" + } + }, launcherPath: "deploy/runtime/launcher/hwlab-env-reuse-launcher.ts", runtimeNodeModulesPath: "/opt/hwlab-env/node_modules", additionalEnvPaths: [ diff --git a/scripts/src/g14-ci-plan-lib.mjs b/scripts/src/g14-ci-plan-lib.mjs index 07b3523d..b609a66a 100644 --- a/scripts/src/g14-ci-plan-lib.mjs +++ b/scripts/src/g14-ci-plan-lib.mjs @@ -108,7 +108,7 @@ export async function createG14CiPlan(options = {}) { ...buildSystemMatches ].filter((item) => !isTestOnlyPath(item) && !isDocsOnlyPath(item))); const catalogRecord = catalogByServiceId.get(model.serviceId) ?? null; - const environmentInputHash = envReuse ? await hashGitPaths(repoRoot, targetRef, envInputPaths, { + const environmentInputHash = envReuse ? await hashEnvReuseInputs(repoRoot, targetRef, envInputPaths, envReuseRecipe.publicRecipe, { skipPath: (filePath) => isTestOnlyPath(filePath) || isDocsOnlyPath(filePath), semanticPackageJson: true }) : null; @@ -339,6 +339,7 @@ function envReuseRecipeForLaneConfig(laneConfig, lane = "v02") { ...additionalEnvPaths ]); const hwpodAliases = normalizeHwpodAliases(configured.hwpodAliases); + const downloadStack = normalizeDownloadStack(configured.downloadStack, lane); if (osPackages.length === 0) throw new Error(`deploy.lanes.${lane}.envRecipe.osPackages is required`); if (additionalEnvPaths.length === 0) throw new Error(`deploy.lanes.${lane}.envRecipe.additionalEnvPaths is required`); if (hwpodAliases.length === 0) throw new Error(`deploy.lanes.${lane}.envRecipe.hwpodAliases is required`); @@ -349,6 +350,7 @@ function envReuseRecipeForLaneConfig(laneConfig, lane = "v02") { runtimeNodeModulesPath, additionalEnvPaths, launcherInputPaths, + downloadStack, hwpodAliases, publicRecipe: { osPackages, @@ -356,11 +358,52 @@ function envReuseRecipeForLaneConfig(laneConfig, lane = "v02") { launcherPath, runtimeNodeModulesPath, additionalEnvPaths, + downloadStack, hwpodAliases } }; } +function normalizeDownloadStack(value, lane) { + const configured = value && typeof value === "object" ? value : null; + if (!configured) throw new Error(`deploy.lanes.${lane}.envRecipe.downloadStack is required`); + const httpProxy = normalizeOptionalHttpUrl(configured.httpProxy, `deploy.lanes.${lane}.envRecipe.downloadStack.httpProxy`); + const httpsProxy = normalizeOptionalHttpUrl(configured.httpsProxy, `deploy.lanes.${lane}.envRecipe.downloadStack.httpsProxy`); + const npmRegistry = normalizeOptionalHttpUrl(configured.npmRegistry, `deploy.lanes.${lane}.envRecipe.downloadStack.npmRegistry`); + const npmFetchTimeoutMs = Number(configured.npmFetchTimeoutMs); + const bunPackages = configured.bunPackages && typeof configured.bunPackages === "object" ? configured.bunPackages : null; + const x64 = normalizePackageName(bunPackages?.x64); + const arm64 = normalizePackageName(bunPackages?.arm64); + if (!npmRegistry) throw new Error(`deploy.lanes.${lane}.envRecipe.downloadStack.npmRegistry is required`); + if (!Number.isInteger(npmFetchTimeoutMs) || npmFetchTimeoutMs <= 0) { + throw new Error(`deploy.lanes.${lane}.envRecipe.downloadStack.npmFetchTimeoutMs must be a positive integer`); + } + if (!x64 || !arm64) throw new Error(`deploy.lanes.${lane}.envRecipe.downloadStack.bunPackages.x64/arm64 are required`); + return { + httpProxy, + httpsProxy, + noProxy: normalizeStringList(configured.noProxy, []), + npmRegistry, + npmFetchTimeoutMs, + bunPackages: { x64, arm64 } + }; +} + +function normalizeOptionalHttpUrl(value, label) { + const text = normalizeDeclarationString(value); + if (!text) return null; + try { + const url = new URL(text); + if (url.protocol === "http:" || url.protocol === "https:") return text; + } catch {} + throw new Error(`${label} must be an http(s) URL`); +} + +function normalizePackageName(value) { + const text = normalizeDeclarationString(value); + return text && !text.includes(" ") ? text : null; +} + function normalizeStringList(value, fallback) { const list = Array.isArray(value) ? value : fallback; return list.map((item) => String(item ?? "").trim()).filter(Boolean); @@ -532,6 +575,13 @@ export async function hashGitPaths(repoRoot, targetRef, paths, options = {}) { return stableHash({ entries: lines }); } +async function hashEnvReuseInputs(repoRoot, targetRef, paths, recipe, options = {}) { + return stableHash({ + gitPaths: await hashGitPaths(repoRoot, targetRef, paths, options), + envRecipe: recipe + }); +} + export async function lastCommitForPaths(repoRoot, targetRef, paths) { const normalizedPaths = uniqueSorted(paths.map(normalizeRepoPath).filter(Boolean)); if (normalizedPaths.length === 0) return null;