Configure runtime lane download stack

2026-06-08 15:39:17 +08:00
parent e63e2afbe4
commit f4b6c12488
4 changed files with 166 additions and 5 deletions
@@ -178,6 +178,20 @@ lanes:
        - ca-certificates
        - git
      bunVersion: 1.3.13
+      downloadStack:
+        httpProxy: http://127.0.0.1:10808
+        httpsProxy: http://127.0.0.1:10808
+        noProxy:
+          - 127.0.0.1
+          - localhost
+          - ::1
+          - hyueapi.com
+          - .hyueapi.com
+        npmRegistry: https://registry.npmjs.org/
+        npmFetchTimeoutMs: 300000
+        bunPackages:
+          x64: '@oven/bun-linux-x64'
+          arm64: '@oven/bun-linux-aarch64'
      launcherPath: deploy/runtime/launcher/hwlab-env-reuse-launcher.ts
      runtimeNodeModulesPath: /opt/hwlab-env/node_modules
      additionalEnvPaths:
@@ -314,6 +328,20 @@ lanes:
        - ca-certificates
        - git
      bunVersion: 1.3.13
+      downloadStack:
+        httpProxy: http://127.0.0.1:10808
+        httpsProxy: http://127.0.0.1:10808
+        noProxy:
+          - 127.0.0.1
+          - localhost
+          - ::1
+          - hyueapi.com
+          - .hyueapi.com
+        npmRegistry: https://registry.npmjs.org/
+        npmFetchTimeoutMs: 300000
+        bunPackages:
+          x64: '@oven/bun-linux-x64'
+          arm64: '@oven/bun-linux-aarch64'
      launcherPath: deploy/runtime/launcher/hwlab-env-reuse-launcher.ts
      runtimeNodeModulesPath: /opt/hwlab-env/node_modules
      additionalEnvPaths:
@@ -865,18 +865,22 @@ function dockerfile(baseImage, port, labels = []) {

 function envReuseDockerfile(baseImage, labels = [], recipe) {
  const normalizedRecipe = normalizeEnvReuseRecipe(recipe);
+  const runtimeRoot = normalizedRecipe.runtimeNodeModulesPath.replace(/\/node_modules$/u, "");
+  const downloadEnvPrefix = shellEnvPrefix(downloadEnvPairs(normalizedRecipe.downloadStack));
+  const npmConfigScript = npmDownloadConfigScript(normalizedRecipe.downloadStack);
+  const downloadDependencyScript = `set -eu; ${npmConfigScript}; if [ -f package-lock.json ]; then npm ci --omit=dev --include=optional --ignore-scripts; else npm install --omit=dev --include=optional --ignore-scripts; fi; if command -v bun >/dev/null 2>&1; then bun_path="$(command -v bun)"; if [ "$bun_path" != "/usr/local/bin/bun" ]; then ln -sf "$bun_path" /usr/local/bin/bun; fi; fi; if [ ! -x /usr/local/bin/bun ]; then bun_pkg=""; case "$(uname -m)" in x86_64|amd64) bun_pkg="${normalizedRecipe.downloadStack.bunPackages.x64}@${normalizedRecipe.bunVersion}" ;; aarch64|arm64) bun_pkg="${normalizedRecipe.downloadStack.bunPackages.arm64}@${normalizedRecipe.bunVersion}" ;; *) echo "unsupported bun architecture: $(uname -m)" >&2; false ;; esac; npm install --no-save --omit=dev --include=optional --ignore-scripts "$bun_pkg"; bun_bin="node_modules/\${bun_pkg%@*}/bin/bun"; test -x "$bun_bin"; ln -sf "${runtimeRoot}/$bun_bin" /usr/local/bin/bun; fi`;
  const osPackageScript = `set -eu; apt-get update; apt-get install -y --no-install-recommends ${normalizedRecipe.osPackages.map(shellQuote).join(" ")}; rm -rf /var/lib/apt/lists/*`;
  const dependencyScript = `set -eu; if [ -f package-lock.json ]; then npm ci --omit=dev --include=optional --ignore-scripts; else npm install --omit=dev --include=optional --ignore-scripts; fi; if command -v bun >/dev/null 2>&1; then bun_path=\"$(command -v bun)\"; if [ \"$bun_path\" != \"/usr/local/bin/bun\" ]; then ln -sf \"$bun_path\" /usr/local/bin/bun; fi; fi; if [ ! -x /usr/local/bin/bun ]; then bun_pkg=\"\"; case \"$(uname -m)\" in x86_64|amd64) bun_pkg=\"@oven/bun-linux-x64@${normalizedRecipe.bunVersion}\" ;; aarch64|arm64) bun_pkg=\"@oven/bun-linux-aarch64@${normalizedRecipe.bunVersion}\" ;; *) echo \"unsupported bun architecture: $(uname -m)\" >&2; false ;; esac; npm install --no-save --omit=dev --include=optional --ignore-scripts \"$bun_pkg\"; bun_bin=\"node_modules/\${bun_pkg%@*}/bin/bun\"; test -x \"$bun_bin\"; ln -sf \"${normalizedRecipe.runtimeNodeModulesPath}/$bun_bin\" /usr/local/bin/bun; fi`;
  const aliasScript = normalizedRecipe.hwpodAliases.map((alias) => `printf '%s\\n' '#!/usr/bin/env sh' 'exec node /workspace/hwlab-boot/repo/scripts/run-bun.mjs /workspace/hwlab-boot/repo/${alias.script} \"$@\"' > /usr/local/bin/${alias.command}; chmod 755 /usr/local/bin/${alias.command}`).join("; ");
  const readinessScript = `set -eu; command -v node >/dev/null; command -v npm >/dev/null; command -v git >/dev/null; command -v sh >/dev/null; test -d ${shellQuote(normalizedRecipe.runtimeNodeModulesPath)}; test -x /usr/local/bin/bun; /usr/local/bin/bun --version >/tmp/hwlab-env-bun-version.txt; ${aliasScript}`;
  return [
    `FROM ${baseImage}`,
-    `WORKDIR ${normalizedRecipe.runtimeNodeModulesPath.replace(/\/node_modules$/u, "")}`,
+    `WORKDIR ${runtimeRoot}`,
    ...labels.map(([name, value]) => `LABEL ${name}=${dockerfileQuote(value)}`),
    "COPY package.json ./package.json",
    "COPY package-lock.json* ./",
-    `RUN ${osPackageScript}`,
-    `RUN ${dependencyScript}`,
+    `RUN ${downloadEnvPrefix}${osPackageScript}`,
+    `RUN ${downloadEnvPrefix}${downloadDependencyScript}`,
    `COPY ${normalizedRecipe.launcherPath} /usr/local/bin/hwlab-env-reuse-launcher.ts`,
    `RUN ${readinessScript}`,
    "ENV HWLAB_RUNTIME_MODE=env-reuse-git-mirror-checkout",
@@ -888,12 +892,13 @@ function envReuseDockerfile(baseImage, labels = [], recipe) {
 }

 function normalizeEnvReuseRecipe(recipe) {
-  assert.ok(recipe && typeof recipe === "object", "envRecipe must be provided by deploy.lanes.v02.envRecipe");
+  assert.ok(recipe && typeof recipe === "object", "envRecipe must be provided by deploy.lanes.<lane>.envRecipe");
  const normalized = {
    osPackages: normalizeStringList(recipe.osPackages),
    bunVersion: String(recipe.bunVersion ?? "").trim(),
    launcherPath: normalizeRepoPath(recipe.launcherPath),
    runtimeNodeModulesPath: normalizeAbsolutePath(recipe.runtimeNodeModulesPath),
+    downloadStack: normalizeDownloadStack(recipe.downloadStack),
    hwpodAliases: normalizeHwpodAliases(recipe.hwpodAliases)
  };
  assert.ok(normalized.osPackages.length > 0, "envRecipe.osPackages must not be empty");
@@ -904,6 +909,66 @@ function normalizeEnvReuseRecipe(recipe) {
  return normalized;
 }

+function normalizeDownloadStack(value) {
+  assert.ok(value && typeof value === "object", "envRecipe.downloadStack must be configured in deploy/deploy.yaml");
+  const normalized = {
+    httpProxy: normalizeOptionalHttpUrl(value.httpProxy),
+    httpsProxy: normalizeOptionalHttpUrl(value.httpsProxy),
+    noProxy: normalizeStringList(value.noProxy),
+    npmRegistry: normalizeOptionalHttpUrl(value.npmRegistry),
+    npmFetchTimeoutMs: Number(value.npmFetchTimeoutMs),
+    bunPackages: {
+      x64: normalizePackageName(value.bunPackages?.x64),
+      arm64: normalizePackageName(value.bunPackages?.arm64)
+    }
+  };
+  assert.ok(normalized.npmRegistry, "envRecipe.downloadStack.npmRegistry must be an http(s) URL");
+  assert.ok(Number.isInteger(normalized.npmFetchTimeoutMs) && normalized.npmFetchTimeoutMs > 0, "envRecipe.downloadStack.npmFetchTimeoutMs must be a positive integer");
+  assert.ok(normalized.bunPackages.x64 && normalized.bunPackages.arm64, "envRecipe.downloadStack.bunPackages.x64/arm64 are required");
+  return normalized;
+}
+
+function normalizeOptionalHttpUrl(value) {
+  const text = String(value ?? "").trim();
+  if (!text) return null;
+  try {
+    const url = new URL(text);
+    if (url.protocol === "http:" || url.protocol === "https:") return text;
+  } catch {}
+  return null;
+}
+
+function normalizePackageName(value) {
+  const text = String(value ?? "").trim();
+  return text && !text.includes(" ") ? text : null;
+}
+
+function downloadEnvPairs(stack) {
+  return [
+    ["HTTP_PROXY", stack.httpProxy],
+    ["http_proxy", stack.httpProxy],
+    ["HTTPS_PROXY", stack.httpsProxy],
+    ["https_proxy", stack.httpsProxy],
+    ["NO_PROXY", stack.noProxy.join(",")],
+    ["no_proxy", stack.noProxy.join(",")]
+  ].filter(([, value]) => value);
+}
+
+function shellEnvPrefix(pairs) {
+  return pairs.length === 0 ? "" : `${pairs.map(([name, value]) => `${name}=${shellQuote(value)}`).join(" ")} `;
+}
+
+function npmDownloadConfigScript(stack) {
+  const commands = [
+    `npm config set registry ${shellQuote(stack.npmRegistry)}`,
+    `npm config set fetch-timeout ${Number(stack.npmFetchTimeoutMs)}`
+  ];
+  if (stack.httpProxy) commands.push(`npm config set proxy ${shellQuote(stack.httpProxy)}`);
+  if (stack.httpsProxy) commands.push(`npm config set https-proxy ${shellQuote(stack.httpsProxy)}`);
+  if (stack.noProxy.length > 0) commands.push(`npm config set noproxy ${shellQuote(stack.noProxy.join(","))}`);
+  return commands.join("; ");
+}
+
 function normalizeStringList(value) {
  const source = Array.isArray(value) ? value : [];
  return source.map((item) => String(item ?? "").trim()).filter(Boolean);
@@ -134,6 +134,10 @@ test("v02 planner rebuilds env image when an env reuse catalog digest is missing

 test("artifact reuse paths preserve catalog provenance instead of current planner provenance", async () => {
  const artifactPublish = await readFile("scripts/artifact-publish.mjs", "utf8");
+  assert.match(artifactPublish, /downloadDependencyScript/u);
+  assert.match(artifactPublish, /RUN \$\{downloadEnvPrefix\}\$\{downloadDependencyScript\}/u);
+  assert.match(artifactPublish, /ln -sf "\$\{runtimeRoot\}\/\$bun_bin" \/usr\/local\/bin\/bun/u);
+  assert.doesNotMatch(artifactPublish, /RUN \$\{dependencyScript\}/u);
  assert.match(artifactPublish, /const catalogProvenance = serviceImageCatalogProvenance\(catalogService\);/u);
  assert.match(artifactPublish, /\.\.\.catalogProvenance,[\s\S]{0,80}buildBackend: "reused-catalog"/u);
  assert.match(artifactPublish, /envReuseRecipe: servicePlan\.envReuseRecipe \?\? null/u);
@@ -606,6 +610,8 @@ test("v02 planner reads env reuse declarations and recipe from deploy config", a

  const recipe = envReuseRecipeForLane(deploy, "v02");
  assert.equal(recipe.bunVersion, "1.3.14");
+  assert.equal(recipe.downloadStack.npmRegistry, "https://registry.npmjs.org/");
+  assert.equal(recipe.downloadStack.bunPackages.x64, "@oven/bun-linux-x64");
  assert.equal(recipe.launcherInputPaths.includes("deploy/runtime/launcher/hwlab-env-reuse-launcher.ts"), true);
  assert.equal(recipe.launcherInputPaths.includes("custom/env-tool.mjs"), true);
 });
@@ -627,6 +633,7 @@ test("v03 planner reads env reuse declarations and catalog from v03 lane config"
  assert.equal(plan.compatibility.serviceDeclarationSource, "deploy.lanes.v03.serviceDeclarations");
  assert.equal(plan.compatibility.envRecipeSource, "deploy.lanes.v03.envRecipe");
  assert.deepEqual(plan.compatibility.envReuseServiceIds, ["hwlab-cloud-api", "hwlab-cloud-web"]);
+  assert.equal(plan.services.every((service) => service.envReuseRecipe.downloadStack.npmFetchTimeoutMs === 300000), true);
  for (const service of plan.services) {
    assert.equal(service.runtimeMode, "env-reuse-git-mirror-checkout", service.serviceId);
    assert.equal(service.envReuse, true, service.serviceId);
@@ -830,6 +837,17 @@ function createDeployFixture({ serviceIds = V02_RUNTIME_SERVICE_IDS } = {}) {
        envRecipe: {
          osPackages: ["ca-certificates", "git"],
          bunVersion: "1.3.13",
+          downloadStack: {
+            httpProxy: "http://127.0.0.1:10808",
+            httpsProxy: "http://127.0.0.1:10808",
+            noProxy: ["127.0.0.1", "localhost", "::1", "hyueapi.com", ".hyueapi.com"],
+            npmRegistry: "https://registry.npmjs.org/",
+            npmFetchTimeoutMs: 300000,
+            bunPackages: {
+              x64: "@oven/bun-linux-x64",
+              arm64: "@oven/bun-linux-aarch64"
+            }
+          },
          launcherPath: "deploy/runtime/launcher/hwlab-env-reuse-launcher.ts",
          runtimeNodeModulesPath: "/opt/hwlab-env/node_modules",
          additionalEnvPaths: [
@@ -108,7 +108,7 @@ export async function createG14CiPlan(options = {}) {
      ...buildSystemMatches
    ].filter((item) => !isTestOnlyPath(item) && !isDocsOnlyPath(item)));
    const catalogRecord = catalogByServiceId.get(model.serviceId) ?? null;
-    const environmentInputHash = envReuse ? await hashGitPaths(repoRoot, targetRef, envInputPaths, {
+    const environmentInputHash = envReuse ? await hashEnvReuseInputs(repoRoot, targetRef, envInputPaths, envReuseRecipe.publicRecipe, {
      skipPath: (filePath) => isTestOnlyPath(filePath) || isDocsOnlyPath(filePath),
      semanticPackageJson: true
    }) : null;
@@ -339,6 +339,7 @@ function envReuseRecipeForLaneConfig(laneConfig, lane = "v02") {
    ...additionalEnvPaths
  ]);
  const hwpodAliases = normalizeHwpodAliases(configured.hwpodAliases);
+  const downloadStack = normalizeDownloadStack(configured.downloadStack, lane);
  if (osPackages.length === 0) throw new Error(`deploy.lanes.${lane}.envRecipe.osPackages is required`);
  if (additionalEnvPaths.length === 0) throw new Error(`deploy.lanes.${lane}.envRecipe.additionalEnvPaths is required`);
  if (hwpodAliases.length === 0) throw new Error(`deploy.lanes.${lane}.envRecipe.hwpodAliases is required`);
@@ -349,6 +350,7 @@ function envReuseRecipeForLaneConfig(laneConfig, lane = "v02") {
    runtimeNodeModulesPath,
    additionalEnvPaths,
    launcherInputPaths,
+    downloadStack,
    hwpodAliases,
    publicRecipe: {
      osPackages,
@@ -356,11 +358,52 @@ function envReuseRecipeForLaneConfig(laneConfig, lane = "v02") {
      launcherPath,
      runtimeNodeModulesPath,
      additionalEnvPaths,
+      downloadStack,
      hwpodAliases
    }
  };
 }

+function normalizeDownloadStack(value, lane) {
+  const configured = value && typeof value === "object" ? value : null;
+  if (!configured) throw new Error(`deploy.lanes.${lane}.envRecipe.downloadStack is required`);
+  const httpProxy = normalizeOptionalHttpUrl(configured.httpProxy, `deploy.lanes.${lane}.envRecipe.downloadStack.httpProxy`);
+  const httpsProxy = normalizeOptionalHttpUrl(configured.httpsProxy, `deploy.lanes.${lane}.envRecipe.downloadStack.httpsProxy`);
+  const npmRegistry = normalizeOptionalHttpUrl(configured.npmRegistry, `deploy.lanes.${lane}.envRecipe.downloadStack.npmRegistry`);
+  const npmFetchTimeoutMs = Number(configured.npmFetchTimeoutMs);
+  const bunPackages = configured.bunPackages && typeof configured.bunPackages === "object" ? configured.bunPackages : null;
+  const x64 = normalizePackageName(bunPackages?.x64);
+  const arm64 = normalizePackageName(bunPackages?.arm64);
+  if (!npmRegistry) throw new Error(`deploy.lanes.${lane}.envRecipe.downloadStack.npmRegistry is required`);
+  if (!Number.isInteger(npmFetchTimeoutMs) || npmFetchTimeoutMs <= 0) {
+    throw new Error(`deploy.lanes.${lane}.envRecipe.downloadStack.npmFetchTimeoutMs must be a positive integer`);
+  }
+  if (!x64 || !arm64) throw new Error(`deploy.lanes.${lane}.envRecipe.downloadStack.bunPackages.x64/arm64 are required`);
+  return {
+    httpProxy,
+    httpsProxy,
+    noProxy: normalizeStringList(configured.noProxy, []),
+    npmRegistry,
+    npmFetchTimeoutMs,
+    bunPackages: { x64, arm64 }
+  };
+}
+
+function normalizeOptionalHttpUrl(value, label) {
+  const text = normalizeDeclarationString(value);
+  if (!text) return null;
+  try {
+    const url = new URL(text);
+    if (url.protocol === "http:" || url.protocol === "https:") return text;
+  } catch {}
+  throw new Error(`${label} must be an http(s) URL`);
+}
+
+function normalizePackageName(value) {
+  const text = normalizeDeclarationString(value);
+  return text && !text.includes(" ") ? text : null;
+}
+
 function normalizeStringList(value, fallback) {
  const list = Array.isArray(value) ? value : fallback;
  return list.map((item) => String(item ?? "").trim()).filter(Boolean);
@@ -532,6 +575,13 @@ export async function hashGitPaths(repoRoot, targetRef, paths, options = {}) {
  return stableHash({ entries: lines });
 }

+async function hashEnvReuseInputs(repoRoot, targetRef, paths, recipe, options = {}) {
+  return stableHash({
+    gitPaths: await hashGitPaths(repoRoot, targetRef, paths, options),
+    envRecipe: recipe
+  });
+}
+
 export async function lastCommitForPaths(repoRoot, targetRef, paths) {
  const normalizedPaths = uniqueSorted(paths.map(normalizeRepoPath).filter(Boolean));
  if (normalizedPaths.length === 0) return null;