pikastech
c6e41380a1
为 web-probe 增加 YAML-first native origin，并同步 Workbench SPEC、WebDev skill、帮助与 L0 测试。
2704 lines
152 KiB
TypeScript
2704 lines
152 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. web-probe module for scripts/src/hwlab-node-impl.ts.
|
||
|
||
// Moved mechanically from scripts/src/hwlab-node-impl.ts:6032-7426 for #903.
|
||
|
||
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
||
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
|
||
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
|
||
import { createHash, randomBytes } from "node:crypto";
|
||
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
||
import { dirname, isAbsolute, join } from "node:path";
|
||
import { repoRoot, rootPath, type Config } from "../config";
|
||
import { runCommand, type CommandResult } from "../command";
|
||
import { startJob } from "../jobs";
|
||
import { classifySshTcpPoolFailure, shellQuote } from "../ssh";
|
||
import { CliInputError } from "../output";
|
||
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneCiGitWorkspaceSecret, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
|
||
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
|
||
import { nodeWebProbeConsoleVerifyScript } from "../hwlab-node-web-probe-console-verify";
|
||
import { nodeWebProbeNativeReadinessScript } from "../hwlab-node-web-probe-native-readiness";
|
||
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
|
||
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
|
||
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
||
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
|
||
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
||
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
||
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
|
||
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
|
||
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
|
||
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
|
||
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
|
||
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
|
||
import type { RenderedCliResult } from "../output";
|
||
|
||
import type { BootstrapAdminPasswordMaterial, BootstrapAdminSecretMaterial, NodeRuntimeGitMirrorEgressProxySpec, NodeRuntimeGitMirrorGithubTransportSpec, NodeRuntimeGitMirrorTargetSpec, NodeRuntimeRenderResult, NodeWebProbeOptions, RuntimeSecretSpec } from "./entry";
|
||
import { isCommandSuccess, nodeRuntimeGitopsRoot, runNodeK3sArgs, runNodeK3sScript } from "./cleanup";
|
||
import { nodeRuntimeCicdWaitWarning } from "./git-mirror";
|
||
import { parseNodeScopedDelegatedOptions } from "./plan";
|
||
import { nodeRuntimePipelineFailureSummary, nodeRuntimePipelineRunDiagnostics } from "./render";
|
||
import { compactRuntimeCommand, runNodeHostScript, runNodeHostScriptAsync } from "./runtime-common";
|
||
import { assertLane, assertNodeId, keyValueLinesFromText, numericField, optionValue, optionalStringValue, positiveIntegerOption, positiveIntegerValue, record, requiredOption, shellQuote, statusText, stringValue, stripOptions } from "./utils";
|
||
import { discoverWebObserveIndexEntry, readWebObserveIndexEntry } from "./web-observe-render";
|
||
import { assertKnownOptions, nodeWebProbeAutoCommandTimeoutSeconds, normalizeNodeWebProbeObserveArgs, parseNodeWebProbeObserveOptions, parseNodeWebProbeSentinelOptions, parseWebProbeBrowserProxyMode } from "./web-probe-observe";
|
||
import { resolveNodeWebProbeCliOrigin } from "./web-probe-origin";
|
||
import { hwlabRuntimeActiveExternalPostgres } from "../hwlab-node-lanes";
|
||
import { resolveEgressProxySourceRef } from "../egress-proxy-sources";
|
||
import { hwlabRuntimePipelineProvenanceAnnotations } from "./pipeline-provenance";
|
||
|
||
export function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
||
const gitSshProxy = httpProxyEndpoint(spec.networkProfile.proxy.http);
|
||
const gitMirror = nodeRuntimeGitMirrorTarget(spec);
|
||
const activeExternalPostgres = hwlabRuntimeActiveExternalPostgres(spec);
|
||
const renderGitMirror = {
|
||
...gitMirror,
|
||
egressProxy: gitMirror.egressProxy.mode === "direct" ? {
|
||
mode: "direct",
|
||
required: false,
|
||
} : gitMirror.egressProxy.mode === "host-route" ? {
|
||
...gitMirror.egressProxy,
|
||
mode: "host-route",
|
||
required: true,
|
||
} : {
|
||
...gitMirror.egressProxy,
|
||
mode: "node-global",
|
||
required: true,
|
||
},
|
||
};
|
||
const deployYamlGitMirror = {
|
||
...renderGitMirror,
|
||
egressProxy: renderGitMirror.egressProxy.mode !== "host-route" ? renderGitMirror.egressProxy : {
|
||
mode: "node-global",
|
||
required: true,
|
||
clientName: renderGitMirror.egressProxy.clientName,
|
||
namespace: "platform-infra",
|
||
serviceName: renderGitMirror.egressProxy.clientName,
|
||
port: httpProxyEndpoint(renderGitMirror.egressProxy.proxyUrl)?.port ?? 10808,
|
||
proxyUrl: renderGitMirror.egressProxy.proxyUrl,
|
||
noProxy: renderGitMirror.egressProxy.noProxy,
|
||
},
|
||
};
|
||
return {
|
||
pipelineProvenanceAnnotations: hwlabRuntimePipelineProvenanceAnnotations(spec),
|
||
nodeId: spec.nodeId,
|
||
lane: spec.lane,
|
||
environment: spec.environment,
|
||
sourceBranch: spec.sourceBranch,
|
||
gitopsBranch: spec.gitopsBranch,
|
||
gitopsRoot: nodeRuntimeGitopsRoot(spec),
|
||
runtimePath: spec.runtimePath,
|
||
runtimeRenderDir: spec.runtimeRenderDir,
|
||
runtimeNamespace: spec.runtimeNamespace,
|
||
catalogPath: spec.catalogPath,
|
||
tektonDir: spec.tektonDir,
|
||
pipelineName: spec.pipeline,
|
||
argoApplicationFile: spec.argoApplicationFile,
|
||
argoRepoUrl: spec.argoRepoUrl,
|
||
gitUrl: spec.gitUrl,
|
||
gitReadUrl: spec.gitReadUrl,
|
||
gitWriteUrl: spec.gitWriteUrl,
|
||
toolsImage: gitMirror.toolsImage,
|
||
toolsImagePullPolicy: gitMirror.toolsImagePullPolicy,
|
||
gitMirror: renderGitMirror,
|
||
deployYamlGitMirror,
|
||
networkProfileId: spec.networkProfileId,
|
||
downloadProfileId: spec.downloadProfileId,
|
||
gitSshProxyHost: gitSshProxy?.host,
|
||
gitSshProxyPort: gitSshProxy?.port,
|
||
proxyHttp: spec.networkProfile.proxy.http,
|
||
proxyHttps: spec.networkProfile.proxy.https,
|
||
proxyAll: spec.networkProfile.proxy.all,
|
||
noProxy: spec.networkProfile.proxy.noProxy.join(","),
|
||
dockerProxyHttp: spec.networkProfile.dockerBuildProxy.http,
|
||
dockerProxyHttps: spec.networkProfile.dockerBuildProxy.https,
|
||
dockerProxyAll: spec.networkProfile.dockerBuildProxy.all,
|
||
dockerNoProxy: spec.networkProfile.dockerBuildProxy.noProxy.join(","),
|
||
dockerNoProxyList: spec.networkProfile.dockerBuildProxy.noProxy,
|
||
npmRegistry: spec.downloadProfile.npm.registry,
|
||
npmFetchTimeoutMs: spec.downloadProfile.npm.fetchTimeoutSeconds * 1000,
|
||
npmRetries: spec.downloadProfile.npm.retries,
|
||
stepEnv: spec.stepEnv,
|
||
ciResourceGovernance: spec.ciResourceGovernance === undefined ? undefined : {
|
||
configRef: `${hwlabRuntimeLaneConfigPath()}#lanes.${spec.lane}.targets.${spec.nodeId}.ciResourceGovernance`,
|
||
...spec.ciResourceGovernance,
|
||
},
|
||
observability: spec.observability,
|
||
runtimeStore: spec.runtimeStore,
|
||
codeAgentRuntime: spec.codeAgentRuntime === undefined ? undefined : {
|
||
enabled: spec.codeAgentRuntime.enabled,
|
||
adapter: spec.codeAgentRuntime.adapter,
|
||
managerUrl: spec.codeAgentRuntime.managerUrl,
|
||
apiKeySecretName: spec.codeAgentRuntime.apiKeySecretName,
|
||
apiKeySecretKey: spec.codeAgentRuntime.apiKeySecretKey,
|
||
runnerNamespace: spec.codeAgentRuntime.runnerNamespace,
|
||
secretNamespace: spec.codeAgentRuntime.secretNamespace,
|
||
...(spec.codeAgentRuntime.providerSecretNames === undefined ? {} : { providerSecretNames: spec.codeAgentRuntime.providerSecretNames }),
|
||
toolSecretNames: spec.codeAgentRuntime.toolSecretNames,
|
||
repoUrlFrom: spec.codeAgentRuntime.repoUrlFrom,
|
||
repoUrl: spec.gitReadUrl,
|
||
providerIdFrom: spec.codeAgentRuntime.providerIdFrom,
|
||
providerId: spec.nodeId,
|
||
defaultProviderProfile: spec.codeAgentRuntime.defaultProviderProfile,
|
||
codexStdioSupervisor: spec.codeAgentRuntime.codexStdioSupervisor,
|
||
kafkaShadowProducer: spec.codeAgentRuntime.kafkaShadowProducer ?? null,
|
||
kafkaEventBridge: spec.codeAgentRuntime.kafkaEventBridge ?? null,
|
||
valuesPrinted: false,
|
||
},
|
||
runtimeImageRewrites: spec.runtimeImageRewrites,
|
||
runtimeImageBuilds: spec.runtimeImageBuilds,
|
||
registryPrefix: spec.registryPrefix,
|
||
buildkitSidecarImage: spec.buildkit?.sidecarImage,
|
||
publicWebUrl: spec.publicWebUrl,
|
||
publicApiUrl: spec.publicApiUrl,
|
||
publicExposure: spec.publicExposure === null ? undefined : spec.publicExposure.mode === "node-port-shared-edge" ? {
|
||
enabled: spec.publicExposure.enabled,
|
||
mode: spec.publicExposure.mode,
|
||
publicBaseUrl: spec.publicExposure.publicBaseUrl,
|
||
hostname: spec.publicExposure.hostname,
|
||
expectedA: spec.publicExposure.expectedA,
|
||
upstream: spec.publicExposure.upstream,
|
||
healthPath: spec.publicExposure.healthPath,
|
||
serviceName: spec.publicExposure.serviceName,
|
||
selector: spec.publicExposure.selector,
|
||
port: spec.publicExposure.port,
|
||
targetPort: spec.publicExposure.targetPort,
|
||
nodePort: spec.publicExposure.nodePort,
|
||
} : {
|
||
enabled: spec.publicExposure.enabled,
|
||
mode: spec.publicExposure.mode,
|
||
publicBaseUrl: spec.publicExposure.publicBaseUrl,
|
||
hostname: spec.publicExposure.hostname,
|
||
expectedA: spec.publicExposure.expectedA,
|
||
serverAddr: spec.publicExposure.serverAddr,
|
||
serverPort: spec.publicExposure.serverPort,
|
||
secretName: spec.publicExposure.secretName,
|
||
secretKey: spec.publicExposure.secretKey,
|
||
tokenKey: spec.publicExposure.tokenKey,
|
||
webProxy: spec.publicExposure.webProxy,
|
||
apiProxy: spec.publicExposure.apiProxy,
|
||
extraProxies: spec.publicExposure.extraProxies,
|
||
},
|
||
externalPostgres: activeExternalPostgres === undefined ? undefined : {
|
||
enabled: true,
|
||
serviceName: activeExternalPostgres.serviceName,
|
||
endpointAddress: activeExternalPostgres.endpointAddress,
|
||
port: activeExternalPostgres.port,
|
||
runtimeAccess: activeExternalPostgres.runtimeAccess ?? null,
|
||
sslmode: activeExternalPostgres.sslmode,
|
||
...(activeExternalPostgres.cloudApi.migrationSecretKey === undefined ? {} : {
|
||
migrationSecretName: activeExternalPostgres.cloudApi.secretName,
|
||
migrationSecretKey: activeExternalPostgres.cloudApi.migrationSecretKey,
|
||
}),
|
||
},
|
||
};
|
||
}
|
||
|
||
export function httpProxyEndpoint(value: string): { host: string; port: number } | null {
|
||
let parsed: URL;
|
||
try {
|
||
parsed = new URL(value);
|
||
} catch {
|
||
return null;
|
||
}
|
||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
|
||
const port = parsed.port === "" ? parsed.protocol === "https:" ? 443 : 80 : Number.parseInt(parsed.port, 10);
|
||
if (!Number.isInteger(port) || port <= 0) return null;
|
||
return { host: parsed.hostname, port };
|
||
}
|
||
|
||
export function nodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string): string[] {
|
||
return [
|
||
`${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`,
|
||
`${renderDir}/${spec.tektonDir}/rbac.yaml`,
|
||
`${renderDir}/${spec.tektonDir}/runtime-gitops-scripts.yaml`,
|
||
`${renderDir}/${spec.tektonDir}/pipeline.yaml`,
|
||
`${renderDir}/argocd/project.yaml`,
|
||
`${renderDir}/argocd/${spec.argoApplicationFile}`,
|
||
];
|
||
}
|
||
|
||
export function applyNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult {
|
||
const files = nodeRuntimeControlPlaneFiles(spec, renderDir);
|
||
const args = [
|
||
"kubectl",
|
||
"apply",
|
||
...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]),
|
||
...files.flatMap((file) => ["-f", file]),
|
||
];
|
||
return runNodeK3sArgs(spec, args, timeoutSeconds);
|
||
}
|
||
|
||
export function applyLocalNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult {
|
||
const manifest = nodeRuntimeControlPlaneFiles(spec, renderDir)
|
||
.map((file) => `---\n# Source: ${file}\n${readFileSync(file, "utf8").trimEnd()}\n`)
|
||
.join("");
|
||
const args = [
|
||
"kubectl",
|
||
"apply",
|
||
...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]),
|
||
"-f",
|
||
"-",
|
||
];
|
||
return runNodeK3sScript(spec, args.map(shellQuote).join(" "), timeoutSeconds, manifest);
|
||
}
|
||
|
||
export function cleanupNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, renderDir: string): CommandResult {
|
||
const prefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`;
|
||
const script = [
|
||
"set -eu",
|
||
`render_dir=${shellQuote(renderDir)}`,
|
||
`prefix=${shellQuote(prefix)}`,
|
||
"case \"$render_dir\" in",
|
||
" \"$prefix\"*) rm -rf \"$render_dir\" ;;",
|
||
" *) echo \"refusing to remove unexpected render dir: $render_dir\" >&2; exit 44 ;;",
|
||
"esac",
|
||
].join("\n");
|
||
return runNodeHostScript(spec, script, 60);
|
||
}
|
||
|
||
export function cleanupLocalNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, render: NodeRuntimeRenderResult): CommandResult {
|
||
const renderPrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`;
|
||
const worktreePrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-`;
|
||
if (!render.renderDir.startsWith(renderPrefix) || !render.worktreeDir.startsWith(worktreePrefix)) {
|
||
return {
|
||
command: ["rm", "-rf", "<refused>"],
|
||
cwd: repoRoot,
|
||
exitCode: 44,
|
||
stdout: "",
|
||
stderr: `refusing to remove unexpected local render paths: ${render.renderDir} ${render.worktreeDir}`,
|
||
signal: null,
|
||
timedOut: false,
|
||
};
|
||
}
|
||
return runCommand(["rm", "-rf", render.renderDir, render.worktreeDir], repoRoot, { timeoutMs: 60_000 });
|
||
}
|
||
|
||
export function nodeRuntimePipelineRunManifest(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string): Record<string, unknown> {
|
||
const gitWorkspaceSecret = hwlabNodeControlPlaneCiGitWorkspaceSecret(spec.nodeId, spec.lane);
|
||
return {
|
||
apiVersion: "tekton.dev/v1",
|
||
kind: "PipelineRun",
|
||
metadata: {
|
||
name: pipelineRun,
|
||
namespace: gitWorkspaceSecret.namespace,
|
||
labels: {
|
||
"app.kubernetes.io/part-of": "hwlab",
|
||
"hwlab.pikastech.local/gitops-target": spec.lane,
|
||
"hwlab.pikastech.local/source-commit": sourceCommit,
|
||
"hwlab.pikastech.local/trigger": "unidesk-node-cli",
|
||
},
|
||
annotations: {
|
||
"hwlab.pikastech.local/node": spec.nodeId,
|
||
"hwlab.pikastech.local/source-branch": spec.sourceBranch,
|
||
"hwlab.pikastech.local/gitops-branch": spec.gitopsBranch,
|
||
"hwlab.pikastech.local/runtime-path": spec.runtimePath,
|
||
"hwlab.pikastech.local/network-profile": spec.networkProfileId,
|
||
"hwlab.pikastech.local/download-profile": spec.downloadProfileId,
|
||
},
|
||
},
|
||
spec: {
|
||
pipelineRef: { name: spec.pipeline },
|
||
taskRunTemplate: {
|
||
serviceAccountName: spec.serviceAccountName,
|
||
podTemplate: {
|
||
hostNetwork: true,
|
||
dnsPolicy: "ClusterFirstWithHostNet",
|
||
securityContext: { fsGroup: 1000 },
|
||
},
|
||
},
|
||
params: [
|
||
{ name: "git-url", value: spec.gitUrl },
|
||
{ name: "git-read-url", value: spec.gitReadUrl },
|
||
{ name: "git-write-url", value: spec.gitWriteUrl },
|
||
{ name: "source-branch", value: spec.sourceBranch },
|
||
{ name: "gitops-branch", value: spec.gitopsBranch },
|
||
{ name: "lane", value: spec.lane },
|
||
{ name: "catalog-path", value: spec.catalogPath },
|
||
{ name: "image-tag-mode", value: "full" },
|
||
{ name: "runtime-path", value: spec.runtimePath },
|
||
{ name: "source-commit", value: sourceCommit },
|
||
{ name: "revision", value: sourceCommit },
|
||
{ name: "registry-prefix", value: spec.registryPrefix },
|
||
{ name: "services", value: spec.serviceIds.join(",") },
|
||
{ name: "base-image", value: spec.baseImage },
|
||
],
|
||
workspaces: [
|
||
{ name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } },
|
||
{ name: "git-ssh", secret: { secretName: gitWorkspaceSecret.name } },
|
||
],
|
||
},
|
||
};
|
||
}
|
||
|
||
export function createNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string, timeoutSeconds: number, rerun: boolean): CommandResult {
|
||
const gitWorkspaceSecret = hwlabNodeControlPlaneCiGitWorkspaceSecret(spec.nodeId, spec.lane);
|
||
const manifestB64 = Buffer.from(JSON.stringify(nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun)), "utf8").toString("base64");
|
||
const script = [
|
||
"set -eu",
|
||
`manifest_b64=${shellQuote(manifestB64)}`,
|
||
`pipeline_run=${shellQuote(pipelineRun)}`,
|
||
`ci_namespace=${shellQuote(gitWorkspaceSecret.namespace)}`,
|
||
`git_workspace_secret=${shellQuote(gitWorkspaceSecret.name)}`,
|
||
`rerun=${rerun ? "true" : "false"}`,
|
||
"manifest_path=\"/tmp/$pipeline_run.json\"",
|
||
"if ! kubectl -n \"$ci_namespace\" get secret \"$git_workspace_secret\" >/dev/null 2>&1; then",
|
||
" printf 'missing Tekton git workspace Secret %s/%s; run: bun scripts/cli.ts hwlab nodes control-plane infra apply --node %s --lane %s --confirm\\n' \"$ci_namespace\" \"$git_workspace_secret\" " + shellQuote(spec.nodeId) + " " + shellQuote(spec.lane) + " >&2",
|
||
" exit 44",
|
||
"fi",
|
||
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
||
"existing_status=$(kubectl get pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" -o 'jsonpath={.status.conditions[0].status}' 2>/dev/null || true)",
|
||
"if [ \"$rerun\" = true ] && [ -n \"$existing_status\" ]; then",
|
||
" kubectl delete pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
||
" kubectl delete taskrun -n \"$ci_namespace\" -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
||
" kubectl delete pod -n \"$ci_namespace\" -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
||
" kubectl -n \"$ci_namespace\" get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n \"$ci_namespace\" delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
||
"elif [ \"$existing_status\" = False ]; then",
|
||
" kubectl delete pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
||
" kubectl delete taskrun -n \"$ci_namespace\" -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
||
" kubectl delete pod -n \"$ci_namespace\" -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
||
" kubectl -n \"$ci_namespace\" get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n \"$ci_namespace\" delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
||
"fi",
|
||
"if kubectl create -f \"$manifest_path\"; then",
|
||
" :",
|
||
"else",
|
||
" code=$?",
|
||
" if kubectl get pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" >/dev/null 2>&1; then",
|
||
" printf 'PipelineRun %s already exists; reusing existing object\\n' \"$pipeline_run\" >&2",
|
||
" else",
|
||
" exit \"$code\"",
|
||
" fi",
|
||
"fi",
|
||
"kubectl get pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'",
|
||
].join("\n");
|
||
return runNodeK3sScript(spec, script, timeoutSeconds);
|
||
}
|
||
|
||
export function waitForNodeRuntimePipelineRunTerminal(
|
||
spec: HwlabRuntimeLaneSpec,
|
||
pipelineRun: string,
|
||
timeoutSeconds: number,
|
||
options: { opportunisticPostFlush?: () => Record<string, unknown> | null; opportunisticPostSync?: () => Record<string, unknown> | null } = {},
|
||
): Record<string, unknown> {
|
||
const severeWarningThresholdMs = 120_000;
|
||
const effectiveTimeoutSeconds = Math.max(0, timeoutSeconds);
|
||
const startedAt = Date.now();
|
||
const deadline = startedAt + effectiveTimeoutSeconds * 1000;
|
||
let polls = 0;
|
||
let last: Record<string, unknown> = { exists: false, name: pipelineRun };
|
||
let lastOpportunisticPostFlushAt = 0;
|
||
let severeWarningEmitted = false;
|
||
let opportunisticPostSyncAttempted = false;
|
||
const opportunisticPostFlushes: Record<string, unknown>[] = [];
|
||
const opportunisticPostSyncs: Record<string, unknown>[] = [];
|
||
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "started", pipelineRun, timeoutSeconds: effectiveTimeoutSeconds });
|
||
if (effectiveTimeoutSeconds <= 0) {
|
||
const warning = nodeRuntimeCicdWaitWarning(spec, pipelineRun, { polls, elapsedMs: 0 });
|
||
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "warning", pipelineRun, polls, elapsedMs: 0, waitLimitSeconds: effectiveTimeoutSeconds });
|
||
return {
|
||
ok: false,
|
||
status: "pending",
|
||
pipelineRun: last,
|
||
polls,
|
||
elapsedMs: 0,
|
||
waitLimitSeconds: effectiveTimeoutSeconds,
|
||
warning,
|
||
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun}` },
|
||
};
|
||
}
|
||
while (Date.now() <= deadline) {
|
||
polls += 1;
|
||
last = getNodeRuntimePipelineRun(spec, pipelineRun);
|
||
const status = typeof last.status === "string" ? last.status : null;
|
||
const reason = typeof last.reason === "string" ? last.reason : null;
|
||
const elapsedMs = Date.now() - startedAt;
|
||
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "poll", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs });
|
||
if (!severeWarningEmitted && elapsedMs >= severeWarningThresholdMs && status !== "True" && status !== "False") {
|
||
severeWarningEmitted = true;
|
||
printNodeRuntimeTriggerProgress(spec, {
|
||
stage: "pipelinerun-wait",
|
||
status: "warning",
|
||
warning: "pipelinerun-wait-severe-timeout",
|
||
severity: "warning",
|
||
pipelineRun,
|
||
pipelineStatus: status,
|
||
reason,
|
||
polls,
|
||
elapsedMs,
|
||
thresholdMs: severeWarningThresholdMs,
|
||
message: "PipelineRun has been non-terminal for more than 120s; this is a severe timeout and requires further investigation of Tekton taskRuns/pods instead of treating it as normal wait time.",
|
||
next: {
|
||
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
|
||
},
|
||
});
|
||
}
|
||
if (status === "True" || status === "False") {
|
||
const ok = status === "True";
|
||
const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun);
|
||
const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics);
|
||
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: ok ? "succeeded" : "failed", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs: Date.now() - startedAt });
|
||
return {
|
||
ok,
|
||
status: ok ? "succeeded" : "failed",
|
||
pipelineRun: last,
|
||
diagnostics,
|
||
failureSummary,
|
||
polls,
|
||
elapsedMs: Date.now() - startedAt,
|
||
opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined,
|
||
opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined,
|
||
degradedReason: ok ? undefined : "node-runtime-pipelinerun-failed",
|
||
};
|
||
}
|
||
if (
|
||
options.opportunisticPostSync !== undefined
|
||
&& !opportunisticPostSyncAttempted
|
||
&& nodeRuntimePipelineRunSourceCloneCompleted(spec, pipelineRun)
|
||
) {
|
||
opportunisticPostSyncAttempted = true;
|
||
const sync = options.opportunisticPostSync();
|
||
if (sync !== null) opportunisticPostSyncs.push(sync);
|
||
}
|
||
if (options.opportunisticPostFlush !== undefined && Date.now() - lastOpportunisticPostFlushAt >= 15_000) {
|
||
lastOpportunisticPostFlushAt = Date.now();
|
||
const flush = options.opportunisticPostFlush();
|
||
if (flush !== null) opportunisticPostFlushes.push(flush);
|
||
last = getNodeRuntimePipelineRun(spec, pipelineRun);
|
||
const refreshedStatus = typeof last.status === "string" ? last.status : null;
|
||
const refreshedReason = typeof last.reason === "string" ? last.reason : null;
|
||
const refreshedElapsedMs = Date.now() - startedAt;
|
||
if (refreshedStatus === "True" || refreshedStatus === "False") {
|
||
const ok = refreshedStatus === "True";
|
||
const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun);
|
||
const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics);
|
||
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: ok ? "succeeded" : "failed", pipelineRun, pipelineStatus: refreshedStatus, reason: refreshedReason, polls, elapsedMs: refreshedElapsedMs, observedAfterPostFlush: true });
|
||
return {
|
||
ok,
|
||
status: ok ? "succeeded" : "failed",
|
||
pipelineRun: last,
|
||
diagnostics,
|
||
failureSummary,
|
||
polls,
|
||
elapsedMs: refreshedElapsedMs,
|
||
opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined,
|
||
opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined,
|
||
observedAfterPostFlush: true,
|
||
degradedReason: ok ? undefined : "node-runtime-pipelinerun-failed",
|
||
};
|
||
}
|
||
}
|
||
sleepSync(Math.min(10_000, Math.max(1000, deadline - Date.now())));
|
||
}
|
||
const elapsedMs = Date.now() - startedAt;
|
||
const warning = nodeRuntimeCicdWaitWarning(spec, pipelineRun, { polls, elapsedMs });
|
||
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "warning", pipelineRun, polls, elapsedMs, waitLimitSeconds: timeoutSeconds });
|
||
return {
|
||
ok: false,
|
||
status: "pending",
|
||
pipelineRun: last,
|
||
polls,
|
||
elapsedMs,
|
||
waitLimitSeconds: timeoutSeconds,
|
||
warning,
|
||
opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined,
|
||
opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined,
|
||
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun}` },
|
||
};
|
||
}
|
||
|
||
export function nodeRuntimePipelineRunSourceCloneCompleted(spec: HwlabRuntimeLaneSpec, pipelineRun: string): boolean {
|
||
const result = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "taskrun", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", "json"], 60);
|
||
if (result.exitCode !== 0) return false;
|
||
let parsed: unknown;
|
||
try {
|
||
parsed = JSON.parse(result.stdout);
|
||
} catch {
|
||
return false;
|
||
}
|
||
const items = Array.isArray(record(parsed).items) ? record(parsed).items as unknown[] : [];
|
||
return items.some((item) => {
|
||
const itemRecord = record(item);
|
||
const metadata = record(itemRecord.metadata);
|
||
const labels = record(metadata.labels);
|
||
const name = typeof metadata.name === "string" ? metadata.name : "";
|
||
const pipelineTask = typeof labels["tekton.dev/pipelineTask"] === "string" ? labels["tekton.dev/pipelineTask"] : "";
|
||
const taskName = `${pipelineTask} ${name}`;
|
||
if (!/(clone|checkout|local-git|source[-_ ]*worktree)/iu.test(taskName)) return false;
|
||
const status = record(itemRecord.status);
|
||
const conditions = Array.isArray(status.conditions) ? status.conditions as unknown[] : [];
|
||
return conditions.some((condition) => {
|
||
const conditionRecord = record(condition);
|
||
return conditionRecord.type === "Succeeded" && conditionRecord.status === "True";
|
||
});
|
||
});
|
||
}
|
||
|
||
export function printNodeRuntimeTriggerProgress(spec: HwlabRuntimeLaneSpec, data: Record<string, unknown> = {}): void {
|
||
process.stderr.write(`${JSON.stringify({ event: "hwlab.runtime-lane.trigger.progress", at: new Date().toISOString(), lane: spec.lane, node: spec.nodeId, ...data })}\n`);
|
||
}
|
||
|
||
export function getNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, pipelineRun: string): Record<string, unknown> {
|
||
const result = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipelinerun", pipelineRun, "-o", "jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}{.metadata.creationTimestamp}{\"\\n\"}"], 60);
|
||
if (result.exitCode !== 0) return { exists: false, name: pipelineRun, result: compactRuntimeCommand(result) };
|
||
const [status = "", reason = "", message = "", createdAt = ""] = result.stdout.split(/\r?\n/u);
|
||
return {
|
||
exists: true,
|
||
name: pipelineRun,
|
||
status: status || null,
|
||
reason: reason || null,
|
||
message: message || null,
|
||
createdAt: createdAt || null,
|
||
result: compactRuntimeCommand(result),
|
||
};
|
||
}
|
||
|
||
export function sleepSync(ms: number): void {
|
||
const buffer = new SharedArrayBuffer(4);
|
||
Atomics.wait(new Int32Array(buffer), 0, 0, Math.max(0, ms));
|
||
}
|
||
|
||
export function syncNodeExternalPostgresSecrets(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record<string, unknown> | null {
|
||
const pg = hwlabRuntimeActiveExternalPostgres(spec);
|
||
if (pg === undefined) return null;
|
||
const secretRoot = externalPostgresSecretSourceRoot(spec);
|
||
const cloudApi = readSecretSourceValue(secretRoot, pg.cloudApi.sourceRef, pg.cloudApi.envKey);
|
||
const migration = pg.cloudApi.migrationEnvKey === undefined
|
||
? null
|
||
: readSecretSourceValue(secretRoot, pg.cloudApi.sourceRef, pg.cloudApi.migrationEnvKey);
|
||
const openfga = readSecretSourceValue(secretRoot, pg.openfga.sourceRef, pg.openfga.envKey);
|
||
const missing = [
|
||
...(cloudApi.ok ? [] : [`${pg.cloudApi.sourceRef}:${pg.cloudApi.envKey}`]),
|
||
...(migration === null || migration.ok ? [] : [`${pg.cloudApi.sourceRef}:${pg.cloudApi.migrationEnvKey}`]),
|
||
...(openfga.ok ? [] : [`${pg.openfga.sourceRef}:${pg.openfga.envKey}`]),
|
||
];
|
||
if (missing.length > 0) {
|
||
if (dryRun) {
|
||
return {
|
||
ok: true,
|
||
mode: "dry-run",
|
||
mutation: false,
|
||
skipped: true,
|
||
reason: "external-postgres-secret-source-missing",
|
||
missing,
|
||
sourceRoot: secretRoot,
|
||
valuesPrinted: false,
|
||
next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` },
|
||
};
|
||
}
|
||
return {
|
||
ok: false,
|
||
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
||
mutation: false,
|
||
degradedReason: "external-postgres-secret-source-missing",
|
||
missing,
|
||
sourceRoot: secretRoot,
|
||
valuesPrinted: false,
|
||
next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` },
|
||
};
|
||
}
|
||
const setup = runNodeK3sScript(spec, externalPostgresSecretSetupScript(spec, dryRun), timeoutSeconds);
|
||
if (!isCommandSuccess(setup)) {
|
||
return {
|
||
ok: false,
|
||
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
||
mutation: false,
|
||
phase: "namespace-authn-setup",
|
||
setup: compactRuntimeCommand(setup),
|
||
valuesPrinted: false,
|
||
};
|
||
}
|
||
const manifest = {
|
||
apiVersion: "v1",
|
||
kind: "List",
|
||
items: [
|
||
{
|
||
apiVersion: "v1",
|
||
kind: "Secret",
|
||
metadata: { name: pg.cloudApi.secretName, namespace: spec.runtimeNamespace },
|
||
type: "Opaque",
|
||
stringData: {
|
||
[pg.cloudApi.secretKey]: cloudApi.value,
|
||
...(migration === null || pg.cloudApi.migrationSecretKey === undefined ? {} : { [pg.cloudApi.migrationSecretKey]: migration.value }),
|
||
},
|
||
},
|
||
{
|
||
apiVersion: "v1",
|
||
kind: "Secret",
|
||
metadata: { name: pg.openfga.secretName, namespace: spec.runtimeNamespace },
|
||
type: "Opaque",
|
||
stringData: { [pg.openfga.secretKey]: openfga.value },
|
||
},
|
||
],
|
||
};
|
||
const applyScript = [
|
||
"set -eu",
|
||
[
|
||
"kubectl",
|
||
"apply",
|
||
"--server-side",
|
||
"--force-conflicts",
|
||
"--field-manager=unidesk-hwlab-node-external-postgres-secret",
|
||
...(dryRun ? ["--dry-run=server"] : []),
|
||
"-f",
|
||
"-",
|
||
].map(shellQuote).join(" "),
|
||
].join("\n");
|
||
const apply = runNodeK3sScript(spec, applyScript, timeoutSeconds, JSON.stringify(manifest));
|
||
return {
|
||
ok: isCommandSuccess(apply),
|
||
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
||
mutation: !dryRun && isCommandSuccess(apply),
|
||
namespace: spec.runtimeNamespace,
|
||
sourceRoot: secretRoot,
|
||
secrets: [
|
||
{ name: pg.cloudApi.secretName, key: pg.cloudApi.secretKey, sourceRef: pg.cloudApi.sourceRef, envKey: pg.cloudApi.envKey },
|
||
...(migration === null || pg.cloudApi.migrationSecretKey === undefined ? [] : [{ name: pg.cloudApi.secretName, key: pg.cloudApi.migrationSecretKey, sourceRef: pg.cloudApi.sourceRef, envKey: pg.cloudApi.migrationEnvKey }]),
|
||
{ name: pg.openfga.secretName, key: pg.openfga.secretKey, sourceRef: pg.openfga.sourceRef, envKey: pg.openfga.envKey, authnKey: pg.openfga.authnKey ?? null },
|
||
],
|
||
setup: compactRuntimeCommand(setup),
|
||
apply: compactRuntimeCommand(apply),
|
||
valuesPrinted: false,
|
||
};
|
||
}
|
||
|
||
export function syncNodeLocalPostgresBootstrapSecret(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record<string, unknown> | null {
|
||
const pg = spec.runtimeStore?.postgres;
|
||
if (pg?.mode !== "local-k3s") return null;
|
||
const secretName = pg.secretName ?? `${spec.runtimeNamespace}-postgres`;
|
||
const sourceRef = pg.adminPasswordSourceRef ?? "";
|
||
const sourceKey = pg.adminPasswordSourceKey ?? "";
|
||
if (sourceRef.length === 0 || sourceKey.length === 0) {
|
||
return {
|
||
ok: false,
|
||
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
||
mutation: false,
|
||
namespace: spec.runtimeNamespace,
|
||
secretName,
|
||
secretKey: "POSTGRES_PASSWORD",
|
||
degradedReason: "local-postgres-secret-source-not-configured",
|
||
valuesPrinted: false,
|
||
next: { fixYaml: `set runtimeStore.postgres.adminPasswordSourceRef/adminPasswordSourceKey for node=${spec.nodeId} lane=${spec.lane}` },
|
||
};
|
||
}
|
||
const material = readLocalPostgresPasswordMaterial({ sourceRef, sourceKey, dryRun });
|
||
if (material.ok !== true) {
|
||
return {
|
||
ok: false,
|
||
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
||
mutation: false,
|
||
namespace: spec.runtimeNamespace,
|
||
secretName,
|
||
secretKey: "POSTGRES_PASSWORD",
|
||
source: material,
|
||
degradedReason: material.error ?? "local-postgres-secret-source-missing",
|
||
valuesPrinted: false,
|
||
next: { createSource: `create ${material.sourcePath ?? join(".state", "secrets", sourceRef)} with ${sourceKey}=<redacted>` },
|
||
};
|
||
}
|
||
const result = runNodeK3sScript(spec, localPostgresBootstrapSecretScript(spec, secretName, sourceRef, sourceKey, dryRun, material.fingerprint), timeoutSeconds, material.value ?? "");
|
||
const fields = keyValueLinesFromText(statusText(result));
|
||
const afterBytes = numericField(fields.afterPasswordBytes) ?? 0;
|
||
const ok = isCommandSuccess(result) && fields.afterSecretExists === "yes" && afterBytes > 0;
|
||
return {
|
||
ok,
|
||
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
||
mutation: !dryRun && fields.mutation === "true",
|
||
namespace: spec.runtimeNamespace,
|
||
secretName,
|
||
secretKey: "POSTGRES_PASSWORD",
|
||
source: {
|
||
ok: material.ok,
|
||
sourceRef,
|
||
sourceKey,
|
||
sourcePath: material.sourcePath === null ? null : displayRepoPath(material.sourcePath),
|
||
generated: material.generated,
|
||
fingerprint: material.fingerprint,
|
||
valueBytes: material.value?.length ?? 0,
|
||
valuesPrinted: false,
|
||
},
|
||
before: {
|
||
namespaceExists: fields.beforeNamespaceExists === "yes",
|
||
secretExists: fields.beforeSecretExists === "yes",
|
||
passwordBytes: numericField(fields.beforePasswordBytes),
|
||
fingerprint: fields.beforePasswordFingerprint || null,
|
||
},
|
||
after: {
|
||
namespaceExists: fields.afterNamespaceExists === "yes",
|
||
secretExists: fields.afterSecretExists === "yes",
|
||
passwordBytes: numericField(fields.afterPasswordBytes),
|
||
fingerprint: fields.afterPasswordFingerprint || null,
|
||
},
|
||
action: fields.action || null,
|
||
applyExitCode: numericField(fields.applyExitCode),
|
||
namespaceCreateExitCode: numericField(fields.namespaceCreateExitCode),
|
||
degradedReason: ok ? undefined : fields.action === "source-mismatch" ? "local-postgres-secret-source-mismatch" : "local-postgres-secret-apply-failed",
|
||
result: compactRuntimeCommand(result),
|
||
valuesPrinted: false,
|
||
};
|
||
}
|
||
|
||
export function readLocalPostgresPasswordMaterial(input: { sourceRef: string; sourceKey: string; dryRun: boolean }): {
|
||
ok: boolean;
|
||
sourceRef: string;
|
||
sourceKey: string;
|
||
sourcePath: string | null;
|
||
checkedPaths: string[];
|
||
value: string | null;
|
||
fingerprint: string | null;
|
||
generated: boolean;
|
||
error?: string;
|
||
} {
|
||
const checkedPaths = localSecretSourcePaths(input.sourceRef);
|
||
const existingPath = checkedPaths.find((candidate) => existsSync(candidate)) ?? null;
|
||
const sourcePath = existingPath ?? checkedPaths[0] ?? null;
|
||
if (sourcePath === null) {
|
||
return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: null, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-path-unresolved" };
|
||
}
|
||
if (existingPath === null && input.dryRun) {
|
||
return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-missing" };
|
||
}
|
||
if (existingPath === null) {
|
||
const value = randomBytes(32).toString("base64url");
|
||
mkdirSync(dirname(sourcePath), { recursive: true });
|
||
writeFileSync(sourcePath, `${input.sourceKey}=${value}\n`, { mode: 0o600 });
|
||
return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath, checkedPaths, value, fingerprint: shortSecretFingerprint(value), generated: true };
|
||
}
|
||
const text = readFileSync(existingPath, "utf8");
|
||
const values = parseEnvFile(text);
|
||
const existingValue = values[input.sourceKey];
|
||
if (existingValue !== undefined && existingValue.length > 0) {
|
||
return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value: existingValue, fingerprint: shortSecretFingerprint(existingValue), generated: false };
|
||
}
|
||
if (input.dryRun) {
|
||
return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-key-missing" };
|
||
}
|
||
const value = randomBytes(32).toString("base64url");
|
||
const prefix = text.length === 0 || text.endsWith("\n") ? "" : "\n";
|
||
writeFileSync(existingPath, `${text}${prefix}${input.sourceKey}=${value}\n`, { mode: 0o600 });
|
||
return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value, fingerprint: shortSecretFingerprint(value), generated: true };
|
||
}
|
||
|
||
export function localSecretSourcePaths(sourceRef: string): string[] {
|
||
if (isAbsolute(sourceRef)) return [sourceRef];
|
||
return [rootPath(".state", "secrets", sourceRef)];
|
||
}
|
||
|
||
export function shortSecretFingerprint(value: string): string {
|
||
return `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`;
|
||
}
|
||
|
||
export function localPostgresBootstrapSecretScript(spec: HwlabRuntimeLaneSpec, secretName: string, sourceRef: string, sourceKey: string, dryRun: boolean, sourceFingerprint: string | null): string {
|
||
return [
|
||
"set +e",
|
||
`namespace=${shellQuote(spec.runtimeNamespace)}`,
|
||
`secret=${shellQuote(secretName)}`,
|
||
"secret_key=POSTGRES_PASSWORD",
|
||
`source_ref=${shellQuote(sourceRef)}`,
|
||
`source_key=${shellQuote(sourceKey)}`,
|
||
`source_fingerprint=${shellQuote(sourceFingerprint ?? "")}`,
|
||
`dry_run=${shellQuote(dryRun ? "true" : "false")}`,
|
||
"field_manager=unidesk-hwlab-node-local-postgres-secret",
|
||
"password=$(cat)",
|
||
"password_bytes=$(printf '%s' \"$password\" | wc -c | tr -d ' ')",
|
||
"exists_flag() { kubectl -n \"$namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }",
|
||
"namespace_exists_flag() { kubectl get namespace \"$namespace\" >/dev/null 2>&1 && printf yes || printf no; }",
|
||
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$secret\" -o \"go-template={{ index .data \\\"$secret_key\\\" }}\" 2>/dev/null || true; }",
|
||
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
|
||
"decoded_fingerprint() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | sha256sum | awk '{print \"sha256:\" substr($1,1,16)}'; fi; }",
|
||
"before_namespace_exists=$(namespace_exists_flag)",
|
||
"before_secret_exists=no",
|
||
"before_b64=",
|
||
"if [ \"$before_namespace_exists\" = yes ]; then before_secret_exists=$(exists_flag secret \"$secret\"); before_b64=$(secret_b64_key); fi",
|
||
"before_password_bytes=$(decoded_length \"$before_b64\")",
|
||
"before_password_fingerprint=$(decoded_fingerprint \"$before_b64\")",
|
||
"after_namespace_exists=$before_namespace_exists",
|
||
"after_secret_exists=$before_secret_exists",
|
||
"after_password_bytes=$before_password_bytes",
|
||
"after_password_fingerprint=$before_password_fingerprint",
|
||
"action=observed",
|
||
"mutation=false",
|
||
"namespace_create_exit=",
|
||
"apply_exit=",
|
||
"if [ \"$dry_run\" = true ]; then",
|
||
" if [ \"$before_secret_exists\" != yes ] || [ \"$before_password_bytes\" -le 0 ]; then action=would-ensure; else action=kept; fi",
|
||
"elif [ \"$password_bytes\" -le 0 ]; then",
|
||
" action=source-empty",
|
||
" apply_exit=42",
|
||
"elif [ \"$before_password_bytes\" -gt 0 ] && [ -n \"$source_fingerprint\" ] && [ \"$before_password_fingerprint\" != \"$source_fingerprint\" ]; then",
|
||
" action=source-mismatch",
|
||
" apply_exit=47",
|
||
"else",
|
||
" if [ \"$before_namespace_exists\" != yes ]; then",
|
||
" kubectl create namespace \"$namespace\" >/tmp/hwlab-local-postgres-namespace.out 2>/tmp/hwlab-local-postgres-namespace.err",
|
||
" namespace_create_exit=$?",
|
||
" else",
|
||
" namespace_create_exit=0",
|
||
" fi",
|
||
" if [ \"$namespace_create_exit\" = 0 ]; then",
|
||
" tmp=$(mktemp /tmp/hwlab-local-postgres-secret.XXXXXX.yaml)",
|
||
" password_b64=$(printf '%s' \"$password\" | base64 | tr -d '\\n')",
|
||
" cat >\"$tmp\" <<EOF_SECRET",
|
||
"apiVersion: v1",
|
||
"kind: Secret",
|
||
"metadata:",
|
||
" name: $secret",
|
||
" namespace: $namespace",
|
||
" labels:",
|
||
" app.kubernetes.io/part-of: hwlab",
|
||
" app.kubernetes.io/managed-by: unidesk",
|
||
" annotations:",
|
||
" hwlab.pikastech.local/secret-source-ref: $source_ref",
|
||
" hwlab.pikastech.local/secret-source-key: $source_key",
|
||
" hwlab.pikastech.local/secret-source-fingerprint: $source_fingerprint",
|
||
"type: Opaque",
|
||
"data:",
|
||
" $secret_key: $password_b64",
|
||
"EOF_SECRET",
|
||
" kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f \"$tmp\" >/tmp/hwlab-local-postgres-secret.apply.out 2>/tmp/hwlab-local-postgres-secret.apply.err",
|
||
" apply_exit=$?",
|
||
" rm -f \"$tmp\"",
|
||
" if [ \"$apply_exit\" = 0 ]; then action=ensured; mutation=true; else action=apply-failed; fi",
|
||
" else",
|
||
" action=namespace-create-failed",
|
||
" apply_exit=$namespace_create_exit",
|
||
" fi",
|
||
"fi",
|
||
"after_namespace_exists=$(namespace_exists_flag)",
|
||
"if [ \"$after_namespace_exists\" = yes ]; then after_secret_exists=$(exists_flag secret \"$secret\"); after_b64=$(secret_b64_key); else after_secret_exists=no; after_b64=; fi",
|
||
"after_password_bytes=$(decoded_length \"$after_b64\")",
|
||
"after_password_fingerprint=$(decoded_fingerprint \"$after_b64\")",
|
||
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
||
"printf 'secret\\t%s\\n' \"$secret\"",
|
||
"printf 'key\\t%s\\n' \"$secret_key\"",
|
||
"printf 'sourceRef\\t%s\\n' \"$source_ref\"",
|
||
"printf 'sourceKey\\t%s\\n' \"$source_key\"",
|
||
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
||
"printf 'action\\t%s\\n' \"$action\"",
|
||
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
||
"printf 'beforeNamespaceExists\\t%s\\n' \"$before_namespace_exists\"",
|
||
"printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"",
|
||
"printf 'beforePasswordBytes\\t%s\\n' \"$before_password_bytes\"",
|
||
"printf 'beforePasswordFingerprint\\t%s\\n' \"$before_password_fingerprint\"",
|
||
"printf 'afterNamespaceExists\\t%s\\n' \"$after_namespace_exists\"",
|
||
"printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"",
|
||
"printf 'afterPasswordBytes\\t%s\\n' \"$after_password_bytes\"",
|
||
"printf 'afterPasswordFingerprint\\t%s\\n' \"$after_password_fingerprint\"",
|
||
"printf 'namespaceCreateExitCode\\t%s\\n' \"$namespace_create_exit\"",
|
||
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
|
||
"password=",
|
||
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
|
||
"if [ \"$after_secret_exists\" != yes ] || [ \"$after_password_bytes\" -le 0 ]; then exit 48; fi",
|
||
].join("\n");
|
||
}
|
||
|
||
interface NodeRuntimeImageDependency {
|
||
id: string;
|
||
target: string;
|
||
source: string;
|
||
mode: "pull" | "build-moonbridge" | "build-opencode-git";
|
||
sourceRepo?: string;
|
||
sourceRef?: string;
|
||
builderImage?: string;
|
||
goProxy?: string;
|
||
dockerNetworkMode?: "default" | "host";
|
||
}
|
||
|
||
function nodeRuntimeImageDependencies(spec: HwlabRuntimeLaneSpec): NodeRuntimeImageDependency[] {
|
||
const images: NodeRuntimeImageDependency[] = [
|
||
{ id: "base", target: spec.baseImage, source: spec.baseImageSource ?? "", mode: "pull" },
|
||
];
|
||
if (spec.buildkit !== undefined) {
|
||
images.push({ id: "buildkit", target: spec.buildkit.sidecarImage, source: spec.buildkit.sourceImage, mode: "pull" });
|
||
}
|
||
spec.runtimeImageRewrites.forEach((rewrite, index) => {
|
||
images.push({ id: `rewrite-${index + 1}`, target: rewrite.target, source: rewrite.source, mode: "pull" });
|
||
});
|
||
spec.runtimeImageBuilds.forEach((build) => {
|
||
if (build.kind === "moonbridge") {
|
||
images.push({
|
||
id: build.id,
|
||
target: build.target,
|
||
source: `${build.sourceRepo}#${build.sourceRef}`,
|
||
mode: "build-moonbridge",
|
||
sourceRepo: build.sourceRepo,
|
||
sourceRef: build.sourceRef,
|
||
builderImage: build.builderImage,
|
||
goProxy: build.goProxy,
|
||
dockerNetworkMode: build.dockerNetworkMode,
|
||
});
|
||
return;
|
||
}
|
||
images.push({
|
||
id: build.id,
|
||
target: build.target,
|
||
source: build.sourceImage ?? "",
|
||
mode: "build-opencode-git",
|
||
dockerNetworkMode: build.dockerNetworkMode,
|
||
});
|
||
});
|
||
return images;
|
||
}
|
||
|
||
function nodeRuntimeImageCalls(spec: HwlabRuntimeLaneSpec, functionName: string): string {
|
||
return nodeRuntimeImageDependencies(spec)
|
||
.map((image) => [
|
||
functionName,
|
||
image.id,
|
||
image.target,
|
||
image.source,
|
||
image.mode,
|
||
image.sourceRepo ?? "",
|
||
image.sourceRef ?? "",
|
||
image.builderImage ?? "",
|
||
image.goProxy ?? "",
|
||
image.dockerNetworkMode ?? "",
|
||
].map(shellQuote).join(" "))
|
||
.join("\n");
|
||
}
|
||
|
||
function parseNodeRuntimeImageRows(text: string): Record<string, unknown>[] {
|
||
return text.split(/\r?\n/u).flatMap((line) => {
|
||
if (!line.startsWith("imageJson\t")) return [];
|
||
try {
|
||
return [JSON.parse(line.slice("imageJson\t".length)) as Record<string, unknown>];
|
||
} catch {
|
||
return [];
|
||
}
|
||
});
|
||
}
|
||
|
||
export function nodeRuntimeBaseImageStatus(spec: HwlabRuntimeLaneSpec, timeoutSeconds: number): Record<string, unknown> {
|
||
const script = [
|
||
"set -eu",
|
||
"check_image() {",
|
||
" id=\"$1\"; target=\"$2\"; source=\"$3\"; mode=\"$4\"",
|
||
" repo_tag=${target#*/}",
|
||
" repo=${repo_tag%:*}",
|
||
" tag=${repo_tag##*:}",
|
||
" if [ \"$repo\" = \"$repo_tag\" ]; then tag=latest; fi",
|
||
" registry_url=\"http://127.0.0.1:5000/v2/$repo/tags/list\"",
|
||
" registry_present=false",
|
||
" if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then registry_present=true; fi",
|
||
" target_present=false",
|
||
" if docker image inspect \"$target\" >/dev/null 2>&1; then target_present=true; fi",
|
||
" source_present=false",
|
||
" if [ \"$mode\" = build-moonbridge ]; then source_present=true; elif [ -n \"$source\" ] && docker image inspect \"$source\" >/dev/null 2>&1; then source_present=true; fi",
|
||
" source_configured=false",
|
||
" if [ -n \"$source\" ]; then source_configured=true; fi",
|
||
" python3 - \"$id\" \"$target\" \"$source\" \"$mode\" \"$source_configured\" \"$registry_url\" \"$registry_present\" \"$target_present\" \"$source_present\" <<'PY'",
|
||
"import json, sys",
|
||
"keys=['id','target','source','mode','sourceConfigured','registryUrl','registryTagPresent','targetImagePresent','sourceImagePresent']",
|
||
"data=dict(zip(keys, sys.argv[1:]))",
|
||
"for key in ['sourceConfigured','registryTagPresent','targetImagePresent','sourceImagePresent']:",
|
||
" data[key] = data[key] == 'true'",
|
||
"print('imageJson\\t' + json.dumps(data, separators=(',', ':')))",
|
||
"PY",
|
||
"}",
|
||
nodeRuntimeImageCalls(spec, "check_image"),
|
||
].join("\n");
|
||
const result = runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300));
|
||
const imageRows = parseNodeRuntimeImageRows(statusText(result));
|
||
const base = imageRows.find((image) => image.id === "base") ?? {};
|
||
return {
|
||
ok: isCommandSuccess(result) && imageRows.length === nodeRuntimeImageDependencies(spec).length && imageRows.every((image) => image.sourceConfigured === true && image.registryTagPresent === true),
|
||
target: base.target ?? spec.baseImage,
|
||
source: base.source || null,
|
||
sourceConfigured: base.sourceConfigured === true,
|
||
registryUrl: base.registryUrl ?? null,
|
||
registryTagPresent: base.registryTagPresent === true,
|
||
targetImagePresent: base.targetImagePresent === true,
|
||
sourceImagePresent: base.sourceImagePresent === true,
|
||
images: imageRows,
|
||
result: compactRuntimeCommand(result),
|
||
};
|
||
}
|
||
|
||
export function ensureNodeBaseImage(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record<string, unknown> | null {
|
||
if (spec.baseImageSource === undefined) return null;
|
||
const script = [
|
||
"set -eu",
|
||
`dry_run=${shellQuote(dryRun ? "true" : "false")}`,
|
||
`pull_retries=${shellQuote(String(Math.max(1, spec.downloadProfile.docker.pullRetries)))}`,
|
||
`build_http_proxy=${shellQuote(spec.networkProfile.dockerBuildProxy.http)}`,
|
||
`build_https_proxy=${shellQuote(spec.networkProfile.dockerBuildProxy.https)}`,
|
||
`build_all_proxy=${shellQuote(spec.networkProfile.dockerBuildProxy.all)}`,
|
||
`build_no_proxy=${shellQuote(spec.networkProfile.dockerBuildProxy.noProxy.join(","))}`,
|
||
"resolve_host_proxy_url() {",
|
||
" raw_url=\"$1\"",
|
||
" if [ -z \"$raw_url\" ]; then printf '\\n'; return 0; fi",
|
||
" info=$(python3 - \"$raw_url\" <<'PY'",
|
||
"from urllib.parse import urlparse",
|
||
"import sys",
|
||
"url = sys.argv[1]",
|
||
"parsed = urlparse(url)",
|
||
"host = parsed.hostname or ''",
|
||
"port = parsed.port",
|
||
"scheme = parsed.scheme or 'http'",
|
||
"parts = host.split('.')",
|
||
"if len(parts) >= 4 and parts[2] == 'svc':",
|
||
" print('\\t'.join(['service', scheme, parts[0], parts[1], str(port or 80)]))",
|
||
"else:",
|
||
" print('\\t'.join(['literal', url]))",
|
||
"PY",
|
||
" )",
|
||
" kind=$(printf '%s' \"$info\" | cut -f1)",
|
||
" service_cluster_ip() {",
|
||
" service_namespace=\"$1\"; service_name=\"$2\"",
|
||
" cluster_ip=\"\"",
|
||
" if command -v kubectl >/dev/null 2>&1; then",
|
||
" cluster_ip=$(kubectl -n \"$service_namespace\" get svc \"$service_name\" -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true)",
|
||
" fi",
|
||
" if { [ -z \"$cluster_ip\" ] || [ \"$cluster_ip\" = None ]; } && command -v k3s >/dev/null 2>&1; then",
|
||
" cluster_ip=$(k3s kubectl -n \"$service_namespace\" get svc \"$service_name\" -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true)",
|
||
" fi",
|
||
" printf '%s\\n' \"$cluster_ip\"",
|
||
" }",
|
||
" if [ \"$kind\" = service ]; then",
|
||
" scheme=$(printf '%s' \"$info\" | cut -f2)",
|
||
" service_name=$(printf '%s' \"$info\" | cut -f3)",
|
||
" service_namespace=$(printf '%s' \"$info\" | cut -f4)",
|
||
" service_port=$(printf '%s' \"$info\" | cut -f5)",
|
||
" cluster_ip=$(service_cluster_ip \"$service_namespace\" \"$service_name\")",
|
||
" if [ -n \"$cluster_ip\" ] && [ \"$cluster_ip\" != None ]; then printf '%s://%s:%s\\n' \"$scheme\" \"$cluster_ip\" \"$service_port\"; else printf '%s\\n' \"$raw_url\"; fi",
|
||
" else",
|
||
" printf '%s\\n' \"$raw_url\"",
|
||
" fi",
|
||
"}",
|
||
"build_http_proxy=$(resolve_host_proxy_url \"$build_http_proxy\")",
|
||
"build_https_proxy=$(resolve_host_proxy_url \"$build_https_proxy\")",
|
||
"build_all_proxy=$(resolve_host_proxy_url \"$build_all_proxy\")",
|
||
"build_container_proxy_url() {",
|
||
" raw_url=\"$1\"",
|
||
" if [ -z \"$raw_url\" ]; then printf '\\n'; return 0; fi",
|
||
" python3 - \"$raw_url\" <<'PY'",
|
||
"from urllib.parse import urlparse, urlunparse",
|
||
"import sys",
|
||
"url = sys.argv[1]",
|
||
"parsed = urlparse(url)",
|
||
"host = parsed.hostname or ''",
|
||
"if host in {'127.0.0.1', 'localhost'}:",
|
||
" netloc = 'host.docker.internal'",
|
||
" if parsed.port is not None:",
|
||
" netloc = f'{netloc}:{parsed.port}'",
|
||
" print(urlunparse(parsed._replace(netloc=netloc)))",
|
||
"else:",
|
||
" print(url)",
|
||
"PY",
|
||
"}",
|
||
"build_container_http_proxy=$(build_container_proxy_url \"$build_http_proxy\")",
|
||
"build_container_https_proxy=$(build_container_proxy_url \"$build_https_proxy\")",
|
||
"build_container_all_proxy=$(build_container_proxy_url \"$build_all_proxy\")",
|
||
"docker_build_add_host_args=\"\"",
|
||
"case \"$build_container_http_proxy $build_container_https_proxy $build_container_all_proxy\" in",
|
||
" *host.docker.internal*) docker_build_add_host_args=\"--add-host host.docker.internal:host-gateway\" ;;",
|
||
"esac",
|
||
"failed=false",
|
||
"ensure_image() {",
|
||
" id=\"$1\"; target=\"$2\"; source=\"$3\"; mode=\"$4\"; source_repo=\"$5\"; source_ref=\"$6\"; builder_image=\"$7\"; go_proxy=\"$8\"; docker_network_mode=\"$9\"",
|
||
" repo_tag=${target#*/}",
|
||
" repo=${repo_tag%:*}",
|
||
" tag=${repo_tag##*:}",
|
||
" if [ \"$repo\" = \"$repo_tag\" ]; then tag=latest; fi",
|
||
" registry_url=\"http://127.0.0.1:5000/v2/$repo/tags/list\"",
|
||
" present=false",
|
||
" if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then present=true; fi",
|
||
" action=none",
|
||
" source_present_before=unknown",
|
||
" pulled_source=false",
|
||
" pull_attempts=0",
|
||
" if [ \"$present\" = false ]; then",
|
||
" action=seed",
|
||
" if [ \"$dry_run\" = false ]; then",
|
||
" if [ \"$mode\" = build-moonbridge ]; then",
|
||
" action=build",
|
||
" source_present_before=build-source",
|
||
" effective_build_container_http_proxy=\"$build_container_http_proxy\"",
|
||
" effective_build_container_https_proxy=\"$build_container_https_proxy\"",
|
||
" effective_build_container_all_proxy=\"$build_container_all_proxy\"",
|
||
" effective_docker_build_add_host_args=\"$docker_build_add_host_args\"",
|
||
" if [ \"$docker_network_mode\" = host ]; then",
|
||
" effective_build_container_http_proxy=\"$build_http_proxy\"",
|
||
" effective_build_container_https_proxy=\"$build_https_proxy\"",
|
||
" effective_build_container_all_proxy=\"$build_all_proxy\"",
|
||
" effective_docker_build_add_host_args=\"\"",
|
||
" fi",
|
||
" tmpdir=$(mktemp -d /tmp/hwlab-node-runtime-image-$id.XXXXXX)",
|
||
" dockerfile=\"$tmpdir/Dockerfile\"",
|
||
" cat > \"$dockerfile\" <<'DOCKERFILE'",
|
||
"ARG BUILDER_IMAGE",
|
||
"FROM ${BUILDER_IMAGE} AS builder",
|
||
"ARG MOONBRIDGE_REPO",
|
||
"ARG MOONBRIDGE_REF",
|
||
"ARG GOPROXY_VALUE",
|
||
"ARG HTTP_PROXY",
|
||
"ARG HTTPS_PROXY",
|
||
"ARG ALL_PROXY",
|
||
"ARG NO_PROXY",
|
||
"ENV GOPROXY=${GOPROXY_VALUE}",
|
||
"ENV HTTP_PROXY=${HTTP_PROXY}",
|
||
"ENV HTTPS_PROXY=${HTTPS_PROXY}",
|
||
"ENV ALL_PROXY=${ALL_PROXY}",
|
||
"ENV NO_PROXY=${NO_PROXY}",
|
||
"ENV http_proxy=${HTTP_PROXY}",
|
||
"ENV https_proxy=${HTTPS_PROXY}",
|
||
"ENV all_proxy=${ALL_PROXY}",
|
||
"ENV no_proxy=${NO_PROXY}",
|
||
"WORKDIR /src",
|
||
"RUN git clone --filter=blob:none \"${MOONBRIDGE_REPO}\" moon-bridge",
|
||
"WORKDIR /src/moon-bridge",
|
||
"RUN git checkout \"${MOONBRIDGE_REF}\" \\",
|
||
" && go mod download \\",
|
||
" && CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags=\"-s -w\" -o /out/moonbridge ./cmd/moonbridge",
|
||
"FROM scratch",
|
||
"WORKDIR /app",
|
||
"COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt",
|
||
"COPY --from=builder /out/moonbridge /app/moonbridge",
|
||
"EXPOSE 4001",
|
||
"USER 65532:65532",
|
||
"ENTRYPOINT [\"/app/moonbridge\"]",
|
||
"DOCKERFILE",
|
||
" if env HTTP_PROXY=\"$build_http_proxy\" HTTPS_PROXY=\"$build_https_proxy\" ALL_PROXY=\"$build_all_proxy\" NO_PROXY=\"$build_no_proxy\" http_proxy=\"$build_http_proxy\" https_proxy=\"$build_https_proxy\" all_proxy=\"$build_all_proxy\" no_proxy=\"$build_no_proxy\" docker build $effective_docker_build_add_host_args --network \"$docker_network_mode\" --build-arg BUILDER_IMAGE=\"$builder_image\" --build-arg MOONBRIDGE_REPO=\"$source_repo\" --build-arg MOONBRIDGE_REF=\"$source_ref\" --build-arg GOPROXY_VALUE=\"$go_proxy\" --build-arg HTTP_PROXY=\"$effective_build_container_http_proxy\" --build-arg HTTPS_PROXY=\"$effective_build_container_https_proxy\" --build-arg ALL_PROXY=\"$effective_build_container_all_proxy\" --build-arg NO_PROXY=\"$build_no_proxy\" --build-arg http_proxy=\"$effective_build_container_http_proxy\" --build-arg https_proxy=\"$effective_build_container_https_proxy\" --build-arg all_proxy=\"$effective_build_container_all_proxy\" --build-arg no_proxy=\"$build_no_proxy\" -t \"$target\" -f \"$dockerfile\" \"$tmpdir\" >/tmp/hwlab-node-runtime-image-$id-build.out 2>&1; then",
|
||
" docker push \"$target\" >/tmp/hwlab-node-runtime-image-$id-push.out 2>&1 || { cat /tmp/hwlab-node-runtime-image-$id-push.out >&2 2>/dev/null || true; failed=true; }",
|
||
" else",
|
||
" cat /tmp/hwlab-node-runtime-image-$id-build.out >&2 2>/dev/null || true",
|
||
" failed=true",
|
||
" fi",
|
||
" rm -rf \"$tmpdir\"",
|
||
" elif [ \"$mode\" = build-opencode-git ]; then",
|
||
" action=build",
|
||
" source_present_before=false",
|
||
" if [ -n \"$source\" ] && docker image inspect \"$source\" >/dev/null 2>&1; then source_present_before=true; fi",
|
||
" effective_build_container_http_proxy=\"$build_container_http_proxy\"",
|
||
" effective_build_container_https_proxy=\"$build_container_https_proxy\"",
|
||
" effective_build_container_all_proxy=\"$build_container_all_proxy\"",
|
||
" effective_docker_build_add_host_args=\"$docker_build_add_host_args\"",
|
||
" if [ \"$docker_network_mode\" = host ]; then",
|
||
" effective_build_container_http_proxy=\"$build_http_proxy\"",
|
||
" effective_build_container_https_proxy=\"$build_https_proxy\"",
|
||
" effective_build_container_all_proxy=\"$build_all_proxy\"",
|
||
" effective_docker_build_add_host_args=\"\"",
|
||
" fi",
|
||
" tmpdir=$(mktemp -d /tmp/hwlab-node-runtime-image-$id.XXXXXX)",
|
||
" dockerfile=\"$tmpdir/Dockerfile\"",
|
||
" cat > \"$dockerfile\" <<'DOCKERFILE'",
|
||
"ARG SOURCE_IMAGE",
|
||
"FROM ${SOURCE_IMAGE}",
|
||
"ARG HTTP_PROXY",
|
||
"ARG HTTPS_PROXY",
|
||
"ARG ALL_PROXY",
|
||
"ARG NO_PROXY",
|
||
"ENV HTTP_PROXY=${HTTP_PROXY}",
|
||
"ENV HTTPS_PROXY=${HTTPS_PROXY}",
|
||
"ENV ALL_PROXY=${ALL_PROXY}",
|
||
"ENV NO_PROXY=${NO_PROXY}",
|
||
"ENV http_proxy=${HTTP_PROXY}",
|
||
"ENV https_proxy=${HTTPS_PROXY}",
|
||
"ENV all_proxy=${ALL_PROXY}",
|
||
"ENV no_proxy=${NO_PROXY}",
|
||
"RUN apk add --no-cache git",
|
||
"DOCKERFILE",
|
||
" if env HTTP_PROXY=\"$build_http_proxy\" HTTPS_PROXY=\"$build_https_proxy\" ALL_PROXY=\"$build_all_proxy\" NO_PROXY=\"$build_no_proxy\" http_proxy=\"$build_http_proxy\" https_proxy=\"$build_https_proxy\" all_proxy=\"$build_all_proxy\" no_proxy=\"$build_no_proxy\" docker build $effective_docker_build_add_host_args --network \"$docker_network_mode\" --build-arg SOURCE_IMAGE=\"$source\" --build-arg HTTP_PROXY=\"$effective_build_container_http_proxy\" --build-arg HTTPS_PROXY=\"$effective_build_container_https_proxy\" --build-arg ALL_PROXY=\"$effective_build_container_all_proxy\" --build-arg NO_PROXY=\"$build_no_proxy\" --build-arg http_proxy=\"$effective_build_container_http_proxy\" --build-arg https_proxy=\"$effective_build_container_https_proxy\" --build-arg all_proxy=\"$effective_build_container_all_proxy\" --build-arg no_proxy=\"$build_no_proxy\" -t \"$target\" -f \"$dockerfile\" \"$tmpdir\" >/tmp/hwlab-node-runtime-image-$id-build.out 2>&1; then",
|
||
" docker push \"$target\" >/tmp/hwlab-node-runtime-image-$id-push.out 2>&1 || { cat /tmp/hwlab-node-runtime-image-$id-push.out >&2 2>/dev/null || true; failed=true; }",
|
||
" else",
|
||
" cat /tmp/hwlab-node-runtime-image-$id-build.out >&2 2>/dev/null || true",
|
||
" failed=true",
|
||
" fi",
|
||
" rm -rf \"$tmpdir\"",
|
||
" else",
|
||
" source_present_before=true",
|
||
" if ! docker image inspect \"$source\" >/dev/null 2>&1; then",
|
||
" source_present_before=false",
|
||
" attempt=1",
|
||
" while [ \"$attempt\" -le \"$pull_retries\" ]; do",
|
||
" pull_attempts=$attempt",
|
||
" if env HTTP_PROXY=\"$build_http_proxy\" HTTPS_PROXY=\"$build_https_proxy\" ALL_PROXY=\"$build_all_proxy\" NO_PROXY=\"$build_no_proxy\" http_proxy=\"$build_http_proxy\" https_proxy=\"$build_https_proxy\" all_proxy=\"$build_all_proxy\" no_proxy=\"$build_no_proxy\" docker pull \"$source\" >/tmp/hwlab-node-runtime-image-$id-pull.out 2>&1; then pulled_source=true; break; fi",
|
||
" attempt=$((attempt + 1))",
|
||
" sleep 2",
|
||
" done",
|
||
" if [ \"$pulled_source\" != true ]; then cat /tmp/hwlab-node-runtime-image-$id-pull.out >&2 2>/dev/null || true; failed=true; fi",
|
||
" fi",
|
||
" if [ \"$failed\" != true ]; then",
|
||
" if docker image inspect \"$source\" >/dev/null 2>&1; then",
|
||
" docker tag \"$source\" \"$target\"",
|
||
" docker push \"$target\" >/tmp/hwlab-node-runtime-image-$id-push.out 2>&1 || failed=true",
|
||
" else",
|
||
" failed=true",
|
||
" fi",
|
||
" fi",
|
||
" fi",
|
||
" fi",
|
||
" fi",
|
||
" after=false",
|
||
" if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then after=true; fi",
|
||
" python3 - \"$id\" \"$target\" \"$source\" \"$mode\" \"$dry_run\" \"$present\" \"$after\" \"$action\" \"$pull_retries\" \"$source_present_before\" \"$pulled_source\" \"$pull_attempts\" <<'PY'",
|
||
"import json, sys",
|
||
"keys=['id','target','source','mode','dryRun','presentBefore','presentAfter','action','pullRetries','sourcePresentBefore','pulledSource','pullAttempts']",
|
||
"data=dict(zip(keys, sys.argv[1:]))",
|
||
"for key in ['dryRun','presentBefore','presentAfter','pulledSource']:",
|
||
" data[key] = data[key] == 'true'",
|
||
"for key in ['pullRetries','pullAttempts']:",
|
||
" data[key] = int(data[key]) if str(data[key]).isdigit() else None",
|
||
"print('imageJson\\t' + json.dumps(data, separators=(',', ':')))",
|
||
"PY",
|
||
" if [ \"$dry_run\" = false ] && [ \"$after\" != true ]; then failed=true; fi",
|
||
"}",
|
||
nodeRuntimeImageCalls(spec, "ensure_image"),
|
||
"if [ \"$failed\" = true ]; then exit 1; fi",
|
||
].join("\n");
|
||
const hasBuild = nodeRuntimeImageDependencies(spec).some((image) => image.mode === "build-moonbridge" || image.mode === "build-opencode-git");
|
||
const result = hasBuild && !dryRun
|
||
? runNodeHostScriptAsync(spec, script, Math.min(timeoutSeconds, 600), "runtime-image-build")
|
||
: runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300));
|
||
const imageRows = parseNodeRuntimeImageRows(statusText(result));
|
||
const dependencyCount = nodeRuntimeImageDependencies(spec).length;
|
||
const imageRowsComplete = imageRows.length === dependencyCount
|
||
&& imageRows.every((image) => image.presentAfter === true || image.registryTagPresent === true);
|
||
const base = imageRows.find((image) => image.id === "base") ?? {};
|
||
return {
|
||
ok: isCommandSuccess(result) || imageRowsComplete,
|
||
dryRun,
|
||
target: base.target ?? spec.baseImage,
|
||
source: base.source ?? spec.baseImageSource,
|
||
presentBefore: base.presentBefore === true,
|
||
presentAfter: base.presentAfter === true,
|
||
action: base.action ?? null,
|
||
pullRetries: numericField(typeof base.pullRetries === "number" ? String(base.pullRetries) : undefined),
|
||
sourcePresentBefore: typeof base.sourcePresentBefore === "string" ? base.sourcePresentBefore : null,
|
||
pulledSource: base.pulledSource === true,
|
||
pullAttempts: numericField(typeof base.pullAttempts === "number" ? String(base.pullAttempts) : undefined),
|
||
images: imageRows,
|
||
result: compactRuntimeCommand(result),
|
||
};
|
||
}
|
||
|
||
export function externalPostgresSecretSetupScript(spec: HwlabRuntimeLaneSpec, dryRun: boolean): string {
|
||
const pg = hwlabRuntimeActiveExternalPostgres(spec);
|
||
if (pg === undefined) return "true";
|
||
const authnKey = pg.openfga.authnKey ?? "authn-preshared-key";
|
||
return [
|
||
"set -eu",
|
||
`namespace=${shellQuote(spec.runtimeNamespace)}`,
|
||
`secret=${shellQuote(pg.openfga.secretName)}`,
|
||
`authn_key=${shellQuote(authnKey)}`,
|
||
`dry_run=${shellQuote(dryRun ? "true" : "false")}`,
|
||
"namespace_apply_args=\"--server-side --field-manager=unidesk-hwlab-node-runtime-namespace\"",
|
||
"if [ \"$dry_run\" = true ]; then namespace_apply_args=\"$namespace_apply_args --dry-run=server\"; fi",
|
||
"kubectl create namespace \"$namespace\" --dry-run=client -o yaml | kubectl apply $namespace_apply_args -f -",
|
||
"authn_present=$(kubectl -n \"$namespace\" get secret \"$secret\" -o \"go-template={{ index .data \\\"$authn_key\\\" }}\" 2>/dev/null | wc -c | tr -d ' ')",
|
||
"if [ \"${authn_present:-0}\" -gt 0 ]; then",
|
||
" action=kept",
|
||
"elif [ \"$dry_run\" = true ]; then",
|
||
" action=would-generate-authn-key",
|
||
"else",
|
||
" if command -v openssl >/dev/null 2>&1; then authn_value=$(openssl rand -base64 48); else authn_value=$(head -c 48 /dev/urandom | base64); fi",
|
||
" kubectl -n \"$namespace\" create secret generic \"$secret\" --from-literal=\"$authn_key=$authn_value\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-node-openfga-authn -f - >/dev/null",
|
||
" authn_value=",
|
||
" action=generated-authn-key",
|
||
"fi",
|
||
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
||
"printf 'secret\\t%s\\n' \"$secret\"",
|
||
"printf 'authnKey\\t%s\\n' \"$authn_key\"",
|
||
"printf 'action\\t%s\\n' \"$action\"",
|
||
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
||
"printf 'valuesPrinted\\tfalse\\n'",
|
||
].join("\n");
|
||
}
|
||
|
||
export function externalPostgresSecretSourceRoot(spec: HwlabRuntimeLaneSpec): string {
|
||
const configRef = spec.externalPostgres?.configRef;
|
||
if (configRef === undefined) return rootPath(".state", "secrets");
|
||
const configPath = rootPath(configRef);
|
||
const parsed = Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown;
|
||
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error(`${configRef} must be a YAML object`);
|
||
const secrets = (parsed as Record<string, unknown>).secrets;
|
||
if (typeof secrets !== "object" || secrets === null || Array.isArray(secrets)) throw new Error(`${configRef}.secrets must be an object`);
|
||
const root = (secrets as Record<string, unknown>).root;
|
||
if (typeof root !== "string" || root.length === 0) throw new Error(`${configRef}.secrets.root must be a non-empty string`);
|
||
return root.startsWith("/") ? root : rootPath(root);
|
||
}
|
||
|
||
export function readSecretSourceValue(secretRoot: string, sourceRef: string, key: string): { ok: true; value: string; sourcePath: string } | { ok: false; sourcePath: string } {
|
||
const sourcePath = isAbsolute(sourceRef) ? sourceRef : join(secretRoot, sourceRef);
|
||
if (!existsSync(sourcePath)) return { ok: false, sourcePath };
|
||
const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
|
||
const value = values[key];
|
||
if (value === undefined || value.length === 0) return { ok: false, sourcePath };
|
||
return { ok: true, value, sourcePath };
|
||
}
|
||
|
||
export function secretSourcePaths(sourceRef: string): string[] {
|
||
if (isAbsolute(sourceRef)) return [sourceRef];
|
||
return [rootPath(".state", "secrets", sourceRef)];
|
||
}
|
||
|
||
export function displayRepoPath(path: string): string {
|
||
const normalizedRoot = repoRoot.replace(/\/+$/u, "");
|
||
if (path === normalizedRoot) return ".";
|
||
if (path.startsWith(`${normalizedRoot}/`)) return path.slice(normalizedRoot.length + 1);
|
||
const marker = "/.worktree/";
|
||
const index = normalizedRoot.indexOf(marker);
|
||
if (index >= 0) {
|
||
const mainRoot = normalizedRoot.slice(0, index);
|
||
if (path === mainRoot) return ".";
|
||
if (path.startsWith(`${mainRoot}/`)) return path.slice(mainRoot.length + 1);
|
||
}
|
||
return path;
|
||
}
|
||
|
||
export function hwlabPasswordHash(password: string): string {
|
||
const salt = randomBytes(16).toString("hex");
|
||
return `sha256:${salt}:${createHash("sha256").update(`${salt}:${password}`).digest("hex")}`;
|
||
}
|
||
|
||
export function readBootstrapAdminSecretMaterial(spec: RuntimeSecretSpec): BootstrapAdminSecretMaterial {
|
||
const sourceRef = spec.bootstrapAdminPasswordSourceRef;
|
||
const sourceKey = spec.bootstrapAdminPasswordSourceKey;
|
||
const sourceLine = spec.bootstrapAdminPasswordSourceLine ?? null;
|
||
if (sourceRef === undefined || sourceKey === undefined || spec.bootstrapAdminPasswordHashTransform === undefined) {
|
||
return bootstrapAdminSecretMaterialError(spec, sourceRef ?? null, sourceKey ?? null, sourceLine, null, null, "bootstrap-admin-yaml-source-missing");
|
||
}
|
||
const password = readSecretSourceScalar(sourceRef, sourceKey, sourceLine);
|
||
if (!password.ok) return bootstrapAdminSecretMaterialError(spec, sourceRef, sourceKey, sourceLine, password.sourcePath, password.sourcePresent, password.error);
|
||
const username = readBootstrapAdminUsername(spec);
|
||
if (!username.ok) return bootstrapAdminSecretMaterialError(spec, sourceRef, sourceKey, sourceLine, password.sourcePath, password.sourcePresent, username.error);
|
||
return {
|
||
ok: true,
|
||
username: username.value,
|
||
usernameSourceRef: spec.bootstrapAdminUsernameSourceRef ?? null,
|
||
usernameSourceKey: spec.bootstrapAdminUsernameSourceKey ?? null,
|
||
usernameSourceLine: spec.bootstrapAdminUsernameSourceLine ?? null,
|
||
usernameFingerprint: shortSecretFingerprint(username.value),
|
||
sourceRef,
|
||
sourceKey,
|
||
sourceLine,
|
||
sourcePath: password.sourcePath,
|
||
sourcePresent: password.sourcePresent,
|
||
sourceFingerprint: shortSecretFingerprint(password.value),
|
||
passwordHash: hwlabPasswordHash(password.value),
|
||
error: null,
|
||
};
|
||
}
|
||
|
||
export function readBootstrapAdminPasswordMaterial(spec: RuntimeSecretSpec): BootstrapAdminPasswordMaterial {
|
||
const sourceRef = spec.bootstrapAdminPasswordSourceRef;
|
||
const sourceKey = spec.bootstrapAdminPasswordSourceKey;
|
||
const sourceLine = spec.bootstrapAdminPasswordSourceLine ?? null;
|
||
if (sourceRef === undefined || sourceKey === undefined || spec.bootstrapAdminPasswordHashTransform === undefined) {
|
||
return bootstrapAdminPasswordMaterialError(spec, sourceRef ?? null, sourceKey ?? null, sourceLine, null, null, "bootstrap-admin-yaml-source-missing");
|
||
}
|
||
const password = readSecretSourceScalar(sourceRef, sourceKey, sourceLine);
|
||
if (!password.ok) return bootstrapAdminPasswordMaterialError(spec, sourceRef, sourceKey, sourceLine, password.sourcePath, password.sourcePresent, password.error);
|
||
const username = readBootstrapAdminUsername(spec);
|
||
if (!username.ok) return bootstrapAdminPasswordMaterialError(spec, sourceRef, sourceKey, sourceLine, password.sourcePath, password.sourcePresent, username.error);
|
||
return {
|
||
ok: true,
|
||
username: username.value,
|
||
usernameSourceRef: spec.bootstrapAdminUsernameSourceRef ?? null,
|
||
usernameSourceKey: spec.bootstrapAdminUsernameSourceKey ?? null,
|
||
usernameSourceLine: spec.bootstrapAdminUsernameSourceLine ?? null,
|
||
usernameFingerprint: shortSecretFingerprint(username.value),
|
||
sourceRef,
|
||
sourceKey,
|
||
sourceLine,
|
||
sourcePath: password.sourcePath,
|
||
sourcePresent: password.sourcePresent,
|
||
sourceFingerprint: shortSecretFingerprint(password.value),
|
||
password: password.value,
|
||
error: null,
|
||
};
|
||
}
|
||
|
||
function readBootstrapAdminUsername(spec: RuntimeSecretSpec): { ok: true; value: string } | { ok: false; error: string } {
|
||
const ref = spec.bootstrapAdminUsernameSourceRef;
|
||
if (ref === undefined) return { ok: true, value: spec.bootstrapAdminUsername };
|
||
const key = spec.bootstrapAdminUsernameSourceKey ?? "username";
|
||
const material = readSecretSourceScalar(ref, key, spec.bootstrapAdminUsernameSourceLine ?? null);
|
||
if (!material.ok && material.error === "secret-source-missing" && spec.bootstrapAdminUsername.length > 0) return { ok: true, value: spec.bootstrapAdminUsername };
|
||
if (!material.ok) return { ok: false, error: `username-${material.error}` };
|
||
return { ok: true, value: material.value };
|
||
}
|
||
|
||
function readSecretSourceScalar(sourceRef: string, sourceKey: string, sourceLine: number | null): { ok: true; value: string; sourcePath: string; sourcePresent: boolean } | { ok: false; sourcePath: string; sourcePresent: boolean; error: string } {
|
||
const paths = secretSourcePaths(sourceRef);
|
||
const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? rootPath(".state", "secrets", sourceRef);
|
||
const runtimeValue = process.env[sourceKey];
|
||
if (!existsSync(sourcePath)) {
|
||
if (runtimeValue !== undefined && runtimeValue.length > 0) return { ok: true, value: runtimeValue, sourcePath, sourcePresent: false };
|
||
return { ok: false, sourcePath, sourcePresent: false, error: "secret-source-missing" };
|
||
}
|
||
const text = readFileSync(sourcePath, "utf8");
|
||
if (sourceLine !== null) {
|
||
const line = text.split(/\r?\n/u)[sourceLine - 1]?.replace(/\r$/u, "") ?? "";
|
||
if (line.length === 0 && runtimeValue !== undefined && runtimeValue.length > 0) return { ok: true, value: runtimeValue, sourcePath, sourcePresent: true };
|
||
if (line.length === 0) return { ok: false, sourcePath, sourcePresent: true, error: "secret-line-missing" };
|
||
return { ok: true, value: line, sourcePath, sourcePresent: true };
|
||
}
|
||
const values = parseEnvFile(text);
|
||
const value = values[sourceKey] ?? runtimeValue;
|
||
if (value === undefined || value.length === 0) return { ok: false, sourcePath, sourcePresent: true, error: "secret-key-missing" };
|
||
return { ok: true, value, sourcePath, sourcePresent: true };
|
||
}
|
||
|
||
function bootstrapAdminSecretMaterialError(spec: RuntimeSecretSpec, sourceRef: string | null, sourceKey: string | null, sourceLine: number | null, sourcePath: string | null, sourcePresent: boolean | null, error: string): BootstrapAdminSecretMaterial {
|
||
return {
|
||
ok: false,
|
||
username: null,
|
||
usernameSourceRef: spec.bootstrapAdminUsernameSourceRef ?? null,
|
||
usernameSourceKey: spec.bootstrapAdminUsernameSourceKey ?? null,
|
||
usernameSourceLine: spec.bootstrapAdminUsernameSourceLine ?? null,
|
||
usernameFingerprint: null,
|
||
sourceRef,
|
||
sourceKey,
|
||
sourceLine,
|
||
sourcePath,
|
||
sourcePresent: sourcePresent === true,
|
||
sourceFingerprint: null,
|
||
passwordHash: null,
|
||
error,
|
||
};
|
||
}
|
||
|
||
function bootstrapAdminPasswordMaterialError(spec: RuntimeSecretSpec, sourceRef: string | null, sourceKey: string | null, sourceLine: number | null, sourcePath: string | null, sourcePresent: boolean | null, error: string): BootstrapAdminPasswordMaterial {
|
||
return {
|
||
ok: false,
|
||
username: null,
|
||
usernameSourceRef: spec.bootstrapAdminUsernameSourceRef ?? null,
|
||
usernameSourceKey: spec.bootstrapAdminUsernameSourceKey ?? null,
|
||
usernameSourceLine: spec.bootstrapAdminUsernameSourceLine ?? null,
|
||
usernameFingerprint: null,
|
||
sourceRef,
|
||
sourceKey,
|
||
sourceLine,
|
||
sourcePath,
|
||
sourcePresent: sourcePresent === true,
|
||
sourceFingerprint: null,
|
||
password: null,
|
||
error,
|
||
};
|
||
}
|
||
|
||
export function parseEnvFile(text: string): Record<string, string> {
|
||
const values: Record<string, string> = {};
|
||
for (const rawLine of text.split(/\r?\n/u)) {
|
||
const line = rawLine.trim();
|
||
if (line.length === 0 || line.startsWith("#")) continue;
|
||
const eq = line.indexOf("=");
|
||
if (eq <= 0) continue;
|
||
const key = line.slice(0, eq).trim();
|
||
const rawValue = line.slice(eq + 1).trim();
|
||
values[key] = rawValue.startsWith("'") && rawValue.endsWith("'")
|
||
? rawValue.slice(1, -1).replace(/'"'"'/gu, "'")
|
||
: rawValue.startsWith("\"") && rawValue.endsWith("\"")
|
||
? rawValue.slice(1, -1)
|
||
: rawValue;
|
||
}
|
||
return values;
|
||
}
|
||
|
||
export function isLocalPostgresObject(name: string, spec: HwlabRuntimeLaneSpec): boolean {
|
||
if (!/postgres|pgsql|pgdata/iu.test(name)) return false;
|
||
const externalServiceName = spec.externalPostgres?.serviceName;
|
||
if (externalServiceName !== undefined && (name === `service/${externalServiceName}` || name === `svc/${externalServiceName}`)) return false;
|
||
return true;
|
||
}
|
||
|
||
export function nodeRuntimeCodeAgentRuntimeStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record<string, unknown> {
|
||
const runtime = spec.codeAgentRuntime;
|
||
if (runtime === undefined || runtime.enabled === false) return { required: false, ready: true, enabled: runtime?.enabled ?? false };
|
||
if (!namespaceExists) return { required: true, ready: false, enabled: true, degradedReason: "runtime-namespace-missing" };
|
||
const managerTarget = nodeRuntimeCodeAgentManagerTarget(runtime.managerUrl, runtime.runnerNamespace);
|
||
const runnerNamespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", runtime.runnerNamespace, "-o", "name"], 60);
|
||
const secretNamespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", runtime.secretNamespace, "-o", "name"], 60);
|
||
const managerNamespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", managerTarget.namespace, "-o", "name"], 60);
|
||
const managerService = runNodeK3sArgs(spec, ["kubectl", "-n", managerTarget.namespace, "get", "service", managerTarget.serviceName, "-o", "name"], 60);
|
||
const managerDeployment = runNodeK3sArgs(spec, ["kubectl", "-n", managerTarget.namespace, "get", "deployment", managerTarget.serviceName, "-o", "jsonpath={.status.readyReplicas}{\"\\n\"}{.spec.replicas}{\"\\n\"}"], 60);
|
||
const [managerReadyReplicasRaw = "", managerDesiredReplicasRaw = ""] = managerDeployment.stdout.split(/\r?\n/u);
|
||
const managerReadyReplicas = numericField(managerReadyReplicasRaw);
|
||
const managerDesiredReplicas = numericField(managerDesiredReplicasRaw);
|
||
const managerDeploymentReady = managerDeployment.exitCode === 0 && (managerReadyReplicas ?? 0) >= Math.max(1, managerDesiredReplicas ?? 1);
|
||
const apiKey = secretKeyStatus(spec, runtime.apiKeySecretName, runtime.apiKeySecretKey);
|
||
const cloudApiEnv = nodeRuntimeCodeAgentCloudApiEnvStatus(spec, runtime);
|
||
const cloudWebEnv = nodeRuntimeCodeAgentCloudWebEnvStatus(spec, runtime);
|
||
const ready = runnerNamespace.exitCode === 0
|
||
&& secretNamespace.exitCode === 0
|
||
&& managerNamespace.exitCode === 0
|
||
&& managerService.exitCode === 0
|
||
&& managerDeploymentReady
|
||
&& apiKey.present === true
|
||
&& cloudApiEnv.ready === true
|
||
&& cloudWebEnv.ready === true;
|
||
const degradedReason = ready
|
||
? undefined
|
||
: runnerNamespace.exitCode !== 0
|
||
? "agentrun-runner-namespace-missing"
|
||
: secretNamespace.exitCode !== 0
|
||
? "agentrun-secret-namespace-missing"
|
||
: managerNamespace.exitCode !== 0
|
||
? "agentrun-manager-namespace-missing"
|
||
: managerService.exitCode !== 0
|
||
? "agentrun-manager-service-missing"
|
||
: !managerDeploymentReady
|
||
? "agentrun-manager-deployment-not-ready"
|
||
: apiKey.present !== true
|
||
? "agentrun-api-key-secret-missing"
|
||
: cloudApiEnv.ready !== true
|
||
? "hwlab-cloud-api-code-agent-env-mismatch"
|
||
: "hwlab-cloud-web-realtime-env-mismatch";
|
||
return {
|
||
required: true,
|
||
enabled: true,
|
||
ready,
|
||
degradedReason,
|
||
adapter: runtime.adapter,
|
||
managerUrl: runtime.managerUrl,
|
||
manager: {
|
||
namespace: managerTarget.namespace,
|
||
serviceName: managerTarget.serviceName,
|
||
serviceExists: managerService.exitCode === 0,
|
||
deploymentReady: managerDeploymentReady,
|
||
readyReplicas: managerReadyReplicas,
|
||
desiredReplicas: managerDesiredReplicas,
|
||
namespaceResult: compactRuntimeCommand(managerNamespace),
|
||
serviceResult: compactRuntimeCommand(managerService),
|
||
deploymentResult: compactRuntimeCommand(managerDeployment),
|
||
},
|
||
runnerNamespace: {
|
||
name: runtime.runnerNamespace,
|
||
exists: runnerNamespace.exitCode === 0,
|
||
result: compactRuntimeCommand(runnerNamespace),
|
||
},
|
||
secretNamespace: {
|
||
name: runtime.secretNamespace,
|
||
exists: secretNamespace.exitCode === 0,
|
||
result: compactRuntimeCommand(secretNamespace),
|
||
},
|
||
apiKey,
|
||
cloudApiEnv,
|
||
cloudWebEnv,
|
||
repoUrlFrom: runtime.repoUrlFrom,
|
||
providerIdFrom: runtime.providerIdFrom,
|
||
valuesPrinted: false,
|
||
};
|
||
}
|
||
|
||
function nodeRuntimeCodeAgentManagerTarget(managerUrl: string, fallbackNamespace: string): { namespace: string; serviceName: string } {
|
||
let parsed: URL | null = null;
|
||
try {
|
||
parsed = new URL(managerUrl);
|
||
} catch {
|
||
parsed = null;
|
||
}
|
||
const host = parsed?.hostname ?? "";
|
||
const parts = host.split(".").filter(Boolean);
|
||
if (parts.length >= 3 && parts[2] === "svc") return { serviceName: parts[0], namespace: parts[1] };
|
||
return { serviceName: parts[0] || "agentrun-mgr", namespace: parts[1] || fallbackNamespace };
|
||
}
|
||
|
||
function nodeRuntimeCodeAgentCloudApiEnvStatus(spec: HwlabRuntimeLaneSpec, runtime: NonNullable<HwlabRuntimeLaneSpec["codeAgentRuntime"]>): Record<string, unknown> {
|
||
const envProbe = runNodeK3sArgs(spec, [
|
||
"kubectl",
|
||
"-n",
|
||
spec.runtimeNamespace,
|
||
"get",
|
||
"deployment",
|
||
"hwlab-cloud-api",
|
||
"-o",
|
||
'jsonpath={range .spec.template.spec.containers[?(@.name=="hwlab-cloud-api")].env[*]}{.name}{"\\t"}{.value}{"\\t"}{.valueFrom.secretKeyRef.name}{"\\t"}{.valueFrom.secretKeyRef.key}{"\\n"}{end}',
|
||
], 60);
|
||
if (envProbe.exitCode !== 0) {
|
||
return {
|
||
ready: false,
|
||
deploymentExists: false,
|
||
mismatches: [{ name: "hwlab-cloud-api", reason: "deployment-missing" }],
|
||
result: compactCodeAgentDeploymentProbeResult(envProbe),
|
||
valuesPrinted: false,
|
||
};
|
||
}
|
||
const envByName = parseCodeAgentEnvProbe(envProbe.stdout);
|
||
if (envByName.size === 0) {
|
||
return {
|
||
ready: false,
|
||
deploymentExists: true,
|
||
deploymentName: "hwlab-cloud-api",
|
||
containerName: "hwlab-cloud-api",
|
||
mismatches: [{ name: "hwlab-cloud-api", reason: "container-env-not-found" }],
|
||
result: compactCodeAgentDeploymentProbeResult(envProbe),
|
||
valuesPrinted: false,
|
||
};
|
||
}
|
||
const mismatches: Array<Record<string, unknown>> = [];
|
||
const expectValue = (name: string, expected: string): void => {
|
||
const item = envByName.get(name);
|
||
if (item === undefined) {
|
||
mismatches.push({ name, expected, present: false, kind: "value" });
|
||
return;
|
||
}
|
||
if (item.value !== expected) mismatches.push({ name, expected, value: item.value, present: true, kind: "value" });
|
||
};
|
||
const expectAbsent = (name: string): void => {
|
||
const item = envByName.get(name);
|
||
if (item !== undefined) mismatches.push({ name, expected: null, value: item.value, present: true, kind: "unexpected-value" });
|
||
};
|
||
const expectSecret = (name: string, expectedSecret: string, expectedKey: string): void => {
|
||
const item = envByName.get(name);
|
||
if (item === undefined) {
|
||
mismatches.push({ name, expectedSecret, expectedKey, present: false, kind: "secretKeyRef" });
|
||
return;
|
||
}
|
||
if (item.secretName !== expectedSecret || item.secretKey !== expectedKey) {
|
||
mismatches.push({ name, expectedSecret, expectedKey, secret: item.secretName, key: item.secretKey, present: true, kind: "secretKeyRef" });
|
||
}
|
||
};
|
||
expectValue("HWLAB_CODE_AGENT_ADAPTER", runtime.adapter);
|
||
expectValue("AGENTRUN_MGR_URL", runtime.managerUrl);
|
||
expectSecret("AGENTRUN_API_KEY", runtime.apiKeySecretName, runtime.apiKeySecretKey);
|
||
expectValue("HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE", runtime.runnerNamespace);
|
||
expectValue("HWLAB_CODE_AGENT_AGENTRUN_SECRET_NAMESPACE", runtime.secretNamespace);
|
||
for (const [profile, secretName] of Object.entries(runtime.providerSecretNames ?? {})) {
|
||
const profileKey = profile.toUpperCase().replace(/[^A-Z0-9]+/gu, "_");
|
||
expectValue(`HWLAB_CODE_AGENT_AGENTRUN_${profileKey}_SECRET_NAME`, secretName);
|
||
}
|
||
expectValue("HWLAB_CODE_AGENT_AGENTRUN_GITHUB_TOOL_SECRET_NAME", runtime.toolSecretNames.githubPr);
|
||
expectValue("HWLAB_CODE_AGENT_AGENTRUN_UNIDESK_SSH_TOOL_SECRET_NAME", runtime.toolSecretNames.unideskSsh);
|
||
expectValue("HWLAB_CODE_AGENT_AGENTRUN_REPO_URL", spec.gitReadUrl);
|
||
expectValue("HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID", spec.nodeId);
|
||
expectValue("HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE", runtime.defaultProviderProfile);
|
||
expectValue("HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR", runtime.codexStdioSupervisor);
|
||
const kafkaShadowProducer = runtime.kafkaShadowProducer;
|
||
if (kafkaShadowProducer !== undefined) {
|
||
expectValue("HWLAB_KAFKA_SHADOW_PRODUCE_ENABLED", String(kafkaShadowProducer.enabled));
|
||
expectValue("HWLAB_KAFKA_SHADOW_CONSUME_ENABLED", String(kafkaShadowProducer.consumeEnabled));
|
||
expectValue("HWLAB_KAFKA_BOOTSTRAP_SERVERS", kafkaShadowProducer.bootstrapServers);
|
||
expectValue("HWLAB_KAFKA_COMMAND_TOPIC", kafkaShadowProducer.commandTopic);
|
||
expectValue("HWLAB_KAFKA_CLIENT_ID", kafkaShadowProducer.clientId);
|
||
}
|
||
const kafkaEventBridge = runtime.kafkaEventBridge;
|
||
if (kafkaEventBridge !== undefined) {
|
||
expectValue("HWLAB_KAFKA_ENABLED", String(kafkaEventBridge.enabled));
|
||
expectValue("HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED", String(kafkaEventBridge.enabled && (kafkaEventBridge.features.directPublish || kafkaEventBridge.features.transactionalProjector)));
|
||
expectValue("HWLAB_KAFKA_BOOTSTRAP_SERVERS", kafkaEventBridge.bootstrapServers);
|
||
expectValue("HWLAB_KAFKA_STDIO_TOPIC", kafkaEventBridge.stdioTopic);
|
||
expectValue("HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC", kafkaEventBridge.agentRunEventTopic);
|
||
expectValue("HWLAB_KAFKA_EVENT_TOPIC", kafkaEventBridge.hwlabEventTopic);
|
||
expectValue("HWLAB_KAFKA_CLIENT_ID", kafkaEventBridge.clientId);
|
||
expectValue("HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID", kafkaEventBridge.directPublishConsumerGroupId);
|
||
expectValue("HWLAB_KAFKA_PROJECTOR_GROUP_ID", kafkaEventBridge.transactionalProjectorConsumerGroupId);
|
||
expectValue("HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID", kafkaEventBridge.hwlabEventConsumerGroupId);
|
||
expectValue("HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED", String(kafkaEventBridge.enabled && kafkaEventBridge.features.directPublish));
|
||
expectValue("HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED", String(kafkaEventBridge.enabled && kafkaEventBridge.features.liveKafkaSse));
|
||
const refreshReplayEnabled = kafkaEventBridge.enabled && kafkaEventBridge.features.kafkaRefreshReplay;
|
||
expectValue("HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED", String(refreshReplayEnabled));
|
||
const refreshReplayEnvNames = [
|
||
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_GROUP_PREFIX",
|
||
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS",
|
||
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT",
|
||
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_MATCHED_EVENT_LIMIT",
|
||
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_LIVE_BUFFER_LIMIT",
|
||
];
|
||
if (refreshReplayEnabled) {
|
||
const refreshReplay = kafkaEventBridge.refreshReplay;
|
||
if (refreshReplay === undefined) throw new Error("codeAgentRuntime.kafkaEventBridge.refreshReplay is required when Kafka refresh replay is enabled");
|
||
expectValue("HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_GROUP_PREFIX", refreshReplay.groupIdPrefix);
|
||
expectValue("HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS", String(refreshReplay.timeoutMs));
|
||
expectValue("HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT", String(refreshReplay.scanLimit));
|
||
expectValue("HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_MATCHED_EVENT_LIMIT", String(refreshReplay.matchedEventLimit));
|
||
expectValue("HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_LIVE_BUFFER_LIMIT", String(refreshReplay.liveBufferLimit));
|
||
} else {
|
||
for (const name of refreshReplayEnvNames) expectAbsent(name);
|
||
}
|
||
expectValue("HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED", String(kafkaEventBridge.enabled && kafkaEventBridge.features.transactionalProjector));
|
||
expectValue("HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED", String(kafkaEventBridge.enabled && kafkaEventBridge.features.projectionOutboxRelay));
|
||
expectValue("HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED", String(kafkaEventBridge.enabled && kafkaEventBridge.features.projectionRealtime));
|
||
}
|
||
return {
|
||
ready: mismatches.length === 0,
|
||
deploymentExists: true,
|
||
deploymentName: "hwlab-cloud-api",
|
||
containerName: "hwlab-cloud-api",
|
||
checkedEnvCount: 9 + (kafkaShadowProducer === undefined ? 0 : 5) + (kafkaEventBridge === undefined ? 0 : 21),
|
||
envCount: envByName.size,
|
||
mismatches,
|
||
result: compactCodeAgentDeploymentProbeResult(envProbe),
|
||
valuesPrinted: false,
|
||
};
|
||
}
|
||
|
||
function nodeRuntimeCodeAgentCloudWebEnvStatus(spec: HwlabRuntimeLaneSpec, runtime: NonNullable<HwlabRuntimeLaneSpec["codeAgentRuntime"]>): Record<string, unknown> {
|
||
const kafkaEventBridge = runtime.kafkaEventBridge;
|
||
if (kafkaEventBridge === undefined) return { required: false, ready: true, checkedEnvCount: 0, valuesPrinted: false };
|
||
const envProbe = runNodeK3sArgs(spec, [
|
||
"kubectl",
|
||
"-n",
|
||
spec.runtimeNamespace,
|
||
"get",
|
||
"deployment",
|
||
"hwlab-cloud-web",
|
||
"-o",
|
||
'jsonpath={range .spec.template.spec.containers[?(@.name=="hwlab-cloud-web")].env[*]}{.name}{"\\t"}{.value}{"\\t"}{.valueFrom.secretKeyRef.name}{"\\t"}{.valueFrom.secretKeyRef.key}{"\\n"}{end}',
|
||
], 60);
|
||
if (envProbe.exitCode !== 0) {
|
||
return {
|
||
required: true,
|
||
ready: false,
|
||
deploymentExists: false,
|
||
mismatches: [{ name: "hwlab-cloud-web", reason: "deployment-missing" }],
|
||
result: compactCodeAgentDeploymentProbeResult(envProbe),
|
||
valuesPrinted: false,
|
||
};
|
||
}
|
||
const envByName = parseCodeAgentEnvProbe(envProbe.stdout);
|
||
const mismatches: Array<Record<string, unknown>> = [];
|
||
const expectValue = (name: string, expected: string): void => {
|
||
const item = envByName.get(name);
|
||
if (item === undefined) {
|
||
mismatches.push({ name, expected, present: false, kind: "value" });
|
||
return;
|
||
}
|
||
if (item.value !== expected) mismatches.push({ name, expected, value: item.value, present: true, kind: "value" });
|
||
};
|
||
expectValue("HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED", String(kafkaEventBridge.enabled && kafkaEventBridge.features.liveKafkaSse));
|
||
expectValue("HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED", String(kafkaEventBridge.enabled && kafkaEventBridge.features.kafkaRefreshReplay));
|
||
expectValue("HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED", String(kafkaEventBridge.enabled && kafkaEventBridge.features.projectionRealtime));
|
||
return {
|
||
required: true,
|
||
ready: mismatches.length === 0,
|
||
deploymentExists: true,
|
||
deploymentName: "hwlab-cloud-web",
|
||
containerName: "hwlab-cloud-web",
|
||
checkedEnvCount: 3,
|
||
envCount: envByName.size,
|
||
mismatches,
|
||
result: compactCodeAgentDeploymentProbeResult(envProbe),
|
||
valuesPrinted: false,
|
||
};
|
||
}
|
||
|
||
type CodeAgentEnvProbeRow = {
|
||
value: string | null;
|
||
secretName: string | null;
|
||
secretKey: string | null;
|
||
};
|
||
|
||
function parseCodeAgentEnvProbe(stdout: string): Map<string, CodeAgentEnvProbeRow> {
|
||
const rows = new Map<string, CodeAgentEnvProbeRow>();
|
||
for (const line of stdout.split(/\r?\n/u)) {
|
||
const trimmed = line.trimEnd();
|
||
if (trimmed.length === 0) continue;
|
||
const [name = "", value = "", secretName = "", secretKey = ""] = trimmed.split("\t");
|
||
if (name.length === 0) continue;
|
||
rows.set(name, {
|
||
value: value.length > 0 ? value : null,
|
||
secretName: secretName.length > 0 ? secretName : null,
|
||
secretKey: secretKey.length > 0 ? secretKey : null,
|
||
});
|
||
}
|
||
return rows;
|
||
}
|
||
|
||
function compactCodeAgentDeploymentProbeResult(result: CommandResult): Record<string, unknown> {
|
||
return {
|
||
exitCode: result.exitCode,
|
||
stdoutBytes: Buffer.byteLength(result.stdout, "utf8"),
|
||
stderrBytes: Buffer.byteLength(result.stderr, "utf8"),
|
||
stderrTail: result.stderr.slice(-600),
|
||
timedOut: result.timedOut === true,
|
||
};
|
||
}
|
||
|
||
export function externalPostgresBridgeStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record<string, unknown> {
|
||
const pg = hwlabRuntimeActiveExternalPostgres(spec);
|
||
if (pg === undefined) return { required: false, ready: true };
|
||
if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" };
|
||
const runtimeAccess = pg.runtimeAccess ?? { endpointAddress: pg.endpointAddress, port: pg.port };
|
||
const endpointIsIpv4 = /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/u.test(runtimeAccess.endpointAddress);
|
||
const service = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "service", pg.serviceName, "-o", "jsonpath={.spec.type}{\"\\n\"}{.spec.externalName}{\"\\n\"}{.spec.ports[0].port}{\"\\n\"}"], 60);
|
||
const [serviceType = "", externalName = "", servicePort = ""] = service.stdout.split(/\r?\n/u);
|
||
if (!endpointIsIpv4) {
|
||
const ready = service.exitCode === 0 && serviceType === "ExternalName" && externalName === runtimeAccess.endpointAddress && Number(servicePort) === runtimeAccess.port;
|
||
return {
|
||
required: true,
|
||
ready,
|
||
serviceName: pg.serviceName,
|
||
serviceType: serviceType || null,
|
||
externalName: externalName || null,
|
||
expectedExternalName: runtimeAccess.endpointAddress,
|
||
port: numericField(servicePort),
|
||
expectedPort: runtimeAccess.port,
|
||
providerEndpointAddress: pg.endpointAddress,
|
||
providerPort: pg.port,
|
||
runtimeAccess: pg.runtimeAccess ?? null,
|
||
service: compactRuntimeCommand(service),
|
||
};
|
||
}
|
||
const endpoints = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "endpoints", pg.serviceName, "-o", "jsonpath={.subsets[0].addresses[0].ip}{\"\\n\"}{.subsets[0].ports[0].port}{\"\\n\"}"], 60);
|
||
const [endpointsAddress = "", endpointsPort = ""] = endpoints.stdout.split(/\r?\n/u);
|
||
const endpointSlice = endpoints.exitCode === 0 && endpointsAddress.length > 0
|
||
? { exitCode: 1, stdout: "", stderr: "", timedOut: false }
|
||
: runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "endpointslice", `${pg.serviceName}-host`, "-o", "jsonpath={.addressType}{\"\\n\"}{.endpoints[0].addresses[0]}{\"\\n\"}{.ports[0].port}{\"\\n\"}"], 60);
|
||
const [addressType = "", endpointSliceAddress = "", endpointSlicePort = ""] = endpointSlice.stdout.split(/\r?\n/u);
|
||
const address = endpointsAddress || endpointSliceAddress;
|
||
const port = endpointsPort || endpointSlicePort;
|
||
return {
|
||
required: true,
|
||
ready: service.exitCode === 0 && (endpoints.exitCode === 0 || endpointSlice.exitCode === 0) && serviceType !== "ExternalName" && address === runtimeAccess.endpointAddress && Number(port) === runtimeAccess.port,
|
||
serviceName: pg.serviceName,
|
||
serviceType: serviceType || null,
|
||
addressType: endpoints.exitCode === 0 ? "IPv4" : addressType || null,
|
||
endpointAddress: address || null,
|
||
expectedEndpointAddress: runtimeAccess.endpointAddress,
|
||
port: numericField(port),
|
||
expectedPort: runtimeAccess.port,
|
||
providerEndpointAddress: pg.endpointAddress,
|
||
providerPort: pg.port,
|
||
runtimeAccess: pg.runtimeAccess ?? null,
|
||
service: compactRuntimeCommand(service),
|
||
endpoints: compactRuntimeCommand(endpoints),
|
||
endpointSlice: compactRuntimeCommand(endpointSlice),
|
||
};
|
||
}
|
||
|
||
export function externalPostgresSecretStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record<string, unknown> {
|
||
const pg = hwlabRuntimeActiveExternalPostgres(spec);
|
||
if (pg === undefined) return { required: false, ready: true };
|
||
if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" };
|
||
const cloudApi = secretKeyStatus(spec, pg.cloudApi.secretName, pg.cloudApi.secretKey);
|
||
const openfgaUri = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.secretKey);
|
||
const openfgaAuthn = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.authnKey ?? "authn-preshared-key");
|
||
return {
|
||
required: true,
|
||
ready: cloudApi.present && openfgaUri.present && openfgaAuthn.present,
|
||
valuesPrinted: false,
|
||
cloudApi,
|
||
openfga: {
|
||
datastoreUri: openfgaUri,
|
||
authnPresharedKey: openfgaAuthn,
|
||
},
|
||
};
|
||
}
|
||
|
||
export function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeGitMirrorTargetSpec {
|
||
const configPath = rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH);
|
||
const parsed = record(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown);
|
||
const nodes = record(parsed.nodes);
|
||
const node = record(nodes[spec.nodeId]);
|
||
const targets = Array.isArray(parsed.targets) ? parsed.targets : [];
|
||
const target = targets.map((item) => record(item)).find((item) => item.node === spec.nodeId && item.lane === spec.lane);
|
||
if (target === undefined) throw new Error(`no gitMirror target for node=${spec.nodeId} lane=${spec.lane} in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`);
|
||
const gitMirror = record(target.gitMirror);
|
||
const gitMirrorEgressProxy = record(gitMirror.egressProxy);
|
||
const gitMirrorEgressMode = stringValue(gitMirrorEgressProxy.mode, "gitMirror.egressProxy.mode");
|
||
if (gitMirrorEgressMode !== "node-global" && gitMirrorEgressMode !== "host-route" && gitMirrorEgressMode !== "direct") throw new Error(`gitMirror.egressProxy.mode must be node-global, host-route, or direct for node=${spec.nodeId} lane=${spec.lane}`);
|
||
const nodeEgressProxy = gitMirrorEgressMode === "direct"
|
||
? { mode: "direct" as const, required: false as const }
|
||
: nodeRuntimeGitMirrorEgressProxySpec(record(node.egressProxy), `nodes.${spec.nodeId}.egressProxy`);
|
||
if ((gitMirrorEgressMode === "node-global" || gitMirrorEgressMode === "host-route") && gitMirrorEgressProxy.required !== true) throw new Error(`gitMirror.egressProxy.required must be true for node=${spec.nodeId} lane=${spec.lane}`);
|
||
if (gitMirrorEgressMode === "node-global" && nodeEgressProxy.mode !== "k8s-service-cluster-ip") throw new Error(`gitMirror.egressProxy.mode=node-global requires nodes.${spec.nodeId}.egressProxy.mode=k8s-service-cluster-ip`);
|
||
if (gitMirrorEgressMode === "host-route" && nodeEgressProxy.mode !== "host-route") throw new Error(`gitMirror.egressProxy.mode=host-route requires nodes.${spec.nodeId}.egressProxy.mode=host-route`);
|
||
const githubTransport = nodeRuntimeGitMirrorGithubTransportSpec(record(gitMirror.githubTransport), "gitMirror.githubTransport");
|
||
const source = record(target.source);
|
||
const gitops = record(target.gitops);
|
||
const tekton = record(target.tekton);
|
||
const toolsImage = record(tekton.toolsImage);
|
||
const toolsImagePullPolicy = optionalStringValue(toolsImage.imagePullPolicy, "tekton.toolsImage.imagePullPolicy") ?? "IfNotPresent";
|
||
if (toolsImagePullPolicy !== "Always" && toolsImagePullPolicy !== "IfNotPresent" && toolsImagePullPolicy !== "Never") throw new Error("tekton.toolsImage.imagePullPolicy must be Always, IfNotPresent, or Never");
|
||
return {
|
||
id: stringValue(target.id, "target.id"),
|
||
node: stringValue(target.node, "target.node"),
|
||
lane: stringValue(target.lane, "target.lane"),
|
||
namespace: stringValue(gitMirror.namespace, "gitMirror.namespace"),
|
||
serviceReadName: stringValue(gitMirror.serviceReadName, "gitMirror.serviceReadName"),
|
||
serviceWriteName: stringValue(gitMirror.serviceWriteName, "gitMirror.serviceWriteName"),
|
||
cachePvcName: stringValue(gitMirror.cachePvcName, "gitMirror.cachePvcName"),
|
||
cachePvcStorage: stringValue(gitMirror.cachePvcStorage, "gitMirror.cachePvcStorage"),
|
||
cacheHostPath: optionalStringValue(gitMirror.cacheHostPath, "gitMirror.cacheHostPath"),
|
||
servicePort: positiveIntegerValue(gitMirror.servicePort, "gitMirror.servicePort"),
|
||
secretName: stringValue(gitMirror.secretName, "gitMirror.secretName"),
|
||
syncConfigMapName: stringValue(gitMirror.syncConfigMapName, "gitMirror.syncConfigMapName"),
|
||
syncJobPrefix: stringValue(gitMirror.syncJobPrefix, "gitMirror.syncJobPrefix"),
|
||
flushJobPrefix: stringValue(gitMirror.flushJobPrefix, "gitMirror.flushJobPrefix"),
|
||
toolsImage: stringValue(toolsImage.output, "tekton.toolsImage.output"),
|
||
toolsImagePullPolicy,
|
||
sourceRepository: stringValue(source.repository, "source.repository"),
|
||
sourceBranch: stringValue(source.branch, "source.branch"),
|
||
gitopsBranch: stringValue(gitops.branch, "gitops.branch"),
|
||
egressProxy: nodeEgressProxy,
|
||
githubTransport,
|
||
};
|
||
}
|
||
|
||
export function nodeRuntimeGitMirrorGithubTransportSpec(raw: Record<string, unknown>, path: string): NodeRuntimeGitMirrorGithubTransportSpec {
|
||
const mode = stringValue(raw.mode, `${path}.mode`);
|
||
if (mode === "ssh") {
|
||
const knownHostsSecretKey = optionalStringValue(raw.knownHostsSecretKey, `${path}.knownHostsSecretKey`);
|
||
const knownHostsSourceRef = optionalStringValue(raw.knownHostsSourceRef, `${path}.knownHostsSourceRef`);
|
||
const knownHostsSourceKey = optionalStringValue(raw.knownHostsSourceKey, `${path}.knownHostsSourceKey`);
|
||
const knownHostsSourceEncoding = raw.knownHostsSourceEncoding === undefined ? null : gitMirrorSecretSourceEncoding(raw.knownHostsSourceEncoding, `${path}.knownHostsSourceEncoding`);
|
||
const knownHostsValues = [knownHostsSecretKey, knownHostsSourceRef, knownHostsSourceKey, knownHostsSourceEncoding];
|
||
if (knownHostsValues.some((value) => value !== null) && knownHostsValues.some((value) => value === null)) {
|
||
throw new Error(`${path}.knownHostsSecretKey/sourceRef/sourceKey/sourceEncoding must be declared together`);
|
||
}
|
||
return {
|
||
mode,
|
||
privateKeySecretKey: stringValue(raw.privateKeySecretKey, `${path}.privateKeySecretKey`),
|
||
privateKeySourceRef: stringValue(raw.privateKeySourceRef, `${path}.privateKeySourceRef`),
|
||
privateKeySourceKey: stringValue(raw.privateKeySourceKey, `${path}.privateKeySourceKey`),
|
||
privateKeySourceEncoding: gitMirrorSecretSourceEncoding(raw.privateKeySourceEncoding, `${path}.privateKeySourceEncoding`),
|
||
knownHostsSecretKey,
|
||
knownHostsSourceRef,
|
||
knownHostsSourceKey,
|
||
knownHostsSourceEncoding,
|
||
};
|
||
}
|
||
if (mode !== "https") throw new Error(`${path}.mode must be ssh or https`);
|
||
return {
|
||
mode,
|
||
username: stringValue(raw.username, `${path}.username`),
|
||
tokenSecretName: stringValue(raw.tokenSecretName, `${path}.tokenSecretName`),
|
||
tokenSecretKey: stringValue(raw.tokenSecretKey, `${path}.tokenSecretKey`),
|
||
tokenSourceRef: stringValue(raw.tokenSourceRef, `${path}.tokenSourceRef`),
|
||
tokenSourceKey: stringValue(raw.tokenSourceKey, `${path}.tokenSourceKey`),
|
||
};
|
||
}
|
||
|
||
function gitMirrorSecretSourceEncoding(raw: unknown, path: string): "plain" | "base64" {
|
||
const value = stringValue(raw, path);
|
||
if (value !== "plain" && value !== "base64") throw new Error(`${path} must be plain or base64`);
|
||
return value;
|
||
}
|
||
|
||
export function nodeRuntimeGitMirrorEgressProxySpec(raw: Record<string, unknown>, path: string): NodeRuntimeGitMirrorEgressProxySpec {
|
||
const mode = stringValue(raw.mode, `${path}.mode`);
|
||
if (mode === "host-route") {
|
||
const noProxyRaw = raw.noProxy;
|
||
if (!Array.isArray(noProxyRaw)) throw new Error(`${path}.noProxy must be an array`);
|
||
return {
|
||
mode,
|
||
clientName: stringValue(raw.clientName, `${path}.clientName`),
|
||
hostProxyConfigRef: stringValue(raw.hostProxyConfigRef, `${path}.hostProxyConfigRef`),
|
||
proxyEnvPath: stringValue(raw.proxyEnvPath, `${path}.proxyEnvPath`),
|
||
proxyUrl: stringValue(raw.proxyUrl, `${path}.proxyUrl`),
|
||
noProxy: noProxyRaw.map((item, index) => stringValue(item, `${path}.noProxy[${index}]`)),
|
||
};
|
||
}
|
||
if (mode !== "k8s-service-cluster-ip") throw new Error(`${path}.mode must be k8s-service-cluster-ip or host-route`);
|
||
const port = Number(raw.port);
|
||
if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error(`${path}.port must be a TCP port`);
|
||
const sourceConfigRef = optionalStringValue(raw.sourceConfigRef, `${path}.sourceConfigRef`) ?? null;
|
||
const source = sourceConfigRef === null ? null : resolveEgressProxySourceRef(sourceConfigRef, `${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}.${path}.sourceConfigRef`);
|
||
const sourceType = source?.sourceType ?? stringValue(raw.sourceType, `${path}.sourceType`);
|
||
if (sourceType !== "subscription-url" && sourceType !== "master-shadowsocks") throw new Error(`${path}.sourceType must be subscription-url or master-shadowsocks`);
|
||
if (raw.sourceType !== undefined && raw.sourceType !== sourceType) throw new Error(`${path}.sourceType must match sourceConfigRef value ${sourceType}`);
|
||
const sourceRef = source?.sourceRef ?? stringValue(raw.sourceRef, `${path}.sourceRef`);
|
||
const sourceKey = source?.sourceKey ?? stringValue(raw.sourceKey, `${path}.sourceKey`);
|
||
if (source !== null) {
|
||
if (raw.sourceRef !== undefined && raw.sourceRef !== source.sourceRef) throw new Error(`${path}.sourceRef must match sourceConfigRef value ${source.sourceRef}`);
|
||
if (raw.sourceKey !== undefined && raw.sourceKey !== source.sourceKey) throw new Error(`${path}.sourceKey must match sourceConfigRef value ${source.sourceKey}`);
|
||
}
|
||
const preferredOutbound = source?.preferredOutbound ?? (raw.preferredOutbound === undefined ? null : stringValue(raw.preferredOutbound, `${path}.preferredOutbound`));
|
||
if (preferredOutbound !== null && preferredOutbound !== "vless-reality" && preferredOutbound !== "hysteria2") throw new Error(`${path}.preferredOutbound must be vless-reality or hysteria2`);
|
||
if (source !== null && source.preferredOutbound !== null && raw.preferredOutbound !== undefined && raw.preferredOutbound !== source.preferredOutbound) {
|
||
throw new Error(`${path}.preferredOutbound must match sourceConfigRef value ${source.preferredOutbound}`);
|
||
}
|
||
const noProxyRaw = raw.noProxy;
|
||
if (!Array.isArray(noProxyRaw)) throw new Error(`${path}.noProxy must be an array`);
|
||
const noProxy = noProxyRaw.map((item, index) => stringValue(item, `${path}.noProxy[${index}]`));
|
||
return {
|
||
mode,
|
||
clientName: stringValue(raw.clientName, `${path}.clientName`),
|
||
namespace: stringValue(raw.namespace, `${path}.namespace`),
|
||
serviceName: stringValue(raw.serviceName, `${path}.serviceName`),
|
||
port,
|
||
sourceConfigRef,
|
||
sourceRef,
|
||
sourceKey,
|
||
sourceType,
|
||
preferredOutbound,
|
||
sourceFingerprint: source?.fingerprint ?? null,
|
||
noProxy,
|
||
};
|
||
}
|
||
|
||
export function secretKeyStatus(spec: HwlabRuntimeLaneSpec, secret: string, key: string): Record<string, unknown> {
|
||
const result = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "secret", secret, "-o", `go-template={{ index .data ${JSON.stringify(key)} }}`], 60);
|
||
return {
|
||
secret,
|
||
key,
|
||
present: result.exitCode === 0 && result.stdout.trim().length > 0,
|
||
valueBytesBase64: result.stdout.trim().length,
|
||
exitCode: result.exitCode,
|
||
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 500),
|
||
valuesPrinted: false,
|
||
};
|
||
}
|
||
|
||
export function startNodeDelegatedJob(options: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
||
const commandArgs = [
|
||
"hwlab",
|
||
"nodes",
|
||
options.domain,
|
||
...stripOptions(options.originalArgs, ["--node", "--lane", "--confirm", "--dry-run", "--wait", "--timeout-seconds"]),
|
||
"--node",
|
||
options.node,
|
||
"--lane",
|
||
options.lane,
|
||
"--confirm",
|
||
"--timeout-seconds",
|
||
String(options.timeoutSeconds),
|
||
"--wait",
|
||
];
|
||
const command = ["bun", "scripts/cli.ts", ...commandArgs];
|
||
const job = startJob(
|
||
`hwlab_nodes_${options.lane}_${options.domain}_${options.action}`,
|
||
command,
|
||
`Run HWLAB ${options.lane} ${options.domain} ${options.action} for node ${options.node}`,
|
||
);
|
||
return {
|
||
ok: true,
|
||
command: `hwlab nodes ${options.domain} ${options.action} --node ${options.node} --lane ${options.lane}`,
|
||
node: options.node,
|
||
lane: options.lane,
|
||
mode: "async-job",
|
||
reason: "confirmed control-plane/mirror actions can spend tens of seconds on remote work; default is fire-and-forget to avoid silent blocking",
|
||
job,
|
||
statusCommand: `bun scripts/cli.ts job status ${job.id}`,
|
||
waitCommand: command.join(" "),
|
||
};
|
||
}
|
||
|
||
export function rewriteDelegatedNodeResult(value: unknown, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
||
const rewritten = rewriteDelegatedNodeValue(value, scoped);
|
||
const result = typeof rewritten === "object" && rewritten !== null && !Array.isArray(rewritten) ? rewritten as Record<string, unknown> : { value: rewritten };
|
||
return {
|
||
...result,
|
||
node: scoped.node,
|
||
commandFamily: "hwlab nodes",
|
||
};
|
||
}
|
||
|
||
export function rewriteDelegatedNodeValue(value: unknown, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): unknown {
|
||
if (typeof value === "string") return rewriteDelegatedNodeString(value, scoped);
|
||
if (Array.isArray(value)) return value.map((item) => rewriteDelegatedNodeValue(item, scoped));
|
||
if (typeof value !== "object" || value === null) return value;
|
||
return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, rewriteDelegatedNodeValue(item, scoped)]));
|
||
}
|
||
|
||
export function rewriteDelegatedNodeString(value: string, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): string {
|
||
const replaceCommand = (text: string, domain: DelegatedNodeDomain) => {
|
||
const escapedDomain = domain.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
||
return text
|
||
.replace(new RegExp(`bun scripts/cli\\.ts hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `bun scripts/cli.ts hwlab nodes ${domain} $1 --node ${scoped.node}`)
|
||
.replace(new RegExp(`hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `hwlab nodes ${domain} $1 --node ${scoped.node}`);
|
||
};
|
||
return replaceCommand(replaceCommand(value, "control-plane"), "git-mirror");
|
||
}
|
||
|
||
export function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions {
|
||
const [actionRaw] = args;
|
||
if (actionRaw !== "run" && actionRaw !== "script" && actionRaw !== "screenshot" && actionRaw !== "observe" && actionRaw !== "sentinel" && actionRaw !== "opencode-smoke" && actionRaw !== "console-verify" && actionRaw !== "native-readiness") throw new Error("web-probe usage: run|script|screenshot|observe|sentinel|opencode-smoke|console-verify|native-readiness --node NODE --lane vNN [--origin internal|public|native]");
|
||
if (actionRaw === "sentinel") return parseNodeWebProbeSentinelOptions(args.slice(1));
|
||
if (actionRaw === "observe") {
|
||
const normalized = normalizeNodeWebProbeObserveArgs(args.slice(1));
|
||
const explicitNode = optionValue(args, "--node");
|
||
const explicitLane = optionValue(args, "--lane");
|
||
const requestedId = normalized.id ?? optionValue(normalized.args, "--job-id") ?? null;
|
||
const indexed = requestedId === null
|
||
? null
|
||
: readWebObserveIndexEntry(requestedId) ?? (explicitNode === undefined || explicitLane === undefined ? discoverWebObserveIndexEntry(requestedId) : null);
|
||
const node = explicitNode ?? indexed?.node;
|
||
const lane = explicitLane ?? indexed?.lane;
|
||
if (node === undefined || lane === undefined) {
|
||
if (requestedId !== null) {
|
||
throw new Error(`web-probe observe observer-id-unknown: ${requestedId}; run observe start first, or pass --node/--lane with --job-id/--state-dir to re-index this observer`);
|
||
}
|
||
throw new Error("web-probe observe node-lane-required-for-start: observe start requires --node/--lane; status|command|stop|collect|analyze should pass an observer id");
|
||
}
|
||
assertNodeId(node);
|
||
assertLane(lane);
|
||
if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`);
|
||
const spec = hwlabRuntimeLaneSpecForNode(lane, node);
|
||
return parseNodeWebProbeObserveOptions(normalized.args, node, lane, spec, normalized.id, indexed);
|
||
}
|
||
const node = requiredOption(args, "--node");
|
||
assertNodeId(node);
|
||
const lane = requiredOption(args, "--lane");
|
||
assertLane(lane);
|
||
if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`);
|
||
const spec = hwlabRuntimeLaneSpecForNode(lane, node);
|
||
const resolveOrigin = () => {
|
||
const browserProxyModeRaw = optionValue(args, "--browser-proxy-mode");
|
||
const browserProxyMode = browserProxyModeRaw === undefined ? undefined : parseWebProbeBrowserProxyMode(browserProxyModeRaw);
|
||
return resolveNodeWebProbeCliOrigin(spec, optionValue(args, "--origin"), optionValue(args, "--url"), browserProxyMode);
|
||
};
|
||
if (actionRaw === "native-readiness") {
|
||
assertKnownOptions(args.slice(1), new Set([
|
||
"--node",
|
||
"--lane",
|
||
"--profile",
|
||
"--timeout-ms",
|
||
"--command-timeout-seconds",
|
||
]), new Set([]));
|
||
const profiles = spec.webProbe?.nativeReadinessProfiles;
|
||
const profileName = optionValue(args, "--profile") ?? spec.webProbe?.defaultNativeReadinessProfile;
|
||
if (profiles === undefined || profileName === undefined) {
|
||
throw new Error(`${hwlabRuntimeLaneConfigPath()} node=${node} lane=${lane} does not declare a default native readiness profile`);
|
||
}
|
||
const profile = profiles[profileName];
|
||
if (profile === undefined) {
|
||
throw new Error(`web-probe native-readiness unknown --profile ${profileName}; configured profiles: ${Object.keys(profiles).sort().join(", ")}`);
|
||
}
|
||
const native = spec.nativeDevelopment?.[profile.application];
|
||
if (native === undefined) throw new Error(`${hwlabRuntimeLaneConfigPath()} node=${node} lane=${lane} does not declare nativeDevelopment.${profile.application}`);
|
||
const timeoutMs = positiveIntegerOption(args, "--timeout-ms", profile.navigationTimeoutMs, 120_000);
|
||
const commandTimeoutSeconds = positiveIntegerOption(args, "--command-timeout-seconds", profile.commandTimeoutSeconds, 3600);
|
||
const configRef = `${hwlabRuntimeLaneConfigPath()}#templates.*.webProbeWorkbench.nativeReadinessProfiles.${profileName}`;
|
||
const scriptText = nodeWebProbeNativeReadinessScript(profileName, profile, configRef);
|
||
const url = native.publicExposure.publicBaseUrl;
|
||
const selectedOrigin = resolveNodeWebProbeCliOrigin(spec, undefined, url, "direct");
|
||
const commandLabel = `web-probe native-readiness --node ${node} --lane ${lane} --profile ${profileName}`;
|
||
return {
|
||
action: "script",
|
||
node,
|
||
lane,
|
||
url: selectedOrigin.url,
|
||
originName: selectedOrigin.originName,
|
||
originMode: selectedOrigin.originMode,
|
||
originConfigPath: `${hwlabRuntimeLaneConfigPath()}#lanes.${lane}.targets.${node}.nativeDevelopment.${profile.application}`,
|
||
timeoutMs,
|
||
viewport: `${profile.viewport.width}x${profile.viewport.height}`,
|
||
browserProxyMode: selectedOrigin.browserProxyMode,
|
||
browserProxyModeSource: selectedOrigin.browserProxyModeSource,
|
||
commandTimeoutSeconds,
|
||
scriptText,
|
||
authentication: profile.authentication ?? "required",
|
||
commandLabel,
|
||
suppressAdHocWarning: true,
|
||
generatedHints: [
|
||
"L1 native Web readiness is a repo-owned typed command; fixed HTTPS origin, path, DOM, interaction and failure rules come from owning YAML.",
|
||
"Pass requires mounted DOM, a completed interaction, no browser errors, no critical failed responses and a screenshot artifact.",
|
||
],
|
||
generatedPreferredCommands: { rerun: commandLabel },
|
||
scriptSource: {
|
||
kind: "generated",
|
||
path: `builtin:native-readiness/${profileName}`,
|
||
byteCount: Buffer.byteLength(scriptText),
|
||
sha256: `sha256:${createHash("sha256").update(scriptText).digest("hex")}`,
|
||
},
|
||
};
|
||
}
|
||
if (actionRaw === "console-verify") {
|
||
assertKnownOptions(args.slice(1), new Set([
|
||
"--node",
|
||
"--lane",
|
||
"--origin",
|
||
"--url",
|
||
"--profile",
|
||
"--timeout-ms",
|
||
"--browser-proxy-mode",
|
||
"--command-timeout-seconds",
|
||
]), new Set([]));
|
||
const profiles = spec.webProbe?.consoleVerificationProfiles;
|
||
const profileName = optionValue(args, "--profile") ?? spec.webProbe?.defaultConsoleVerificationProfile;
|
||
if (profiles === undefined || profileName === undefined) {
|
||
throw new Error(`${hwlabRuntimeLaneConfigPath()} node=${node} lane=${lane} does not declare a default Cloud Console verification profile`);
|
||
}
|
||
const profile = profiles[profileName];
|
||
if (profile === undefined) {
|
||
throw new Error(`web-probe console-verify unknown --profile ${profileName}; configured profiles: ${Object.keys(profiles).sort().join(", ")}`);
|
||
}
|
||
const firstViewport = profile.viewports[0];
|
||
if (firstViewport === undefined) throw new Error(`web-probe console-verify profile ${profileName} has no viewport`);
|
||
const timeoutMs = positiveIntegerOption(args, "--timeout-ms", profile.navigationTimeoutMs, 120_000);
|
||
const commandTimeoutSeconds = positiveIntegerOption(args, "--command-timeout-seconds", profile.commandTimeoutSeconds, 3600);
|
||
const configRef = `${hwlabRuntimeLaneConfigPath()}#templates.*.webProbeWorkbench.consoleVerificationProfiles.${profileName}`;
|
||
const scriptText = nodeWebProbeConsoleVerifyScript(profileName, profile, configRef);
|
||
const selectedOrigin = resolveOrigin();
|
||
const commandLabel = `web-probe console-verify --node ${node} --lane ${lane}${selectedOrigin.originName === "custom" ? "" : ` --origin ${selectedOrigin.originName}`} --profile ${profileName}`;
|
||
return {
|
||
action: "script",
|
||
node,
|
||
lane,
|
||
url: selectedOrigin.url,
|
||
originName: selectedOrigin.originName,
|
||
originMode: selectedOrigin.originMode,
|
||
originConfigPath: selectedOrigin.originConfigPath,
|
||
timeoutMs,
|
||
viewport: `${firstViewport.width}x${firstViewport.height}`,
|
||
browserProxyMode: selectedOrigin.browserProxyMode,
|
||
browserProxyModeSource: selectedOrigin.browserProxyModeSource,
|
||
commandTimeoutSeconds,
|
||
scriptText,
|
||
commandLabel,
|
||
suppressAdHocWarning: true,
|
||
generatedHints: [
|
||
"Cloud Console verification is a repo-owned typed command; route, viewport, selector, workflow and evidence bounds come from the selected owning YAML profile.",
|
||
"The command reuses the managed semantic origin, authentication, remote browser, physical MemAvailable guard, report recovery and screenshot contracts.",
|
||
],
|
||
generatedPreferredCommands: {
|
||
rerun: commandLabel,
|
||
},
|
||
scriptSource: {
|
||
kind: "generated",
|
||
path: `builtin:console-verify/${profileName}`,
|
||
byteCount: Buffer.byteLength(scriptText),
|
||
sha256: `sha256:${createHash("sha256").update(scriptText).digest("hex")}`,
|
||
},
|
||
};
|
||
}
|
||
if (actionRaw === "screenshot") {
|
||
assertKnownOptions(args.slice(1), new Set([
|
||
"--node",
|
||
"--lane",
|
||
"--origin",
|
||
"--url",
|
||
"--path",
|
||
"--viewport",
|
||
"--browser-proxy-mode",
|
||
"--local-dir",
|
||
"--name",
|
||
"--timeout-ms",
|
||
"--wait-until",
|
||
"--selector",
|
||
"--wait-timeout-ms",
|
||
"--command-timeout-seconds",
|
||
]), new Set([
|
||
"--full-page",
|
||
"--no-full-page",
|
||
"--keep-remote",
|
||
]));
|
||
const url = optionValue(args, "--url");
|
||
const targetPath = optionValue(args, "--path") ?? null;
|
||
if (url !== undefined && targetPath !== null) {
|
||
const correctionCommand = `bun scripts/cli.ts web-probe ${removeOptionWithValue(args, "--path").map(shellQuote).join(" ")}`;
|
||
throw new CliInputError("web-probe screenshot 的 --url 与 --path 互斥；custom/local URL 已包含完整路径，请移除 --path", {
|
||
code: "web-probe-screenshot-url-path-mutually-exclusive",
|
||
reason: "custom-url-already-includes-full-path-remove-path",
|
||
mutation: false,
|
||
argument: "--url/--path",
|
||
usage: correctionCommand,
|
||
hint: "保留 --url 中的完整路径并移除 --path；semantic origin 调用则保留 --origin 与 --path，并移除 --url。",
|
||
correctionCommands: [correctionCommand],
|
||
});
|
||
}
|
||
const selectedOrigin = resolveOrigin();
|
||
const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 120000);
|
||
const waitTimeoutMs = positiveIntegerOption(args, "--wait-timeout-ms", Math.max(90000, timeoutMs + 30000), 600000);
|
||
const commandTimeoutSeconds = positiveIntegerOption(args, "--command-timeout-seconds", Math.ceil(waitTimeoutMs / 1000) + 45, 900);
|
||
return {
|
||
action: "screenshot",
|
||
node,
|
||
lane,
|
||
url: url ?? resolveWebProbeScreenshotUrl(selectedOrigin.url, targetPath ?? "/"),
|
||
originName: selectedOrigin.originName,
|
||
originMode: selectedOrigin.originMode,
|
||
originConfigPath: selectedOrigin.originConfigPath,
|
||
browserProxyMode: selectedOrigin.browserProxyMode,
|
||
browserProxyModeSource: selectedOrigin.browserProxyModeSource,
|
||
path: targetPath,
|
||
viewport: parseWebProbeScreenshotViewport(optionValue(args, "--viewport") ?? "1440x900"),
|
||
localDir: optionValue(args, "--local-dir") ?? "/tmp",
|
||
name: parseWebProbeScreenshotName(optionValue(args, "--name") ?? `web-probe-${node.toLowerCase()}-${lane}.png`),
|
||
timeoutMs,
|
||
waitUntil: parseWebProbeScreenshotWaitUntil(optionValue(args, "--wait-until") ?? "networkidle"),
|
||
fullPage: args.includes("--full-page") && !args.includes("--no-full-page"),
|
||
selector: optionValue(args, "--selector") ?? null,
|
||
keepRemote: args.includes("--keep-remote"),
|
||
waitTimeoutMs,
|
||
commandTimeoutSeconds,
|
||
};
|
||
}
|
||
if (actionRaw === "script") {
|
||
assertKnownOptions(args.slice(1), new Set([
|
||
"--node",
|
||
"--lane",
|
||
"--origin",
|
||
"--url",
|
||
"--timeout-ms",
|
||
"--viewport",
|
||
"--browser-proxy-mode",
|
||
"--script-file",
|
||
"--command-timeout-seconds",
|
||
]), new Set([]));
|
||
const scriptFile = optionValue(args, "--script-file");
|
||
if (scriptFile === undefined && process.stdin.isTTY) {
|
||
throw new Error("web-probe script requires a stdin heredoc or --script-file <path>");
|
||
}
|
||
const scriptText = scriptFile === undefined ? readFileSync(0, "utf8") : readFileSync(scriptFile, "utf8");
|
||
if (scriptText.trim().length === 0) throw new Error("web-probe script received an empty script");
|
||
const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 60000);
|
||
const commandTimeoutSeconds = positiveIntegerOption(args, "--command-timeout-seconds", Math.max(60, Math.ceil(timeoutMs / 1000) + 30), 3600);
|
||
const selectedOrigin = resolveOrigin();
|
||
return {
|
||
action: "script",
|
||
node,
|
||
lane,
|
||
url: selectedOrigin.url,
|
||
originName: selectedOrigin.originName,
|
||
originMode: selectedOrigin.originMode,
|
||
originConfigPath: selectedOrigin.originConfigPath,
|
||
timeoutMs,
|
||
viewport: optionValue(args, "--viewport") ?? "1440x900",
|
||
browserProxyMode: selectedOrigin.browserProxyMode,
|
||
browserProxyModeSource: selectedOrigin.browserProxyModeSource,
|
||
commandTimeoutSeconds,
|
||
scriptText,
|
||
scriptSource: {
|
||
kind: scriptFile === undefined ? "stdin" : "file",
|
||
path: scriptFile ?? null,
|
||
byteCount: Buffer.byteLength(scriptText),
|
||
sha256: `sha256:${createHash("sha256").update(scriptText).digest("hex")}`,
|
||
},
|
||
};
|
||
}
|
||
if (actionRaw === "opencode-smoke") {
|
||
assertKnownOptions(args.slice(1), new Set([
|
||
"--node",
|
||
"--lane",
|
||
"--origin",
|
||
"--url",
|
||
"--path",
|
||
"--message",
|
||
"--expect-text",
|
||
"--timeout-ms",
|
||
"--response-timeout-ms",
|
||
"--viewport",
|
||
"--browser-proxy-mode",
|
||
"--command-timeout-seconds",
|
||
]), new Set([]));
|
||
const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 40000, 300000);
|
||
const commandTimeoutSeconds = positiveIntegerOption(args, "--command-timeout-seconds", 55, 3600);
|
||
const requestedResponseTimeoutMs = positiveIntegerOption(args, "--response-timeout-ms", Math.min(30000, timeoutMs), timeoutMs);
|
||
const responseTimeoutMs = Math.min(requestedResponseTimeoutMs, Math.max(5000, (commandTimeoutSeconds * 1000) - 20000));
|
||
const path = optionValue(args, "--path") ?? "/opencode";
|
||
if (!path.startsWith("/") || path.includes("\0") || path.length > 200) throw new Error("web-probe opencode-smoke --path must be an absolute path up to 200 chars");
|
||
const message = optionValue(args, "--message") ?? "hi";
|
||
if (message.includes("\0") || message.length > 500) throw new Error("web-probe opencode-smoke --message must be 0-500 non-NUL chars");
|
||
const expectText = optionValue(args, "--expect-text") ?? null;
|
||
if (expectText !== null && (expectText.includes("\0") || expectText.length > 500)) throw new Error("web-probe opencode-smoke --expect-text must be 0-500 non-NUL chars");
|
||
const scriptText = nodeWebProbeOpencodeSmokeScript({
|
||
path,
|
||
message,
|
||
expectText,
|
||
responseTimeoutMs,
|
||
});
|
||
const selectedOrigin = resolveOrigin();
|
||
const commandLabel = `web-probe opencode-smoke --node ${node} --lane ${lane}${selectedOrigin.originName === "custom" ? "" : ` --origin ${selectedOrigin.originName}`}`;
|
||
return {
|
||
action: "script",
|
||
node,
|
||
lane,
|
||
url: selectedOrigin.url,
|
||
originName: selectedOrigin.originName,
|
||
originMode: selectedOrigin.originMode,
|
||
originConfigPath: selectedOrigin.originConfigPath,
|
||
timeoutMs,
|
||
viewport: optionValue(args, "--viewport") ?? "1440x900",
|
||
browserProxyMode: selectedOrigin.browserProxyMode,
|
||
browserProxyModeSource: selectedOrigin.browserProxyModeSource,
|
||
commandTimeoutSeconds,
|
||
scriptText,
|
||
commandLabel,
|
||
suppressAdHocWarning: true,
|
||
generatedHints: [
|
||
"OpenCode smoke is a repo-owned typed web-probe command; it uses the same managed auth, remote browser, report recovery and artifact contract as web-probe script.",
|
||
"Passing means the browser DOM reached assistant text and the OpenCode EventSource produced update, finish and idle evidence.",
|
||
],
|
||
generatedPreferredCommands: {
|
||
withExpectedText: `${commandLabel} --expect-text <assistant-substring>`,
|
||
},
|
||
scriptSource: {
|
||
kind: "generated",
|
||
path: "builtin:opencode-smoke",
|
||
byteCount: Buffer.byteLength(scriptText),
|
||
sha256: `sha256:${createHash("sha256").update(scriptText).digest("hex")}`,
|
||
},
|
||
};
|
||
}
|
||
assertKnownOptions(args.slice(1), new Set([
|
||
"--node",
|
||
"--lane",
|
||
"--origin",
|
||
"--url",
|
||
"--browser-proxy-mode",
|
||
"--timeout-ms",
|
||
"--wait-after-submit-ms",
|
||
"--wait-messages-ms",
|
||
"--wait-agent-terminal-ms",
|
||
"--trace-sample-count",
|
||
"--trace-sample-interval-ms",
|
||
"--message",
|
||
"--conversation-id",
|
||
"--command-timeout-seconds",
|
||
]), new Set([
|
||
"--fresh-session",
|
||
"--no-cancel-running",
|
||
]));
|
||
const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 60000);
|
||
const waitAfterSubmitMs = positiveIntegerOption(args, "--wait-after-submit-ms", 1500, 60000);
|
||
const waitMessagesMs = positiveIntegerOption(args, "--wait-messages-ms", 2500, 60000);
|
||
const waitAgentTerminalMs = positiveIntegerOption(args, "--wait-agent-terminal-ms", 0, 600000);
|
||
const message = optionValue(args, "--message") ?? null;
|
||
const hasMessage = message !== null;
|
||
const traceSampleCountDefault = hasMessage ? 2 : 0;
|
||
const traceSampleIntervalDefault = hasMessage ? 1500 : 0;
|
||
const traceSampleCount = positiveIntegerOption(args, "--trace-sample-count", traceSampleCountDefault, 200);
|
||
const traceSampleIntervalMs = positiveIntegerOption(args, "--trace-sample-interval-ms", traceSampleIntervalDefault, 60000);
|
||
const commandTimeoutAutoSeconds = nodeWebProbeAutoCommandTimeoutSeconds({
|
||
timeoutMs,
|
||
waitAfterSubmitMs,
|
||
waitMessagesMs,
|
||
waitAgentTerminalMs,
|
||
traceSampleCount,
|
||
traceSampleIntervalMs,
|
||
freshSession: args.includes("--fresh-session"),
|
||
hasMessage,
|
||
});
|
||
const commandTimeoutRaw = optionValue(args, "--command-timeout-seconds");
|
||
const commandTimeoutUserProvided = commandTimeoutRaw !== undefined;
|
||
const commandTimeoutSeconds = commandTimeoutUserProvided
|
||
? Math.max(positiveIntegerOption(args, "--command-timeout-seconds", commandTimeoutAutoSeconds, 3600), commandTimeoutAutoSeconds)
|
||
: commandTimeoutAutoSeconds;
|
||
const selectedOrigin = resolveOrigin();
|
||
return {
|
||
action: "run",
|
||
node,
|
||
lane,
|
||
url: selectedOrigin.url,
|
||
originName: selectedOrigin.originName,
|
||
originMode: selectedOrigin.originMode,
|
||
originConfigPath: selectedOrigin.originConfigPath,
|
||
browserProxyMode: selectedOrigin.browserProxyMode,
|
||
browserProxyModeSource: selectedOrigin.browserProxyModeSource,
|
||
timeoutMs,
|
||
waitAfterSubmitMs,
|
||
waitMessagesMs,
|
||
waitAgentTerminalMs,
|
||
traceSampleCount,
|
||
traceSampleIntervalMs,
|
||
message,
|
||
conversationId: optionValue(args, "--conversation-id") ?? null,
|
||
freshSession: args.includes("--fresh-session"),
|
||
cancelRunning: !args.includes("--no-cancel-running"),
|
||
commandTimeoutSeconds,
|
||
commandTimeoutAutoSeconds,
|
||
commandTimeoutUserProvided,
|
||
};
|
||
}
|
||
|
||
function removeOptionWithValue(args: string[], option: string): string[] {
|
||
const result: string[] = [];
|
||
for (let index = 0; index < args.length; index += 1) {
|
||
const item = args[index] ?? "";
|
||
if (item === option) {
|
||
index += 1;
|
||
continue;
|
||
}
|
||
if (item.startsWith(`${option}=`)) continue;
|
||
result.push(item);
|
||
}
|
||
return result;
|
||
}
|
||
|
||
function nodeWebProbeOpencodeSmokeScript(input: { path: string; message: string; expectText: string | null; responseTimeoutMs: number }): string {
|
||
const config = {
|
||
path: input.path,
|
||
message: input.message,
|
||
expectText: input.expectText,
|
||
responseTimeoutMs: input.responseTimeoutMs,
|
||
};
|
||
return patchOpenCodeSmokeScript(`const config = ${JSON.stringify(config)};\n\nexport default async function opencodeSmoke({ page, baseUrl, recordStep, wait, screenshot, jsonArtifact }) {\n await page.addInitScript(() => {\n const key = \"__unideskOpenCodeEvents\";\n window[key] = Array.isArray(window[key]) ? window[key] : [];\n if (window.__unideskOpenCodeEventSourceWrapped) return;\n window.__unideskOpenCodeEventSourceWrapped = true;\n const OriginalEventSource = window.EventSource;\n if (!OriginalEventSource) return;\n const push = (source, type, event) => {\n let parsedType = null;\n let parsedStatus = null;\n let dataBytes = 0;\n const rawData = typeof event?.data === \"string\" ? event.data : \"\";\n dataBytes = rawData.length;\n if (rawData) {\n try {\n const parsed = JSON.parse(rawData);\n parsedType = typeof parsed?.type === \"string\" ? parsed.type : typeof parsed?.event === \"string\" ? parsed.event : typeof parsed?.name === \"string\" ? parsed.name : null;\n parsedStatus = typeof parsed?.status === \"string\" ? parsed.status : typeof parsed?.part?.status === \"string\" ? parsed.part.status : null;\n } catch {\n parsedType = null;\n }\n }\n const label = parsedType || (parsedStatus ? type + \":\" + parsedStatus : type);\n window[key].push({ at: Date.now(), source, type, parsedType, status: parsedStatus, label, dataBytes });\n if (window[key].length > 240) window[key].splice(0, window[key].length - 240);\n };\n const originalAddEventListener = OriginalEventSource.prototype.addEventListener;\n OriginalEventSource.prototype.addEventListener = function wrappedAddEventListener(type, listener, options) {\n if (typeof listener !== \"function\") return originalAddEventListener.call(this, type, listener, options);\n const wrapped = function wrappedOpenCodeEvent(event) {\n push(String(this?.url || \"eventsource\"), String(type || \"message\"), event);\n return listener.call(this, event);\n };\n return originalAddEventListener.call(this, type, wrapped, options);\n };\n function WrappedEventSource(url, options) {\n const instance = new OriginalEventSource(url, options);\n originalAddEventListener.call(instance, \"message\", (event) => push(String(url || \"eventsource\"), \"message\", event));\n return instance;\n }\n WrappedEventSource.prototype = OriginalEventSource.prototype;\n WrappedEventSource.CONNECTING = OriginalEventSource.CONNECTING;\n WrappedEventSource.OPEN = OriginalEventSource.OPEN;\n WrappedEventSource.CLOSED = OriginalEventSource.CLOSED;\n window.EventSource = WrappedEventSource;\n });\n\n const targetUrl = new URL(config.path, baseUrl).toString();\n await page.goto(targetUrl, { waitUntil: \"domcontentloaded\", timeout: Math.min(Math.max(config.responseTimeoutMs, 1000), 120000) });\n recordStep(\"opencode-navigate\", { ok: true, targetPath: config.path, finalUrl: page.url() });\n\n const frame = await findOpenCodeFrame(page, config.responseTimeoutMs);\n const before = await collectDomState(frame, [], config);\n const beforeLines = before.lines;\n recordStep(\"opencode-frame\", { ok: true, frameUrl: before.url, title: before.title, beforeLineCount: beforeLines.length, hasSubmit: before.hasSubmit });\n\n const prompt = await fillOpenCodePrompt(frame, config.message);\n recordStep(\"opencode-prompt-filled\", { ok: true, selector: prompt.selector, textBytes: config.message.length });\n\n const submit = frame.locator(\"[data-action='prompt-submit']\").first();\n await submit.waitFor({ state: \"visible\", timeout: Math.min(config.responseTimeoutMs, 10000) });\n await frame.waitForFunction(() => {\n const button = document.querySelector(\"[data-action='prompt-submit']\");\n return Boolean(button && !button.disabled && button.getAttribute(\"aria-disabled\") !== \"true\");\n }, null, { timeout: 5000 }).catch(() => null);\n await submit.click({ timeout: 10000 });\n recordStep(\"opencode-submit\", { ok: true, selector: \"[data-action='prompt-submit']\" });\n\n const final = await waitForOpenCodeFinal(page, frame, beforeLines, config);\n await jsonArtifact(\"opencode-smoke-events.json\", final.events);\n const finalScreenshot = await screenshot(\"opencode-smoke-final.png\").catch((error) => ({ error: error instanceof Error ? error.message : String(error) }));\n const ok = final.conditions.assistantText && final.conditions.notThinking && final.conditions.updateEvent && final.conditions.finishEvent && final.conditions.idleEvent && final.conditions.expectedText;\n recordStep(\"opencode-final\", { ok, conditions: final.conditions, assistantTextPreview: final.assistantTextPreview, eventTypes: final.events.eventTypes });\n return {\n ok,\n status: ok ? \"pass\" : \"blocked\",\n failedCondition: ok ? null : failedCondition(final.conditions),\n frameUrl: final.url,\n finalUrl: page.url(),\n assistantTextPreview: final.assistantTextPreview,\n bodyPreview: final.bodyPreview,\n eventTypes: final.events.eventTypes,\n eventCount: final.events.eventCount,\n conditions: final.conditions,\n screenshot: finalScreenshot,\n issueEvidence: {\n frameUrl: final.url,\n assistantTextPreview: final.assistantTextPreview,\n eventTypes: final.events.eventTypes,\n conditions: final.conditions,\n screenshotSha256: typeof finalScreenshot?.sha256 === \"string\" ? finalScreenshot.sha256 : null,\n valuesRedacted: true,\n },\n valuesRedacted: true,\n };\n}\n\nasync function findOpenCodeFrame(page, timeoutMs) {\n const deadline = Date.now() + timeoutMs;\n let best = null;\n while (Date.now() < deadline) {\n for (const frame of page.frames()) {\n const summary = await frameSummary(frame);\n const score = (summary.hasSubmit ? 100 : 0) + (summary.textboxCount > 0 ? 20 : 0) + (/opencode/iu.test(summary.url) ? 20 : 0) + (/opencode/iu.test(summary.title) ? 10 : 0);\n if (best === null || score > best.score) best = { frame, summary, score };\n if (summary.hasSubmit && summary.textboxCount > 0) return frame;\n }\n await waitFor(500);\n }\n throw new Error(\"OpenCode frame with prompt submit button was not found; best=\" + JSON.stringify(best?.summary || null));\n}\n\nasync function frameSummary(frame) {\n return await frame.evaluate(() => ({\n url: location.href,\n title: document.title,\n hasSubmit: Boolean(document.querySelector(\"[data-action='prompt-submit']\")),\n textboxCount: document.querySelectorAll(\"[contenteditable='true'], textarea, [role='textbox'], input:not([type]), input[type='text']\").length,\n bodyPreview: (document.body?.innerText || \"\").replace(/\\s+/gu, \" \").trim().slice(0, 240),\n })).catch((error) => ({ url: frame.url(), title: \"\", hasSubmit: false, textboxCount: 0, bodyPreview: String(error).slice(0, 160) }));\n}\n\nasync function fillOpenCodePrompt(frame, message) {\n const selectors = [\n \"[contenteditable='true']\",\n \"textarea\",\n \"[role='textbox']\",\n \"input:not([type])\",\n \"input[type='text']\",\n ];\n let lastError = null;\n for (const selector of selectors) {\n const locator = frame.locator(selector).last();\n const count = await locator.count().catch(() => 0);\n if (count < 1) continue;\n try {\n await locator.click({ timeout: 5000 });\n await locator.fill(message, { timeout: 5000 });\n return { selector };\n } catch (error) {\n lastError = error;\n try {\n await locator.evaluate((element, text) => {\n if (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement) {\n element.value = text;\n element.dispatchEvent(new InputEvent(\"input\", { bubbles: true, data: text, inputType: \"insertText\" }));\n element.dispatchEvent(new Event(\"change\", { bubbles: true }));\n return;\n }\n element.textContent = text;\n element.dispatchEvent(new InputEvent(\"input\", { bubbles: true, data: text, inputType: \"insertText\" }));\n }, message);\n return { selector, fallback: \"dom-input-event\" };\n } catch (fallbackError) {\n lastError = fallbackError;\n }\n }\n }\n throw new Error(\"OpenCode prompt input was not fillable: \" + (lastError instanceof Error ? lastError.message : String(lastError || \"not found\")));\n}\n\nasync function waitForOpenCodeFinal(page, frame, beforeLines, config) {\n const deadline = Date.now() + config.responseTimeoutMs;\n let latest = await collectState(page, frame, beforeLines, config);\n while (Date.now() < deadline) {\n latest = await collectState(page, frame, beforeLines, config);\n if (latest.conditions.assistantText && latest.conditions.notThinking && latest.conditions.updateEvent && latest.conditions.finishEvent && latest.conditions.idleEvent && latest.conditions.expectedText) return latest;\n await waitFor(750);\n }\n return latest;\n}\n\nasync function collectState(page, frame, beforeLines, config) {\n const dom = await collectDomState(frame, beforeLines, config);\n const events = await collectEventSummary(page);\n const labels = events.eventTypes.join(\"\\n\");\n const conditions = {\n assistantText: Boolean(dom.assistantTextPreview),\n notThinking: dom.thinkingVisible !== true,\n updateEvent: /message[._-]?part[._-]?updated|part[._-]?updated|delta|updated/iu.test(labels),\n finishEvent: /step[._-]?finish|finish|completed|done/iu.test(labels),\n idleEvent: /session[._-]?idle|session[._-]?status[:/._-]?idle|\\bidle\\b/iu.test(labels),\n expectedText: config.expectText === null || dom.expectedTextPresent === true,\n };\n return { ...dom, events, conditions };\n}\n\nasync function collectDomState(frame, beforeLines, config) {\n return await frame.evaluate(({ beforeLines, message, expectText }) => {\n const normalize = (value) => String(value || \"\").replace(/\\r/g, \"\").replace(/[ \\t]+/g, \" \").trim();\n const bodyText = normalize(document.body?.innerText || \"\");\n const lines = bodyText.split(\"\\n\").map((line) => normalize(line)).filter(Boolean).slice(-400);\n const beforeSet = new Set(Array.isArray(beforeLines) ? beforeLines : []);\n const prompt = normalize(message).toLowerCase();\n const blocked = /^(send|stop|cancel|thinking|loading|new session|settings)$/iu;\n const selectorTexts = Array.from(document.querySelectorAll(\"[data-role='assistant'], [data-message-role='assistant'], [data-testid*='assistant' i], [aria-label*='assistant' i], [class*='assistant' i], main article, main [role='article'], main [data-message-id]\")).map((element) => normalize(element.textContent || \"\")).filter((text) => text.length > 0 && text.toLowerCase() !== prompt && !blocked.test(text)).slice(-20);\n const newLines = lines.filter((line) => !beforeSet.has(line) && line.toLowerCase() !== prompt && !blocked.test(line) && !/^you\\b/iu.test(line)).slice(-40);\n const assistantText = selectorTexts.find((text) => text.length > 0) || newLines.find((line) => line.length > 0) || null;\n return {\n url: location.href,\n title: document.title,\n hasSubmit: Boolean(document.querySelector(\"[data-action='prompt-submit']\")),\n lines,\n lineCount: lines.length,\n newLineCount: newLines.length,\n assistantCandidateCount: selectorTexts.length,\n assistantTextPreview: assistantText === null ? null : assistantText.slice(0, 500),\n thinkingVisible: /(^|\\n|\\b)Thinking(\\b|\\n)/u.test(bodyText),\n expectedTextPresent: expectText === null ? true : bodyText.includes(expectText),\n bodyPreview: bodyText.slice(-1200),\n };\n }, { beforeLines, message: config.message, expectText: config.expectText });\n}\n\nasync function collectEventSummary(page) {\n const frameEvents = [];\n for (const frame of page.frames()) {\n const events = await frame.evaluate(() => Array.isArray(window.__unideskOpenCodeEvents) ? window.__unideskOpenCodeEvents.slice(-120) : []).catch(() => []);\n if (events.length > 0) frameEvents.push({ frameUrl: frame.url(), events });\n }\n const labels = [];\n for (const item of frameEvents) {\n for (const event of item.events) {\n const raw = [event.label, event.parsedType, event.status, event.type].filter(Boolean).join(\":\");\n if (raw && !labels.includes(raw)) labels.push(raw);\n }\n }\n return {\n eventCount: frameEvents.reduce((sum, item) => sum + item.events.length, 0),\n eventTypes: labels.slice(0, 40),\n frames: frameEvents.map((item) => ({ frameUrl: item.frameUrl, count: item.events.length, last: item.events.slice(-5) })),\n valuesRedacted: true,\n };\n}\n\nfunction failedCondition(conditions) {\n return Object.entries(conditions).filter(([, ok]) => ok !== true).map(([key]) => key).join(\",\") || \"unknown\";\n}\n\nfunction waitFor(ms) {\n return new Promise((resolve) => setTimeout(resolve, ms));\n}\n`);
|
||
}
|
||
|
||
function patchOpenCodeSmokeScript(source: string): string {
|
||
const oldFlow = ` const frame = await findOpenCodeFrame(page, config.responseTimeoutMs);
|
||
const before = await collectDomState(frame, [], config);
|
||
const beforeLines = before.lines;
|
||
recordStep("opencode-frame", { ok: true, frameUrl: before.url, title: before.title, beforeLineCount: beforeLines.length, hasSubmit: before.hasSubmit });
|
||
|
||
const prompt = await fillOpenCodePrompt(frame, config.message);`;
|
||
const newFlow = ` const frame = await findOpenCodeFrame(page, config.responseTimeoutMs);
|
||
const project = await ensureOpenCodeProjectOpen(frame, config.responseTimeoutMs);
|
||
recordStep("opencode-project", project);
|
||
const composer = await waitForOpenCodeComposer(frame, config.responseTimeoutMs);
|
||
recordStep("opencode-composer", composer);
|
||
const eventProbe = await startOpenCodeEventProbe(frame);
|
||
recordStep("opencode-event-probe", eventProbe);
|
||
const before = await collectDomState(frame, [], config);
|
||
const beforeLines = before.lines;
|
||
recordStep("opencode-frame", { ok: true, frameUrl: before.url, title: before.title, beforeLineCount: beforeLines.length, hasSubmit: before.hasSubmit });
|
||
|
||
const prompt = await fillOpenCodePrompt(frame, config.message);`;
|
||
const oldFrameFinder = `async function findOpenCodeFrame(page, timeoutMs) {
|
||
const deadline = Date.now() + timeoutMs;
|
||
let best = null;
|
||
while (Date.now() < deadline) {
|
||
for (const frame of page.frames()) {
|
||
const summary = await frameSummary(frame);
|
||
const score = (summary.hasSubmit ? 100 : 0) + (summary.textboxCount > 0 ? 20 : 0) + (/opencode/iu.test(summary.url) ? 20 : 0) + (/opencode/iu.test(summary.title) ? 10 : 0);
|
||
if (best === null || score > best.score) best = { frame, summary, score };
|
||
if (summary.hasSubmit && summary.textboxCount > 0) return frame;
|
||
}
|
||
await waitFor(500);
|
||
}
|
||
throw new Error("OpenCode frame with prompt submit button was not found; best=" + JSON.stringify(best?.summary || null));
|
||
}`;
|
||
const newFrameFinder = `async function findOpenCodeFrame(page, timeoutMs) {
|
||
const deadline = Date.now() + timeoutMs;
|
||
let best = null;
|
||
while (Date.now() < deadline) {
|
||
for (const frame of page.frames()) {
|
||
const summary = await frameSummary(frame);
|
||
const score = (summary.hasSubmit ? 100 : 0) + (summary.textboxCount > 0 ? 20 : 0) + (/opencode/iu.test(summary.url) ? 20 : 0) + (/opencode/iu.test(summary.title) ? 10 : 0) + (isOpenCodeFrame(frame, summary) ? 50 : 0);
|
||
if (best === null || score > best.score) best = { frame, summary, score };
|
||
if (summary.hasSubmit && summary.textboxCount > 0) return frame;
|
||
if (isOpenCodeFrame(frame, summary)) return frame;
|
||
}
|
||
await waitFor(500);
|
||
}
|
||
throw new Error("OpenCode frame was not found; best=" + JSON.stringify(best?.summary || null));
|
||
}
|
||
|
||
function isOpenCodeFrame(frame, summary) {
|
||
const rawUrl = String(summary?.url || frame.url() || "");
|
||
let hostname = "";
|
||
try {
|
||
hostname = new URL(rawUrl).hostname;
|
||
} catch {
|
||
hostname = "";
|
||
}
|
||
const hasParent = typeof frame.parentFrame === "function" && frame.parentFrame() !== null;
|
||
if (hasParent && (/opencode/iu.test(rawUrl) || String(summary?.title || "") === "OpenCode")) return true;
|
||
return /^opencode[-.]/iu.test(hostname);
|
||
}
|
||
|
||
async function ensureOpenCodeProjectOpen(frame, timeoutMs) {
|
||
const deadline = Date.now() + Math.min(timeoutMs, 20000);
|
||
let clickedProject = null;
|
||
let dismissed = [];
|
||
let lastSummary = null;
|
||
while (Date.now() < deadline) {
|
||
lastSummary = await frameSummary(frame);
|
||
if (lastSummary.hasSubmit && lastSummary.textboxCount > 0) {
|
||
return { ok: true, alreadyOpen: true, frameUrl: lastSummary.url, dismissed, clickedProject };
|
||
}
|
||
const dismissedButton = await clickButtonMatching(frame, (text) => /^(not yet|skip|maybe later)$/iu.test(text), "dismiss-provider");
|
||
if (dismissedButton) {
|
||
dismissed.push(dismissedButton.text);
|
||
await waitFor(500);
|
||
continue;
|
||
}
|
||
if (/No projects open|Recent projects|Open a project/iu.test(String(lastSummary.bodyPreview || ""))) {
|
||
const clicked = await clickButtonMatching(frame, (text) => text === "/" || /^\\/\\s*\\d/iu.test(text) || /^\\/.*ago$/iu.test(text), "recent-project");
|
||
if (clicked) {
|
||
clickedProject = clicked.text;
|
||
await waitFor(1500);
|
||
return { ok: true, alreadyOpen: false, frameUrl: lastSummary.url, dismissed, clickedProject };
|
||
}
|
||
}
|
||
await waitFor(500);
|
||
}
|
||
return { ok: false, alreadyOpen: false, frameUrl: lastSummary?.url ?? frame.url(), dismissed, clickedProject, bodyPreview: lastSummary?.bodyPreview ?? null };
|
||
}
|
||
|
||
async function waitForOpenCodeComposer(frame, timeoutMs) {
|
||
const deadline = Date.now() + timeoutMs;
|
||
let lastSummary = null;
|
||
while (Date.now() < deadline) {
|
||
lastSummary = await frameSummary(frame);
|
||
if (lastSummary.hasSubmit && lastSummary.textboxCount > 0) {
|
||
return { ok: true, frameUrl: lastSummary.url, textboxCount: lastSummary.textboxCount, hasSubmit: true };
|
||
}
|
||
await waitFor(750);
|
||
}
|
||
throw new Error("OpenCode composer was not ready; last=" + JSON.stringify(lastSummary));
|
||
}
|
||
|
||
async function startOpenCodeEventProbe(frame) {
|
||
return await frame.evaluate(() => {
|
||
const key = "__unideskOpenCodeEvents";
|
||
window[key] = Array.isArray(window[key]) ? window[key] : [];
|
||
if (window.__unideskManualOpenCodeEventSourceStarted) {
|
||
return { ok: true, reused: true, eventCount: window[key].length };
|
||
}
|
||
window.__unideskManualOpenCodeEventSourceStarted = true;
|
||
const eventUrl = new URL("/global/event", location.origin).toString();
|
||
const source = new EventSource(eventUrl);
|
||
window.__unideskManualOpenCodeEventSource = source;
|
||
const push = (type, event) => {
|
||
const rawData = typeof event?.data === "string" ? event.data : "";
|
||
let parsedType = null;
|
||
let status = null;
|
||
if (rawData) {
|
||
try {
|
||
const parsed = JSON.parse(rawData);
|
||
parsedType = typeof parsed?.type === "string" ? parsed.type : typeof parsed?.event === "string" ? parsed.event : typeof parsed?.name === "string" ? parsed.name : null;
|
||
status = typeof parsed?.status === "string" ? parsed.status : typeof parsed?.part?.status === "string" ? parsed.part.status : null;
|
||
} catch {
|
||
parsedType = null;
|
||
}
|
||
}
|
||
const label = parsedType || (status ? String(type) + ":" + status : String(type));
|
||
window[key].push({ at: Date.now(), source: eventUrl, type: String(type), parsedType, status, label, dataBytes: rawData.length, manualProbe: true });
|
||
if (window[key].length > 240) window[key].splice(0, window[key].length - 240);
|
||
};
|
||
const eventTypes = ["message", "message.part.updated", "message.part.added", "message.updated", "step-finish", "step.finish", "session.idle", "session.status", "session.updated", "session.error", "error"];
|
||
for (const type of eventTypes) source.addEventListener(type, (event) => push(type, event));
|
||
source.addEventListener("open", () => push("open", { data: "" }));
|
||
return { ok: true, reused: false, eventUrl, listenedTypes: eventTypes };
|
||
});
|
||
}
|
||
|
||
async function clickButtonMatching(frame, predicate, label) {
|
||
const buttons = frame.locator("button,[role='button']");
|
||
const count = Math.min(await buttons.count().catch(() => 0), 80);
|
||
for (let index = 0; index < count; index += 1) {
|
||
const button = buttons.nth(index);
|
||
const visible = await button.isVisible().catch(() => false);
|
||
if (!visible) continue;
|
||
const text = normalizeButtonText(await button.textContent().catch(() => ""));
|
||
if (!predicate(text)) continue;
|
||
await button.click({ timeout: 5000 });
|
||
return { ok: true, label, index, text };
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function normalizeButtonText(value) {
|
||
return String(value || "").replace(/\\s+/gu, " ").trim();
|
||
}`;
|
||
const oldBlocked = "const blocked = /^(send|stop|cancel|thinking|loading|new session|settings)$/iu;";
|
||
const newBlocked = "const blocked = /^(send|stop|cancel|thinking|loading|new session|settings|build|shell|review|last turn changes|create git repository|all files|main branch|deepseek-v4-flash)$/iu;";
|
||
const withFlow = source.replace(oldFlow, newFlow);
|
||
const withFrameFinder = withFlow.replace(oldFrameFinder, newFrameFinder);
|
||
const patched = withFrameFinder.replace(oldBlocked, newBlocked);
|
||
if (withFlow === source) throw new Error("internal opencode-smoke generated script patch did not apply: flow");
|
||
if (withFrameFinder === withFlow) throw new Error("internal opencode-smoke generated script patch did not apply: frame finder");
|
||
if (patched === withFrameFinder) throw new Error("internal opencode-smoke generated script patch did not apply: assistant text filter");
|
||
if (!patched.includes("ensureOpenCodeProjectOpen") || !patched.includes("startOpenCodeEventProbe")) throw new Error("internal opencode-smoke generated script patch is missing required helpers");
|
||
return patched;
|
||
}
|
||
|
||
function resolveWebProbeScreenshotUrl(origin: string, targetPath: string): string {
|
||
try {
|
||
return new URL(targetPath || "/", origin).toString();
|
||
} catch {
|
||
throw new Error(`web-probe screenshot --path cannot be resolved against ${origin}: ${targetPath}`);
|
||
}
|
||
}
|
||
|
||
function parseWebProbeScreenshotViewport(value: string): string {
|
||
if (!/^[1-9][0-9]{1,4}x[1-9][0-9]{1,4}$/u.test(value)) throw new Error(`web-probe screenshot --viewport must look like 1440x900, got ${value}`);
|
||
const [widthRaw, heightRaw] = value.split("x");
|
||
const width = Number(widthRaw);
|
||
const height = Number(heightRaw);
|
||
if (width < 240 || width > 7680 || height < 240 || height > 4320) throw new Error(`web-probe screenshot --viewport out of range: ${value}`);
|
||
return value;
|
||
}
|
||
|
||
function parseWebProbeScreenshotName(value: string): string {
|
||
if (!/^[A-Za-z0-9._-]{1,120}$/u.test(value)) throw new Error("web-probe screenshot --name must contain only letters, numbers, dot, underscore, or dash, max 120 chars");
|
||
return value.endsWith(".png") ? value : `${value}.png`;
|
||
}
|
||
|
||
function parseWebProbeScreenshotWaitUntil(value: string): "load" | "domcontentloaded" | "networkidle" | "commit" {
|
||
if (value === "load" || value === "domcontentloaded" || value === "networkidle" || value === "commit") return value;
|
||
throw new Error(`web-probe screenshot --wait-until must be load, domcontentloaded, networkidle, or commit; got ${value}`);
|
||
}
|