Lyon
13825 lines
778 KiB
TypeScript
13825 lines
778 KiB
TypeScript
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
|
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { repoRoot, rootPath, type Config } from "./config";
|
|
import { runCommand, type CommandResult } from "./command";
|
|
import { startJob } from "./jobs";
|
|
import { classifySshTcpPoolFailure } from "./ssh";
|
|
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "./hwlab-node-control-plane";
|
|
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec } from "./hwlab-node-lanes";
|
|
import { nodeWebProbeScriptRunnerSource } from "./hwlab-node-web-probe-runner-source";
|
|
import { nodeWebObserveAnalyzerSource } from "./hwlab-node-web-observe-analyzer-source";
|
|
import { nodeWebObserveRunnerSource } from "./hwlab-node-web-observe-runner-source";
|
|
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "./hwlab-node-web-observe-collect";
|
|
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "./hwlab-node-web-observe-render";
|
|
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "./hwlab-node-help";
|
|
import { compactWebProbeResult, compactWebProbeScriptResult } from "./hwlab-node-web-probe-summary";
|
|
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "./hwlab-node-observability-promql";
|
|
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "./hwlab-node-transport";
|
|
import type { RenderedCliResult } from "./output";
|
|
|
|
type SecretAction = "status" | "ensure" | "cleanup-owned-postgres" | "cleanup-obsolete";
|
|
type SecretPreset = "openfga" | "master-server-admin-api-key" | "bootstrap-admin" | "code-agent-provider" | "cloud-api-db" | "owned-postgres-cleanup" | "obsolete-secret-cleanup";
|
|
type NodeRuntimeRenderLocation = "node-host" | "local";
|
|
type WebProbeBrowserProxyMode = "auto" | "direct";
|
|
|
|
interface NodeWebProbeRunOptions {
|
|
action: "run";
|
|
node: string;
|
|
lane: string;
|
|
url: string;
|
|
timeoutMs: number;
|
|
waitAfterSubmitMs: number;
|
|
waitMessagesMs: number;
|
|
waitAgentTerminalMs: number;
|
|
traceSampleCount: number;
|
|
traceSampleIntervalMs: number;
|
|
message: string | null;
|
|
conversationId: string | null;
|
|
freshSession: boolean;
|
|
cancelRunning: boolean;
|
|
commandTimeoutSeconds: number;
|
|
commandTimeoutAutoSeconds: number;
|
|
commandTimeoutUserProvided: boolean;
|
|
}
|
|
|
|
interface NodeWebProbeScriptOptions {
|
|
action: "script";
|
|
node: string;
|
|
lane: string;
|
|
url: string;
|
|
timeoutMs: number;
|
|
viewport: string;
|
|
browserProxyMode: WebProbeBrowserProxyMode;
|
|
commandTimeoutSeconds: number;
|
|
scriptText: string;
|
|
scriptSource: {
|
|
kind: "stdin" | "file";
|
|
path: string | null;
|
|
byteCount: number;
|
|
sha256: string;
|
|
};
|
|
}
|
|
|
|
type NodeWebProbeObserveAction = "start" | "status" | "command" | "stop" | "collect" | "analyze";
|
|
type NodeWebProbeObserveCommandType = "login" | "preflight" | "goto" | "newSession" | "sendPrompt" | "steer" | "cancel" | "selectProvider" | "clickSession" | "screenshot" | "mark" | "stop";
|
|
|
|
interface NodeWebProbeObserveOptions {
|
|
action: "observe";
|
|
observeAction: NodeWebProbeObserveAction;
|
|
id: string | null;
|
|
node: string;
|
|
lane: string;
|
|
url: string;
|
|
targetPath: string;
|
|
viewport: string;
|
|
browserProxyMode: WebProbeBrowserProxyMode;
|
|
sampleIntervalMs: number;
|
|
screenshotIntervalMs: number;
|
|
observerRefreshIntervalMs: number;
|
|
maxSamples: number;
|
|
commandTimeoutSeconds: number;
|
|
waitMs: number;
|
|
tailLines: number;
|
|
maxFiles: number;
|
|
collectView: NodeWebProbeObserveCollectView;
|
|
collectFile: string | null;
|
|
collectFinding: string | null;
|
|
collectGrep: string | null;
|
|
collectTraceId: string | null;
|
|
collectSampleSeq: number | null;
|
|
collectTimestamp: string | null;
|
|
collectTurn: number | null;
|
|
analyzeArchivePrefix: string | null;
|
|
analyzeTailSamples: number | null;
|
|
full: boolean;
|
|
stateDir: string | null;
|
|
jobId: string | null;
|
|
force: boolean;
|
|
commandType: NodeWebProbeObserveCommandType | null;
|
|
commandText: string | null;
|
|
commandPath: string | null;
|
|
commandLabel: string | null;
|
|
commandSessionId: string | null;
|
|
commandProvider: string | null;
|
|
}
|
|
|
|
type NodeWebProbeOptions = NodeWebProbeRunOptions | NodeWebProbeScriptOptions | NodeWebProbeObserveOptions;
|
|
|
|
interface WebObserveIndexEntry {
|
|
id: string;
|
|
node: string;
|
|
lane: string;
|
|
workspace: string;
|
|
stateDir: string;
|
|
url: string;
|
|
targetPath: string;
|
|
status: string | null;
|
|
pid: number | null;
|
|
startedAt: string | null;
|
|
updatedAt: string;
|
|
}
|
|
|
|
type NodeObservabilityAction = "plan" | "apply" | "status" | "workbench-summary" | "performance-summary";
|
|
|
|
interface NodeObservabilityOptions {
|
|
action: NodeObservabilityAction;
|
|
node: string;
|
|
lane: HwlabRuntimeLane;
|
|
confirm: boolean;
|
|
dryRun: boolean;
|
|
full: boolean;
|
|
timeoutSeconds: number;
|
|
spec: HwlabRuntimeLaneSpec;
|
|
}
|
|
|
|
interface NodeRuntimeRenderResult {
|
|
readonly result: CommandResult;
|
|
readonly renderDir: string;
|
|
readonly worktreeDir: string;
|
|
readonly location: NodeRuntimeRenderLocation;
|
|
}
|
|
|
|
interface NodeRuntimeCleanupPipelineRunRow {
|
|
name: string;
|
|
createdAt: string | null;
|
|
ageMinutes: number | null;
|
|
status: string | null;
|
|
reason: string | null;
|
|
selected?: boolean;
|
|
selectedReason?: string;
|
|
}
|
|
|
|
interface NodeRuntimeCleanupOptions {
|
|
minAgeMinutes: number;
|
|
limit: number;
|
|
sourceCommit?: string;
|
|
pipelineRun?: string;
|
|
targetPipelineRun?: string;
|
|
includeActive: boolean;
|
|
dryRun: boolean;
|
|
}
|
|
|
|
interface NodeSecretOptions {
|
|
action: SecretAction;
|
|
node: string;
|
|
lane: string;
|
|
name: string;
|
|
key?: string;
|
|
preset: SecretPreset;
|
|
dryRun: boolean;
|
|
confirm: boolean;
|
|
force?: boolean;
|
|
timeoutSeconds: number;
|
|
}
|
|
|
|
interface BootstrapAdminSecretMaterial {
|
|
ok: boolean;
|
|
sourceRef: string | null;
|
|
sourceKey: string | null;
|
|
sourcePath: string | null;
|
|
sourcePresent: boolean;
|
|
sourceFingerprint: string | null;
|
|
passwordHash: string | null;
|
|
error: string | null;
|
|
}
|
|
|
|
interface BootstrapAdminPasswordMaterial {
|
|
ok: boolean;
|
|
sourceRef: string | null;
|
|
sourceKey: string | null;
|
|
sourcePath: string | null;
|
|
sourcePresent: boolean;
|
|
sourceFingerprint: string | null;
|
|
password: string | null;
|
|
error: string | null;
|
|
}
|
|
|
|
interface NodePublicExposureOptions {
|
|
action: "public-exposure";
|
|
node: string;
|
|
lane: HwlabRuntimeLane;
|
|
dryRun: boolean;
|
|
confirm: boolean;
|
|
timeoutSeconds: number;
|
|
spec: HwlabRuntimeLaneSpec;
|
|
}
|
|
|
|
interface RuntimeSecretSpec {
|
|
node: string;
|
|
lane: string;
|
|
namespace: string;
|
|
platformDb: boolean;
|
|
runtimeLaneSpec?: HwlabRuntimeLaneSpec;
|
|
externalPostgres?: NonNullable<HwlabRuntimeLaneSpec["externalPostgres"]>;
|
|
platformPostgresService: string;
|
|
platformPostgresEndpointAddress?: string;
|
|
platformPostgresEndpointSlice: string;
|
|
postgresSecret: string;
|
|
postgresStatefulSet: string;
|
|
postgresAdminUser: string;
|
|
openFgaSecret: string;
|
|
openFgaDbName: string;
|
|
openFgaDbUser: string;
|
|
openFgaDbHost: string;
|
|
masterAdminApiKeySecret: string;
|
|
bootstrapAdminSecret: string;
|
|
bootstrapAdminPasswordHashKey: string;
|
|
bootstrapAdminUsername: string;
|
|
bootstrapAdminDisplayName: string;
|
|
bootstrapAdminPasswordSourceRef?: string;
|
|
bootstrapAdminPasswordSourceKey?: string;
|
|
bootstrapAdminPasswordHashTransform?: "hwlab-sha256";
|
|
bootstrapAdminSourceNamespace: string;
|
|
bootstrapAdminSourceSecret: string;
|
|
cloudApiDbSecret: string;
|
|
cloudApiDbKey: string;
|
|
cloudApiDbName: string;
|
|
cloudApiDbUser: string;
|
|
cloudApiDbHost: string;
|
|
cloudApiDeployment: string;
|
|
obsoleteHwpodDbSecret: string;
|
|
obsoleteHwpodDbName: string;
|
|
obsoleteHwpodDbUser: string;
|
|
codeAgentProviderSecret: string;
|
|
codeAgentProviderSourceNamespace: string;
|
|
codeAgentProviderSourceSecret: string;
|
|
fieldManager: string;
|
|
}
|
|
|
|
interface NodeRuntimeGitMirrorTargetSpec {
|
|
id: string;
|
|
node: string;
|
|
lane: string;
|
|
namespace: string;
|
|
serviceReadName: string;
|
|
serviceWriteName: string;
|
|
cachePvcName: string;
|
|
cachePvcStorage: string;
|
|
cacheHostPath: string | null;
|
|
servicePort: number;
|
|
secretName: string;
|
|
syncConfigMapName: string;
|
|
syncJobPrefix: string;
|
|
flushJobPrefix: string;
|
|
toolsImage: string;
|
|
toolsImagePullPolicy: "Always" | "IfNotPresent" | "Never";
|
|
sourceRepository: string;
|
|
sourceBranch: string;
|
|
gitopsBranch: string;
|
|
egressProxy: NodeRuntimeGitMirrorEgressProxySpec;
|
|
githubTransport: NodeRuntimeGitMirrorGithubTransportSpec;
|
|
}
|
|
|
|
type NodeRuntimeGitMirrorGithubTransportSpec =
|
|
| { mode: "ssh" }
|
|
| {
|
|
mode: "https";
|
|
username: string;
|
|
tokenSecretName: string;
|
|
tokenSecretKey: string;
|
|
tokenSourceRef: string;
|
|
tokenSourceKey: string;
|
|
};
|
|
|
|
type NodeRuntimeGitMirrorEgressProxySpec =
|
|
| { mode: "direct"; required: false }
|
|
| {
|
|
mode: "k8s-service-cluster-ip";
|
|
clientName: string;
|
|
namespace: string;
|
|
serviceName: string;
|
|
port: number;
|
|
sourceRef: string;
|
|
sourceKey: string;
|
|
sourceType: "subscription-url";
|
|
noProxy: string[];
|
|
};
|
|
|
|
const MASTER_ADMIN_API_KEY_KEY = "api-key";
|
|
const BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY = "password-hash";
|
|
const BOOTSTRAP_ADMIN_SOURCE_NAMESPACE = "hwlab-v02";
|
|
const BOOTSTRAP_ADMIN_SOURCE_SECRET = "hwlab-v02-bootstrap-admin";
|
|
const OPENFGA_AUTHN_KEY = "authn-preshared-key";
|
|
const OPENFGA_DATASTORE_URI_KEY = "datastore-uri";
|
|
const OPENFGA_POSTGRES_PASSWORD_KEY = "postgres-password";
|
|
const CLOUD_API_DB_KEY = "database-url";
|
|
const CODE_AGENT_PROVIDER_OPENAI_KEY = "openai-api-key";
|
|
const CODE_AGENT_PROVIDER_OPENCODE_KEY = "opencode-api-key";
|
|
const CODE_AGENT_PROVIDER_SOURCE_NAMESPACE = "hwlab-v02";
|
|
const CODE_AGENT_PROVIDER_SOURCE_SECRET = "hwlab-v02-code-agent-provider";
|
|
const HWLAB_CI_NAMESPACE = "hwlab-ci";
|
|
const NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS = 120;
|
|
const NODE_RUNTIME_TRIGGER_SEVERE_WARNING_MS = NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS * 1000;
|
|
|
|
export async function runHwlabNodeCommand(_config: Config, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
|
|
if (args.length === 0) return hwlabNodeHelp();
|
|
const [domain] = args;
|
|
if (domain === "control-plane" && args[1] === "infra") {
|
|
if (args.length === 2 || args.includes("--help") || args.includes("-h") || args[2] === "help") return hwlabNodeControlPlaneInfraHelp();
|
|
return runHwlabNodeControlPlaneInfra(args.slice(2));
|
|
}
|
|
if (domain === "test-accounts") {
|
|
const { runHwlabTestAccountsCommand } = await import("./hwlab-test-accounts");
|
|
return runHwlabTestAccountsCommand(args.slice(1));
|
|
}
|
|
if (domain === "web-probe") {
|
|
if (args.length === 1 || args.includes("--help") || args.includes("-h") || args[1] === "help") return hwlabNodeWebProbeHelp();
|
|
return runNodeWebProbe(parseNodeWebProbeOptions(args.slice(1)));
|
|
}
|
|
if (domain === "observability") {
|
|
if (args.length === 1 || args.includes("--help") || args.includes("-h") || args[1] === "help") return hwlabNodeObservabilityHelp();
|
|
return runNodeObservability(parseNodeObservabilityOptions(args.slice(1)));
|
|
}
|
|
if (args.includes("--help") || args.includes("-h")) return hwlabNodeHelp();
|
|
if (domain === "control-plane" || domain === "git-mirror") {
|
|
return runNodeDelegatedDomain(_config, domain, args.slice(1));
|
|
}
|
|
if (domain !== "secret") {
|
|
return { ok: false, command: `hwlab nodes ${domain ?? ""}`.trim(), message: "supported commands: hwlab nodes control-plane, hwlab nodes git-mirror, hwlab nodes observability, hwlab nodes secret, hwlab nodes test-accounts, hwlab nodes web-probe" };
|
|
}
|
|
const options = parseSecretOptions(args.slice(1));
|
|
return runNodeSecret(options);
|
|
}
|
|
|
|
export { hwlabNodeHelp, hwlabNodeWebProbeHelp, hwlabNodeObservabilityHelp } from "./hwlab-node-help";
|
|
|
|
function parseNodeObservabilityOptions(args: string[]): NodeObservabilityOptions {
|
|
const [actionRaw] = args;
|
|
if (typeof actionRaw !== "string" || actionRaw.startsWith("--")) {
|
|
throw new Error("observability usage: observability ACTION --node NODE --lane vNN [--dry-run|--confirm]");
|
|
}
|
|
if (actionRaw !== "plan" && actionRaw !== "apply" && actionRaw !== "status" && actionRaw !== "workbench-summary" && actionRaw !== "performance-summary") {
|
|
throw new Error(`observability action must be plan, apply, status, workbench-summary, or performance-summary; got ${actionRaw}`);
|
|
}
|
|
assertKnownOptions(args, new Set(["--node", "--lane", "--timeout-seconds"]), new Set(["--dry-run", "--confirm", "--full", "--raw"]));
|
|
const node = requiredOption(args, "--node");
|
|
assertNodeId(node);
|
|
const laneRaw = requiredOption(args, "--lane");
|
|
if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`--lane must be one of v02, v03; got ${laneRaw}`);
|
|
const confirm = args.includes("--confirm");
|
|
const dryRun = args.includes("--dry-run");
|
|
if (confirm && dryRun) throw new Error("observability accepts only one of --confirm or --dry-run");
|
|
if (actionRaw === "apply" && !confirm && !dryRun) throw new Error("observability apply requires --dry-run or --confirm");
|
|
if (actionRaw !== "apply" && (confirm || dryRun)) throw new Error(`observability ${actionRaw} is read-only and does not accept --confirm or --dry-run`);
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane: laneRaw,
|
|
confirm,
|
|
dryRun,
|
|
full: args.includes("--full") || args.includes("--raw"),
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
|
|
spec: hwlabRuntimeLaneSpecForNode(laneRaw, node),
|
|
};
|
|
}
|
|
|
|
async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomain, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
|
|
const scoped = parseNodeScopedDelegatedOptions(domain, args);
|
|
const defaultSpec = hwlabRuntimeLaneSpec(scoped.lane);
|
|
if (domain === "control-plane" && scoped.action === "public-exposure") {
|
|
return runNodePublicExposure({
|
|
action: "public-exposure",
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
dryRun: scoped.dryRun || !scoped.confirm,
|
|
confirm: scoped.confirm,
|
|
timeoutSeconds: scoped.timeoutSeconds,
|
|
spec: scoped.spec,
|
|
});
|
|
}
|
|
if (domain === "control-plane" && scoped.action === "runtime-image") {
|
|
return nodeRuntimeBaseImageCommand(scoped);
|
|
}
|
|
if (domain === "control-plane" && scoped.action === "plan") {
|
|
const result = nodeRuntimeControlPlanePlan(scoped);
|
|
return nodeScopedFullOutput(scoped) ? result : withNodeRuntimeControlPlanePlanRendered(result, scoped);
|
|
}
|
|
if (domain === "control-plane" && scoped.node !== defaultSpec.nodeId) {
|
|
if (scoped.action === "status") {
|
|
const result = nodeRuntimeControlPlaneStatus(scoped);
|
|
return nodeScopedFullOutput(scoped) ? result : withNodeRuntimeControlPlaneStatusRendered(result, scoped);
|
|
}
|
|
if (scoped.action === "apply" || scoped.action === "trigger-current" || scoped.action === "refresh" || scoped.action === "sync" || scoped.action === "runtime-migration" || scoped.action === "cleanup-runs") {
|
|
if (scoped.confirm && !scoped.dryRun && !scoped.wait) return startNodeDelegatedJob(scoped);
|
|
return nodeRuntimeControlPlaneRun(scoped);
|
|
}
|
|
return nodeRuntimeUnsupportedAction(scoped);
|
|
}
|
|
if (domain === "git-mirror" && scoped.node !== defaultSpec.nodeId) {
|
|
if (scoped.action === "status") {
|
|
const result = nodeRuntimeGitMirrorStatus(scoped);
|
|
return nodeScopedFullOutput(scoped) ? result : withNodeRuntimeGitMirrorRendered(result, scoped);
|
|
}
|
|
if (scoped.action === "sync" || scoped.action === "flush") {
|
|
if (scoped.confirm && !scoped.dryRun && !scoped.wait) return startNodeDelegatedJob(scoped);
|
|
const result = nodeRuntimeGitMirrorRun(scoped);
|
|
return nodeScopedFullOutput(scoped) ? result : withNodeRuntimeGitMirrorRendered(result, scoped);
|
|
}
|
|
return nodeRuntimeUnsupportedAction(scoped);
|
|
}
|
|
if (domain === "control-plane" && scoped.action === "allow-endpoint-bridge") {
|
|
return runNodeEndpointBridge(scoped);
|
|
}
|
|
if (domain === "control-plane" && scoped.action === "trigger-current" && scoped.confirm && !scoped.dryRun && !scoped.wait) {
|
|
return startNodeDelegatedJob(scoped);
|
|
}
|
|
if (domain === "git-mirror" && (scoped.action === "sync" || scoped.action === "flush") && scoped.confirm && !scoped.dryRun && !scoped.wait) {
|
|
return startNodeDelegatedJob(scoped);
|
|
}
|
|
const delegatedArgs = stripOption(args, "--node");
|
|
const result = await runDelegatedHwlabNodeCommand(config, domain, delegatedArgs);
|
|
return rewriteDelegatedNodeResult(result, scoped);
|
|
}
|
|
|
|
function parseNodeScopedDelegatedOptions(domain: DelegatedNodeDomain, args: string[]): {
|
|
domain: DelegatedNodeDomain;
|
|
action: string;
|
|
runtimeImageAction: string | null;
|
|
node: string;
|
|
lane: HwlabRuntimeLane;
|
|
confirm: boolean;
|
|
dryRun: boolean;
|
|
wait: boolean;
|
|
rerun: boolean;
|
|
allowLiveDbRead: boolean;
|
|
timeoutSeconds: number;
|
|
originalArgs: string[];
|
|
spec: HwlabRuntimeLaneSpec;
|
|
} {
|
|
const [actionRaw] = args;
|
|
if (typeof actionRaw !== "string" || actionRaw.startsWith("--")) throw new Error(`${domain} usage: ${domain} ACTION --node NODE --lane vNN [--dry-run|--confirm]`);
|
|
const runtimeImageAction = actionRaw === "runtime-image" && typeof args[1] === "string" && !args[1].startsWith("--") ? args[1] : null;
|
|
const node = requiredOption(args, "--node");
|
|
assertNodeId(node);
|
|
const laneRaw = requiredOption(args, "--lane");
|
|
if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`--lane must be one of v02, v03; got ${laneRaw}`);
|
|
const spec = hwlabRuntimeLaneSpecForNode(laneRaw, node);
|
|
const confirm = args.includes("--confirm");
|
|
const dryRun = args.includes("--dry-run");
|
|
if (confirm && dryRun) throw new Error(`${domain} accepts only one of --confirm or --dry-run`);
|
|
return {
|
|
domain,
|
|
action: actionRaw,
|
|
runtimeImageAction,
|
|
node,
|
|
lane: laneRaw,
|
|
confirm,
|
|
dryRun,
|
|
wait: args.includes("--wait"),
|
|
rerun: args.includes("--rerun"),
|
|
allowLiveDbRead: args.includes("--allow-live-db-read"),
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS, 3600),
|
|
originalArgs: [...args],
|
|
spec,
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeLocalPostgresExpectedAbsent(spec: HwlabRuntimeLaneSpec): boolean {
|
|
return spec.externalPostgres !== undefined || spec.runtimeStore?.postgres?.mode === "platform-service";
|
|
}
|
|
|
|
function nodeRuntimeExpected(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
|
return {
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
node: spec.nodeId,
|
|
nodeRoute: spec.nodeRoute,
|
|
nodeKubeRoute: spec.nodeKubeRoute,
|
|
lane: spec.lane,
|
|
sourceBranch: spec.sourceBranch,
|
|
workspace: spec.workspace,
|
|
cicdRepo: spec.cicdRepo,
|
|
git: {
|
|
sourceUrl: spec.gitUrl,
|
|
readUrl: spec.gitReadUrl,
|
|
writeUrl: spec.gitWriteUrl,
|
|
},
|
|
argo: {
|
|
repoURL: spec.argoRepoUrl,
|
|
},
|
|
gitopsBranch: spec.gitopsBranch,
|
|
catalogPath: spec.catalogPath,
|
|
runtimePath: spec.runtimePath,
|
|
runtimeNamespace: spec.runtimeNamespace,
|
|
renderDir: spec.runtimeRenderDir,
|
|
pipeline: spec.pipeline,
|
|
pipelineRunPrefix: spec.pipelineRunPrefix,
|
|
serviceAccount: spec.serviceAccountName,
|
|
argoApplication: spec.app,
|
|
registryPrefix: spec.registryPrefix,
|
|
baseImage: {
|
|
image: spec.baseImage,
|
|
sourceImage: spec.baseImageSource ?? null,
|
|
},
|
|
serviceIds: spec.serviceIds,
|
|
buildkit: spec.buildkit === undefined ? null : {
|
|
sidecarImage: spec.buildkit.sidecarImage,
|
|
},
|
|
dockerBuildProxy: {
|
|
http: spec.networkProfile.dockerBuildProxy.http,
|
|
https: spec.networkProfile.dockerBuildProxy.https,
|
|
all: spec.networkProfile.dockerBuildProxy.all,
|
|
noProxy: spec.networkProfile.dockerBuildProxy.noProxy,
|
|
},
|
|
stepEnv: spec.stepEnv,
|
|
public: {
|
|
webUrl: spec.publicWebUrl,
|
|
apiUrl: spec.publicApiUrl,
|
|
},
|
|
webProbe: spec.webProbe === undefined ? null : {
|
|
browserProxyMode: spec.webProbe.browserProxyMode ?? null,
|
|
defaultOrigin: spec.webProbe.defaultOrigin ?? null,
|
|
},
|
|
bootstrapAdmin: spec.bootstrapAdmin === undefined ? null : {
|
|
username: spec.bootstrapAdmin.username,
|
|
displayName: spec.bootstrapAdmin.displayName,
|
|
passwordSourceRef: spec.bootstrapAdmin.passwordSourceRef,
|
|
passwordSourceKey: spec.bootstrapAdmin.passwordSourceKey,
|
|
passwordHashTransform: spec.bootstrapAdmin.passwordHashTransform,
|
|
secretName: spec.bootstrapAdmin.secretName,
|
|
secretKey: spec.bootstrapAdmin.secretKey,
|
|
rolloutDeployment: spec.bootstrapAdmin.rolloutDeployment,
|
|
valuesPrinted: false,
|
|
},
|
|
publicExposure: spec.publicExposure === null ? null : publicExposureSummary(spec.publicExposure),
|
|
runtimeStore: spec.runtimeStore ?? null,
|
|
downloadProfile: {
|
|
id: spec.downloadProfileId,
|
|
git: spec.downloadProfile.git,
|
|
npm: spec.downloadProfile.npm,
|
|
},
|
|
observability: spec.observability,
|
|
runtimeImageRewrites: spec.runtimeImageRewrites,
|
|
externalPostgres: spec.externalPostgres === undefined ? null : {
|
|
provider: spec.externalPostgres.provider,
|
|
configRef: spec.externalPostgres.configRef,
|
|
serviceName: spec.externalPostgres.serviceName,
|
|
endpointAddress: spec.externalPostgres.endpointAddress,
|
|
port: spec.externalPostgres.port,
|
|
sslmode: spec.externalPostgres.sslmode,
|
|
database: spec.externalPostgres.database,
|
|
cloudApi: {
|
|
secretName: spec.externalPostgres.cloudApi.secretName,
|
|
secretKey: spec.externalPostgres.cloudApi.secretKey,
|
|
sourceRef: spec.externalPostgres.cloudApi.sourceRef,
|
|
envKey: spec.externalPostgres.cloudApi.envKey,
|
|
role: spec.externalPostgres.cloudApi.role,
|
|
},
|
|
openfga: {
|
|
secretName: spec.externalPostgres.openfga.secretName,
|
|
secretKey: spec.externalPostgres.openfga.secretKey,
|
|
sourceRef: spec.externalPostgres.openfga.sourceRef,
|
|
envKey: spec.externalPostgres.openfga.envKey,
|
|
authnKey: spec.externalPostgres.openfga.authnKey ?? null,
|
|
role: spec.externalPostgres.openfga.role,
|
|
schema: spec.externalPostgres.openfga.schema ?? null,
|
|
},
|
|
valuesPrinted: false,
|
|
},
|
|
localPostgres: {
|
|
shouldRender: spec.externalPostgres === undefined,
|
|
expectedAbsent: spec.externalPostgres !== undefined,
|
|
},
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeControlPlanePlan(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
return {
|
|
ok: true,
|
|
command: `hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`,
|
|
mode: "plan",
|
|
mutation: false,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
expected: nodeRuntimeExpected(scoped.spec),
|
|
checks: {
|
|
nodeScopedTargetConfigured: true,
|
|
externalPostgresDeclared: scoped.spec.externalPostgres !== undefined,
|
|
secretValuesPrinted: false,
|
|
runtimeNamespace: scoped.spec.runtimeNamespace,
|
|
localPostgresExpectedAbsent: nodeRuntimeLocalPostgresExpectedAbsent(scoped.spec),
|
|
publicExposureDeclared: scoped.spec.publicExposure !== null,
|
|
},
|
|
next: {
|
|
infraStatus: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
platformDbStatus: scoped.spec.externalPostgres === undefined
|
|
? null
|
|
: `bun scripts/cli.ts platform-db postgres status --config ${scoped.spec.externalPostgres.configRef}`,
|
|
publicExposure: scoped.spec.publicExposure === null ? null : `bun scripts/cli.ts hwlab nodes control-plane public-exposure --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
|
|
function runNodeObservability(options: NodeObservabilityOptions): Record<string, unknown> {
|
|
if (options.action === "plan") return nodeObservabilityPlan(options);
|
|
if (options.action === "apply") return nodeObservabilityApply(options);
|
|
if (options.action === "status") return nodeObservabilityStatus(options);
|
|
if (options.action === "performance-summary") return nodeObservabilityPerformanceSummary(options);
|
|
return nodeObservabilityWorkbenchSummary(options);
|
|
}
|
|
|
|
function nodeObservabilityPlan(options: NodeObservabilityOptions): Record<string, unknown> {
|
|
const configProblem = nodeObservabilityConfigProblem(options.spec.observability);
|
|
return {
|
|
ok: configProblem === null,
|
|
command: `hwlab nodes observability plan --node ${options.node} --lane ${options.lane}`,
|
|
mode: "node-observability-plan",
|
|
mutation: false,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
expected: nodeObservabilityExpected(options.spec),
|
|
renderPlan: nodeObservabilityRenderPlan(options.spec.observability),
|
|
degradedReason: configProblem,
|
|
next: {
|
|
dryRun: `bun scripts/cli.ts hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --dry-run`,
|
|
apply: `bun scripts/cli.ts hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`,
|
|
status: `bun scripts/cli.ts hwlab nodes observability status --node ${options.node} --lane ${options.lane}`,
|
|
workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
|
|
performanceSummary: `bun scripts/cli.ts hwlab nodes observability performance-summary --node ${options.node} --lane ${options.lane}`,
|
|
},
|
|
};
|
|
}
|
|
|
|
function nodeObservabilityApply(options: NodeObservabilityOptions): Record<string, unknown> {
|
|
const configProblem = nodeObservabilityConfigProblem(options.spec.observability);
|
|
const applyPlan = nodeObservabilityApplyPlan(options.spec.observability);
|
|
if (options.dryRun) {
|
|
return {
|
|
ok: configProblem === null && applyPlan.supported === true,
|
|
command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --dry-run`,
|
|
mode: "node-observability-apply-dry-run",
|
|
mutation: false,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
expected: nodeObservabilityExpected(options.spec),
|
|
applyPlan,
|
|
degradedReason: configProblem ?? (applyPlan.supported === true ? undefined : "node-observability-apply-mode-unsupported"),
|
|
};
|
|
}
|
|
if (configProblem !== null || applyPlan.supported !== true) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`,
|
|
mode: "node-observability-apply",
|
|
mutation: false,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
applyPlan,
|
|
degradedReason: configProblem ?? "node-observability-apply-mode-unsupported",
|
|
};
|
|
}
|
|
const status = nodeObservabilityStatus({ ...options, full: true });
|
|
return {
|
|
ok: status.ok === true,
|
|
command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`,
|
|
mode: "node-observability-apply",
|
|
mutation: false,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
applyPlan,
|
|
appliedResources: [],
|
|
applyResult: {
|
|
mode: applyPlan.mode,
|
|
reason: applyPlan.reason,
|
|
runtimeValidationOnly: true,
|
|
},
|
|
status: options.full ? status : summarizeNodeObservabilityStatus(status),
|
|
degradedReason: status.ok === true ? undefined : "node-observability-status-not-ready",
|
|
};
|
|
}
|
|
|
|
function nodeObservabilityStatus(options: NodeObservabilityOptions): Record<string, unknown> {
|
|
const configProblem = nodeObservabilityConfigProblem(options.spec.observability);
|
|
const serviceStatus = nodeObservabilityServiceStatus(options);
|
|
const publicRawMetrics = nodeObservabilityPublicRawMetricsProbe(options);
|
|
const workbenchSummary = nodeObservabilityWorkbenchSummary(options, serviceStatus);
|
|
const ok = configProblem === null && serviceStatus.ready === true && publicRawMetrics.ok === true && workbenchSummary.ok === true;
|
|
const status = {
|
|
ok,
|
|
command: `hwlab nodes observability status --node ${options.node} --lane ${options.lane}`,
|
|
mode: "node-observability-status",
|
|
mutation: false,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
expected: nodeObservabilityExpected(options.spec),
|
|
service: serviceStatus,
|
|
publicRawMetrics,
|
|
workbenchSummary,
|
|
degradedReason: configProblem
|
|
?? (serviceStatus.ready === true
|
|
? publicRawMetrics.ok === true
|
|
? workbenchSummary.ok === true ? undefined : "node-observability-workbench-metrics-not-ready"
|
|
: "node-observability-public-raw-metrics-boundary-failed"
|
|
: "node-observability-service-not-ready"),
|
|
next: {
|
|
plan: `bun scripts/cli.ts hwlab nodes observability plan --node ${options.node} --lane ${options.lane}`,
|
|
workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
|
|
performanceSummary: `bun scripts/cli.ts hwlab nodes observability performance-summary --node ${options.node} --lane ${options.lane}`,
|
|
},
|
|
};
|
|
return options.full ? status : summarizeNodeObservabilityStatus(status);
|
|
}
|
|
|
|
function nodeObservabilityWorkbenchSummary(options: NodeObservabilityOptions, existingServiceStatus?: Record<string, unknown>): Record<string, unknown> {
|
|
const configProblem = nodeObservabilityConfigProblem(options.spec.observability);
|
|
const serviceStatus = existingServiceStatus ?? nodeObservabilityServiceStatus(options);
|
|
const podName = typeof serviceStatus.podName === "string" && serviceStatus.podName.length > 0 ? serviceStatus.podName : null;
|
|
const metricsProbe = podName === null ? null : nodeObservabilityMetricsProbe(options, podName);
|
|
const metrics = metricsProbe?.summary ?? summarizeNodeObservabilityMetrics(options.spec.observability, "", null);
|
|
const ok = configProblem === null && serviceStatus.ready === true && metricsProbe?.ok === true && metrics.ready === true;
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
|
|
mode: "node-observability-workbench-summary",
|
|
mutation: false,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
metricsEndpoint: nodeObservabilityEndpointSummary(options.spec.observability),
|
|
service: {
|
|
ready: serviceStatus.ready === true,
|
|
namespace: serviceStatus.namespace,
|
|
serviceName: serviceStatus.serviceName,
|
|
podName,
|
|
containerName: serviceStatus.containerName,
|
|
},
|
|
recordingRules: nodeObservabilityRecordingRuleSummaries(options.spec.observability),
|
|
warningAlerts: nodeObservabilityWarningAlertSummaries(options.spec.observability),
|
|
metrics,
|
|
probe: metricsProbe === null ? null : {
|
|
ok: metricsProbe.ok,
|
|
httpStatus: metricsProbe.httpStatus,
|
|
bodyBytes: metricsProbe.bodyBytes,
|
|
result: compactRuntimeCommand(metricsProbe.result),
|
|
error: metricsProbe.error,
|
|
},
|
|
degradedReason: configProblem
|
|
?? (serviceStatus.ready === true
|
|
? metricsProbe?.ok === true
|
|
? metrics.ready === true ? undefined : "node-observability-required-series-missing"
|
|
: "node-observability-metrics-endpoint-not-readable"
|
|
: "node-observability-service-not-ready"),
|
|
};
|
|
}
|
|
|
|
function nodeObservabilityPerformanceSummary(options: NodeObservabilityOptions): Record<string, unknown> {
|
|
const workbench = options.spec.observability.workbench;
|
|
const summaryPath = workbench?.summaryPath ?? "/v1/web-performance/summary";
|
|
const endpoint = joinUrlPath(options.spec.publicWebUrl, summaryPath);
|
|
const secretSpec = runtimeSecretSpec({ node: options.node, lane: options.lane });
|
|
const material = readBootstrapAdminPasswordMaterial(secretSpec);
|
|
const credential = webProbeCredential(secretSpec, material);
|
|
const command = `hwlab nodes observability performance-summary --node ${options.node} --lane ${options.lane}`;
|
|
if (material.ok !== true || material.password === null) {
|
|
return {
|
|
ok: false,
|
|
command,
|
|
mode: "node-observability-performance-summary",
|
|
mutation: false,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
target: {
|
|
workspace: options.spec.workspace,
|
|
webBaseUrl: options.spec.publicWebUrl,
|
|
endpoint,
|
|
},
|
|
credential,
|
|
degradedReason: material.error ?? "web-login-secret-unavailable",
|
|
next: {
|
|
secretStatus: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name bootstrap-admin`,
|
|
webProbe: `bun scripts/cli.ts hwlab nodes web-probe run --node ${options.node} --lane ${options.lane}`,
|
|
},
|
|
};
|
|
}
|
|
const timeoutSeconds = Math.max(10, Math.min(options.timeoutSeconds, 55));
|
|
const script = nodePerformanceSummaryRemoteScript(options.spec.publicWebUrl, summaryPath, secretSpec.bootstrapAdminUsername, material.password);
|
|
const result = runTransWorkspaceStdinScript(options.node, options.spec.workspace, script, timeoutSeconds);
|
|
const report = parseJsonRecordFromText(result.stdout);
|
|
const performance = record(report.performance);
|
|
const rows = nodePerformanceRows(performance);
|
|
const analysis = nodePerformanceAnalysis(performance, rows);
|
|
const ok = result.exitCode === 0 && report.ok === true && performance.ok !== false;
|
|
return {
|
|
ok,
|
|
command,
|
|
mode: "node-observability-performance-summary",
|
|
mutation: false,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
target: {
|
|
workspace: options.spec.workspace,
|
|
webBaseUrl: options.spec.publicWebUrl,
|
|
endpoint,
|
|
summaryPath,
|
|
lowSampleThreshold: workbench?.lowSampleThreshold ?? null,
|
|
},
|
|
credential,
|
|
auth: record(report.auth),
|
|
httpStatus: report.httpStatus ?? null,
|
|
fetchAttempts: report.fetchAttempts ?? null,
|
|
summary: performance,
|
|
analysis,
|
|
probe: compactCommandResultRedacted(result, [material.password]),
|
|
degradedReason: ok
|
|
? undefined
|
|
: typeof report.error === "string" && report.error.length > 0
|
|
? report.error
|
|
: result.exitCode === 0 ? "web-performance-summary-not-ready" : "web-performance-summary-probe-failed",
|
|
next: {
|
|
webProbe: `bun scripts/cli.ts hwlab nodes web-probe run --node ${options.node} --lane ${options.lane}`,
|
|
workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
|
|
apiMetrics: `bun scripts/cli.ts hwlab nodes observability status --node ${options.node} --lane ${options.lane} --full`,
|
|
},
|
|
};
|
|
}
|
|
|
|
function nodePerformanceSummaryRemoteScript(baseUrl: string, summaryPath: string, username: string, password: string): string {
|
|
const nodeScript = [
|
|
"const baseUrl = process.env.HWLAB_WEB_BASE_URL;",
|
|
"const summaryPath = process.env.HWLAB_PERFORMANCE_SUMMARY_PATH || '/v1/web-performance/summary';",
|
|
"const username = process.env.HWLAB_WEB_USER;",
|
|
"const password = process.env.HWLAB_WEB_PASS;",
|
|
"function compactNumber(value) { const n = Number(value); return Number.isFinite(n) ? Math.round(n * 10) / 10 : null; }",
|
|
"function compactSeconds(msValue, secondsValue) { const ms = Number(msValue); if (Number.isFinite(ms)) return Math.round((ms / 1000) * 10) / 10; return compactNumber(secondsValue); }",
|
|
"function compactString(value) { return typeof value === 'string' && value.length > 0 ? value : null; }",
|
|
"function compactRow(row) {",
|
|
" if (!row || typeof row !== 'object' || Array.isArray(row)) return null;",
|
|
" const route = compactString(row.route) || compactString(row.path) || compactString(row.name) || compactString(row.metric) || compactString(row.title);",
|
|
" return {",
|
|
" source: compactString(row.source) || compactString(row.kind) || compactString(row.category),",
|
|
" metric: compactString(row.metric) || compactString(row.eventKind) || compactString(row.phase),",
|
|
" route,",
|
|
" problem: compactString(row.problem) || compactString(row.diagnosis) || compactString(row.reason),",
|
|
" outcome: compactString(row.outcome) || compactString(row.result) || compactString(row.status),",
|
|
" tone: compactString(row.tone) || compactString(row.severity),",
|
|
" sampleState: compactString(row.sampleState) || compactString(row.sampleStatus),",
|
|
" count: compactNumber(row.count ?? row.sampleCount ?? row.samples),",
|
|
" durationUnit: 'seconds',",
|
|
" p50Seconds: compactSeconds(row.p50Ms, row.p50Seconds ?? row.p50 ?? row.medianSeconds ?? row.median),",
|
|
" p75Seconds: compactSeconds(row.p75Ms, row.p75Seconds ?? row.p75),",
|
|
" p95Seconds: compactSeconds(row.p95Ms, row.p95Seconds ?? row.p95),",
|
|
" maxSeconds: compactSeconds(row.maxMs, row.maxSeconds ?? row.max),",
|
|
" };",
|
|
"}",
|
|
"function rowHasSignal(row) { return Boolean(row && (row.source || row.metric || row.route || row.problem || row.outcome)); }",
|
|
"function compactRows(value, limit = 16) { return Array.isArray(value) ? value.map(compactRow).filter(rowHasSignal).slice(0, limit) : []; }",
|
|
"function compactTopN(value) {",
|
|
" if (!Array.isArray(value)) return [];",
|
|
" return value.slice(0, 8).map((group) => {",
|
|
" const rows = compactRows(group?.rows || group?.items || group?.data, 8);",
|
|
" return { id: compactString(group?.id) || compactString(group?.key), title: compactString(group?.title) || compactString(group?.label), rows };",
|
|
" }).filter((group) => group.rows.length > 0);",
|
|
"}",
|
|
"function compactCards(value) {",
|
|
" if (!Array.isArray(value)) return [];",
|
|
" return value.slice(0, 12).map((card) => ({ id: compactString(card?.id), title: compactString(card?.title), value: card?.value ?? null, tone: compactString(card?.tone) || compactString(card?.status), detail: compactString(card?.detail) || compactString(card?.hint) }));",
|
|
"}",
|
|
"function compactPerformance(body) {",
|
|
" if (!body || typeof body !== 'object' || Array.isArray(body)) return { ok: false, degradedReason: 'non-json-summary' };",
|
|
" const dashboard = body.dashboard && typeof body.dashboard === 'object' ? body.dashboard : {};",
|
|
" const summary = body.summary && typeof body.summary === 'object' ? body.summary : {};",
|
|
" const rows = compactRows(body.rows || body.tableRows || body.performanceRows, 24);",
|
|
" const workbenchRows = compactRows(body.workbenchRows || body.workbench || body.workbenchTableRows, 12);",
|
|
" const problems = compactRows(body.problems || body.problemRows || dashboard.problems, 24);",
|
|
" const displayRows = rows.length > 0 ? rows.slice(0, 16) : problems.slice(0, 16);",
|
|
" const displayProblems = rows.length > 0 ? [] : problems.slice(0, 16);",
|
|
" const topN = compactTopN(dashboard.topN || body.topN);",
|
|
" return {",
|
|
" ok: true,",
|
|
" schemaVersion: body.schemaVersion || dashboard.schemaVersion || null,",
|
|
" source: body.source || dashboard.source || null,",
|
|
" observedAt: body.observedAt || dashboard.observedAt || body.generatedAt || dashboard.generatedAt || null,",
|
|
" window: body.window || body.sampleWindow || dashboard.window || null,",
|
|
" freshness: body.freshness || dashboard.freshness || null,",
|
|
" sampleCount: summary.sampleCount ?? body.sampleCount ?? dashboard.sampleCount ?? null,",
|
|
" problemCount: summary.problemCount ?? body.problemCount ?? dashboard.problemCount ?? null,",
|
|
" status: summary.status || body.status || dashboard.status || null,",
|
|
" cards: compactCards(dashboard.cards || body.cards),",
|
|
" rows: displayRows,",
|
|
" workbenchRows,",
|
|
" problems: displayProblems,",
|
|
" topN,",
|
|
" rowCount: rows.length,",
|
|
" workbenchRowCount: workbenchRows.length,",
|
|
" problemRowCount: problems.length,",
|
|
" };",
|
|
"}",
|
|
"function splitSetCookie(raw) { return raw ? raw.split(/,(?=[^;,]+=)/g).map((item) => item.trim()).filter(Boolean) : []; }",
|
|
"async function fetchWithTimeout(url, init, timeoutMs = 12000) {",
|
|
" const controller = new AbortController();",
|
|
" const timer = setTimeout(() => controller.abort(), timeoutMs);",
|
|
" try {",
|
|
" const response = await fetch(url, { ...init, signal: controller.signal });",
|
|
" const text = await response.text();",
|
|
" let body = null;",
|
|
" try { body = JSON.parse(text); } catch {}",
|
|
" return { response, text, body };",
|
|
" } finally { clearTimeout(timer); }",
|
|
"}",
|
|
"function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); }",
|
|
"async function fetchWithRetry(url, init, attempts = 3) {",
|
|
" let last = null;",
|
|
" for (let attempt = 1; attempt <= attempts; attempt += 1) {",
|
|
" try {",
|
|
" const result = await fetchWithTimeout(url, init);",
|
|
" result.attempt = attempt;",
|
|
" last = result;",
|
|
" if (![502, 503, 504].includes(result.response.status)) return result;",
|
|
" } catch (error) {",
|
|
" last = { attempt, error: error instanceof Error ? error.message : String(error), response: { status: 0, ok: false, headers: new Headers() }, body: null, text: '' };",
|
|
" }",
|
|
" if (attempt < attempts) await sleep(400 * attempt);",
|
|
" }",
|
|
" return last;",
|
|
"}",
|
|
"(async () => {",
|
|
" try {",
|
|
" const loginUrl = new URL('/auth/login', baseUrl).toString();",
|
|
" const login = await fetchWithRetry(loginUrl, { method: 'POST', headers: { accept: 'application/json', 'content-type': 'application/json' }, body: JSON.stringify({ username, password }) });",
|
|
" const setCookie = splitSetCookie(login.response.headers.get('set-cookie'));",
|
|
" const cookie = setCookie.map((item) => item.split(';')[0].trim()).filter(Boolean).join('; ');",
|
|
" const auth = { ok: login.response.ok && cookie.length > 0, httpStatus: login.response.status, cookiePresent: cookie.length > 0, attempts: login.attempt || 1 };",
|
|
" if (!auth.ok) {",
|
|
" console.log(JSON.stringify({ ok: false, auth, httpStatus: login.response.status, error: 'web-login-failed' }));",
|
|
" process.exitCode = 1;",
|
|
" return;",
|
|
" }",
|
|
" const summaryUrl = new URL(summaryPath, baseUrl).toString();",
|
|
" const summary = await fetchWithRetry(summaryUrl, { method: 'GET', headers: { accept: 'application/json', cookie } });",
|
|
" const performance = compactPerformance(summary.body);",
|
|
" const ok = summary.response.ok && performance.ok === true;",
|
|
" console.log(JSON.stringify({ ok, auth, httpStatus: summary.response.status, fetchAttempts: summary.attempt || 1, finalUrl: summaryUrl, performance, error: ok ? null : 'web-performance-summary-fetch-failed' }));",
|
|
" process.exitCode = ok ? 0 : 1;",
|
|
" } catch (error) {",
|
|
" console.log(JSON.stringify({ ok: false, error: error instanceof Error ? error.message : String(error) }));",
|
|
" process.exitCode = 1;",
|
|
" }",
|
|
"})();",
|
|
].join("\n");
|
|
return [
|
|
"set -eu",
|
|
`HWLAB_WEB_BASE_URL=${shellQuote(baseUrl)} HWLAB_PERFORMANCE_SUMMARY_PATH=${shellQuote(summaryPath)} HWLAB_WEB_USER=${shellQuote(username)} HWLAB_WEB_PASS=${shellQuote(password)} node <<'NODE'`,
|
|
nodeScript,
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function nodePerformanceRows(performance: Record<string, unknown>): Record<string, unknown>[] {
|
|
const rows = [
|
|
...nodePerformanceArray(performance.problems),
|
|
...nodePerformanceArray(performance.rows),
|
|
...nodePerformanceArray(performance.workbenchRows),
|
|
];
|
|
const topN = Array.isArray(performance.topN) ? performance.topN.map(record) : [];
|
|
for (const group of topN) {
|
|
for (const row of nodePerformanceArray(group.rows)) rows.push(row);
|
|
}
|
|
const seen = new Set<string>();
|
|
return rows.filter((row) => {
|
|
const key = [
|
|
nodePerformanceString(row.source),
|
|
nodePerformanceString(row.metric),
|
|
nodePerformanceString(row.route),
|
|
nodePerformanceString(row.outcome),
|
|
nodePerformanceString(row.problem),
|
|
].join("|");
|
|
if (seen.has(key)) return false;
|
|
seen.add(key);
|
|
return true;
|
|
});
|
|
}
|
|
|
|
function nodePerformanceArray(value: unknown): Record<string, unknown>[] {
|
|
return Array.isArray(value) ? value.map(record).filter((item) => Object.keys(item).length > 0) : [];
|
|
}
|
|
|
|
function nodePerformanceAnalysis(performance: Record<string, unknown>, rows: Record<string, unknown>[]): Record<string, unknown> {
|
|
const actionableRows = rows.filter(nodePerformanceActionableProblem);
|
|
const networkErrors = actionableRows.filter((row) => nodePerformanceString(row.outcome) === "network_error" || nodePerformanceString(row.problem) === "network_error" || nodePerformanceString(row.problem) === "network error");
|
|
const blockedRows = actionableRows.filter((row) => nodePerformanceString(row.tone) === "blocked" || nodePerformanceString(row.status) === "blocked");
|
|
const slowApiRows = rows.filter((row) => {
|
|
const source = nodePerformanceString(row.source) ?? "";
|
|
const metric = nodePerformanceString(row.metric) ?? "";
|
|
const p95Seconds = nodePerformanceNumber(row.p95Seconds);
|
|
return nodePerformanceActionableProblem(row)
|
|
&& (source.includes("api") || metric.includes("api") || (nodePerformanceString(row.route) ?? "").startsWith("/v1") || (nodePerformanceString(row.route) ?? "").startsWith("/health"))
|
|
&& (p95Seconds === null || p95Seconds >= 5);
|
|
});
|
|
const lcpRows = actionableRows.filter((row) => {
|
|
const metric = `${nodePerformanceString(row.metric) ?? ""} ${nodePerformanceString(row.problem) ?? ""}`.toLowerCase();
|
|
return metric.includes("lcp") || metric.includes("largest contentful") || metric.includes("最大内容");
|
|
});
|
|
const maxP95Seconds = Math.max(0, ...rows.map((row) => nodePerformanceNumber(row.p95Seconds) ?? 0));
|
|
const headline = networkErrors.length > 0
|
|
? "Public same-origin API has intermittent network_error samples; treat edge/proxy/API reachability as P0 before tuning page render."
|
|
: lcpRows.length > 0
|
|
? "Workbench page LCP is blocked, likely by first-load API waterfalls and late visible content."
|
|
: slowApiRows.length > 0
|
|
? "Same-origin API p95 is above the interactive budget; split edge wait, server work, and dependency latency."
|
|
: "No blocking performance rows were returned by the current summary window.";
|
|
const priorities = [
|
|
networkErrors.length > 0
|
|
? {
|
|
priority: "P0",
|
|
area: "public-edge-api-reachability",
|
|
action: "Correlate network_error rows with Caddy/FRP/ingress, cloud-api pod restarts, response-header timeout, and browser Resource Timing failure stage.",
|
|
evidence: nodePerformanceEvidence(networkErrors),
|
|
}
|
|
: null,
|
|
slowApiRows.length > 0
|
|
? {
|
|
priority: "P1",
|
|
area: "same-origin-api-latency",
|
|
action: "Add server-side route histograms and dependency spans for slow /health, /v1/live-builds, /v1/workbench/sessions, trace events, hwpod specs, and hwpod-node-ops paths; then optimize the highest p95 route first.",
|
|
evidence: nodePerformanceEvidence(slowApiRows),
|
|
}
|
|
: null,
|
|
lcpRows.length > 0
|
|
? {
|
|
priority: "P1",
|
|
area: "workbench-lcp",
|
|
action: "Render a stable first screen before slow session/detail API calls complete, and split LCP by route plus blocking API waterfall.",
|
|
evidence: nodePerformanceEvidence(lcpRows),
|
|
}
|
|
: null,
|
|
blockedRows.length > 0
|
|
? {
|
|
priority: "P2",
|
|
area: "alert-thresholds-and-sample-quality",
|
|
action: "Keep low-sample rows visible but avoid closing incidents on count=1; require repeated windows or matching server metrics before labeling a regression fixed.",
|
|
evidence: nodePerformanceEvidence(blockedRows),
|
|
}
|
|
: null,
|
|
].filter(Boolean);
|
|
return {
|
|
headline,
|
|
sampleCount: performance.sampleCount ?? null,
|
|
problemCount: performance.problemCount ?? null,
|
|
rowCount: rows.length,
|
|
blockedRowCount: blockedRows.length,
|
|
networkErrorRowCount: networkErrors.length,
|
|
slowApiRowCount: slowApiRows.length,
|
|
lcpRowCount: lcpRows.length,
|
|
maxP95Seconds,
|
|
priorities,
|
|
};
|
|
}
|
|
|
|
function nodePerformanceActionableProblem(row: Record<string, unknown>): boolean {
|
|
const tone = nodePerformanceString(row.tone) ?? "";
|
|
const status = nodePerformanceString(row.status) ?? "";
|
|
if (tone === "blocked" || tone === "warn" || status === "blocked" || status === "warn") return true;
|
|
const sampleState = nodePerformanceString(row.sampleState) ?? "";
|
|
if (sampleState && sampleState !== "ok") return false;
|
|
const problem = (nodePerformanceString(row.problem) ?? "").trim().toLowerCase();
|
|
return Boolean(problem && problem !== "ok" && problem !== "low-sample" && problem !== "no-sample");
|
|
}
|
|
|
|
function nodePerformanceEvidence(rows: Record<string, unknown>[]): Record<string, unknown>[] {
|
|
return rows.slice(0, 8).map((row) => ({
|
|
source: nodePerformanceString(row.source),
|
|
metric: nodePerformanceString(row.metric),
|
|
route: nodePerformanceString(row.route),
|
|
outcome: nodePerformanceString(row.outcome),
|
|
problem: nodePerformanceString(row.problem),
|
|
count: nodePerformanceNumber(row.count),
|
|
durationUnit: nodePerformanceString(row.durationUnit) ?? "seconds",
|
|
p50Seconds: nodePerformanceNumber(row.p50Seconds),
|
|
p75Seconds: nodePerformanceNumber(row.p75Seconds),
|
|
p95Seconds: nodePerformanceNumber(row.p95Seconds),
|
|
}));
|
|
}
|
|
|
|
function nodePerformanceString(value: unknown): string | null {
|
|
return typeof value === "string" && value.length > 0 ? value : null;
|
|
}
|
|
|
|
function nodePerformanceNumber(value: unknown): number | null {
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
}
|
|
|
|
function summarizeNodeObservabilityStatus(status: Record<string, unknown>): Record<string, unknown> {
|
|
const service = record(status.service);
|
|
const publicRawMetrics = record(status.publicRawMetrics);
|
|
const workbenchSummary = record(status.workbenchSummary);
|
|
const metrics = record(workbenchSummary.metrics);
|
|
const recordingRules = Array.isArray(workbenchSummary.recordingRules) ? workbenchSummary.recordingRules : [];
|
|
const warningAlerts = Array.isArray(workbenchSummary.warningAlerts) ? workbenchSummary.warningAlerts : [];
|
|
return {
|
|
ok: status.ok === true,
|
|
command: status.command,
|
|
mode: "node-observability-status-summary",
|
|
mutation: false,
|
|
node: status.node,
|
|
lane: status.lane,
|
|
degradedReason: typeof status.degradedReason === "string" ? status.degradedReason : null,
|
|
service: {
|
|
ready: service.ready === true,
|
|
namespace: service.namespace ?? null,
|
|
serviceName: service.serviceName ?? null,
|
|
podName: service.podName ?? null,
|
|
containerName: service.containerName ?? null,
|
|
},
|
|
publicRawMetrics: {
|
|
ok: publicRawMetrics.ok === true,
|
|
expected: publicRawMetrics.expected ?? null,
|
|
httpStatus: publicRawMetrics.httpStatus ?? null,
|
|
exposesPrometheus: publicRawMetrics.exposesPrometheus === true,
|
|
},
|
|
workbench: {
|
|
ok: workbenchSummary.ok === true,
|
|
httpStatus: metrics.httpStatus ?? null,
|
|
bodyBytes: metrics.bodyBytes ?? null,
|
|
recordingRuleCount: recordingRules.length,
|
|
warningAlertCount: warningAlerts.length,
|
|
seriesByPrefix: metrics.seriesByPrefix ?? {},
|
|
missingSeries: metrics.missingSeries ?? [],
|
|
topSlowDimensions: metrics.topSlowDimensions ?? [],
|
|
deniedBackendEventLineCount: metrics.deniedBackendEventLineCount ?? null,
|
|
maxDeniedBackendEventLines: metrics.maxDeniedBackendEventLines ?? null,
|
|
},
|
|
next: status.next,
|
|
};
|
|
}
|
|
|
|
function nodeObservabilityExpected(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
|
return {
|
|
node: spec.nodeId,
|
|
lane: spec.lane,
|
|
namespace: spec.runtimeNamespace,
|
|
publicApiUrl: spec.publicApiUrl,
|
|
observability: spec.observability,
|
|
sourceOfTruth: hwlabRuntimeLaneConfigPath(),
|
|
runtimeAsSourceOfTruth: false,
|
|
};
|
|
}
|
|
|
|
function nodeObservabilityRenderPlan(observability: HwlabRuntimeObservabilitySpec): Record<string, unknown> {
|
|
const endpoint = observability.metricsEndpoint;
|
|
return {
|
|
prometheusOperator: observability.prometheusOperator,
|
|
metricsEndpoint: nodeObservabilityEndpointSummary(observability),
|
|
workbench: observability.workbench ?? null,
|
|
recordingRules: nodeObservabilityRecordingRuleSummaries(observability),
|
|
warningAlerts: nodeObservabilityWarningAlertSummaries(observability),
|
|
clusterResources: observability.prometheusOperator
|
|
? {
|
|
source: "HWLAB GitOps rendered manifests",
|
|
serviceMonitor: "YAML-rendered only when declared by target lane",
|
|
prometheusRule: "YAML-rendered only when declared by target lane",
|
|
}
|
|
: {
|
|
source: "none",
|
|
reason: endpoint?.scrapeMode === "pod-loopback" ? "target lane declares pod-loopback scrape mode" : "no YAML-declared scrape target",
|
|
},
|
|
publicRawMetrics: endpoint?.publicRawMetrics ?? null,
|
|
};
|
|
}
|
|
|
|
function nodeObservabilityApplyPlan(observability: HwlabRuntimeObservabilitySpec): Record<string, unknown> {
|
|
const endpoint = observability.metricsEndpoint;
|
|
if (endpoint === undefined) {
|
|
return {
|
|
supported: false,
|
|
mode: "unconfigured",
|
|
reason: "metricsEndpoint is missing from YAML",
|
|
};
|
|
}
|
|
if (!observability.prometheusOperator && endpoint.scrapeMode === "pod-loopback") {
|
|
return {
|
|
supported: true,
|
|
mode: "validated-noop",
|
|
reason: "target lane declares pod-loopback metrics collection and no Prometheus Operator resources",
|
|
resources: [],
|
|
validates: ["kubernetes-service", "selected-pod", "pod-loopback-metrics", "public-raw-metrics-boundary", "workbench-required-series"],
|
|
};
|
|
}
|
|
if (observability.prometheusOperator) {
|
|
return {
|
|
supported: false,
|
|
mode: "prometheus-operator",
|
|
reason: "Prometheus Operator object rendering must be declared in the target lane GitOps YAML before this node-scoped apply can own it",
|
|
};
|
|
}
|
|
return {
|
|
supported: false,
|
|
mode: endpoint.scrapeMode,
|
|
reason: "unsupported metricsEndpoint.scrapeMode",
|
|
};
|
|
}
|
|
|
|
function nodeObservabilityConfigProblem(observability: HwlabRuntimeObservabilitySpec): string | null {
|
|
if (observability.metricsEndpoint === undefined) return "node-observability-metrics-endpoint-not-declared";
|
|
if (observability.workbench === undefined) return "node-observability-workbench-not-declared";
|
|
if (!observability.workbench.enabled) return "node-observability-workbench-disabled";
|
|
return null;
|
|
}
|
|
|
|
function nodeObservabilityEndpointSummary(observability: HwlabRuntimeObservabilitySpec): Record<string, unknown> | null {
|
|
const endpoint = observability.metricsEndpoint;
|
|
if (endpoint === undefined) return null;
|
|
return {
|
|
serviceName: endpoint.serviceName,
|
|
containerName: endpoint.containerName,
|
|
port: endpoint.port,
|
|
scheme: endpoint.scheme,
|
|
path: endpoint.path,
|
|
scrapeMode: endpoint.scrapeMode,
|
|
publicRawMetrics: endpoint.publicRawMetrics,
|
|
};
|
|
}
|
|
|
|
function nodeObservabilityServiceStatus(options: NodeObservabilityOptions): Record<string, unknown> {
|
|
const endpoint = options.spec.observability.metricsEndpoint;
|
|
if (endpoint === undefined) {
|
|
return {
|
|
ready: false,
|
|
namespace: options.spec.runtimeNamespace,
|
|
serviceName: null,
|
|
podName: null,
|
|
containerName: null,
|
|
degradedReason: "node-observability-metrics-endpoint-not-declared",
|
|
};
|
|
}
|
|
const namespace = runNodeK3sArgs(options.spec, ["kubectl", "get", "ns", options.spec.runtimeNamespace, "-o", "name"], 60);
|
|
const namespaceExists = namespace.exitCode === 0;
|
|
const service = namespaceExists
|
|
? runNodeK3sArgs(options.spec, ["kubectl", "-n", options.spec.runtimeNamespace, "get", "svc", endpoint.serviceName, "-o", "json"], 60)
|
|
: null;
|
|
const serviceJson = service !== null && service.exitCode === 0 ? parseJsonRecordFromText(service.stdout) : {};
|
|
const selector = k8sServiceSelector(serviceJson);
|
|
const selectorText = labelSelectorText(selector);
|
|
const pods = namespaceExists && service !== null && service.exitCode === 0 && selectorText.length > 0
|
|
? runNodeK3sArgs(options.spec, ["kubectl", "-n", options.spec.runtimeNamespace, "get", "pods", "-l", selectorText, "-o", "json"], 60)
|
|
: null;
|
|
const pod = pods !== null && pods.exitCode === 0 ? selectK8sPod(parseJsonRecordFromText(pods.stdout), endpoint.containerName) : null;
|
|
const ready = namespaceExists && service !== null && service.exitCode === 0 && pod?.ready === true;
|
|
return {
|
|
ready,
|
|
namespace: options.spec.runtimeNamespace,
|
|
namespaceExists,
|
|
serviceName: endpoint.serviceName,
|
|
serviceExists: service !== null && service.exitCode === 0,
|
|
selector,
|
|
selectorText: selectorText.length > 0 ? selectorText : null,
|
|
podName: pod?.name ?? null,
|
|
podPhase: pod?.phase ?? null,
|
|
podReady: pod?.ready ?? false,
|
|
containerName: endpoint.containerName,
|
|
containerReady: pod?.containerReady ?? false,
|
|
probes: {
|
|
namespace: compactRuntimeCommand(namespace),
|
|
service: service === null ? null : compactRuntimeCommand(service),
|
|
pods: pods === null ? null : compactRuntimeCommand(pods),
|
|
},
|
|
degradedReason: ready ? undefined : namespaceExists ? "node-observability-pod-or-service-not-ready" : "node-observability-namespace-missing",
|
|
};
|
|
}
|
|
|
|
interface NodeObservabilityMetricsProbeResult {
|
|
readonly ok: boolean;
|
|
readonly httpStatus: number | null;
|
|
readonly bodyBytes: number;
|
|
readonly summary: Record<string, unknown>;
|
|
readonly result: CommandResult;
|
|
readonly error: string | null;
|
|
}
|
|
|
|
function nodeObservabilityMetricsProbe(options: NodeObservabilityOptions, podName: string): NodeObservabilityMetricsProbeResult {
|
|
const endpoint = options.spec.observability.metricsEndpoint;
|
|
if (endpoint === undefined) throw new Error("metricsEndpoint is required before probing metrics");
|
|
const workbench = options.spec.observability.workbench;
|
|
const summaryConfig = Buffer.from(JSON.stringify({
|
|
metricPrefixes: workbench?.metricPrefixes ?? [],
|
|
requiredSeries: workbench?.requiredSeries ?? [],
|
|
backendLabelDenylist: workbench?.backendLabelDenylist ?? [],
|
|
maxDeniedBackendEventLines: workbench?.maxUnknownEventLines ?? 0,
|
|
lowSampleThreshold: workbench?.lowSampleThreshold ?? 0,
|
|
histogramMetrics: [
|
|
{ id: "workbench_journey", metric: "hwlab_workbench_journey_duration_seconds", kind: "workbench_journey", dimensionLabels: ["namespace", "gitops_target", "journey", "route", "backend", "transport", "target_state", "cache", "source", "entry", "outcome"] },
|
|
{ id: "workbench_event_phase", metric: "hwlab_workbench_event_phase_duration_seconds", kind: "workbench_event_phase", dimensionLabels: ["namespace", "gitops_target", "phase", "event_type", "backend", "transport", "outcome"] },
|
|
{ id: "workbench_backend_event_visible", metric: "hwlab_workbench_backend_event_visible_latency_seconds", kind: "workbench_backend_event_visible", dimensionLabels: ["namespace", "gitops_target", "event_type", "backend", "transport", "outcome"] },
|
|
{ id: "workbench_projection_lag_events", metric: "hwlab_workbench_projection_lag_events", kind: "workbench_projection_lag_events", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "projection_status", "source", "status", "reason"] },
|
|
{ id: "workbench_projection_lag_seconds", metric: "hwlab_workbench_projection_lag_seconds", kind: "workbench_projection_lag_seconds", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "projection_status", "source", "status", "reason"] },
|
|
{ id: "workbench_terminal_projection_delay", metric: "hwlab_workbench_terminal_projection_delay_seconds", kind: "workbench_terminal_projection_delay", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "projection_status", "source", "status", "reason"] },
|
|
{ id: "workbench_turn_get", metric: "hwlab_workbench_turn_get_duration_seconds", kind: "workbench_turn_get", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "route", "status", "degraded_reason"] },
|
|
{ id: "agentrun_result_duration", metric: "hwlab_agentrun_result_duration_seconds", kind: "agentrun_result_duration", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "event_count_bucket", "status"] },
|
|
{ id: "agentrun_result_pages", metric: "hwlab_agentrun_result_pages_scanned", kind: "agentrun_result_pages", dimensionLabels: ["namespace", "gitops_target", "node", "lane"] },
|
|
{ id: "agentrun_result_events", metric: "hwlab_agentrun_result_events_scanned", kind: "agentrun_result_events", dimensionLabels: ["namespace", "gitops_target", "node", "lane"] },
|
|
{ id: "workbench_projector_batch", metric: "hwlab_workbench_projector_batch_duration_seconds", kind: "workbench_projector_batch", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "phase", "status"] },
|
|
],
|
|
}), "utf8").toString("base64");
|
|
const source = [
|
|
"const http = require('node:http');",
|
|
"const port = Number(process.argv[1]);",
|
|
"const path = process.argv[2];",
|
|
"const config = JSON.parse(Buffer.from(process.argv[3], 'base64').toString('utf8'));",
|
|
"function metricNameFromLine(line) { const match = /^([A-Za-z_:][A-Za-z0-9_:]*)/.exec(line); return match ? match[1] : null; }",
|
|
"function lineMatchesMetric(line, name) { if (!line.startsWith(name)) return false; const next = line.charAt(name.length); return next === '' || next === '{' || /\\s/.test(next); }",
|
|
"function lineHasLabel(line, name, value) { return line.includes(`${name}=\\\"${String(value).replace(/\\\\/g, '\\\\\\\\').replace(/\\\"/g, '\\\\\\\"')}\\\"`); }",
|
|
"function isBackendEventMetric(name) { return name.startsWith('hwlab_workbench_event_') || name.startsWith('hwlab_workbench_backend_event_'); }",
|
|
"function compactLines(lines) { return lines.slice(0, 16).map((line) => line.length > 320 ? `${line.slice(0, 317)}...` : line); }",
|
|
"function parseLabels(raw) { const labels = {}; for (const part of String(raw || '').split(',')) { const index = part.indexOf('='); if (index <= 0) continue; const key = part.slice(0, index); let value = part.slice(index + 1); if (value.startsWith('\\\"') && value.endsWith('\\\"')) value = value.slice(1, -1); labels[key] = value.replace(/\\\\\\\"/g, '\\\"').replace(/\\\\\\\\/g, '\\\\'); } return labels; }",
|
|
"function labelKey(metric, labels) { const copy = { ...labels }; delete copy.le; return `${metric}|${Object.keys(copy).sort().map((key) => `${key}=${copy[key]}`).join('|')}`; }",
|
|
"function pickLabels(labels, names) { return Object.fromEntries(names.map((name) => [name, labels[name] || 'unknown'])); }",
|
|
"function quantile(row, q) { const count = Number(row.count || 0); if (count <= 0) return 0; const rank = Math.max(1, Math.ceil(count * q)); const buckets = row.buckets.filter((item) => item.le !== '+Inf').sort((left, right) => Number(left.le) - Number(right.le)); for (const bucket of buckets) { if (bucket.value >= rank) return Number(bucket.le); } return buckets.length ? Number(buckets[buckets.length - 1].le) : 0; }",
|
|
"function summarizeHistograms(sampleLines) {",
|
|
" const metricConfig = new Map(config.histogramMetrics.map((item) => [item.metric, item]));",
|
|
" const series = new Map();",
|
|
" for (const line of sampleLines) {",
|
|
" const match = /^([A-Za-z_:][A-Za-z0-9_:]*?)_(bucket|sum|count)\\{([^}]*)\\}\\s+([0-9.eE+-]+)/.exec(line);",
|
|
" if (!match || !metricConfig.has(match[1])) continue;",
|
|
" const metric = match[1];",
|
|
" const suffix = match[2];",
|
|
" const labels = parseLabels(match[3]);",
|
|
" const value = Number(match[4]);",
|
|
" if (!Number.isFinite(value)) continue;",
|
|
" const key = labelKey(metric, labels);",
|
|
" const row = series.get(key) || { metric, labels: { ...labels }, buckets: [], sum: 0, count: 0 };",
|
|
" if (suffix === 'bucket') row.buckets.push({ le: labels.le || '+Inf', value });",
|
|
" if (suffix === 'sum') row.sum = value;",
|
|
" if (suffix === 'count') row.count = value;",
|
|
" series.set(key, row);",
|
|
" }",
|
|
" const lowSampleThreshold = Number(config.lowSampleThreshold || 0);",
|
|
" const groups = Object.fromEntries(config.histogramMetrics.map((item) => [item.id, []]));",
|
|
" const rows = [];",
|
|
" for (const row of series.values()) {",
|
|
" const cfg = metricConfig.get(row.metric);",
|
|
" const sampleCount = Number(row.count || 0);",
|
|
" const output = { kind: cfg.kind, metric: row.metric, sampleCount, average: sampleCount > 0 ? Number((row.sum / sampleCount).toFixed(4)) : 0, p50: quantile(row, 0.5), p75: quantile(row, 0.75), p95: quantile(row, 0.95), lowSample: sampleCount > 0 && sampleCount < lowSampleThreshold, sampleState: sampleCount <= 0 ? 'empty' : sampleCount < lowSampleThreshold ? 'low-sample' : 'ok', dimensions: pickLabels(row.labels, cfg.dimensionLabels) };",
|
|
" groups[cfg.id].push(output);",
|
|
" rows.push(output);",
|
|
" }",
|
|
" for (const key of Object.keys(groups)) groups[key].sort((left, right) => right.p95 - left.p95 || right.sampleCount - left.sampleCount);",
|
|
" rows.sort((left, right) => right.p95 - left.p95 || right.sampleCount - left.sampleCount);",
|
|
" return { groups: Object.fromEntries(Object.entries(groups).map(([key, value]) => [key, value.slice(0, 24)])), topSlowDimensions: rows.slice(0, 12) };",
|
|
"}",
|
|
"function summarize(text, statusCode) {",
|
|
" const lines = text.split(/\\r?\\n/).map((line) => line.trim()).filter(Boolean);",
|
|
" const sampleLines = lines.filter((line) => !line.startsWith('#'));",
|
|
" const seriesByPrefix = Object.fromEntries(config.metricPrefixes.map((prefix) => [prefix, sampleLines.filter((line) => (metricNameFromLine(line) || '').startsWith(prefix)).length]));",
|
|
" const requiredSeries = config.requiredSeries.map((name) => { const matching = sampleLines.filter((line) => lineMatchesMetric(line, name)); return { name, present: matching.length > 0, sampleCount: matching.length }; });",
|
|
" const missingSeries = requiredSeries.filter((series) => !series.present).map((series) => series.name);",
|
|
" const deniedBackendEventLines = sampleLines.filter((line) => { const name = metricNameFromLine(line) || ''; if (!isBackendEventMetric(name)) return false; return config.backendLabelDenylist.some((label) => lineHasLabel(line, 'backend', label)); });",
|
|
" const histogramSummary = summarizeHistograms(sampleLines);",
|
|
" const ready = statusCode >= 200 && statusCode < 300 && missingSeries.length === 0 && deniedBackendEventLines.length <= config.maxDeniedBackendEventLines;",
|
|
" return {",
|
|
" ready,",
|
|
" httpStatus: statusCode,",
|
|
" bodyBytes: Buffer.byteLength(text),",
|
|
" lineCount: lines.length,",
|
|
" sampleLineCount: sampleLines.length,",
|
|
" metricPrefixes: config.metricPrefixes,",
|
|
" seriesByPrefix,",
|
|
" requiredSeries,",
|
|
" missingSeries,",
|
|
" backendLabelDenylist: config.backendLabelDenylist,",
|
|
" lowSampleThreshold: config.lowSampleThreshold,",
|
|
" workbenchHistograms: histogramSummary.groups,",
|
|
" topSlowDimensions: histogramSummary.topSlowDimensions,",
|
|
" deniedBackendEventLineCount: deniedBackendEventLines.length,",
|
|
" maxDeniedBackendEventLines: config.maxDeniedBackendEventLines,",
|
|
" deniedBackendEventLines: compactLines(deniedBackendEventLines),",
|
|
" backendVisibleCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_backend_event_visible_latency_seconds_count'))),",
|
|
" phaseCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_event_phase_duration_seconds_count'))),",
|
|
" projectionLagCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_projection_lag_events_count') || lineMatchesMetric(line, 'hwlab_workbench_projection_lag_seconds_count'))),",
|
|
" turnGetCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_turn_get_duration_seconds_count'))),",
|
|
" agentRunResultCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_agentrun_result_duration_seconds_count'))),",
|
|
" projectorBatchCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_projector_batch_duration_seconds_count'))),",
|
|
" projectorLastSuccessLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_projector_last_success_unixtime'))),",
|
|
" degradedReason: ready ? undefined : missingSeries.length > 0 ? 'node-observability-required-series-missing' : 'node-observability-denied-backend-label-present',",
|
|
" };",
|
|
"}",
|
|
"let settled = false;",
|
|
"const finish = (payload, code = 0) => { if (settled) return; settled = true; console.log(JSON.stringify(payload)); process.exitCode = code; };",
|
|
"const req = http.get({ host: '127.0.0.1', port, path, timeout: 8000 }, (res) => {",
|
|
" let body = '';",
|
|
" res.setEncoding('utf8');",
|
|
" res.on('data', (chunk) => { body += chunk; if (body.length > 2000000) req.destroy(new Error('metrics body exceeded 2MB')); });",
|
|
" res.on('end', () => finish({ ok: res.statusCode >= 200 && res.statusCode < 300, statusCode: res.statusCode, bodyBytes: Buffer.byteLength(body), summary: summarize(body, res.statusCode || 0) }));",
|
|
"});",
|
|
"req.on('timeout', () => req.destroy(new Error('timeout')));",
|
|
"req.on('error', (error) => finish({ ok: false, statusCode: null, bodyBytes: 0, summary: null, error: String(error && error.message || error) }, 1));",
|
|
].join("\n");
|
|
const result = runNodeK3sArgs(options.spec, [
|
|
"kubectl",
|
|
"-n", options.spec.runtimeNamespace,
|
|
"exec", podName,
|
|
"-c", endpoint.containerName,
|
|
"--",
|
|
"node", "-e", source,
|
|
String(endpoint.port),
|
|
endpoint.path,
|
|
summaryConfig,
|
|
], options.timeoutSeconds);
|
|
const payload = parseJsonRecordFromText(result.stdout);
|
|
const httpStatus = typeof payload.statusCode === "number" ? payload.statusCode : null;
|
|
const error = typeof payload.error === "string" ? payload.error : null;
|
|
const summary = record(payload.summary);
|
|
return {
|
|
ok: result.exitCode === 0 && payload.ok === true && httpStatus !== null && httpStatus >= 200 && httpStatus < 300,
|
|
httpStatus,
|
|
bodyBytes: typeof payload.bodyBytes === "number" ? payload.bodyBytes : 0,
|
|
summary,
|
|
result,
|
|
error,
|
|
};
|
|
}
|
|
|
|
function nodeObservabilityPublicRawMetricsProbe(options: NodeObservabilityOptions): Record<string, unknown> {
|
|
const endpoint = options.spec.observability.metricsEndpoint;
|
|
if (endpoint === undefined) {
|
|
return {
|
|
ok: false,
|
|
expected: null,
|
|
degradedReason: "node-observability-metrics-endpoint-not-declared",
|
|
};
|
|
}
|
|
const url = joinUrlPath(options.spec.publicApiUrl, endpoint.path);
|
|
const result = runCommand(["curl", "-k", "-sS", "--connect-timeout", "5", "--max-time", "12", "-o", "-", "-w", "\nUNIDESK_HTTP_STATUS:%{http_code}", url], repoRoot, { timeoutMs: 15_000 });
|
|
const parsed = splitCurlBodyStatus(result.stdout);
|
|
const exposesPrometheus = looksLikePrometheusMetrics(parsed.body, options.spec.observability.workbench?.metricPrefixes ?? []);
|
|
const denied = endpoint.publicRawMetrics === "denied" && (parsed.httpStatus === null || parsed.httpStatus >= 400 || !exposesPrometheus);
|
|
return {
|
|
ok: result.exitCode === 0 && denied,
|
|
expected: endpoint.publicRawMetrics,
|
|
url,
|
|
httpStatus: parsed.httpStatus,
|
|
exposesPrometheus,
|
|
bodyBytes: Buffer.byteLength(parsed.body),
|
|
result: compactRuntimeCommand(result),
|
|
degradedReason: denied ? undefined : "node-observability-public-raw-metrics-exposed",
|
|
};
|
|
}
|
|
|
|
function summarizeNodeObservabilityMetrics(observability: HwlabRuntimeObservabilitySpec, text: string, httpStatus: number | null): Record<string, unknown> {
|
|
const workbench = observability.workbench;
|
|
if (workbench === undefined) {
|
|
return {
|
|
ready: false,
|
|
httpStatus,
|
|
degradedReason: "node-observability-workbench-not-declared",
|
|
};
|
|
}
|
|
const lines = text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
|
const sampleLines = lines.filter((line) => !line.startsWith("#"));
|
|
const seriesByPrefix = Object.fromEntries(workbench.metricPrefixes.map((prefix) => [
|
|
prefix,
|
|
sampleLines.filter((line) => (metricNameFromPrometheusLine(line) ?? "").startsWith(prefix)).length,
|
|
]));
|
|
const requiredSeries = workbench.requiredSeries.map((name) => {
|
|
const matching = sampleLines.filter((line) => prometheusLineMatchesMetric(line, name));
|
|
return {
|
|
name,
|
|
present: matching.length > 0,
|
|
sampleCount: matching.length,
|
|
};
|
|
});
|
|
const missingSeries = requiredSeries.filter((series) => !series.present).map((series) => series.name);
|
|
const deniedBackendEventLines = sampleLines.filter((line) => {
|
|
const name = metricNameFromPrometheusLine(line) ?? "";
|
|
if (!isWorkbenchBackendEventMetric(name)) return false;
|
|
return workbench.backendLabelDenylist.some((label) => prometheusLineHasLabel(line, "backend", label));
|
|
});
|
|
const ready = httpStatus !== null
|
|
&& httpStatus >= 200
|
|
&& httpStatus < 300
|
|
&& missingSeries.length === 0
|
|
&& deniedBackendEventLines.length <= workbench.maxUnknownEventLines;
|
|
return {
|
|
ready,
|
|
httpStatus,
|
|
bodyBytes: Buffer.byteLength(text),
|
|
lineCount: lines.length,
|
|
sampleLineCount: sampleLines.length,
|
|
metricPrefixes: workbench.metricPrefixes,
|
|
seriesByPrefix,
|
|
requiredSeries,
|
|
missingSeries,
|
|
backendLabelDenylist: workbench.backendLabelDenylist,
|
|
deniedBackendEventLineCount: deniedBackendEventLines.length,
|
|
maxDeniedBackendEventLines: workbench.maxUnknownEventLines,
|
|
deniedBackendEventLines: compactPrometheusLines(deniedBackendEventLines),
|
|
backendVisibleCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_backend_event_visible_latency_seconds_count"))),
|
|
phaseCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_event_phase_duration_seconds_count"))),
|
|
projectionLagCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_projection_lag_events_count") || prometheusLineMatchesMetric(line, "hwlab_workbench_projection_lag_seconds_count"))),
|
|
turnGetCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_turn_get_duration_seconds_count"))),
|
|
agentRunResultCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_agentrun_result_duration_seconds_count"))),
|
|
projectorBatchCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_projector_batch_duration_seconds_count"))),
|
|
projectorLastSuccessLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_projector_last_success_unixtime"))),
|
|
degradedReason: ready ? undefined : missingSeries.length > 0 ? "node-observability-required-series-missing" : "node-observability-denied-backend-label-present",
|
|
};
|
|
}
|
|
|
|
function splitCurlBodyStatus(stdout: string): { body: string; httpStatus: number | null } {
|
|
const marker = "\nUNIDESK_HTTP_STATUS:";
|
|
const index = stdout.lastIndexOf(marker);
|
|
if (index < 0) return { body: stdout, httpStatus: null };
|
|
const rawStatus = stdout.slice(index + marker.length).trim();
|
|
const status = nullableInteger(rawStatus);
|
|
return {
|
|
body: stdout.slice(0, index),
|
|
httpStatus: status,
|
|
};
|
|
}
|
|
|
|
function looksLikePrometheusMetrics(text: string, prefixes: readonly string[]): boolean {
|
|
const lines = text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
|
return lines.some((line) => {
|
|
if (line.startsWith("# HELP ") || line.startsWith("# TYPE ")) return true;
|
|
const name = metricNameFromPrometheusLine(line);
|
|
return name !== null && (prefixes.length === 0 || prefixes.some((prefix) => name.startsWith(prefix)));
|
|
});
|
|
}
|
|
|
|
function metricNameFromPrometheusLine(line: string): string | null {
|
|
const match = /^([A-Za-z_:][A-Za-z0-9_:]*)/u.exec(line);
|
|
return match?.[1] ?? null;
|
|
}
|
|
|
|
function prometheusLineMatchesMetric(line: string, name: string): boolean {
|
|
if (!line.startsWith(name)) return false;
|
|
const next = line.charAt(name.length);
|
|
return next === "" || next === "{" || /\s/u.test(next);
|
|
}
|
|
|
|
function prometheusLineHasLabel(line: string, name: string, value: string): boolean {
|
|
return line.includes(`${name}="${value.replace(/\\/gu, "\\\\").replace(/"/gu, '\\"')}"`);
|
|
}
|
|
|
|
function isWorkbenchBackendEventMetric(name: string): boolean {
|
|
return name.startsWith("hwlab_workbench_event_") || name.startsWith("hwlab_workbench_backend_event_");
|
|
}
|
|
|
|
function compactPrometheusLines(lines: readonly string[]): string[] {
|
|
return lines.slice(0, 16).map((line) => line.length > 320 ? `${line.slice(0, 317)}...` : line);
|
|
}
|
|
|
|
function parseJsonRecordFromText(text: string): Record<string, unknown> {
|
|
const trimmed = text.trim();
|
|
if (trimmed.length === 0) return {};
|
|
try {
|
|
const parsed = JSON.parse(trimmed) as unknown;
|
|
return record(parsed);
|
|
} catch {
|
|
const start = trimmed.indexOf("{");
|
|
const end = trimmed.lastIndexOf("}");
|
|
if (start >= 0 && end > start) {
|
|
try {
|
|
return record(JSON.parse(trimmed.slice(start, end + 1)) as unknown);
|
|
} catch {}
|
|
}
|
|
}
|
|
return {};
|
|
}
|
|
|
|
function k8sServiceSelector(serviceJson: Record<string, unknown>): Record<string, string> {
|
|
const spec = record(serviceJson.spec);
|
|
const selector = record(spec.selector);
|
|
return Object.fromEntries(Object.entries(selector).filter((entry): entry is [string, string] => typeof entry[1] === "string" && entry[1].length > 0));
|
|
}
|
|
|
|
function labelSelectorText(selector: Record<string, string>): string {
|
|
return Object.entries(selector).map(([key, value]) => `${key}=${value}`).join(",");
|
|
}
|
|
|
|
function selectK8sPod(podsJson: Record<string, unknown>, containerName: string): { name: string; phase: string | null; ready: boolean; containerReady: boolean } | null {
|
|
const items = Array.isArray(podsJson.items) ? podsJson.items.map(record) : [];
|
|
const pods = items.map((item) => {
|
|
const metadata = record(item.metadata);
|
|
const status = record(item.status);
|
|
const name = typeof metadata.name === "string" ? metadata.name : "";
|
|
const phase = typeof status.phase === "string" ? status.phase : null;
|
|
const containerStatuses = Array.isArray(status.containerStatuses) ? status.containerStatuses.map(record) : [];
|
|
const targetContainer = containerStatuses.find((container) => container.name === containerName);
|
|
const containerReady = targetContainer?.ready === true;
|
|
return {
|
|
name,
|
|
phase,
|
|
ready: name.length > 0 && phase === "Running" && containerReady,
|
|
containerReady,
|
|
};
|
|
}).filter((pod) => pod.name.length > 0);
|
|
return pods.find((pod) => pod.ready) ?? pods.find((pod) => pod.phase === "Running") ?? pods[0] ?? null;
|
|
}
|
|
|
|
function compactRuntimeCommand(result: CommandResult): Record<string, unknown> {
|
|
return {
|
|
exitCode: result.exitCode,
|
|
stdoutBytes: Buffer.byteLength(result.stdout),
|
|
stderrBytes: Buffer.byteLength(result.stderr),
|
|
stdoutTail: result.stdout.trim().slice(-2000),
|
|
stderrTail: result.stderr.trim().slice(-2000),
|
|
timedOut: result.timedOut,
|
|
};
|
|
}
|
|
|
|
function compactRuntimeCommandStats(result: CommandResult): Record<string, unknown> {
|
|
return {
|
|
exitCode: result.exitCode,
|
|
stdoutBytes: Buffer.byteLength(result.stdout),
|
|
stderrBytes: Buffer.byteLength(result.stderr),
|
|
timedOut: result.timedOut,
|
|
};
|
|
}
|
|
|
|
function parseLastJsonLineObject(text: string): Record<string, unknown> {
|
|
for (const line of text.split(/\r?\n/u).reverse()) {
|
|
const trimmed = line.trim();
|
|
if (!trimmed.startsWith("{") || !trimmed.endsWith("}")) continue;
|
|
try {
|
|
return record(JSON.parse(trimmed) as unknown);
|
|
} catch {
|
|
// Keep scanning: command tails can mix kubectl status, git output, and JSON payloads.
|
|
}
|
|
}
|
|
return {};
|
|
}
|
|
|
|
function escapeRegExp(value: string): string {
|
|
return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
|
}
|
|
|
|
function nodeRuntimeGitMirrorRefSources(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>, mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, unknown> {
|
|
return {
|
|
localSource: `refs/heads/${mirror.sourceBranch}`,
|
|
githubSource: `refs/mirror-stage/heads/${mirror.sourceBranch}`,
|
|
localGitops: `refs/heads/${mirror.gitopsBranch}`,
|
|
githubGitops: `refs/mirror-stage/heads/${mirror.gitopsBranch}`,
|
|
githubFieldsAreMirrorStageCache: true,
|
|
cacheRefresh: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`,
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeGitMirrorFlushPartialSuccess(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>, mirror: NodeRuntimeGitMirrorTargetSpec, result: CommandResult): Record<string, unknown> | null {
|
|
if (scoped.action !== "flush") return null;
|
|
const text = `${result.stdout}\n${result.stderr}`;
|
|
const payload = parseLastJsonLineObject(text);
|
|
const partialSuccess = typeof payload.partialSuccess === "string" && payload.partialSuccess.length > 0
|
|
? payload.partialSuccess
|
|
: null;
|
|
const payloadStatus = typeof payload.status === "string" && payload.status.length > 0 ? payload.status : null;
|
|
const branch = escapeRegExp(mirror.gitopsBranch);
|
|
const pushSucceededInGitOutput = new RegExp(`\\b${branch}\\s*->\\s*${branch}\\b`, "u").test(text);
|
|
const postPushFetchFailed = /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|fatal:.*fetch|fetch-pack|early EOF/iu.test(text);
|
|
if (partialSuccess !== "push-succeeded-fetch-failed" && !(result.exitCode !== 0 && pushSucceededInGitOutput && postPushFetchFailed)) {
|
|
return null;
|
|
}
|
|
return {
|
|
status: payloadStatus ?? "partial-success",
|
|
partialSuccess: "push-succeeded-fetch-failed",
|
|
degradedReason: "node-runtime-git-mirror-flush-post-push-fetch-failed",
|
|
retryable: true,
|
|
message: "GitOps push appears to have succeeded, but the post-push fetch/recheck failed. Refresh mirror-stage cache before deciding another flush is needed.",
|
|
payload: Object.keys(payload).length > 0 ? payload : null,
|
|
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
|
|
next: {
|
|
sync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`,
|
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
},
|
|
};
|
|
}
|
|
|
|
function sshTcpPoolDiagnosticsFromCommand(spec: HwlabRuntimeLaneSpec, result: CommandResult): Record<string, unknown> | null {
|
|
if (isCommandSuccess(result)) return null;
|
|
const failureKind = classifySshTcpPoolFailure(`${result.stderr}\n${result.stdout}`);
|
|
if (failureKind === null) return null;
|
|
return {
|
|
classification: "ssh-tcp-pool-transient",
|
|
failureKind,
|
|
providerId: spec.nodeId,
|
|
route: spec.nodeKubeRoute,
|
|
message: "SSH tcp data pool failed while running the controlled command; treat this as transport/data-pool transient until provider labels and retry say otherwise.",
|
|
next: {
|
|
poolStatus: `bun scripts/cli.ts debug ssh-pool ${spec.nodeId}`,
|
|
retrySmoke: `trans ${spec.nodeId} argv true`,
|
|
retryApply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
|
|
retryTriggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeUnsupportedAction(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mutation: false,
|
|
degradedReason: "unsupported-node-scoped-runtime-action",
|
|
message: "node-scoped runtime currently supports plan/status/apply/refresh/sync/trigger-current/cleanup-runs/runtime-image/runtime-migration",
|
|
expected: nodeRuntimeExpected(scoped.spec),
|
|
};
|
|
}
|
|
|
|
function transPath(): string {
|
|
return join(repoRoot, "scripts", "trans");
|
|
}
|
|
|
|
function runNodeHostScript(spec: HwlabRuntimeLaneSpec, script: string, timeoutSeconds: number, input = ""): CommandResult {
|
|
return runCommand([transPath(), spec.nodeRoute, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
|
|
}
|
|
|
|
function runNodeHostScriptAsync(spec: HwlabRuntimeLaneSpec, script: string, timeoutSeconds: number, label: string): CommandResult {
|
|
const token = nodeRuntimeRenderToken();
|
|
const stateDir = `/tmp/unidesk-hwlab-node-runtime/${label}-${token}`;
|
|
const statusPath = `${stateDir}/status.json`;
|
|
const stdoutPath = `${stateDir}/stdout.log`;
|
|
const stderrPath = `${stateDir}/stderr.log`;
|
|
const runnerPath = `${stateDir}/run.sh`;
|
|
const startScript = [
|
|
"set -eu",
|
|
`state_dir=${shellQuote(stateDir)}`,
|
|
`status_path=${shellQuote(statusPath)}`,
|
|
`stdout_path=${shellQuote(stdoutPath)}`,
|
|
`stderr_path=${shellQuote(stderrPath)}`,
|
|
`runner_path=${shellQuote(runnerPath)}`,
|
|
"rm -rf \"$state_dir\"",
|
|
"mkdir -p \"$state_dir\"",
|
|
"cat > \"$runner_path\" <<'UNIDESK_REMOTE_RUNNER'",
|
|
script,
|
|
"UNIDESK_REMOTE_RUNNER",
|
|
"chmod 700 \"$runner_path\"",
|
|
"python3 - \"$status_path\" <<'PY'",
|
|
"import datetime, json, sys",
|
|
"json.dump({'state':'running','ok':None,'pid':None,'startedAt':datetime.datetime.utcnow().isoformat()+'Z','finishedAt':None,'exitCode':None}, open(sys.argv[1], 'w'), indent=2)",
|
|
"PY",
|
|
"(",
|
|
" set +e",
|
|
" sh \"$runner_path\" >\"$stdout_path\" 2>\"$stderr_path\"",
|
|
" code=$?",
|
|
" python3 - \"$status_path\" \"$stdout_path\" \"$stderr_path\" \"$code\" <<'PY'",
|
|
"import datetime, json, pathlib, sys",
|
|
"status_path, stdout_path, stderr_path, code = sys.argv[1:5]",
|
|
"def tail(path):",
|
|
" try:",
|
|
" text = pathlib.Path(path).read_text(errors='replace')",
|
|
" except FileNotFoundError:",
|
|
" return ''",
|
|
" return text[-12000:]",
|
|
"payload = {'state':'finished','ok':code == '0','pid':None,'startedAt':None,'finishedAt':datetime.datetime.utcnow().isoformat()+'Z','exitCode':int(code),'stdout':tail(stdout_path),'stderr':tail(stderr_path)}",
|
|
"try:",
|
|
" current = json.load(open(status_path))",
|
|
" payload['startedAt'] = current.get('startedAt')",
|
|
"except Exception:",
|
|
" pass",
|
|
"json.dump(payload, open(status_path, 'w'), indent=2)",
|
|
"PY",
|
|
") >/dev/null 2>&1 &",
|
|
"pid=$!",
|
|
"python3 - \"$status_path\" \"$pid\" <<'PY'",
|
|
"import json, sys",
|
|
"path, pid = sys.argv[1:3]",
|
|
"payload = json.load(open(path))",
|
|
"payload['pid'] = int(pid)",
|
|
"json.dump(payload, open(path, 'w'), indent=2)",
|
|
"PY",
|
|
"printf '{\"ok\":true,\"state\":\"started\",\"pid\":%s,\"statusPath\":\"%s\"}\\n' \"$pid\" \"$status_path\"",
|
|
].join("\n");
|
|
const start = runNodeHostScript(spec, startScript, 55);
|
|
if (!isCommandSuccess(start)) return start;
|
|
const deadline = Date.now() + Math.max(1, timeoutSeconds) * 1000;
|
|
let lastStatus: CommandResult | null = null;
|
|
let payload: Record<string, unknown> = {};
|
|
while (Date.now() <= deadline) {
|
|
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
|
|
const status = runNodeHostScript(spec, `cat ${shellQuote(statusPath)} 2>/dev/null || printf '{"state":"missing","ok":false,"exitCode":127,"stdout":"","stderr":"missing async status"}\\n'`, 55);
|
|
lastStatus = status;
|
|
payload = parseJsonObject(status.stdout.trim());
|
|
if (payload.state === "finished" || payload.ok === true || payload.ok === false) {
|
|
return commandResultFromAsync(spec, payload, statusPath, false);
|
|
}
|
|
}
|
|
const kill = runNodeHostScript(spec, [
|
|
"set +e",
|
|
`status_path=${shellQuote(statusPath)}`,
|
|
"pid=$(python3 - \"$status_path\" <<'PY' 2>/dev/null",
|
|
"import json, sys",
|
|
"try: print(json.load(open(sys.argv[1])).get('pid') or '')",
|
|
"except Exception: print('')",
|
|
"PY",
|
|
")",
|
|
"if [ -n \"$pid\" ]; then kill \"$pid\" 2>/dev/null || true; fi",
|
|
"cat \"$status_path\" 2>/dev/null || true",
|
|
].join("\n"), 55);
|
|
return {
|
|
command: [transPath(), spec.nodeRoute, "sh", "--", "<remote async script>"],
|
|
cwd: repoRoot,
|
|
exitCode: 124,
|
|
stdout: typeof payload.stdout === "string" ? payload.stdout : "",
|
|
stderr: [
|
|
`remote async command timed out after ${timeoutSeconds}s; statusPath=${statusPath}`,
|
|
typeof payload.stderr === "string" ? payload.stderr : "",
|
|
lastStatus?.stderr.trim() ?? "",
|
|
kill.stderr.trim(),
|
|
].filter(Boolean).join("\n"),
|
|
signal: null,
|
|
timedOut: true,
|
|
};
|
|
}
|
|
|
|
function parseJsonObject(text: string): Record<string, unknown> {
|
|
try {
|
|
const parsed = JSON.parse(text) as unknown;
|
|
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record<string, unknown> : {};
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
|
|
function commandResultFromAsync(spec: HwlabRuntimeLaneSpec, payload: Record<string, unknown>, statusPath: string, timedOut: boolean): CommandResult {
|
|
return {
|
|
command: [transPath(), spec.nodeRoute, "sh", "--", "<remote async script>"],
|
|
cwd: repoRoot,
|
|
exitCode: typeof payload.exitCode === "number" ? payload.exitCode : payload.ok === true ? 0 : 1,
|
|
stdout: typeof payload.stdout === "string" ? payload.stdout : "",
|
|
stderr: typeof payload.stderr === "string" ? payload.stderr : `remote async statusPath=${statusPath}`,
|
|
signal: null,
|
|
timedOut,
|
|
};
|
|
}
|
|
|
|
function runNodeK3sArgs(spec: HwlabRuntimeLaneSpec, args: string[], timeoutSeconds: number): CommandResult {
|
|
return runCommand([transPath(), spec.nodeKubeRoute, ...args], repoRoot, { timeoutMs: timeoutSeconds * 1000 });
|
|
}
|
|
|
|
function runNodeK3sScript(spec: HwlabRuntimeLaneSpec, script: string, timeoutSeconds: number, input = ""): CommandResult {
|
|
return runCommand([transPath(), spec.nodeKubeRoute, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
|
|
}
|
|
|
|
function isCommandSuccess(result: CommandResult): boolean {
|
|
return result.exitCode === 0 && !result.timedOut;
|
|
}
|
|
|
|
function shortSha(value: string): string {
|
|
return value.slice(0, 12).toLowerCase();
|
|
}
|
|
|
|
function shortValue(value: unknown): string {
|
|
if (typeof value !== "string" || value.length === 0) return "-";
|
|
if (/^[0-9a-f]{40}$/iu.test(value)) return shortSha(value);
|
|
return value.length > 30 ? `${value.slice(0, 27)}~` : value;
|
|
}
|
|
|
|
function formatElapsedMs(value: unknown): string {
|
|
const ms = Number(value);
|
|
if (!Number.isFinite(ms) || ms < 0) return "-";
|
|
const seconds = Math.round(ms / 1000);
|
|
const minutes = Math.floor(seconds / 60);
|
|
const rest = seconds % 60;
|
|
return minutes > 0 ? `${minutes}m${String(rest).padStart(2, "0")}s` : `${seconds}s`;
|
|
}
|
|
|
|
function nodeRuntimePipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string {
|
|
return `${spec.pipelineRunPrefix}-${shortSha(sourceCommit)}`;
|
|
}
|
|
|
|
function nodeRuntimeRerunPipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string {
|
|
return `${nodeRuntimePipelineRunName(spec, sourceCommit)}-r${Date.now().toString(36)}`.slice(0, 63);
|
|
}
|
|
|
|
function nodeRuntimeGitopsRoot(spec: HwlabRuntimeLaneSpec): string {
|
|
const suffix = `/${spec.runtimeRenderDir}`;
|
|
if (spec.runtimePath.endsWith(suffix)) return spec.runtimePath.slice(0, -suffix.length);
|
|
return spec.gitopsRoot;
|
|
}
|
|
|
|
function resolveNodeRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandResult } {
|
|
const result = runCommand(["git", "ls-remote", spec.gitUrl, `refs/heads/${spec.sourceBranch}`], repoRoot, { timeoutMs: 45_000 });
|
|
if (!isCommandSuccess(result)) return { sourceCommit: null, result };
|
|
const match = /[0-9a-f]{40}/iu.exec(statusText(result));
|
|
return { sourceCommit: match?.[0].toLowerCase() ?? null, result };
|
|
}
|
|
|
|
function runtimeLaneCicdRepoEnsureScript(spec: HwlabRuntimeLaneSpec): string {
|
|
const gitRetries = Math.max(1, spec.downloadProfile.git.retries);
|
|
const gitTimeoutSeconds = Math.max(30, spec.downloadProfile.git.timeoutSeconds);
|
|
return [
|
|
`cicd_repo=${shellQuote(spec.cicdRepo)}`,
|
|
`cicd_url=${shellQuote(spec.gitUrl)}`,
|
|
`cicd_branch=${shellQuote(spec.sourceBranch)}`,
|
|
`cicd_repo_lock=${shellQuote(spec.cicdRepoLock)}`,
|
|
`git_retries=${shellQuote(String(gitRetries))}`,
|
|
`git_timeout=${shellQuote(String(gitTimeoutSeconds))}`,
|
|
"run_git() { if command -v timeout >/dev/null 2>&1; then timeout \"$git_timeout\" git -c protocol.version=2 \"$@\"; else git -c protocol.version=2 \"$@\"; fi; }",
|
|
"retry_git() {",
|
|
" op=$1",
|
|
" shift",
|
|
" attempt=1",
|
|
" while [ \"$attempt\" -le \"$git_retries\" ]; do",
|
|
" echo \"phase=git-$op attempt=$attempt timeoutSeconds=$git_timeout\" >&2",
|
|
" run_git \"$@\"",
|
|
" code=$?",
|
|
" if [ \"$code\" -eq 0 ]; then return 0; fi",
|
|
" echo \"phase=git-$op attempt=$attempt exitCode=$code\" >&2",
|
|
" attempt=$((attempt + 1))",
|
|
" done",
|
|
" return 1",
|
|
"}",
|
|
"mkdir -p \"$(dirname \"$cicd_repo\")\"",
|
|
"if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then",
|
|
" :",
|
|
"elif [ -e \"$cicd_repo\" ]; then",
|
|
" echo \"CI/CD repo path exists but is not a bare git repo: $cicd_repo\" >&2",
|
|
" exit 41",
|
|
"else",
|
|
" retry_git clone-ci-cache clone --bare --filter=blob:none --single-branch --branch \"$cicd_branch\" \"$cicd_url\" \"$cicd_repo\"",
|
|
"fi",
|
|
"git --git-dir=\"$cicd_repo\" remote set-url origin \"$cicd_url\" 2>/dev/null || git --git-dir=\"$cicd_repo\" remote add origin \"$cicd_url\"",
|
|
"git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'",
|
|
"if ! retry_git fetch-ci-cache --git-dir=\"$cicd_repo\" fetch --filter=blob:none --depth=1 origin \"+refs/heads/$cicd_branch:refs/remotes/origin/$cicd_branch\" --prune; then",
|
|
" rm -rf \"$cicd_repo\"",
|
|
" retry_git clone-ci-cache-retry clone --bare --filter=blob:none --single-branch --branch \"$cicd_branch\" \"$cicd_url\" \"$cicd_repo\"",
|
|
" git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'",
|
|
" retry_git fetch-ci-cache-retry --git-dir=\"$cicd_repo\" fetch --filter=blob:none --depth=1 origin \"+refs/heads/$cicd_branch:refs/remotes/origin/$cicd_branch\" --prune",
|
|
"fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function nodeRuntimeControlPlaneRun(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> | RenderedCliResult {
|
|
if (scoped.action === "refresh") return nodeRuntimeRefresh(scoped);
|
|
if (scoped.action === "sync") return nodeRuntimeSync(scoped);
|
|
if (scoped.action === "apply") return nodeRuntimeApply(scoped);
|
|
if (scoped.action === "trigger-current") return nodeRuntimeTriggerCurrentOutput(scoped);
|
|
if (scoped.action === "runtime-migration") return nodeRuntimeMigration(scoped);
|
|
if (scoped.action === "cleanup-runs") return nodeRuntimeCleanupRuns(scoped);
|
|
return nodeRuntimeUnsupportedAction(scoped);
|
|
}
|
|
|
|
function parseNodeRuntimeCleanupOptions(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): NodeRuntimeCleanupOptions {
|
|
const sourceCommitRaw = optionValue(scoped.originalArgs, "--source-commit");
|
|
const pipelineRunRaw = optionValue(scoped.originalArgs, "--pipeline-run");
|
|
if (sourceCommitRaw !== undefined && pipelineRunRaw !== undefined) {
|
|
throw new Error("control-plane cleanup-runs accepts only one of --source-commit or --pipeline-run");
|
|
}
|
|
const sourceCommit = sourceCommitRaw?.toLowerCase();
|
|
if (sourceCommit !== undefined && !/^[0-9a-f]{40}$/u.test(sourceCommit)) {
|
|
throw new Error("--source-commit must be a full 40-character git sha for cleanup-runs");
|
|
}
|
|
const pipelineRun = pipelineRunRaw === undefined ? undefined : validateNodeRuntimePipelineRunName(scoped.spec, pipelineRunRaw);
|
|
return {
|
|
minAgeMinutes: positiveIntegerOption(scoped.originalArgs, "--min-age-minutes", 60, 10080),
|
|
limit: positiveIntegerOption(scoped.originalArgs, "--limit", 20, 200),
|
|
sourceCommit,
|
|
pipelineRun,
|
|
targetPipelineRun: pipelineRun ?? (sourceCommit === undefined ? undefined : nodeRuntimePipelineRunName(scoped.spec, sourceCommit)),
|
|
includeActive: scoped.originalArgs.includes("--include-active"),
|
|
dryRun: scoped.dryRun || !scoped.confirm,
|
|
};
|
|
}
|
|
|
|
function validateNodeRuntimePipelineRunName(spec: HwlabRuntimeLaneSpec, value: string): string {
|
|
const escapedPrefix = spec.pipelineRunPrefix.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
|
if (!new RegExp(`^${escapedPrefix}-[0-9a-f]{7,40}(?:-[a-z0-9][a-z0-9-]{0,24})?$`, "iu").test(value)) {
|
|
throw new Error(`--pipeline-run must be a ${spec.pipelineRunPrefix}-<sha>[-rerun] PipelineRun name for --lane ${spec.lane}`);
|
|
}
|
|
return value.toLowerCase();
|
|
}
|
|
|
|
function nodeRuntimeCleanupRuns(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const options = parseNodeRuntimeCleanupOptions(scoped);
|
|
const beforeCounts = nodeRuntimeCiObjectCounts(scoped.spec);
|
|
const candidates = listNodeRuntimeCleanupPipelineRuns(scoped.spec, options);
|
|
const selectedPipelineRuns = candidates
|
|
.filter((item) => item.selected !== false)
|
|
.map((item) => item.name);
|
|
const ownedResources = listNodeRuntimeCleanupOwnedResources(scoped.spec, selectedPipelineRuns);
|
|
const command = `hwlab nodes control-plane cleanup-runs --node ${scoped.node} --lane ${scoped.lane}`;
|
|
if (options.dryRun) {
|
|
return {
|
|
ok: true,
|
|
command,
|
|
mode: "dry-run",
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
namespace: HWLAB_CI_NAMESPACE,
|
|
minAgeMinutes: options.minAgeMinutes,
|
|
limit: options.limit,
|
|
sourceCommit: options.sourceCommit,
|
|
pipelineRun: options.pipelineRun,
|
|
includeActive: options.includeActive,
|
|
candidates,
|
|
candidateCount: candidates.length,
|
|
selectedPipelineRuns,
|
|
selectedPipelineRunCount: selectedPipelineRuns.length,
|
|
ownedResources,
|
|
ciObjectCounts: beforeCounts,
|
|
mutation: false,
|
|
next: {
|
|
confirm: [
|
|
command,
|
|
`--min-age-minutes ${options.minAgeMinutes}`,
|
|
`--limit ${options.limit}`,
|
|
options.pipelineRun === undefined ? "" : `--pipeline-run ${options.pipelineRun}`,
|
|
options.sourceCommit === undefined ? "" : `--source-commit ${options.sourceCommit}`,
|
|
options.includeActive ? "--include-active" : "",
|
|
"--confirm",
|
|
"--wait",
|
|
].filter(Boolean).join(" "),
|
|
},
|
|
};
|
|
}
|
|
const deletion = deleteNodeRuntimeCleanupRuns(scoped.spec, selectedPipelineRuns, scoped.timeoutSeconds);
|
|
const afterCounts = nodeRuntimeCiObjectCounts(scoped.spec);
|
|
return {
|
|
ok: isCommandSuccess(deletion),
|
|
command,
|
|
mode: "confirmed-cleanup",
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
namespace: HWLAB_CI_NAMESPACE,
|
|
minAgeMinutes: options.minAgeMinutes,
|
|
limit: options.limit,
|
|
sourceCommit: options.sourceCommit,
|
|
pipelineRun: options.pipelineRun,
|
|
includeActive: options.includeActive,
|
|
deletedPipelineRuns: selectedPipelineRuns,
|
|
deletedPipelineRunCount: selectedPipelineRuns.length,
|
|
ownedResourcesBefore: ownedResources,
|
|
ciObjectCountsBefore: beforeCounts,
|
|
ciObjectCountsAfter: afterCounts,
|
|
deletion: compactRuntimeCommand(deletion),
|
|
mutation: isCommandSuccess(deletion),
|
|
degradedReason: isCommandSuccess(deletion) ? undefined : "node-runtime-ci-cleanup-delete-failed",
|
|
next: {
|
|
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
rerunStatus: options.targetPipelineRun === undefined
|
|
? undefined
|
|
: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${options.targetPipelineRun}`,
|
|
},
|
|
};
|
|
}
|
|
|
|
function listNodeRuntimeCleanupPipelineRuns(spec: HwlabRuntimeLaneSpec, options: NodeRuntimeCleanupOptions): NodeRuntimeCleanupPipelineRunRow[] {
|
|
if (options.targetPipelineRun !== undefined) {
|
|
const target = getNodeRuntimeCleanupPipelineRun(spec, options.targetPipelineRun);
|
|
if (target === null) {
|
|
return [{
|
|
name: options.targetPipelineRun,
|
|
createdAt: null,
|
|
ageMinutes: null,
|
|
status: null,
|
|
reason: "target-pipelinerun-not-found",
|
|
selected: false,
|
|
}];
|
|
}
|
|
if (target.status !== "True" && target.status !== "False" && !options.includeActive) {
|
|
return [{ ...target, selected: false, selectedReason: "target-pipelinerun-not-terminal" }];
|
|
}
|
|
if ((target.status === "True" || target.status === "False") && (target.ageMinutes === null || target.ageMinutes < options.minAgeMinutes)) {
|
|
return [{ ...target, selected: false, selectedReason: target.ageMinutes === null ? "missing-creation-timestamp" : "below-min-age" }];
|
|
}
|
|
return [target];
|
|
}
|
|
const result = runNodeK3sArgs(spec, [
|
|
"kubectl",
|
|
"-n",
|
|
HWLAB_CI_NAMESPACE,
|
|
"get",
|
|
"pipelinerun",
|
|
"-o",
|
|
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\n"}{end}',
|
|
], 60);
|
|
if (!isCommandSuccess(result)) throw new Error(`failed to list ${HWLAB_CI_NAMESPACE} PipelineRuns: ${result.stderr.trim().slice(0, 1000)}`);
|
|
const rows = nodeRuntimeCleanupPipelineRunRowsFromText(result.stdout);
|
|
const prefix = `${spec.pipelineRunPrefix}-`;
|
|
const terminalRuns = rows
|
|
.filter((item) => item.name.startsWith(prefix))
|
|
.filter((item) => item.status === "True" || item.status === "False")
|
|
.sort((left, right) => String(left.createdAt ?? "").localeCompare(String(right.createdAt ?? "")));
|
|
const protectedLatest = terminalRuns
|
|
.slice()
|
|
.sort((left, right) => String(right.createdAt ?? "").localeCompare(String(left.createdAt ?? "")))[0]?.name ?? null;
|
|
return terminalRuns
|
|
.filter((item) => typeof item.ageMinutes === "number" && item.ageMinutes >= options.minAgeMinutes)
|
|
.map((item) => item.name === protectedLatest ? { ...item, selected: false, selectedReason: "protected-latest-pipelinerun" } : item)
|
|
.slice(0, options.limit);
|
|
}
|
|
|
|
function getNodeRuntimeCleanupPipelineRun(spec: HwlabRuntimeLaneSpec, pipelineRun: string): NodeRuntimeCleanupPipelineRunRow | null {
|
|
const result = runNodeK3sArgs(spec, [
|
|
"kubectl",
|
|
"-n",
|
|
HWLAB_CI_NAMESPACE,
|
|
"get",
|
|
"pipelinerun",
|
|
pipelineRun,
|
|
"-o",
|
|
'jsonpath={.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\n"}',
|
|
], 60);
|
|
if (result.exitCode !== 0) return null;
|
|
return nodeRuntimeCleanupPipelineRunRowsFromText(result.stdout)[0] ?? null;
|
|
}
|
|
|
|
function nodeRuntimeCleanupPipelineRunRowsFromText(text: string): NodeRuntimeCleanupPipelineRunRow[] {
|
|
const now = Date.now();
|
|
return text.split(/\r?\n/u).map((line) => {
|
|
const [name = "", createdAtRaw = "", statusRaw = "", reasonRaw = ""] = line.trim().split("\t");
|
|
const createdAt = createdAtRaw.length > 0 ? createdAtRaw : null;
|
|
const createdMs = createdAt === null ? NaN : Date.parse(createdAt);
|
|
return {
|
|
name,
|
|
createdAt,
|
|
ageMinutes: Number.isFinite(createdMs) ? Math.floor((now - createdMs) / 60000) : null,
|
|
status: statusRaw || null,
|
|
reason: reasonRaw || null,
|
|
};
|
|
}).filter((item) => item.name.length > 0);
|
|
}
|
|
|
|
function listNodeRuntimeCleanupOwnedResources(spec: HwlabRuntimeLaneSpec, pipelineRunNames: string[]): Record<string, unknown> {
|
|
const previewLimit = 24;
|
|
if (pipelineRunNames.length === 0) {
|
|
return {
|
|
taskRunPreview: [],
|
|
podPreview: [],
|
|
pvcPreview: [],
|
|
previewLimit,
|
|
truncated: false,
|
|
taskRunCount: 0,
|
|
podCount: 0,
|
|
pvcCount: 0,
|
|
};
|
|
}
|
|
const wanted = new Set(pipelineRunNames);
|
|
const taskRunsResult = runNodeK3sArgs(spec, [
|
|
"kubectl",
|
|
"-n",
|
|
HWLAB_CI_NAMESPACE,
|
|
"get",
|
|
"taskrun",
|
|
"-o",
|
|
'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/pipelineRun"}}{{"\\t"}}{{range .status.conditions}}{{if eq .type "Succeeded"}}{{.status}}{{"\\t"}}{{.reason}}{{end}}{{end}}{{"\\n"}}{{end}}',
|
|
], 60);
|
|
const podsResult = runNodeK3sArgs(spec, [
|
|
"kubectl",
|
|
"-n",
|
|
HWLAB_CI_NAMESPACE,
|
|
"get",
|
|
"pod",
|
|
"-o",
|
|
'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/pipelineRun"}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{.spec.nodeName}}{{"\\n"}}{{end}}',
|
|
], 60);
|
|
const pvcsResult = runNodeK3sArgs(spec, [
|
|
"kubectl",
|
|
"-n",
|
|
HWLAB_CI_NAMESPACE,
|
|
"get",
|
|
"pvc",
|
|
"-o",
|
|
'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{.spec.volumeName}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{range .metadata.ownerReferences}}{{if eq .kind "PipelineRun"}}{{.kind}}{{"\\t"}}{{.name}}{{end}}{{end}}{{"\\t"}}{{.spec.resources.requests.storage}}{{"\\n"}}{{end}}',
|
|
], 60);
|
|
const taskRuns = isCommandSuccess(taskRunsResult) ? nodeRuntimeCleanupOwnedTaskRunsFromText(taskRunsResult.stdout, wanted) : [];
|
|
const pods = isCommandSuccess(podsResult) ? nodeRuntimeCleanupOwnedPodsFromText(podsResult.stdout, wanted) : [];
|
|
const pvcs = isCommandSuccess(pvcsResult) ? nodeRuntimeCleanupOwnedPvcsFromText(pvcsResult.stdout, wanted) : [];
|
|
return {
|
|
taskRunPreview: taskRuns.slice(0, previewLimit),
|
|
podPreview: pods.slice(0, previewLimit),
|
|
pvcPreview: pvcs.slice(0, previewLimit),
|
|
previewLimit,
|
|
truncated: taskRuns.length > previewLimit || pods.length > previewLimit || pvcs.length > previewLimit,
|
|
taskRunCount: taskRuns.length,
|
|
podCount: pods.length,
|
|
pvcCount: pvcs.length,
|
|
query: {
|
|
taskRuns: compactRuntimeCommandStats(taskRunsResult),
|
|
pods: compactRuntimeCommandStats(podsResult),
|
|
pvcs: compactRuntimeCommandStats(pvcsResult),
|
|
},
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeCleanupOwnedTaskRunsFromText(text: string, wanted: Set<string>): Record<string, unknown>[] {
|
|
return text.split(/\r?\n/u).map((line) => {
|
|
const [name = "", pipelineRun = "", status = "", reason = ""] = line.trim().split("\t");
|
|
if (name.length === 0 || !wanted.has(pipelineRun)) return null;
|
|
return {
|
|
name,
|
|
pipelineRun,
|
|
status: status || null,
|
|
reason: reason || null,
|
|
};
|
|
}).filter((item): item is Record<string, unknown> => item !== null);
|
|
}
|
|
|
|
function nodeRuntimeCleanupOwnedPodsFromText(text: string, wanted: Set<string>): Record<string, unknown>[] {
|
|
return text.split(/\r?\n/u).map((line) => {
|
|
const [name = "", pipelineRun = "", phase = "", nodeName = ""] = line.trim().split("\t");
|
|
if (name.length === 0 || !wanted.has(pipelineRun)) return null;
|
|
return {
|
|
name,
|
|
pipelineRun,
|
|
phase: phase || null,
|
|
nodeName: nodeName || null,
|
|
};
|
|
}).filter((item): item is Record<string, unknown> => item !== null);
|
|
}
|
|
|
|
function nodeRuntimeCleanupOwnedPvcsFromText(text: string, wanted: Set<string>): Record<string, unknown>[] {
|
|
return text.split(/\r?\n/u).map((line) => {
|
|
const [name = "", volumeName = "", phase = "", ownerKind = "", ownerName = "", storage = ""] = line.trim().split("\t");
|
|
if (name.length === 0 || ownerKind !== "PipelineRun" || !wanted.has(ownerName)) return null;
|
|
return {
|
|
name,
|
|
pipelineRun: ownerName,
|
|
phase: phase || null,
|
|
volumeName: volumeName || null,
|
|
storage: storage || null,
|
|
};
|
|
}).filter((item): item is Record<string, unknown> => item !== null);
|
|
}
|
|
|
|
function deleteNodeRuntimeCleanupRuns(spec: HwlabRuntimeLaneSpec, pipelineRunNames: string[], timeoutSeconds: number): CommandResult {
|
|
if (pipelineRunNames.length === 0) {
|
|
return {
|
|
command: [],
|
|
cwd: repoRoot,
|
|
exitCode: 0,
|
|
stdout: "no candidates",
|
|
stderr: "",
|
|
signal: null,
|
|
timedOut: false,
|
|
};
|
|
}
|
|
const script = [
|
|
"set -eu",
|
|
`namespace=${shellQuote(HWLAB_CI_NAMESPACE)}`,
|
|
"names_file=$(mktemp)",
|
|
"cat > \"$names_file\"",
|
|
"deleted_pipeline_runs=$(grep -c . \"$names_file\" | tr -d ' ')",
|
|
"xargs -r -n 50 kubectl -n \"$namespace\" delete pipelinerun --ignore-not-found=true --wait=false < \"$names_file\"",
|
|
"deleted_pod_groups=0",
|
|
"deleted_taskrun_groups=0",
|
|
"explicit_owned_cleanup=skipped-large-batch",
|
|
"if [ \"$deleted_pipeline_runs\" -le 40 ]; then",
|
|
" explicit_owned_cleanup=executed",
|
|
" while IFS= read -r pipeline_run; do",
|
|
" [ -n \"$pipeline_run\" ] || continue",
|
|
" kubectl -n \"$namespace\" delete pod -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
" deleted_pod_groups=$((deleted_pod_groups + 1))",
|
|
" kubectl -n \"$namespace\" delete taskrun -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
" deleted_taskrun_groups=$((deleted_taskrun_groups + 1))",
|
|
" done < \"$names_file\"",
|
|
"fi",
|
|
"printf 'deletedPipelineRunCount\\t%s\\n' \"$deleted_pipeline_runs\"",
|
|
"printf 'deletedTaskRunLabelGroups\\t%s\\n' \"$deleted_taskrun_groups\"",
|
|
"printf 'deletedPodLabelGroups\\t%s\\n' \"$deleted_pod_groups\"",
|
|
"printf 'explicitOwnedCleanup\\t%s\\n' \"$explicit_owned_cleanup\"",
|
|
"rm -f \"$names_file\"",
|
|
].join("\n");
|
|
return runNodeK3sScript(spec, script, timeoutSeconds, pipelineRunNames.join("\n") + "\n");
|
|
}
|
|
|
|
function nodeRuntimeCiObjectCounts(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
|
const script = [
|
|
"set +e",
|
|
`namespace=${shellQuote(HWLAB_CI_NAMESPACE)}`,
|
|
"count_kind() { kubectl -n \"$namespace\" get \"$1\" -o name 2>/dev/null | wc -l | tr -d ' '; }",
|
|
"printf 'pipelineRuns\\t%s\\n' \"$(count_kind pipelinerun)\"",
|
|
"printf 'taskRuns\\t%s\\n' \"$(count_kind taskrun)\"",
|
|
"printf 'pods\\t%s\\n' \"$(count_kind pod)\"",
|
|
"printf 'pvcs\\t%s\\n' \"$(count_kind pvc)\"",
|
|
].join("\n");
|
|
const result = runNodeK3sScript(spec, script, 30);
|
|
const fields = keyValueLinesFromText(statusText(result));
|
|
return {
|
|
pipelineRuns: numericField(fields.pipelineRuns),
|
|
taskRuns: numericField(fields.taskRuns),
|
|
pods: numericField(fields.pods),
|
|
pvcs: numericField(fields.pvcs),
|
|
result: compactRuntimeCommand(result),
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeBaseImageCommand(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const action = scoped.runtimeImageAction;
|
|
if (action === null) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane runtime-image --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mutation: false,
|
|
degradedReason: "node-runtime-image-action-missing",
|
|
message: "runtime-image requires one of: status, preload, build",
|
|
expected: nodeRuntimeExpected(scoped.spec),
|
|
};
|
|
}
|
|
if (action !== "status" && action !== "preload" && action !== "build") {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane runtime-image ${action} --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mutation: false,
|
|
degradedReason: "unsupported-node-runtime-image-action",
|
|
message: "runtime-image currently supports status/preload/build",
|
|
expected: nodeRuntimeExpected(scoped.spec),
|
|
};
|
|
}
|
|
const statusBefore = nodeRuntimeBaseImageStatus(scoped.spec, scoped.timeoutSeconds);
|
|
if (action === "status") {
|
|
return {
|
|
ok: statusBefore.ok,
|
|
command: `hwlab nodes control-plane runtime-image status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "status",
|
|
mutation: false,
|
|
status: statusBefore,
|
|
degradedReason: statusBefore.ok ? undefined : "node-runtime-base-image-not-ready",
|
|
next: statusBefore.ok ? undefined : {
|
|
preload: `bun scripts/cli.ts hwlab nodes control-plane runtime-image preload --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
if (!scoped.confirm && !scoped.dryRun) throw new Error("control-plane runtime-image preload/build requires --dry-run or --confirm");
|
|
const preload = ensureNodeBaseImage(scoped.spec, scoped.dryRun, scoped.timeoutSeconds);
|
|
const statusAfter = nodeRuntimeBaseImageStatus(scoped.spec, scoped.timeoutSeconds);
|
|
return {
|
|
ok: preload !== null && preload.ok === true && (scoped.dryRun || statusAfter.ok === true),
|
|
command: `hwlab nodes control-plane runtime-image ${action} --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: scoped.dryRun ? "dry-run" : "confirmed-preload",
|
|
requestedAction: action,
|
|
effectiveAction: "preload",
|
|
mutation: !scoped.dryRun && preload !== null && preload.ok === true,
|
|
statusBefore,
|
|
preload,
|
|
statusAfter,
|
|
degradedReason: preload === null
|
|
? "node-runtime-base-image-source-missing"
|
|
: preload.ok === true && (scoped.dryRun || statusAfter.ok === true)
|
|
? undefined
|
|
: "node-runtime-base-image-seed-failed",
|
|
next: scoped.dryRun
|
|
? { preload: `bun scripts/cli.ts hwlab nodes control-plane runtime-image preload --node ${scoped.node} --lane ${scoped.lane} --confirm` }
|
|
: { triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm --rerun` },
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeMigration(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const spec = scoped.spec;
|
|
if (scoped.allowLiveDbRead && scoped.confirm) throw new Error("control-plane runtime-migration accepts --allow-live-db-read only with dry-run/source-check mode, not --confirm");
|
|
if (!scoped.confirm && !scoped.dryRun) throw new Error("control-plane runtime-migration requires --dry-run or --confirm");
|
|
const head = resolveNodeRuntimeLaneHead(spec);
|
|
const sourceCommit = head.sourceCommit;
|
|
if (sourceCommit === null) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
phase: "source-head",
|
|
mutation: false,
|
|
degradedReason: "node-runtime-source-head-unresolved",
|
|
headProbe: compactRuntimeCommand(head.result),
|
|
};
|
|
}
|
|
const reportPath = `/tmp/hwlab-${scoped.node.toLowerCase()}-${scoped.lane}-runtime-migration-${shortSha(sourceCommit)}.json`;
|
|
const migrationArgs = scoped.dryRun
|
|
? [
|
|
...(scoped.allowLiveDbRead ? ["--dry-run", "--allow-live-db-read", "--confirm-dev"] : ["--check"]),
|
|
"--report",
|
|
reportPath,
|
|
]
|
|
: [
|
|
"--apply",
|
|
"--confirm-dev",
|
|
"--confirmed-non-production",
|
|
"--report",
|
|
reportPath,
|
|
];
|
|
const result = runNodeK3sArgs(spec, [
|
|
"kubectl",
|
|
"exec",
|
|
"-n",
|
|
spec.runtimeNamespace,
|
|
"deployment/hwlab-cloud-api",
|
|
"-c",
|
|
"hwlab-cloud-api",
|
|
"--",
|
|
"sh",
|
|
"-lc",
|
|
`cd /workspace/hwlab-boot/repo && exec bun cmd/hwlab-cloud-api/migrate.ts ${migrationArgs.map(shellQuote).join(" ")}`,
|
|
], scoped.timeoutSeconds);
|
|
const ok = isCommandSuccess(result);
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: scoped.dryRun ? scoped.allowLiveDbRead ? "live-read-dry-run" : "source-check" : "confirmed-apply",
|
|
sourceCommit,
|
|
runtimeNamespace: spec.runtimeNamespace,
|
|
target: "deployment/hwlab-cloud-api -c hwlab-cloud-api",
|
|
workingDirectory: "/workspace/hwlab-boot/repo",
|
|
migrationCommand: ["bun", "cmd/hwlab-cloud-api/migrate.ts", ...migrationArgs],
|
|
reportPath,
|
|
mutation: !scoped.dryRun && ok,
|
|
result: compactRuntimeCommand(result),
|
|
degradedReason: ok ? undefined : "node-runtime-migration-failed",
|
|
next: scoped.dryRun
|
|
? {
|
|
liveReadDryRun: `bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane} --allow-live-db-read --dry-run`,
|
|
apply: `bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
}
|
|
: {
|
|
health: `curl -fsS --max-time 20 ${spec.publicApiUrl}/health/live`,
|
|
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
},
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeApply(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const spec = scoped.spec;
|
|
const head = resolveNodeRuntimeLaneHead(spec);
|
|
const sourceCommit = head.sourceCommit;
|
|
if (sourceCommit === null) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
phase: "source-head",
|
|
degradedReason: "node-runtime-source-head-unresolved",
|
|
headProbe: compactRuntimeCommand(head.result),
|
|
};
|
|
}
|
|
const localPostgres = syncNodeLocalPostgresBootstrapSecret(spec, scoped.dryRun, scoped.timeoutSeconds);
|
|
if (localPostgres !== null && localPostgres.ok !== true) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
|
|
phase: "local-postgres-secret-sync",
|
|
sourceCommit,
|
|
localPostgres,
|
|
degradedReason: "local-postgres-secret-sync-failed",
|
|
};
|
|
}
|
|
const secrets = syncNodeExternalPostgresSecrets(spec, scoped.dryRun, scoped.timeoutSeconds);
|
|
if (secrets !== null && secrets.ok !== true) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
|
|
phase: "external-postgres-secret-sync",
|
|
sourceCommit,
|
|
localPostgres,
|
|
secrets,
|
|
degradedReason: "external-postgres-secret-sync-failed",
|
|
};
|
|
}
|
|
const baseImage = ensureNodeBaseImage(spec, scoped.dryRun, scoped.timeoutSeconds);
|
|
if (baseImage !== null && baseImage.ok !== true) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
|
|
phase: "base-image-seed",
|
|
sourceCommit,
|
|
localPostgres,
|
|
secrets,
|
|
baseImage,
|
|
degradedReason: "node-runtime-base-image-seed-failed",
|
|
};
|
|
}
|
|
const render = renderNodeRuntimeControlPlane(spec, sourceCommit, scoped.timeoutSeconds);
|
|
if (!isCommandSuccess(render.result)) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
|
|
phase: "source-render",
|
|
sourceCommit,
|
|
localPostgres,
|
|
secrets,
|
|
baseImage,
|
|
renderDir: render.renderDir,
|
|
renderLocation: render.location,
|
|
render: compactRuntimeCommand(render.result),
|
|
degradedReason: "node-runtime-control-plane-render-failed",
|
|
};
|
|
}
|
|
const apply = render.location === "local"
|
|
? applyLocalNodeRuntimeControlPlaneFiles(spec, render.renderDir, scoped.dryRun, scoped.timeoutSeconds)
|
|
: applyNodeRuntimeControlPlaneFiles(spec, render.renderDir, scoped.dryRun, scoped.timeoutSeconds);
|
|
const cleanup = render.location === "local"
|
|
? cleanupLocalNodeRuntimeRenderDir(spec, render)
|
|
: cleanupNodeRuntimeRenderDir(spec, render.renderDir);
|
|
const sshTcpPoolDiagnostics = sshTcpPoolDiagnosticsFromCommand(spec, apply);
|
|
return {
|
|
ok: isCommandSuccess(apply),
|
|
command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
|
|
mutation: !scoped.dryRun && isCommandSuccess(apply),
|
|
sourceCommit,
|
|
expected: nodeRuntimeExpected(spec),
|
|
localPostgres,
|
|
secrets,
|
|
baseImage,
|
|
renderDir: render.renderDir,
|
|
renderLocation: render.location,
|
|
render: compactRuntimeCommand(render.result),
|
|
apply: compactRuntimeCommand(apply),
|
|
cleanupRenderDir: compactRuntimeCommand(cleanup),
|
|
diagnostics: sshTcpPoolDiagnostics,
|
|
degradedReason: isCommandSuccess(apply) ? undefined : "node-runtime-control-plane-apply-failed",
|
|
next: scoped.dryRun
|
|
? { apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm` }
|
|
: {
|
|
triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
...(sshTcpPoolDiagnostics === null ? {} : { sshPoolStatus: `bun scripts/cli.ts debug ssh-pool ${scoped.node}` }),
|
|
},
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeRefresh(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const spec = scoped.spec;
|
|
const script = [
|
|
"set +e",
|
|
`app=${shellQuote(spec.app)}`,
|
|
"argo_namespace=argocd",
|
|
`dry_run=${shellQuote(scoped.dryRun ? "true" : "false")}`,
|
|
"before=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}' 2>/dev/null || true)",
|
|
"if [ \"$dry_run\" = true ]; then",
|
|
" output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite --dry-run=server -o name 2>&1)",
|
|
"else",
|
|
" output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite -o name 2>&1)",
|
|
"fi",
|
|
"code=$?",
|
|
"after=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}' 2>/dev/null || true)",
|
|
"printf 'app\\t%s\\n' \"$app\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'before\\t%s\\n' \"$before\"",
|
|
"printf 'after\\t%s\\n' \"$after\"",
|
|
"printf 'annotateExitCode\\t%s\\n' \"$code\"",
|
|
"printf 'annotateOutput\\t%s\\n' \"$(printf '%s' \"$output\" | tr '\\n\\t' ' ' | cut -c1-500)\"",
|
|
"exit \"$code\"",
|
|
].join("\n");
|
|
const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
|
|
const fields = keyValueLinesFromText(statusText(result));
|
|
return {
|
|
ok: isCommandSuccess(result),
|
|
command: `hwlab nodes control-plane refresh --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: scoped.dryRun ? "dry-run" : "confirmed-refresh",
|
|
mutation: !scoped.dryRun && isCommandSuccess(result),
|
|
argoApplication: fields.app || spec.app,
|
|
before: fields.before || null,
|
|
after: fields.after || null,
|
|
annotateExitCode: numericField(fields.annotateExitCode),
|
|
result: compactRuntimeCommand(result),
|
|
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}` },
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeSync(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const spec = scoped.spec;
|
|
const localPostgres = syncNodeLocalPostgresBootstrapSecret(spec, scoped.dryRun, scoped.timeoutSeconds);
|
|
if (localPostgres !== null && localPostgres.ok !== true) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane sync --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: scoped.dryRun ? "dry-run" : "confirmed-sync",
|
|
phase: "local-postgres-secret-sync",
|
|
localPostgres,
|
|
degradedReason: "local-postgres-secret-sync-failed",
|
|
next: { apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm` },
|
|
};
|
|
}
|
|
const operation = {
|
|
operation: {
|
|
initiatedBy: { username: "unidesk-hwlab-node-cli" },
|
|
retry: { limit: 1 },
|
|
sync: {
|
|
prune: true,
|
|
revision: spec.gitopsBranch,
|
|
source: {
|
|
repoURL: spec.argoRepoUrl,
|
|
targetRevision: spec.gitopsBranch,
|
|
path: spec.runtimePath,
|
|
},
|
|
syncOptions: ["CreateNamespace=true"],
|
|
syncStrategy: { hook: {} },
|
|
},
|
|
},
|
|
};
|
|
const patchB64 = Buffer.from(JSON.stringify(operation), "utf8").toString("base64");
|
|
const script = [
|
|
"set +e",
|
|
`app=${shellQuote(spec.app)}`,
|
|
"argo_namespace=argocd",
|
|
`runtime_namespace=${shellQuote(spec.runtimeNamespace)}`,
|
|
`dry_run=${shellQuote(scoped.dryRun ? "true" : "false")}`,
|
|
`patch_b64=${shellQuote(patchB64)}`,
|
|
"terminate_patch_file=$(mktemp /tmp/hwlab-node-argocd-terminate.XXXXXX.json)",
|
|
"patch_file=$(mktemp /tmp/hwlab-node-argocd-sync.XXXXXX.json)",
|
|
"printf '{\"operation\":null}' > \"$terminate_patch_file\"",
|
|
"printf '%s' \"$patch_b64\" | base64 -d > \"$patch_file\"",
|
|
"before=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}{\"\\t\"}{.status.operationState.phase}{\"\\t\"}{.status.operationState.message}' 2>/dev/null || true)",
|
|
"operation_phase=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.operationState.phase}' 2>/dev/null || true)",
|
|
"terminate_code=0",
|
|
"terminate_output=",
|
|
"terminated_operation=false",
|
|
"if [ \"$operation_phase\" = Running ]; then",
|
|
" if [ \"$dry_run\" = true ]; then",
|
|
" terminated_operation=would-terminate",
|
|
" else",
|
|
" terminate_output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$terminate_patch_file\" -o name 2>&1)",
|
|
" terminate_code=$?",
|
|
" if [ \"$terminate_code\" -eq 0 ]; then terminated_operation=true; else terminated_operation=failed; fi",
|
|
" sleep 2",
|
|
" fi",
|
|
"fi",
|
|
"failed_hook_count=0",
|
|
"deleted_hook_count=0",
|
|
"failed_hooks=",
|
|
"deleted_hooks=",
|
|
"hook_delete_errors=",
|
|
"for job in $(kubectl -n \"$runtime_namespace\" get job -o name 2>/dev/null || true); do",
|
|
" tracking=$(kubectl -n \"$runtime_namespace\" get \"$job\" -o go-template='{{ index .metadata.annotations \"argocd.argoproj.io/tracking-id\" }}' 2>/dev/null || true)",
|
|
" failed=$(kubectl -n \"$runtime_namespace\" get \"$job\" -o go-template='{{ range .status.conditions }}{{ if or (eq .type \"Failed\") (eq .type \"FailureTarget\") }}{{ .status }} {{ end }}{{ end }}' 2>/dev/null || true)",
|
|
" case \"$tracking\" in \"$app:\"*) ;; *) continue ;; esac",
|
|
" case \"$failed\" in *True*) ;; *) continue ;; esac",
|
|
" hook_name=${job#job.batch/}",
|
|
" failed_hook_count=$((failed_hook_count + 1))",
|
|
" if [ -z \"$failed_hooks\" ]; then failed_hooks=$hook_name; else failed_hooks=$failed_hooks,$hook_name; fi",
|
|
" if [ \"$dry_run\" = false ]; then",
|
|
" delete_output=$(kubectl -n \"$runtime_namespace\" delete \"$job\" --wait=false 2>&1)",
|
|
" delete_code=$?",
|
|
" if [ \"$delete_code\" -eq 0 ]; then",
|
|
" deleted_hook_count=$((deleted_hook_count + 1))",
|
|
" if [ -z \"$deleted_hooks\" ]; then deleted_hooks=$hook_name; else deleted_hooks=$deleted_hooks,$hook_name; fi",
|
|
" else",
|
|
" if [ -z \"$hook_delete_errors\" ]; then hook_delete_errors=\"$hook_name:$delete_code\"; else hook_delete_errors=\"$hook_delete_errors,$hook_name:$delete_code\"; fi",
|
|
" fi",
|
|
" fi",
|
|
"done",
|
|
"stale_statefulset_pod_count=0",
|
|
"deleted_stale_statefulset_pod_count=0",
|
|
"stale_statefulset_pods=",
|
|
"deleted_stale_statefulset_pods=",
|
|
"stale_statefulset_pod_delete_errors=",
|
|
"for sts in $(kubectl -n \"$runtime_namespace\" get statefulset -o name 2>/dev/null || true); do",
|
|
" sts_name=${sts#statefulset.apps/}",
|
|
" current_revision=$(kubectl -n \"$runtime_namespace\" get \"$sts\" -o 'jsonpath={.status.currentRevision}' 2>/dev/null || true)",
|
|
" update_revision=$(kubectl -n \"$runtime_namespace\" get \"$sts\" -o 'jsonpath={.status.updateRevision}' 2>/dev/null || true)",
|
|
" if [ -z \"$current_revision\" ] || [ -z \"$update_revision\" ] || [ \"$current_revision\" = \"$update_revision\" ]; then continue; fi",
|
|
" for pod in $(kubectl -n \"$runtime_namespace\" get pod -l \"controller-revision-hash=$current_revision\" -o name 2>/dev/null || true); do",
|
|
" pod_name=${pod#pod/}",
|
|
" owner=$(kubectl -n \"$runtime_namespace\" get \"$pod\" -o 'jsonpath={.metadata.ownerReferences[0].kind}/{.metadata.ownerReferences[0].name}' 2>/dev/null || true)",
|
|
" if [ \"$owner\" != \"StatefulSet/$sts_name\" ]; then continue; fi",
|
|
" ready=$(kubectl -n \"$runtime_namespace\" get \"$pod\" -o 'jsonpath={.status.conditions[?(@.type==\"Ready\")].status}' 2>/dev/null || true)",
|
|
" waiting=$(kubectl -n \"$runtime_namespace\" get \"$pod\" -o 'jsonpath={range .status.containerStatuses[*]}{.state.waiting.reason}{\" \"}{end}' 2>/dev/null || true)",
|
|
" case \"$waiting\" in *ImagePullBackOff*|*ErrImagePull*|*CrashLoopBackOff*) ;; *) continue ;; esac",
|
|
" if [ \"$ready\" = True ]; then continue; fi",
|
|
" stale_statefulset_pod_count=$((stale_statefulset_pod_count + 1))",
|
|
" stale_item=\"$sts_name/$pod_name:$waiting\"",
|
|
" if [ -z \"$stale_statefulset_pods\" ]; then stale_statefulset_pods=$stale_item; else stale_statefulset_pods=$stale_statefulset_pods,$stale_item; fi",
|
|
" if [ \"$dry_run\" = false ]; then",
|
|
" delete_output=$(kubectl -n \"$runtime_namespace\" delete \"$pod\" --wait=false 2>&1)",
|
|
" delete_code=$?",
|
|
" if [ \"$delete_code\" -eq 0 ]; then",
|
|
" deleted_stale_statefulset_pod_count=$((deleted_stale_statefulset_pod_count + 1))",
|
|
" if [ -z \"$deleted_stale_statefulset_pods\" ]; then deleted_stale_statefulset_pods=$pod_name; else deleted_stale_statefulset_pods=$deleted_stale_statefulset_pods,$pod_name; fi",
|
|
" else",
|
|
" if [ -z \"$stale_statefulset_pod_delete_errors\" ]; then stale_statefulset_pod_delete_errors=\"$pod_name:$delete_code\"; else stale_statefulset_pod_delete_errors=\"$stale_statefulset_pod_delete_errors,$pod_name:$delete_code\"; fi",
|
|
" fi",
|
|
" fi",
|
|
" done",
|
|
"done",
|
|
"if [ \"$dry_run\" = true ]; then",
|
|
" output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$patch_file\" --dry-run=server -o name 2>&1)",
|
|
"else",
|
|
" output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$patch_file\" -o name 2>&1)",
|
|
"fi",
|
|
"code=$?",
|
|
"after=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}{\"\\t\"}{.operation.sync.revision}{\"\\t\"}{.status.operationState.phase}{\"\\t\"}{.status.operationState.message}' 2>/dev/null || true)",
|
|
"printf 'app\\t%s\\n' \"$app\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'before\\t%s\\n' \"$before\"",
|
|
"printf 'after\\t%s\\n' \"$after\"",
|
|
"printf 'operationPhase\\t%s\\n' \"$operation_phase\"",
|
|
"printf 'terminatedOperation\\t%s\\n' \"$terminated_operation\"",
|
|
"printf 'terminateExitCode\\t%s\\n' \"$terminate_code\"",
|
|
"printf 'terminateOutput\\t%s\\n' \"$(printf '%s' \"$terminate_output\" | tr '\\n\\t' ' ' | cut -c1-500)\"",
|
|
"printf 'failedHookCount\\t%s\\n' \"$failed_hook_count\"",
|
|
"printf 'failedHooks\\t%s\\n' \"$failed_hooks\"",
|
|
"printf 'deletedHookCount\\t%s\\n' \"$deleted_hook_count\"",
|
|
"printf 'deletedHooks\\t%s\\n' \"$deleted_hooks\"",
|
|
"printf 'hookDeleteErrors\\t%s\\n' \"$hook_delete_errors\"",
|
|
"printf 'staleStatefulSetPodCount\\t%s\\n' \"$stale_statefulset_pod_count\"",
|
|
"printf 'staleStatefulSetPods\\t%s\\n' \"$stale_statefulset_pods\"",
|
|
"printf 'deletedStaleStatefulSetPodCount\\t%s\\n' \"$deleted_stale_statefulset_pod_count\"",
|
|
"printf 'deletedStaleStatefulSetPods\\t%s\\n' \"$deleted_stale_statefulset_pods\"",
|
|
"printf 'staleStatefulSetPodDeleteErrors\\t%s\\n' \"$stale_statefulset_pod_delete_errors\"",
|
|
"printf 'patchExitCode\\t%s\\n' \"$code\"",
|
|
"printf 'patchOutput\\t%s\\n' \"$(printf '%s' \"$output\" | tr '\\n\\t' ' ' | cut -c1-500)\"",
|
|
"rm -f \"$patch_file\" \"$terminate_patch_file\"",
|
|
"exit \"$code\"",
|
|
].join("\n");
|
|
const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
|
|
const fields = keyValueLinesFromText(statusText(result));
|
|
return {
|
|
ok: isCommandSuccess(result),
|
|
command: `hwlab nodes control-plane sync --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: scoped.dryRun ? "dry-run" : "confirmed-sync",
|
|
mutation: !scoped.dryRun && (isCommandSuccess(result) || localPostgres?.mutation === true),
|
|
localPostgres,
|
|
argoApplication: fields.app || spec.app,
|
|
syncSource: {
|
|
repoURL: spec.argoRepoUrl,
|
|
targetRevision: spec.gitopsBranch,
|
|
path: spec.runtimePath,
|
|
},
|
|
before: fields.before || null,
|
|
after: fields.after || null,
|
|
operationRecovery: {
|
|
operationPhase: fields.operationPhase || null,
|
|
terminatedOperation: fields.terminatedOperation || "false",
|
|
terminateExitCode: numericField(fields.terminateExitCode),
|
|
terminateOutput: fields.terminateOutput || null,
|
|
failedHookCount: numericField(fields.failedHookCount),
|
|
failedHooks: commaListField(fields.failedHooks),
|
|
deletedHookCount: numericField(fields.deletedHookCount),
|
|
deletedHooks: commaListField(fields.deletedHooks),
|
|
hookDeleteErrors: fields.hookDeleteErrors || null,
|
|
staleStatefulSetPodCount: numericField(fields.staleStatefulSetPodCount),
|
|
staleStatefulSetPods: commaListField(fields.staleStatefulSetPods),
|
|
deletedStaleStatefulSetPodCount: numericField(fields.deletedStaleStatefulSetPodCount),
|
|
deletedStaleStatefulSetPods: commaListField(fields.deletedStaleStatefulSetPods),
|
|
staleStatefulSetPodDeleteErrors: fields.staleStatefulSetPodDeleteErrors || null,
|
|
},
|
|
patchExitCode: numericField(fields.patchExitCode),
|
|
patchOutput: fields.patchOutput || null,
|
|
result: compactRuntimeCommand(result),
|
|
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}` },
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeTriggerCurrentOutput(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> | RenderedCliResult {
|
|
const result = nodeRuntimeTriggerCurrent(scoped);
|
|
if (nodeScopedFullOutput(scoped)) return result;
|
|
return withNodeRuntimeTriggerRendered(result, scoped);
|
|
}
|
|
|
|
function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const spec = scoped.spec;
|
|
const pipelineWaitSeconds = nodeRuntimeCicdWaitSeconds(scoped);
|
|
const triggerStartedAt = Date.now();
|
|
const triggerElapsedMs = () => Date.now() - triggerStartedAt;
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "started" });
|
|
const head = resolveNodeRuntimeLaneHead(spec);
|
|
const sourceCommit = head.sourceCommit;
|
|
if (sourceCommit === null) {
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "failed" });
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
phase: "source-head",
|
|
degradedReason: "node-runtime-source-head-unresolved",
|
|
headProbe: compactRuntimeCommand(head.result),
|
|
};
|
|
}
|
|
const basePipelineRun = nodeRuntimePipelineRunName(spec, sourceCommit);
|
|
const pipelineRun = scoped.rerun ? nodeRuntimeRerunPipelineRunName(spec, sourceCommit) : basePipelineRun;
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "succeeded", sourceCommit, pipelineRun });
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "probe-existing-pipelinerun", status: "started", sourceCommit, pipelineRun });
|
|
const before = getNodeRuntimePipelineRun(spec, pipelineRun);
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "probe-existing-pipelinerun", status: "succeeded", sourceCommit, pipelineRun, existingStatus: before.status ?? null });
|
|
if (scoped.dryRun) {
|
|
const gitMirror = nodeRuntimeGitMirrorStatus(scoped);
|
|
return {
|
|
ok: true,
|
|
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "dry-run",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
rerunOf: scoped.rerun ? basePipelineRun : null,
|
|
rerun: scoped.rerun,
|
|
before,
|
|
gitMirror: nodeScopedFullOutput(scoped) ? gitMirror : compactNodeRuntimeGitMirrorObservation(gitMirror),
|
|
manifest: nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun),
|
|
next: { triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm` },
|
|
};
|
|
}
|
|
if (!scoped.rerun && before.exists === true && (before.status === "True" || before.status === "Unknown")) {
|
|
const pipelineWait = before.status === "Unknown"
|
|
? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, pipelineWaitSeconds, {
|
|
opportunisticPostSync: () => nodeRuntimeOpportunisticGitMirrorSync(scoped, sourceCommit, pipelineRun),
|
|
opportunisticPostFlush: () => nodeRuntimeOpportunisticGitMirrorFlush(scoped, sourceCommit, pipelineRun),
|
|
})
|
|
: { ok: true, status: "already-succeeded", pipelineRun: before, polls: 0, elapsedMs: 0 };
|
|
const waitedPipelineRun = record(pipelineWait.pipelineRun);
|
|
const pipelineFailureSummary = nodeRuntimePipelineFailureSummary(pipelineWait);
|
|
const postFlush = waitedPipelineRun.status === "True"
|
|
? nodeRuntimeEnsureGitMirrorFlushed(scoped, "post", sourceCommit, pipelineRun)
|
|
: null;
|
|
const pipelinePending = pipelineWait.status === "pending";
|
|
const ok = pipelineWait.ok === true && (postFlush === null || postFlush.ok === true);
|
|
const elapsedMs = triggerElapsedMs();
|
|
const triggerWarning = pipelinePending ? null : nodeRuntimeTriggerElapsedWarning(spec, pipelineRun, elapsedMs);
|
|
if (triggerWarning !== null) printNodeRuntimeTriggerProgress(spec, { stage: "trigger-current", status: "warning", ...triggerWarning });
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "confirmed-trigger",
|
|
completion: pipelinePending ? "pending" : ok ? "completed" : "failed",
|
|
warning: pipelinePending ? pipelineWait.warning ?? nodeRuntimeCicdWaitWarning(spec, pipelineRun, pipelineWait) : triggerWarning ?? undefined,
|
|
triggerElapsedMs: elapsedMs,
|
|
sourceCommit,
|
|
pipelineRun,
|
|
before,
|
|
pipelineWait,
|
|
pipelineFailureSummary,
|
|
postFlush,
|
|
skipped: true,
|
|
reason: before.status === "True" ? "existing-pipelinerun-succeeded" : "existing-pipelinerun-running",
|
|
skipPolicy: "source-commit-only",
|
|
skipExplanation: "An existing PipelineRun for this HWLAB source commit was reused. If UniDesk node/lane YAML or other render inputs changed without a HWLAB source commit change, rerun the controlled publish with --rerun.",
|
|
rerunAvailable: true,
|
|
rerunCommand: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm --rerun`,
|
|
degradedReason: ok ? undefined : postFlush !== null && postFlush.ok !== true ? "node-runtime-git-mirror-post-flush-failed" : "node-runtime-existing-pipelinerun-not-succeeded",
|
|
next: {
|
|
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${pipelineRun}`,
|
|
rerun: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm --rerun`,
|
|
},
|
|
};
|
|
}
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "git-mirror-pre-sync", status: "started", sourceCommit, pipelineRun });
|
|
const gitMirror = nodeRuntimeEnsureGitMirrorSourceCurrent(scoped, sourceCommit, pipelineRun);
|
|
if (gitMirror.ok !== true) {
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "git-mirror-pre-sync", status: "failed", sourceCommit, pipelineRun, reason: gitMirror.degradedReason ?? null });
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
phase: "git-mirror-pre-sync",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
before,
|
|
gitMirror,
|
|
degradedReason: "node-runtime-git-mirror-pre-sync-failed",
|
|
};
|
|
}
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "git-mirror-pre-sync", status: "succeeded", sourceCommit, pipelineRun });
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "started", sourceCommit, pipelineRun });
|
|
const refresh = nodeRuntimeApply({ ...scoped, action: "apply", dryRun: false });
|
|
if (refresh.ok !== true) {
|
|
const diagnostics = record(refresh.diagnostics);
|
|
printNodeRuntimeTriggerProgress(spec, {
|
|
stage: "control-plane-refresh",
|
|
status: "failed",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
...(diagnostics.classification === "ssh-tcp-pool-transient"
|
|
? { reason: "ssh-tcp-pool-transient", failureKind: diagnostics.failureKind ?? null }
|
|
: {}),
|
|
});
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
phase: "control-plane-apply",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
refresh,
|
|
diagnostics: Object.keys(diagnostics).length > 0 ? diagnostics : null,
|
|
degradedReason: "node-runtime-control-plane-apply-before-trigger-failed",
|
|
next: {
|
|
retryTriggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
...(diagnostics.classification === "ssh-tcp-pool-transient" ? { sshPoolStatus: `bun scripts/cli.ts debug ssh-pool ${scoped.node}` } : {}),
|
|
},
|
|
};
|
|
}
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "succeeded", sourceCommit, pipelineRun });
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "create-pipelinerun", status: "started", sourceCommit, pipelineRun, rerun: scoped.rerun });
|
|
const create = createNodeRuntimePipelineRun(spec, sourceCommit, pipelineRun, scoped.timeoutSeconds, scoped.rerun);
|
|
const after = getNodeRuntimePipelineRun(spec, pipelineRun);
|
|
const createObserved = after.exists === true && (after.status === "Unknown" || after.status === "True");
|
|
const createOk = isCommandSuccess(create) || createObserved;
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "create-pipelinerun", status: createOk ? "succeeded" : "failed", sourceCommit, pipelineRun, exitCode: create.exitCode, createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined });
|
|
const pipelineWait = createOk
|
|
? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, pipelineWaitSeconds, {
|
|
opportunisticPostSync: () => nodeRuntimeOpportunisticGitMirrorSync(scoped, sourceCommit, pipelineRun),
|
|
opportunisticPostFlush: () => nodeRuntimeOpportunisticGitMirrorFlush(scoped, sourceCommit, pipelineRun),
|
|
})
|
|
: null;
|
|
const waitedPipelineRun = record(pipelineWait?.pipelineRun);
|
|
const pipelineFailureSummary = nodeRuntimePipelineFailureSummary(pipelineWait);
|
|
const postFlush = waitedPipelineRun.status === "True"
|
|
? nodeRuntimeEnsureGitMirrorFlushed(scoped, "post", sourceCommit, pipelineRun)
|
|
: null;
|
|
const pipelineReady = pipelineWait !== null && pipelineWait.ok === true;
|
|
const postFlushOk = postFlush === null || postFlush.ok === true;
|
|
const pipelinePending = pipelineWait !== null && pipelineWait.status === "pending";
|
|
const ok = createOk && (pipelineReady || pipelinePending) && postFlushOk;
|
|
const elapsedMs = triggerElapsedMs();
|
|
const triggerWarning = pipelinePending ? null : nodeRuntimeTriggerElapsedWarning(spec, pipelineRun, elapsedMs);
|
|
if (triggerWarning !== null) printNodeRuntimeTriggerProgress(spec, { stage: "trigger-current", status: "warning", ...triggerWarning });
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "confirmed-trigger",
|
|
completion: pipelinePending ? "pending" : ok ? "completed" : "failed",
|
|
warning: pipelinePending ? pipelineWait?.warning ?? nodeRuntimeCicdWaitWarning(spec, pipelineRun, pipelineWait) : triggerWarning ?? undefined,
|
|
triggerElapsedMs: elapsedMs,
|
|
mutation: createOk,
|
|
sourceCommit,
|
|
pipelineRun,
|
|
rerunOf: scoped.rerun ? basePipelineRun : null,
|
|
rerun: scoped.rerun,
|
|
before,
|
|
gitMirror,
|
|
refresh,
|
|
create: compactRuntimeCommand(create),
|
|
after,
|
|
pipelineWait,
|
|
pipelineFailureSummary,
|
|
postFlush,
|
|
createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined,
|
|
degradedReason: ok
|
|
? undefined
|
|
: !createOk
|
|
? "node-runtime-pipelinerun-create-failed"
|
|
: !pipelineReady
|
|
? "node-runtime-pipelinerun-not-succeeded"
|
|
: "node-runtime-git-mirror-post-flush-failed",
|
|
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${pipelineRun}` },
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeCicdWaitSeconds(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): number {
|
|
return Math.min(scoped.timeoutSeconds, NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS);
|
|
}
|
|
|
|
function nodeRuntimeCicdWaitWarning(spec: HwlabRuntimeLaneSpec, pipelineRun: string, pipelineWait: unknown): Record<string, unknown> {
|
|
const wait = record(pipelineWait);
|
|
return {
|
|
code: "node-runtime-cicd-wait-over-120s",
|
|
message: `PipelineRun ${pipelineRun} is still non-terminal after ${NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS}s; this is a severe timeout and requires further investigation of Tekton TaskRuns/Pods instead of treating it as normal wait time.`,
|
|
waitedSeconds: NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS,
|
|
elapsedMs: wait.elapsedMs ?? null,
|
|
polls: wait.polls ?? null,
|
|
inspectEnvReuse: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
|
|
inspectGitMirror: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeTriggerElapsedWarning(spec: HwlabRuntimeLaneSpec, pipelineRun: string, elapsedMs: number): Record<string, unknown> | null {
|
|
if (elapsedMs < NODE_RUNTIME_TRIGGER_SEVERE_WARNING_MS) return null;
|
|
return {
|
|
code: "node-runtime-trigger-over-120s",
|
|
message: `trigger-current total elapsed time exceeded ${NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS}s; this is a severe timeout even if the PipelineRun eventually succeeded and requires follow-up investigation of slow control-plane, Tekton and git-mirror stages.`,
|
|
thresholdMs: NODE_RUNTIME_TRIGGER_SEVERE_WARNING_MS,
|
|
elapsedMs,
|
|
inspectEnvReuse: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
|
|
inspectGitMirror: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
};
|
|
}
|
|
|
|
function withNodeRuntimeTriggerRendered(result: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): RenderedCliResult {
|
|
const pipelineWait = record(result.pipelineWait);
|
|
const pipelineRunRecord = record(pipelineWait.pipelineRun ?? result.after ?? result.before);
|
|
const warning = record(result.warning ?? pipelineWait.warning);
|
|
const postFlush = record(result.postFlush);
|
|
const gitMirror = record(result.gitMirror);
|
|
const gitMirrorSummary = record(postFlush.afterSummary ?? postFlush.beforeSummary ?? gitMirror.statusSummary ?? gitMirror.afterSummary ?? gitMirror.beforeSummary ?? gitMirror.summary);
|
|
const refresh = record(result.refresh);
|
|
const create = record(result.create);
|
|
const next = record(result.next);
|
|
const pipelineStatus = pipelineRunRecord.status ?? pipelineWait.status ?? "-";
|
|
const completion = result.completion ?? (result.ok === true ? "completed" : "failed");
|
|
const renderedText = [
|
|
"hwlab nodes control-plane trigger-current",
|
|
"",
|
|
webObserveTable(
|
|
["NODE", "LANE", "STATUS", "COMPLETION", "SOURCE", "PIPELINERUN"],
|
|
[[scoped.node, scoped.lane, result.ok === true ? "ok" : "failed", webObserveText(completion), shortValue(result.sourceCommit), webObserveText(result.pipelineRun)]],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["STAGE", "STATUS", "DETAIL"],
|
|
[
|
|
["git-mirror", gitMirror.ok === true ? "ok" : gitMirror.ok === false ? "failed" : "-", webObserveText(gitMirror.mode ?? gitMirror.degradedReason)],
|
|
["refresh", refresh.ok === true ? "ok" : refresh.ok === false ? "failed" : "-", webObserveText(refresh.degradedReason)],
|
|
["create", create.exitCode === 0 || result.mutation === true ? "ok" : create.exitCode === undefined ? "-" : "failed", result.createObservedAfterTimeout === true ? "observed-after-timeout" : `exit=${webObserveText(create.exitCode)}`],
|
|
["pipeline-wait", webObserveText(pipelineWait.status), `pipeline=${webObserveText(pipelineStatus)} polls=${webObserveText(pipelineWait.polls)} elapsed=${formatElapsedMs(pipelineWait.elapsedMs)}`],
|
|
["post-flush", postFlush.ok === true ? "ok" : postFlush.ok === false ? "failed" : "-", webObserveText(postFlush.mode ?? postFlush.degradedReason)],
|
|
["total", result.triggerElapsedMs === undefined ? "-" : "elapsed", formatElapsedMs(result.triggerElapsedMs)],
|
|
],
|
|
),
|
|
"",
|
|
Object.keys(warning).length === 0
|
|
? "WARNING\n-"
|
|
: webObserveTable(
|
|
["CODE", "MESSAGE"],
|
|
[[webObserveText(warning.code ?? "warning"), webObserveText(warning.message ?? "PipelineRun is still active after the default wait window.")]],
|
|
),
|
|
"",
|
|
Object.keys(warning).length === 0
|
|
? "INSPECT\n-"
|
|
: webObserveTable(
|
|
["CHECK", "COMMAND"],
|
|
[
|
|
["env-reuse", webObserveText(warning.inspectEnvReuse ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${result.pipelineRun} --full`)],
|
|
["git-mirror", webObserveText(warning.inspectGitMirror ?? `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`)],
|
|
],
|
|
),
|
|
"",
|
|
Object.keys(gitMirrorSummary).length === 0
|
|
? "GIT_MIRROR\n-"
|
|
: webObserveTable(
|
|
["LOCAL_SOURCE", "GITHUB_SOURCE", "LOCAL_GITOPS", "GITHUB_GITOPS", "PENDING", "IN_SYNC"],
|
|
[[
|
|
shortValue(gitMirrorSummary.localSource),
|
|
shortValue(gitMirrorSummary.githubSource),
|
|
shortValue(gitMirrorSummary.localGitops),
|
|
shortValue(gitMirrorSummary.githubGitops),
|
|
webObserveText(gitMirrorSummary.pendingFlush),
|
|
webObserveText(gitMirrorSummary.githubInSync),
|
|
]],
|
|
),
|
|
"",
|
|
"NEXT",
|
|
` ${next.status ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${result.pipelineRun}`}`,
|
|
"",
|
|
"Disclosure:",
|
|
" default view is a bounded CICD summary; use --full or --raw for the complete JSON payload.",
|
|
` ${NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS}s is the severe timeout warning threshold for both PipelineRun wait and total trigger elapsed time.`,
|
|
].join("\n");
|
|
return { ...result, renderedText, contentType: "text/plain" };
|
|
}
|
|
|
|
function withNodeRuntimeControlPlanePlanRendered(result: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): RenderedCliResult {
|
|
const checks = record(result.checks);
|
|
const next = record(result.next);
|
|
const renderedText = [
|
|
"hwlab nodes control-plane plan",
|
|
"",
|
|
webObserveTable(
|
|
["NODE", "LANE", "STATUS", "MODE", "NAMESPACE", "EXT_PG", "PUBLIC"],
|
|
[[
|
|
scoped.node,
|
|
scoped.lane,
|
|
result.ok === true ? "ok" : "failed",
|
|
webObserveText(result.mode),
|
|
webObserveText(checks.runtimeNamespace),
|
|
webObserveText(checks.externalPostgresDeclared),
|
|
webObserveText(checks.publicExposureDeclared),
|
|
]],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["CHECK", "VALUE"],
|
|
[
|
|
["node-scoped-target", webObserveText(checks.nodeScopedTargetConfigured)],
|
|
["local-postgres-absent", webObserveText(checks.localPostgresExpectedAbsent)],
|
|
["secret-values-printed", webObserveText(checks.secretValuesPrinted)],
|
|
],
|
|
),
|
|
"",
|
|
"NEXT",
|
|
` ${next.status ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`}`,
|
|
"",
|
|
"Disclosure:",
|
|
" default view is a bounded control-plane plan summary; use --full or --raw for the complete JSON payload.",
|
|
].join("\n");
|
|
return { ...result, renderedText, contentType: "text/plain" };
|
|
}
|
|
|
|
function withNodeRuntimeControlPlaneStatusRendered(result: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): RenderedCliResult {
|
|
const pipelineRun = record(result.pipelineRun);
|
|
const pipelineRunDiagnostics = record(pipelineRun.diagnostics);
|
|
const argo = record(result.argo);
|
|
const runtime = record(result.runtime);
|
|
const publicProbe = record(result.publicProbe);
|
|
const gitMirror = record(result.gitMirror);
|
|
const next = record(result.next);
|
|
const diagnostic = record(publicProbe.diagnostic);
|
|
const notReadyWorkloads = Array.isArray(runtime.notReadyWorkloads) ? runtime.notReadyWorkloads.map((item) => webObserveText(item)).filter(Boolean) : [];
|
|
const runtimeDetail = [
|
|
`workloads=${webObserveText(runtime.workloadReady)}`,
|
|
`pg=${webObserveText(runtime.externalPostgresReady ?? runtime.localPostgresReady)}`,
|
|
...(notReadyWorkloads.length > 0 ? [`notReady=${webObserveShort(notReadyWorkloads.join(","), 80)}`] : []),
|
|
].join(" ");
|
|
const failedTaskRuns = Array.isArray(pipelineRunDiagnostics.failedTaskRuns) ? pipelineRunDiagnostics.failedTaskRuns.map(compactNodeRuntimeTaskRunDiagnostic).filter(Boolean) : [];
|
|
const pendingTaskRuns = Array.isArray(pipelineRunDiagnostics.pendingTaskRuns) ? pipelineRunDiagnostics.pendingTaskRuns.map(compactNodeRuntimeTaskRunDiagnostic).filter(Boolean) : [];
|
|
const pipelineDetail = [
|
|
webObserveShort(webObserveText(pipelineRun.reason ?? pipelineRun.message), 50),
|
|
...(failedTaskRuns.length > 0 ? [`failed=${webObserveShort(failedTaskRuns.join(","), 80)}`] : []),
|
|
...(pendingTaskRuns.length > 0 ? [`pending=${webObserveShort(pendingTaskRuns.join(","), 80)}`] : []),
|
|
].filter(Boolean).join(" ");
|
|
const status = result.ok === true ? "ok" : result.ok === false ? "failed" : "-";
|
|
const renderedText = [
|
|
"hwlab nodes control-plane status",
|
|
"",
|
|
webObserveTable(
|
|
["NODE", "LANE", "STATUS", "REASON", "SOURCE", "PIPELINERUN"],
|
|
[[
|
|
scoped.node,
|
|
scoped.lane,
|
|
status,
|
|
webObserveShort(webObserveText(result.degradedReason), 40),
|
|
shortValue(result.sourceCommit),
|
|
webObserveShort(webObserveText(pipelineRun.name), 36),
|
|
]],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["STAGE", "STATUS", "DETAIL"],
|
|
[
|
|
["pipeline", pipelineRun.ready === true ? "ok" : pipelineRun.exists === true ? webObserveText(pipelineRun.status) : "missing", pipelineDetail],
|
|
["argo", argo.ready === true ? "ok" : "failed", `${webObserveText(argo.syncStatus)}/${webObserveText(argo.health)} rev=${shortValue(argo.syncRevision)}`],
|
|
["runtime", runtime.ready === true ? "ok" : "failed", runtimeDetail],
|
|
["public", publicProbe.ready === true ? "ok" : "failed", webObserveShort(webObserveText(diagnostic.kind ?? diagnostic.message), 80)],
|
|
["git-mirror", gitMirror.ready === true ? "ok" : "failed", `pending=${webObserveText(gitMirror.pendingFlush)} inSync=${webObserveText(gitMirror.githubInSync)}`],
|
|
],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["LOCAL_SOURCE", "GITHUB_SOURCE", "LOCAL_GITOPS", "GITHUB_GITOPS"],
|
|
[[
|
|
shortValue(gitMirror.localSource),
|
|
shortValue(gitMirror.githubSource),
|
|
shortValue(gitMirror.localGitops),
|
|
shortValue(gitMirror.githubGitops),
|
|
]],
|
|
),
|
|
"",
|
|
"NEXT",
|
|
` ${result.nextAction ?? next.full ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --full`}`,
|
|
"",
|
|
"Disclosure:",
|
|
" default view is a bounded control-plane status summary; use --full or --raw for the complete JSON payload.",
|
|
].join("\n");
|
|
return { ...result, renderedText, contentType: "text/plain" };
|
|
}
|
|
|
|
function nodeRuntimeGitMirrorStatus(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const spec = scoped.spec;
|
|
const mirror = nodeRuntimeGitMirrorTarget(spec);
|
|
const script = [
|
|
"set +e",
|
|
`namespace=${shellQuote(mirror.namespace)}`,
|
|
`read_deploy=${shellQuote(mirror.serviceReadName)}`,
|
|
`write_deploy=${shellQuote(mirror.serviceWriteName)}`,
|
|
`read_svc=${shellQuote(mirror.serviceReadName)}`,
|
|
`write_svc=${shellQuote(mirror.serviceWriteName)}`,
|
|
`cache_pvc=${shellQuote(mirror.cachePvcName)}`,
|
|
`cache_host_path=${shellQuote(mirror.cacheHostPath ?? "")}`,
|
|
`github_transport_mode=${shellQuote(mirror.githubTransport.mode)}`,
|
|
`github_token_secret=${shellQuote(mirror.githubTransport.mode === "https" ? mirror.githubTransport.tokenSecretName : "")}`,
|
|
`github_token_key=${shellQuote(mirror.githubTransport.mode === "https" ? mirror.githubTransport.tokenSecretKey : "")}`,
|
|
`github_token_source_ref=${shellQuote(mirror.githubTransport.mode === "https" ? mirror.githubTransport.tokenSourceRef : "")}`,
|
|
`github_token_source_key=${shellQuote(mirror.githubTransport.mode === "https" ? mirror.githubTransport.tokenSourceKey : "")}`,
|
|
"deploy_ready() { desired=$(kubectl -n \"$1\" get deploy \"$2\" -o 'jsonpath={.spec.replicas}' 2>/dev/null || true); ready=$(kubectl -n \"$1\" get deploy \"$2\" -o 'jsonpath={.status.readyReplicas}' 2>/dev/null || true); [ -n \"$desired\" ] && [ \"$desired\" -gt 0 ] 2>/dev/null && [ \"${ready:-0}\" = \"$desired\" ] && printf true || printf false; }",
|
|
"exists_res() { kubectl -n \"$1\" get \"$2\" \"$3\" >/dev/null 2>&1 && printf true || printf false; }",
|
|
"endpoint_ready() { endpoints=$(kubectl -n \"$1\" get endpoints \"$2\" -o 'jsonpath={.subsets[*].addresses[*].ip}' 2>/dev/null || true); [ -n \"$endpoints\" ] && printf true || printf false; }",
|
|
"github_transport_json=$(python3 - \"$github_transport_mode\" \"$namespace\" \"$github_token_secret\" \"$github_token_key\" \"$github_token_source_ref\" \"$github_token_source_key\" <<'PY'",
|
|
"import hashlib, json, subprocess, sys",
|
|
"mode, namespace, secret_name, secret_key, source_ref, source_key = sys.argv[1:7]",
|
|
"if mode != 'https':",
|
|
" print(json.dumps({'mode': mode, 'required': False, 'ready': True, 'valuesPrinted': False}))",
|
|
" raise SystemExit(0)",
|
|
"proc = subprocess.run(['kubectl', '-n', namespace, 'get', 'secret', secret_name, '-o', 'json'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)",
|
|
"exists = proc.returncode == 0",
|
|
"data = {}",
|
|
"if exists:",
|
|
" try:",
|
|
" data = json.loads(proc.stdout).get('data') or {}",
|
|
" except Exception:",
|
|
" data = {}",
|
|
"encoded = data.get(secret_key) if isinstance(data, dict) else None",
|
|
"present = isinstance(encoded, str) and len(encoded) > 0",
|
|
"fingerprint = 'sha256:' + hashlib.sha256(encoded.encode()).hexdigest()[:16] if present else None",
|
|
"print(json.dumps({'mode': mode, 'required': True, 'ready': exists and present, 'tokenSecretName': secret_name, 'tokenSecretKey': secret_key, 'tokenSourceRef': source_ref, 'tokenSourceKey': source_key, 'tokenSecretExists': exists, 'tokenKeyPresent': present, 'tokenKeyBytes': len(encoded) if present else 0, 'tokenFingerprint': fingerprint, 'valuesPrinted': False}))",
|
|
"PY",
|
|
")",
|
|
"summary_json=$(kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc '/etc/git-mirror/status.sh' 2>/tmp/hwlab-node-gitmirror-status.err || true)",
|
|
"if [ -z \"$summary_json\" ]; then summary_json='{}'; fi",
|
|
"read_deployment_ready=$(deploy_ready \"$namespace\" \"$read_deploy\")",
|
|
"write_deployment_ready=$(deploy_ready \"$namespace\" \"$write_deploy\")",
|
|
"read_service_exists=$(exists_res \"$namespace\" service \"$read_svc\")",
|
|
"write_service_exists=$(exists_res \"$namespace\" service \"$write_svc\")",
|
|
"read_endpoints_ready=$(endpoint_ready \"$namespace\" \"$read_svc\")",
|
|
"write_endpoints_ready=$(endpoint_ready \"$namespace\" \"$write_svc\")",
|
|
"cache_pvc_exists=$(exists_res \"$namespace\" pvc \"$cache_pvc\")",
|
|
"cache_host_path_exists=false",
|
|
"if [ -n \"$cache_host_path\" ] && kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc 'test -d /cache' >/dev/null 2>&1; then cache_host_path_exists=true; fi",
|
|
"SUMMARY_JSON=\"$summary_json\" GITHUB_TRANSPORT_JSON=\"$github_transport_json\" read_deployment_ready=\"$read_deployment_ready\" write_deployment_ready=\"$write_deployment_ready\" read_service_exists=\"$read_service_exists\" write_service_exists=\"$write_service_exists\" read_endpoints_ready=\"$read_endpoints_ready\" write_endpoints_ready=\"$write_endpoints_ready\" cache_pvc_exists=\"$cache_pvc_exists\" cache_host_path=\"$cache_host_path\" cache_host_path_exists=\"$cache_host_path_exists\" node <<'NODE'",
|
|
"const summary = (() => { try { return JSON.parse(process.env.SUMMARY_JSON || '{}'); } catch { return {}; } })();",
|
|
"const githubTransport = (() => { try { return JSON.parse(process.env.GITHUB_TRANSPORT_JSON || '{}'); } catch { return {}; } })();",
|
|
"const env = process.env;",
|
|
"const ok = env.read_deployment_ready === 'true' && env.write_deployment_ready === 'true' && env.read_service_exists === 'true' && env.write_service_exists === 'true' && env.read_endpoints_ready === 'true' && env.write_endpoints_ready === 'true' && (env.cache_pvc_exists === 'true' || env.cache_host_path_exists === 'true') && githubTransport.ready !== false && summary.localSource;",
|
|
"console.log(JSON.stringify({",
|
|
" ok: Boolean(ok),",
|
|
" resources: {",
|
|
" readDeploymentReady: env.read_deployment_ready === 'true',",
|
|
" writeDeploymentReady: env.write_deployment_ready === 'true',",
|
|
" readServiceExists: env.read_service_exists === 'true',",
|
|
" writeServiceExists: env.write_service_exists === 'true',",
|
|
" readEndpointsReady: env.read_endpoints_ready === 'true',",
|
|
" writeEndpointsReady: env.write_endpoints_ready === 'true',",
|
|
" cachePvcExists: env.cache_pvc_exists === 'true',",
|
|
" cacheHostPathConfigured: Boolean(env.cache_host_path),",
|
|
" cacheHostPathExists: env.cache_host_path_exists === 'true'",
|
|
" },",
|
|
" githubTransport,",
|
|
" summary,",
|
|
" valuesPrinted: false",
|
|
"}));",
|
|
"NODE",
|
|
].join("\n");
|
|
const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
|
|
const parsed = record(parseJsonObject(statusText(result)));
|
|
const summary = record(parsed.summary);
|
|
const resources = record(parsed.resources);
|
|
const githubTransport = record(parsed.githubTransport);
|
|
const ok = result.exitCode === 0 && parsed.ok === true;
|
|
const remoteText = `${result.stderr ?? ""}\n${result.stdout ?? ""}`;
|
|
const remoteUnavailable = /provider is not online|UNIDESK_SSH_PROVIDER_OFFLINE|ssh.*unreachable/iu.test(remoteText);
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "status",
|
|
mutation: false,
|
|
namespace: mirror.namespace,
|
|
readUrl: spec.gitReadUrl,
|
|
writeUrl: spec.gitWriteUrl,
|
|
sourceBranch: mirror.sourceBranch,
|
|
gitopsBranch: mirror.gitopsBranch,
|
|
resources,
|
|
githubTransport,
|
|
summary,
|
|
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
|
|
result: compactRuntimeCommand(result),
|
|
degradedReason: ok ? undefined : remoteUnavailable ? "node-provider-unavailable" : "node-runtime-git-mirror-not-ready",
|
|
valuesPrinted: false,
|
|
next: {
|
|
sync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
flush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
if (scoped.action !== "sync" && scoped.action !== "flush") return nodeRuntimeUnsupportedAction(scoped);
|
|
if (!scoped.confirm && !scoped.dryRun) throw new Error(`git-mirror ${scoped.action} requires --dry-run or --confirm`);
|
|
const spec = scoped.spec;
|
|
const mirror = nodeRuntimeGitMirrorTarget(spec);
|
|
const retryMaxAttempts = !scoped.dryRun && (scoped.action === "sync" || scoped.action === "flush") ? 5 : 1;
|
|
const attempts: Record<string, unknown>[] = [];
|
|
let finalAttempt: {
|
|
attempt: number;
|
|
retryLabel: string;
|
|
jobName: string;
|
|
manifest: Record<string, unknown>;
|
|
result: CommandResult;
|
|
partialSuccess: Record<string, unknown> | null;
|
|
retryableFailure: Record<string, unknown> | null;
|
|
actionSucceeded: boolean;
|
|
} | null = null;
|
|
|
|
for (let attempt = 1; attempt <= retryMaxAttempts; attempt += 1) {
|
|
const retryLabel = `${attempt}/${retryMaxAttempts}`;
|
|
const jobName = nodeRuntimeGitMirrorJobName(mirror, scoped.action);
|
|
const manifest = nodeRuntimeGitMirrorJobManifest(mirror, scoped.action, jobName);
|
|
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
|
const waitTimeoutSeconds = Math.max(5, Math.min(45, Math.max(5, scoped.timeoutSeconds - 10)));
|
|
const script = [
|
|
"set -eu",
|
|
`namespace=${shellQuote(mirror.namespace)}`,
|
|
`job=${shellQuote(jobName)}`,
|
|
`manifest_b64=${shellQuote(manifestB64)}`,
|
|
"manifest_path=\"/tmp/$job.json\"",
|
|
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
|
scoped.dryRun
|
|
? "kubectl create --dry-run=server -f \"$manifest_path\" -o name"
|
|
: [
|
|
"kubectl delete job -n \"$namespace\" \"$job\" --ignore-not-found=true >/dev/null",
|
|
"kubectl create -f \"$manifest_path\"",
|
|
`deadline=$(( $(date +%s) + ${waitTimeoutSeconds} ))`,
|
|
"while :; do",
|
|
" status=$(kubectl get job -n \"$namespace\" \"$job\" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)",
|
|
" succeeded=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')",
|
|
" failed=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')",
|
|
" if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi",
|
|
" if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then kubectl logs -n \"$namespace\" \"job/$job\" --tail=200 || true; exit 44; fi",
|
|
" if [ \"$(date +%s)\" -ge \"$deadline\" ]; then kubectl get job,pod -n \"$namespace\" -l job-name=\"$job\" -o wide || true; exit 45; fi",
|
|
" sleep 2",
|
|
"done",
|
|
"kubectl logs -n \"$namespace\" \"job/$job\" --tail=200 || true",
|
|
].join("\n"),
|
|
].join("\n");
|
|
const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
|
|
const partialSuccess = nodeRuntimeGitMirrorFlushPartialSuccess(scoped, mirror, result);
|
|
const actionSucceeded = isCommandSuccess(result);
|
|
const retryableFailure = !actionSucceeded
|
|
? nodeRuntimeGitMirrorRetryableFailure(scoped, mirror, result, partialSuccess, attempt, retryMaxAttempts)
|
|
: null;
|
|
const retryable = retryableFailure?.retryable === true;
|
|
const exhausted = retryable && attempt >= retryMaxAttempts;
|
|
attempts.push({
|
|
attempt,
|
|
retryLabel,
|
|
jobName,
|
|
ok: actionSucceeded,
|
|
exitCode: result.exitCode,
|
|
retryable,
|
|
exhausted,
|
|
partialSuccess: partialSuccess?.partialSuccess ?? null,
|
|
degradedReason: partialSuccess !== null
|
|
? "node-runtime-git-mirror-flush-post-push-fetch-failed"
|
|
: actionSucceeded ? null : `node-runtime-git-mirror-${scoped.action}-failed`,
|
|
result: compactRuntimeCommand(result),
|
|
valuesPrinted: false,
|
|
});
|
|
finalAttempt = { attempt, retryLabel, jobName, manifest, result, partialSuccess, retryableFailure, actionSucceeded };
|
|
if (actionSucceeded || scoped.dryRun || !retryable || exhausted) break;
|
|
const backoffMs = nodeRuntimeGitMirrorRetryDelayMs(attempt);
|
|
printNodeRuntimeTriggerProgress(spec, {
|
|
stage: `git-mirror-${scoped.action}-retry`,
|
|
status: "waiting",
|
|
retryLabel,
|
|
nextRetryLabel: `${attempt + 1}/${retryMaxAttempts}`,
|
|
backoffMs,
|
|
jobName,
|
|
});
|
|
sleepSync(backoffMs);
|
|
}
|
|
|
|
if (finalAttempt === null) throw new Error("git-mirror run produced no attempts");
|
|
const { jobName, manifest, result, partialSuccess, retryableFailure, actionSucceeded } = finalAttempt;
|
|
const status = scoped.dryRun || !actionSucceeded
|
|
? undefined
|
|
: nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
|
|
const retryExhausted = retryableFailure?.retryable === true && attempts.length >= retryMaxAttempts;
|
|
const stopped = !actionSucceeded && (retryExhausted || retryableFailure?.stopped === true || retryableFailure === null);
|
|
return {
|
|
ok: actionSucceeded && (status === undefined || status.ok === true),
|
|
command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: scoped.dryRun ? "dry-run" : `confirmed-${scoped.action}`,
|
|
mutation: !scoped.dryRun && actionSucceeded,
|
|
namespace: mirror.namespace,
|
|
githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror),
|
|
jobName,
|
|
manifest: scoped.dryRun ? manifest : undefined,
|
|
result: compactRuntimeCommand(result),
|
|
status,
|
|
retry: retryMaxAttempts > 1 ? {
|
|
policy: "exponential-backoff",
|
|
maxAttempts: retryMaxAttempts,
|
|
attempts,
|
|
exhausted: retryExhausted,
|
|
stopped,
|
|
valuesPrinted: false,
|
|
} : undefined,
|
|
retryableFailure: retryableFailure ?? undefined,
|
|
partialSuccess: partialSuccess?.partialSuccess ?? undefined,
|
|
recovery: partialSuccess ?? undefined,
|
|
degradedReason: partialSuccess !== null
|
|
? "node-runtime-git-mirror-flush-post-push-fetch-failed"
|
|
: actionSucceeded ? status?.ok === false ? "node-runtime-git-mirror-status-failed-after-action" : undefined : `node-runtime-git-mirror-${scoped.action}-failed`,
|
|
next: partialSuccess !== null
|
|
? partialSuccess.next
|
|
: scoped.dryRun
|
|
? { run: `bun scripts/cli.ts hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane} --confirm` }
|
|
: retryableFailure?.next ?? { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeGitMirrorRetryableFailure(
|
|
scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
|
|
mirror: NodeRuntimeGitMirrorTargetSpec,
|
|
result: CommandResult,
|
|
partialSuccess: Record<string, unknown> | null,
|
|
attempt: number,
|
|
retryMaxAttempts: number,
|
|
): Record<string, unknown> | null {
|
|
const text = `${result.stdout}\n${result.stderr}`;
|
|
const proxyConnectFailure = /hwlab git-mirror proxy-connect:/iu.test(text);
|
|
const sshProxyBannerFailure = mirror.egressProxy.mode !== "direct" && /Connection timed out during banner exchange|Connection to UNKNOWN port 65535 timed out|kex_exchange_identification|Connection closed by remote host/iu.test(text);
|
|
const waitTimeoutFailure = result.exitCode === 45
|
|
|| result.exitCode === 124
|
|
|| /UNIDESK_SSH_RUNTIME_TIMEOUT|ssh-runtime-timeout|ssh\/tran operation exceeded|job,pod/iu.test(text);
|
|
const scriptTransportMismatch =
|
|
(mirror.githubTransport.mode === "ssh" && /transport=https[\s\S]*https auth: missing GITHUB_TOKEN/iu.test(text))
|
|
|| (mirror.githubTransport.mode === "https" && /transport=ssh\b/iu.test(text));
|
|
if (scriptTransportMismatch) {
|
|
return {
|
|
retryable: false,
|
|
retryAttempt: attempt,
|
|
retryMaxAttempts,
|
|
retryLabel: `${attempt}/${retryMaxAttempts}`,
|
|
retryExhausted: false,
|
|
stopped: true,
|
|
reason: `Git mirror script transport does not match YAML githubTransport.mode=${mirror.githubTransport.mode}; apply the node control-plane so the git-mirror ConfigMap is updated before retrying.`,
|
|
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
|
|
githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror),
|
|
next: {
|
|
applyControlPlane: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const authFailure = /Authentication failed|Invalid username or password|Repository not found|could not read Username|could not read Password|terminal prompts disabled|https auth: missing GITHUB_TOKEN/iu.test(text);
|
|
if (authFailure) {
|
|
const authFix = mirror.githubTransport.mode === "ssh"
|
|
? {
|
|
fixSecret: `apply the YAML-declared SSH private key Secret ${mirror.secretName} through the node control-plane infra apply path`,
|
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
}
|
|
: {
|
|
fixSecret: "apply the YAML-declared githubTransport token source through the node control-plane infra apply path",
|
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
};
|
|
return {
|
|
retryable: false,
|
|
retryAttempt: attempt,
|
|
retryMaxAttempts,
|
|
retryLabel: `${attempt}/${retryMaxAttempts}`,
|
|
retryExhausted: false,
|
|
stopped: true,
|
|
reason: `Git mirror GitHub ${mirror.githubTransport.mode} authentication failed. This is not retryable; fix the YAML-declared ${mirror.githubTransport.mode === "ssh" ? "SSH key Secret" : "token source/Secret"} instead of consuming retry budget.`,
|
|
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
|
|
githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror),
|
|
next: authFix,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const retryable = partialSuccess !== null
|
|
|| proxyConnectFailure
|
|
|| sshProxyBannerFailure
|
|
|| waitTimeoutFailure
|
|
|| /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|ssh.github.com|github.com|fetch-pack|early EOF/iu.test(text);
|
|
if (!retryable) return null;
|
|
const retryLabel = `${attempt}/${retryMaxAttempts}`;
|
|
const exhausted = attempt >= retryMaxAttempts;
|
|
return {
|
|
retryable: true,
|
|
retryAttempt: attempt,
|
|
retryMaxAttempts,
|
|
retryLabel,
|
|
retryExhausted: exhausted,
|
|
stopped: exhausted,
|
|
backoffPolicy: "exponential",
|
|
nextRetryDelayMs: exhausted ? null : nodeRuntimeGitMirrorRetryDelayMs(attempt),
|
|
reason: partialSuccess !== null
|
|
? "GitOps push appears to have succeeded, but the post-push fetch/recheck failed. Standard git-mirror stops without host workspace fallback."
|
|
: waitTimeoutFailure
|
|
? "Git mirror job wait exceeded the controlled short-connection budget. Standard git-mirror retries with exponential backoff and stops when retry budget is exhausted."
|
|
: proxyConnectFailure || sshProxyBannerFailure
|
|
? "Git mirror job hit a retryable YAML-first SSH-over-proxy failure. Standard git-mirror keeps using the configured node-global proxy and stops when retry budget is exhausted."
|
|
: `Git mirror job hit a retryable upstream GitHub ${mirror.githubTransport.mode} transport/fetch failure. Standard git-mirror stops without host workspace fallback.`,
|
|
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
|
|
githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror),
|
|
next: exhausted
|
|
? {
|
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
stopped: "retry budget exhausted; standard git-mirror stopped instead of silently continuing",
|
|
}
|
|
: {
|
|
retry: `bun scripts/cli.ts hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`,
|
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeGitMirrorRetryDelayMs(attempt: number): number {
|
|
return Math.min(15_000, 1000 * (2 ** Math.max(0, attempt - 1)));
|
|
}
|
|
|
|
function nodeRuntimeGitMirrorHostWriteUrl(spec: HwlabRuntimeLaneSpec, mirror: NodeRuntimeGitMirrorTargetSpec): string | null {
|
|
const result = runNodeK3sArgs(spec, ["kubectl", "-n", mirror.namespace, "get", "service", mirror.serviceWriteName, "-o", "jsonpath={.spec.clusterIP}{\":\"}{.spec.ports[0].port}"], 60);
|
|
const value = result.stdout.trim();
|
|
if (result.exitCode !== 0 || value.length === 0 || value.startsWith(":")) return null;
|
|
return `http://${value}/${mirror.sourceRepository}.git`;
|
|
}
|
|
|
|
function nodeRuntimeEnsureGitMirrorSourceCurrent(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>, sourceCommit: string, pipelineRun: string): Record<string, unknown> {
|
|
const full = nodeScopedFullOutput(scoped);
|
|
const before = nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
|
|
const beforeSummary = compactNodeRuntimeGitMirrorStatus(before);
|
|
const beforeGithubTransport = record(before.githubTransport);
|
|
if (beforeGithubTransport.required === true && beforeGithubTransport.ready === false) {
|
|
return {
|
|
ok: false,
|
|
mode: "github-transport-auth-secret-not-ready",
|
|
sourceCommit,
|
|
beforeSummary,
|
|
before: full ? before : undefined,
|
|
githubTransport: beforeGithubTransport,
|
|
degradedReason: "node-runtime-git-mirror-auth-secret-not-ready",
|
|
next: {
|
|
applyBootstrap: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
},
|
|
};
|
|
}
|
|
if (before.ok === true && beforeSummary.localSource === sourceCommit && beforeSummary.githubSource === sourceCommit) {
|
|
const flush = nodeRuntimeEnsureGitMirrorFlushed(scoped, "pre", sourceCommit, pipelineRun, before);
|
|
return {
|
|
ok: flush.ok === true,
|
|
mode: "already-current",
|
|
sourceCommit,
|
|
beforeSummary,
|
|
before: full ? before : undefined,
|
|
flush,
|
|
degradedReason: flush.ok === true ? undefined : "node-runtime-git-mirror-pre-flush-failed",
|
|
};
|
|
}
|
|
const sync = nodeRuntimeGitMirrorRun({ ...scoped, domain: "git-mirror", action: "sync", confirm: true, dryRun: false, wait: true });
|
|
const after = record(sync.status);
|
|
const afterSummary = Object.keys(after).length > 0 ? compactNodeRuntimeGitMirrorStatus(after) : {};
|
|
const sourceOk = sync.ok === true && afterSummary.localSource === sourceCommit && afterSummary.githubSource === sourceCommit;
|
|
const flush = sourceOk ? nodeRuntimeEnsureGitMirrorFlushed(scoped, "pre", sourceCommit, pipelineRun, after) : null;
|
|
const ok = sourceOk && (flush === null || flush.ok === true);
|
|
return {
|
|
ok,
|
|
mode: "synced-before-trigger",
|
|
sourceCommit,
|
|
beforeSummary,
|
|
before: full ? before : undefined,
|
|
sync: full ? sync : compactNodeRuntimeGitMirrorRun(sync),
|
|
afterSummary: Object.keys(afterSummary).length > 0 ? afterSummary : null,
|
|
after: full ? sync.status ?? null : undefined,
|
|
flush,
|
|
degradedReason: ok
|
|
? undefined
|
|
: sourceOk
|
|
? "node-runtime-git-mirror-pre-flush-failed"
|
|
: "node-runtime-git-mirror-local-source-not-current-after-sync",
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeEnsureGitMirrorFlushed(
|
|
scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
|
|
phase: "pre" | "post" | "parallel",
|
|
sourceCommit: string,
|
|
pipelineRun: string | null,
|
|
statusInput: Record<string, unknown> | null = null,
|
|
): Record<string, unknown> {
|
|
const stage = `git-mirror-${phase}-flush`;
|
|
const full = nodeScopedFullOutput(scoped);
|
|
const before = statusInput ?? nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
|
|
const beforeSummary = compactNodeRuntimeGitMirrorStatus(before);
|
|
if (before.ok !== true) {
|
|
printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "failed", sourceCommit, pipelineRun, reason: "git-mirror-status-failed", ...beforeSummary });
|
|
return {
|
|
ok: false,
|
|
phase,
|
|
mode: "status-failed",
|
|
executed: false,
|
|
before: full ? before : undefined,
|
|
beforeSummary,
|
|
degradedReason: `node-runtime-git-mirror-${phase}-status-failed`,
|
|
next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
|
|
};
|
|
}
|
|
const flushNeeded = nodeRuntimeGitMirrorNeedsFlush(before);
|
|
if (!flushNeeded) {
|
|
printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "skipped", sourceCommit, pipelineRun, flushNeeded: false, ...beforeSummary });
|
|
return {
|
|
ok: true,
|
|
phase,
|
|
mode: "already-flushed",
|
|
executed: false,
|
|
before: full ? before : undefined,
|
|
beforeSummary,
|
|
after: full ? before : undefined,
|
|
afterSummary: beforeSummary,
|
|
};
|
|
}
|
|
printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "started", sourceCommit, pipelineRun, flushNeeded: true, ...beforeSummary });
|
|
const flush = nodeRuntimeGitMirrorRun({ ...scoped, domain: "git-mirror", action: "flush", confirm: true, dryRun: false, wait: true });
|
|
const after = record(flush.status);
|
|
const afterSummary = Object.keys(after).length > 0 ? compactNodeRuntimeGitMirrorStatus(after) : {};
|
|
const ok = flush.ok === true && Object.keys(after).length > 0 && !nodeRuntimeGitMirrorNeedsFlush(after);
|
|
printNodeRuntimeTriggerProgress(scoped.spec, {
|
|
stage,
|
|
status: ok ? "succeeded" : "failed",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
flushNeeded: true,
|
|
jobName: flush.jobName ?? null,
|
|
...afterSummary,
|
|
});
|
|
return {
|
|
ok,
|
|
phase,
|
|
mode: "flushed",
|
|
executed: true,
|
|
before: full ? before : undefined,
|
|
beforeSummary,
|
|
flush: full ? flush : compactNodeRuntimeGitMirrorRun(flush),
|
|
jobName: flush.jobName ?? null,
|
|
after: full ? (Object.keys(after).length > 0 ? after : null) : undefined,
|
|
afterSummary: Object.keys(afterSummary).length > 0 ? afterSummary : null,
|
|
degradedReason: ok ? undefined : `node-runtime-git-mirror-${phase}-flush-failed`,
|
|
next: ok ? undefined : flush.next ?? { flush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` },
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeOpportunisticGitMirrorSync(
|
|
scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
|
|
sourceCommit: string,
|
|
pipelineRun: string,
|
|
): Record<string, unknown> | null {
|
|
const stage = "git-mirror-parallel-sync";
|
|
const full = nodeScopedFullOutput(scoped);
|
|
printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "started", sourceCommit, pipelineRun });
|
|
const sync = nodeRuntimeGitMirrorRun({ ...scoped, domain: "git-mirror", action: "sync", confirm: true, dryRun: false, wait: true });
|
|
const after = record(sync.status);
|
|
const afterSummary = Object.keys(after).length > 0 ? compactNodeRuntimeGitMirrorStatus(after) : {};
|
|
const ok = sync.ok === true;
|
|
printNodeRuntimeTriggerProgress(scoped.spec, {
|
|
stage,
|
|
status: ok ? "succeeded" : "failed",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
jobName: sync.jobName ?? null,
|
|
...afterSummary,
|
|
});
|
|
return {
|
|
ok,
|
|
phase: "parallel",
|
|
mode: "synced-during-pipelinerun-wait",
|
|
executed: true,
|
|
sourceCommit,
|
|
pipelineRun,
|
|
sync: full ? sync : compactNodeRuntimeGitMirrorRun(sync),
|
|
jobName: sync.jobName ?? null,
|
|
after: full ? (Object.keys(after).length > 0 ? after : null) : undefined,
|
|
afterSummary: Object.keys(afterSummary).length > 0 ? afterSummary : null,
|
|
degradedReason: ok ? undefined : "node-runtime-git-mirror-parallel-sync-failed",
|
|
next: ok ? undefined : sync.next ?? { sync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` },
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeOpportunisticGitMirrorFlush(
|
|
scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
|
|
sourceCommit: string,
|
|
pipelineRun: string,
|
|
): Record<string, unknown> | null {
|
|
const status = nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
|
|
if (status.ok !== true || !nodeRuntimeGitMirrorNeedsFlush(status)) return null;
|
|
return nodeRuntimeEnsureGitMirrorFlushed(scoped, "parallel", sourceCommit, pipelineRun, status);
|
|
}
|
|
|
|
function nodeRuntimeGitMirrorNeedsFlush(status: Record<string, unknown>): boolean {
|
|
const summary = record(status.summary);
|
|
const localGitops = typeof summary.localGitops === "string" ? summary.localGitops : null;
|
|
const githubGitops = typeof summary.githubGitops === "string" ? summary.githubGitops : null;
|
|
return summary.pendingFlush === true
|
|
|| summary.flushNeeded === true
|
|
|| summary.githubInSync === false
|
|
|| (localGitops !== null && githubGitops !== null && localGitops !== githubGitops);
|
|
}
|
|
|
|
function nodeScopedFullOutput(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): boolean {
|
|
return scoped.originalArgs.includes("--full") || scoped.originalArgs.includes("--raw");
|
|
}
|
|
|
|
function compactNodeRuntimeGitMirrorObservation(status: Record<string, unknown>): Record<string, unknown> {
|
|
return {
|
|
ok: status.ok === true,
|
|
mode: status.mode ?? null,
|
|
mutation: status.mutation === true,
|
|
summary: compactNodeRuntimeGitMirrorStatus(status),
|
|
degradedReason: status.degradedReason ?? null,
|
|
next: status.next ?? null,
|
|
};
|
|
}
|
|
|
|
function compactNodeRuntimeGitMirrorRun(result: Record<string, unknown>): Record<string, unknown> {
|
|
const status = record(result.status);
|
|
return {
|
|
ok: result.ok === true,
|
|
action: result.action ?? null,
|
|
mode: result.mode ?? null,
|
|
mutation: result.mutation === true,
|
|
jobName: result.jobName ?? null,
|
|
partialSuccess: result.partialSuccess ?? null,
|
|
fallback: result.fallback ?? null,
|
|
retry: result.retry ?? null,
|
|
githubTransport: result.githubTransport ?? null,
|
|
retryableFailure: result.retryableFailure ?? null,
|
|
degradedReason: result.degradedReason ?? null,
|
|
statusSummary: Object.keys(status).length > 0 ? compactNodeRuntimeGitMirrorStatus(status) : null,
|
|
next: result.next ?? null,
|
|
};
|
|
}
|
|
|
|
function withNodeRuntimeGitMirrorRendered(result: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): RenderedCliResult {
|
|
const status = record(result.status);
|
|
const summary = scoped.action === "status" ? record(result.summary) : record(status.summary);
|
|
const retry = record(result.retry);
|
|
const retryAttempts = Array.isArray(retry.attempts) ? retry.attempts.map(record).filter((item): item is Record<string, unknown> => item !== null) : [];
|
|
const retryText = typeof retry.maxAttempts === "number"
|
|
? `${retryAttempts.length}/${retry.maxAttempts}${retry.exhausted === true ? " exhausted" : ""}`
|
|
: "-";
|
|
const transport = record(result.githubTransport);
|
|
const commandResult = record(result.result);
|
|
const retryableFailure = record(result.retryableFailure);
|
|
const next = record(result.next);
|
|
const resultStatus = result.ok === true ? "ok" : result.ok === false ? "failed" : "-";
|
|
const lines = [
|
|
`hwlab nodes git-mirror ${scoped.action}`,
|
|
"",
|
|
webObserveTable(
|
|
["NODE", "LANE", "ACTION", "STATUS", "MODE", "JOB", "RETRY"],
|
|
[[
|
|
scoped.node,
|
|
scoped.lane,
|
|
scoped.action,
|
|
resultStatus,
|
|
webObserveText(result.mode),
|
|
webObserveShort(webObserveText(result.jobName), 36),
|
|
retryText,
|
|
]],
|
|
),
|
|
"",
|
|
Object.keys(summary).length === 0
|
|
? "REFS\n-"
|
|
: webObserveTable(
|
|
["LOCAL_SOURCE", "GITHUB_SOURCE", "LOCAL_GITOPS", "GITHUB_GITOPS", "PENDING", "IN_SYNC"],
|
|
[[
|
|
shortValue(summary.localSource),
|
|
shortValue(summary.githubSource),
|
|
shortValue(summary.localGitops),
|
|
shortValue(summary.githubGitops),
|
|
webObserveText(summary.pendingFlush),
|
|
webObserveText(summary.githubInSync),
|
|
]],
|
|
),
|
|
"",
|
|
Object.keys(transport).length === 0
|
|
? "TRANSPORT\n-"
|
|
: webObserveTable(
|
|
["MODE", "READY", "REQUIRED"],
|
|
[[webObserveText(transport.mode), webObserveText(transport.ready), webObserveText(transport.required)]],
|
|
),
|
|
"",
|
|
Object.keys(commandResult).length === 0
|
|
? "RESULT\n-"
|
|
: webObserveTable(
|
|
["EXIT", "TIMED_OUT", "STDOUT_BYTES", "STDERR_BYTES"],
|
|
[[
|
|
webObserveText(commandResult.exitCode),
|
|
webObserveText(commandResult.timedOut),
|
|
webObserveText(commandResult.stdoutBytes),
|
|
webObserveText(commandResult.stderrBytes),
|
|
]],
|
|
),
|
|
"",
|
|
Object.keys(retryableFailure).length === 0
|
|
? "RETRYABLE\n-"
|
|
: webObserveTable(
|
|
["RETRYABLE", "EXHAUSTED", "REASON"],
|
|
[[
|
|
webObserveText(retryableFailure.retryable),
|
|
webObserveText(retryableFailure.exhausted),
|
|
webObserveShort(webObserveText(retryableFailure.reason ?? retryableFailure.degradedReason), 100),
|
|
]],
|
|
),
|
|
"",
|
|
"NEXT",
|
|
` ${next.status ?? next.sync ?? next.flush ?? next.retry ?? next.run ?? `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`}`,
|
|
"",
|
|
"Disclosure:",
|
|
" default view is a bounded git-mirror summary; use --full or --raw for the complete JSON payload.",
|
|
];
|
|
return { ...result, renderedText: lines.join("\n"), contentType: "text/plain" };
|
|
}
|
|
|
|
function compactNodeRuntimeGitMirrorStatus(status: Record<string, unknown>): Record<string, unknown> {
|
|
const summary = record(status.summary);
|
|
return {
|
|
ok: status.ok === true,
|
|
localSource: summary.localSource ?? null,
|
|
githubSource: summary.githubSource ?? null,
|
|
localGitops: summary.localGitops ?? null,
|
|
githubGitops: summary.githubGitops ?? null,
|
|
pendingFlush: summary.pendingFlush === true,
|
|
flushNeeded: summary.flushNeeded === true,
|
|
githubInSync: summary.githubInSync === true,
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeGitMirrorJobName(mirror: NodeRuntimeGitMirrorTargetSpec, action: "sync" | "flush"): string {
|
|
const prefix = action === "sync" ? mirror.syncJobPrefix : mirror.flushJobPrefix;
|
|
return `${prefix}-${Date.now().toString(36)}`.slice(0, 63);
|
|
}
|
|
|
|
function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTargetSpec, action: "sync" | "flush", jobName: string): Record<string, unknown> {
|
|
const volumes: Record<string, unknown>[] = [
|
|
{ name: "cache", ...(mirror.cacheHostPath === null ? { persistentVolumeClaim: { claimName: mirror.cachePvcName } } : { hostPath: { path: mirror.cacheHostPath, type: "DirectoryOrCreate" } }) },
|
|
{ name: "script", configMap: { name: mirror.syncConfigMapName, defaultMode: 0o755 } },
|
|
];
|
|
const volumeMounts: Record<string, unknown>[] = [
|
|
{ name: "cache", mountPath: "/cache" },
|
|
{ name: "script", mountPath: "/script", readOnly: true },
|
|
];
|
|
if (mirror.githubTransport.mode === "ssh") {
|
|
volumes.splice(1, 0, { name: "git-ssh", secret: { secretName: mirror.secretName, defaultMode: 0o400 } });
|
|
volumeMounts.splice(1, 0, { name: "git-ssh", mountPath: "/git-ssh", readOnly: true });
|
|
}
|
|
return {
|
|
apiVersion: "batch/v1",
|
|
kind: "Job",
|
|
metadata: {
|
|
name: jobName,
|
|
namespace: mirror.namespace,
|
|
labels: {
|
|
"app.kubernetes.io/name": "git-mirror",
|
|
"app.kubernetes.io/part-of": "hwlab-node-control-plane",
|
|
"app.kubernetes.io/component": `${action}-controller`,
|
|
"hwlab.pikastech.local/node": mirror.node,
|
|
"hwlab.pikastech.local/lane": mirror.lane,
|
|
"hwlab.pikastech.local/trigger": "manual-cli",
|
|
},
|
|
},
|
|
spec: {
|
|
backoffLimit: 0,
|
|
activeDeadlineSeconds: 600,
|
|
ttlSecondsAfterFinished: 3600,
|
|
template: {
|
|
metadata: {
|
|
labels: {
|
|
"app.kubernetes.io/name": "git-mirror",
|
|
"app.kubernetes.io/part-of": "hwlab-node-control-plane",
|
|
"app.kubernetes.io/component": `${action}-controller`,
|
|
"hwlab.pikastech.local/node": mirror.node,
|
|
"hwlab.pikastech.local/lane": mirror.lane,
|
|
},
|
|
},
|
|
spec: {
|
|
restartPolicy: "Never",
|
|
volumes,
|
|
containers: [{
|
|
name: action,
|
|
image: mirror.toolsImage,
|
|
imagePullPolicy: mirror.toolsImagePullPolicy,
|
|
command: [action === "sync" ? "/script/sync.sh" : "/script/flush.sh"],
|
|
env: [...nodeRuntimeGitMirrorProxyEnv(mirror), ...nodeRuntimeGitMirrorGithubTransportEnv(mirror)],
|
|
volumeMounts,
|
|
}],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeGitMirrorProxyEnv(mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, unknown>[] {
|
|
const proxy = mirror.egressProxy;
|
|
if (proxy.mode === "direct") return [];
|
|
const proxyUrl = `http://${proxy.serviceName}.${proxy.namespace}.svc.cluster.local:${proxy.port}`;
|
|
const noProxy = proxy.noProxy.join(",");
|
|
return [
|
|
{ name: "HTTP_PROXY", value: proxyUrl },
|
|
{ name: "HTTPS_PROXY", value: proxyUrl },
|
|
{ name: "ALL_PROXY", value: proxyUrl },
|
|
{ name: "http_proxy", value: proxyUrl },
|
|
{ name: "https_proxy", value: proxyUrl },
|
|
{ name: "all_proxy", value: proxyUrl },
|
|
{ name: "NO_PROXY", value: noProxy },
|
|
{ name: "no_proxy", value: noProxy },
|
|
];
|
|
}
|
|
|
|
function nodeRuntimeGitMirrorGithubTransportEnv(mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, unknown>[] {
|
|
if (mirror.githubTransport.mode !== "https") return [];
|
|
return [{
|
|
name: "GITHUB_TOKEN",
|
|
valueFrom: { secretKeyRef: { name: mirror.githubTransport.tokenSecretName, key: mirror.githubTransport.tokenSecretKey } },
|
|
}];
|
|
}
|
|
|
|
function nodeRuntimeGitMirrorGithubTransportSummary(mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, unknown> {
|
|
const transport = mirror.githubTransport;
|
|
if (transport.mode === "ssh") return { mode: "ssh", secretName: mirror.secretName, valuesPrinted: false };
|
|
return {
|
|
mode: "https",
|
|
username: transport.username,
|
|
tokenSecretName: transport.tokenSecretName,
|
|
tokenSecretKey: transport.tokenSecretKey,
|
|
tokenSourceRef: transport.tokenSourceRef,
|
|
tokenSourceKey: transport.tokenSourceKey,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const spec = scoped.spec;
|
|
const sourceCommitOverride = optionValue(scoped.originalArgs, "--source-commit");
|
|
const pipelineRunOverride = optionValue(scoped.originalArgs, "--pipeline-run");
|
|
const head = sourceCommitOverride === undefined ? resolveNodeRuntimeLaneHead(spec) : null;
|
|
const sourceCommit = sourceCommitOverride ?? head?.sourceCommit ?? null;
|
|
const pipelineRun = pipelineRunOverride ?? (sourceCommit === null ? null : nodeRuntimePipelineRunName(spec, sourceCommit));
|
|
const namespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", spec.runtimeNamespace, "-o", "name"], 60);
|
|
const namespaceExists = namespace.exitCode === 0;
|
|
const postgresObjects = namespaceExists
|
|
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "statefulset,svc,pvc", "-o", "name"], 60)
|
|
: null;
|
|
const localPostgresObjects = postgresObjects === null
|
|
? []
|
|
: postgresObjects.stdout.split(/\r?\n/u).map((line) => line.trim()).filter((line) => isLocalPostgresObject(line, spec));
|
|
const serviceAccount = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "serviceaccount", spec.serviceAccountName, "-o", "name"], 60);
|
|
const pipeline = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipeline", spec.pipeline, "-o", "name"], 60);
|
|
const argo = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", "jsonpath={.spec.source.repoURL}{\"\\n\"}{.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}"], 60);
|
|
const [repoURL = "", targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = argo.stdout.split(/\r?\n/u);
|
|
const pipelineRunProbe = pipelineRun === null ? null : getNodeRuntimePipelineRun(spec, pipelineRun);
|
|
const pipelineRunDiagnostics = pipelineRun !== null && pipelineRunProbe?.exists === true && pipelineRunProbe?.status !== "True"
|
|
? nodeRuntimePipelineRunDiagnostics(spec, pipelineRun)
|
|
: null;
|
|
const workloads = namespaceExists
|
|
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset,svc,ingress,configmap", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "name"], 60)
|
|
: null;
|
|
const workloadNames = workloads === null ? [] : workloads.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
|
const workloadReadinessProbe = namespaceExists
|
|
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "jsonpath={range .items[*]}{.kind}{\"/\"}{.metadata.name}{\"\\t\"}{.status.readyReplicas}{\"/\"}{.status.replicas}{\"/\"}{.spec.replicas}{\"\\n\"}{end}"], 60)
|
|
: null;
|
|
const workloadReadiness = parseNodeRuntimeWorkloadReadiness(workloadReadinessProbe?.stdout ?? "");
|
|
const bridge = externalPostgresBridgeStatus(spec, namespaceExists);
|
|
const secrets = externalPostgresSecretStatus(spec, namespaceExists);
|
|
const publicProbes = nodeRuntimePublicProbeStatus(spec);
|
|
const gitMirror = nodeRuntimeGitMirrorStatus(scoped);
|
|
const gitMirrorCompact = compactNodeRuntimeGitMirrorStatus(gitMirror);
|
|
const controlPlaneReady = serviceAccount.exitCode === 0 && pipeline.exitCode === 0 && argo.exitCode === 0;
|
|
const workloadsReady = workloadReadiness.length > 0 && workloadReadiness.every((item) => item.ready);
|
|
const localPostgresExpectedAbsent = nodeRuntimeLocalPostgresExpectedAbsent(spec);
|
|
const localPostgresReady = localPostgresExpectedAbsent ? localPostgresObjects.length === 0 : localPostgresObjects.length > 0;
|
|
const runtimeReady = namespaceExists && localPostgresReady && workloadsReady && (spec.externalPostgres === undefined || (bridge.ready && secrets.ready));
|
|
const argoReady = argo.exitCode === 0 && repoURL === spec.argoRepoUrl && targetRevision === spec.gitopsBranch && path === spec.runtimePath && syncStatus === "Synced" && health === "Healthy";
|
|
const pipelineRunReady = pipelineRunProbe !== null && pipelineRunProbe.status === "True";
|
|
const pipelineRunDegradedReason = typeof pipelineRunDiagnostics?.degradedReason === "string"
|
|
? pipelineRunDiagnostics.degradedReason
|
|
: "pipelinerun-not-succeeded";
|
|
const publicReady = publicProbes.ready === true;
|
|
const gitMirrorReady = gitMirror.ok === true && gitMirrorCompact.pendingFlush === false && gitMirrorCompact.githubInSync === true;
|
|
const fullStatus = {
|
|
ok: controlPlaneReady && runtimeReady && argoReady && pipelineRunReady && publicReady && gitMirrorReady,
|
|
command: `hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
mode: "node-scoped-runtime-status",
|
|
mutation: false,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
sourceCommit,
|
|
pipelineRun,
|
|
expected: nodeRuntimeExpected(spec),
|
|
sourceHead: head === null
|
|
? { ok: sourceCommit !== null, value: sourceCommit, source: "option" }
|
|
: { ok: head.sourceCommit !== null, value: head.sourceCommit, probe: compactRuntimeCommand(head.result) },
|
|
controlPlane: {
|
|
ready: controlPlaneReady,
|
|
serviceAccount: { exists: serviceAccount.exitCode === 0, result: compactRuntimeCommand(serviceAccount) },
|
|
pipeline: { exists: pipeline.exitCode === 0, result: compactRuntimeCommand(pipeline) },
|
|
},
|
|
argo: {
|
|
ready: argoReady,
|
|
application: spec.app,
|
|
repoURL,
|
|
expectedRepoURL: spec.argoRepoUrl,
|
|
targetRevision,
|
|
path,
|
|
syncRevision,
|
|
syncStatus,
|
|
health,
|
|
result: compactRuntimeCommand(argo),
|
|
},
|
|
pipelineRun: pipelineRunProbe,
|
|
pipelineRunDiagnostics,
|
|
runtime: {
|
|
ready: runtimeReady,
|
|
namespace: spec.runtimeNamespace,
|
|
namespaceExists,
|
|
localPostgresObjects,
|
|
localPostgresAbsent: namespaceExists && localPostgresObjects.length === 0,
|
|
localPostgresExpectedAbsent,
|
|
localPostgresReady,
|
|
workloadNames,
|
|
workloadCount: workloadNames.length,
|
|
workloadReadiness,
|
|
workloadsReady,
|
|
workloadReadinessResult: workloadReadinessProbe === null ? null : compactRuntimeCommand(workloadReadinessProbe),
|
|
workloadResult: workloads === null ? null : compactRuntimeCommand(workloads),
|
|
externalPostgresBridge: bridge,
|
|
externalPostgresSecrets: secrets,
|
|
},
|
|
publicProbes,
|
|
gitMirror: {
|
|
...gitMirror,
|
|
compact: gitMirrorCompact,
|
|
ready: gitMirrorReady,
|
|
},
|
|
probes: {
|
|
namespace: compactRuntimeCommand(namespace),
|
|
postgresObjects: postgresObjects === null ? null : compactRuntimeCommand(postgresObjects),
|
|
},
|
|
degradedReason: controlPlaneReady
|
|
? runtimeReady
|
|
? argoReady
|
|
? pipelineRunReady
|
|
? publicReady
|
|
? gitMirrorReady ? undefined : "git-mirror-pending-flush"
|
|
: "public-probe-not-ready"
|
|
: pipelineRunDegradedReason
|
|
: "argo-not-synced-healthy"
|
|
: namespaceExists ? "runtime-not-ready" : "runtime-namespace-missing"
|
|
: "control-plane-not-ready",
|
|
next: {
|
|
plan: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`,
|
|
apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
},
|
|
};
|
|
const summary = summarizeNodeRuntimeControlPlaneStatus(fullStatus, scoped);
|
|
if (scoped.originalArgs.includes("--full") || scoped.originalArgs.includes("--raw")) return { ...fullStatus, summary };
|
|
return summary;
|
|
}
|
|
|
|
function parseNodeRuntimeWorkloadReadiness(text: string): Array<Record<string, unknown>> {
|
|
return text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean).map((line) => {
|
|
const [ref = "", counts = ""] = line.split(/\t/u);
|
|
const [readyRaw = "", currentRaw = "", desiredRaw = ""] = counts.split("/");
|
|
const readyReplicas = nullableInteger(readyRaw);
|
|
const currentReplicas = nullableInteger(currentRaw);
|
|
const desiredReplicas = nullableInteger(desiredRaw);
|
|
const ready = desiredReplicas !== null
|
|
? (readyReplicas ?? 0) >= desiredReplicas
|
|
: readyReplicas !== null && currentReplicas !== null && readyReplicas >= currentReplicas;
|
|
return {
|
|
ref,
|
|
readyReplicas,
|
|
currentReplicas,
|
|
desiredReplicas,
|
|
ready,
|
|
};
|
|
});
|
|
}
|
|
|
|
function nullableInteger(value: string): number | null {
|
|
if (!/^[0-9]+$/u.test(value)) return null;
|
|
return Number(value);
|
|
}
|
|
|
|
function nodeRuntimePublicProbeStatus(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
|
const web = publicHttpProbe("web", spec.publicWebUrl);
|
|
const apiHealth = publicHttpProbe("apiHealth", joinUrlPath(spec.publicApiUrl, "/health/live"));
|
|
const ready = web.ok === true && apiHealth.ok === true;
|
|
const targetHost = nodeRuntimeTargetHostPublicProbeStatus(spec);
|
|
return {
|
|
ready,
|
|
web,
|
|
apiHealth,
|
|
targetHost,
|
|
diagnostic: nodeRuntimePublicProbeDiagnostic(ready, targetHost),
|
|
};
|
|
}
|
|
|
|
function publicHttpProbe(kind: string, url: string): Record<string, unknown> {
|
|
const result = runCommand(["curl", "-k", "-sS", "--connect-timeout", "5", "--max-time", "12", "-o", "/dev/null", "-w", "%{http_code}", url], repoRoot, { timeoutMs: 15_000 });
|
|
const httpStatus = nullableInteger(result.stdout.trim().slice(-3));
|
|
return {
|
|
kind,
|
|
url,
|
|
ok: result.exitCode === 0 && httpStatus !== null && httpStatus >= 200 && httpStatus < 400,
|
|
httpStatus,
|
|
result: compactRuntimeCommand(result),
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeTargetHostPublicProbeStatus(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
|
const webUrl = spec.publicWebUrl;
|
|
const apiHealthUrl = joinUrlPath(spec.publicApiUrl, "/health/live");
|
|
const script = [
|
|
"set -eu",
|
|
`web_url=${shellQuote(webUrl)}`,
|
|
`api_url=${shellQuote(apiHealthUrl)}`,
|
|
"probe() {",
|
|
" name=\"$1\"",
|
|
" url=\"$2\"",
|
|
" err_file=$(mktemp)",
|
|
" set +e",
|
|
" http_status=$(curl -k -sS --connect-timeout 5 --max-time 12 -o /dev/null -w '%{http_code}' \"$url\" 2>\"$err_file\")",
|
|
" rc=$?",
|
|
" set -e",
|
|
" error_text=$(tr '\\r\\n\\t' ' ' <\"$err_file\" | tail -c 600)",
|
|
" rm -f \"$err_file\"",
|
|
" printf '%sUrl\\t%s\\n' \"$name\" \"$url\"",
|
|
" printf '%sExitCode\\t%s\\n' \"$name\" \"$rc\"",
|
|
" printf '%sHttpStatus\\t%s\\n' \"$name\" \"$http_status\"",
|
|
" printf '%sError\\t%s\\n' \"$name\" \"$error_text\"",
|
|
"}",
|
|
"probe web \"$web_url\"",
|
|
"probe apiHealth \"$api_url\"",
|
|
].join("\n");
|
|
const result = runTransHostScript(spec.nodeId, script, "", 35);
|
|
const fields = keyValueLinesFromText(result.stdout);
|
|
const web = targetHostPublicHttpProbeFromFields("web", fields, webUrl, result.exitCode === 0);
|
|
const apiHealth = targetHostPublicHttpProbeFromFields("apiHealth", fields, apiHealthUrl, result.exitCode === 0);
|
|
return {
|
|
node: spec.nodeId,
|
|
ready: web.ok === true && apiHealth.ok === true,
|
|
probeAvailable: result.exitCode === 0 && result.timedOut !== true,
|
|
web,
|
|
apiHealth,
|
|
result: compactRuntimeCommand(result),
|
|
};
|
|
}
|
|
|
|
function targetHostPublicHttpProbeFromFields(kind: string, fields: Record<string, string>, fallbackUrl: string, transportOk: boolean): Record<string, unknown> {
|
|
const exitCode = numericField(fields[`${kind}ExitCode`]);
|
|
const httpStatus = numericField(fields[`${kind}HttpStatus`]);
|
|
const error = fields[`${kind}Error`] ?? "";
|
|
return {
|
|
kind,
|
|
url: fields[`${kind}Url`] ?? fallbackUrl,
|
|
ok: transportOk && exitCode === 0 && httpStatus !== null && httpStatus >= 200 && httpStatus < 400,
|
|
httpStatus,
|
|
exitCode,
|
|
error: error.length > 0 ? error.slice(0, 600) : null,
|
|
};
|
|
}
|
|
|
|
function nodeRuntimePublicProbeDiagnostic(publicReady: boolean, targetHost: Record<string, unknown>): Record<string, unknown> {
|
|
const targetWeb = record(targetHost.web);
|
|
const targetApiHealth = record(targetHost.apiHealth);
|
|
const targetReady = targetHost.ready === true;
|
|
if (!publicReady) {
|
|
return {
|
|
kind: "public-entry-probe-failed",
|
|
affectsUserEntry: true,
|
|
targetHostReady: targetReady,
|
|
message: "control-plane public probe failed; treat this as a public endpoint readiness failure before using web-probe closeout evidence.",
|
|
nextAction: "run hwlab nodes web-probe run for the same node/lane after checking publicProbe.web and publicProbe.apiHealth",
|
|
};
|
|
}
|
|
if (targetHost.probeAvailable !== true) {
|
|
return {
|
|
kind: "target-host-public-probe-unavailable",
|
|
affectsUserEntry: false,
|
|
targetHostReady: false,
|
|
message: "control-plane public probe passed, but the target host diagnostic probe could not run; this does not invalidate user-entry evidence.",
|
|
nextAction: "use control-plane publicProbe and web-probe evidence for closeout; inspect target host SSH/trans health separately if host-side CLI must call the public URL",
|
|
};
|
|
}
|
|
if (!targetReady) {
|
|
return {
|
|
kind: "target-host-public-egress-mismatch",
|
|
affectsUserEntry: false,
|
|
targetHostReady: false,
|
|
failed: {
|
|
web: targetWeb.ok === true ? null : { httpStatus: targetWeb.httpStatus ?? null, exitCode: targetWeb.exitCode ?? null, error: targetWeb.error ?? null },
|
|
apiHealth: targetApiHealth.ok === true ? null : { httpStatus: targetApiHealth.httpStatus ?? null, exitCode: targetApiHealth.exitCode ?? null, error: targetApiHealth.error ?? null },
|
|
},
|
|
message: "control-plane public probe passed, but the target host cannot reach the same public URLs; classify this as target-host egress/hairpin diagnostics, not a public endpoint failure.",
|
|
nextAction: "use control-plane publicProbe and web-probe evidence for issue closeout; track host-side public URL access separately if hwlab-cli must run on that host",
|
|
};
|
|
}
|
|
return {
|
|
kind: "public-entry-and-target-host-ok",
|
|
affectsUserEntry: false,
|
|
targetHostReady: true,
|
|
message: "control-plane public probe and target-host public URL probe both passed.",
|
|
nextAction: null,
|
|
};
|
|
}
|
|
|
|
function joinUrlPath(baseUrl: string, suffix: string): string {
|
|
return `${baseUrl.replace(/\/+$/u, "")}/${suffix.replace(/^\/+/u, "")}`;
|
|
}
|
|
|
|
function compactNodeRuntimeTaskRunDiagnostic(value: unknown): string {
|
|
const item = record(value);
|
|
const pipelineTask = webObserveText(item.pipelineTask ?? item.pipelineTaskName);
|
|
const taskRun = webObserveText(item.taskRun ?? item.name);
|
|
const reason = webObserveText(item.reason ?? item.status ?? item.message);
|
|
const left = [pipelineTask, taskRun].filter(Boolean).join("/");
|
|
return [left, reason ? `(${webObserveShort(reason, 36)})` : ""].filter(Boolean).join("");
|
|
}
|
|
|
|
function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const pipelineRun = record(status.pipelineRun);
|
|
const pipelineRunDiagnostics = record(status.pipelineRunDiagnostics);
|
|
const argo = record(status.argo);
|
|
const runtime = record(status.runtime);
|
|
const publicProbes = record(status.publicProbes);
|
|
const gitMirror = record(status.gitMirror);
|
|
const gitMirrorCompact = record(gitMirror.compact);
|
|
const workloadReadiness = Array.isArray(runtime.workloadReadiness) ? runtime.workloadReadiness.map(record) : [];
|
|
const readyWorkloads = workloadReadiness.filter((item) => item.ready === true).length;
|
|
const notReadyWorkloads = workloadReadiness.filter((item) => item.ready !== true).map((item) => item.ref).filter(Boolean);
|
|
const workloadCount = typeof runtime.workloadCount === "number" ? runtime.workloadCount : workloadReadiness.length;
|
|
const webProbe = record(publicProbes.web);
|
|
const apiProbe = record(publicProbes.apiHealth);
|
|
const targetHostProbe = record(publicProbes.targetHost);
|
|
const targetHostWebProbe = record(targetHostProbe.web);
|
|
const targetHostApiProbe = record(targetHostProbe.apiHealth);
|
|
const publicProbeDiagnostic = record(publicProbes.diagnostic);
|
|
return {
|
|
ok: status.ok === true,
|
|
command: status.command,
|
|
mode: "node-scoped-runtime-status-summary",
|
|
mutation: false,
|
|
node: status.node,
|
|
lane: status.lane,
|
|
sourceCommit: status.sourceCommit,
|
|
degradedReason: typeof status.degradedReason === "string" ? status.degradedReason : null,
|
|
pipelineRun: {
|
|
name: pipelineRun.name ?? null,
|
|
exists: pipelineRun.exists === true,
|
|
status: pipelineRun.status ?? null,
|
|
reason: pipelineRun.reason ?? null,
|
|
message: pipelineRun.message ?? null,
|
|
createdAt: pipelineRun.createdAt ?? null,
|
|
ready: pipelineRun.status === "True",
|
|
diagnostics: Object.keys(pipelineRunDiagnostics).length === 0 ? null : {
|
|
degradedReason: pipelineRunDiagnostics.degradedReason ?? null,
|
|
taskRunCount: pipelineRunDiagnostics.taskRunCount ?? null,
|
|
podCount: pipelineRunDiagnostics.podCount ?? null,
|
|
failedTaskRunCount: pipelineRunDiagnostics.failedTaskRunCount ?? null,
|
|
failedTaskRuns: pipelineRunDiagnostics.failedTaskRuns ?? [],
|
|
failureSummary: pipelineRunDiagnostics.failureSummary ?? null,
|
|
pendingTaskRuns: pipelineRunDiagnostics.pendingTaskRuns ?? [],
|
|
unscheduledPods: pipelineRunDiagnostics.unscheduledPods ?? [],
|
|
schedulingMessages: pipelineRunDiagnostics.schedulingMessages ?? [],
|
|
next: pipelineRunDiagnostics.next ?? null,
|
|
},
|
|
},
|
|
argo: {
|
|
application: argo.application ?? null,
|
|
ready: argo.ready === true,
|
|
syncRevision: argo.syncRevision ?? null,
|
|
syncStatus: argo.syncStatus ?? null,
|
|
health: argo.health ?? null,
|
|
},
|
|
runtime: {
|
|
namespace: runtime.namespace ?? null,
|
|
namespaceExists: runtime.namespaceExists === true,
|
|
ready: runtime.ready === true,
|
|
workloadCount,
|
|
workloadReady: `${readyWorkloads}/${workloadReadiness.length}`,
|
|
notReadyWorkloads,
|
|
localPostgresAbsent: runtime.localPostgresAbsent === true,
|
|
localPostgresExpectedAbsent: runtime.localPostgresExpectedAbsent === true,
|
|
localPostgresReady: runtime.localPostgresReady === true,
|
|
externalPostgresReady: runtime.externalPostgresBridge === undefined && runtime.externalPostgresSecrets === undefined
|
|
? null
|
|
: record(runtime.externalPostgresBridge).ready === true && record(runtime.externalPostgresSecrets).ready === true,
|
|
},
|
|
publicProbe: {
|
|
ready: publicProbes.ready === true,
|
|
web: { url: webProbe.url ?? null, ok: webProbe.ok === true, httpStatus: webProbe.httpStatus ?? null },
|
|
apiHealth: { url: apiProbe.url ?? null, ok: apiProbe.ok === true, httpStatus: apiProbe.httpStatus ?? null },
|
|
targetHost: Object.keys(targetHostProbe).length === 0 ? null : {
|
|
node: targetHostProbe.node ?? status.node ?? null,
|
|
ready: targetHostProbe.ready === true,
|
|
probeAvailable: targetHostProbe.probeAvailable === true,
|
|
web: {
|
|
url: targetHostWebProbe.url ?? null,
|
|
ok: targetHostWebProbe.ok === true,
|
|
httpStatus: targetHostWebProbe.httpStatus ?? null,
|
|
exitCode: targetHostWebProbe.exitCode ?? null,
|
|
error: targetHostWebProbe.error ?? null,
|
|
},
|
|
apiHealth: {
|
|
url: targetHostApiProbe.url ?? null,
|
|
ok: targetHostApiProbe.ok === true,
|
|
httpStatus: targetHostApiProbe.httpStatus ?? null,
|
|
exitCode: targetHostApiProbe.exitCode ?? null,
|
|
error: targetHostApiProbe.error ?? null,
|
|
},
|
|
},
|
|
diagnostic: Object.keys(publicProbeDiagnostic).length === 0 ? null : publicProbeDiagnostic,
|
|
},
|
|
gitMirror: {
|
|
ready: gitMirror.ready === true,
|
|
localSource: gitMirrorCompact.localSource ?? null,
|
|
githubSource: gitMirrorCompact.githubSource ?? null,
|
|
localGitops: gitMirrorCompact.localGitops ?? null,
|
|
githubGitops: gitMirrorCompact.githubGitops ?? null,
|
|
pendingFlush: gitMirrorCompact.pendingFlush === true,
|
|
flushNeeded: gitMirrorCompact.flushNeeded === true,
|
|
githubInSync: gitMirrorCompact.githubInSync === true,
|
|
},
|
|
nextAction: nodeRuntimeStatusNextAction(status, scoped),
|
|
next: {
|
|
full: `${nodeRuntimeStatusCommand(scoped)} --full`,
|
|
plan: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`,
|
|
triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`,
|
|
webProbe: `bun scripts/cli.ts hwlab nodes web-probe run --node ${scoped.node} --lane ${scoped.lane}`,
|
|
},
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeStatusNextAction(status: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): string {
|
|
const reason = typeof status.degradedReason === "string" ? status.degradedReason : null;
|
|
if (reason === null) return `${nodeRuntimeStatusCommand(scoped)} --full`;
|
|
if (reason === "control-plane-not-ready" || reason === "runtime-namespace-missing") {
|
|
return `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`;
|
|
}
|
|
if (reason === "runtime-not-ready") return `${nodeRuntimeStatusCommand(scoped)} --full`;
|
|
if (reason === "argo-not-synced-healthy") {
|
|
return `bun scripts/cli.ts hwlab nodes control-plane refresh --node ${scoped.node} --lane ${scoped.lane} --confirm`;
|
|
}
|
|
if (reason === "pipelinerun-not-succeeded") {
|
|
return `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`;
|
|
}
|
|
if (reason === "node-runtime-ci-step-publish-failed") {
|
|
return `bun scripts/cli.ts platform-infra sub2api status --target ${scoped.node}`;
|
|
}
|
|
if (reason === "node-runtime-ci-taskrun-failed") {
|
|
const next = record(record(status.pipelineRunDiagnostics).next);
|
|
const failedStepLogs = typeof next.failedStepLogs === "string" ? next.failedStepLogs : null;
|
|
return failedStepLogs ?? `${nodeRuntimeStatusCommand(scoped)} --full`;
|
|
}
|
|
if (reason === "node-runtime-ci-pod-capacity-exhausted" || reason === "node-runtime-ci-pod-unschedulable") {
|
|
return `bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node ${scoped.node} --lane ${scoped.lane} --min-age-minutes 60 --limit 20 --dry-run`;
|
|
}
|
|
if (reason === "public-probe-not-ready") {
|
|
return `bun scripts/cli.ts hwlab nodes web-probe run --node ${scoped.node} --lane ${scoped.lane}`;
|
|
}
|
|
if (reason === "git-mirror-pending-flush") {
|
|
return `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`;
|
|
}
|
|
return `${nodeRuntimeStatusCommand(scoped)} --full`;
|
|
}
|
|
|
|
function nodeRuntimeStatusCommand(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): string {
|
|
const sourceCommit = optionValue(scoped.originalArgs, "--source-commit");
|
|
const pipelineRun = optionValue(scoped.originalArgs, "--pipeline-run");
|
|
return [
|
|
`bun scripts/cli.ts hwlab nodes control-plane status --node ${shellQuote(scoped.node)} --lane ${shellQuote(scoped.lane)}`,
|
|
sourceCommit === undefined ? "" : `--source-commit ${shellQuote(sourceCommit)}`,
|
|
pipelineRun === undefined ? "" : `--pipeline-run ${shellQuote(pipelineRun)}`,
|
|
].filter(Boolean).join(" ");
|
|
}
|
|
|
|
function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pipelineRun: string): Record<string, unknown> {
|
|
const taskRunTemplate = `{{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/pipelineTask"}}{{"\\t"}}{{if .spec.taskRef}}{{.spec.taskRef.name}}{{end}}{{"\\t"}}{{with index .status.conditions 0}}{{.status}}{{"\\t"}}{{.reason}}{{else}}{{"\\t"}}{{end}}{{"\\t"}}{{.status.podName}}{{"\\t"}}{{with index .status.conditions 0}}{{printf "%.600s" .message}}{{end}}{{"\\n"}}{{end}}`;
|
|
const podTemplate = `{{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/taskRun"}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{.spec.nodeName}}{{"\\t"}}{{range .status.conditions}}{{if eq .type "PodScheduled"}}{{.status}}|{{.reason}}|{{printf "%.600s" .message}}{{end}}{{end}}{{"\\t"}}{{range .status.initContainerStatuses}}{{.name}}:{{if .state.terminated}}{{.state.terminated.exitCode}}:{{.state.terminated.reason}}{{else if .state.waiting}}waiting:{{.state.waiting.reason}}{{else if .state.running}}running:{{.state.running.startedAt}}{{end}},{{end}}{{"\\t"}}{{range .status.containerStatuses}}{{.name}}:{{if .state.terminated}}{{.state.terminated.exitCode}}:{{.state.terminated.reason}}{{else if .state.waiting}}waiting:{{.state.waiting.reason}}{{else if .state.running}}running:{{.state.running.startedAt}}{{end}},{{end}}{{"\\n"}}{{end}}`;
|
|
const taskRunsResult = runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "taskrun", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", `go-template=${taskRunTemplate}`], 60);
|
|
const podsResult = runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pod", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", `go-template=${podTemplate}`], 60);
|
|
const taskRuns = isCommandSuccess(taskRunsResult) ? nodeRuntimePipelineDiagnosticTaskRunsFromTsv(taskRunsResult.stdout) : [];
|
|
const pods = isCommandSuccess(podsResult) ? nodeRuntimePipelineDiagnosticPodsFromTsv(podsResult.stdout) : [];
|
|
const pendingTaskRuns = taskRuns.filter((item) => item.status !== "True" && item.status !== "False");
|
|
const failedTaskRuns = taskRuns.filter((item) => item.status === "False");
|
|
const failedTaskRunSummaries = nodeRuntimePipelineFailedTaskRunSummaries(spec, failedTaskRuns, pods);
|
|
const stepPublishFailures = failedTaskRunSummaries.filter((item) => item.container === "step-publish" || item.step === "publish" || item.step === "step-publish");
|
|
const unscheduledPods = pods.filter((item) => item.scheduled === false);
|
|
const schedulingMessages = unscheduledPods
|
|
.map((item) => typeof item.scheduledMessage === "string" ? item.scheduledMessage : "")
|
|
.filter((message) => message.length > 0);
|
|
const tooManyPods = schedulingMessages.some((message) => /too many pods/iu.test(message));
|
|
const failureSummary = failedTaskRunSummaries.length > 0
|
|
? {
|
|
failedTaskRunCount: failedTaskRuns.length,
|
|
failedStepCount: failedTaskRunSummaries.length,
|
|
stepPublishFailureCount: stepPublishFailures.length,
|
|
firstFailure: failedTaskRunSummaries[0] ?? null,
|
|
stepFailures: failedTaskRunSummaries.slice(0, 8),
|
|
nextAction: stepPublishFailures.length > 0
|
|
? "step-publish failed in a service build; first distinguish platform-infra Sub2API/proxy health from a single upstream transient, then choose controlled rerun or artifact-publish/envRecipe retry fix."
|
|
: failedTaskRunSummaries.length > 0
|
|
? "Inspect the failed TaskRun and bounded pod log command before rerunning the control-plane trigger."
|
|
: null,
|
|
}
|
|
: null;
|
|
return {
|
|
ok: taskRunsResult.exitCode === 0 && podsResult.exitCode === 0,
|
|
pipelineRun,
|
|
taskRuns,
|
|
pods,
|
|
taskRunCount: taskRuns.length,
|
|
podCount: pods.length,
|
|
failedTaskRunCount: failedTaskRuns.length,
|
|
failedTaskRuns: failedTaskRunSummaries,
|
|
stepPublishFailures,
|
|
failureSummary,
|
|
pendingTaskRuns,
|
|
unscheduledPods,
|
|
schedulingMessages,
|
|
degradedReason: tooManyPods
|
|
? "node-runtime-ci-pod-capacity-exhausted"
|
|
: unscheduledPods.length > 0
|
|
? "node-runtime-ci-pod-unschedulable"
|
|
: stepPublishFailures.length > 0
|
|
? "node-runtime-ci-step-publish-failed"
|
|
: failedTaskRunSummaries.length > 0
|
|
? "node-runtime-ci-taskrun-failed"
|
|
: pendingTaskRuns.length > 0
|
|
? "node-runtime-ci-taskrun-pending"
|
|
: undefined,
|
|
query: {
|
|
taskRuns: compactRuntimeCommand(taskRunsResult),
|
|
pods: compactRuntimeCommand(podsResult),
|
|
},
|
|
next: tooManyPods || unscheduledPods.length > 0
|
|
? { cleanupRuns: `bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes 60 --limit 20 --dry-run` }
|
|
: stepPublishFailures.length > 0
|
|
? {
|
|
sub2apiStatus: `bun scripts/cli.ts platform-infra sub2api status --target ${spec.nodeId}`,
|
|
sub2apiValidate: `bun scripts/cli.ts platform-infra sub2api validate --target ${spec.nodeId}`,
|
|
failedStepLogs: stepPublishFailures[0]?.logCommand ?? null,
|
|
rerun: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm --rerun`,
|
|
}
|
|
: failedTaskRunSummaries.length > 0
|
|
? {
|
|
failedStepLogs: failedTaskRunSummaries[0]?.logCommand ?? null,
|
|
failedTaskRun: failedTaskRunSummaries[0]?.taskRunCommand ?? null,
|
|
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
|
|
}
|
|
: undefined,
|
|
};
|
|
}
|
|
|
|
function nodeRuntimePipelineDiagnosticTaskRunsFromTsv(text: string): Array<Record<string, unknown>> {
|
|
return text.split(/\r?\n/u).map((line) => line.trimEnd()).filter(Boolean).map((line) => {
|
|
const parts = line.split("\t");
|
|
const [name = "", pipelineTask = "", taskRef = "", status = "", reason = "", podName = "", ...messageParts] = parts;
|
|
const message = messageParts.join("\t");
|
|
return {
|
|
name: stringOrNull(name),
|
|
pipelineTask: stringOrNull(pipelineTask),
|
|
taskRef: stringOrNull(taskRef),
|
|
status: stringOrNull(status),
|
|
reason: stringOrNull(reason),
|
|
message: diagnosticText(message),
|
|
podName: stringOrNull(podName),
|
|
steps: [],
|
|
failedSteps: [],
|
|
};
|
|
});
|
|
}
|
|
|
|
function nodeRuntimePipelineDiagnosticPodsFromTsv(text: string): Array<Record<string, unknown>> {
|
|
return text.split(/\r?\n/u).map((line) => line.trimEnd()).filter(Boolean).map((line) => {
|
|
const [name = "", taskRun = "", phase = "", nodeName = "", scheduledRaw = "", initRaw = "", containersRaw = ""] = line.split("\t");
|
|
const [scheduledStatus = "", scheduledReason = "", scheduledMessage = ""] = scheduledRaw.split("|");
|
|
const initContainers = nodeRuntimePipelineContainerStatusesFromCompact(initRaw);
|
|
const containers = nodeRuntimePipelineContainerStatusesFromCompact(containersRaw);
|
|
const failedContainers = [...initContainers, ...containers].filter((container) => container.failed === true);
|
|
return {
|
|
name: stringOrNull(name),
|
|
taskRun: stringOrNull(taskRun),
|
|
phase: stringOrNull(phase),
|
|
nodeName: stringOrNull(nodeName),
|
|
scheduled: scheduledStatus.length === 0 ? null : scheduledStatus === "True",
|
|
scheduledReason: stringOrNull(scheduledReason),
|
|
scheduledMessage: diagnosticText(scheduledMessage),
|
|
initContainers,
|
|
containers,
|
|
failedContainers,
|
|
};
|
|
});
|
|
}
|
|
|
|
function nodeRuntimePipelineContainerStatusesFromCompact(text: string): Array<Record<string, unknown>> {
|
|
return text.split(",").map((item) => item.trim()).filter(Boolean).map((item) => {
|
|
const [name = "", stateOrExit = "", reason = ""] = item.split(":");
|
|
const exitCode = /^[0-9]+$/u.test(stateOrExit) ? Number(stateOrExit) : null;
|
|
const state = exitCode !== null ? "terminated" : stateOrExit === "waiting" ? "waiting" : stateOrExit === "running" ? "running" : null;
|
|
return {
|
|
name: stringOrNull(name),
|
|
ready: null,
|
|
restartCount: null,
|
|
state,
|
|
failed: exitCode !== null && exitCode !== 0,
|
|
exitCode,
|
|
reason: stringOrNull(reason),
|
|
message: null,
|
|
startedAt: null,
|
|
finishedAt: null,
|
|
};
|
|
});
|
|
}
|
|
|
|
function nodeRuntimePipelineDiagnosticTaskRuns(json: Record<string, unknown>): Array<Record<string, unknown>> {
|
|
const items = Array.isArray(json.items) ? json.items.map(record) : [];
|
|
return items.map((item) => {
|
|
const metadata = record(item.metadata);
|
|
const labels = record(metadata.labels);
|
|
const spec = record(item.spec);
|
|
const taskRef = record(spec.taskRef);
|
|
const status = record(item.status);
|
|
const conditions = Array.isArray(status.conditions) ? status.conditions.map(record) : [];
|
|
const condition = conditions[0] ?? {};
|
|
const steps = nodeRuntimePipelineDiagnosticSteps(status.steps);
|
|
const failedSteps = steps.filter((step) => step.failed === true);
|
|
return {
|
|
name: metadata.name ?? null,
|
|
pipelineTask: labels["tekton.dev/pipelineTask"] ?? null,
|
|
taskRef: taskRef.name ?? null,
|
|
status: condition.status ?? null,
|
|
reason: condition.reason ?? null,
|
|
message: diagnosticText(condition.message),
|
|
podName: status.podName ?? null,
|
|
steps,
|
|
failedSteps,
|
|
};
|
|
});
|
|
}
|
|
|
|
function nodeRuntimePipelineDiagnosticPods(json: Record<string, unknown>): Array<Record<string, unknown>> {
|
|
const items = Array.isArray(json.items) ? json.items.map(record) : [];
|
|
return items.map((item) => {
|
|
const metadata = record(item.metadata);
|
|
const labels = record(metadata.labels);
|
|
const spec = record(item.spec);
|
|
const status = record(item.status);
|
|
const conditions = Array.isArray(status.conditions) ? status.conditions.map(record) : [];
|
|
const scheduled = conditions.find((condition) => condition.type === "PodScheduled");
|
|
const initContainers = nodeRuntimeContainerStatusSummaries(status.initContainerStatuses);
|
|
const containers = nodeRuntimeContainerStatusSummaries(status.containerStatuses);
|
|
const failedContainers = [...initContainers, ...containers].filter((container) => container.failed === true);
|
|
return {
|
|
name: metadata.name ?? null,
|
|
taskRun: labels["tekton.dev/taskRun"] ?? null,
|
|
phase: status.phase ?? null,
|
|
nodeName: spec.nodeName ?? null,
|
|
scheduled: scheduled === undefined ? null : scheduled.status === "True",
|
|
scheduledReason: scheduled?.reason ?? null,
|
|
scheduledMessage: diagnosticText(scheduled?.message),
|
|
initContainers,
|
|
containers,
|
|
failedContainers,
|
|
};
|
|
});
|
|
}
|
|
|
|
function nodeRuntimePipelineDiagnosticSteps(value: unknown): Array<Record<string, unknown>> {
|
|
const steps = Array.isArray(value) ? value.map(record) : [];
|
|
return steps.map((step) => {
|
|
const terminated = record(step.terminated);
|
|
const running = record(step.running);
|
|
const waiting = record(step.waiting);
|
|
const exitCode = typeof terminated.exitCode === "number" ? terminated.exitCode : null;
|
|
const terminatedReason = typeof terminated.reason === "string" ? terminated.reason : null;
|
|
const waitingReason = typeof waiting.reason === "string" ? waiting.reason : null;
|
|
const state = Object.keys(terminated).length > 0 ? "terminated" : Object.keys(waiting).length > 0 ? "waiting" : Object.keys(running).length > 0 ? "running" : null;
|
|
return {
|
|
name: step.name ?? null,
|
|
container: step.container ?? null,
|
|
state,
|
|
failed: exitCode !== null && exitCode !== 0,
|
|
exitCode,
|
|
reason: terminatedReason ?? waitingReason,
|
|
message: diagnosticText(terminated.message ?? waiting.message),
|
|
startedAt: terminated.startedAt ?? running.startedAt ?? null,
|
|
finishedAt: terminated.finishedAt ?? null,
|
|
};
|
|
});
|
|
}
|
|
|
|
function nodeRuntimeContainerStatusSummaries(value: unknown): Array<Record<string, unknown>> {
|
|
const containers = Array.isArray(value) ? value.map(record) : [];
|
|
return containers.map((container) => {
|
|
const state = record(container.state);
|
|
const terminated = record(state.terminated);
|
|
const waiting = record(state.waiting);
|
|
const running = record(state.running);
|
|
const exitCode = typeof terminated.exitCode === "number" ? terminated.exitCode : null;
|
|
const terminatedReason = typeof terminated.reason === "string" ? terminated.reason : null;
|
|
const waitingReason = typeof waiting.reason === "string" ? waiting.reason : null;
|
|
const stateName = Object.keys(terminated).length > 0 ? "terminated" : Object.keys(waiting).length > 0 ? "waiting" : Object.keys(running).length > 0 ? "running" : null;
|
|
return {
|
|
name: container.name ?? null,
|
|
ready: container.ready === true,
|
|
restartCount: typeof container.restartCount === "number" ? container.restartCount : null,
|
|
state: stateName,
|
|
failed: exitCode !== null && exitCode !== 0,
|
|
exitCode,
|
|
reason: terminatedReason ?? waitingReason,
|
|
message: diagnosticText(terminated.message ?? waiting.message),
|
|
startedAt: terminated.startedAt ?? running.startedAt ?? null,
|
|
finishedAt: terminated.finishedAt ?? null,
|
|
};
|
|
});
|
|
}
|
|
|
|
function nodeRuntimePipelineFailedTaskRunSummaries(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
failedTaskRuns: Array<Record<string, unknown>>,
|
|
pods: Array<Record<string, unknown>>,
|
|
): Array<Record<string, unknown>> {
|
|
const summaries: Array<Record<string, unknown>> = [];
|
|
for (const taskRun of failedTaskRuns) {
|
|
const taskRunName = stringOrNull(taskRun.name);
|
|
const podName = stringOrNull(taskRun.podName);
|
|
const pod = pods.find((item) => item.name === podName || (taskRunName !== null && item.taskRun === taskRunName)) ?? {};
|
|
const failedSteps = Array.isArray(taskRun.failedSteps) ? taskRun.failedSteps.map(record) : [];
|
|
const failedContainers = Array.isArray(pod.failedContainers) ? pod.failedContainers.map(record) : [];
|
|
const failures = failedSteps.length > 0
|
|
? failedSteps
|
|
: failedContainers.map((container) => ({
|
|
name: typeof container.name === "string" && container.name.startsWith("step-") ? container.name.slice("step-".length) : container.name ?? null,
|
|
container: container.name ?? null,
|
|
state: container.state ?? null,
|
|
exitCode: container.exitCode ?? null,
|
|
reason: container.reason ?? null,
|
|
message: container.message ?? null,
|
|
}));
|
|
const effectiveFailures = failures.length > 0 ? failures : [{ name: null, container: null, state: null, exitCode: null, reason: taskRun.reason ?? null, message: taskRun.message ?? null }];
|
|
for (const failure of effectiveFailures) {
|
|
const stepName = stringOrNull(failure.name);
|
|
const containerName = stringOrNull(failure.container) ?? (stepName === null ? null : `step-${stepName}`);
|
|
summaries.push({
|
|
taskRun: taskRunName,
|
|
pipelineTask: taskRun.pipelineTask ?? null,
|
|
taskRef: taskRun.taskRef ?? null,
|
|
taskRunStatus: taskRun.status ?? null,
|
|
taskRunReason: taskRun.reason ?? null,
|
|
taskRunMessage: diagnosticText(taskRun.message),
|
|
pod: podName,
|
|
podPhase: pod.phase ?? null,
|
|
nodeName: pod.nodeName ?? null,
|
|
step: stepName,
|
|
container: containerName,
|
|
containerState: failure.state ?? null,
|
|
terminationReason: failure.reason ?? null,
|
|
exitCode: typeof failure.exitCode === "number" ? failure.exitCode : null,
|
|
message: diagnosticText(failure.message),
|
|
logCommand: podName === null ? null : nodeRuntimePipelineLogsCommand(spec, podName, containerName),
|
|
taskRunCommand: taskRunName === null ? null : nodeRuntimeK3sCommand(spec, ["get", "taskrun", "-n", HWLAB_CI_NAMESPACE, taskRunName, "-o", "yaml"]),
|
|
taskRunDescribeCommand: taskRunName === null ? null : nodeRuntimeK3sCommand(spec, ["describe", "taskrun", "-n", HWLAB_CI_NAMESPACE, taskRunName]),
|
|
podDescribeCommand: podName === null ? null : nodeRuntimeK3sCommand(spec, ["describe", "pod", "-n", HWLAB_CI_NAMESPACE, podName]),
|
|
});
|
|
}
|
|
}
|
|
return summaries.slice(0, 16);
|
|
}
|
|
|
|
function nodeRuntimePipelineFailureSummary(value: unknown): Record<string, unknown> | null {
|
|
const recordValue = record(value);
|
|
const direct = record(recordValue.failureSummary);
|
|
if (Object.keys(direct).length > 0) return direct;
|
|
const diagnostics = record(recordValue.diagnostics);
|
|
const fromDiagnostics = record(diagnostics.failureSummary);
|
|
return Object.keys(fromDiagnostics).length > 0 ? fromDiagnostics : null;
|
|
}
|
|
|
|
function nodeRuntimePipelineLogsCommand(spec: HwlabRuntimeLaneSpec, podName: string, containerName: string | null): string {
|
|
return nodeRuntimeK3sCommand(spec, [
|
|
"logs",
|
|
"--namespace", HWLAB_CI_NAMESPACE,
|
|
"--pod", podName,
|
|
...(containerName === null ? ["--all-containers"] : ["--container", containerName]),
|
|
"--tail", "200",
|
|
]);
|
|
}
|
|
|
|
function nodeRuntimeK3sCommand(spec: HwlabRuntimeLaneSpec, args: string[]): string {
|
|
return ["trans", spec.nodeKubeRoute, ...args].map(shellQuote).join(" ");
|
|
}
|
|
|
|
function stringOrNull(value: unknown): string | null {
|
|
return typeof value === "string" && value.length > 0 ? value : null;
|
|
}
|
|
|
|
function diagnosticText(value: unknown): string | null {
|
|
if (typeof value !== "string") return null;
|
|
const trimmed = value.trim();
|
|
if (trimmed.length === 0) return null;
|
|
return trimmed
|
|
.replace(/postgres(?:ql)?:\/\/[^\s"'`]+/giu, "postgres://<redacted>")
|
|
.replace(/\b([A-Za-z0-9_.-]*(?:TOKEN|PASSWORD|SECRET|API_KEY|DATABASE_URL)[A-Za-z0-9_.-]*)=([^\s"'`]+)/giu, "$1=<redacted>")
|
|
.replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gu, "Bearer <redacted>")
|
|
.slice(0, 600);
|
|
}
|
|
|
|
function nodeRuntimeRenderToken(): string {
|
|
return `${process.pid}-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 10)}`.replace(/[^A-Za-z0-9_.-]/gu, "-");
|
|
}
|
|
|
|
function renderNodeRuntimeControlPlane(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult {
|
|
if (shouldRenderNodeRuntimeControlPlaneLocally(spec)) return renderNodeRuntimeControlPlaneLocal(spec, sourceCommit, timeoutSeconds);
|
|
return renderNodeRuntimeControlPlaneOnNode(spec, sourceCommit, timeoutSeconds);
|
|
}
|
|
|
|
function shouldRenderNodeRuntimeControlPlaneLocally(spec: HwlabRuntimeLaneSpec): boolean {
|
|
return hwlabRuntimeLaneSpec(spec.lane).nodeId !== spec.nodeId;
|
|
}
|
|
|
|
function yamlDependencyInstallScript(registry: string, fetchTimeoutSeconds: number, retries: number, context: string): string[] {
|
|
const timeoutSeconds = Math.max(15, Math.ceil(fetchTimeoutSeconds));
|
|
const retryCount = Math.max(0, Math.floor(retries));
|
|
const maxAttempts = retryCount + 1;
|
|
const safeContext = context.replace(/[^A-Za-z0-9_.-]/gu, "-");
|
|
return [
|
|
`yaml_registry=${shellQuote(registry)}`,
|
|
`yaml_fetch_timeout=${shellQuote(String(timeoutSeconds))}`,
|
|
`yaml_fetch_retries=${shellQuote(String(retryCount))}`,
|
|
`yaml_max_attempts=${shellQuote(String(maxAttempts))}`,
|
|
`yaml_dependency_context=${shellQuote(safeContext)}`,
|
|
"yaml_dependency_log() { echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"'\"$1\"'\",\"manager\":\"'\"${2:-}\"'\"}' >&2; }",
|
|
"yaml_prepare_node_package_proxy() {",
|
|
" yaml_http_proxy=\"${HTTP_PROXY:-${http_proxy:-${ALL_PROXY:-${all_proxy:-}}}}\"",
|
|
" yaml_https_proxy=\"${HTTPS_PROXY:-${https_proxy:-${yaml_http_proxy}}}\"",
|
|
" yaml_all_proxy=\"${ALL_PROXY:-${all_proxy:-${yaml_http_proxy}}}\"",
|
|
" if [ -n \"$yaml_http_proxy\" ]; then export HTTP_PROXY=\"$yaml_http_proxy\" http_proxy=\"$yaml_http_proxy\" npm_config_proxy=\"$yaml_http_proxy\"; fi",
|
|
" if [ -n \"$yaml_https_proxy\" ]; then export HTTPS_PROXY=\"$yaml_https_proxy\" https_proxy=\"$yaml_https_proxy\" npm_config_https_proxy=\"$yaml_https_proxy\"; fi",
|
|
" if [ -n \"$yaml_all_proxy\" ]; then export ALL_PROXY=\"$yaml_all_proxy\" all_proxy=\"$yaml_all_proxy\"; fi",
|
|
" export npm_config_registry=\"$yaml_registry\"",
|
|
" export BUN_CONFIG_REGISTRY=\"$yaml_registry\"",
|
|
" export npm_config_noproxy=\"${NO_PROXY:-${no_proxy:-}}\"",
|
|
" export npm_config_fetch_retries=\"$yaml_fetch_retries\"",
|
|
" export npm_config_fetch_retry_mintimeout=2000",
|
|
" export npm_config_fetch_retry_maxtimeout=16000",
|
|
" export npm_config_fetch_timeout=$((yaml_fetch_timeout * 1000))",
|
|
"}",
|
|
"yaml_run_dependency_manager() {",
|
|
" yaml_manager=\"$1\"; shift",
|
|
" yaml_attempt=1",
|
|
" yaml_delay=2",
|
|
" while [ \"$yaml_attempt\" -le \"$yaml_max_attempts\" ]; do",
|
|
" echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"attempt\",\"manager\":\"'\"$yaml_manager\"'\",\"attempt\":\"'\"$yaml_attempt/$yaml_max_attempts\"'\",\"proxy\":\"'\"${yaml_https_proxy:-$yaml_http_proxy}\"'\"}' >&2",
|
|
" if timeout \"$yaml_fetch_timeout\" \"$@\"; then return 0; fi",
|
|
" if [ \"$yaml_attempt\" -ge \"$yaml_max_attempts\" ]; then return 1; fi",
|
|
" echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"retrying\",\"manager\":\"'\"$yaml_manager\"'\",\"attempt\":\"'\"$yaml_attempt/$yaml_max_attempts\"'\",\"sleepSeconds\":'\"$yaml_delay\"'}' >&2",
|
|
" sleep \"$yaml_delay\"",
|
|
" yaml_delay=$((yaml_delay * 2))",
|
|
" yaml_attempt=$((yaml_attempt + 1))",
|
|
" done",
|
|
" return 1",
|
|
"}",
|
|
"yaml_npm_debug_log_tail() {",
|
|
" yaml_npm_log_dir=\"${HOME:-/tmp}/.npm/_logs\"",
|
|
" if [ ! -d \"$yaml_npm_log_dir\" ]; then return 0; fi",
|
|
" yaml_npm_log=\"$(find \"$yaml_npm_log_dir\" -type f -name '*debug*.log' | sort | tail -n 1 || true)\"",
|
|
" if [ -n \"$yaml_npm_log\" ] && [ -f \"$yaml_npm_log\" ]; then",
|
|
" echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"npm-debug-log\",\"path\":\"'\"$yaml_npm_log\"'\"}' >&2",
|
|
" tail -n 80 \"$yaml_npm_log\" >&2 || true",
|
|
" fi",
|
|
"}",
|
|
"if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1; then",
|
|
" yaml_prepare_node_package_proxy",
|
|
"fi",
|
|
"if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1 && command -v bun >/dev/null 2>&1; then",
|
|
" rm -rf node_modules/yaml",
|
|
" if yaml_run_dependency_manager bun bun add --no-save --ignore-scripts --registry \"$yaml_registry\" yaml@2.8.3; then",
|
|
" yaml_dependency_log installed bun",
|
|
" else",
|
|
" yaml_dependency_log bun-failed bun",
|
|
" fi",
|
|
"fi",
|
|
"if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1; then",
|
|
" rm -rf node_modules/yaml",
|
|
" command -v npm >/dev/null 2>&1 || { yaml_dependency_log failed missing-tool; exit 31; }",
|
|
" if yaml_run_dependency_manager npm npm install --package-lock=false --no-save --ignore-scripts --no-audit --no-fund --omit=dev --registry \"$yaml_registry\" yaml@2.8.3; then",
|
|
" yaml_dependency_log installed npm",
|
|
" else",
|
|
" yaml_dependency_log npm-failed npm",
|
|
" yaml_npm_debug_log_tail",
|
|
" exit 34",
|
|
" fi",
|
|
"fi",
|
|
"node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1 || { yaml_dependency_log failed unresolved; exit 34; }",
|
|
];
|
|
}
|
|
|
|
function renderNodeRuntimeControlPlaneOnNode(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult {
|
|
const token = nodeRuntimeRenderToken();
|
|
const renderDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-${shortSha(sourceCommit)}-${token}`;
|
|
const worktreeDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-${shortSha(sourceCommit)}-${token}`;
|
|
const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64");
|
|
const script = [
|
|
"set -eu",
|
|
runtimeLaneCicdRepoEnsureScript(spec),
|
|
`source_commit=${shellQuote(sourceCommit)}`,
|
|
`render_dir=${shellQuote(renderDir)}`,
|
|
`worktree_dir=${shellQuote(worktreeDir)}`,
|
|
`overlay_b64=${shellQuote(overlay)}`,
|
|
"cleanup_render_worktree() { rm -rf \"$worktree_dir\"; }",
|
|
"trap cleanup_render_worktree EXIT",
|
|
`test "$(git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/${spec.sourceBranch})" = "$source_commit"`,
|
|
"rm -rf \"$render_dir\" \"$worktree_dir\"",
|
|
"mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"",
|
|
"git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"",
|
|
"git -C \"$worktree_dir\" checkout --detach \"$source_commit\"",
|
|
"cd \"$worktree_dir\"",
|
|
...yamlDependencyInstallScript(spec.downloadProfile.npm.registry, spec.downloadProfile.npm.fetchTimeoutSeconds, spec.downloadProfile.npm.retries, "control-plane-render"),
|
|
"node - \"$overlay_b64\" <<'NODE'",
|
|
"const fs = require('fs');",
|
|
"const YAML = require('yaml');",
|
|
"const overlay = JSON.parse(Buffer.from(process.argv[2], 'base64').toString('utf8'));",
|
|
"const path = 'deploy/deploy.yaml';",
|
|
"const doc = YAML.parse(fs.readFileSync(path, 'utf8'));",
|
|
"doc.nodes = doc.nodes || {};",
|
|
"doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };",
|
|
"doc.lanes = doc.lanes || {};",
|
|
"const lane = doc.lanes[overlay.lane] || {};",
|
|
"const downloadStack = {",
|
|
" ...(lane.envRecipe?.downloadStack || {}),",
|
|
" httpProxy: overlay.dockerProxyHttp,",
|
|
" httpsProxy: overlay.dockerProxyHttps,",
|
|
" noProxy: overlay.dockerNoProxyList,",
|
|
"};",
|
|
"doc.lanes[overlay.lane] = {",
|
|
" ...lane,",
|
|
" node: overlay.nodeId,",
|
|
" sourceBranch: overlay.sourceBranch,",
|
|
" gitopsBranch: overlay.gitopsBranch,",
|
|
" namespace: overlay.runtimeNamespace,",
|
|
" endpoint: overlay.publicApiUrl,",
|
|
" publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },",
|
|
" artifactCatalog: overlay.catalogPath,",
|
|
" runtimePath: overlay.runtimePath,",
|
|
" imageTagMode: 'full',",
|
|
" sourceRepo: overlay.gitUrl,",
|
|
" externalPostgres: overlay.externalPostgres,",
|
|
" observability: overlay.observability,",
|
|
" envRecipe: { ...(lane.envRecipe || {}), downloadStack },",
|
|
"};",
|
|
"if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;",
|
|
"if (overlay.gitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.gitMirror;",
|
|
"fs.writeFileSync(path, YAML.stringify(doc));",
|
|
"NODE",
|
|
"if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi",
|
|
[
|
|
"node scripts/run-bun.mjs \"$render_script\"",
|
|
`--lane ${shellQuote(spec.lane)}`,
|
|
`--node ${shellQuote(spec.nodeId)}`,
|
|
`--gitops-root ${shellQuote(nodeRuntimeGitopsRoot(spec))}`,
|
|
`--catalog-path ${shellQuote(spec.catalogPath)}`,
|
|
"--image-tag-mode full",
|
|
`--source-revision ${shellQuote(sourceCommit)}`,
|
|
`--source-repo ${shellQuote(spec.gitUrl)}`,
|
|
`--source-branch ${shellQuote(spec.sourceBranch)}`,
|
|
`--gitops-branch ${shellQuote(spec.gitopsBranch)}`,
|
|
`--git-read-url ${shellQuote(spec.gitReadUrl)}`,
|
|
`--git-write-url ${shellQuote(spec.gitWriteUrl)}`,
|
|
`--registry-prefix ${shellQuote(spec.registryPrefix)}`,
|
|
`--runtime-endpoint ${shellQuote(spec.publicApiUrl)}`,
|
|
`--web-endpoint ${shellQuote(spec.publicWebUrl)}`,
|
|
`--out ${shellQuote(renderDir)}`,
|
|
].join(" "),
|
|
...nodeRuntimePipelinePostprocessScript(),
|
|
].join("\n");
|
|
return { result: runNodeHostScriptAsync(spec, script, timeoutSeconds, `${spec.nodeId.toLowerCase()}-${spec.lane}-render`), renderDir, worktreeDir, location: "node-host" };
|
|
}
|
|
|
|
function renderNodeRuntimeControlPlaneLocal(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult {
|
|
const token = nodeRuntimeRenderToken();
|
|
const renderDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-${shortSha(sourceCommit)}-${token}`;
|
|
const worktreeDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-${shortSha(sourceCommit)}-${token}`;
|
|
const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64");
|
|
const gitTimeoutSeconds = Math.max(30, spec.downloadProfile.git.timeoutSeconds);
|
|
const script = [
|
|
"set -eu",
|
|
`source_url=${shellQuote(spec.gitUrl)}`,
|
|
`source_branch=${shellQuote(spec.sourceBranch)}`,
|
|
`source_commit=${shellQuote(sourceCommit)}`,
|
|
`render_dir=${shellQuote(renderDir)}`,
|
|
`worktree_dir=${shellQuote(worktreeDir)}`,
|
|
`overlay_b64=${shellQuote(overlay)}`,
|
|
`git_timeout=${shellQuote(String(gitTimeoutSeconds))}`,
|
|
"run_git() { if command -v timeout >/dev/null 2>&1; then timeout \"$git_timeout\" git -c protocol.version=2 \"$@\"; else git -c protocol.version=2 \"$@\"; fi; }",
|
|
"rm -rf \"$render_dir\" \"$worktree_dir\"",
|
|
"mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"",
|
|
"echo \"phase=local-git-clone-worktree\" >&2",
|
|
"run_git clone --depth 1 --single-branch --branch \"$source_branch\" \"$source_url\" \"$worktree_dir\"",
|
|
"test \"$(git -C \"$worktree_dir\" rev-parse HEAD)\" = \"$source_commit\"",
|
|
"cd \"$worktree_dir\"",
|
|
"echo \"phase=local-install-yaml\" >&2",
|
|
...yamlDependencyInstallScript(spec.downloadProfile.npm.registry, spec.downloadProfile.npm.fetchTimeoutSeconds, spec.downloadProfile.npm.retries, "local-control-plane-render"),
|
|
"node - \"$overlay_b64\" <<'NODE'",
|
|
"const fs = require('fs');",
|
|
"const YAML = require('yaml');",
|
|
"const overlay = JSON.parse(Buffer.from(process.argv[2], 'base64').toString('utf8'));",
|
|
"const path = 'deploy/deploy.yaml';",
|
|
"const doc = YAML.parse(fs.readFileSync(path, 'utf8'));",
|
|
"doc.nodes = doc.nodes || {};",
|
|
"doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };",
|
|
"doc.lanes = doc.lanes || {};",
|
|
"const lane = doc.lanes[overlay.lane] || {};",
|
|
"const downloadStack = {",
|
|
" ...(lane.envRecipe?.downloadStack || {}),",
|
|
" httpProxy: overlay.dockerProxyHttp,",
|
|
" httpsProxy: overlay.dockerProxyHttps,",
|
|
" noProxy: overlay.dockerNoProxyList,",
|
|
"};",
|
|
"doc.lanes[overlay.lane] = {",
|
|
" ...lane,",
|
|
" node: overlay.nodeId,",
|
|
" sourceBranch: overlay.sourceBranch,",
|
|
" gitopsBranch: overlay.gitopsBranch,",
|
|
" namespace: overlay.runtimeNamespace,",
|
|
" endpoint: overlay.publicApiUrl,",
|
|
" publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },",
|
|
" artifactCatalog: overlay.catalogPath,",
|
|
" runtimePath: overlay.runtimePath,",
|
|
" imageTagMode: 'full',",
|
|
" sourceRepo: overlay.gitUrl,",
|
|
" externalPostgres: overlay.externalPostgres,",
|
|
" observability: overlay.observability,",
|
|
" envRecipe: { ...(lane.envRecipe || {}), downloadStack },",
|
|
"};",
|
|
"if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;",
|
|
"if (overlay.gitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.gitMirror;",
|
|
"fs.writeFileSync(path, YAML.stringify(doc));",
|
|
"NODE",
|
|
"if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi",
|
|
"echo \"phase=local-gitops-render\" >&2",
|
|
[
|
|
"node scripts/run-bun.mjs \"$render_script\"",
|
|
`--lane ${shellQuote(spec.lane)}`,
|
|
`--node ${shellQuote(spec.nodeId)}`,
|
|
`--gitops-root ${shellQuote(nodeRuntimeGitopsRoot(spec))}`,
|
|
`--catalog-path ${shellQuote(spec.catalogPath)}`,
|
|
"--image-tag-mode full",
|
|
`--source-revision ${shellQuote(sourceCommit)}`,
|
|
`--source-repo ${shellQuote(spec.gitUrl)}`,
|
|
`--source-branch ${shellQuote(spec.sourceBranch)}`,
|
|
`--gitops-branch ${shellQuote(spec.gitopsBranch)}`,
|
|
`--git-read-url ${shellQuote(spec.gitReadUrl)}`,
|
|
`--git-write-url ${shellQuote(spec.gitWriteUrl)}`,
|
|
`--registry-prefix ${shellQuote(spec.registryPrefix)}`,
|
|
`--runtime-endpoint ${shellQuote(spec.publicApiUrl)}`,
|
|
`--web-endpoint ${shellQuote(spec.publicWebUrl)}`,
|
|
`--out ${shellQuote(renderDir)}`,
|
|
].join(" "),
|
|
...nodeRuntimePipelinePostprocessScript(),
|
|
].join("\n");
|
|
return { result: runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: timeoutSeconds * 1000 }), renderDir, worktreeDir, location: "local" };
|
|
}
|
|
|
|
function nodeRuntimePipelinePostprocessScript(): string[] {
|
|
return [
|
|
"node - \"$render_dir\" \"$overlay_b64\" <<'NODE'",
|
|
"const fs = require('fs');",
|
|
"const path = require('path');",
|
|
"const vm = require('node:vm');",
|
|
"const renderDir = process.argv[2];",
|
|
"const overlay = JSON.parse(Buffer.from(process.argv[3], 'base64').toString('utf8'));",
|
|
"const pipelinePath = path.join(renderDir, overlay.tektonDir, 'pipeline.yaml');",
|
|
"let text = fs.readFileSync(pipelinePath, 'utf8');",
|
|
"let YAML = null;",
|
|
"try { YAML = require('yaml'); } catch {}",
|
|
"const escapeRegExp = (value) => String(value).replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');",
|
|
"const shellSingle = (value) => `'${String(value).replaceAll(\"'\", `'\"'\"'`)}'`;",
|
|
"const yamlString = (value) => JSON.stringify(String(value));",
|
|
"const proxyEnv = {",
|
|
" HTTP_PROXY: overlay.proxyHttp,",
|
|
" HTTPS_PROXY: overlay.proxyHttps,",
|
|
" ALL_PROXY: overlay.proxyAll,",
|
|
" NO_PROXY: overlay.noProxy,",
|
|
" http_proxy: overlay.proxyHttp,",
|
|
" https_proxy: overlay.proxyHttps,",
|
|
" all_proxy: overlay.proxyAll,",
|
|
" no_proxy: overlay.noProxy,",
|
|
"};",
|
|
"const dockerProxyEnv = {",
|
|
" HWLAB_NODE_PROXY_URL: overlay.dockerProxyHttp,",
|
|
" HWLAB_NODE_ALL_PROXY_URL: overlay.dockerProxyAll,",
|
|
" HWLAB_NODE_NO_PROXY: overlay.dockerNoProxy,",
|
|
"};",
|
|
"const stepEnv = { ...proxyEnv, ...dockerProxyEnv, ...(overlay.stepEnv || {}) };",
|
|
"function prepareSourceDependencyScript() {",
|
|
" return `prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"",
|
|
"echo '{\"event\":\"prepare-source-dependencies\",\"status\":\"not-required\",\"dependency\":\"yaml\",\"manager\":\"inline-service-id-parser\"}' >&2",
|
|
"ci_timing_emit prepare-source-dependencies succeeded \"$prepare_source_dependencies_started_ms\"`;",
|
|
"}",
|
|
"function validatePrepareSourceDependencyScript(script) {",
|
|
" const marker = 'NODE_UNIDESK_YAML_DEPENDENCY';",
|
|
" let offset = 0;",
|
|
" while (true) {",
|
|
" const markerStart = script.indexOf(\"node <<'\" + marker + \"'\", offset);",
|
|
" if (markerStart === -1) return;",
|
|
" const bodyStart = script.indexOf('\\n', markerStart);",
|
|
" if (bodyStart === -1) throw new Error('prepare-source dependency heredoc body missing newline');",
|
|
" const bodyEnd = script.indexOf('\\n' + marker, bodyStart + 1);",
|
|
" if (bodyEnd === -1) throw new Error('prepare-source dependency heredoc terminator missing');",
|
|
" const body = script.slice(bodyStart + 1, bodyEnd);",
|
|
" try { new vm.Script(body, { filename: 'NODE_UNIDESK_YAML_DEPENDENCY.js' }); } catch (error) { throw new Error(`generated prepare-source yaml dependency script is invalid: ${error.message}`); }",
|
|
" offset = bodyEnd + marker.length + 1;",
|
|
" }",
|
|
"}",
|
|
"function deployYamlOverlayScript() {",
|
|
" const runtimeOverlay = JSON.stringify({",
|
|
" nodeId: overlay.nodeId,",
|
|
" lane: overlay.lane,",
|
|
" sourceBranch: overlay.sourceBranch,",
|
|
" gitopsBranch: overlay.gitopsBranch,",
|
|
" gitopsRoot: overlay.gitopsRoot,",
|
|
" runtimePath: overlay.runtimePath,",
|
|
" runtimeRenderDir: overlay.runtimeRenderDir,",
|
|
" runtimeNamespace: overlay.runtimeNamespace,",
|
|
" catalogPath: overlay.catalogPath,",
|
|
" gitUrl: overlay.gitUrl,",
|
|
" publicWebUrl: overlay.publicWebUrl,",
|
|
" publicApiUrl: overlay.publicApiUrl,",
|
|
" externalPostgres: overlay.externalPostgres,",
|
|
" runtimeStore: overlay.runtimeStore,",
|
|
" observability: overlay.observability,",
|
|
" runtimeImageRewrites: overlay.runtimeImageRewrites,",
|
|
" dockerProxyHttp: overlay.dockerProxyHttp,",
|
|
" dockerProxyHttps: overlay.dockerProxyHttps,",
|
|
" dockerNoProxyList: overlay.dockerNoProxyList,",
|
|
" npmRegistry: overlay.npmRegistry,",
|
|
" npmFetchTimeoutMs: overlay.npmFetchTimeoutMs,",
|
|
" npmRetries: overlay.npmRetries,",
|
|
" });",
|
|
" return `node - <<'NODE_UNIDESK_DEPLOY_YAML_OVERLAY'",
|
|
"const fs = require('fs');",
|
|
"const YAML = require('yaml');",
|
|
"const overlay = ${runtimeOverlay};",
|
|
"const file = 'deploy/deploy.yaml';",
|
|
"if (!fs.existsSync(file)) {",
|
|
" console.error(JSON.stringify({ event: 'unidesk-deploy-yaml-overlay', ok: false, reason: 'deploy-yaml-missing', file }));",
|
|
" process.exit(45);",
|
|
"}",
|
|
"const doc = YAML.parse(fs.readFileSync(file, 'utf8'));",
|
|
"doc.nodes = doc.nodes || {};",
|
|
"doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };",
|
|
"doc.lanes = doc.lanes || {};",
|
|
"const lane = doc.lanes[overlay.lane] || {};",
|
|
"const envRecipe = lane.envRecipe || {};",
|
|
"const downloadStack = {",
|
|
" ...(envRecipe.downloadStack || {}),",
|
|
" httpProxy: overlay.dockerProxyHttp,",
|
|
" httpsProxy: overlay.dockerProxyHttps,",
|
|
" noProxy: overlay.dockerNoProxyList,",
|
|
"};",
|
|
"if (overlay.npmRegistry) downloadStack.npmRegistry = overlay.npmRegistry;",
|
|
"if (overlay.npmFetchTimeoutMs) downloadStack.npmFetchTimeoutMs = overlay.npmFetchTimeoutMs;",
|
|
"doc.lanes[overlay.lane] = {",
|
|
" ...lane,",
|
|
" node: overlay.nodeId,",
|
|
" sourceBranch: overlay.sourceBranch,",
|
|
" gitopsBranch: overlay.gitopsBranch,",
|
|
" namespace: overlay.runtimeNamespace,",
|
|
" endpoint: overlay.publicApiUrl,",
|
|
" publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },",
|
|
" artifactCatalog: overlay.catalogPath,",
|
|
" runtimePath: overlay.runtimePath,",
|
|
" imageTagMode: 'full',",
|
|
" sourceRepo: overlay.gitUrl,",
|
|
" externalPostgres: overlay.externalPostgres,",
|
|
" observability: overlay.observability,",
|
|
" envRecipe: { ...envRecipe, downloadStack },",
|
|
"};",
|
|
"if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;",
|
|
"fs.writeFileSync(file, YAML.stringify(doc));",
|
|
"console.error(JSON.stringify({ event: 'unidesk-deploy-yaml-overlay', ok: true, lane: overlay.lane, httpProxy: overlay.dockerProxyHttp, noProxyCount: overlay.dockerNoProxyList.length }));",
|
|
"NODE_UNIDESK_DEPLOY_YAML_OVERLAY`;",
|
|
"}",
|
|
"function runtimePathOverlayScript() {",
|
|
" const sourcePath = `${String(overlay.gitopsRoot || '').replace(/\\/+$/u, '')}/${overlay.runtimeRenderDir}`;",
|
|
" const targetPath = String(overlay.runtimePath || '');",
|
|
" if (!targetPath || sourcePath === targetPath) return '';",
|
|
" return [",
|
|
" `if [ ! -d ${shellSingle(targetPath)} ]; then`,",
|
|
" ` if [ ! -d ${shellSingle(sourcePath)} ]; then echo ${shellSingle(JSON.stringify({ event: 'unidesk-runtime-path-overlay', ok: false, reason: 'source-runtime-path-missing' }))} >&2; exit 46; fi`,",
|
|
" ` mkdir -p \"$(dirname ${shellSingle(targetPath)})\"`,",
|
|
" ` cp -a ${shellSingle(sourcePath)} ${shellSingle(targetPath)}` ,",
|
|
" ` echo ${shellSingle(JSON.stringify({ event: 'unidesk-runtime-path-overlay', ok: true }))} >&2`,",
|
|
" `fi`,",
|
|
" ].join('\\n');",
|
|
"}",
|
|
"function stepEnvBootstrapScript() {",
|
|
" const entries = Object.entries(overlay.stepEnv || {}).filter(([, value]) => value !== undefined && value !== null && String(value).length > 0);",
|
|
" const lines = ['# unidesk-step-env-bootstrap'];",
|
|
" for (const [name, value] of entries) lines.push(`export ${name}=${shellSingle(value)}`);",
|
|
" if (Object.prototype.hasOwnProperty.call(overlay.stepEnv || {}, 'HOME')) lines.push('mkdir -p \"$HOME\"');",
|
|
" if (Object.prototype.hasOwnProperty.call(overlay.stepEnv || {}, 'XDG_CONFIG_HOME')) lines.push('mkdir -p \"$XDG_CONFIG_HOME\"');",
|
|
" lines.push('ci_node_deps=\"${HWLAB_CI_NODE_DEPS:-/opt/hwlab-ci-node-deps/node_modules}\"');",
|
|
" lines.push('if [ -d \"$ci_node_deps\" ]; then');",
|
|
" lines.push(' if [ -d /workspace/source/repo ]; then ci_node_deps_target=/workspace/source/repo/node_modules; else ci_node_deps_target=./node_modules; fi');",
|
|
" lines.push(' mkdir -p \"$ci_node_deps_target\"');",
|
|
" lines.push(' for ci_node_dep in yaml; do if [ -d \"$ci_node_deps/$ci_node_dep\" ] && [ ! -e \"$ci_node_deps_target/$ci_node_dep\" ]; then ln -s \"$ci_node_deps/$ci_node_dep\" \"$ci_node_deps_target/$ci_node_dep\"; fi; done');",
|
|
" lines.push('fi');",
|
|
" return lines.join('\\n');",
|
|
"}",
|
|
"function runtimeGitopsPostprocessScript() {",
|
|
" const runtimeOverlay = JSON.stringify({",
|
|
" gitopsRoot: overlay.gitopsRoot,",
|
|
" runtimePath: overlay.runtimePath,",
|
|
" runtimeRenderDir: overlay.runtimeRenderDir,",
|
|
" runtimeNamespace: overlay.runtimeNamespace,",
|
|
" externalPostgres: overlay.externalPostgres,",
|
|
" publicExposure: overlay.publicExposure,",
|
|
" observability: overlay.observability,",
|
|
" runtimeImageRewrites: overlay.runtimeImageRewrites,",
|
|
" gitReadUrl: overlay.gitReadUrl,",
|
|
" publicWebUrl: overlay.publicWebUrl,",
|
|
" publicApiUrl: overlay.publicApiUrl,",
|
|
" });",
|
|
" return `node - <<'NODE_UNIDESK_RUNTIME_GITOPS_POSTPROCESS'",
|
|
"const fs = require('fs');",
|
|
"const path = require('path');",
|
|
"const crypto = require('crypto');",
|
|
"const YAML = require('yaml');",
|
|
"const overlay = ${runtimeOverlay};",
|
|
"const runtimePath = String(overlay.runtimePath || '');",
|
|
"const renderDir = String(overlay.runtimeRenderDir || '');",
|
|
"const legacyRuntimePath = runtimePath ? path.posix.join(path.posix.dirname(path.posix.dirname(runtimePath)), path.posix.basename(runtimePath)) : '';",
|
|
"const candidates = [...new Set([",
|
|
" runtimePath,",
|
|
" renderDir,",
|
|
" overlay.gitopsRoot && renderDir ? path.posix.join(String(overlay.gitopsRoot), renderDir) : '',",
|
|
" legacyRuntimePath,",
|
|
"].filter(Boolean))];",
|
|
"const sourcePath = candidates.find((candidate) => fs.existsSync(candidate)) || runtimePath;",
|
|
"if (!runtimePath || !fs.existsSync(sourcePath)) {",
|
|
" console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-postprocess', ok: false, reason: 'runtime-path-missing', runtimePath, sourcePath }));",
|
|
" process.exit(47);",
|
|
"}",
|
|
"if (sourcePath !== runtimePath) {",
|
|
" fs.rmSync(runtimePath, { recursive: true, force: true });",
|
|
" fs.mkdirSync(path.dirname(runtimePath), { recursive: true });",
|
|
" fs.cpSync(sourcePath, runtimePath, { recursive: true });",
|
|
" fs.rmSync(sourcePath, { recursive: true, force: true });",
|
|
"}",
|
|
"for (const candidate of candidates) {",
|
|
" if (candidate !== runtimePath && candidate !== sourcePath && candidate.endsWith('/' + path.posix.basename(runtimePath))) fs.rmSync(candidate, { recursive: true, force: true });",
|
|
"}",
|
|
"function readYaml(file) { return YAML.parse(fs.readFileSync(file, 'utf8')); }",
|
|
"function writeYaml(file, doc) { fs.writeFileSync(file, YAML.stringify(doc).trimEnd() + '\\\\n'); }",
|
|
"function listItems(doc) { return doc && doc.kind === 'List' && Array.isArray(doc.items) ? doc.items : [doc]; }",
|
|
"function normalizeList(items) { return { apiVersion: 'v1', kind: 'List', items }; }",
|
|
"function isObject(value) { return value && typeof value === 'object' && !Array.isArray(value); }",
|
|
"function yamlFiles(dir) {",
|
|
" const files = [];",
|
|
" for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {",
|
|
" const file = path.join(dir, entry.name);",
|
|
" if (entry.isDirectory()) files.push(...yamlFiles(file));",
|
|
" else if (entry.isFile() && /\\.ya?ml$/u.test(entry.name)) files.push(file);",
|
|
" }",
|
|
" return files;",
|
|
"}",
|
|
"function readYamlDocuments(file) { return YAML.parseAllDocuments(fs.readFileSync(file, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null); }",
|
|
"function writeYamlDocuments(file, docs) { fs.writeFileSync(file, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\\\n---\\\\n') + '\\\\n'); }",
|
|
"function podSpecFor(item) {",
|
|
" if (!isObject(item) || !isObject(item.spec)) return null;",
|
|
" if (item.kind === 'Pod') return item.spec;",
|
|
" if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;",
|
|
" if (item.kind === 'Job') return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;",
|
|
" if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.spec : null;",
|
|
" return null;",
|
|
"}",
|
|
"function templateMetadataFor(item) {",
|
|
" if (!isObject(item) || !isObject(item.spec)) return null;",
|
|
" if (item.kind === 'Pod') return item.metadata || null;",
|
|
" if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template ? item.spec.template.metadata : null;",
|
|
" if (item.kind === 'Job') return item.spec.template ? item.spec.template.metadata : null;",
|
|
" if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.metadata : null;",
|
|
" return null;",
|
|
"}",
|
|
"function stripMonitoringMetadata(metadata) {",
|
|
" if (!isObject(metadata)) return false;",
|
|
" let changed = false;",
|
|
" if (isObject(metadata.labels) && metadata.labels['hwlab.pikastech.local/monitoring'] !== undefined && metadata.labels['hwlab.pikastech.local/monitoring'] !== 'disabled') {",
|
|
" metadata.labels['hwlab.pikastech.local/monitoring'] = 'disabled';",
|
|
" changed = true;",
|
|
" }",
|
|
" if (isObject(metadata.annotations) && metadata.annotations['hwlab.pikastech.local/metrics-sidecar-sha256'] !== undefined) {",
|
|
" delete metadata.annotations['hwlab.pikastech.local/metrics-sidecar-sha256'];",
|
|
" changed = true;",
|
|
" }",
|
|
" return changed;",
|
|
"}",
|
|
"function containerHasVolumeMount(container, name) { return isObject(container) && Array.isArray(container.volumeMounts) && container.volumeMounts.some((mount) => mount && mount.name === name); }",
|
|
"function removeMetricsSidecar(podSpec) {",
|
|
" if (!isObject(podSpec)) return false;",
|
|
" let changed = false;",
|
|
" if (Array.isArray(podSpec.containers)) {",
|
|
" const next = podSpec.containers.filter((container) => !(isObject(container) && (container.name === 'hwlab-metrics' || (Array.isArray(container.command) && container.command.includes('/metrics/metrics-sidecar.mjs')))));",
|
|
" if (next.length !== podSpec.containers.length) { podSpec.containers = next; changed = true; }",
|
|
" }",
|
|
" for (const group of ['containers', 'initContainers']) {",
|
|
" for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {",
|
|
" if (!isObject(container) || !Array.isArray(container.volumeMounts)) continue;",
|
|
" const nextMounts = container.volumeMounts.filter((mount) => !(mount && mount.name === 'hwlab-metrics-sidecar'));",
|
|
" if (nextMounts.length !== container.volumeMounts.length) { container.volumeMounts = nextMounts; changed = true; }",
|
|
" }",
|
|
" }",
|
|
" if (Array.isArray(podSpec.volumes)) {",
|
|
" const nextVolumes = podSpec.volumes.filter((volume) => !(volume && volume.name === 'hwlab-metrics-sidecar'));",
|
|
" if (nextVolumes.length !== podSpec.volumes.length) { podSpec.volumes = nextVolumes; changed = true; }",
|
|
" }",
|
|
" return changed;",
|
|
"}",
|
|
"function envValue(container, name) {",
|
|
" if (!isObject(container) || !Array.isArray(container.env)) return undefined;",
|
|
" const item = container.env.find((env) => env && env.name === name);",
|
|
" return item ? item.value : undefined;",
|
|
"}",
|
|
"function setEnvValue(container, name, value) {",
|
|
" if (!isObject(container) || typeof value !== 'string') return false;",
|
|
" container.env = Array.isArray(container.env) ? container.env : [];",
|
|
" let item = container.env.find((env) => env && env.name === name);",
|
|
" if (!item) { item = { name }; container.env.push(item); }",
|
|
" const changed = item.value !== value || item.valueFrom !== undefined;",
|
|
" item.value = value;",
|
|
" delete item.valueFrom;",
|
|
" return changed;",
|
|
"}",
|
|
"function isEnvReuseContainer(container) { return envValue(container, 'HWLAB_RUNTIME_MODE') === 'env-reuse-git-mirror-checkout' || envValue(container, 'HWLAB_BOOT_SH') !== undefined || envValue(container, 'HWLAB_BOOT_COMMIT') !== undefined; }",
|
|
"function workloadName(item) { return item && item.metadata && item.metadata.labels && item.metadata.labels['app.kubernetes.io/name'] ? String(item.metadata.labels['app.kubernetes.io/name']) : String(item && item.metadata && item.metadata.name || ''); }",
|
|
"function expectedPublicEndpoint(item) { return workloadName(item) === 'hwlab-cloud-web' ? overlay.publicWebUrl : overlay.publicApiUrl; }",
|
|
"function startupProbeFrom(probe) {",
|
|
" const next = JSON.parse(JSON.stringify(probe));",
|
|
" next.periodSeconds = 10;",
|
|
" next.timeoutSeconds = Math.max(Number(next.timeoutSeconds || 0), 2);",
|
|
" next.failureThreshold = 30;",
|
|
" next.successThreshold = 1;",
|
|
" delete next.initialDelaySeconds;",
|
|
" return next;",
|
|
"}",
|
|
"function addEnvReuseStartupProbe(podSpec) {",
|
|
" if (!isObject(podSpec) || !Array.isArray(podSpec.containers)) return false;",
|
|
" let changed = false;",
|
|
" for (const container of podSpec.containers) {",
|
|
" if (!isObject(container) || !isEnvReuseContainer(container) || container.startupProbe) continue;",
|
|
" const sourceProbe = container.readinessProbe || container.livenessProbe;",
|
|
" if (!sourceProbe) continue;",
|
|
" container.startupProbe = startupProbeFrom(sourceProbe);",
|
|
" changed = true;",
|
|
" }",
|
|
" return changed;",
|
|
"}",
|
|
"function rewriteRuntimeImage(image) {",
|
|
" if (typeof image !== 'string') return image;",
|
|
" const match = (overlay.runtimeImageRewrites || []).find((item) => item && item.source === image);",
|
|
" return match ? match.target : image;",
|
|
"}",
|
|
"function patchRuntimeImages(podSpec) {",
|
|
" if (!isObject(podSpec)) return false;",
|
|
" let changed = false;",
|
|
" for (const group of ['containers', 'initContainers']) {",
|
|
" for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {",
|
|
" if (!isObject(container) || typeof container.image !== 'string') continue;",
|
|
" const nextImage = rewriteRuntimeImage(container.image);",
|
|
" if (nextImage !== container.image) { container.image = nextImage; changed = true; }",
|
|
" }",
|
|
" }",
|
|
" return changed;",
|
|
"}",
|
|
"function rewriteEnvValue(value) {",
|
|
" if (typeof value !== 'string') return value;",
|
|
" return value.replaceAll('http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git', overlay.gitReadUrl);",
|
|
"}",
|
|
"function patchGitReadUrlEnv(podSpec) {",
|
|
" if (!isObject(podSpec)) return false;",
|
|
" let changed = false;",
|
|
" for (const group of ['containers', 'initContainers']) {",
|
|
" for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {",
|
|
" if (!isObject(container) || !Array.isArray(container.env)) continue;",
|
|
" for (const env of container.env) {",
|
|
" if (!isObject(env) || typeof env.value !== 'string') continue;",
|
|
" const nextValue = rewriteEnvValue(env.value);",
|
|
" if (nextValue !== env.value) { env.value = nextValue; changed = true; }",
|
|
" }",
|
|
" }",
|
|
" }",
|
|
" return changed;",
|
|
"}",
|
|
"function patchRuntimeEnv(item, podSpec) {",
|
|
" if (!isObject(podSpec)) return { publicEndpointChanged: false, dbSslModeChanged: false };",
|
|
" let publicEndpointChanged = false;",
|
|
" let dbSslModeChanged = false;",
|
|
" const pg = overlay.externalPostgres;",
|
|
" for (const group of ['containers', 'initContainers']) {",
|
|
" for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {",
|
|
" if (!isObject(container)) continue;",
|
|
" if (envValue(container, 'HWLAB_PUBLIC_ENDPOINT') !== undefined) publicEndpointChanged = setEnvValue(container, 'HWLAB_PUBLIC_ENDPOINT', expectedPublicEndpoint(item)) || publicEndpointChanged;",
|
|
" if (pg && pg.sslmode && envValue(container, 'HWLAB_CLOUD_DB_SSL_MODE') !== undefined) dbSslModeChanged = setEnvValue(container, 'HWLAB_CLOUD_DB_SSL_MODE', pg.sslmode) || dbSslModeChanged;",
|
|
" }",
|
|
" }",
|
|
" return { publicEndpointChanged, dbSslModeChanged };",
|
|
"}",
|
|
"function patchRuntimeWorkloads() {",
|
|
" let observabilityChanged = false;",
|
|
" let startupProbeChanged = false;",
|
|
" let imageRewriteChanged = false;",
|
|
" let gitReadUrlChanged = false;",
|
|
" let publicEndpointChanged = false;",
|
|
" let dbSslModeChanged = false;",
|
|
" for (const file of yamlFiles(runtimePath)) {",
|
|
" if (path.basename(file) === 'kustomization.yaml') continue;",
|
|
" const docs = readYamlDocuments(file);",
|
|
" let changed = false;",
|
|
" for (const doc of docs) {",
|
|
" for (const item of listItems(doc).filter(Boolean)) {",
|
|
" if (!isObject(item)) continue;",
|
|
" if (overlay.observability && overlay.observability.prometheusOperator === false) {",
|
|
" const metadataChanged = stripMonitoringMetadata(item.metadata);",
|
|
" const templateChanged = stripMonitoringMetadata(templateMetadataFor(item));",
|
|
" const sidecarChanged = removeMetricsSidecar(podSpecFor(item));",
|
|
" const monitoringChanged = metadataChanged || templateChanged || sidecarChanged;",
|
|
" changed = monitoringChanged || changed;",
|
|
" observabilityChanged = observabilityChanged || monitoringChanged;",
|
|
" }",
|
|
" const probeChanged = addEnvReuseStartupProbe(podSpecFor(item));",
|
|
" changed = probeChanged || changed;",
|
|
" startupProbeChanged = startupProbeChanged || probeChanged;",
|
|
" const imageChanged = patchRuntimeImages(podSpecFor(item));",
|
|
" changed = imageChanged || changed;",
|
|
" imageRewriteChanged = imageRewriteChanged || imageChanged;",
|
|
" const gitUrlChanged = patchGitReadUrlEnv(podSpecFor(item));",
|
|
" changed = gitUrlChanged || changed;",
|
|
" gitReadUrlChanged = gitReadUrlChanged || gitUrlChanged;",
|
|
" const envChanged = patchRuntimeEnv(item, podSpecFor(item));",
|
|
" changed = envChanged.publicEndpointChanged || envChanged.dbSslModeChanged || changed;",
|
|
" publicEndpointChanged = publicEndpointChanged || envChanged.publicEndpointChanged;",
|
|
" dbSslModeChanged = dbSslModeChanged || envChanged.dbSslModeChanged;",
|
|
" }",
|
|
" }",
|
|
" if (changed) writeYamlDocuments(file, docs);",
|
|
" }",
|
|
" return { observabilityChanged, startupProbeChanged, imageRewriteChanged, gitReadUrlChanged, publicEndpointChanged, dbSslModeChanged };",
|
|
"}",
|
|
"function patchKustomization() {",
|
|
" const file = path.join(runtimePath, 'kustomization.yaml');",
|
|
" if (!fs.existsSync(file)) return false;",
|
|
" const doc = readYaml(file) || {};",
|
|
" const resources = Array.isArray(doc.resources) ? doc.resources : [];",
|
|
" const next = resources.filter((item) => !(overlay.observability && overlay.observability.prometheusOperator === false && item === 'observability.yaml'));",
|
|
" let changed = false;",
|
|
" if (next.length !== resources.length) { doc.resources = next; writeYaml(file, doc); changed = true; }",
|
|
" const observabilityFile = path.join(runtimePath, 'observability.yaml');",
|
|
" if (overlay.observability && overlay.observability.prometheusOperator === false && fs.existsSync(observabilityFile)) { fs.rmSync(observabilityFile, { force: true }); changed = true; }",
|
|
" return changed;",
|
|
"}",
|
|
"function patchExternalPostgres() {",
|
|
" const pg = overlay.externalPostgres;",
|
|
" if (!pg || !pg.serviceName) return false;",
|
|
" const file = path.join(runtimePath, 'external-postgres.yaml');",
|
|
" if (!fs.existsSync(file)) return false;",
|
|
" const doc = readYaml(file);",
|
|
" const items = listItems(doc).filter(Boolean);",
|
|
" let changed = false;",
|
|
" const endpointSliceName = String(pg.serviceName) + '-host';",
|
|
" for (const item of items) {",
|
|
" if (!item || typeof item !== 'object') continue;",
|
|
" item.metadata = item.metadata || {};",
|
|
" item.metadata.namespace = overlay.runtimeNamespace;",
|
|
" item.metadata.labels = item.metadata.labels || {};",
|
|
" item.metadata.labels['app.kubernetes.io/name'] = pg.serviceName;",
|
|
" if (item.kind === 'Service') {",
|
|
" item.metadata.name = pg.serviceName;",
|
|
" item.spec = item.spec || {};",
|
|
" item.spec.ports = [{ name: 'postgres', port: pg.port, targetPort: pg.port, protocol: 'TCP' }];",
|
|
" delete item.spec.selector;",
|
|
" changed = true;",
|
|
" }",
|
|
" if (item.kind === 'EndpointSlice') {",
|
|
" item.metadata.name = endpointSliceName;",
|
|
" item.metadata.labels['kubernetes.io/service-name'] = pg.serviceName;",
|
|
" item.addressType = 'IPv4';",
|
|
" item.ports = [{ name: 'postgres', port: pg.port, protocol: 'TCP' }];",
|
|
" item.endpoints = [{ addresses: [pg.endpointAddress], conditions: { ready: true } }];",
|
|
" changed = true;",
|
|
" }",
|
|
" }",
|
|
" if (changed) writeYaml(file, normalizeList(items));",
|
|
" return changed;",
|
|
"}",
|
|
"function patchHealthContract() {",
|
|
" const file = path.join(runtimePath, 'health-contract.yaml');",
|
|
" if (!fs.existsSync(file)) return false;",
|
|
" const doc = readYaml(file);",
|
|
" const items = listItems(doc).filter(Boolean);",
|
|
" let changed = false;",
|
|
" const pg = overlay.externalPostgres;",
|
|
" for (const item of items) {",
|
|
" if (!item || item.kind !== 'ConfigMap') continue;",
|
|
" item.data = item.data || {};",
|
|
" if (item.data.endpoint !== overlay.publicWebUrl) { item.data.endpoint = overlay.publicWebUrl; changed = true; }",
|
|
" const cloudApi = 'GET /health/live through ' + overlay.publicApiUrl;",
|
|
" if (item.data['cloud-api'] !== cloudApi) { item.data['cloud-api'] = cloudApi; changed = true; }",
|
|
" const cloudWeb = 'GET /health/live on ' + overlay.publicWebUrl + '; consumes cloud-api only';",
|
|
" if (item.data['cloud-web'] !== cloudWeb) { item.data['cloud-web'] = cloudWeb; changed = true; }",
|
|
" if (pg && pg.sslmode && typeof item.data['cloud-api-db'] === 'string') {",
|
|
" const next = item.data['cloud-api-db'].replace(/HWLAB_CLOUD_DB_SSL_MODE=[A-Za-z0-9_-]+/g, 'HWLAB_CLOUD_DB_SSL_MODE=' + pg.sslmode);",
|
|
" if (next !== item.data['cloud-api-db']) { item.data['cloud-api-db'] = next; changed = true; }",
|
|
" }",
|
|
" }",
|
|
" if (changed) writeYaml(file, normalizeList(items));",
|
|
" return changed;",
|
|
"}",
|
|
"function renderPublicExposureFrpcToml(exposure) {",
|
|
" return [",
|
|
" 'serverAddr = ' + JSON.stringify(String(exposure.serverAddr)),",
|
|
" 'serverPort = ' + Number(exposure.serverPort),",
|
|
" 'loginFailExit = true',",
|
|
" 'auth.token = \"{{ .Envs.HWLAB_FRP_TOKEN }}\"',",
|
|
" '',",
|
|
" '[[proxies]]',",
|
|
" 'name = ' + JSON.stringify(String(exposure.webProxy.name)),",
|
|
" 'type = \"tcp\"',",
|
|
" 'localIP = ' + JSON.stringify(String(exposure.webProxy.localIP)),",
|
|
" 'localPort = ' + Number(exposure.webProxy.localPort),",
|
|
" 'remotePort = ' + Number(exposure.webProxy.remotePort),",
|
|
" '',",
|
|
" '[[proxies]]',",
|
|
" 'name = ' + JSON.stringify(String(exposure.apiProxy.name)),",
|
|
" 'type = \"tcp\"',",
|
|
" 'localIP = ' + JSON.stringify(String(exposure.apiProxy.localIP)),",
|
|
" 'localPort = ' + Number(exposure.apiProxy.localPort),",
|
|
" 'remotePort = ' + Number(exposure.apiProxy.remotePort),",
|
|
" '',",
|
|
" ].join('\\\\n');",
|
|
"}",
|
|
"function setEnvFromSecret(container, name, secretName, secretKey) {",
|
|
" if (!isObject(container)) return false;",
|
|
" container.env = Array.isArray(container.env) ? container.env : [];",
|
|
" let item = container.env.find((env) => env && env.name === name);",
|
|
" if (!item) { item = { name }; container.env.push(item); }",
|
|
" const nextValueFrom = { secretKeyRef: { name: secretName, key: secretKey } };",
|
|
" const changed = item.value !== undefined || JSON.stringify(item.valueFrom || {}) !== JSON.stringify(nextValueFrom);",
|
|
" delete item.value;",
|
|
" item.valueFrom = nextValueFrom;",
|
|
" return changed;",
|
|
"}",
|
|
"function patchPublicExposure() {",
|
|
" const exposure = overlay.publicExposure;",
|
|
" if (!exposure || !exposure.enabled) return { configured: false, changed: false };",
|
|
" const file = path.join(runtimePath, 'node-frpc.yaml');",
|
|
" if (!fs.existsSync(file)) {",
|
|
" console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: false, reason: 'node-frpc-yaml-missing', filePath: file, hostname: exposure.hostname }));",
|
|
" process.exit(49);",
|
|
" }",
|
|
" const docs = readYamlDocuments(file);",
|
|
" const configName = String(overlay.runtimeNamespace) + '-frpc-config';",
|
|
" const deploymentName = String(overlay.runtimeNamespace) + '-frpc';",
|
|
" const configKey = String(exposure.secretKey || 'frpc.toml');",
|
|
" const tokenKey = String(exposure.tokenKey || 'token');",
|
|
" const toml = renderPublicExposureFrpcToml(exposure);",
|
|
" let changed = false;",
|
|
" let foundConfigMap = false;",
|
|
" let foundDeployment = false;",
|
|
" for (const doc of docs) {",
|
|
" for (const item of listItems(doc).filter(Boolean)) {",
|
|
" if (!isObject(item)) continue;",
|
|
" item.metadata = item.metadata || {};",
|
|
" if (item.kind === 'ConfigMap' && item.metadata.name === configName) {",
|
|
" foundConfigMap = true;",
|
|
" item.data = item.data || {};",
|
|
" if (item.data[configKey] !== toml) { item.data[configKey] = toml; changed = true; }",
|
|
" }",
|
|
" if (item.kind === 'Deployment' && item.metadata.name === deploymentName) {",
|
|
" foundDeployment = true;",
|
|
" item.spec = item.spec || {};",
|
|
" const nextStrategy = { type: 'Recreate' };",
|
|
" if (JSON.stringify(item.spec.strategy || {}) !== JSON.stringify(nextStrategy)) { item.spec.strategy = nextStrategy; changed = true; }",
|
|
" const podSpec = podSpecFor(item);",
|
|
" for (const container of Array.isArray(podSpec && podSpec.containers) ? podSpec.containers : []) {",
|
|
" if (!isObject(container)) continue;",
|
|
" if (container.name === 'frpc' || String(container.image || '').includes('frpc')) changed = setEnvFromSecret(container, 'HWLAB_FRP_TOKEN', exposure.secretName, tokenKey) || changed;",
|
|
" }",
|
|
" }",
|
|
" }",
|
|
" }",
|
|
" if (!foundConfigMap || !foundDeployment) {",
|
|
" console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: false, reason: 'frpc-resource-missing', filePath: file, configName, deploymentName, foundConfigMap, foundDeployment }));",
|
|
" process.exit(50);",
|
|
" }",
|
|
" if (changed) writeYamlDocuments(file, docs);",
|
|
" console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: true, applied: true, changed, filePath: file, hostname: exposure.hostname, serverAddr: exposure.serverAddr, serverPort: exposure.serverPort, webProxy: exposure.webProxy.name, apiProxy: exposure.apiProxy.name, configSha256: crypto.createHash('sha256').update(toml).digest('hex') }));",
|
|
" return { configured: true, changed, foundConfigMap, foundDeployment };",
|
|
"}",
|
|
"const kustomizationChanged = patchKustomization();",
|
|
"const runtimeWorkloadsChanged = patchRuntimeWorkloads();",
|
|
"const externalPostgresChanged = patchExternalPostgres();",
|
|
"const healthContractChanged = patchHealthContract();",
|
|
"const publicExposureChanged = patchPublicExposure();",
|
|
"console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-postprocess', ok: true, runtimePath, sourcePath, pathRelocated: sourcePath !== runtimePath, observabilityPrometheusOperator: overlay.observability ? overlay.observability.prometheusOperator : null, runtimeImageRewriteCount: (overlay.runtimeImageRewrites || []).length, kustomizationChanged, observabilityWorkloadsChanged: runtimeWorkloadsChanged.observabilityChanged, startupProbeChanged: runtimeWorkloadsChanged.startupProbeChanged, runtimeImageRewriteChanged: runtimeWorkloadsChanged.imageRewriteChanged, gitReadUrlChanged: runtimeWorkloadsChanged.gitReadUrlChanged, publicEndpointChanged: runtimeWorkloadsChanged.publicEndpointChanged, dbSslModeChanged: runtimeWorkloadsChanged.dbSslModeChanged, externalPostgresChanged, healthContractChanged, publicExposureChanged }));",
|
|
"NODE_UNIDESK_RUNTIME_GITOPS_POSTPROCESS`;",
|
|
"}",
|
|
"function runtimeGitopsVerifyScript() {",
|
|
" const runtimeOverlay = JSON.stringify({",
|
|
" runtimePath: overlay.runtimePath,",
|
|
" runtimeNamespace: overlay.runtimeNamespace,",
|
|
" externalPostgres: overlay.externalPostgres,",
|
|
" publicExposure: overlay.publicExposure,",
|
|
" observability: overlay.observability,",
|
|
" runtimeImageRewrites: overlay.runtimeImageRewrites,",
|
|
" gitReadUrl: overlay.gitReadUrl,",
|
|
" publicWebUrl: overlay.publicWebUrl,",
|
|
" publicApiUrl: overlay.publicApiUrl,",
|
|
" });",
|
|
" return `node - <<'NODE_UNIDESK_RUNTIME_GITOPS_VERIFY'",
|
|
"const fs = require('fs');",
|
|
"const path = require('path');",
|
|
"const YAML = require('yaml');",
|
|
"const overlay = ${runtimeOverlay};",
|
|
"const runtimePath = String(overlay.runtimePath || '');",
|
|
"function fail(reason, extra = {}) {",
|
|
" console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-verify', ok: false, reason, runtimePath, ...extra }));",
|
|
" process.exit(48);",
|
|
"}",
|
|
"if (!runtimePath || !fs.existsSync(runtimePath)) fail('runtime-path-missing');",
|
|
"function readYaml(file) { return YAML.parse(fs.readFileSync(file, 'utf8')); }",
|
|
"function listItems(doc) { return doc && doc.kind === 'List' && Array.isArray(doc.items) ? doc.items : [doc]; }",
|
|
"function isObject(value) { return value && typeof value === 'object' && !Array.isArray(value); }",
|
|
"function yamlFiles(dir) {",
|
|
" const files = [];",
|
|
" for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {",
|
|
" const file = path.join(dir, entry.name);",
|
|
" if (entry.isDirectory()) files.push(...yamlFiles(file));",
|
|
" else if (entry.isFile() && /\\.ya?ml$/u.test(entry.name)) files.push(file);",
|
|
" }",
|
|
" return files;",
|
|
"}",
|
|
"function readYamlDocuments(file) { return YAML.parseAllDocuments(fs.readFileSync(file, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null); }",
|
|
"function allItemsFromFile(file) { return readYamlDocuments(file).flatMap((doc) => listItems(doc).filter(Boolean)); }",
|
|
"function podSpecFor(item) {",
|
|
" if (!isObject(item) || !isObject(item.spec)) return null;",
|
|
" if (item.kind === 'Pod') return item.spec;",
|
|
" if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;",
|
|
" if (item.kind === 'Job') return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;",
|
|
" if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.spec : null;",
|
|
" return null;",
|
|
"}",
|
|
"function envValue(container, name) {",
|
|
" if (!isObject(container) || !Array.isArray(container.env)) return undefined;",
|
|
" const item = container.env.find((env) => env && env.name === name);",
|
|
" return item ? item.value : undefined;",
|
|
"}",
|
|
"function isEnvReuseContainer(container) { return envValue(container, 'HWLAB_RUNTIME_MODE') === 'env-reuse-git-mirror-checkout' || envValue(container, 'HWLAB_BOOT_SH') !== undefined || envValue(container, 'HWLAB_BOOT_COMMIT') !== undefined; }",
|
|
"function workloadName(item) { return item && item.metadata && item.metadata.labels && item.metadata.labels['app.kubernetes.io/name'] ? String(item.metadata.labels['app.kubernetes.io/name']) : String(item && item.metadata && item.metadata.name || ''); }",
|
|
"function expectedPublicEndpoint(item) { return workloadName(item) === 'hwlab-cloud-web' ? overlay.publicWebUrl : overlay.publicApiUrl; }",
|
|
"function workloadRef(item, file, container) { return { file, kind: item && item.kind, name: item && item.metadata && item.metadata.name, container: container && container.name }; }",
|
|
"function workloadChecks() {",
|
|
" const metricsRefs = [];",
|
|
" const missingStartupProbes = [];",
|
|
" const publicRuntimeImages = [];",
|
|
" const staleGitReadUrls = [];",
|
|
" const wrongPublicEndpoints = [];",
|
|
" const wrongDbSslModes = [];",
|
|
" const rewriteSources = new Set((overlay.runtimeImageRewrites || []).map((item) => item && item.source).filter(Boolean));",
|
|
" for (const file of yamlFiles(runtimePath)) {",
|
|
" if (path.basename(file) === 'kustomization.yaml') continue;",
|
|
" for (const doc of readYamlDocuments(file)) {",
|
|
" for (const item of listItems(doc).filter(Boolean)) {",
|
|
" const podSpec = podSpecFor(item);",
|
|
" if (!isObject(podSpec)) continue;",
|
|
" for (const container of Array.isArray(podSpec.containers) ? podSpec.containers : []) {",
|
|
" if (!isObject(container)) continue;",
|
|
" if (container.name === 'hwlab-metrics' || (Array.isArray(container.volumeMounts) && container.volumeMounts.some((mount) => mount && mount.name === 'hwlab-metrics-sidecar'))) metricsRefs.push(workloadRef(item, file, container));",
|
|
" if (isEnvReuseContainer(container) && (container.readinessProbe || container.livenessProbe) && !container.startupProbe) missingStartupProbes.push(workloadRef(item, file, container));",
|
|
" if (typeof container.image === 'string' && rewriteSources.has(container.image)) publicRuntimeImages.push({ ...workloadRef(item, file, container), image: container.image });",
|
|
" if (Array.isArray(container.env) && container.env.some((env) => env && typeof env.value === 'string' && env.value.includes('git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git') && env.value !== overlay.gitReadUrl)) staleGitReadUrls.push(workloadRef(item, file, container));",
|
|
" const publicEndpoint = envValue(container, 'HWLAB_PUBLIC_ENDPOINT');",
|
|
" if (publicEndpoint !== undefined && publicEndpoint !== expectedPublicEndpoint(item)) wrongPublicEndpoints.push({ ...workloadRef(item, file, container), value: publicEndpoint, expected: expectedPublicEndpoint(item) });",
|
|
" const dbSslMode = envValue(container, 'HWLAB_CLOUD_DB_SSL_MODE');",
|
|
" if (overlay.externalPostgres && overlay.externalPostgres.sslmode && dbSslMode !== undefined && dbSslMode !== overlay.externalPostgres.sslmode) wrongDbSslModes.push({ ...workloadRef(item, file, container), value: dbSslMode, expected: overlay.externalPostgres.sslmode });",
|
|
" }",
|
|
" if (Array.isArray(podSpec.volumes) && podSpec.volumes.some((volume) => volume && volume.name === 'hwlab-metrics-sidecar')) metricsRefs.push(workloadRef(item, file, { name: 'volume/hwlab-metrics-sidecar' }));",
|
|
" }",
|
|
" }",
|
|
" }",
|
|
" return { metricsRefs, missingStartupProbes, publicRuntimeImages, staleGitReadUrls, wrongPublicEndpoints, wrongDbSslModes };",
|
|
"}",
|
|
"const checks = [];",
|
|
"const workloadCheck = workloadChecks();",
|
|
"const kustomizationPath = path.join(runtimePath, 'kustomization.yaml');",
|
|
"if (overlay.observability && overlay.observability.prometheusOperator === false) {",
|
|
" if (!fs.existsSync(kustomizationPath)) fail('kustomization-missing');",
|
|
" const resources = readYaml(kustomizationPath).resources || [];",
|
|
" if (resources.includes('observability.yaml')) fail('observability-resource-still-rendered', { file: kustomizationPath });",
|
|
" if (workloadCheck.metricsRefs.length > 0) fail('observability-sidecar-still-rendered', { refs: workloadCheck.metricsRefs.slice(0, 12), count: workloadCheck.metricsRefs.length });",
|
|
" checks.push('observability-disabled');",
|
|
"}",
|
|
"if (workloadCheck.missingStartupProbes.length > 0) fail('env-reuse-startup-probe-missing', { refs: workloadCheck.missingStartupProbes.slice(0, 12), count: workloadCheck.missingStartupProbes.length });",
|
|
"checks.push('env-reuse-startup-probes');",
|
|
"if (workloadCheck.publicRuntimeImages.length > 0) fail('runtime-image-rewrite-missing', { refs: workloadCheck.publicRuntimeImages.slice(0, 12), count: workloadCheck.publicRuntimeImages.length });",
|
|
"if ((overlay.runtimeImageRewrites || []).length > 0) checks.push('runtime-image-rewrites');",
|
|
"if (workloadCheck.staleGitReadUrls.length > 0) fail('runtime-git-read-url-stale', { refs: workloadCheck.staleGitReadUrls.slice(0, 12), count: workloadCheck.staleGitReadUrls.length, expected: overlay.gitReadUrl });",
|
|
"checks.push('runtime-git-read-url');",
|
|
"if (workloadCheck.wrongPublicEndpoints.length > 0) fail('runtime-public-endpoint-mismatch', { refs: workloadCheck.wrongPublicEndpoints.slice(0, 12), count: workloadCheck.wrongPublicEndpoints.length });",
|
|
"checks.push('runtime-public-endpoint');",
|
|
"if (workloadCheck.wrongDbSslModes.length > 0) fail('runtime-db-ssl-mode-mismatch', { refs: workloadCheck.wrongDbSslModes.slice(0, 12), count: workloadCheck.wrongDbSslModes.length });",
|
|
"if (overlay.externalPostgres && overlay.externalPostgres.sslmode) checks.push('runtime-db-ssl-mode');",
|
|
"const pg = overlay.externalPostgres;",
|
|
"if (pg && pg.serviceName) {",
|
|
" const file = path.join(runtimePath, 'external-postgres.yaml');",
|
|
" if (!fs.existsSync(file)) fail('external-postgres-missing');",
|
|
" const items = listItems(readYaml(file)).filter(Boolean);",
|
|
" const service = items.find((item) => item && item.kind === 'Service' && item.metadata && item.metadata.name === pg.serviceName);",
|
|
" const endpointSlice = items.find((item) => item && item.kind === 'EndpointSlice' && item.metadata && item.metadata.name === String(pg.serviceName) + '-host');",
|
|
" if (!service) fail('external-postgres-service-missing', { expected: pg.serviceName });",
|
|
" if (!endpointSlice) fail('external-postgres-endpointslice-missing', { expected: String(pg.serviceName) + '-host' });",
|
|
" const servicePort = service.spec && Array.isArray(service.spec.ports) && service.spec.ports[0] ? service.spec.ports[0].port : null;",
|
|
" const endpointPort = Array.isArray(endpointSlice.ports) && endpointSlice.ports[0] ? endpointSlice.ports[0].port : null;",
|
|
" const endpointAddress = Array.isArray(endpointSlice.endpoints) && endpointSlice.endpoints[0] && Array.isArray(endpointSlice.endpoints[0].addresses) ? endpointSlice.endpoints[0].addresses[0] : null;",
|
|
" if (Number(servicePort) !== Number(pg.port) || Number(endpointPort) !== Number(pg.port)) fail('external-postgres-port-mismatch', { servicePort, endpointPort, expectedPort: pg.port });",
|
|
" if (String(endpointAddress) !== String(pg.endpointAddress)) fail('external-postgres-address-mismatch', { endpointAddress, expectedEndpointAddress: pg.endpointAddress });",
|
|
" checks.push('external-postgres-bridge');",
|
|
"}",
|
|
"const exposure = overlay.publicExposure;",
|
|
"if (exposure && exposure.enabled) {",
|
|
" const file = path.join(runtimePath, 'node-frpc.yaml');",
|
|
" if (!fs.existsSync(file)) fail('public-exposure-frpc-missing');",
|
|
" const items = allItemsFromFile(file);",
|
|
" const configName = String(overlay.runtimeNamespace) + '-frpc-config';",
|
|
" const deploymentName = String(overlay.runtimeNamespace) + '-frpc';",
|
|
" const configKey = String(exposure.secretKey || 'frpc.toml');",
|
|
" const tokenKey = String(exposure.tokenKey || 'token');",
|
|
" const configMap = items.find((item) => item && item.kind === 'ConfigMap' && item.metadata && item.metadata.name === configName);",
|
|
" const deployment = items.find((item) => item && item.kind === 'Deployment' && item.metadata && item.metadata.name === deploymentName);",
|
|
" if (!configMap) fail('public-exposure-frpc-configmap-missing', { expected: configName });",
|
|
" if (!deployment) fail('public-exposure-frpc-deployment-missing', { expected: deploymentName });",
|
|
" const toml = configMap.data && configMap.data[configKey];",
|
|
" if (typeof toml !== 'string') fail('public-exposure-frpc-config-missing', { expectedKey: configKey });",
|
|
" for (const expected of [String(exposure.serverAddr), String(exposure.serverPort), String(exposure.webProxy.name), String(exposure.webProxy.remotePort), String(exposure.apiProxy.name), String(exposure.apiProxy.remotePort), 'HWLAB_FRP_TOKEN']) {",
|
|
" if (!toml.includes(expected)) fail('public-exposure-frpc-config-mismatch', { expected });",
|
|
" }",
|
|
" const podSpec = podSpecFor(deployment);",
|
|
" const containers = Array.isArray(podSpec && podSpec.containers) ? podSpec.containers : [];",
|
|
" const strategyType = deployment.spec && deployment.spec.strategy && deployment.spec.strategy.type;",
|
|
" if (strategyType !== 'Recreate') fail('public-exposure-frpc-strategy-mismatch', { expected: 'Recreate', actual: strategyType || null });",
|
|
" const frpc = containers.find((container) => container && (container.name === 'frpc' || String(container.image || '').includes('frpc')));",
|
|
" const env = frpc && Array.isArray(frpc.env) ? frpc.env.find((item) => item && item.name === 'HWLAB_FRP_TOKEN') : null;",
|
|
" const secretRef = env && env.valueFrom && env.valueFrom.secretKeyRef;",
|
|
" if (!secretRef || secretRef.name !== exposure.secretName || secretRef.key !== tokenKey) fail('public-exposure-frpc-token-env-mismatch', { expectedSecret: exposure.secretName, expectedKey: tokenKey });",
|
|
" checks.push('public-exposure-frpc');",
|
|
"}",
|
|
"console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-verify', ok: true, runtimePath, checks }));",
|
|
"NODE_UNIDESK_RUNTIME_GITOPS_VERIFY`;",
|
|
"}",
|
|
"function patchScript(script) {",
|
|
" let result = String(script || '');",
|
|
" const bootstrap = stepEnvBootstrapScript();",
|
|
" if (bootstrap && !result.includes('unidesk-step-env-bootstrap')) result = `${bootstrap}\\n${result}`;",
|
|
" const inlineDeployServiceIdParser = `const deployText = fs.readFileSync(deployPath, 'utf8');\\nconst deploy = { services: [...deployText.matchAll(/(?:^|\\\\n)\\\\s*serviceId:\\\\s*[\\\"']?([^\\\"'\\\\n#]+)[\\\"']?/g)].map((match) => ({ serviceId: match[1].trim() })) };`;",
|
|
" result = result.split('import { readStructuredFile } from \"./scripts/src/structured-config.mjs\";\\n').join('');",
|
|
" result = result.split('const deploy = await readStructuredFile(process.cwd(), deployPath);').join(inlineDeployServiceIdParser);",
|
|
" if (result.includes('npm run gitops:ts:check')) {",
|
|
" result = result.replace(/\\n[ \\t]*npm run gitops:ts:check\\n/g, '\\n echo \\'{\"event\":\"unidesk-node-contract-check\",\"status\":\"skipped\",\"reason\":\"d601-yaml-render-check-replaces-tsc-gate\"}\\' >&2\\n');",
|
|
" }",
|
|
" const isNodeContractCheck = result.includes('\\\"event\\\":\\\"unidesk-node-contract-check\\\"');",
|
|
" if (isNodeContractCheck) {",
|
|
" result = result.replace(/(^|\\n)([ \\t]*)node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs[^\\n]*(?=\\n|$)/g, '$1$2echo \\'{\"event\":\"unidesk-node-contract-check\",\"status\":\"skipped\",\"reason\":\"d601-yaml-render-check-disabled-gitops-render\"}\\' >&2');",
|
|
" }",
|
|
" const prepareSourceDependencyPattern = new RegExp(String.raw`prepare_source_dependencies_started_ms=\"\\$\\(ci_now_ms\\)\"\\nif node -e 'require\\.resolve\\(\"yaml\"\\)'[\\s\\S]*?\\nci_timing_emit prepare-source-dependencies succeeded \"\\$prepare_source_dependencies_started_ms\"`, 'g');",
|
|
" if (result.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"')) {",
|
|
" result = result.replace(prepareSourceDependencyPattern, prepareSourceDependencyScript());",
|
|
" }",
|
|
" const artifactPublishNeedle = 'node scripts/artifact-publish.mjs --publish';",
|
|
" if (result.includes(artifactPublishNeedle) && !result.includes('unidesk-deploy-yaml-overlay')) {",
|
|
" result = result.replace(artifactPublishNeedle, `${deployYamlOverlayScript()}\\n${artifactPublishNeedle}`);",
|
|
" }",
|
|
" const gitopsRenderNeedle = 'node scripts/run-bun.mjs scripts/gitops-render.mjs';",
|
|
" if (!isNodeContractCheck && result.includes(gitopsRenderNeedle) && !result.includes('unidesk-deploy-yaml-overlay')) {",
|
|
" result = result.replace(gitopsRenderNeedle, `${deployYamlOverlayScript()}\\n${gitopsRenderNeedle}`);",
|
|
" }",
|
|
" result = result.replaceAll('node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808', `node /tmp/hwlab-github-proxy-connect.mjs ${overlay.gitSshProxyHost || '127.0.0.1'} ${overlay.gitSshProxyPort || 10808}`);",
|
|
" result = result.replace(/--git-read-url '[^']*'/g, `--git-read-url ${shellSingle(overlay.gitReadUrl)}`);",
|
|
" result = result.replace(/--git-write-url '[^']*'/g, `--git-write-url ${shellSingle(overlay.gitWriteUrl)}`);",
|
|
" result = result.replace(/--runtime-endpoint '[^']*'/g, `--runtime-endpoint ${shellSingle(overlay.publicApiUrl)}`);",
|
|
" result = result.replace(/--web-endpoint '[^']*'/g, `--web-endpoint ${shellSingle(overlay.publicWebUrl)}`);",
|
|
" result = result.replace(/--gitops-root \"\\$gitops_root\"/g, `--gitops-root ${JSON.stringify(overlay.gitopsRoot)}`);",
|
|
" result = result.replace(/--out \"\\$gitops_root\"/g, `--out ${JSON.stringify(overlay.gitopsRoot)}`);",
|
|
" const legacyRuntimePath = (() => {",
|
|
" const runtimePath = String(overlay.runtimePath || '');",
|
|
" const parts = runtimePath.split('/').filter(Boolean);",
|
|
" if (parts.length < 2) return '';",
|
|
" const leaf = parts.at(-1);",
|
|
" const parent = parts.slice(0, -2).join('/');",
|
|
" return parent && leaf ? `${parent}/${leaf}` : '';",
|
|
" })();",
|
|
" if (legacyRuntimePath && legacyRuntimePath !== overlay.runtimePath) {",
|
|
" result = result.replace('runtime_path=\"$(params.runtime-path)\"', `runtime_path=\"$(params.runtime-path)\"\\nunidesk_legacy_runtime_path=${shellSingle(legacyRuntimePath)}`);",
|
|
" result = result.replace('rm -rf \"$runtime_path\" \"$catalog_path\"; else rm -rf deploy/gitops/node \"$catalog_path\"; fi', 'rm -rf \"$runtime_path\" \"$catalog_path\" \"$unidesk_legacy_runtime_path\"; else rm -rf deploy/gitops/node \"$catalog_path\"; fi');",
|
|
" result = result.replace('git add \"$catalog_path\" \"$runtime_path\"', 'git add \"$catalog_path\" \"$runtime_path\"\\n if [ -n \"${unidesk_legacy_runtime_path:-}\" ]; then git add -A \"$unidesk_legacy_runtime_path\" || true; fi');",
|
|
" }",
|
|
" if (bootstrap && result.includes('git config --global') && !result.includes('unidesk-step-env-bootstrap')) {",
|
|
" result = result.replace(/\\n([ \\t]*)git config --global/g, (match, indent) => `\\n${indent}${bootstrap.split('\\n').join(`\\n${indent}`)}\\n${indent}git config --global`);",
|
|
" }",
|
|
" result = result.replace(/node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs([^\\n]*?)--use-deploy-images/g, (match) => {",
|
|
" let next = match;",
|
|
" if (!next.includes('--gitops-root ')) next = next.replace(' --use-deploy-images', ` --gitops-root ${JSON.stringify(overlay.gitopsRoot)} --use-deploy-images`);",
|
|
" if (!next.includes('--out ')) next = next.replace(' --use-deploy-images', ` --out ${JSON.stringify(overlay.gitopsRoot)} --use-deploy-images`);",
|
|
" return next;",
|
|
" });",
|
|
" result = result.replace(/(node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs[^\\n]*--use-deploy-images[^\\n]*)/g, (match) => {",
|
|
" if (match.includes('--check')) return runtimeGitopsVerifyScript();",
|
|
" return `${match}\\n${[runtimePathOverlayScript(), runtimeGitopsPostprocessScript()].filter(Boolean).join('\\n')}`;",
|
|
" });",
|
|
" result = result.replace(/node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs([^\\n]*?)--out \"\\$render_check_dir\"/g, (match) => match.includes('--gitops-root ') ? match : `${match} --gitops-root ${JSON.stringify(overlay.gitopsRoot)} --node ${shellSingle(overlay.nodeId)} --git-read-url ${shellSingle(overlay.gitReadUrl)} --git-write-url ${shellSingle(overlay.gitWriteUrl)} --runtime-endpoint ${shellSingle(overlay.publicApiUrl)} --web-endpoint ${shellSingle(overlay.publicWebUrl)}`);",
|
|
" validatePrepareSourceDependencyScript(result);",
|
|
" return result;",
|
|
"}",
|
|
"function patchManifestObject(doc) {",
|
|
" if (!doc || typeof doc !== 'object') return false;",
|
|
" if (doc.kind !== 'Pipeline' || !doc.spec) return false;",
|
|
" const defaults = {",
|
|
" 'git-url': overlay.gitUrl,",
|
|
" 'git-read-url': overlay.gitReadUrl,",
|
|
" 'git-write-url': overlay.gitWriteUrl,",
|
|
" 'catalog-path': overlay.catalogPath,",
|
|
" 'runtime-path': overlay.runtimePath,",
|
|
" 'registry-prefix': overlay.registryPrefix,",
|
|
" };",
|
|
" for (const param of doc.spec?.params || []) {",
|
|
" if (Object.prototype.hasOwnProperty.call(defaults, param.name)) param.default = defaults[param.name];",
|
|
" }",
|
|
" doc.metadata = doc.metadata || {};",
|
|
" doc.metadata.annotations = doc.metadata.annotations || {};",
|
|
" doc.metadata.annotations['hwlab.pikastech.local/download-profile'] = overlay.downloadProfileId;",
|
|
" doc.metadata.annotations['hwlab.pikastech.local/network-profile'] = overlay.networkProfileId;",
|
|
" for (const task of doc.spec?.tasks || []) {",
|
|
" for (const sidecar of task.taskSpec?.sidecars || []) {",
|
|
" if (overlay.buildkitSidecarImage && typeof sidecar.image === 'string' && sidecar.image.includes('buildkit')) sidecar.image = overlay.buildkitSidecarImage;",
|
|
" }",
|
|
" for (const step of task.taskSpec?.steps || []) {",
|
|
" if (step.image === overlay.toolsImage && overlay.toolsImagePullPolicy) step.imagePullPolicy = overlay.toolsImagePullPolicy;",
|
|
" if (Array.isArray(step.env)) {",
|
|
" for (const env of step.env) {",
|
|
" if (Object.prototype.hasOwnProperty.call(stepEnv, env.name) && stepEnv[env.name] !== undefined) env.value = stepEnv[env.name];",
|
|
" }",
|
|
" }",
|
|
" step.env = Array.isArray(step.env) ? step.env : [];",
|
|
" const existingEnv = new Set(step.env.map((env) => env.name));",
|
|
" for (const [name, value] of Object.entries(stepEnv)) {",
|
|
" if (value !== undefined && !existingEnv.has(name)) step.env.push({ name, value });",
|
|
" }",
|
|
" if (typeof step.script === 'string') step.script = patchScript(step.script);",
|
|
" }",
|
|
" }",
|
|
" return true;",
|
|
"}",
|
|
"function patchStructuredPipeline() {",
|
|
" try {",
|
|
" const doc = JSON.parse(text);",
|
|
" if (!patchManifestObject(doc)) return false;",
|
|
" text = JSON.stringify(doc, null, 2) + '\\n';",
|
|
" return true;",
|
|
" } catch {}",
|
|
" if (YAML) {",
|
|
" try {",
|
|
" const docs = YAML.parseAllDocuments(text).map((document) => document.toJS()).filter((doc) => doc !== null);",
|
|
" const changed = docs.some((doc) => patchManifestObject(doc));",
|
|
" if (!changed) return false;",
|
|
" text = docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n';",
|
|
" return true;",
|
|
" } catch {}",
|
|
" }",
|
|
" return false;",
|
|
"}",
|
|
"function patchGitMirrorTransportYaml() {",
|
|
" const mirror = overlay.gitMirror || {};",
|
|
" const transport = mirror.githubTransport || {};",
|
|
" if (transport.mode !== 'https') return;",
|
|
" if (!YAML) throw new Error('yaml module is required to patch git-mirror githubTransport=https');",
|
|
" const requireString = (pathName, value) => {",
|
|
" if (typeof value !== 'string' || value.length === 0) throw new Error('overlay.' + pathName + ' is required for git-mirror githubTransport=https');",
|
|
" return value;",
|
|
" };",
|
|
" const repository = requireString('gitMirror.sourceRepository', mirror.sourceRepository);",
|
|
" const configMapName = requireString('gitMirror.syncConfigMapName', mirror.syncConfigMapName);",
|
|
" const tokenSecretName = requireString('gitMirror.githubTransport.tokenSecretName', transport.tokenSecretName);",
|
|
" const tokenSecretKey = requireString('gitMirror.githubTransport.tokenSecretKey', transport.tokenSecretKey);",
|
|
" const tokenSourceRef = requireString('gitMirror.githubTransport.tokenSourceRef', transport.tokenSourceRef);",
|
|
" const username = typeof transport.username === 'string' && transport.username ? transport.username : 'x-access-token';",
|
|
" const remoteUrl = `https://github.com/${repository}.git`;",
|
|
" const proxySummary = `transport=https proxy=HTTP_PROXY authSecret=${tokenSecretName} authKey=${tokenSecretKey} authSourceRef=${tokenSourceRef} source=yaml`;",
|
|
" const askpassBlock = [",
|
|
" 'if [ -z \"${GITHUB_TOKEN:-}\" ]; then echo \\'hwlab git-mirror https auth: missing GITHUB_TOKEN secret env\\' >&2; exit 64; fi',",
|
|
" \"cat > /tmp/hwlab-git-askpass.sh <<'SH_ASKPASS'\",",
|
|
" '#!/bin/sh',",
|
|
" 'case \"$1\" in',",
|
|
" ' *Username*) printf \\'%s\\\\n\\' \"${GITHUB_USERNAME:-x-access-token}\" ;;',",
|
|
" ' *Password*) printf \\'%s\\\\n\\' \"$GITHUB_TOKEN\" ;;',",
|
|
" \" *) printf '\\\\n' ;;\",",
|
|
" 'esac',",
|
|
" 'SH_ASKPASS',",
|
|
" 'chmod 0700 /tmp/hwlab-git-askpass.sh',",
|
|
" `export GITHUB_USERNAME=${shellSingle(username)}`,",
|
|
" 'export GIT_ASKPASS=/tmp/hwlab-git-askpass.sh',",
|
|
" 'export GIT_TERMINAL_PROMPT=0',",
|
|
" 'unset GIT_SSH',",
|
|
" 'unset GIT_SSH_COMMAND',",
|
|
" ].join('\\n');",
|
|
" function patchScript(script, key) {",
|
|
" let next = String(script || '');",
|
|
" if (next.length === 0) throw new Error(`generated git-mirror ConfigMap ${configMapName} missing ${key}`);",
|
|
" let remoteReplaced = false;",
|
|
" next = next.replace(/repo_url=(?:\"[^\"]*\"|'[^']*')/g, () => { remoteReplaced = true; return `repo_url=${shellSingle(remoteUrl)}`; });",
|
|
" next = next.replace(/remote=(?:\"[^\"]*\"|'[^']*')/g, () => { remoteReplaced = true; return `remote=${shellSingle(remoteUrl)}`; });",
|
|
" if (!remoteReplaced) throw new Error(`generated git-mirror ${key} missing remote url assignment for githubTransport=https`);",
|
|
" next = next.replace(/transport=ssh ssh=GIT_SSH-wrapper source=yaml/g, proxySummary);",
|
|
" next = next.replace(/ssh=GIT_SSH-wrapper source=yaml/g, proxySummary);",
|
|
" next = next.replace(/mkdir -p ([^\\n]*?) \\/root\\/\\.ssh/g, 'mkdir -p $1');",
|
|
" next = next.replace(/\\n[ \\t]*mkdir -p \\/root\\/\\.ssh\\n/g, '\\n');",
|
|
" next = next.replace(/\\n[ \\t]*cp \\/git-ssh\\/ssh-privatekey[^\\n]*\\n[ \\t]*chmod 0?400[^\\n]*\\n/g, '\\n');",
|
|
" next = next.replace(/\\n[ \\t]*cat > \\/tmp\\/hwlab-github-proxy-connect\\.(?:mjs|cjs) <<'NODE_PROXY'[\\s\\S]*?\\nNODE_PROXY\\n[ \\t]*chmod [^\\n]*\\/tmp\\/hwlab-github-proxy-connect\\.(?:mjs|cjs)\\n/g, '\\n');",
|
|
" next = next.replace(/\\n[ \\t]*cat > \\/tmp\\/hwlab-git-ssh-proxy\\.sh <<'SH_PROXY'[\\s\\S]*?\\nSH_PROXY\\n[ \\t]*chmod [^\\n]*\\/tmp\\/hwlab-git-ssh-proxy\\.sh\\n/g, '\\n');",
|
|
" next = next.replace(/\\n[ \\t]*export GIT_SSH=.*\\n/g, '\\n');",
|
|
" next = next.replace(/\\n[ \\t]*export GIT_SSH_COMMAND=.*\\n/g, '\\n');",
|
|
" next = next.replace(/\\n[ \\t]*unset GIT_SSH_COMMAND\\n/g, '\\n');",
|
|
" if (!next.includes('hwlab git-mirror https auth: missing GITHUB_TOKEN secret env')) {",
|
|
" const noProxyExport = /\\nexport no_proxy=[^\\n]*\\n/;",
|
|
" if (noProxyExport.test(next)) next = next.replace(noProxyExport, (match) => `${match}${askpassBlock}\\n`);",
|
|
" else if (next.includes('\\nset -eu\\n')) next = next.replace('\\nset -eu\\n', `\\nset -eu\\n${askpassBlock}\\n`);",
|
|
" else next = `${askpassBlock}\\n${next}`;",
|
|
" }",
|
|
" if (next.includes('/git-ssh') || next.includes('ssh://git@') || next.includes('GIT_SSH=')) throw new Error(`generated git-mirror ${key} still contains ssh transport after githubTransport=https patch`);",
|
|
" if (!next.includes('GIT_ASKPASS') || !next.includes('GITHUB_TOKEN')) throw new Error(`generated git-mirror ${key} missing https auth after githubTransport=https patch`);",
|
|
" return next;",
|
|
" }",
|
|
" const gitMirrorFile = path.join(renderDir, 'devops-infra', 'git-mirror.yaml');",
|
|
" if (!fs.existsSync(gitMirrorFile)) throw new Error(`generated git-mirror manifest missing: ${gitMirrorFile}`);",
|
|
" const docs = YAML.parseAllDocuments(fs.readFileSync(gitMirrorFile, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null);",
|
|
" const manifests = [];",
|
|
" for (const doc of docs) {",
|
|
" if (doc && typeof doc === 'object' && doc.kind === 'List' && Array.isArray(doc.items)) manifests.push(...doc.items);",
|
|
" else manifests.push(doc);",
|
|
" }",
|
|
" let changed = false;",
|
|
" for (const doc of manifests) {",
|
|
" if (!doc || typeof doc !== 'object' || doc.kind !== 'ConfigMap') continue;",
|
|
" if (!doc.metadata || doc.metadata.name !== configMapName) continue;",
|
|
" doc.data = doc.data || {};",
|
|
" doc.data['sync.sh'] = patchScript(doc.data['sync.sh'], 'sync.sh');",
|
|
" doc.data['flush.sh'] = patchScript(doc.data['flush.sh'], 'flush.sh');",
|
|
" changed = true;",
|
|
" }",
|
|
" if (!changed) throw new Error(`generated git-mirror ConfigMap ${configMapName} was not found in ${gitMirrorFile}`);",
|
|
" fs.writeFileSync(gitMirrorFile, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n');",
|
|
"}",
|
|
"const structured = patchStructuredPipeline();",
|
|
"function replaceParamDefault(name, value) {",
|
|
" const namePattern = escapeRegExp(name);",
|
|
" text = text.replace(new RegExp(`(- default: )[^\\n]*(\\n\\\\s+name: ${namePattern}\\n\\\\s+type: string)`, 'g'), `$1${yamlString(value)}$2`);",
|
|
" text = text.replace(new RegExp(`(- name: ${namePattern}\\n\\\\s+type: string\\n\\\\s+default: )[^\\n]*`, 'g'), `$1${yamlString(value)}`);",
|
|
"}",
|
|
"replaceParamDefault('git-url', overlay.gitUrl);",
|
|
"replaceParamDefault('git-read-url', overlay.gitReadUrl);",
|
|
"replaceParamDefault('git-write-url', overlay.gitWriteUrl);",
|
|
"replaceParamDefault('catalog-path', overlay.catalogPath);",
|
|
"replaceParamDefault('runtime-path', overlay.runtimePath);",
|
|
"replaceParamDefault('registry-prefix', overlay.registryPrefix);",
|
|
"text = text.replace(/hwlab\\.pikastech\\.local\\/download-profile: [^\\n]+/g, `hwlab.pikastech.local/download-profile: ${overlay.downloadProfileId}`);",
|
|
"text = text.replace(/hwlab\\.pikastech\\.local\\/network-profile: [^\\n]+/g, `hwlab.pikastech.local/network-profile: ${overlay.networkProfileId}`);",
|
|
"if (overlay.buildkitSidecarImage) text = text.replace(/moby\\/buildkit:rootless/g, overlay.buildkitSidecarImage);",
|
|
"text = text.replace(/--git-read-url '[^']*'/g, `--git-read-url ${shellSingle(overlay.gitReadUrl)}`);",
|
|
"text = text.replace(/--git-write-url '[^']*'/g, `--git-write-url ${shellSingle(overlay.gitWriteUrl)}`);",
|
|
"text = text.replace(/--runtime-endpoint '[^']*'/g, `--runtime-endpoint ${shellSingle(overlay.publicApiUrl)}`);",
|
|
"text = text.replace(/--web-endpoint '[^']*'/g, `--web-endpoint ${shellSingle(overlay.publicWebUrl)}`);",
|
|
"for (const [name, value] of Object.entries(stepEnv)) {",
|
|
" if (value === undefined) continue;",
|
|
" text = text.replace(new RegExp(`(- name: ${escapeRegExp(name)}\\\\n\\\\s+value: )[^\\\\n]+`, 'g'), `$1${yamlString(value)}`);",
|
|
" text = text.replace(new RegExp(`(\\\"name\\\": ${JSON.stringify(name)},\\\\n\\\\s+\\\"value\\\": )\\\"[^\\\"]*\\\"`, 'g'), `$1${yamlString(value)}`);",
|
|
"}",
|
|
"const bootstrap = stepEnvBootstrapScript();",
|
|
"if (bootstrap) {",
|
|
" text = text.replace(/\\n([ \\t]*)git config --global/g, (match, indent, offset, fullText) => {",
|
|
" const previous = fullText.slice(Math.max(0, offset - 500), offset);",
|
|
" if (previous.includes('unidesk-step-env-bootstrap')) return match;",
|
|
" return `\\n${indent}${bootstrap.split('\\n').join(`\\n${indent}`)}\\n${indent}git config --global`;",
|
|
" });",
|
|
"}",
|
|
"if (overlay.gitSshProxyHost && overlay.gitSshProxyPort) {",
|
|
" text = text.split('node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808').join(`node /tmp/hwlab-github-proxy-connect.mjs ${overlay.gitSshProxyHost} ${overlay.gitSshProxyPort}`);",
|
|
"}",
|
|
"const quotedRoot = JSON.stringify(overlay.gitopsRoot);",
|
|
"const escapedQuotedRoot = quotedRoot.replaceAll('\"', '\\\\\"');",
|
|
"const replacements = [",
|
|
" ['--registry-prefix \"$(params.registry-prefix)\" --use-deploy-images', text.includes('--gitops-root ') ? '--registry-prefix \"$(params.registry-prefix)\" --use-deploy-images' : `--registry-prefix \"$(params.registry-prefix)\" --gitops-root ${quotedRoot} --use-deploy-images`],",
|
|
" ['--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --use-deploy-images', text.includes('--gitops-root ') ? '--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --use-deploy-images' : `--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --gitops-root ${escapedQuotedRoot} --use-deploy-images`],",
|
|
"];",
|
|
"let changed = false;",
|
|
"for (const [needle, replacement] of replacements) {",
|
|
" if (!text.includes(needle)) continue;",
|
|
" text = text.split(needle).join(replacement);",
|
|
" changed = true;",
|
|
"}",
|
|
"if (!structured && !changed && !text.includes(`--gitops-root ${quotedRoot}`) && !text.includes(`--gitops-root ${escapedQuotedRoot}`)) { throw new Error(`generated pipeline missing expected gitops-render invocation in ${pipelinePath}`); }",
|
|
"if (text.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"') && text.includes('npm ci --ignore-scripts --no-audit --prefer-offline')) { throw new Error(`generated pipeline still uses full npm ci prepare-source dependency install in ${pipelinePath}`); }",
|
|
"if (text.includes('readStructuredFile(process.cwd(), deployPath)')) { throw new Error(`generated pipeline still imports YAML parser for prepare-source catalog check in ${pipelinePath}`); }",
|
|
"if (text.includes('/yaml/-/yaml-') || text.includes('bun add --no-save --ignore-scripts') || text.includes('npm install --package-lock=false --no-save')) { throw new Error(`generated pipeline still downloads yaml during prepare-source in ${pipelinePath}`); }",
|
|
"if (text.includes('npm run gitops:ts:check')) { throw new Error(`generated pipeline still uses npm gitops:ts:check gate in ${pipelinePath}`); }",
|
|
"fs.writeFileSync(pipelinePath, text);",
|
|
"patchGitMirrorTransportYaml();",
|
|
"function patchArgoYaml(filePath) {",
|
|
" if (!YAML || !fs.existsSync(filePath)) return;",
|
|
" const docs = YAML.parseAllDocuments(fs.readFileSync(filePath, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null);",
|
|
" let changed = false;",
|
|
" for (const doc of docs) {",
|
|
" if (!doc || typeof doc !== 'object') continue;",
|
|
" if (doc.kind === 'AppProject') {",
|
|
" doc.spec = doc.spec || {};",
|
|
" doc.spec.sourceRepos = [overlay.argoRepoUrl];",
|
|
" changed = true;",
|
|
" }",
|
|
" if (doc.kind === 'Application') {",
|
|
" doc.spec = doc.spec || {};",
|
|
" doc.spec.source = { ...(doc.spec.source || {}), repoURL: overlay.argoRepoUrl, targetRevision: overlay.gitopsBranch, path: overlay.runtimePath };",
|
|
" changed = true;",
|
|
" }",
|
|
" }",
|
|
" if (changed) fs.writeFileSync(filePath, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n');",
|
|
"}",
|
|
"patchArgoYaml(path.join(renderDir, 'argocd', 'project.yaml'));",
|
|
"patchArgoYaml(path.join(renderDir, 'argocd', overlay.argoApplicationFile));",
|
|
"NODE",
|
|
];
|
|
}
|
|
|
|
function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
|
const gitSshProxy = httpProxyEndpoint(spec.networkProfile.proxy.http);
|
|
const gitMirror = nodeRuntimeGitMirrorTarget(spec);
|
|
const renderGitMirror = {
|
|
...gitMirror,
|
|
egressProxy: gitMirror.egressProxy.mode === "direct" ? {
|
|
mode: "direct",
|
|
required: false,
|
|
} : {
|
|
...gitMirror.egressProxy,
|
|
mode: "node-global",
|
|
required: true,
|
|
},
|
|
};
|
|
return {
|
|
nodeId: spec.nodeId,
|
|
lane: spec.lane,
|
|
sourceBranch: spec.sourceBranch,
|
|
gitopsBranch: spec.gitopsBranch,
|
|
gitopsRoot: nodeRuntimeGitopsRoot(spec),
|
|
runtimePath: spec.runtimePath,
|
|
runtimeRenderDir: spec.runtimeRenderDir,
|
|
runtimeNamespace: spec.runtimeNamespace,
|
|
catalogPath: spec.catalogPath,
|
|
tektonDir: spec.tektonDir,
|
|
argoApplicationFile: spec.argoApplicationFile,
|
|
argoRepoUrl: spec.argoRepoUrl,
|
|
gitUrl: spec.gitUrl,
|
|
gitReadUrl: spec.gitReadUrl,
|
|
gitWriteUrl: spec.gitWriteUrl,
|
|
toolsImage: gitMirror.toolsImage,
|
|
toolsImagePullPolicy: gitMirror.toolsImagePullPolicy,
|
|
gitMirror: renderGitMirror,
|
|
networkProfileId: spec.networkProfileId,
|
|
downloadProfileId: spec.downloadProfileId,
|
|
gitSshProxyHost: gitSshProxy?.host,
|
|
gitSshProxyPort: gitSshProxy?.port,
|
|
proxyHttp: spec.networkProfile.proxy.http,
|
|
proxyHttps: spec.networkProfile.proxy.https,
|
|
proxyAll: spec.networkProfile.proxy.all,
|
|
noProxy: spec.networkProfile.proxy.noProxy.join(","),
|
|
dockerProxyHttp: spec.networkProfile.dockerBuildProxy.http,
|
|
dockerProxyHttps: spec.networkProfile.dockerBuildProxy.https,
|
|
dockerProxyAll: spec.networkProfile.dockerBuildProxy.all,
|
|
dockerNoProxy: spec.networkProfile.dockerBuildProxy.noProxy.join(","),
|
|
dockerNoProxyList: spec.networkProfile.dockerBuildProxy.noProxy,
|
|
npmRegistry: spec.downloadProfile.npm.registry,
|
|
npmFetchTimeoutMs: spec.downloadProfile.npm.fetchTimeoutSeconds * 1000,
|
|
npmRetries: spec.downloadProfile.npm.retries,
|
|
stepEnv: spec.stepEnv,
|
|
observability: spec.observability,
|
|
runtimeStore: spec.runtimeStore,
|
|
runtimeImageRewrites: spec.runtimeImageRewrites,
|
|
registryPrefix: spec.registryPrefix,
|
|
buildkitSidecarImage: spec.buildkit?.sidecarImage,
|
|
publicWebUrl: spec.publicWebUrl,
|
|
publicApiUrl: spec.publicApiUrl,
|
|
publicExposure: spec.publicExposure === null ? undefined : {
|
|
enabled: spec.publicExposure.enabled,
|
|
mode: spec.publicExposure.mode,
|
|
publicBaseUrl: spec.publicExposure.publicBaseUrl,
|
|
hostname: spec.publicExposure.hostname,
|
|
expectedA: spec.publicExposure.expectedA,
|
|
serverAddr: spec.publicExposure.serverAddr,
|
|
serverPort: spec.publicExposure.serverPort,
|
|
secretName: spec.publicExposure.secretName,
|
|
secretKey: spec.publicExposure.secretKey,
|
|
tokenKey: spec.publicExposure.tokenKey,
|
|
webProxy: spec.publicExposure.webProxy,
|
|
apiProxy: spec.publicExposure.apiProxy,
|
|
},
|
|
externalPostgres: spec.externalPostgres === undefined ? undefined : {
|
|
enabled: true,
|
|
serviceName: spec.externalPostgres.serviceName,
|
|
endpointAddress: spec.externalPostgres.endpointAddress,
|
|
port: spec.externalPostgres.port,
|
|
sslmode: spec.externalPostgres.sslmode,
|
|
},
|
|
};
|
|
}
|
|
|
|
function httpProxyEndpoint(value: string): { host: string; port: number } | null {
|
|
let parsed: URL;
|
|
try {
|
|
parsed = new URL(value);
|
|
} catch {
|
|
return null;
|
|
}
|
|
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
|
|
const port = parsed.port === "" ? parsed.protocol === "https:" ? 443 : 80 : Number.parseInt(parsed.port, 10);
|
|
if (!Number.isInteger(port) || port <= 0) return null;
|
|
return { host: parsed.hostname, port };
|
|
}
|
|
|
|
|
|
function nodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string): string[] {
|
|
return [
|
|
`${renderDir}/devops-infra/git-mirror.yaml`,
|
|
`${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`,
|
|
`${renderDir}/${spec.tektonDir}/rbac.yaml`,
|
|
`${renderDir}/${spec.tektonDir}/pipeline.yaml`,
|
|
`${renderDir}/argocd/project.yaml`,
|
|
`${renderDir}/argocd/${spec.argoApplicationFile}`,
|
|
];
|
|
}
|
|
|
|
function applyNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult {
|
|
const files = nodeRuntimeControlPlaneFiles(spec, renderDir);
|
|
const args = [
|
|
"kubectl",
|
|
"apply",
|
|
...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]),
|
|
...files.flatMap((file) => ["-f", file]),
|
|
];
|
|
return runNodeK3sArgs(spec, args, timeoutSeconds);
|
|
}
|
|
|
|
function applyLocalNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult {
|
|
const manifest = nodeRuntimeControlPlaneFiles(spec, renderDir)
|
|
.map((file) => `---\n# Source: ${file}\n${readFileSync(file, "utf8").trimEnd()}\n`)
|
|
.join("");
|
|
const args = [
|
|
"kubectl",
|
|
"apply",
|
|
...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]),
|
|
"-f",
|
|
"-",
|
|
];
|
|
return runNodeK3sScript(spec, args.map(shellQuote).join(" "), timeoutSeconds, manifest);
|
|
}
|
|
|
|
function cleanupNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, renderDir: string): CommandResult {
|
|
const prefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`;
|
|
const script = [
|
|
"set -eu",
|
|
`render_dir=${shellQuote(renderDir)}`,
|
|
`prefix=${shellQuote(prefix)}`,
|
|
"case \"$render_dir\" in",
|
|
" \"$prefix\"*) rm -rf \"$render_dir\" ;;",
|
|
" *) echo \"refusing to remove unexpected render dir: $render_dir\" >&2; exit 44 ;;",
|
|
"esac",
|
|
].join("\n");
|
|
return runNodeHostScript(spec, script, 60);
|
|
}
|
|
|
|
function cleanupLocalNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, render: NodeRuntimeRenderResult): CommandResult {
|
|
const renderPrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`;
|
|
const worktreePrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-`;
|
|
if (!render.renderDir.startsWith(renderPrefix) || !render.worktreeDir.startsWith(worktreePrefix)) {
|
|
return {
|
|
command: ["rm", "-rf", "<refused>"],
|
|
cwd: repoRoot,
|
|
exitCode: 44,
|
|
stdout: "",
|
|
stderr: `refusing to remove unexpected local render paths: ${render.renderDir} ${render.worktreeDir}`,
|
|
signal: null,
|
|
timedOut: false,
|
|
};
|
|
}
|
|
return runCommand(["rm", "-rf", render.renderDir, render.worktreeDir], repoRoot, { timeoutMs: 60_000 });
|
|
}
|
|
|
|
function nodeRuntimePipelineRunManifest(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string): Record<string, unknown> {
|
|
return {
|
|
apiVersion: "tekton.dev/v1",
|
|
kind: "PipelineRun",
|
|
metadata: {
|
|
name: pipelineRun,
|
|
namespace: "hwlab-ci",
|
|
labels: {
|
|
"app.kubernetes.io/part-of": "hwlab",
|
|
"hwlab.pikastech.local/gitops-target": spec.lane,
|
|
"hwlab.pikastech.local/source-commit": sourceCommit,
|
|
"hwlab.pikastech.local/trigger": "unidesk-node-cli",
|
|
},
|
|
annotations: {
|
|
"hwlab.pikastech.local/node": spec.nodeId,
|
|
"hwlab.pikastech.local/source-branch": spec.sourceBranch,
|
|
"hwlab.pikastech.local/gitops-branch": spec.gitopsBranch,
|
|
"hwlab.pikastech.local/runtime-path": spec.runtimePath,
|
|
"hwlab.pikastech.local/network-profile": spec.networkProfileId,
|
|
"hwlab.pikastech.local/download-profile": spec.downloadProfileId,
|
|
},
|
|
},
|
|
spec: {
|
|
pipelineRef: { name: spec.pipeline },
|
|
taskRunTemplate: {
|
|
serviceAccountName: spec.serviceAccountName,
|
|
podTemplate: {
|
|
hostNetwork: true,
|
|
dnsPolicy: "ClusterFirstWithHostNet",
|
|
securityContext: { fsGroup: 1000 },
|
|
},
|
|
},
|
|
params: [
|
|
{ name: "git-url", value: spec.gitUrl },
|
|
{ name: "git-read-url", value: spec.gitReadUrl },
|
|
{ name: "git-write-url", value: spec.gitWriteUrl },
|
|
{ name: "source-branch", value: spec.sourceBranch },
|
|
{ name: "gitops-branch", value: spec.gitopsBranch },
|
|
{ name: "lane", value: spec.lane },
|
|
{ name: "catalog-path", value: spec.catalogPath },
|
|
{ name: "image-tag-mode", value: "full" },
|
|
{ name: "runtime-path", value: spec.runtimePath },
|
|
{ name: "revision", value: sourceCommit },
|
|
{ name: "registry-prefix", value: spec.registryPrefix },
|
|
{ name: "services", value: spec.serviceIds.join(",") },
|
|
{ name: "base-image", value: spec.baseImage },
|
|
],
|
|
workspaces: [
|
|
{ name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } },
|
|
{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } },
|
|
],
|
|
},
|
|
};
|
|
}
|
|
|
|
function createNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string, timeoutSeconds: number, rerun: boolean): CommandResult {
|
|
const manifestB64 = Buffer.from(JSON.stringify(nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun)), "utf8").toString("base64");
|
|
const script = [
|
|
"set -eu",
|
|
`manifest_b64=${shellQuote(manifestB64)}`,
|
|
`pipeline_run=${shellQuote(pipelineRun)}`,
|
|
`rerun=${rerun ? "true" : "false"}`,
|
|
"manifest_path=\"/tmp/$pipeline_run.json\"",
|
|
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
|
"existing_status=$(kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" -o 'jsonpath={.status.conditions[0].status}' 2>/dev/null || true)",
|
|
"if [ \"$rerun\" = true ] && [ -n \"$existing_status\" ]; then",
|
|
" kubectl delete pipelinerun -n hwlab-ci \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
" kubectl delete taskrun -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
" kubectl delete pod -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
" kubectl -n hwlab-ci get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n hwlab-ci delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
"elif [ \"$existing_status\" = False ]; then",
|
|
" kubectl delete pipelinerun -n hwlab-ci \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
" kubectl delete taskrun -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
" kubectl delete pod -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
" kubectl -n hwlab-ci get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n hwlab-ci delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
"fi",
|
|
"if kubectl create -f \"$manifest_path\"; then",
|
|
" :",
|
|
"else",
|
|
" code=$?",
|
|
" if kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" >/dev/null 2>&1; then",
|
|
" printf 'PipelineRun %s already exists; reusing existing object\\n' \"$pipeline_run\" >&2",
|
|
" else",
|
|
" exit \"$code\"",
|
|
" fi",
|
|
"fi",
|
|
"kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'",
|
|
].join("\n");
|
|
return runNodeK3sScript(spec, script, timeoutSeconds);
|
|
}
|
|
|
|
function waitForNodeRuntimePipelineRunTerminal(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
pipelineRun: string,
|
|
timeoutSeconds: number,
|
|
options: { opportunisticPostFlush?: () => Record<string, unknown> | null; opportunisticPostSync?: () => Record<string, unknown> | null } = {},
|
|
): Record<string, unknown> {
|
|
const severeWarningThresholdMs = 120_000;
|
|
const startedAt = Date.now();
|
|
const deadline = startedAt + timeoutSeconds * 1000;
|
|
let polls = 0;
|
|
let last: Record<string, unknown> = { exists: false, name: pipelineRun };
|
|
let lastOpportunisticPostFlushAt = 0;
|
|
let severeWarningEmitted = false;
|
|
let opportunisticPostSyncAttempted = false;
|
|
const opportunisticPostFlushes: Record<string, unknown>[] = [];
|
|
const opportunisticPostSyncs: Record<string, unknown>[] = [];
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "started", pipelineRun, timeoutSeconds });
|
|
while (Date.now() <= deadline) {
|
|
polls += 1;
|
|
last = getNodeRuntimePipelineRun(spec, pipelineRun);
|
|
const status = typeof last.status === "string" ? last.status : null;
|
|
const reason = typeof last.reason === "string" ? last.reason : null;
|
|
const elapsedMs = Date.now() - startedAt;
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "poll", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs });
|
|
if (!severeWarningEmitted && elapsedMs >= severeWarningThresholdMs && status !== "True" && status !== "False") {
|
|
severeWarningEmitted = true;
|
|
printNodeRuntimeTriggerProgress(spec, {
|
|
stage: "pipelinerun-wait",
|
|
status: "warning",
|
|
warning: "pipelinerun-wait-severe-timeout",
|
|
severity: "warning",
|
|
pipelineRun,
|
|
pipelineStatus: status,
|
|
reason,
|
|
polls,
|
|
elapsedMs,
|
|
thresholdMs: severeWarningThresholdMs,
|
|
message: "PipelineRun has been non-terminal for more than 120s; this is a severe timeout and requires further investigation of Tekton taskRuns/pods instead of treating it as normal wait time.",
|
|
next: {
|
|
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
|
|
},
|
|
});
|
|
}
|
|
if (status === "True" || status === "False") {
|
|
const ok = status === "True";
|
|
const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun);
|
|
const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics);
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: ok ? "succeeded" : "failed", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs: Date.now() - startedAt });
|
|
return {
|
|
ok,
|
|
status: ok ? "succeeded" : "failed",
|
|
pipelineRun: last,
|
|
diagnostics,
|
|
failureSummary,
|
|
polls,
|
|
elapsedMs: Date.now() - startedAt,
|
|
opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined,
|
|
opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined,
|
|
degradedReason: ok ? undefined : "node-runtime-pipelinerun-failed",
|
|
};
|
|
}
|
|
if (
|
|
options.opportunisticPostSync !== undefined
|
|
&& !opportunisticPostSyncAttempted
|
|
&& nodeRuntimePipelineRunSourceCloneCompleted(spec, pipelineRun)
|
|
) {
|
|
opportunisticPostSyncAttempted = true;
|
|
const sync = options.opportunisticPostSync();
|
|
if (sync !== null) opportunisticPostSyncs.push(sync);
|
|
}
|
|
if (options.opportunisticPostFlush !== undefined && Date.now() - lastOpportunisticPostFlushAt >= 15_000) {
|
|
lastOpportunisticPostFlushAt = Date.now();
|
|
const flush = options.opportunisticPostFlush();
|
|
if (flush !== null) opportunisticPostFlushes.push(flush);
|
|
last = getNodeRuntimePipelineRun(spec, pipelineRun);
|
|
const refreshedStatus = typeof last.status === "string" ? last.status : null;
|
|
const refreshedReason = typeof last.reason === "string" ? last.reason : null;
|
|
const refreshedElapsedMs = Date.now() - startedAt;
|
|
if (refreshedStatus === "True" || refreshedStatus === "False") {
|
|
const ok = refreshedStatus === "True";
|
|
const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun);
|
|
const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics);
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: ok ? "succeeded" : "failed", pipelineRun, pipelineStatus: refreshedStatus, reason: refreshedReason, polls, elapsedMs: refreshedElapsedMs, observedAfterPostFlush: true });
|
|
return {
|
|
ok,
|
|
status: ok ? "succeeded" : "failed",
|
|
pipelineRun: last,
|
|
diagnostics,
|
|
failureSummary,
|
|
polls,
|
|
elapsedMs: refreshedElapsedMs,
|
|
opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined,
|
|
opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined,
|
|
observedAfterPostFlush: true,
|
|
degradedReason: ok ? undefined : "node-runtime-pipelinerun-failed",
|
|
};
|
|
}
|
|
}
|
|
sleepSync(Math.min(10_000, Math.max(1000, deadline - Date.now())));
|
|
}
|
|
const elapsedMs = Date.now() - startedAt;
|
|
const warning = nodeRuntimeCicdWaitWarning(spec, pipelineRun, { polls, elapsedMs });
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "warning", pipelineRun, polls, elapsedMs, waitLimitSeconds: timeoutSeconds });
|
|
return {
|
|
ok: true,
|
|
status: "pending",
|
|
pipelineRun: last,
|
|
polls,
|
|
elapsedMs,
|
|
waitLimitSeconds: timeoutSeconds,
|
|
warning,
|
|
opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined,
|
|
opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined,
|
|
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun}` },
|
|
};
|
|
}
|
|
|
|
function nodeRuntimePipelineRunSourceCloneCompleted(spec: HwlabRuntimeLaneSpec, pipelineRun: string): boolean {
|
|
const result = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "taskrun", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", "json"], 60);
|
|
if (result.exitCode !== 0) return false;
|
|
let parsed: unknown;
|
|
try {
|
|
parsed = JSON.parse(result.stdout);
|
|
} catch {
|
|
return false;
|
|
}
|
|
const items = Array.isArray(record(parsed).items) ? record(parsed).items as unknown[] : [];
|
|
return items.some((item) => {
|
|
const itemRecord = record(item);
|
|
const metadata = record(itemRecord.metadata);
|
|
const labels = record(metadata.labels);
|
|
const name = typeof metadata.name === "string" ? metadata.name : "";
|
|
const pipelineTask = typeof labels["tekton.dev/pipelineTask"] === "string" ? labels["tekton.dev/pipelineTask"] : "";
|
|
const taskName = `${pipelineTask} ${name}`;
|
|
if (!/(clone|checkout|local-git|source[-_ ]*worktree)/iu.test(taskName)) return false;
|
|
const status = record(itemRecord.status);
|
|
const conditions = Array.isArray(status.conditions) ? status.conditions as unknown[] : [];
|
|
return conditions.some((condition) => {
|
|
const conditionRecord = record(condition);
|
|
return conditionRecord.type === "Succeeded" && conditionRecord.status === "True";
|
|
});
|
|
});
|
|
}
|
|
|
|
function printNodeRuntimeTriggerProgress(spec: HwlabRuntimeLaneSpec, data: Record<string, unknown> = {}): void {
|
|
process.stderr.write(`${JSON.stringify({ event: "hwlab.runtime-lane.trigger.progress", at: new Date().toISOString(), lane: spec.lane, node: spec.nodeId, ...data })}\n`);
|
|
}
|
|
|
|
function getNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, pipelineRun: string): Record<string, unknown> {
|
|
const result = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipelinerun", pipelineRun, "-o", "jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}{.metadata.creationTimestamp}{\"\\n\"}"], 60);
|
|
if (result.exitCode !== 0) return { exists: false, name: pipelineRun, result: compactRuntimeCommand(result) };
|
|
const [status = "", reason = "", message = "", createdAt = ""] = result.stdout.split(/\r?\n/u);
|
|
return {
|
|
exists: true,
|
|
name: pipelineRun,
|
|
status: status || null,
|
|
reason: reason || null,
|
|
message: message || null,
|
|
createdAt: createdAt || null,
|
|
result: compactRuntimeCommand(result),
|
|
};
|
|
}
|
|
|
|
function sleepSync(ms: number): void {
|
|
const buffer = new SharedArrayBuffer(4);
|
|
Atomics.wait(new Int32Array(buffer), 0, 0, Math.max(0, ms));
|
|
}
|
|
|
|
function syncNodeExternalPostgresSecrets(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record<string, unknown> | null {
|
|
const pg = spec.externalPostgres;
|
|
if (pg === undefined) return null;
|
|
const secretRoot = externalPostgresSecretSourceRoot(spec);
|
|
const cloudApi = readSecretSourceValue(secretRoot, pg.cloudApi.sourceRef, pg.cloudApi.envKey);
|
|
const openfga = readSecretSourceValue(secretRoot, pg.openfga.sourceRef, pg.openfga.envKey);
|
|
const missing = [
|
|
...(cloudApi.ok ? [] : [`${pg.cloudApi.sourceRef}:${pg.cloudApi.envKey}`]),
|
|
...(openfga.ok ? [] : [`${pg.openfga.sourceRef}:${pg.openfga.envKey}`]),
|
|
];
|
|
if (missing.length > 0) {
|
|
if (dryRun) {
|
|
return {
|
|
ok: true,
|
|
mode: "dry-run",
|
|
mutation: false,
|
|
skipped: true,
|
|
reason: "external-postgres-secret-source-missing",
|
|
missing,
|
|
sourceRoot: secretRoot,
|
|
valuesPrinted: false,
|
|
next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` },
|
|
};
|
|
}
|
|
return {
|
|
ok: false,
|
|
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
|
mutation: false,
|
|
degradedReason: "external-postgres-secret-source-missing",
|
|
missing,
|
|
sourceRoot: secretRoot,
|
|
valuesPrinted: false,
|
|
next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` },
|
|
};
|
|
}
|
|
const setup = runNodeK3sScript(spec, externalPostgresSecretSetupScript(spec, dryRun), timeoutSeconds);
|
|
if (!isCommandSuccess(setup)) {
|
|
return {
|
|
ok: false,
|
|
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
|
mutation: false,
|
|
phase: "namespace-authn-setup",
|
|
setup: compactRuntimeCommand(setup),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const manifest = {
|
|
apiVersion: "v1",
|
|
kind: "List",
|
|
items: [
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "Secret",
|
|
metadata: { name: pg.cloudApi.secretName, namespace: spec.runtimeNamespace },
|
|
type: "Opaque",
|
|
stringData: { [pg.cloudApi.secretKey]: cloudApi.value },
|
|
},
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "Secret",
|
|
metadata: { name: pg.openfga.secretName, namespace: spec.runtimeNamespace },
|
|
type: "Opaque",
|
|
stringData: { [pg.openfga.secretKey]: openfga.value },
|
|
},
|
|
],
|
|
};
|
|
const applyScript = [
|
|
"set -eu",
|
|
[
|
|
"kubectl",
|
|
"apply",
|
|
"--server-side",
|
|
"--force-conflicts",
|
|
"--field-manager=unidesk-hwlab-node-external-postgres-secret",
|
|
...(dryRun ? ["--dry-run=server"] : []),
|
|
"-f",
|
|
"-",
|
|
].map(shellQuote).join(" "),
|
|
].join("\n");
|
|
const apply = runNodeK3sScript(spec, applyScript, timeoutSeconds, JSON.stringify(manifest));
|
|
return {
|
|
ok: isCommandSuccess(apply),
|
|
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
|
mutation: !dryRun && isCommandSuccess(apply),
|
|
namespace: spec.runtimeNamespace,
|
|
sourceRoot: secretRoot,
|
|
secrets: [
|
|
{ name: pg.cloudApi.secretName, key: pg.cloudApi.secretKey, sourceRef: pg.cloudApi.sourceRef, envKey: pg.cloudApi.envKey },
|
|
{ name: pg.openfga.secretName, key: pg.openfga.secretKey, sourceRef: pg.openfga.sourceRef, envKey: pg.openfga.envKey, authnKey: pg.openfga.authnKey ?? null },
|
|
],
|
|
setup: compactRuntimeCommand(setup),
|
|
apply: compactRuntimeCommand(apply),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function syncNodeLocalPostgresBootstrapSecret(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record<string, unknown> | null {
|
|
const pg = spec.runtimeStore?.postgres;
|
|
if (pg?.mode !== "local-k3s") return null;
|
|
const secretName = pg.secretName ?? `${spec.runtimeNamespace}-postgres`;
|
|
const sourceRef = pg.adminPasswordSourceRef ?? "";
|
|
const sourceKey = pg.adminPasswordSourceKey ?? "";
|
|
if (sourceRef.length === 0 || sourceKey.length === 0) {
|
|
return {
|
|
ok: false,
|
|
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
|
mutation: false,
|
|
namespace: spec.runtimeNamespace,
|
|
secretName,
|
|
secretKey: "POSTGRES_PASSWORD",
|
|
degradedReason: "local-postgres-secret-source-not-configured",
|
|
valuesPrinted: false,
|
|
next: { fixYaml: `set runtimeStore.postgres.adminPasswordSourceRef/adminPasswordSourceKey for node=${spec.nodeId} lane=${spec.lane}` },
|
|
};
|
|
}
|
|
const material = readLocalPostgresPasswordMaterial({ sourceRef, sourceKey, dryRun });
|
|
if (material.ok !== true) {
|
|
return {
|
|
ok: false,
|
|
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
|
mutation: false,
|
|
namespace: spec.runtimeNamespace,
|
|
secretName,
|
|
secretKey: "POSTGRES_PASSWORD",
|
|
source: material,
|
|
degradedReason: material.error ?? "local-postgres-secret-source-missing",
|
|
valuesPrinted: false,
|
|
next: { createSource: `create ${material.sourcePath ?? join(".state", "secrets", sourceRef)} with ${sourceKey}=<redacted>` },
|
|
};
|
|
}
|
|
const result = runNodeK3sScript(spec, localPostgresBootstrapSecretScript(spec, secretName, sourceRef, sourceKey, dryRun, material.fingerprint), timeoutSeconds, material.value ?? "");
|
|
const fields = keyValueLinesFromText(statusText(result));
|
|
const afterBytes = numericField(fields.afterPasswordBytes) ?? 0;
|
|
const ok = isCommandSuccess(result) && fields.afterSecretExists === "yes" && afterBytes > 0;
|
|
return {
|
|
ok,
|
|
mode: dryRun ? "dry-run" : "confirmed-secret-sync",
|
|
mutation: !dryRun && fields.mutation === "true",
|
|
namespace: spec.runtimeNamespace,
|
|
secretName,
|
|
secretKey: "POSTGRES_PASSWORD",
|
|
source: {
|
|
ok: material.ok,
|
|
sourceRef,
|
|
sourceKey,
|
|
sourcePath: material.sourcePath === null ? null : displayRepoPath(material.sourcePath),
|
|
generated: material.generated,
|
|
fingerprint: material.fingerprint,
|
|
valueBytes: material.value?.length ?? 0,
|
|
valuesPrinted: false,
|
|
},
|
|
before: {
|
|
namespaceExists: fields.beforeNamespaceExists === "yes",
|
|
secretExists: fields.beforeSecretExists === "yes",
|
|
passwordBytes: numericField(fields.beforePasswordBytes),
|
|
fingerprint: fields.beforePasswordFingerprint || null,
|
|
},
|
|
after: {
|
|
namespaceExists: fields.afterNamespaceExists === "yes",
|
|
secretExists: fields.afterSecretExists === "yes",
|
|
passwordBytes: numericField(fields.afterPasswordBytes),
|
|
fingerprint: fields.afterPasswordFingerprint || null,
|
|
},
|
|
action: fields.action || null,
|
|
applyExitCode: numericField(fields.applyExitCode),
|
|
namespaceCreateExitCode: numericField(fields.namespaceCreateExitCode),
|
|
degradedReason: ok ? undefined : fields.action === "source-mismatch" ? "local-postgres-secret-source-mismatch" : "local-postgres-secret-apply-failed",
|
|
result: compactRuntimeCommand(result),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function readLocalPostgresPasswordMaterial(input: { sourceRef: string; sourceKey: string; dryRun: boolean }): {
|
|
ok: boolean;
|
|
sourceRef: string;
|
|
sourceKey: string;
|
|
sourcePath: string | null;
|
|
checkedPaths: string[];
|
|
value: string | null;
|
|
fingerprint: string | null;
|
|
generated: boolean;
|
|
error?: string;
|
|
} {
|
|
const checkedPaths = localSecretSourcePaths(input.sourceRef);
|
|
const existingPath = checkedPaths.find((candidate) => existsSync(candidate)) ?? null;
|
|
const sourcePath = existingPath ?? checkedPaths[0] ?? null;
|
|
if (sourcePath === null) {
|
|
return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: null, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-path-unresolved" };
|
|
}
|
|
if (existingPath === null && input.dryRun) {
|
|
return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-missing" };
|
|
}
|
|
if (existingPath === null) {
|
|
const value = randomBytes(32).toString("base64url");
|
|
mkdirSync(dirname(sourcePath), { recursive: true });
|
|
writeFileSync(sourcePath, `${input.sourceKey}=${value}\n`, { mode: 0o600 });
|
|
return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath, checkedPaths, value, fingerprint: shortSecretFingerprint(value), generated: true };
|
|
}
|
|
const text = readFileSync(existingPath, "utf8");
|
|
const values = parseEnvFile(text);
|
|
const existingValue = values[input.sourceKey];
|
|
if (existingValue !== undefined && existingValue.length > 0) {
|
|
return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value: existingValue, fingerprint: shortSecretFingerprint(existingValue), generated: false };
|
|
}
|
|
if (input.dryRun) {
|
|
return { ok: false, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value: null, fingerprint: null, generated: false, error: "local-postgres-secret-source-key-missing" };
|
|
}
|
|
const value = randomBytes(32).toString("base64url");
|
|
const prefix = text.length === 0 || text.endsWith("\n") ? "" : "\n";
|
|
writeFileSync(existingPath, `${text}${prefix}${input.sourceKey}=${value}\n`, { mode: 0o600 });
|
|
return { ok: true, sourceRef: input.sourceRef, sourceKey: input.sourceKey, sourcePath: existingPath, checkedPaths, value, fingerprint: shortSecretFingerprint(value), generated: true };
|
|
}
|
|
|
|
function localSecretSourcePaths(sourceRef: string): string[] {
|
|
const marker = "/.worktree/";
|
|
const index = repoRoot.indexOf(marker);
|
|
const paths = index >= 0
|
|
? [
|
|
join(repoRoot.slice(0, index), ".state", "secrets", sourceRef),
|
|
join(repoRoot, ".state", "secrets", sourceRef),
|
|
]
|
|
: [join(repoRoot, ".state", "secrets", sourceRef)];
|
|
return [...new Set(paths)];
|
|
}
|
|
|
|
function shortSecretFingerprint(value: string): string {
|
|
return `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`;
|
|
}
|
|
|
|
function localPostgresBootstrapSecretScript(spec: HwlabRuntimeLaneSpec, secretName: string, sourceRef: string, sourceKey: string, dryRun: boolean, sourceFingerprint: string | null): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(spec.runtimeNamespace)}`,
|
|
`secret=${shellQuote(secretName)}`,
|
|
"secret_key=POSTGRES_PASSWORD",
|
|
`source_ref=${shellQuote(sourceRef)}`,
|
|
`source_key=${shellQuote(sourceKey)}`,
|
|
`source_fingerprint=${shellQuote(sourceFingerprint ?? "")}`,
|
|
`dry_run=${shellQuote(dryRun ? "true" : "false")}`,
|
|
"field_manager=unidesk-hwlab-node-local-postgres-secret",
|
|
"password=$(cat)",
|
|
"password_bytes=$(printf '%s' \"$password\" | wc -c | tr -d ' ')",
|
|
"exists_flag() { kubectl -n \"$namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"namespace_exists_flag() { kubectl get namespace \"$namespace\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$secret\" -o \"go-template={{ index .data \\\"$secret_key\\\" }}\" 2>/dev/null || true; }",
|
|
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
|
|
"decoded_fingerprint() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | sha256sum | awk '{print \"sha256:\" substr($1,1,16)}'; fi; }",
|
|
"before_namespace_exists=$(namespace_exists_flag)",
|
|
"before_secret_exists=no",
|
|
"before_b64=",
|
|
"if [ \"$before_namespace_exists\" = yes ]; then before_secret_exists=$(exists_flag secret \"$secret\"); before_b64=$(secret_b64_key); fi",
|
|
"before_password_bytes=$(decoded_length \"$before_b64\")",
|
|
"before_password_fingerprint=$(decoded_fingerprint \"$before_b64\")",
|
|
"after_namespace_exists=$before_namespace_exists",
|
|
"after_secret_exists=$before_secret_exists",
|
|
"after_password_bytes=$before_password_bytes",
|
|
"after_password_fingerprint=$before_password_fingerprint",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"namespace_create_exit=",
|
|
"apply_exit=",
|
|
"if [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$before_secret_exists\" != yes ] || [ \"$before_password_bytes\" -le 0 ]; then action=would-ensure; else action=kept; fi",
|
|
"elif [ \"$password_bytes\" -le 0 ]; then",
|
|
" action=source-empty",
|
|
" apply_exit=42",
|
|
"elif [ \"$before_password_bytes\" -gt 0 ] && [ -n \"$source_fingerprint\" ] && [ \"$before_password_fingerprint\" != \"$source_fingerprint\" ]; then",
|
|
" action=source-mismatch",
|
|
" apply_exit=47",
|
|
"else",
|
|
" if [ \"$before_namespace_exists\" != yes ]; then",
|
|
" kubectl create namespace \"$namespace\" >/tmp/hwlab-local-postgres-namespace.out 2>/tmp/hwlab-local-postgres-namespace.err",
|
|
" namespace_create_exit=$?",
|
|
" else",
|
|
" namespace_create_exit=0",
|
|
" fi",
|
|
" if [ \"$namespace_create_exit\" = 0 ]; then",
|
|
" tmp=$(mktemp /tmp/hwlab-local-postgres-secret.XXXXXX.yaml)",
|
|
" password_b64=$(printf '%s' \"$password\" | base64 | tr -d '\\n')",
|
|
" cat >\"$tmp\" <<EOF_SECRET",
|
|
"apiVersion: v1",
|
|
"kind: Secret",
|
|
"metadata:",
|
|
" name: $secret",
|
|
" namespace: $namespace",
|
|
" labels:",
|
|
" app.kubernetes.io/part-of: hwlab",
|
|
" app.kubernetes.io/managed-by: unidesk",
|
|
" annotations:",
|
|
" hwlab.pikastech.local/secret-source-ref: $source_ref",
|
|
" hwlab.pikastech.local/secret-source-key: $source_key",
|
|
" hwlab.pikastech.local/secret-source-fingerprint: $source_fingerprint",
|
|
"type: Opaque",
|
|
"data:",
|
|
" $secret_key: $password_b64",
|
|
"EOF_SECRET",
|
|
" kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f \"$tmp\" >/tmp/hwlab-local-postgres-secret.apply.out 2>/tmp/hwlab-local-postgres-secret.apply.err",
|
|
" apply_exit=$?",
|
|
" rm -f \"$tmp\"",
|
|
" if [ \"$apply_exit\" = 0 ]; then action=ensured; mutation=true; else action=apply-failed; fi",
|
|
" else",
|
|
" action=namespace-create-failed",
|
|
" apply_exit=$namespace_create_exit",
|
|
" fi",
|
|
"fi",
|
|
"after_namespace_exists=$(namespace_exists_flag)",
|
|
"if [ \"$after_namespace_exists\" = yes ]; then after_secret_exists=$(exists_flag secret \"$secret\"); after_b64=$(secret_b64_key); else after_secret_exists=no; after_b64=; fi",
|
|
"after_password_bytes=$(decoded_length \"$after_b64\")",
|
|
"after_password_fingerprint=$(decoded_fingerprint \"$after_b64\")",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$secret\"",
|
|
"printf 'key\\t%s\\n' \"$secret_key\"",
|
|
"printf 'sourceRef\\t%s\\n' \"$source_ref\"",
|
|
"printf 'sourceKey\\t%s\\n' \"$source_key\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeNamespaceExists\\t%s\\n' \"$before_namespace_exists\"",
|
|
"printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"",
|
|
"printf 'beforePasswordBytes\\t%s\\n' \"$before_password_bytes\"",
|
|
"printf 'beforePasswordFingerprint\\t%s\\n' \"$before_password_fingerprint\"",
|
|
"printf 'afterNamespaceExists\\t%s\\n' \"$after_namespace_exists\"",
|
|
"printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"",
|
|
"printf 'afterPasswordBytes\\t%s\\n' \"$after_password_bytes\"",
|
|
"printf 'afterPasswordFingerprint\\t%s\\n' \"$after_password_fingerprint\"",
|
|
"printf 'namespaceCreateExitCode\\t%s\\n' \"$namespace_create_exit\"",
|
|
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
|
|
"password=",
|
|
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
|
|
"if [ \"$after_secret_exists\" != yes ] || [ \"$after_password_bytes\" -le 0 ]; then exit 48; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function nodeRuntimeBaseImageStatus(spec: HwlabRuntimeLaneSpec, timeoutSeconds: number): Record<string, unknown> {
|
|
const source = spec.baseImageSource ?? "";
|
|
const script = [
|
|
"set -eu",
|
|
`target=${shellQuote(spec.baseImage)}`,
|
|
`source=${shellQuote(source)}`,
|
|
"repo_tag=${target#*/}",
|
|
"repo=${repo_tag%:*}",
|
|
"tag=${repo_tag##*:}",
|
|
"if [ \"$repo\" = \"$repo_tag\" ]; then tag=latest; fi",
|
|
"registry_url=\"http://127.0.0.1:5000/v2/$repo/tags/list\"",
|
|
"registry_present=false",
|
|
"if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then registry_present=true; fi",
|
|
"target_present=false",
|
|
"if docker image inspect \"$target\" >/dev/null 2>&1; then target_present=true; fi",
|
|
"source_present=false",
|
|
"if [ -n \"$source\" ] && docker image inspect \"$source\" >/dev/null 2>&1; then source_present=true; fi",
|
|
"printf 'target\\t%s\\n' \"$target\"",
|
|
"printf 'source\\t%s\\n' \"$source\"",
|
|
"printf 'sourceConfigured\\t%s\\n' \"$([ -n \"$source\" ] && printf true || printf false)\"",
|
|
"printf 'registryUrl\\t%s\\n' \"$registry_url\"",
|
|
"printf 'registryTagPresent\\t%s\\n' \"$registry_present\"",
|
|
"printf 'targetImagePresent\\t%s\\n' \"$target_present\"",
|
|
"printf 'sourceImagePresent\\t%s\\n' \"$source_present\"",
|
|
].join("\n");
|
|
const result = runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300));
|
|
const fields = keyValueLinesFromText(statusText(result));
|
|
const sourceConfigured = fields.sourceConfigured === "true";
|
|
const registryTagPresent = fields.registryTagPresent === "true";
|
|
return {
|
|
ok: isCommandSuccess(result) && sourceConfigured && registryTagPresent,
|
|
target: fields.target ?? spec.baseImage,
|
|
source: fields.source || null,
|
|
sourceConfigured,
|
|
registryUrl: fields.registryUrl ?? null,
|
|
registryTagPresent,
|
|
targetImagePresent: fields.targetImagePresent === "true",
|
|
sourceImagePresent: fields.sourceImagePresent === "true",
|
|
result: compactRuntimeCommand(result),
|
|
};
|
|
}
|
|
|
|
function ensureNodeBaseImage(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record<string, unknown> | null {
|
|
if (spec.baseImageSource === undefined) return null;
|
|
const script = [
|
|
"set -eu",
|
|
`target=${shellQuote(spec.baseImage)}`,
|
|
`source=${shellQuote(spec.baseImageSource)}`,
|
|
`dry_run=${shellQuote(dryRun ? "true" : "false")}`,
|
|
`pull_retries=${shellQuote(String(Math.max(1, spec.downloadProfile.docker.pullRetries)))}`,
|
|
"repo_tag=${target#*/}",
|
|
"repo=${repo_tag%:*}",
|
|
"tag=${repo_tag##*:}",
|
|
"if [ \"$repo\" = \"$repo_tag\" ]; then tag=latest; fi",
|
|
"registry_url=\"http://127.0.0.1:5000/v2/$repo/tags/list\"",
|
|
"present=false",
|
|
"if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then present=true; fi",
|
|
"action=none",
|
|
"if [ \"$present\" = false ]; then",
|
|
" action=seed",
|
|
" if [ \"$dry_run\" = false ]; then",
|
|
" source_present_before=true",
|
|
" if ! docker image inspect \"$source\" >/dev/null 2>&1; then",
|
|
" source_present_before=false",
|
|
" attempt=1",
|
|
" pulled_source=false",
|
|
" while [ \"$attempt\" -le \"$pull_retries\" ]; do",
|
|
" pull_attempts=$attempt",
|
|
" if docker pull \"$source\" >/tmp/hwlab-node-base-image-pull.out 2>&1; then pulled_source=true; break; fi",
|
|
" attempt=$((attempt + 1))",
|
|
" sleep 2",
|
|
" done",
|
|
" if [ \"$pulled_source\" != true ]; then cat /tmp/hwlab-node-base-image-pull.out >&2 2>/dev/null || true; fi",
|
|
" fi",
|
|
" docker image inspect \"$source\" >/dev/null",
|
|
" docker tag \"$source\" \"$target\"",
|
|
" docker push \"$target\" >/tmp/hwlab-node-base-image-push.out",
|
|
" fi",
|
|
"fi",
|
|
"after=false",
|
|
"if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then after=true; fi",
|
|
"printf 'target\\t%s\\n' \"$target\"",
|
|
"printf 'source\\t%s\\n' \"$source\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'presentBefore\\t%s\\n' \"$present\"",
|
|
"printf 'presentAfter\\t%s\\n' \"$after\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'pullRetries\\t%s\\n' \"$pull_retries\"",
|
|
"printf 'sourcePresentBefore\\t%s\\n' \"${source_present_before:-unknown}\"",
|
|
"printf 'pulledSource\\t%s\\n' \"${pulled_source:-false}\"",
|
|
"printf 'pullAttempts\\t%s\\n' \"${pull_attempts:-0}\"",
|
|
"if [ \"$dry_run\" = true ] || [ \"$after\" = true ]; then exit 0; fi",
|
|
"exit 1",
|
|
].join("\n");
|
|
const result = runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300));
|
|
const fields = keyValueLinesFromText(statusText(result));
|
|
return {
|
|
ok: isCommandSuccess(result),
|
|
dryRun,
|
|
target: fields.target ?? spec.baseImage,
|
|
source: fields.source ?? spec.baseImageSource,
|
|
presentBefore: fields.presentBefore === "true",
|
|
presentAfter: fields.presentAfter === "true",
|
|
action: fields.action ?? null,
|
|
pullRetries: numericField(fields.pullRetries),
|
|
sourcePresentBefore: fields.sourcePresentBefore ?? null,
|
|
pulledSource: fields.pulledSource === "true",
|
|
pullAttempts: numericField(fields.pullAttempts),
|
|
result: compactRuntimeCommand(result),
|
|
};
|
|
}
|
|
|
|
function externalPostgresSecretSetupScript(spec: HwlabRuntimeLaneSpec, dryRun: boolean): string {
|
|
const pg = spec.externalPostgres;
|
|
if (pg === undefined) return "true";
|
|
const authnKey = pg.openfga.authnKey ?? "authn-preshared-key";
|
|
return [
|
|
"set -eu",
|
|
`namespace=${shellQuote(spec.runtimeNamespace)}`,
|
|
`secret=${shellQuote(pg.openfga.secretName)}`,
|
|
`authn_key=${shellQuote(authnKey)}`,
|
|
`dry_run=${shellQuote(dryRun ? "true" : "false")}`,
|
|
"namespace_apply_args=\"--server-side --field-manager=unidesk-hwlab-node-runtime-namespace\"",
|
|
"if [ \"$dry_run\" = true ]; then namespace_apply_args=\"$namespace_apply_args --dry-run=server\"; fi",
|
|
"kubectl create namespace \"$namespace\" --dry-run=client -o yaml | kubectl apply $namespace_apply_args -f -",
|
|
"authn_present=$(kubectl -n \"$namespace\" get secret \"$secret\" -o \"go-template={{ index .data \\\"$authn_key\\\" }}\" 2>/dev/null | wc -c | tr -d ' ')",
|
|
"if [ \"${authn_present:-0}\" -gt 0 ]; then",
|
|
" action=kept",
|
|
"elif [ \"$dry_run\" = true ]; then",
|
|
" action=would-generate-authn-key",
|
|
"else",
|
|
" if command -v openssl >/dev/null 2>&1; then authn_value=$(openssl rand -base64 48); else authn_value=$(head -c 48 /dev/urandom | base64); fi",
|
|
" kubectl -n \"$namespace\" create secret generic \"$secret\" --from-literal=\"$authn_key=$authn_value\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-node-openfga-authn -f - >/dev/null",
|
|
" authn_value=",
|
|
" action=generated-authn-key",
|
|
"fi",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$secret\"",
|
|
"printf 'authnKey\\t%s\\n' \"$authn_key\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'valuesPrinted\\tfalse\\n'",
|
|
].join("\n");
|
|
}
|
|
|
|
function externalPostgresSecretSourceRoot(spec: HwlabRuntimeLaneSpec): string {
|
|
const configRef = spec.externalPostgres?.configRef;
|
|
if (configRef === undefined) return join(repoRoot, ".state", "secrets");
|
|
const configPath = rootPath(configRef);
|
|
const parsed = Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown;
|
|
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error(`${configRef} must be a YAML object`);
|
|
const secrets = (parsed as Record<string, unknown>).secrets;
|
|
if (typeof secrets !== "object" || secrets === null || Array.isArray(secrets)) throw new Error(`${configRef}.secrets must be an object`);
|
|
const root = (secrets as Record<string, unknown>).root;
|
|
if (typeof root !== "string" || root.length === 0) throw new Error(`${configRef}.secrets.root must be a non-empty string`);
|
|
return root.startsWith("/") ? root : rootPath(root);
|
|
}
|
|
|
|
function readSecretSourceValue(secretRoot: string, sourceRef: string, key: string): { ok: true; value: string; sourcePath: string } | { ok: false; sourcePath: string } {
|
|
const sourcePath = join(secretRoot, sourceRef);
|
|
if (!existsSync(sourcePath)) return { ok: false, sourcePath };
|
|
const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
|
|
const value = values[key];
|
|
if (value === undefined || value.length === 0) return { ok: false, sourcePath };
|
|
return { ok: true, value, sourcePath };
|
|
}
|
|
|
|
function secretSourcePaths(sourceRef: string): string[] {
|
|
const paths = [join(repoRoot, ".state", "secrets", sourceRef)];
|
|
const marker = "/.worktree/";
|
|
const index = repoRoot.indexOf(marker);
|
|
if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", sourceRef));
|
|
return [...new Set(paths)];
|
|
}
|
|
|
|
function displayRepoPath(path: string): string {
|
|
const normalizedRoot = repoRoot.replace(/\/+$/u, "");
|
|
if (path === normalizedRoot) return ".";
|
|
if (path.startsWith(`${normalizedRoot}/`)) return path.slice(normalizedRoot.length + 1);
|
|
const marker = "/.worktree/";
|
|
const index = normalizedRoot.indexOf(marker);
|
|
if (index >= 0) {
|
|
const mainRoot = normalizedRoot.slice(0, index);
|
|
if (path === mainRoot) return ".";
|
|
if (path.startsWith(`${mainRoot}/`)) return path.slice(mainRoot.length + 1);
|
|
}
|
|
return path;
|
|
}
|
|
|
|
function hwlabPasswordHash(password: string): string {
|
|
const salt = randomBytes(16).toString("hex");
|
|
return `sha256:${salt}:${createHash("sha256").update(`${salt}:${password}`).digest("hex")}`;
|
|
}
|
|
|
|
function readBootstrapAdminSecretMaterial(spec: RuntimeSecretSpec): BootstrapAdminSecretMaterial {
|
|
const sourceRef = spec.bootstrapAdminPasswordSourceRef;
|
|
const sourceKey = spec.bootstrapAdminPasswordSourceKey;
|
|
if (sourceRef === undefined || sourceKey === undefined || spec.bootstrapAdminPasswordHashTransform === undefined) {
|
|
return { ok: false, sourceRef: sourceRef ?? null, sourceKey: sourceKey ?? null, sourcePath: null, sourcePresent: false, sourceFingerprint: null, passwordHash: null, error: "bootstrap-admin-yaml-source-missing" };
|
|
}
|
|
const paths = secretSourcePaths(sourceRef);
|
|
const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
|
|
if (!existsSync(sourcePath)) {
|
|
return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: false, sourceFingerprint: null, passwordHash: null, error: "secret-source-missing" };
|
|
}
|
|
const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
|
|
const password = values[sourceKey];
|
|
if (password === undefined || password.length === 0) {
|
|
return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: true, sourceFingerprint: null, passwordHash: null, error: "secret-key-missing" };
|
|
}
|
|
return {
|
|
ok: true,
|
|
sourceRef,
|
|
sourceKey,
|
|
sourcePath,
|
|
sourcePresent: true,
|
|
sourceFingerprint: `sha256:${createHash("sha256").update(password).digest("hex").slice(0, 16)}`,
|
|
passwordHash: hwlabPasswordHash(password),
|
|
error: null,
|
|
};
|
|
}
|
|
|
|
function readBootstrapAdminPasswordMaterial(spec: RuntimeSecretSpec): BootstrapAdminPasswordMaterial {
|
|
const sourceRef = spec.bootstrapAdminPasswordSourceRef;
|
|
const sourceKey = spec.bootstrapAdminPasswordSourceKey;
|
|
if (sourceRef === undefined || sourceKey === undefined || spec.bootstrapAdminPasswordHashTransform === undefined) {
|
|
return { ok: false, sourceRef: sourceRef ?? null, sourceKey: sourceKey ?? null, sourcePath: null, sourcePresent: false, sourceFingerprint: null, password: null, error: "bootstrap-admin-yaml-source-missing" };
|
|
}
|
|
const paths = secretSourcePaths(sourceRef);
|
|
const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
|
|
if (!existsSync(sourcePath)) {
|
|
return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: false, sourceFingerprint: null, password: null, error: "secret-source-missing" };
|
|
}
|
|
const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
|
|
const password = values[sourceKey];
|
|
if (password === undefined || password.length === 0) {
|
|
return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: true, sourceFingerprint: null, password: null, error: "secret-key-missing" };
|
|
}
|
|
return {
|
|
ok: true,
|
|
sourceRef,
|
|
sourceKey,
|
|
sourcePath,
|
|
sourcePresent: true,
|
|
sourceFingerprint: `sha256:${createHash("sha256").update(password).digest("hex").slice(0, 16)}`,
|
|
password,
|
|
error: null,
|
|
};
|
|
}
|
|
|
|
function parseEnvFile(text: string): Record<string, string> {
|
|
const values: Record<string, string> = {};
|
|
for (const rawLine of text.split(/\r?\n/u)) {
|
|
const line = rawLine.trim();
|
|
if (line.length === 0 || line.startsWith("#")) continue;
|
|
const eq = line.indexOf("=");
|
|
if (eq <= 0) continue;
|
|
const key = line.slice(0, eq).trim();
|
|
const rawValue = line.slice(eq + 1).trim();
|
|
values[key] = rawValue.startsWith("'") && rawValue.endsWith("'")
|
|
? rawValue.slice(1, -1).replace(/'"'"'/gu, "'")
|
|
: rawValue.startsWith("\"") && rawValue.endsWith("\"")
|
|
? rawValue.slice(1, -1)
|
|
: rawValue;
|
|
}
|
|
return values;
|
|
}
|
|
|
|
function isLocalPostgresObject(name: string, spec: HwlabRuntimeLaneSpec): boolean {
|
|
if (!/postgres|pgsql|pgdata/iu.test(name)) return false;
|
|
const externalServiceName = spec.externalPostgres?.serviceName;
|
|
if (externalServiceName !== undefined && (name === `service/${externalServiceName}` || name === `svc/${externalServiceName}`)) return false;
|
|
return true;
|
|
}
|
|
|
|
function externalPostgresBridgeStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record<string, unknown> {
|
|
const pg = spec.externalPostgres;
|
|
if (pg === undefined) return { required: false, ready: true };
|
|
if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" };
|
|
const service = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "service", pg.serviceName, "-o", "name"], 60);
|
|
const endpointSlice = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "endpointslice", `${pg.serviceName}-host`, "-o", "jsonpath={.endpoints[0].addresses[0]}{\"\\n\"}{.ports[0].port}{\"\\n\"}"], 60);
|
|
const [address = "", port = ""] = endpointSlice.stdout.split(/\r?\n/u);
|
|
return {
|
|
required: true,
|
|
ready: service.exitCode === 0 && endpointSlice.exitCode === 0 && address === pg.endpointAddress && Number(port) === pg.port,
|
|
serviceName: pg.serviceName,
|
|
endpointAddress: address || null,
|
|
expectedEndpointAddress: pg.endpointAddress,
|
|
port: numericField(port),
|
|
expectedPort: pg.port,
|
|
service: compactRuntimeCommand(service),
|
|
endpointSlice: compactRuntimeCommand(endpointSlice),
|
|
};
|
|
}
|
|
|
|
function externalPostgresSecretStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record<string, unknown> {
|
|
const pg = spec.externalPostgres;
|
|
if (pg === undefined) return { required: false, ready: true };
|
|
if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" };
|
|
const cloudApi = secretKeyStatus(spec, pg.cloudApi.secretName, pg.cloudApi.secretKey);
|
|
const openfgaUri = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.secretKey);
|
|
const openfgaAuthn = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.authnKey ?? "authn-preshared-key");
|
|
return {
|
|
required: true,
|
|
ready: cloudApi.present && openfgaUri.present && openfgaAuthn.present,
|
|
valuesPrinted: false,
|
|
cloudApi,
|
|
openfga: {
|
|
datastoreUri: openfgaUri,
|
|
authnPresharedKey: openfgaAuthn,
|
|
},
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeGitMirrorTargetSpec {
|
|
const configPath = rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH);
|
|
const parsed = record(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown);
|
|
const nodes = record(parsed.nodes);
|
|
const node = record(nodes[spec.nodeId]);
|
|
const targets = Array.isArray(parsed.targets) ? parsed.targets : [];
|
|
const target = targets.map((item) => record(item)).find((item) => item.node === spec.nodeId && item.lane === spec.lane);
|
|
if (target === undefined) throw new Error(`no gitMirror target for node=${spec.nodeId} lane=${spec.lane} in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`);
|
|
const gitMirror = record(target.gitMirror);
|
|
const gitMirrorEgressProxy = record(gitMirror.egressProxy);
|
|
const gitMirrorEgressMode = stringValue(gitMirrorEgressProxy.mode, "gitMirror.egressProxy.mode");
|
|
if (gitMirrorEgressMode !== "node-global" && gitMirrorEgressMode !== "direct") throw new Error(`gitMirror.egressProxy.mode must be node-global or direct for node=${spec.nodeId} lane=${spec.lane}`);
|
|
const nodeEgressProxy = gitMirrorEgressMode === "direct"
|
|
? { mode: "direct" as const, required: false as const }
|
|
: nodeRuntimeGitMirrorEgressProxySpec(record(node.egressProxy), `nodes.${spec.nodeId}.egressProxy`);
|
|
if (gitMirrorEgressMode === "node-global" && gitMirrorEgressProxy.required !== true) throw new Error(`gitMirror.egressProxy.required must be true for node=${spec.nodeId} lane=${spec.lane}`);
|
|
const githubTransport = nodeRuntimeGitMirrorGithubTransportSpec(record(gitMirror.githubTransport), "gitMirror.githubTransport");
|
|
const source = record(target.source);
|
|
const gitops = record(target.gitops);
|
|
const tekton = record(target.tekton);
|
|
const toolsImage = record(tekton.toolsImage);
|
|
const toolsImagePullPolicy = optionalStringValue(toolsImage.imagePullPolicy, "tekton.toolsImage.imagePullPolicy") ?? "IfNotPresent";
|
|
if (toolsImagePullPolicy !== "Always" && toolsImagePullPolicy !== "IfNotPresent" && toolsImagePullPolicy !== "Never") throw new Error("tekton.toolsImage.imagePullPolicy must be Always, IfNotPresent, or Never");
|
|
return {
|
|
id: stringValue(target.id, "target.id"),
|
|
node: stringValue(target.node, "target.node"),
|
|
lane: stringValue(target.lane, "target.lane"),
|
|
namespace: stringValue(gitMirror.namespace, "gitMirror.namespace"),
|
|
serviceReadName: stringValue(gitMirror.serviceReadName, "gitMirror.serviceReadName"),
|
|
serviceWriteName: stringValue(gitMirror.serviceWriteName, "gitMirror.serviceWriteName"),
|
|
cachePvcName: stringValue(gitMirror.cachePvcName, "gitMirror.cachePvcName"),
|
|
cachePvcStorage: stringValue(gitMirror.cachePvcStorage, "gitMirror.cachePvcStorage"),
|
|
cacheHostPath: optionalStringValue(gitMirror.cacheHostPath, "gitMirror.cacheHostPath"),
|
|
servicePort: positiveIntegerValue(gitMirror.servicePort, "gitMirror.servicePort"),
|
|
secretName: stringValue(gitMirror.secretName, "gitMirror.secretName"),
|
|
syncConfigMapName: stringValue(gitMirror.syncConfigMapName, "gitMirror.syncConfigMapName"),
|
|
syncJobPrefix: stringValue(gitMirror.syncJobPrefix, "gitMirror.syncJobPrefix"),
|
|
flushJobPrefix: stringValue(gitMirror.flushJobPrefix, "gitMirror.flushJobPrefix"),
|
|
toolsImage: stringValue(toolsImage.output, "tekton.toolsImage.output"),
|
|
toolsImagePullPolicy,
|
|
sourceRepository: stringValue(source.repository, "source.repository"),
|
|
sourceBranch: stringValue(source.branch, "source.branch"),
|
|
gitopsBranch: stringValue(gitops.branch, "gitops.branch"),
|
|
egressProxy: nodeEgressProxy,
|
|
githubTransport,
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeGitMirrorGithubTransportSpec(raw: Record<string, unknown>, path: string): NodeRuntimeGitMirrorGithubTransportSpec {
|
|
const mode = stringValue(raw.mode, `${path}.mode`);
|
|
if (mode === "ssh") return { mode };
|
|
if (mode !== "https") throw new Error(`${path}.mode must be ssh or https`);
|
|
return {
|
|
mode,
|
|
username: stringValue(raw.username, `${path}.username`),
|
|
tokenSecretName: stringValue(raw.tokenSecretName, `${path}.tokenSecretName`),
|
|
tokenSecretKey: stringValue(raw.tokenSecretKey, `${path}.tokenSecretKey`),
|
|
tokenSourceRef: stringValue(raw.tokenSourceRef, `${path}.tokenSourceRef`),
|
|
tokenSourceKey: stringValue(raw.tokenSourceKey, `${path}.tokenSourceKey`),
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeGitMirrorEgressProxySpec(raw: Record<string, unknown>, path: string): NodeRuntimeGitMirrorEgressProxySpec {
|
|
const mode = stringValue(raw.mode, `${path}.mode`);
|
|
if (mode !== "k8s-service-cluster-ip") throw new Error(`${path}.mode must be k8s-service-cluster-ip`);
|
|
const port = Number(raw.port);
|
|
if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error(`${path}.port must be a TCP port`);
|
|
const sourceType = stringValue(raw.sourceType, `${path}.sourceType`);
|
|
if (sourceType !== "subscription-url") throw new Error(`${path}.sourceType must be subscription-url`);
|
|
const noProxyRaw = raw.noProxy;
|
|
if (!Array.isArray(noProxyRaw)) throw new Error(`${path}.noProxy must be an array`);
|
|
const noProxy = noProxyRaw.map((item, index) => stringValue(item, `${path}.noProxy[${index}]`));
|
|
return {
|
|
mode,
|
|
clientName: stringValue(raw.clientName, `${path}.clientName`),
|
|
namespace: stringValue(raw.namespace, `${path}.namespace`),
|
|
serviceName: stringValue(raw.serviceName, `${path}.serviceName`),
|
|
port,
|
|
sourceRef: stringValue(raw.sourceRef, `${path}.sourceRef`),
|
|
sourceKey: stringValue(raw.sourceKey, `${path}.sourceKey`),
|
|
sourceType,
|
|
noProxy,
|
|
};
|
|
}
|
|
|
|
function secretKeyStatus(spec: HwlabRuntimeLaneSpec, secret: string, key: string): Record<string, unknown> {
|
|
const result = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "secret", secret, "-o", `go-template={{ index .data ${JSON.stringify(key)} }}`], 60);
|
|
return {
|
|
secret,
|
|
key,
|
|
present: result.exitCode === 0 && result.stdout.trim().length > 0,
|
|
valueBytesBase64: result.stdout.trim().length,
|
|
exitCode: result.exitCode,
|
|
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 500),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function startNodeDelegatedJob(options: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const commandArgs = [
|
|
"hwlab",
|
|
"nodes",
|
|
options.domain,
|
|
...stripOptions(options.originalArgs, ["--node", "--lane", "--confirm", "--dry-run", "--wait", "--timeout-seconds"]),
|
|
"--node",
|
|
options.node,
|
|
"--lane",
|
|
options.lane,
|
|
"--confirm",
|
|
"--timeout-seconds",
|
|
String(options.timeoutSeconds),
|
|
"--wait",
|
|
];
|
|
const command = ["bun", "scripts/cli.ts", ...commandArgs];
|
|
const job = startJob(
|
|
`hwlab_nodes_${options.lane}_${options.domain}_${options.action}`,
|
|
command,
|
|
`Run HWLAB ${options.lane} ${options.domain} ${options.action} for node ${options.node}`,
|
|
);
|
|
return {
|
|
ok: true,
|
|
command: `hwlab nodes ${options.domain} ${options.action} --node ${options.node} --lane ${options.lane}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
mode: "async-job",
|
|
reason: "confirmed control-plane/mirror actions can spend tens of seconds on remote work; default is fire-and-forget to avoid silent blocking",
|
|
job,
|
|
statusCommand: `bun scripts/cli.ts job status ${job.id}`,
|
|
waitCommand: command.join(" "),
|
|
};
|
|
}
|
|
|
|
function rewriteDelegatedNodeResult(value: unknown, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const rewritten = rewriteDelegatedNodeValue(value, scoped);
|
|
const result = typeof rewritten === "object" && rewritten !== null && !Array.isArray(rewritten) ? rewritten as Record<string, unknown> : { value: rewritten };
|
|
return {
|
|
...result,
|
|
node: scoped.node,
|
|
commandFamily: "hwlab nodes",
|
|
};
|
|
}
|
|
|
|
function rewriteDelegatedNodeValue(value: unknown, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): unknown {
|
|
if (typeof value === "string") return rewriteDelegatedNodeString(value, scoped);
|
|
if (Array.isArray(value)) return value.map((item) => rewriteDelegatedNodeValue(item, scoped));
|
|
if (typeof value !== "object" || value === null) return value;
|
|
return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, rewriteDelegatedNodeValue(item, scoped)]));
|
|
}
|
|
|
|
function rewriteDelegatedNodeString(value: string, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): string {
|
|
const replaceCommand = (text: string, domain: DelegatedNodeDomain) => {
|
|
const escapedDomain = domain.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
|
return text
|
|
.replace(new RegExp(`bun scripts/cli\\.ts hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `bun scripts/cli.ts hwlab nodes ${domain} $1 --node ${scoped.node}`)
|
|
.replace(new RegExp(`hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `hwlab nodes ${domain} $1 --node ${scoped.node}`);
|
|
};
|
|
return replaceCommand(replaceCommand(value, "control-plane"), "git-mirror");
|
|
}
|
|
|
|
function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions {
|
|
const [actionRaw] = args;
|
|
if (actionRaw !== "run" && actionRaw !== "script" && actionRaw !== "observe") throw new Error("web-probe usage: run|script|observe --node NODE --lane vNN [--url URL]");
|
|
if (actionRaw === "observe") {
|
|
const normalized = normalizeNodeWebProbeObserveArgs(args.slice(1));
|
|
const explicitNode = optionValue(args, "--node");
|
|
const explicitLane = optionValue(args, "--lane");
|
|
const requestedId = normalized.id ?? optionValue(normalized.args, "--job-id") ?? null;
|
|
const indexed = requestedId === null
|
|
? null
|
|
: readWebObserveIndexEntry(requestedId) ?? (explicitNode === undefined || explicitLane === undefined ? discoverWebObserveIndexEntry(requestedId) : null);
|
|
const node = explicitNode ?? indexed?.node;
|
|
const lane = explicitLane ?? indexed?.lane;
|
|
if (node === undefined || lane === undefined) {
|
|
if (requestedId !== null) {
|
|
throw new Error(`web-probe observe observer-id-unknown: ${requestedId}; run observe start first, or pass --node/--lane with --job-id/--state-dir to re-index this observer`);
|
|
}
|
|
throw new Error("web-probe observe node-lane-required-for-start: observe start requires --node/--lane; status|command|stop|collect|analyze should pass an observer id");
|
|
}
|
|
assertNodeId(node);
|
|
assertLane(lane);
|
|
if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`);
|
|
const spec = hwlabRuntimeLaneSpecForNode(lane, node);
|
|
return parseNodeWebProbeObserveOptions(normalized.args, node, lane, spec, normalized.id, indexed);
|
|
}
|
|
const node = requiredOption(args, "--node");
|
|
assertNodeId(node);
|
|
const lane = requiredOption(args, "--lane");
|
|
assertLane(lane);
|
|
if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`);
|
|
const spec = hwlabRuntimeLaneSpecForNode(lane, node);
|
|
if (actionRaw === "script") {
|
|
assertKnownOptions(args.slice(1), new Set([
|
|
"--node",
|
|
"--lane",
|
|
"--url",
|
|
"--timeout-ms",
|
|
"--viewport",
|
|
"--browser-proxy-mode",
|
|
"--script-file",
|
|
"--command-timeout-seconds",
|
|
]), new Set([]));
|
|
const scriptFile = optionValue(args, "--script-file");
|
|
if (scriptFile === undefined && process.stdin.isTTY) {
|
|
throw new Error("web-probe script requires a stdin heredoc or --script-file <path>");
|
|
}
|
|
const scriptText = scriptFile === undefined ? readFileSync(0, "utf8") : readFileSync(scriptFile, "utf8");
|
|
if (scriptText.trim().length === 0) throw new Error("web-probe script received an empty script");
|
|
const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 60000);
|
|
const commandTimeoutSeconds = positiveIntegerOption(args, "--command-timeout-seconds", Math.max(60, Math.ceil(timeoutMs / 1000) + 30), 3600);
|
|
return {
|
|
action: "script",
|
|
node,
|
|
lane,
|
|
url: optionValue(args, "--url") ?? nodeWebProbeDefaultUrl(spec),
|
|
timeoutMs,
|
|
viewport: optionValue(args, "--viewport") ?? "1440x900",
|
|
browserProxyMode: parseWebProbeBrowserProxyMode(optionValue(args, "--browser-proxy-mode") ?? spec.webProbe?.browserProxyMode),
|
|
commandTimeoutSeconds,
|
|
scriptText,
|
|
scriptSource: {
|
|
kind: scriptFile === undefined ? "stdin" : "file",
|
|
path: scriptFile ?? null,
|
|
byteCount: Buffer.byteLength(scriptText),
|
|
sha256: `sha256:${createHash("sha256").update(scriptText).digest("hex")}`,
|
|
},
|
|
};
|
|
}
|
|
assertKnownOptions(args.slice(1), new Set([
|
|
"--node",
|
|
"--lane",
|
|
"--url",
|
|
"--timeout-ms",
|
|
"--wait-after-submit-ms",
|
|
"--wait-messages-ms",
|
|
"--wait-agent-terminal-ms",
|
|
"--trace-sample-count",
|
|
"--trace-sample-interval-ms",
|
|
"--message",
|
|
"--conversation-id",
|
|
"--command-timeout-seconds",
|
|
]), new Set([
|
|
"--fresh-session",
|
|
"--no-cancel-running",
|
|
]));
|
|
const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 60000);
|
|
const waitAfterSubmitMs = positiveIntegerOption(args, "--wait-after-submit-ms", 1500, 60000);
|
|
const waitMessagesMs = positiveIntegerOption(args, "--wait-messages-ms", 2500, 60000);
|
|
const waitAgentTerminalMs = positiveIntegerOption(args, "--wait-agent-terminal-ms", 0, 600000);
|
|
const message = optionValue(args, "--message") ?? null;
|
|
const hasMessage = message !== null;
|
|
const traceSampleCountDefault = hasMessage ? 2 : 0;
|
|
const traceSampleIntervalDefault = hasMessage ? 1500 : 0;
|
|
const traceSampleCount = positiveIntegerOption(args, "--trace-sample-count", traceSampleCountDefault, 200);
|
|
const traceSampleIntervalMs = positiveIntegerOption(args, "--trace-sample-interval-ms", traceSampleIntervalDefault, 60000);
|
|
const commandTimeoutAutoSeconds = nodeWebProbeAutoCommandTimeoutSeconds({
|
|
timeoutMs,
|
|
waitAfterSubmitMs,
|
|
waitMessagesMs,
|
|
waitAgentTerminalMs,
|
|
traceSampleCount,
|
|
traceSampleIntervalMs,
|
|
freshSession: args.includes("--fresh-session"),
|
|
hasMessage,
|
|
});
|
|
const commandTimeoutRaw = optionValue(args, "--command-timeout-seconds");
|
|
const commandTimeoutUserProvided = commandTimeoutRaw !== undefined;
|
|
const commandTimeoutSeconds = commandTimeoutUserProvided
|
|
? Math.max(positiveIntegerOption(args, "--command-timeout-seconds", commandTimeoutAutoSeconds, 3600), commandTimeoutAutoSeconds)
|
|
: commandTimeoutAutoSeconds;
|
|
return {
|
|
action: "run",
|
|
node,
|
|
lane,
|
|
url: optionValue(args, "--url") ?? nodeWebProbeDefaultUrl(spec),
|
|
timeoutMs,
|
|
waitAfterSubmitMs,
|
|
waitMessagesMs,
|
|
waitAgentTerminalMs,
|
|
traceSampleCount,
|
|
traceSampleIntervalMs,
|
|
message,
|
|
conversationId: optionValue(args, "--conversation-id") ?? null,
|
|
freshSession: args.includes("--fresh-session"),
|
|
cancelRunning: !args.includes("--no-cancel-running"),
|
|
commandTimeoutSeconds,
|
|
commandTimeoutAutoSeconds,
|
|
commandTimeoutUserProvided,
|
|
};
|
|
}
|
|
|
|
function normalizeNodeWebProbeObserveArgs(args: string[]): { args: string[]; id: string | null } {
|
|
const [observeActionRaw, maybeId, ...rest] = args;
|
|
if (observeActionRaw !== "start" && maybeId !== undefined && !maybeId.startsWith("--")) {
|
|
if (!isSafeWebObserveJobId(maybeId)) throw new Error(`unsafe web-probe observe id: ${maybeId}`);
|
|
return { args: [observeActionRaw, ...rest], id: maybeId };
|
|
}
|
|
return { args, id: null };
|
|
}
|
|
|
|
function parseNodeWebProbeObserveOptions(
|
|
args: string[],
|
|
node: string,
|
|
lane: string,
|
|
spec: HwlabRuntimeLaneSpec,
|
|
observeId: string | null,
|
|
indexed: WebObserveIndexEntry | null,
|
|
): NodeWebProbeObserveOptions {
|
|
const [observeActionRaw] = args;
|
|
if (
|
|
observeActionRaw !== "start"
|
|
&& observeActionRaw !== "status"
|
|
&& observeActionRaw !== "command"
|
|
&& observeActionRaw !== "stop"
|
|
&& observeActionRaw !== "collect"
|
|
&& observeActionRaw !== "analyze"
|
|
) {
|
|
throw new Error("web-probe observe usage: observe start --node NODE --lane vNN [...]; observe status|command|stop|collect|analyze <id> [...]");
|
|
}
|
|
assertKnownOptions(args.slice(1), new Set([
|
|
"--node",
|
|
"--lane",
|
|
"--url",
|
|
"--target-path",
|
|
"--viewport",
|
|
"--browser-proxy-mode",
|
|
"--sample-interval-ms",
|
|
"--screenshot-interval-ms",
|
|
"--observer-refresh-interval-ms",
|
|
"--max-samples",
|
|
"--command-timeout-seconds",
|
|
"--wait-ms",
|
|
"--tail-lines",
|
|
"--max-files",
|
|
"--view",
|
|
"--file",
|
|
"--finding",
|
|
"--grep",
|
|
"--trace-id",
|
|
"--sample-seq",
|
|
"--timestamp",
|
|
"--turn",
|
|
"--archive-prefix",
|
|
"--tail-samples",
|
|
"--state-dir",
|
|
"--job-id",
|
|
"--type",
|
|
"--text",
|
|
"--path",
|
|
"--label",
|
|
"--session-id",
|
|
"--provider",
|
|
]), new Set(["--force", "--full", "--text-stdin"]));
|
|
const commandTypeRaw = optionValue(args, "--type") ?? null;
|
|
const commandType = commandTypeRaw === null ? null : parseNodeWebProbeObserveCommandType(commandTypeRaw);
|
|
const stateDir = optionValue(args, "--state-dir") ?? indexed?.stateDir ?? null;
|
|
const jobId = optionValue(args, "--job-id") ?? observeId ?? indexed?.id ?? null;
|
|
if (stateDir !== null && !isSafeWebObserveStateDir(stateDir)) throw new Error(`unsafe web-probe observe --state-dir: ${stateDir}`);
|
|
if (jobId !== null && !isSafeWebObserveJobId(jobId)) throw new Error(`unsafe web-probe observe --job-id: ${jobId}`);
|
|
const collectView = parseNodeWebProbeObserveCollectView(optionValue(args, "--view") ?? "files");
|
|
const collectFile = optionValue(args, "--file") ?? null;
|
|
if (collectFile !== null && !isSafeWebObserveCollectFile(collectFile)) throw new Error(`unsafe web-probe observe --file: ${collectFile}`);
|
|
const collectFinding = optionValue(args, "--finding") ?? null;
|
|
if (collectFinding !== null && !isSafeWebObserveFindingId(collectFinding)) throw new Error(`unsafe web-probe observe --finding: ${collectFinding}`);
|
|
const collectGrep = optionValue(args, "--grep") ?? null;
|
|
if (collectGrep !== null && (collectGrep.includes("\0") || collectGrep.length > 200)) throw new Error("unsafe web-probe observe --grep: expected 1-200 non-NUL chars");
|
|
const collectTraceId = optionValue(args, "--trace-id") ?? null;
|
|
if (collectTraceId !== null && !isSafeWebObserveTraceId(collectTraceId)) throw new Error(`unsafe web-probe observe --trace-id: ${collectTraceId}`);
|
|
const collectSampleSeqRaw = optionValue(args, "--sample-seq") ?? null;
|
|
const collectSampleSeq = collectSampleSeqRaw === null ? null : Number(collectSampleSeqRaw);
|
|
if (collectSampleSeq !== null && (!Number.isInteger(collectSampleSeq) || collectSampleSeq < 1)) throw new Error("unsafe web-probe observe --sample-seq: expected positive integer");
|
|
const collectTimestamp = optionValue(args, "--timestamp") ?? null;
|
|
if (collectTimestamp !== null && (collectTimestamp.includes("\0") || collectTimestamp.length > 80 || !Number.isFinite(Date.parse(collectTimestamp)))) throw new Error("unsafe web-probe observe --timestamp: expected parseable timestamp");
|
|
const collectTurnRaw = optionValue(args, "--turn") ?? null;
|
|
const collectTurn = collectTurnRaw === null ? null : Number(collectTurnRaw);
|
|
if (collectTurn !== null && (!Number.isInteger(collectTurn) || collectTurn < 1)) throw new Error("unsafe web-probe observe --turn: expected positive integer");
|
|
const analyzeArchivePrefix = optionValue(args, "--archive-prefix") ?? null;
|
|
if (analyzeArchivePrefix !== null && !isSafeWebObserveArchivePrefix(analyzeArchivePrefix)) throw new Error(`unsafe web-probe observe --archive-prefix: ${analyzeArchivePrefix}`);
|
|
const analyzeTailSamplesRaw = optionValue(args, "--tail-samples") ?? null;
|
|
const analyzeTailSamples = analyzeTailSamplesRaw === null ? null : Number(analyzeTailSamplesRaw);
|
|
if (analyzeTailSamples !== null && (!Number.isInteger(analyzeTailSamples) || analyzeTailSamples < 0)) {
|
|
throw new Error("unsafe web-probe observe --tail-samples: expected a non-negative integer; use 0 for all samples");
|
|
}
|
|
if (observeActionRaw !== "start" && stateDir === null && jobId === null) {
|
|
throw new Error("web-probe observe status|command|stop|collect|analyze requires --state-dir or --job-id");
|
|
}
|
|
const commandTextOption = optionValue(args, "--text") ?? null;
|
|
const commandTextFromStdin = args.includes("--text-stdin");
|
|
if (commandTextFromStdin && observeActionRaw !== "command") {
|
|
throw new Error("web-probe observe --text-stdin is only supported for observe command");
|
|
}
|
|
if (commandTextFromStdin && commandTextOption !== null) {
|
|
throw new Error("web-probe observe command accepts either --text or --text-stdin, not both");
|
|
}
|
|
const commandText = commandTextFromStdin ? readFileSync(0, "utf8") : commandTextOption;
|
|
return {
|
|
action: "observe",
|
|
observeAction: observeActionRaw,
|
|
id: observeId ?? jobId,
|
|
node,
|
|
lane,
|
|
url: optionValue(args, "--url") ?? (observeActionRaw === "start" ? nodeWebProbeDefaultUrl(spec) : indexed?.url ?? spec.publicWebUrl),
|
|
targetPath: optionValue(args, "--target-path") ?? "/workbench",
|
|
viewport: optionValue(args, "--viewport") ?? "1440x900",
|
|
browserProxyMode: parseWebProbeBrowserProxyMode(optionValue(args, "--browser-proxy-mode") ?? spec.webProbe?.browserProxyMode),
|
|
sampleIntervalMs: positiveIntegerOption(args, "--sample-interval-ms", 5000, 600000),
|
|
screenshotIntervalMs: positiveIntegerOption(args, "--screenshot-interval-ms", 300000, 86_400_000),
|
|
observerRefreshIntervalMs: positiveIntegerOption(args, "--observer-refresh-interval-ms", 180000, 86_400_000),
|
|
maxSamples: positiveIntegerOption(args, "--max-samples", 0, 10_000_000),
|
|
commandTimeoutSeconds: positiveIntegerOption(args, "--command-timeout-seconds", 55, 3600),
|
|
waitMs: positiveIntegerOption(args, "--wait-ms", 0, 600000),
|
|
tailLines: positiveIntegerOption(args, "--tail-lines", 5, 200),
|
|
maxFiles: positiveIntegerOption(args, "--max-files", 80, 5000),
|
|
collectView,
|
|
collectFile,
|
|
collectFinding,
|
|
collectGrep,
|
|
collectTraceId,
|
|
collectSampleSeq,
|
|
collectTimestamp,
|
|
collectTurn,
|
|
analyzeArchivePrefix,
|
|
analyzeTailSamples,
|
|
full: args.includes("--full"),
|
|
stateDir,
|
|
jobId,
|
|
force: args.includes("--force"),
|
|
commandType,
|
|
commandText,
|
|
commandPath: optionValue(args, "--path") ?? null,
|
|
commandLabel: optionValue(args, "--label") ?? null,
|
|
commandSessionId: optionValue(args, "--session-id") ?? null,
|
|
commandProvider: optionValue(args, "--provider") ?? null,
|
|
};
|
|
}
|
|
|
|
function parseNodeWebProbeObserveCommandType(value: string): NodeWebProbeObserveCommandType {
|
|
if (
|
|
value === "login"
|
|
|| value === "preflight"
|
|
|| value === "goto"
|
|
|| value === "newSession"
|
|
|| value === "sendPrompt"
|
|
|| value === "steer"
|
|
|| value === "cancel"
|
|
|| value === "selectProvider"
|
|
|| value === "clickSession"
|
|
|| value === "screenshot"
|
|
|| value === "mark"
|
|
|| value === "stop"
|
|
) return value;
|
|
throw new Error(`web-probe observe command --type must be login, preflight, goto, newSession, sendPrompt, steer, cancel, selectProvider, clickSession, screenshot, mark, or stop; got ${value}`);
|
|
}
|
|
|
|
function parseWebProbeBrowserProxyMode(value: string | undefined): WebProbeBrowserProxyMode {
|
|
if (value === undefined || value === "auto") return "auto";
|
|
if (value === "direct") return "direct";
|
|
throw new Error(`web-probe --browser-proxy-mode must be auto or direct, got ${value}`);
|
|
}
|
|
|
|
function nodeWebProbeDefaultUrl(spec: HwlabRuntimeLaneSpec): string {
|
|
const origin = spec.webProbe?.defaultOrigin;
|
|
if (origin === undefined || origin.mode === "public") return origin?.baseUrl ?? spec.publicWebUrl;
|
|
const clusterIp = resolveKubernetesServiceClusterIp(spec, origin.namespace, origin.serviceName, new Map());
|
|
return `${origin.scheme}://${clusterIp}:${origin.port}`;
|
|
}
|
|
|
|
function nodeWebProbeAutoCommandTimeoutSeconds(input: {
|
|
timeoutMs: number;
|
|
waitAfterSubmitMs: number;
|
|
waitMessagesMs: number;
|
|
waitAgentTerminalMs: number;
|
|
traceSampleCount: number;
|
|
traceSampleIntervalMs: number;
|
|
freshSession: boolean;
|
|
hasMessage: boolean;
|
|
}): number {
|
|
const traceWindowMs = input.traceSampleCount > 0
|
|
? Math.max(0, input.traceSampleCount - 1) * input.traceSampleIntervalMs
|
|
: 0;
|
|
const startupBudgetMs = input.timeoutMs + 30_000;
|
|
const freshnessBudgetMs = input.freshSession ? Math.min(input.timeoutMs, 30_000) : 0;
|
|
const submitBudgetMs = input.hasMessage ? input.waitAfterSubmitMs + input.waitMessagesMs + 15_000 : 0;
|
|
const terminalBudgetMs = input.waitAgentTerminalMs > 0 ? input.waitAgentTerminalMs : 0;
|
|
const totalMs = startupBudgetMs + freshnessBudgetMs + submitBudgetMs + terminalBudgetMs + traceWindowMs + 15_000;
|
|
return Math.min(3600, Math.max(60, Math.ceil(totalMs / 1000)));
|
|
}
|
|
|
|
function assertKnownOptions(args: string[], valueOptions: Set<string>, flagOptions: Set<string>): void {
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index] ?? "";
|
|
if (!arg.startsWith("--")) continue;
|
|
if (flagOptions.has(arg)) continue;
|
|
if (valueOptions.has(arg)) {
|
|
index += 1;
|
|
continue;
|
|
}
|
|
throw new Error(`unknown option: ${arg}`);
|
|
}
|
|
}
|
|
|
|
function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, unknown> {
|
|
const lane = options.lane;
|
|
if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`);
|
|
const spec = hwlabRuntimeLaneSpecForNode(lane, options.node);
|
|
if (options.action === "observe" && options.observeAction !== "start") return runNodeWebProbeObserve(options, spec, null, null, null);
|
|
const secretSpec = runtimeSecretSpec({ node: options.node, lane });
|
|
const material = readBootstrapAdminPasswordMaterial(secretSpec);
|
|
const credential = webProbeCredential(secretSpec, material);
|
|
if (!material.ok || material.password === null) {
|
|
return {
|
|
ok: false,
|
|
status: "blocked",
|
|
command: options.action === "observe"
|
|
? `hwlab nodes web-probe observe ${options.observeAction} --node ${options.node} --lane ${options.lane}`
|
|
: `hwlab nodes web-probe ${options.action} --node ${options.node} --lane ${options.lane}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
workspace: spec.workspace,
|
|
url: options.url,
|
|
degradedReason: "web_login_secret_missing",
|
|
credential,
|
|
next: { secretStatus: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name ${secretSpec.bootstrapAdminSecret}` },
|
|
};
|
|
}
|
|
if (options.action === "observe") return runNodeWebProbeObserve(options, spec, secretSpec, material, credential);
|
|
if (options.action === "script") return runNodeWebProbeScript(options, spec, secretSpec, material, credential);
|
|
if (options.commandTimeoutSeconds > 55) return runNodeWebProbeAsync(options, spec, secretSpec, material, credential);
|
|
const probeArgs = nodeWebProbeRunArgs(options, "run");
|
|
const webProbeProxy = nodeWebProbeHostProxyEnv(spec);
|
|
const script = [
|
|
"set -eu",
|
|
`${[...webProbeProxy.envAssignments, `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password)}`].join(" ")} ${probeArgs.map(shellQuote).join(" ")}`,
|
|
].join("\n");
|
|
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
|
const probe = compactWebProbeResult(parseJsonObject(result.stdout));
|
|
const passed = result.exitCode === 0 && probe?.status === "pass";
|
|
const summary = nullableRecord(probe?.summary);
|
|
const degradedReason = result.timedOut
|
|
? "web-probe-command-timeout"
|
|
: typeof probe?.degradedReason === "string"
|
|
? probe.degradedReason
|
|
: null;
|
|
return renderWebProbeRunResult({
|
|
ok: passed,
|
|
status: passed ? "pass" : "blocked",
|
|
command: `hwlab nodes web-probe run --node ${options.node} --lane ${options.lane}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
workspace: spec.workspace,
|
|
url: options.url,
|
|
network: webProbeProxy.summary,
|
|
credential,
|
|
commandTimeout: {
|
|
seconds: options.commandTimeoutSeconds,
|
|
autoSeconds: options.commandTimeoutAutoSeconds,
|
|
userProvided: options.commandTimeoutUserProvided,
|
|
timedOut: result.timedOut,
|
|
},
|
|
degradedReason,
|
|
failureKind: typeof summary?.failureKind === "string" ? summary.failureKind : null,
|
|
summary,
|
|
probe,
|
|
result: compactCommandResult(result),
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
|
|
function nodeWebProbeRunArgs(options: NodeWebProbeRunOptions, command: "run" | "start"): string[] {
|
|
const probeArgs = [
|
|
"node",
|
|
"scripts/web-live-dom-probe.mjs",
|
|
command,
|
|
"--url", options.url,
|
|
"--timeout-ms", String(options.timeoutMs),
|
|
"--wait-after-submit-ms", String(options.waitAfterSubmitMs),
|
|
"--wait-messages-ms", String(options.waitMessagesMs),
|
|
];
|
|
if (options.waitAgentTerminalMs > 0) probeArgs.push("--wait-agent-terminal-ms", String(options.waitAgentTerminalMs));
|
|
if (options.traceSampleCount > 0) probeArgs.push("--trace-sample-count", String(options.traceSampleCount), "--trace-sample-interval-ms", String(options.traceSampleIntervalMs));
|
|
if (options.freshSession) probeArgs.push("--fresh-session");
|
|
if (!options.cancelRunning) probeArgs.push("--no-cancel-running");
|
|
if (options.conversationId !== null) probeArgs.push("--conversation-id", options.conversationId);
|
|
if (options.message !== null) probeArgs.push("--message", options.message);
|
|
return probeArgs;
|
|
}
|
|
|
|
function runNodeWebProbeAsync(
|
|
options: NodeWebProbeRunOptions,
|
|
spec: HwlabRuntimeLaneSpec,
|
|
secretSpec: RuntimeSecretSpec,
|
|
material: BootstrapAdminPasswordMaterial,
|
|
credential: Record<string, unknown>,
|
|
): Record<string, unknown> {
|
|
const startArgs = nodeWebProbeRunArgs(options, "start");
|
|
const webProbeProxy = nodeWebProbeHostProxyEnv(spec);
|
|
const startScript = [
|
|
"set -eu",
|
|
`${[...webProbeProxy.envAssignments, `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password ?? "")}`].join(" ")} ${startArgs.map(shellQuote).join(" ")}`,
|
|
].join("\n");
|
|
const startResult = runTransWorkspaceStdinScript(options.node, spec.workspace, startScript, 55);
|
|
const start = parseJsonObject(startResult.stdout);
|
|
const jobId = typeof start?.jobId === "string" ? start.jobId : null;
|
|
if (startResult.exitCode !== 0 || jobId === null) {
|
|
return renderWebProbeRunResult({
|
|
ok: false,
|
|
status: "blocked",
|
|
command: `hwlab nodes web-probe run --node ${options.node} --lane ${options.lane}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
workspace: spec.workspace,
|
|
url: options.url,
|
|
network: webProbeProxy.summary,
|
|
credential,
|
|
mode: "async-start",
|
|
commandTimeout: webProbeCommandTimeoutSummary(options, false),
|
|
degradedReason: startResult.timedOut ? "web-probe-command-timeout" : "web-probe-async-start-failed",
|
|
start: start ?? null,
|
|
result: compactCommandResultRedacted(startResult, [material.password ?? ""]),
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
const poll = pollNodeWebProbeJob(options, spec, jobId);
|
|
const statusReport = record(record(poll.status).report);
|
|
const reportPath = typeof start.reportPath === "string" ? start.reportPath : null;
|
|
const reportLoad = Object.keys(statusReport).length > 0
|
|
? { source: "status", report: statusReport, result: null as CommandResult | null, degradedReason: null as string | null, path: null as string | null }
|
|
: readNodeWebProbeReport(options, spec, reportPath);
|
|
const report = reportLoad.report ?? {};
|
|
const reportRecovered = Object.keys(report).length > 0;
|
|
const probe = compactWebProbeResult(Object.keys(report).length > 0 ? report : null);
|
|
const passed = probe?.status === "pass";
|
|
const summary = nullableRecord(probe?.summary);
|
|
const degradedReason = poll.timedOut
|
|
? "web-probe-command-timeout"
|
|
: typeof probe?.degradedReason === "string"
|
|
? probe.degradedReason
|
|
: poll.status === null
|
|
? reportRecovered
|
|
? reportLoad.degradedReason
|
|
: reportLoad.degradedReason ?? "web-probe-async-status-failed"
|
|
: reportLoad.degradedReason;
|
|
return renderWebProbeRunResult({
|
|
ok: passed,
|
|
status: passed ? "pass" : "blocked",
|
|
command: `hwlab nodes web-probe run --node ${options.node} --lane ${options.lane}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
workspace: spec.workspace,
|
|
url: options.url,
|
|
network: webProbeProxy.summary,
|
|
credential,
|
|
mode: "async",
|
|
commandTimeout: webProbeCommandTimeoutSummary(options, poll.timedOut),
|
|
degradedReason,
|
|
failureKind: typeof summary?.failureKind === "string" ? summary.failureKind : null,
|
|
summary,
|
|
job: {
|
|
jobId,
|
|
startedAt: start.startedAt ?? null,
|
|
polls: poll.polls,
|
|
elapsedMs: poll.elapsedMs,
|
|
statusCommand: start.statusCommand ?? `node scripts/web-live-dom-probe.mjs status ${jobId}`,
|
|
},
|
|
probe,
|
|
start: {
|
|
ok: start.ok === true,
|
|
status: start.status ?? null,
|
|
jobId,
|
|
traceSampling: start.traceSampling ?? null,
|
|
reportPath: start.reportPath ?? null,
|
|
screenshotPath: start.screenshotPath ?? null,
|
|
},
|
|
statusResult: poll.result === null ? null : compactCommandResult(poll.result),
|
|
reportLoad: {
|
|
source: reportLoad.source,
|
|
path: reportLoad.path,
|
|
result: reportLoad.result === null ? null : compactCommandResult(reportLoad.result),
|
|
degradedReason: reportLoad.degradedReason,
|
|
},
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
|
|
function readNodeWebProbeReport(
|
|
options: NodeWebProbeRunOptions,
|
|
spec: HwlabRuntimeLaneSpec,
|
|
reportPath: string | null,
|
|
): { source: string; report: Record<string, unknown> | null; result: CommandResult | null; degradedReason: string | null; path: string | null } {
|
|
if (!reportPath) return { source: "missing", report: null, result: null, degradedReason: "web-probe-report-path-missing", path: null };
|
|
if (!isSafeWebProbeReportPath(reportPath)) return { source: "unsafe-path", report: null, result: null, degradedReason: "web-probe-report-path-invalid", path: reportPath };
|
|
const script = [
|
|
"set -eu",
|
|
`test -f ${shellQuote(reportPath)}`,
|
|
`node - ${shellQuote(reportPath)} <<'NODE'`,
|
|
"const fs = require('fs');",
|
|
"const reportPath = process.argv[2];",
|
|
"const report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));",
|
|
"function rec(value) { return value && typeof value === 'object' && !Array.isArray(value) ? value : {}; }",
|
|
"function compact(value, depth = 0) {",
|
|
" if (value === null || value === undefined) return value ?? null;",
|
|
" if (typeof value === 'string') return value.replace(/\\s+/gu, ' ').trim().slice(0, 600);",
|
|
" if (typeof value === 'number' || typeof value === 'boolean') return value;",
|
|
" if (depth >= 6) return '[max-depth]';",
|
|
" if (Array.isArray(value)) return value.slice(0, 16).map((item) => compact(item, depth + 1));",
|
|
" if (typeof value === 'object') {",
|
|
" const out = {};",
|
|
" for (const [key, nested] of Object.entries(value).slice(0, 32)) out[key] = compact(nested, depth + 1);",
|
|
" return out;",
|
|
" }",
|
|
" return String(value).slice(0, 600);",
|
|
"}",
|
|
"const artifacts = rec(report.artifacts);",
|
|
"const compactReport = {",
|
|
" ok: report.ok === true,",
|
|
" status: typeof report.status === 'string' ? report.status : null,",
|
|
" finalUrl: typeof report.finalUrl === 'string' ? report.finalUrl : null,",
|
|
" error: typeof report.error === 'string' ? report.error : null,",
|
|
" degradedReason: typeof report.degradedReason === 'string' ? report.degradedReason : null,",
|
|
" actions: compact(report.actions),",
|
|
" session: compact(report.session),",
|
|
" trace: compact(report.trace),",
|
|
" promptValidation: compact(report.promptValidation),",
|
|
" performance: compact(report.performance),",
|
|
" traceSamples: compact(report.traceSamples),",
|
|
" dom: compact(report.dom),",
|
|
" failureDom: compact(report.failureDom),",
|
|
" artifacts: {",
|
|
" ...compact(artifacts),",
|
|
" reportPath: typeof artifacts.reportPath === 'string' ? artifacts.reportPath : reportPath,",
|
|
" },",
|
|
" safety: compact(report.safety),",
|
|
"};",
|
|
"console.log(JSON.stringify(compactReport));",
|
|
"NODE",
|
|
].join("\n");
|
|
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, 55);
|
|
if (result.exitCode !== 0 || result.timedOut) return { source: "report-file", report: null, result, degradedReason: result.timedOut ? "web-probe-command-timeout" : "web-probe-report-read-failed", path: reportPath };
|
|
const report = parseJsonObject(result.stdout);
|
|
return {
|
|
source: "report-file",
|
|
report,
|
|
result,
|
|
degradedReason: report === null ? "web-probe-report-parse-failed" : null,
|
|
path: reportPath,
|
|
};
|
|
}
|
|
|
|
function isSafeWebProbeReportPath(reportPath: string): boolean {
|
|
return reportPath.includes("/.state/web-live-dom-probe/") && reportPath.endsWith(".result.json") && !reportPath.includes("\0");
|
|
}
|
|
|
|
function pollNodeWebProbeJob(options: NodeWebProbeRunOptions, spec: HwlabRuntimeLaneSpec, jobId: string): {
|
|
status: Record<string, unknown> | null;
|
|
result: CommandResult | null;
|
|
polls: number;
|
|
elapsedMs: number;
|
|
timedOut: boolean;
|
|
} {
|
|
const startedAt = Date.now();
|
|
const deadline = startedAt + options.commandTimeoutSeconds * 1000;
|
|
let lastStatus: Record<string, unknown> | null = null;
|
|
let lastResult: CommandResult | null = null;
|
|
let polls = 0;
|
|
while (Date.now() < deadline) {
|
|
polls += 1;
|
|
const script = `node scripts/web-live-dom-probe.mjs status ${shellQuote(jobId)}`;
|
|
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, 55);
|
|
lastResult = result;
|
|
lastStatus = parseJsonObject(result.stdout);
|
|
const status = typeof lastStatus?.status === "string" ? lastStatus.status : null;
|
|
if (result.exitCode === 0 && status !== "running") return { status: lastStatus, result, polls, elapsedMs: Date.now() - startedAt, timedOut: false };
|
|
if (result.exitCode !== 0 && status !== "running") return { status: lastStatus, result, polls, elapsedMs: Date.now() - startedAt, timedOut: false };
|
|
sleepSync(Math.min(5000, Math.max(500, deadline - Date.now())));
|
|
}
|
|
return { status: lastStatus, result: lastResult, polls, elapsedMs: Date.now() - startedAt, timedOut: true };
|
|
}
|
|
|
|
function webProbeCommandTimeoutSummary(options: NodeWebProbeRunOptions, timedOut: boolean): Record<string, unknown> {
|
|
return {
|
|
seconds: options.commandTimeoutSeconds,
|
|
autoSeconds: options.commandTimeoutAutoSeconds,
|
|
userProvided: options.commandTimeoutUserProvided,
|
|
transportMode: options.commandTimeoutSeconds > 55 ? "async-start-status" : "direct",
|
|
timedOut,
|
|
};
|
|
}
|
|
|
|
function webProbeCredential(secretSpec: RuntimeSecretSpec, material: BootstrapAdminPasswordMaterial): Record<string, unknown> {
|
|
return {
|
|
username: secretSpec.bootstrapAdminUsername,
|
|
sourceRef: material.sourceRef,
|
|
sourceKey: material.sourceKey,
|
|
sourcePath: material.sourcePath === null ? null : displayRepoPath(material.sourcePath),
|
|
sourcePresent: material.sourcePresent,
|
|
sourceFingerprint: material.sourceFingerprint,
|
|
injectedVia: material.ok ? "stdin-env" : null,
|
|
valuesRedacted: true,
|
|
error: material.error,
|
|
};
|
|
}
|
|
|
|
function nodeWebProbeAlertThresholds(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeAlertThresholdsSpec {
|
|
const thresholds = spec.webProbe?.alertThresholds;
|
|
if (thresholds === undefined) {
|
|
throw new Error(`${hwlabRuntimeLaneConfigPath()} node=${spec.nodeId} lane=${spec.lane} requires webProbe.alertThresholds for web-probe observe`);
|
|
}
|
|
return thresholds;
|
|
}
|
|
|
|
interface NodeWebProbeHostProxyEnv {
|
|
readonly envAssignments: string[];
|
|
readonly summary: Record<string, unknown>;
|
|
}
|
|
|
|
function nodeWebProbeHostProxyEnv(spec: HwlabRuntimeLaneSpec, browserProxyMode: WebProbeBrowserProxyMode = "auto"): NodeWebProbeHostProxyEnv {
|
|
if (browserProxyMode === "direct") {
|
|
return {
|
|
envAssignments: [],
|
|
summary: {
|
|
source: "option",
|
|
mode: "direct",
|
|
networkProfileId: spec.networkProfileId,
|
|
proxy: { enabled: false },
|
|
valuesPrinted: false,
|
|
},
|
|
};
|
|
}
|
|
const proxy = spec.networkProfile.proxy;
|
|
const serviceCache = new Map<string, string>();
|
|
const http = resolveNodeWebProbeHostProxyUrl(spec, proxy.http, serviceCache);
|
|
const https = resolveNodeWebProbeHostProxyUrl(spec, proxy.https, serviceCache);
|
|
const all = resolveNodeWebProbeHostProxyUrl(spec, proxy.all, serviceCache);
|
|
const noProxy = proxy.noProxy.join(",");
|
|
return {
|
|
envAssignments: [
|
|
["HTTP_PROXY", http.url],
|
|
["HTTPS_PROXY", https.url],
|
|
["ALL_PROXY", all.url],
|
|
["http_proxy", http.url],
|
|
["https_proxy", https.url],
|
|
["all_proxy", all.url],
|
|
["NO_PROXY", noProxy],
|
|
["no_proxy", noProxy],
|
|
].map(([key, value]) => `${key}=${shellQuote(value)}`),
|
|
summary: {
|
|
source: "yaml",
|
|
mode: "host-env",
|
|
networkProfileId: spec.networkProfileId,
|
|
proxy: {
|
|
http: http.summary,
|
|
https: https.summary,
|
|
all: all.summary,
|
|
noProxyCount: proxy.noProxy.length,
|
|
},
|
|
valuesPrinted: false,
|
|
},
|
|
};
|
|
}
|
|
|
|
function resolveNodeWebProbeHostProxyUrl(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
rawUrl: string,
|
|
serviceCache: Map<string, string>,
|
|
): { url: string; summary: Record<string, unknown> } {
|
|
let parsed: URL;
|
|
try {
|
|
parsed = new URL(rawUrl);
|
|
} catch (error) {
|
|
throw new Error(`config/hwlab-node-lanes.yaml networkProfiles.${spec.networkProfileId}.proxy contains invalid proxy URL: ${error instanceof Error ? error.message : String(error)}`);
|
|
}
|
|
const service = parseKubernetesServiceDnsHost(parsed.hostname);
|
|
if (service === null) {
|
|
return {
|
|
url: rawUrl,
|
|
summary: {
|
|
mode: "host-url",
|
|
host: parsed.hostname,
|
|
port: parsed.port || null,
|
|
valuesPrinted: false,
|
|
},
|
|
};
|
|
}
|
|
const clusterIp = resolveKubernetesServiceClusterIp(spec, service.namespace, service.name, serviceCache);
|
|
const originalHost = parsed.hostname;
|
|
parsed.hostname = clusterIp;
|
|
const resolvedUrl = normalizedProxyUrl(parsed);
|
|
return {
|
|
url: resolvedUrl,
|
|
summary: {
|
|
mode: "k8s-service-cluster-ip",
|
|
service: service.name,
|
|
namespace: service.namespace,
|
|
originalHost,
|
|
resolvedHost: clusterIp,
|
|
port: parsed.port || null,
|
|
valuesPrinted: false,
|
|
},
|
|
};
|
|
}
|
|
|
|
function parseKubernetesServiceDnsHost(hostname: string): { name: string; namespace: string } | null {
|
|
const match = hostname.toLowerCase().match(/^([a-z0-9]([-a-z0-9]*[a-z0-9])?)\.([a-z0-9]([-a-z0-9]*[a-z0-9])?)\.svc(?:\.cluster\.local)?$/u);
|
|
if (match === null) return null;
|
|
return { name: match[1] ?? "", namespace: match[3] ?? "" };
|
|
}
|
|
|
|
function resolveKubernetesServiceClusterIp(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
namespace: string,
|
|
serviceName: string,
|
|
serviceCache: Map<string, string>,
|
|
): string {
|
|
const cacheKey = `${namespace}/${serviceName}`;
|
|
const cached = serviceCache.get(cacheKey);
|
|
if (cached !== undefined) return cached;
|
|
const result = runCommand([transPath(), spec.nodeKubeRoute, "get", "svc", "-n", namespace, serviceName, "-o", "jsonpath={.spec.clusterIP}"], repoRoot, { timeoutMs: 20_000 });
|
|
const clusterIp = result.stdout.trim();
|
|
if (result.exitCode !== 0 || clusterIp.length === 0) {
|
|
const reason = result.stderr.trim().slice(-500) || result.stdout.trim().slice(-500) || `exitCode=${result.exitCode}`;
|
|
throw new Error(`web-probe proxy service resolution failed for ${spec.nodeId}/${spec.lane} ${namespace}/${serviceName}: ${reason}`);
|
|
}
|
|
serviceCache.set(cacheKey, clusterIp);
|
|
return clusterIp;
|
|
}
|
|
|
|
function normalizedProxyUrl(parsed: URL): string {
|
|
const value = parsed.toString();
|
|
if (parsed.pathname === "/" && parsed.search === "" && parsed.hash === "") return value.replace(/\/$/u, "");
|
|
return value;
|
|
}
|
|
|
|
function runNodeWebProbeObserve(
|
|
options: NodeWebProbeObserveOptions,
|
|
spec: HwlabRuntimeLaneSpec,
|
|
secretSpec: RuntimeSecretSpec | null,
|
|
material: BootstrapAdminPasswordMaterial | null,
|
|
credential: Record<string, unknown> | null,
|
|
): Record<string, unknown> {
|
|
if (options.observeAction === "start") {
|
|
if (secretSpec === null || material === null || credential === null || material.password === null) throw new Error("web-probe observe start requires bootstrap admin credential material");
|
|
return runNodeWebProbeObserveStart(options, spec, secretSpec, material, credential);
|
|
}
|
|
if (options.observeAction === "status") return runNodeWebProbeObserveStatus(options, spec);
|
|
if (options.observeAction === "command") return runNodeWebProbeObserveCommand(options, spec, false);
|
|
if (options.observeAction === "stop") return runNodeWebProbeObserveCommand({ ...options, commandType: "stop" }, spec, true);
|
|
if (options.observeAction === "collect") return runNodeWebProbeObserveCollect(options, spec);
|
|
return runNodeWebProbeObserveAnalyze(options, spec);
|
|
}
|
|
|
|
function runNodeWebProbeObserveStart(
|
|
options: NodeWebProbeObserveOptions,
|
|
spec: HwlabRuntimeLaneSpec,
|
|
secretSpec: RuntimeSecretSpec,
|
|
material: BootstrapAdminPasswordMaterial,
|
|
credential: Record<string, unknown>,
|
|
): Record<string, unknown> | RenderedCliResult {
|
|
const jobId = `webobs-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`;
|
|
const timestamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/[.]\d{3}Z$/u, "Z");
|
|
const day = timestamp.slice(0, 8);
|
|
const defaultStateDir = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}/${day.slice(0, 4)}/${day.slice(4, 6)}/${day.slice(6, 8)}/${timestamp}_${safeWebObserveTargetSegment(options.targetPath)}_${jobId}`;
|
|
const stateDir = options.stateDir ?? defaultStateDir;
|
|
const runnerB64 = Buffer.from(nodeWebObserveRunnerSource(), "utf8").toString("base64");
|
|
const runnerB64Body = runnerB64.match(/.{1,76}/gu)?.join("\n") ?? runnerB64;
|
|
const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode);
|
|
const alertThresholds = nodeWebProbeAlertThresholds(spec);
|
|
const runnerEnvAssignments = [
|
|
...webProbeProxy.envAssignments,
|
|
`HWLAB_WEB_BASE_URL=${shellQuote(options.url)}`,
|
|
`HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`,
|
|
`HWLAB_WEB_PASS=${shellQuote(material.password)}`,
|
|
`UNIDESK_WEB_OBSERVE_STATE_DIR=${shellQuote(stateDir)}`,
|
|
`UNIDESK_WEB_OBSERVE_JOB_ID=${shellQuote(jobId)}`,
|
|
`UNIDESK_WEB_OBSERVE_TARGET_PATH=${shellQuote(options.targetPath)}`,
|
|
`UNIDESK_WEB_OBSERVE_SAMPLE_INTERVAL_MS=${shellQuote(String(options.sampleIntervalMs))}`,
|
|
`UNIDESK_WEB_OBSERVE_SCREENSHOT_INTERVAL_MS=${shellQuote(String(options.screenshotIntervalMs))}`,
|
|
`UNIDESK_WEB_OBSERVE_OBSERVER_REFRESH_INTERVAL_MS=${shellQuote(String(options.observerRefreshIntervalMs))}`,
|
|
`UNIDESK_WEB_OBSERVE_MAX_SAMPLES=${shellQuote(String(options.maxSamples))}`,
|
|
`UNIDESK_WEB_OBSERVE_VIEWPORT=${shellQuote(options.viewport)}`,
|
|
`UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`,
|
|
`UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=${shellQuote(JSON.stringify(alertThresholds))}`,
|
|
].join(" ");
|
|
const script = [
|
|
"set -eu",
|
|
`state_dir=${shellQuote(stateDir)}`,
|
|
"mkdir -p \"$state_dir\"",
|
|
"chmod 700 \"$state_dir\"",
|
|
"runner=\"$state_dir/observer-runner.mjs\"",
|
|
"runner_b64=\"$state_dir/observer-runner.mjs.b64\"",
|
|
"cat >\"$runner_b64\" <<'UNIDESK_WEB_OBSERVE_RUNNER_B64'",
|
|
runnerB64Body,
|
|
"UNIDESK_WEB_OBSERVE_RUNNER_B64",
|
|
"node -e \"const fs=require('fs'); fs.writeFileSync(process.argv[1], Buffer.from(fs.readFileSync(process.argv[2], 'utf8').replace(/\\s+/g, ''), 'base64'))\" \"$runner\" \"$runner_b64\"",
|
|
"rm -f \"$runner_b64\"",
|
|
"chmod 700 \"$runner\"",
|
|
`if command -v setsid >/dev/null 2>&1; then setsid env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" </dev/null & else nohup env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" </dev/null & fi`,
|
|
"pid=$!",
|
|
"printf '%s\\n' \"$pid\" >\"$state_dir/pid\"",
|
|
"sleep 1",
|
|
`node -e ${shellQuote("const fs=require('fs'); const dir=process.argv[1]; const read=(n)=>{try{return JSON.parse(fs.readFileSync(dir+'/'+n,'utf8'))}catch{return null}}; const pid=fs.existsSync(dir+'/pid')?fs.readFileSync(dir+'/pid','utf8').trim():null; console.log(JSON.stringify({ok:true,command:'web-probe-observe start',jobId:process.argv[2],stateDir:dir,pid:Number(pid)||null,manifestPath:dir+'/manifest.json',heartbeat:read('heartbeat.json'),manifest:read('manifest.json'),statusCommand:'bun scripts/cli.ts hwlab nodes web-probe observe status --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,stopCommand:'bun scripts/cli.ts hwlab nodes web-probe observe stop --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,valuesRedacted:true},null,2))")} "$state_dir" ${shellQuote(jobId)} ${shellQuote(options.node)} ${shellQuote(options.lane)}`,
|
|
].join("\n");
|
|
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
|
const started = parseJsonObject(result.stdout);
|
|
const observerId = typeof started?.jobId === "string" ? started.jobId : jobId;
|
|
const index = result.exitCode === 0 && started?.ok === true
|
|
? upsertWebObserveIndexEntry({
|
|
id: observerId,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
workspace: spec.workspace,
|
|
stateDir,
|
|
url: options.url,
|
|
targetPath: options.targetPath,
|
|
status: "running",
|
|
pid: typeof started.pid === "number" ? started.pid : null,
|
|
startedAt: new Date().toISOString(),
|
|
updatedAt: new Date().toISOString(),
|
|
})
|
|
: null;
|
|
return renderWebObserveStartResult({
|
|
ok: result.exitCode === 0 && started?.ok === true,
|
|
status: result.exitCode === 0 && started?.ok === true ? "started" : "blocked",
|
|
command: `hwlab nodes web-probe observe start --node ${options.node} --lane ${options.lane}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
workspace: spec.workspace,
|
|
url: options.url,
|
|
network: webProbeProxy.summary,
|
|
alertThresholds,
|
|
targetPath: options.targetPath,
|
|
id: observerId,
|
|
credential,
|
|
observer: withWebObserveShortcuts(started, observerId),
|
|
index,
|
|
next: webObserveNextCommands(observerId),
|
|
result: compactCommandResultRedacted(result, [material.password ?? ""]),
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
|
|
function runNodeWebProbeObserveStatus(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> | RenderedCliResult {
|
|
const script = [
|
|
"set -eu",
|
|
nodeWebObserveResolveStateDirShell(options),
|
|
nodeWebObserveStatusNodeScript(options.tailLines, options.node, options.lane),
|
|
].join("\n");
|
|
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
|
const status = parseJsonObject(result.stdout);
|
|
const observerId = webObserveIdFromStatus(status, options);
|
|
const statusReadable = status !== null;
|
|
const ok = result.exitCode === 0 && statusReadable && status.ok !== false;
|
|
const degradedReason = result.timedOut
|
|
? "web-probe-command-timeout"
|
|
: result.exitCode !== 0
|
|
? "web-probe-observe-status-failed"
|
|
: !statusReadable
|
|
? "web-probe-observe-status-unreadable"
|
|
: typeof status.degradedReason === "string"
|
|
? status.degradedReason
|
|
: null;
|
|
const index = ok && observerId !== null && options.stateDir !== null
|
|
? upsertWebObserveIndexEntry(webObserveIndexEntryFromOptions(options, spec, observerId, status))
|
|
: null;
|
|
return withWebObserveStatusRendered({
|
|
ok,
|
|
status: ok ? "observed" : "blocked",
|
|
command: webObserveCommandLabel("status", options),
|
|
id: observerId,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
workspace: spec.workspace,
|
|
degradedReason,
|
|
observer: withWebObserveShortcuts(status, observerId),
|
|
index,
|
|
next: observerId === null ? null : webObserveNextCommands(observerId),
|
|
result: compactCommandResultWithStdoutTail(result),
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
|
|
function webObserveText(value: unknown): string {
|
|
if (value === null || value === undefined || value === "") return "-";
|
|
if (typeof value === "string") return value;
|
|
if (typeof value === "number" || typeof value === "boolean") return String(value);
|
|
return JSON.stringify(value);
|
|
}
|
|
|
|
function webObserveShort(value: string, maxLength: number): string {
|
|
if (value.length <= maxLength) return value;
|
|
if (maxLength <= 1) return value.slice(0, maxLength);
|
|
return `${value.slice(0, maxLength - 1)}~`;
|
|
}
|
|
|
|
function runNodeWebProbeObserveCommand(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec, stopCommand: boolean): Record<string, unknown> | RenderedCliResult {
|
|
const type = options.commandType ?? (stopCommand ? "stop" : null);
|
|
if (type === null) throw new Error("web-probe observe command requires --type");
|
|
const commandId = `cmd-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`;
|
|
const payload = {
|
|
id: commandId,
|
|
type,
|
|
createdAt: new Date().toISOString(),
|
|
source: "cli",
|
|
path: options.commandPath,
|
|
text: options.commandText,
|
|
label: options.commandLabel,
|
|
sessionId: options.commandSessionId,
|
|
provider: options.commandProvider,
|
|
};
|
|
const payloadB64 = Buffer.from(JSON.stringify(payload), "utf8").toString("base64");
|
|
const script = [
|
|
"set -eu",
|
|
nodeWebObserveResolveStateDirShell(options),
|
|
"mkdir -p \"$state_dir/commands/pending\"",
|
|
`node -e "const fs=require('fs'),path=require('path'); const dir=process.argv[1], id=process.argv[2], payload=Buffer.from(process.argv[3], 'base64').toString('utf8'); fs.writeFileSync(path.join(dir,'commands','pending',id+'.json'), payload+'\\n', {mode:0o600});" "$state_dir" ${shellQuote(commandId)} ${shellQuote(payloadB64)}`,
|
|
nodeWebObserveWaitCommandShell(commandId, options.waitMs),
|
|
].join("\n");
|
|
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
|
const commandResult = parseJsonObject(result.stdout);
|
|
if (options.force && stopCommand && result.exitCode !== 0) {
|
|
const killResult = runTransWorkspaceStdinScript(options.node, spec.workspace, [
|
|
"set -eu",
|
|
nodeWebObserveResolveStateDirShell(options),
|
|
"if [ -f \"$state_dir/pid\" ]; then kill \"$(cat \"$state_dir/pid\")\" >/dev/null 2>&1 || true; fi",
|
|
"printf '{\"ok\":true,\"forced\":true,\"stateDir\":\"%s\"}\\n' \"$state_dir\"",
|
|
].join("\n"), 55);
|
|
return withWebObserveCommandRendered({
|
|
ok: killResult.exitCode === 0,
|
|
status: killResult.exitCode === 0 ? "forced-stop-requested" : "blocked",
|
|
command: webObserveCommandLabel("stop", options),
|
|
id: webObserveIdFromOptions(options),
|
|
node: options.node,
|
|
lane: options.lane,
|
|
workspace: spec.workspace,
|
|
commandId,
|
|
observerCommand: commandSummaryForOutput(payload),
|
|
gracefulResult: compactCommandResult(result),
|
|
forceResult: compactCommandResult(killResult),
|
|
full: options.full,
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
return withWebObserveCommandRendered({
|
|
ok: result.exitCode === 0 && commandResult?.ok !== false,
|
|
status: result.exitCode === 0 ? (options.waitMs > 0 ? "completed-or-queued" : "queued") : "blocked",
|
|
command: webObserveCommandLabel(stopCommand ? "stop" : "command", options),
|
|
id: webObserveIdFromOptions(options),
|
|
node: options.node,
|
|
lane: options.lane,
|
|
workspace: spec.workspace,
|
|
commandId,
|
|
observerCommand: commandSummaryForOutput(payload),
|
|
observer: commandResult,
|
|
result: compactCommandResult(result),
|
|
full: options.full,
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
|
|
function runNodeWebProbeObserveCollect(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> | RenderedCliResult {
|
|
const collectScript = options.collectView === "files"
|
|
? nodeWebObserveCollectNodeScript(options.maxFiles, options.collectFile, options.collectFinding, options.collectGrep)
|
|
: nodeWebObserveCollectViewNodeScript({
|
|
maxFiles: options.maxFiles,
|
|
view: options.collectView,
|
|
traceId: options.collectTraceId,
|
|
sampleSeq: options.collectSampleSeq,
|
|
timestamp: options.collectTimestamp,
|
|
turn: options.collectTurn,
|
|
});
|
|
const script = [
|
|
"set -eu",
|
|
nodeWebObserveResolveStateDirShell(options),
|
|
collectScript,
|
|
].join("\n");
|
|
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
|
const collect = parseJsonObject(result.stdout);
|
|
return withWebObserveCollectRendered({
|
|
ok: result.exitCode === 0 && collect !== null && collect.ok !== false,
|
|
status: result.exitCode === 0 && collect !== null ? "collected" : "blocked",
|
|
command: webObserveCommandLabel("collect", options),
|
|
id: webObserveIdFromOptions(options),
|
|
node: options.node,
|
|
lane: options.lane,
|
|
workspace: spec.workspace,
|
|
view: options.collectView,
|
|
requestedFile: options.collectFile,
|
|
requestedGrep: options.collectGrep,
|
|
degradedReason: collect === null ? "collect-json-parse-failed" : null,
|
|
collect,
|
|
result: collect === null ? compactCommandResultWithStdoutTail(result) : compactCommandResult(result),
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
|
|
function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> | RenderedCliResult {
|
|
const analyzerB64 = Buffer.from(nodeWebObserveAnalyzerSource(), "utf8").toString("base64");
|
|
const alertThresholds = nodeWebProbeAlertThresholds(spec);
|
|
const script = [
|
|
"set -eu",
|
|
nodeWebObserveResolveStateDirShell(options),
|
|
"analyzer=\"$state_dir/observer-analyzer.mjs\"",
|
|
"analyzer_b64=\"$state_dir/observer-analyzer.mjs.b64\"",
|
|
"cat >\"$analyzer_b64\" <<'UNIDESK_WEB_OBSERVE_ANALYZER_B64'",
|
|
analyzerB64,
|
|
"UNIDESK_WEB_OBSERVE_ANALYZER_B64",
|
|
"node -e \"const fs=require('fs'); fs.writeFileSync(process.argv[1], Buffer.from(fs.readFileSync(process.argv[2], 'utf8').replace(/\\s+/g, ''), 'base64'))\" \"$analyzer\" \"$analyzer_b64\"",
|
|
"rm -f \"$analyzer_b64\"",
|
|
"chmod 700 \"$analyzer\"",
|
|
"mkdir -p \"$state_dir/analysis\"",
|
|
"analysis_stdout=\"$state_dir/analysis/analyzer-stdout.json\"",
|
|
"analysis_stderr=\"$state_dir/analysis/analyzer-stderr.log\"",
|
|
"set +e",
|
|
`UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX=${shellQuote(options.analyzeArchivePrefix ?? "")}`,
|
|
`UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES=${shellQuote(options.analyzeTailSamples === null ? "" : String(options.analyzeTailSamples))}`,
|
|
`UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=${shellQuote(JSON.stringify(alertThresholds))}`,
|
|
"UNIDESK_WEB_OBSERVE_STATE_DIR=\"$state_dir\" UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX=\"$UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX\" UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES=\"$UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES\" UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=\"$UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON\" node \"$analyzer\" >\"$analysis_stdout\" 2>\"$analysis_stderr\"",
|
|
"analyzer_exit=$?",
|
|
"set -e",
|
|
"report_json=\"$state_dir/analysis/report.json\"",
|
|
"report_md=\"$state_dir/analysis/report.md\"",
|
|
"node - \"$analysis_stdout\" \"$analysis_stderr\" \"$report_json\" \"$report_md\" \"$analyzer_exit\" <<'UNIDESK_WEB_OBSERVE_ANALYZE_COMPACT'",
|
|
"const fs = require('fs');",
|
|
"const crypto = require('crypto');",
|
|
"const [stdoutPath, stderrPath, reportJsonPath, reportMdPath, analyzerExitRaw] = process.argv.slice(2);",
|
|
"const analyzerExit = Number(analyzerExitRaw);",
|
|
"const readText = (path) => { try { return fs.readFileSync(path, 'utf8'); } catch { return ''; } };",
|
|
"const readJson = (path) => { const text = readText(path); if (!text.trim()) return null; try { return JSON.parse(text); } catch { return null; } };",
|
|
"const sha256 = (path) => { const text = readText(path); return text ? 'sha256:' + crypto.createHash('sha256').update(text).digest('hex') : null; };",
|
|
"const statSize = (path) => { try { return fs.statSync(path).size; } catch { return 0; } };",
|
|
"const tail = (text, limit = 1200) => String(text || '').slice(-limit);",
|
|
"const takeHead = (value, limit) => Array.isArray(value) ? value.slice(0, limit) : [];",
|
|
"const takeTail = (value, limit) => Array.isArray(value) ? value.slice(-limit) : [];",
|
|
"const firstArray = (...values) => { for (const value of values) if (Array.isArray(value)) return value; return []; };",
|
|
"const firstNonEmptyArray = (...values) => { for (const value of values) if (Array.isArray(value) && value.length > 0) return value; return firstArray(...values); };",
|
|
"const readJsonlTail = (path, limit) => readText(path).split(/\\r?\\n/).filter(Boolean).slice(-limit).map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean);",
|
|
"const mergeArrays = (...values) => { const out = []; const seen = new Set(); for (const value of values) { if (!Array.isArray(value)) continue; for (const item of value) { const key = JSON.stringify([item?.id ?? item?.kind ?? item?.code ?? item?.columnLabel ?? item?.traceId ?? null, item?.path ?? item?.urlPath ?? null, item?.method ?? null, item?.status ?? null, item?.summary ?? item?.message ?? item?.fromSeq ?? item?.firstAt ?? null, item?.toSeq ?? item?.lastAt ?? null]); if (seen.has(key)) continue; seen.add(key); out.push(item); } } return out; };",
|
|
"const clip = (value, limit = 160) => value === null || value === undefined ? null : String(value).slice(0, limit);",
|
|
"const findingRank = (item) => { const id = String(item?.id ?? item?.kind ?? item?.code ?? ''); if (id === 'observer-command-failed') return 0; if (id === 'page-performance-slow-same-origin-api') return 1; if (id === 'session-rail-title-fallback-majority') return 2; if (id.startsWith('turn-timing-total-elapsed')) return 3; if (id.startsWith('turn-timing-recent-update')) return 4; if (id.includes('runtime-execution') || id.includes('prompt-chat-submit-failed')) return 5; return 10; };",
|
|
"const severityRank = (item) => { const severity = String(item?.severity ?? item?.level ?? '').toLowerCase(); if (severity === 'red') return 0; if (severity === 'amber' || severity === 'warning') return 1; if (severity === 'info') return 3; return 2; };",
|
|
"const sortFindings = (items) => (Array.isArray(items) ? items : []).slice().sort((a, b) => findingRank(a) - findingRank(b) || severityRank(a) - severityRank(b));",
|
|
"const stdoutJson = readJson(stdoutPath);",
|
|
"const reportJson = readJson(reportJsonPath);",
|
|
"const source = (stdoutJson && stdoutJson.ok !== false ? stdoutJson : null) || reportJson || stdoutJson || null;",
|
|
"const fullSource = reportJson || source;",
|
|
"const objectOrNull = (value) => value && typeof value === 'object' && !Array.isArray(value) ? value : null;",
|
|
"const slimRound = (item) => { const v = objectOrNull(item) || {}; return { promptIndex: v.promptIndex ?? null, sampleCount: v.sampleCount ?? null, loadingSamples: v.loadingSamples ?? null, maxLoadingCount: v.maxLoadingCount ?? null, loadingOwnerCount: v.loadingOwnerCount ?? null, lastTotalElapsedSeconds: v.lastTotalElapsedSeconds ?? null, lastRecentUpdateSeconds: v.lastRecentUpdateSeconds ?? null, diagnosticSamples: v.diagnosticSamples ?? null, terminalSamples: v.terminalSamples ?? null, turnTimingTotalElapsedForwardJumpCount: v.turnTimingTotalElapsedForwardJumpCount ?? null, turnTimingTotalElapsedForwardJumpMaxSeconds: v.turnTimingTotalElapsedForwardJumpMaxSeconds ?? null, promptTextHash: clip(v.promptTextHash, 80) }; };",
|
|
"const slimTurnColumn = (item) => { const v = objectOrNull(item) || {}; return { label: clip(v.label, 24), pageRole: clip(v.pageRole, 24), pageId: clip(v.pageId, 32), pageEpoch: v.pageEpoch ?? null, promptIndex: v.promptIndex ?? null, lastPromptIndex: v.lastPromptIndex ?? null, traceId: clip(v.traceId, 48), firstSeq: v.firstSeq ?? null, lastSeq: v.lastSeq ?? null, source: clip(v.source, 48) }; };",
|
|
"const slimSlowSample = (item) => { const v = objectOrNull(item) || {}; return { ts: v.ts ?? null, seq: v.seq ?? null, path: clip(v.path ?? v.rawPath, 96), initiatorType: clip(v.initiatorType, 24), durationMs: v.durationMs ?? null, requestToResponseStartMs: v.requestToResponseStartMs ?? v.streamOpenMs ?? null, responseTransferMs: v.responseTransferMs ?? null, nextHopProtocol: clip(v.nextHopProtocol, 24), timingStatus: clip(v.timingStatus, 16), serverTimingNames: Array.isArray(v.serverTimingNames) ? v.serverTimingNames.slice(0, 4).map((x) => clip(x, 32)) : [], otelTraceId: clip(v.otelTraceId, 32) }; };",
|
|
"const slimSlowApi = (item) => { const v = objectOrNull(item) || {}; return { path: clip(v.path ?? v.route, 96), route: clip(v.route ?? v.path, 96), sampleCount: v.sampleCount ?? null, p95Ms: v.p95Ms ?? v.p95 ?? null, maxMs: v.maxMs ?? v.max ?? null, budgetMs: v.budgetMs ?? null, overBudgetCount: v.overBudgetCount ?? null, overFiveSecondCount: v.overFiveSecondCount ?? null, slowSamples: Array.isArray(v.slowSamples) ? v.slowSamples.slice(0, 3).map(slimSlowSample) : [] }; };",
|
|
"const slimFinding = (item) => { const v = objectOrNull(item) || {}; return { kind: clip(v.kind ?? v.id ?? v.code, 48), code: clip(v.code ?? v.id ?? v.kind, 48), severity: clip(v.severity ?? v.level, 24), level: clip(v.level ?? v.severity, 24), count: v.count ?? v.sampleCount ?? null, sampleCount: v.sampleCount ?? v.count ?? null, summary: clip(v.summary ?? v.message, 180), message: clip(v.message ?? v.summary, 180) }; };",
|
|
"const slimDomGroup = (item) => { const v = objectOrNull(item) || {}; return { count: v.count ?? null, firstAt: v.firstAt ?? null, lastAt: v.lastAt ?? null, text: clip(v.text ?? v.preview, 180) }; };",
|
|
"const slimNetworkGroup = (item) => { const v = objectOrNull(item) || {}; return { count: v.count ?? null, method: clip(v.method, 12), status: v.status ?? null, path: clip(v.path ?? v.urlPath, 96), firstAt: v.firstAt ?? null, lastAt: v.lastAt ?? null, promptIndexes: Array.isArray(v.promptIndexes) ? v.promptIndexes.slice(0, 6) : [], failureKinds: Array.isArray(v.failureKinds) ? v.failureKinds.slice(0, 4).map((x) => clip(x, 48)) : [] }; };",
|
|
"const slimDomSample = (item) => { const v = objectOrNull(item) || {}; return { seq: v.seq ?? null, ts: v.ts ?? null, source: clip(v.source, 32), diagnosticCode: clip(v.diagnosticCode, 48), traceId: clip(v.traceId, 64), httpStatus: v.httpStatus ?? null, idleSeconds: v.idleSeconds ?? null, waitingFor: clip(v.waitingFor, 48), lastEventLabel: clip(v.lastEventLabel, 80), text: clip(v.text ?? v.preview, 180) }; };",
|
|
"const slimConsoleGroup = (item) => { const v = objectOrNull(item) || {}; return { count: v.count ?? null, type: clip(v.type, 24), status: v.status ?? null, path: clip(v.path ?? v.urlPath, 96), lastAt: v.lastAt ?? v.firstAt ?? null, firstAt: v.firstAt ?? null, traceIds: Array.isArray(v.traceIds) ? v.traceIds.slice(0, 3).map((x) => clip(x, 64)) : [] }; };",
|
|
"const slimConsoleSample = (item) => { const v = objectOrNull(item) || {}; return { ts: v.ts ?? null, type: clip(v.type, 24), status: v.status ?? null, path: clip(v.path ?? v.urlPath, 96), traceId: clip(v.traceId, 64), text: clip(v.text ?? v.preview, 180) }; };",
|
|
"const slimRunnerError = (item) => { const v = objectOrNull(item) || {}; const readiness = objectOrNull(v.lastReadiness); return { ts: v.ts ?? null, type: clip(v.type, 32), commandId: clip(v.commandId, 80), sampleSeq: v.sampleSeq ?? null, message: clip(v.message, 240), retry: clip(v.retry, 24), retryExhausted: v.retryExhausted === true, lastError: clip(v.lastError, 180), attemptCount: v.attemptCount ?? null, lastFailureKind: clip(v.lastFailureKind, 48), lastReadinessReason: clip(v.lastReadinessReason, 48), lastReadiness: readiness ? { reason: clip(readiness.reason, 48), path: clip(readiness.path, 96), readyState: clip(readiness.readyState, 24), workbenchShellVisible: readiness.workbenchShellVisible === true, sessionCreateVisible: readiness.sessionCreateVisible === true, commandInputPresent: readiness.commandInputPresent === true, activeTabPresent: readiness.activeTabPresent === true, warningPresent: readiness.warningPresent === true, loginVisible: readiness.loginVisible === true, bodyTextHash: clip(readiness.bodyTextHash, 80) } : null }; };",
|
|
"const slimRunnerErrorFromJsonl = (item) => { const v = objectOrNull(item) || {}; const error = objectOrNull(v.error) || {}; const attempts = Array.isArray(error.attempts) ? error.attempts : []; const lastAttempt = attempts.length > 0 ? objectOrNull(attempts[attempts.length - 1]) || {} : {}; const rawReadiness = objectOrNull(lastAttempt.readiness) || objectOrNull(error.navigationReadiness); const readiness = objectOrNull(rawReadiness?.snapshot) || rawReadiness; return { ts: v.ts ?? null, type: v.type ?? null, commandId: v.commandId ?? null, sampleSeq: v.sampleSeq ?? null, message: error.message ?? v.message ?? null, attemptCount: attempts.length, lastFailureKind: lastAttempt.failureKind ?? null, lastReadinessReason: rawReadiness?.reason ?? readiness?.reason ?? null, lastReadiness: readiness ?? null }; };",
|
|
"const slimCommandFailure = (item) => { const v = objectOrNull(item) || {}; return { ts: v.ts ?? null, commandId: clip(v.commandId, 80), type: clip(v.type, 32), source: clip(v.source, 24), durationMs: v.durationMs ?? null, beforePath: clip(v.beforePath, 80), afterPath: clip(v.afterPath, 80), name: clip(v.name, 48), failureKind: clip(v.failureKind, 48), sampleSeq: v.sampleSeq ?? null, failureSampleOk: v.failureSampleOk === true, message: clip(v.message, 240) }; };",
|
|
"const slimJump = (item) => { const v = objectOrNull(item) || {}; return { columnLabel: v.columnLabel ?? null, pageRole: clip(v.pageRole, 24), pageId: clip(v.pageId, 32), pageEpoch: v.pageEpoch ?? null, promptIndex: v.promptIndex ?? null, fromSeq: v.fromSeq ?? null, toSeq: v.toSeq ?? null, fromValue: v.fromValue ?? null, toValue: v.toValue ?? null, delta: v.delta ?? null, sampleDeltaSeconds: v.sampleDeltaSeconds ?? null, allowedIncreaseSeconds: v.allowedIncreaseSeconds ?? null, traceId: v.traceId ?? null }; };",
|
|
"const slimTraceOrderAnomaly = (item) => { const v = objectOrNull(item) || {}; return { sampleIndex: v.sampleIndex ?? v.seq ?? null, seq: v.seq ?? null, timestamp: v.timestamp ?? v.ts ?? null, pageRole: clip(v.pageRole, 24), traceId: clip(v.traceId, 64), previousRowIndex: v.previousRowIndex ?? null, currentRowIndex: v.currentRowIndex ?? null, reasons: Array.isArray(v.reasons) ? v.reasons.slice(0, 6).map((x) => clip(x, 48)) : [], previousTotalSeconds: v.previousTotalSeconds ?? null, currentTotalSeconds: v.currentTotalSeconds ?? null, previousClockSeconds: v.previousClockSeconds ?? null, currentClockSeconds: v.currentClockSeconds ?? null, previousPreview: clip(v.previousPreview, 180), currentPreview: clip(v.currentPreview, 180) }; };",
|
|
"const slimTraceOrderMetrics = (value) => { const v = objectOrNull(value); if (!v) return null; const s = objectOrNull(v.summary); return { summary: { sampleCount: v.sampleCount ?? s?.sampleCount ?? null, traceRowCount: v.traceRowCount ?? s?.traceRowCount ?? null, orderAnomalyCount: v.orderAnomalyCount ?? s?.orderAnomalyCount ?? (Array.isArray(v.orderAnomalies) ? v.orderAnomalies.length : null), completionNotLastCount: v.completionNotLastCount ?? s?.completionNotLastCount ?? (Array.isArray(v.completionNotLast) ? v.completionNotLast.length : null) }, orderAnomalies: takeHead(v.orderAnomalies, 8).map(slimTraceOrderAnomaly), valuesRedacted: true }; };",
|
|
"const slimLoadingOwner = (item) => { const v = objectOrNull(item) || {}; return { ownerKey: clip(v.ownerKey, 120), ownerKind: clip(v.ownerKind, 32), ownerLabel: clip(v.ownerLabel, 120), ownerTraceId: clip(v.ownerTraceId, 64), ownerMessageId: clip(v.ownerMessageId, 64), ownerSessionId: clip(v.ownerSessionId, 64), sampleCount: v.sampleCount ?? null, occurrenceCount: v.occurrenceCount ?? null, maxSimultaneousCount: v.maxSimultaneousCount ?? null, longestContinuousSeconds: v.longestContinuousSeconds ?? null, firstSeq: v.firstSeq ?? null, lastSeq: v.lastSeq ?? null, promptIndexes: Array.isArray(v.promptIndexes) ? v.promptIndexes.slice(0, 8) : [] }; };",
|
|
"const slimLoadingSegment = (item) => { const v = objectOrNull(item) || {}; return { firstAt: v.firstAt ?? null, lastAt: v.lastAt ?? null, endedAt: v.endedAt ?? null, firstSeq: v.firstSeq ?? null, lastSeq: v.lastSeq ?? null, durationSeconds: v.durationSeconds ?? null, upperBoundSeconds: v.upperBoundSeconds ?? null, endedGapSeconds: v.endedGapSeconds ?? null, sampleCount: v.sampleCount ?? null, maxCount: v.maxCount ?? null, ownerCount: v.ownerCount ?? null, ongoing: v.ongoing === true, owners: Array.isArray(v.owners) ? v.owners.slice(0, 6).map((owner) => ({ ownerKind: clip(owner?.ownerKind, 32), ownerLabel: clip(owner?.ownerLabel, 120), ownerTraceId: clip(owner?.ownerTraceId, 64), ownerMessageId: clip(owner?.ownerMessageId, 64), ownerSessionId: clip(owner?.ownerSessionId, 64), count: owner?.count ?? null })) : [] }; };",
|
|
"const slimLoadingMetrics = (value) => { const v = objectOrNull(value); if (!v) return null; return { summary: objectOrNull(v.summary), longestSegments: takeHead(v.longestSegments ?? v.segments, 8).map(slimLoadingSegment), owners: takeHead(v.owners, 8).map(slimLoadingOwner), timeline: takeTail(v.timeline, 12).map((item) => { const row = objectOrNull(item) || {}; return { seq: row.seq ?? null, ts: row.ts ?? null, promptIndex: row.promptIndex ?? null, loadingCount: row.loadingCount ?? null, ownerCount: row.ownerCount ?? null, owners: Array.isArray(row.owners) ? row.owners.slice(0, 6).map((owner) => ({ ownerKind: clip(owner?.ownerKind, 32), ownerLabel: clip(owner?.ownerLabel, 120), ownerTraceId: clip(owner?.ownerTraceId, 64), count: owner?.count ?? null })) : [] }; }), valuesRedacted: true }; };",
|
|
"const srcMetrics = objectOrNull(source?.sampleMetrics);",
|
|
"const fullRecentWindow = objectOrNull(fullSource?.windows?.recent);",
|
|
"const fullRecentMetrics = objectOrNull(fullRecentWindow?.sampleMetrics);",
|
|
"const fullArchiveMetrics = objectOrNull(fullSource?.sampleMetrics);",
|
|
"const metricHasTurnDetail = (value) => { const v = objectOrNull(value); const s = objectOrNull(v?.summary); return !!(v && ((Array.isArray(v.roundItems) && v.roundItems.length > 0) || (Array.isArray(v.rounds) && v.rounds.length > 0) || (Array.isArray(v.turnColumns) && v.turnColumns.length > 0) || (Array.isArray(v.turnTimingRows) && v.turnTimingRows.length > 0) || (Array.isArray(v.turnTimingTotalElapsedForwardJumps) && v.turnTimingTotalElapsedForwardJumps.length > 0) || (Array.isArray(v.turnTimingRecentUpdateSawtoothJumps) && v.turnTimingRecentUpdateSawtoothJumps.length > 0) || Number(v.turnTimingRows ?? s?.turnTimingRows ?? 0) > 0)); };",
|
|
"const selectedMetrics = metricHasTurnDetail(srcMetrics) ? srcMetrics : metricHasTurnDetail(fullRecentMetrics) ? fullRecentMetrics : metricHasTurnDetail(fullArchiveMetrics) ? fullArchiveMetrics : srcMetrics || fullRecentMetrics || fullArchiveMetrics;",
|
|
"const selectedSummary = objectOrNull(selectedMetrics?.summary);",
|
|
"const metrics = selectedMetrics ? {",
|
|
" sampleCount: selectedMetrics.sampleCount ?? selectedSummary?.sampleCount ?? null,",
|
|
" loadingSampleCount: selectedMetrics.loadingSampleCount ?? selectedSummary?.loadingSampleCount ?? null,",
|
|
" loadingMaxCount: selectedMetrics.loadingMaxCount ?? selectedSummary?.loadingMaxCount ?? null,",
|
|
" loadingMaxOwnerCount: selectedMetrics.loadingMaxOwnerCount ?? selectedSummary?.loadingMaxOwnerCount ?? null,",
|
|
" loadingOwnerCount: selectedMetrics.loadingOwnerCount ?? selectedSummary?.loadingOwnerCount ?? null,",
|
|
" loadingConcurrentSampleCount: selectedMetrics.loadingConcurrentSampleCount ?? selectedSummary?.loadingConcurrentSampleCount ?? null,",
|
|
" loadingLongestContinuousSeconds: selectedMetrics.loadingLongestContinuousSeconds ?? selectedSummary?.loadingLongestContinuousSeconds ?? null,",
|
|
" loadingCurrentContinuousSeconds: selectedMetrics.loadingCurrentContinuousSeconds ?? selectedSummary?.loadingCurrentContinuousSeconds ?? null,",
|
|
" loadingOverFiveSecondSegmentCount: selectedMetrics.loadingOverFiveSecondSegmentCount ?? selectedSummary?.loadingOverFiveSecondSegmentCount ?? null,",
|
|
" loading: slimLoadingMetrics(selectedMetrics.loading),",
|
|
" traceOrder: slimTraceOrderMetrics(selectedMetrics.traceOrder ?? fullRecentMetrics?.traceOrder ?? fullArchiveMetrics?.traceOrder),",
|
|
" roundItems: takeTail(firstNonEmptyArray(selectedMetrics.roundItems, selectedMetrics.rounds, fullRecentMetrics?.rounds, fullArchiveMetrics?.rounds), 8).map(slimRound),",
|
|
" rounds: takeTail(firstNonEmptyArray(selectedMetrics.roundItems, selectedMetrics.rounds, fullRecentMetrics?.rounds, fullArchiveMetrics?.rounds), 8).map(slimRound),",
|
|
" turnColumns: takeTail(firstNonEmptyArray(selectedMetrics.turnColumns, fullRecentMetrics?.turnColumns, fullArchiveMetrics?.turnColumns), 8).map(slimTurnColumn),",
|
|
" turnTimingRows: selectedMetrics.turnTimingRows ?? selectedSummary?.turnTimingRows ?? fullRecentMetrics?.summary?.turnTimingRows ?? fullRecentMetrics?.turnTimingRows?.length ?? fullArchiveMetrics?.summary?.turnTimingRows ?? fullArchiveMetrics?.turnTimingRows?.length ?? null,",
|
|
" turnTimingNonMonotonicCount: selectedMetrics.turnTimingNonMonotonicCount ?? selectedSummary?.turnTimingNonMonotonicCount ?? fullRecentMetrics?.summary?.turnTimingNonMonotonicCount ?? fullRecentMetrics?.turnTimingNonMonotonic?.length ?? fullArchiveMetrics?.summary?.turnTimingNonMonotonicCount ?? fullArchiveMetrics?.turnTimingNonMonotonic?.length ?? null,",
|
|
" turnTimingTotalElapsedDecreaseCount: selectedMetrics.turnTimingTotalElapsedDecreaseCount ?? selectedSummary?.turnTimingTotalElapsedDecreaseCount ?? fullRecentMetrics?.summary?.turnTimingTotalElapsedDecreaseCount ?? fullArchiveMetrics?.summary?.turnTimingTotalElapsedDecreaseCount ?? null,",
|
|
" turnTimingTotalElapsedZeroResetCount: selectedMetrics.turnTimingTotalElapsedZeroResetCount ?? selectedSummary?.turnTimingTotalElapsedZeroResetCount ?? fullRecentMetrics?.summary?.turnTimingTotalElapsedZeroResetCount ?? fullRecentMetrics?.turnTimingElapsedZeroResets?.length ?? fullArchiveMetrics?.summary?.turnTimingTotalElapsedZeroResetCount ?? fullArchiveMetrics?.turnTimingElapsedZeroResets?.length ?? null,",
|
|
" turnTimingTotalElapsedForwardJumpCount: selectedMetrics.turnTimingTotalElapsedForwardJumpCount ?? selectedSummary?.turnTimingTotalElapsedForwardJumpCount ?? fullRecentMetrics?.summary?.turnTimingTotalElapsedForwardJumpCount ?? fullRecentMetrics?.turnTimingTotalElapsedForwardJumps?.length ?? fullArchiveMetrics?.summary?.turnTimingTotalElapsedForwardJumpCount ?? fullArchiveMetrics?.turnTimingTotalElapsedForwardJumps?.length ?? null,",
|
|
" turnTimingTotalElapsedForwardJumpMaxSeconds: selectedMetrics.turnTimingTotalElapsedForwardJumpMaxSeconds ?? selectedSummary?.turnTimingTotalElapsedForwardJumpMaxSeconds ?? fullRecentMetrics?.summary?.turnTimingTotalElapsedForwardJumpMaxSeconds ?? fullArchiveMetrics?.summary?.turnTimingTotalElapsedForwardJumpMaxSeconds ?? null,",
|
|
" turnTimingTerminalElapsedGrowthCount: selectedMetrics.turnTimingTerminalElapsedGrowthCount ?? selectedSummary?.turnTimingTerminalElapsedGrowthCount ?? fullRecentMetrics?.summary?.turnTimingTerminalElapsedGrowthCount ?? fullRecentMetrics?.turnTimingTerminalElapsedGrowth?.length ?? fullArchiveMetrics?.summary?.turnTimingTerminalElapsedGrowthCount ?? fullArchiveMetrics?.turnTimingTerminalElapsedGrowth?.length ?? null,",
|
|
" turnTimingRecentUpdateJumpCount: selectedMetrics.turnTimingRecentUpdateJumpCount ?? selectedSummary?.turnTimingRecentUpdateJumpCount ?? selectedSummary?.turnTimingRecentUpdateSawtoothJumpCount ?? fullRecentMetrics?.summary?.turnTimingRecentUpdateJumpCount ?? fullRecentMetrics?.summary?.turnTimingRecentUpdateSawtoothJumpCount ?? fullArchiveMetrics?.summary?.turnTimingRecentUpdateJumpCount ?? fullArchiveMetrics?.summary?.turnTimingRecentUpdateSawtoothJumpCount ?? null,",
|
|
" turnTimingRecentUpdateSawtoothJumpCount: selectedMetrics.turnTimingRecentUpdateSawtoothJumpCount ?? selectedSummary?.turnTimingRecentUpdateSawtoothJumpCount ?? fullRecentMetrics?.summary?.turnTimingRecentUpdateSawtoothJumpCount ?? fullArchiveMetrics?.summary?.turnTimingRecentUpdateSawtoothJumpCount ?? null,",
|
|
" turnTimingRecentUpdateMaxIncreaseSeconds: selectedMetrics.turnTimingRecentUpdateMaxIncreaseSeconds ?? selectedSummary?.turnTimingRecentUpdateMaxIncreaseSeconds ?? fullRecentMetrics?.summary?.turnTimingRecentUpdateMaxIncreaseSeconds ?? fullArchiveMetrics?.summary?.turnTimingRecentUpdateMaxIncreaseSeconds ?? null,",
|
|
" promptSegments: selectedMetrics.promptSegments ?? selectedSummary?.promptSegments ?? null",
|
|
"} : null;",
|
|
"const srcRuntimeAlerts = objectOrNull(source?.runtimeAlerts);",
|
|
"const runtimeAlerts = srcRuntimeAlerts ? { httpErrorCount: srcRuntimeAlerts.httpErrorCount ?? null, requestFailedCount: srcRuntimeAlerts.requestFailedCount ?? null, domDiagnosticSampleCount: srcRuntimeAlerts.domDiagnosticSampleCount ?? null, consoleAlertCount: srcRuntimeAlerts.consoleAlertCount ?? null } : null;",
|
|
"const srcPagePerformance = objectOrNull(source?.pagePerformance);",
|
|
"const srcPagePerformanceSummary = objectOrNull(srcPagePerformance?.summary);",
|
|
"const pagePerformance = srcPagePerformance ? { sameOriginApiPaths: srcPagePerformance.sameOriginApiPaths ?? srcPagePerformanceSummary?.sameOriginApiPathCount ?? null, longLivedStreamPathCount: srcPagePerformance.longLivedStreamPathCount ?? srcPagePerformanceSummary?.longLivedStreamPathCount ?? null, longLivedStreamOpenOverFiveSecondSampleCount: srcPagePerformance.longLivedStreamOpenOverFiveSecondSampleCount ?? srcPagePerformanceSummary?.longLivedStreamOpenOverFiveSecondSampleCount ?? null, slowPathCount: srcPagePerformance.slowPathCount ?? srcPagePerformanceSummary?.slowPathCount ?? null, slowSampleCount: srcPagePerformance.slowSampleCount ?? srcPagePerformanceSummary?.slowSampleCount ?? null, worstP95Ms: srcPagePerformance.worstP95Ms ?? srcPagePerformanceSummary?.worstP95Ms ?? null } : null;",
|
|
"const fullPagePerformance = objectOrNull(fullRecentWindow?.pagePerformance) || objectOrNull(fullSource?.pagePerformance);",
|
|
"const fullArchivePagePerformance = objectOrNull(fullSource?.pagePerformance);",
|
|
"const isLongLivedApi = (item) => item?.isLongLivedStream === true || item?.routeKind === 'same-origin-api-stream' || item?.budgetMetric === 'streamOpenMs';",
|
|
"const sourceSlowApi = Array.isArray(source?.pagePerformanceSlowApi) ? source.pagePerformanceSlowApi.filter((item) => !isLongLivedApi(item) && Number(item?.overBudgetCount ?? item?.overFiveSecondCount ?? 0) > 0) : (Array.isArray(fullPagePerformance?.sameOriginApiByPath) ? fullPagePerformance.sameOriginApiByPath.filter((item) => !isLongLivedApi(item) && Number(item?.overBudgetCount ?? item?.overFiveSecondCount ?? 0) > 0) : []);",
|
|
"const archiveSlowApi = firstNonEmptyArray(source?.archivePagePerformanceSlowApi, Array.isArray(fullArchivePagePerformance?.sameOriginApiByPath) ? fullArchivePagePerformance.sameOriginApiByPath : []).filter((item) => !isLongLivedApi(item) && Number(item?.overBudgetCount ?? item?.overFiveSecondCount ?? 0) > 0);",
|
|
"const sourceSseStreams = Array.isArray(source?.pagePerformanceSseStreams) ? source.pagePerformanceSseStreams : (Array.isArray(fullPagePerformance?.sameOriginApiByPath) ? fullPagePerformance.sameOriginApiByPath.filter((item) => isLongLivedApi(item)) : []);",
|
|
"const combinedFindingSource = firstNonEmptyArray(source?.findings, fullSource?.findings, fullRecentWindow?.findings, source?.archiveSummary?.redFindings, fullSource?.archiveSummary?.redFindings);",
|
|
"const combinedFindings = sortFindings(combinedFindingSource);",
|
|
"const archiveRedFindings = sortFindings(firstNonEmptyArray(source?.archiveSummary?.redFindings, fullSource?.archiveSummary?.redFindings, fullSource?.findings).filter((item) => String(item?.severity ?? item?.level ?? '').toLowerCase() === 'red'));",
|
|
"const allFindingsForCommands = [...firstArray(source?.findings), ...firstArray(fullSource?.findings), ...firstArray(fullRecentWindow?.findings), ...firstArray(source?.archiveSummary?.redFindings), ...firstArray(fullSource?.archiveSummary?.redFindings)];",
|
|
"const findingSamplesById = (id) => { for (const item of allFindingsForCommands) { if (String(item?.id ?? item?.kind ?? item?.code ?? '') !== id) continue; if (Array.isArray(item?.samples) && item.samples.length > 0) return item.samples; } return []; };",
|
|
"const commandFailuresFromFindings = [...allFindingsForCommands, ...archiveRedFindings].flatMap((item) => Array.isArray(item?.commands) ? item.commands : []);",
|
|
"const srcPromptNetwork = objectOrNull(source?.promptNetwork);",
|
|
"const promptNetwork = srcPromptNetwork ? { promptSegments: srcPromptNetwork.promptSegments ?? null } : null;",
|
|
"const runnerErrorsFromJsonl = readJsonlTail(reportJsonPath.replace(/\\/analysis\\/report\\.json$/u, '/errors.jsonl'), 8).filter((item) => item?.type === 'runner-error').map(slimRunnerErrorFromJsonl);",
|
|
"const compact = source ? {",
|
|
" ok: analyzerExit === 0,",
|
|
" counts: source.counts ?? null,",
|
|
" jsonlScope: objectOrNull(source.jsonlScope),",
|
|
" analysisWindow: objectOrNull(source.analysisWindow),",
|
|
" archiveSummary: objectOrNull(source.archiveSummary),",
|
|
" sampleMetrics: metrics,",
|
|
" runtimeAlerts,",
|
|
" pagePerformance,",
|
|
" promptNetwork,",
|
|
" pagePerformanceSlowApi: takeHead(sourceSlowApi, 4).map(slimSlowApi),",
|
|
" archivePagePerformanceSlowApi: takeHead(archiveSlowApi, 8).map(slimSlowApi),",
|
|
" pagePerformanceSseStreams: takeHead(sourceSseStreams, 4).map((item) => ({ path: item?.path ?? item?.route ?? null, route: item?.route ?? null, sampleCount: item?.sampleCount ?? null, streamOpenSampleCount: item?.streamOpenSampleCount ?? null, streamOpenP95Ms: item?.streamOpenP95Ms ?? null, streamOpenMaxMs: item?.streamOpenMaxMs ?? null, streamOpenBudgetMs: item?.streamOpenBudgetMs ?? null, streamOpenOverBudgetCount: item?.streamOpenOverBudgetCount ?? null, streamOpenOverFiveSecondCount: item?.streamOpenOverFiveSecondCount ?? null, streamLifetimeOverFiveSecondCount: item?.streamLifetimeOverFiveSecondCount ?? null })),",
|
|
" findings: takeHead(combinedFindings, 8).map(slimFinding),",
|
|
" archiveRedFindings: takeHead(archiveRedFindings, 8).map(slimFinding),",
|
|
" httpErrorGroups: takeHead(firstArray(source.httpErrorGroups, fullRecentWindow?.runtimeAlerts?.networkHttpErrorsByPath, fullSource?.httpErrorGroups, fullSource?.runtimeAlerts?.networkHttpErrorsByPath), 4).map(slimNetworkGroup),",
|
|
" requestFailedGroups: takeHead(firstArray(source.requestFailedGroups, fullRecentWindow?.runtimeAlerts?.networkRequestFailedByPath, fullSource?.requestFailedGroups, fullSource?.runtimeAlerts?.networkRequestFailedByPath), 5).map(slimNetworkGroup),",
|
|
" domDiagnosticGroups: takeHead(firstArray(source.domDiagnosticGroups, fullRecentWindow?.runtimeAlerts?.domDiagnosticsByFingerprint, fullSource?.domDiagnosticGroups, fullSource?.runtimeAlerts?.domDiagnosticsByFingerprint), 3).map(slimDomGroup),",
|
|
" domDiagnosticSamples: takeHead(firstArray(source.domDiagnosticSamples, fullRecentWindow?.runtimeAlerts?.domDiagnostics, fullSource?.domDiagnosticSamples, fullSource?.runtimeAlerts?.domDiagnostics), 5).map(slimDomSample),",
|
|
" consoleAlertGroups: takeHead(firstArray(source.consoleAlertGroups, fullRecentWindow?.runtimeAlerts?.consoleAlertsByPath, fullSource?.consoleAlertGroups, fullSource?.runtimeAlerts?.consoleAlertsByPath), 5).map(slimConsoleGroup),",
|
|
" consoleAlertSamples: takeHead(firstArray(source.consoleAlertSamples, fullRecentWindow?.runtimeAlerts?.consoleAlerts, fullSource?.consoleAlertSamples, fullSource?.runtimeAlerts?.consoleAlerts), 5).map(slimConsoleSample),",
|
|
" runnerErrors: takeTail(firstNonEmptyArray(source.runnerErrors, fullSource?.runnerErrors, runnerErrorsFromJsonl), 8).map(slimRunnerError),",
|
|
" commandFailures: takeTail(firstNonEmptyArray(source.commandFailures, fullSource?.commandFailures, commandFailuresFromFindings), 8).map(slimCommandFailure),",
|
|
" turnTimingRecentUpdateJumps: takeHead(firstNonEmptyArray(source.turnTimingRecentUpdateJumps, selectedMetrics?.turnTimingRecentUpdateJumps, selectedMetrics?.turnTimingRecentUpdateSawtoothJumps, srcMetrics?.turnTimingRecentUpdateJumps, srcMetrics?.turnTimingRecentUpdateSawtoothJumps, fullRecentWindow?.sampleMetrics?.turnTimingRecentUpdateSawtoothJumps, fullRecentWindow?.turnTimingRecentUpdateJumps, fullSource?.sampleMetrics?.turnTimingRecentUpdateSawtoothJumps, fullSource?.turnTimingRecentUpdateJumps, findingSamplesById('turn-timing-recent-update-sawtooth-jump')), 5).map(slimJump),",
|
|
" turnTimingElapsedZeroResets: takeHead(firstNonEmptyArray(source.turnTimingElapsedZeroResets, selectedMetrics?.turnTimingElapsedZeroResets, srcMetrics?.turnTimingElapsedZeroResets, fullRecentWindow?.sampleMetrics?.turnTimingElapsedZeroResets, fullRecentWindow?.turnTimingElapsedZeroResets, fullSource?.sampleMetrics?.turnTimingElapsedZeroResets, fullSource?.turnTimingElapsedZeroResets, findingSamplesById('turn-timing-total-elapsed-zero-reset')), 5).map(slimJump),",
|
|
" turnTimingTotalElapsedForwardJumps: takeHead(firstNonEmptyArray(source.turnTimingTotalElapsedForwardJumps, selectedMetrics?.turnTimingTotalElapsedForwardJumps, srcMetrics?.turnTimingTotalElapsedForwardJumps, fullRecentWindow?.sampleMetrics?.turnTimingTotalElapsedForwardJumps, fullRecentWindow?.turnTimingTotalElapsedForwardJumps, fullSource?.sampleMetrics?.turnTimingTotalElapsedForwardJumps, fullSource?.turnTimingTotalElapsedForwardJumps, findingSamplesById('turn-timing-total-elapsed-forward-jump')), 5).map(slimJump),",
|
|
" reportJsonPath: source.reportJsonPath || reportJsonPath,",
|
|
" reportJsonSha256: source.reportJsonSha256 || sha256(reportJsonPath),",
|
|
" reportMdPath: source.reportMdPath || reportMdPath,",
|
|
" reportMdSha256: source.reportMdSha256 || sha256(reportMdPath),",
|
|
" analyzer: {",
|
|
" exitCode: Number.isFinite(analyzerExit) ? analyzerExit : null,",
|
|
" recoveredFrom: stdoutJson && stdoutJson.ok !== false ? 'stdout' : reportJson ? 'report-file' : stdoutJson ? 'stdout-error' : 'missing-output',",
|
|
" stdoutBytes: statSize(stdoutPath),",
|
|
" stderrBytes: statSize(stderrPath),",
|
|
" stderrTail: tail(readText(stderrPath))",
|
|
" }",
|
|
"} : {",
|
|
" ok: false,",
|
|
" error: 'web-probe-analyzer-output-missing',",
|
|
" reportJsonPath,",
|
|
" reportMdPath,",
|
|
" analyzer: {",
|
|
" exitCode: Number.isFinite(analyzerExit) ? analyzerExit : null,",
|
|
" recoveredFrom: 'missing-output',",
|
|
" stdoutBytes: statSize(stdoutPath),",
|
|
" stderrBytes: statSize(stderrPath),",
|
|
" stderrTail: tail(readText(stderrPath))",
|
|
" }",
|
|
"};",
|
|
"const compactStdoutLimitBytes = 10000;",
|
|
"const compactOutput = (value) => JSON.stringify(value);",
|
|
"let output = compactOutput(compact);",
|
|
"if (Buffer.byteLength(output, 'utf8') > compactStdoutLimitBytes) {",
|
|
" const minimalRounds = takeTail(firstNonEmptyArray(compact.sampleMetrics?.rounds, compact.sampleMetrics?.roundItems, source?.sampleMetrics?.rounds, fullRecentMetrics?.rounds, fullArchiveMetrics?.rounds), 8).map(slimRound);",
|
|
" const minimalTurnColumns = takeTail(firstNonEmptyArray(compact.sampleMetrics?.turnColumns, source?.sampleMetrics?.turnColumns, fullRecentMetrics?.turnColumns, fullArchiveMetrics?.turnColumns), 8).map(slimTurnColumn);",
|
|
" const minimal = {",
|
|
" ok: compact.ok,",
|
|
" counts: compact.counts ?? null,",
|
|
" jsonlScope: compact.jsonlScope ?? null,",
|
|
" analysisWindow: compact.analysisWindow ?? null,",
|
|
" archiveSummary: compact.archiveSummary ?? null,",
|
|
" sampleMetrics: compact.sampleMetrics ? {",
|
|
" roundItems: minimalRounds,",
|
|
" rounds: minimalRounds,",
|
|
" sampleCount: compact.sampleMetrics.sampleCount ?? null,",
|
|
" loadingSampleCount: compact.sampleMetrics.loadingSampleCount ?? null,",
|
|
" loadingMaxCount: compact.sampleMetrics.loadingMaxCount ?? null,",
|
|
" loadingMaxOwnerCount: compact.sampleMetrics.loadingMaxOwnerCount ?? null,",
|
|
" loadingOwnerCount: compact.sampleMetrics.loadingOwnerCount ?? null,",
|
|
" loadingConcurrentSampleCount: compact.sampleMetrics.loadingConcurrentSampleCount ?? null,",
|
|
" loadingLongestContinuousSeconds: compact.sampleMetrics.loadingLongestContinuousSeconds ?? null,",
|
|
" loadingCurrentContinuousSeconds: compact.sampleMetrics.loadingCurrentContinuousSeconds ?? null,",
|
|
" loadingOverFiveSecondSegmentCount: compact.sampleMetrics.loadingOverFiveSecondSegmentCount ?? null,",
|
|
" sessionRailSampleCount: compact.sampleMetrics.sessionRailSampleCount ?? compact.sampleMetrics.sessionRailTitles?.summary?.sampleCount ?? null,",
|
|
" sessionRailVisibleSampleCount: compact.sampleMetrics.sessionRailVisibleSampleCount ?? compact.sampleMetrics.sessionRailTitles?.summary?.visibleSampleCount ?? null,",
|
|
" sessionRailFallbackMajoritySampleCount: compact.sampleMetrics.sessionRailFallbackMajoritySampleCount ?? compact.sampleMetrics.sessionRailTitles?.summary?.majorityFallbackSampleCount ?? null,",
|
|
" sessionRailFallbackMaxRatio: compact.sampleMetrics.sessionRailFallbackMaxRatio ?? compact.sampleMetrics.sessionRailTitles?.summary?.maxFallbackRatio ?? null,",
|
|
" sessionRailFallbackMaxVisibleCount: compact.sampleMetrics.sessionRailFallbackMaxVisibleCount ?? compact.sampleMetrics.sessionRailTitles?.summary?.maxVisibleCount ?? null,",
|
|
" sessionRailFallbackMaxCount: compact.sampleMetrics.sessionRailFallbackMaxCount ?? compact.sampleMetrics.sessionRailTitles?.summary?.maxFallbackTitleCount ?? null,",
|
|
" loading: compact.sampleMetrics.loading ?? null,",
|
|
" sessionRailTitles: compact.sampleMetrics.sessionRailTitles ? { summary: compact.sampleMetrics.sessionRailTitles.summary ?? null, samples: Array.isArray(compact.sampleMetrics.sessionRailTitles.samples) ? compact.sampleMetrics.sessionRailTitles.samples.slice(0, 4) : [], examples: Array.isArray(compact.sampleMetrics.sessionRailTitles.examples) ? compact.sampleMetrics.sessionRailTitles.examples.slice(0, 4) : [] } : null,",
|
|
" turnTimingRows: compact.sampleMetrics.turnTimingRows ?? null,",
|
|
" turnTimingNonMonotonicCount: compact.sampleMetrics.turnTimingNonMonotonicCount ?? null,",
|
|
" turnTimingTotalElapsedDecreaseCount: compact.sampleMetrics.turnTimingTotalElapsedDecreaseCount ?? null,",
|
|
" turnTimingTotalElapsedForwardJumpCount: compact.sampleMetrics.turnTimingTotalElapsedForwardJumpCount ?? null,",
|
|
" turnTimingTotalElapsedForwardJumpMaxSeconds: compact.sampleMetrics.turnTimingTotalElapsedForwardJumpMaxSeconds ?? null,",
|
|
" turnTimingTerminalElapsedGrowthCount: compact.sampleMetrics.turnTimingTerminalElapsedGrowthCount ?? null,",
|
|
" turnTimingRecentUpdateJumpCount: compact.sampleMetrics.turnTimingRecentUpdateJumpCount ?? null,",
|
|
" turnTimingRecentUpdateSawtoothJumpCount: compact.sampleMetrics.turnTimingRecentUpdateSawtoothJumpCount ?? null,",
|
|
" turnTimingRecentUpdateMaxIncreaseSeconds: compact.sampleMetrics.turnTimingRecentUpdateMaxIncreaseSeconds ?? null,",
|
|
" turnColumns: minimalTurnColumns",
|
|
" } : null,",
|
|
" runtimeAlerts: compact.runtimeAlerts ?? null,",
|
|
" pagePerformance: compact.pagePerformance ?? null,",
|
|
" promptNetwork: compact.promptNetwork ?? null,",
|
|
" findings: Array.isArray(compact.findings) ? compact.findings.slice(0, 4) : [],",
|
|
" archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4) : [],",
|
|
" turnTimingRecentUpdateJumps: Array.isArray(compact.turnTimingRecentUpdateJumps) ? compact.turnTimingRecentUpdateJumps.slice(0, 5) : [],",
|
|
" turnTimingElapsedZeroResets: Array.isArray(compact.turnTimingElapsedZeroResets) ? compact.turnTimingElapsedZeroResets.slice(0, 5) : [],",
|
|
" turnTimingTotalElapsedForwardJumps: Array.isArray(compact.turnTimingTotalElapsedForwardJumps) ? compact.turnTimingTotalElapsedForwardJumps.slice(0, 5) : [],",
|
|
" pagePerformanceSlowApi: Array.isArray(compact.pagePerformanceSlowApi) ? compact.pagePerformanceSlowApi.slice(0, 2).map((item) => ({ ...item, slowSamples: Array.isArray(item.slowSamples) ? item.slowSamples.slice(0, 1) : [] })) : [],",
|
|
" archivePagePerformanceSlowApi: Array.isArray(compact.archivePagePerformanceSlowApi) ? compact.archivePagePerformanceSlowApi.slice(0, 4).map((item) => ({ ...item, slowSamples: Array.isArray(item.slowSamples) ? item.slowSamples.slice(0, 1) : [] })) : [],",
|
|
" pagePerformanceSseStreams: Array.isArray(compact.pagePerformanceSseStreams) ? compact.pagePerformanceSseStreams.slice(0, 2).map((item) => ({ path: item.path ?? item.route ?? null, route: item.route ?? item.path ?? null, sampleCount: item.sampleCount ?? null, streamOpenP95Ms: item.streamOpenP95Ms ?? null, streamOpenMaxMs: item.streamOpenMaxMs ?? null, streamOpenBudgetMs: item.streamOpenBudgetMs ?? null, streamOpenOverBudgetCount: item.streamOpenOverBudgetCount ?? null, streamOpenOverFiveSecondCount: item.streamOpenOverFiveSecondCount ?? null, streamLifetimeOverFiveSecondCount: item.streamLifetimeOverFiveSecondCount ?? null })) : [],",
|
|
" httpErrorGroups: Array.isArray(compact.httpErrorGroups) ? compact.httpErrorGroups.slice(0, 3) : [],",
|
|
" requestFailedGroups: Array.isArray(compact.requestFailedGroups) ? compact.requestFailedGroups.slice(0, 3) : [],",
|
|
" domDiagnosticGroups: Array.isArray(compact.domDiagnosticGroups) ? compact.domDiagnosticGroups.slice(0, 2) : [],",
|
|
" domDiagnosticSamples: Array.isArray(compact.domDiagnosticSamples) ? compact.domDiagnosticSamples.slice(0, 2) : [],",
|
|
" consoleAlertGroups: Array.isArray(compact.consoleAlertGroups) ? compact.consoleAlertGroups.slice(0, 3) : [],",
|
|
" consoleAlertSamples: Array.isArray(compact.consoleAlertSamples) ? compact.consoleAlertSamples.slice(0, 2) : [],",
|
|
" runnerErrors: Array.isArray(compact.runnerErrors) ? compact.runnerErrors.slice(-4) : [],",
|
|
" commandFailures: Array.isArray(compact.commandFailures) ? compact.commandFailures.slice(-4) : [],",
|
|
" turnTimingRecentUpdateJumps: Array.isArray(compact.turnTimingRecentUpdateJumps) ? compact.turnTimingRecentUpdateJumps.slice(0, 4) : [],",
|
|
" turnTimingElapsedZeroResets: Array.isArray(compact.turnTimingElapsedZeroResets) ? compact.turnTimingElapsedZeroResets.slice(0, 4) : [],",
|
|
" turnTimingTotalElapsedForwardJumps: Array.isArray(compact.turnTimingTotalElapsedForwardJumps) ? compact.turnTimingTotalElapsedForwardJumps.slice(0, 4) : [],",
|
|
" reportJsonPath: compact.reportJsonPath ?? reportJsonPath,",
|
|
" reportJsonSha256: compact.reportJsonSha256 ?? sha256(reportJsonPath),",
|
|
" reportMdPath: compact.reportMdPath ?? reportMdPath,",
|
|
" reportMdSha256: compact.reportMdSha256 ?? sha256(reportMdPath),",
|
|
" analyzer: { ...(compact.analyzer ?? {}), compactStdoutLimited: true, compactStdoutLimitBytes, originalCompactBytes: Buffer.byteLength(output, 'utf8') },",
|
|
" valuesRedacted: true",
|
|
" };",
|
|
" output = compactOutput(minimal);",
|
|
" if (Buffer.byteLength(output, 'utf8') > compactStdoutLimitBytes) {",
|
|
" const tiny = {",
|
|
" ok: compact.ok,",
|
|
" counts: compact.counts ?? null,",
|
|
" jsonlScope: compact.jsonlScope ?? null,",
|
|
" analysisWindow: compact.analysisWindow ?? null,",
|
|
" archiveSummary: compact.archiveSummary ?? null,",
|
|
" sampleMetrics: compact.sampleMetrics ? {",
|
|
" sampleCount: compact.sampleMetrics.sampleCount ?? null,",
|
|
" loadingSampleCount: compact.sampleMetrics.loadingSampleCount ?? null,",
|
|
" loadingMaxCount: compact.sampleMetrics.loadingMaxCount ?? null,",
|
|
" loadingMaxOwnerCount: compact.sampleMetrics.loadingMaxOwnerCount ?? null,",
|
|
" loadingOwnerCount: compact.sampleMetrics.loadingOwnerCount ?? null,",
|
|
" loadingLongestContinuousSeconds: compact.sampleMetrics.loadingLongestContinuousSeconds ?? null,",
|
|
" loadingCurrentContinuousSeconds: compact.sampleMetrics.loadingCurrentContinuousSeconds ?? null,",
|
|
" loadingOverFiveSecondSegmentCount: compact.sampleMetrics.loadingOverFiveSecondSegmentCount ?? null,",
|
|
" loading: compact.sampleMetrics.loading ? { summary: compact.sampleMetrics.loading.summary ?? null, longestSegments: Array.isArray(compact.sampleMetrics.loading.longestSegments) ? compact.sampleMetrics.loading.longestSegments.slice(0, 4) : [], owners: Array.isArray(compact.sampleMetrics.loading.owners) ? compact.sampleMetrics.loading.owners.slice(0, 4) : [] } : null,",
|
|
" sessionRailSampleCount: compact.sampleMetrics.sessionRailSampleCount ?? compact.sampleMetrics.sessionRailTitles?.summary?.sampleCount ?? null,",
|
|
" sessionRailVisibleSampleCount: compact.sampleMetrics.sessionRailVisibleSampleCount ?? compact.sampleMetrics.sessionRailTitles?.summary?.visibleSampleCount ?? null,",
|
|
" sessionRailFallbackMajoritySampleCount: compact.sampleMetrics.sessionRailFallbackMajoritySampleCount ?? compact.sampleMetrics.sessionRailTitles?.summary?.majorityFallbackSampleCount ?? null,",
|
|
" sessionRailFallbackMaxRatio: compact.sampleMetrics.sessionRailFallbackMaxRatio ?? compact.sampleMetrics.sessionRailTitles?.summary?.maxFallbackRatio ?? null,",
|
|
" sessionRailFallbackMaxVisibleCount: compact.sampleMetrics.sessionRailFallbackMaxVisibleCount ?? compact.sampleMetrics.sessionRailTitles?.summary?.maxVisibleCount ?? null,",
|
|
" sessionRailFallbackMaxCount: compact.sampleMetrics.sessionRailFallbackMaxCount ?? compact.sampleMetrics.sessionRailTitles?.summary?.maxFallbackTitleCount ?? null,",
|
|
" sessionRailTitles: compact.sampleMetrics.sessionRailTitles ? { summary: compact.sampleMetrics.sessionRailTitles.summary ?? null, samples: Array.isArray(compact.sampleMetrics.sessionRailTitles.samples) ? compact.sampleMetrics.sessionRailTitles.samples.slice(0, 4) : [], examples: Array.isArray(compact.sampleMetrics.sessionRailTitles.examples) ? compact.sampleMetrics.sessionRailTitles.examples.slice(0, 4) : [] } : null,",
|
|
" turnTimingRows: compact.sampleMetrics.turnTimingRows ?? null,",
|
|
" turnTimingNonMonotonicCount: compact.sampleMetrics.turnTimingNonMonotonicCount ?? null,",
|
|
" turnTimingTotalElapsedDecreaseCount: compact.sampleMetrics.turnTimingTotalElapsedDecreaseCount ?? null,",
|
|
" turnTimingTotalElapsedForwardJumpCount: compact.sampleMetrics.turnTimingTotalElapsedForwardJumpCount ?? null,",
|
|
" turnTimingTerminalElapsedGrowthCount: compact.sampleMetrics.turnTimingTerminalElapsedGrowthCount ?? null,",
|
|
" turnTimingRecentUpdateJumpCount: compact.sampleMetrics.turnTimingRecentUpdateJumpCount ?? compact.sampleMetrics.turnTimingRecentUpdateSawtoothJumpCount ?? null",
|
|
" } : null,",
|
|
" runtimeAlerts: compact.runtimeAlerts ?? null,",
|
|
" pagePerformance: compact.pagePerformance ?? null,",
|
|
" findings: Array.isArray(compact.findings) ? compact.findings.slice(0, 4) : [],",
|
|
" archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4) : [],",
|
|
" pagePerformanceSlowApi: Array.isArray(compact.pagePerformanceSlowApi) ? compact.pagePerformanceSlowApi.slice(0, 2).map((item) => ({ path: item.path ?? item.route ?? null, route: item.route ?? item.path ?? null, sampleCount: item.sampleCount ?? null, p95Ms: item.p95Ms ?? item.p95 ?? null, maxMs: item.maxMs ?? item.max ?? null, budgetMs: item.budgetMs ?? null, overBudgetCount: item.overBudgetCount ?? null, overFiveSecondCount: item.overFiveSecondCount ?? null, slowSamples: Array.isArray(item.slowSamples) ? item.slowSamples.slice(0, 1) : [] })) : [],",
|
|
" archivePagePerformanceSlowApi: Array.isArray(compact.archivePagePerformanceSlowApi) ? compact.archivePagePerformanceSlowApi.slice(0, 4).map((item) => ({ path: item.path ?? item.route ?? null, route: item.route ?? item.path ?? null, sampleCount: item.sampleCount ?? null, p95Ms: item.p95Ms ?? item.p95 ?? null, maxMs: item.maxMs ?? item.max ?? null, budgetMs: item.budgetMs ?? null, overBudgetCount: item.overBudgetCount ?? null, overFiveSecondCount: item.overFiveSecondCount ?? null, slowSamples: Array.isArray(item.slowSamples) ? item.slowSamples.slice(0, 1) : [] })) : [],",
|
|
" pagePerformanceSseStreams: Array.isArray(compact.pagePerformanceSseStreams) ? compact.pagePerformanceSseStreams.slice(0, 2).map((item) => ({ path: item.path ?? item.route ?? null, route: item.route ?? item.path ?? null, sampleCount: item.sampleCount ?? null, streamOpenP95Ms: item.streamOpenP95Ms ?? null, streamOpenMaxMs: item.streamOpenMaxMs ?? null, streamOpenOverFiveSecondCount: item.streamOpenOverFiveSecondCount ?? null, streamLifetimeOverFiveSecondCount: item.streamLifetimeOverFiveSecondCount ?? null })) : [],",
|
|
" httpErrorGroups: Array.isArray(compact.httpErrorGroups) ? compact.httpErrorGroups.slice(0, 2) : [],",
|
|
" requestFailedGroups: Array.isArray(compact.requestFailedGroups) ? compact.requestFailedGroups.slice(0, 2) : [],",
|
|
" consoleAlertGroups: Array.isArray(compact.consoleAlertGroups) ? compact.consoleAlertGroups.slice(0, 2) : [],",
|
|
" commandFailures: Array.isArray(compact.commandFailures) ? compact.commandFailures.slice(-3) : [],",
|
|
" turnTimingRecentUpdateJumps: Array.isArray(compact.turnTimingRecentUpdateJumps) ? compact.turnTimingRecentUpdateJumps.slice(0, 3) : [],",
|
|
" turnTimingTotalElapsedForwardJumps: Array.isArray(compact.turnTimingTotalElapsedForwardJumps) ? compact.turnTimingTotalElapsedForwardJumps.slice(0, 3) : [],",
|
|
" reportJsonPath: compact.reportJsonPath ?? reportJsonPath,",
|
|
" reportJsonSha256: compact.reportJsonSha256 ?? sha256(reportJsonPath),",
|
|
" reportMdPath: compact.reportMdPath ?? reportMdPath,",
|
|
" reportMdSha256: compact.reportMdSha256 ?? sha256(reportMdPath),",
|
|
" analyzer: { ...(compact.analyzer ?? {}), compactStdoutLimited: true, compactStdoutLimitBytes, originalCompactBytes: Buffer.byteLength(compactOutput(compact), 'utf8'), originalMinimalBytes: Buffer.byteLength(output, 'utf8') },",
|
|
" valuesRedacted: true",
|
|
" };",
|
|
" output = compactOutput(tiny);",
|
|
" if (Buffer.byteLength(output, 'utf8') > compactStdoutLimitBytes) {",
|
|
" const ultratiny = {",
|
|
" ok: compact.ok,",
|
|
" counts: compact.counts ?? null,",
|
|
" analysisWindow: compact.analysisWindow ?? null,",
|
|
" archiveSummary: compact.archiveSummary ?? null,",
|
|
" sampleMetrics: compact.sampleMetrics ? {",
|
|
" sampleCount: compact.sampleMetrics.sampleCount ?? null,",
|
|
" loadingSampleCount: compact.sampleMetrics.loadingSampleCount ?? null,",
|
|
" loadingMaxCount: compact.sampleMetrics.loadingMaxCount ?? null,",
|
|
" loadingOverFiveSecondSegmentCount: compact.sampleMetrics.loadingOverFiveSecondSegmentCount ?? null,",
|
|
" loading: compact.sampleMetrics.loading ? { summary: compact.sampleMetrics.loading.summary ?? null, longestSegments: Array.isArray(compact.sampleMetrics.loading.longestSegments) ? compact.sampleMetrics.loading.longestSegments.slice(0, 4) : [], owners: Array.isArray(compact.sampleMetrics.loading.owners) ? compact.sampleMetrics.loading.owners.slice(0, 4) : [] } : null,",
|
|
" sessionRailFallbackMajoritySampleCount: compact.sampleMetrics.sessionRailFallbackMajoritySampleCount ?? compact.sampleMetrics.sessionRailTitles?.summary?.majorityFallbackSampleCount ?? null,",
|
|
" turnTimingRows: compact.sampleMetrics.turnTimingRows ?? null,",
|
|
" turnTimingTotalElapsedForwardJumpCount: compact.sampleMetrics.turnTimingTotalElapsedForwardJumpCount ?? null,",
|
|
" turnTimingRecentUpdateJumpCount: compact.sampleMetrics.turnTimingRecentUpdateJumpCount ?? compact.sampleMetrics.turnTimingRecentUpdateSawtoothJumpCount ?? null",
|
|
" } : null,",
|
|
" runtimeAlerts: compact.runtimeAlerts ?? null,",
|
|
" pagePerformance: compact.pagePerformance ?? null,",
|
|
" findings: Array.isArray(compact.findings) ? compact.findings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [],",
|
|
" archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [],",
|
|
" httpErrorGroups: Array.isArray(compact.httpErrorGroups) ? compact.httpErrorGroups.slice(0, 2).map((item) => ({ count: item.count ?? null, method: item.method ?? null, status: item.status ?? null, path: item.path ?? null, lastAt: item.lastAt ?? null })) : [],",
|
|
" requestFailedGroups: Array.isArray(compact.requestFailedGroups) ? compact.requestFailedGroups.slice(0, 2).map((item) => ({ count: item.count ?? null, method: item.method ?? null, path: item.path ?? null, failureKinds: item.failureKinds ?? null })) : [],",
|
|
" commandFailures: Array.isArray(compact.commandFailures) ? compact.commandFailures.slice(-3).map((item) => ({ ts: item.ts ?? null, type: item.type ?? null, commandId: item.commandId ?? null, durationMs: item.durationMs ?? null, sampleSeq: item.sampleSeq ?? null, beforePath: item.beforePath ?? null, afterPath: item.afterPath ?? null, message: clip(item.message ?? item.failureKind ?? item.name, 120) })) : [],",
|
|
" turnTimingRecentUpdateJumps: Array.isArray(compact.turnTimingRecentUpdateJumps) ? compact.turnTimingRecentUpdateJumps.slice(0, 3) : [],",
|
|
" turnTimingElapsedZeroResets: Array.isArray(compact.turnTimingElapsedZeroResets) ? compact.turnTimingElapsedZeroResets.slice(0, 3) : [],",
|
|
" turnTimingTotalElapsedForwardJumps: Array.isArray(compact.turnTimingTotalElapsedForwardJumps) ? compact.turnTimingTotalElapsedForwardJumps.slice(0, 3) : [],",
|
|
" pagePerformanceSlowApi: Array.isArray(compact.pagePerformanceSlowApi) ? compact.pagePerformanceSlowApi.slice(0, 2).map((item) => ({ path: item.path ?? item.route ?? null, sampleCount: item.sampleCount ?? null, p95Ms: item.p95Ms ?? null, maxMs: item.maxMs ?? null, budgetMs: item.budgetMs ?? null, overBudgetCount: item.overBudgetCount ?? null, overFiveSecondCount: item.overFiveSecondCount ?? null })) : [],",
|
|
" archivePagePerformanceSlowApi: Array.isArray(compact.archivePagePerformanceSlowApi) ? compact.archivePagePerformanceSlowApi.slice(0, 4).map((item) => ({ path: item.path ?? item.route ?? null, sampleCount: item.sampleCount ?? null, p95Ms: item.p95Ms ?? null, maxMs: item.maxMs ?? null, budgetMs: item.budgetMs ?? null, overBudgetCount: item.overBudgetCount ?? null, overFiveSecondCount: item.overFiveSecondCount ?? null })) : [],",
|
|
" pagePerformanceSseStreams: Array.isArray(compact.pagePerformanceSseStreams) ? compact.pagePerformanceSseStreams.slice(0, 2).map((item) => ({ path: item.path ?? item.route ?? null, sampleCount: item.sampleCount ?? null, streamOpenP95Ms: item.streamOpenP95Ms ?? null, streamOpenMaxMs: item.streamOpenMaxMs ?? null, streamOpenBudgetMs: item.streamOpenBudgetMs ?? null, streamOpenOverBudgetCount: item.streamOpenOverBudgetCount ?? null, streamOpenOverFiveSecondCount: item.streamOpenOverFiveSecondCount ?? null, streamLifetimeOverFiveSecondCount: item.streamLifetimeOverFiveSecondCount ?? null })) : [],",
|
|
" reportJsonPath: compact.reportJsonPath ?? reportJsonPath,",
|
|
" reportJsonSha256: compact.reportJsonSha256 ?? sha256(reportJsonPath),",
|
|
" reportMdPath: compact.reportMdPath ?? reportMdPath,",
|
|
" reportMdSha256: compact.reportMdSha256 ?? sha256(reportMdPath),",
|
|
" analyzer: { ...(compact.analyzer ?? {}), compactStdoutLimited: true, ultratiny: true, compactStdoutLimitBytes, originalCompactBytes: Buffer.byteLength(compactOutput(compact), 'utf8'), originalTinyBytes: Buffer.byteLength(output, 'utf8') },",
|
|
" valuesRedacted: true",
|
|
" };",
|
|
" output = compactOutput(ultratiny);",
|
|
" }",
|
|
" if (Buffer.byteLength(output, 'utf8') > compactStdoutLimitBytes) {",
|
|
" output = compactOutput({ ok: compact.ok, counts: compact.counts ?? null, jsonlScope: compact.jsonlScope ?? null, analysisWindow: compact.analysisWindow ?? null, archiveSummary: compact.archiveSummary ?? null, findings: Array.isArray(compact.findings) ? compact.findings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [], archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [], commandFailures: Array.isArray(compact.commandFailures) ? compact.commandFailures.slice(-3).map((item) => ({ ts: item.ts ?? null, type: item.type ?? null, commandId: item.commandId ?? null, durationMs: item.durationMs ?? null, sampleSeq: item.sampleSeq ?? null, beforePath: item.beforePath ?? null, afterPath: item.afterPath ?? null, message: clip(item.message ?? item.failureKind ?? item.name, 120) })) : [], archivePagePerformanceSlowApi: Array.isArray(compact.archivePagePerformanceSlowApi) ? compact.archivePagePerformanceSlowApi.slice(0, 4).map((item) => ({ path: item.path ?? item.route ?? null, sampleCount: item.sampleCount ?? null, p95Ms: item.p95Ms ?? null, maxMs: item.maxMs ?? null, overFiveSecondCount: item.overFiveSecondCount ?? null })) : [], reportJsonPath: compact.reportJsonPath ?? reportJsonPath, reportJsonSha256: compact.reportJsonSha256 ?? sha256(reportJsonPath), reportMdPath: compact.reportMdPath ?? reportMdPath, reportMdSha256: compact.reportMdSha256 ?? sha256(reportMdPath), analyzer: { ...(compact.analyzer ?? {}), compactStdoutLimited: true, ultratiny: true, hardFallback: true, compactStdoutLimitBytes }, valuesRedacted: true });",
|
|
" }",
|
|
" }",
|
|
"}",
|
|
"console.log(output);",
|
|
"UNIDESK_WEB_OBSERVE_ANALYZE_COMPACT",
|
|
"exit \"$analyzer_exit\"",
|
|
].join("\n");
|
|
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
|
const primaryAnalysis = recoverWebObserveAnalyzeTurnDetails(options, spec, parseJsonObject(result.stdout));
|
|
const artifactAnalysis = (result.timedOut || result.exitCode !== 0 || primaryAnalysis === null)
|
|
? recoverWebObserveAnalyzeFromArtifacts(options, spec, result)
|
|
: null;
|
|
const analysis = recoverWebObserveAnalyzeTurnDetails(options, spec, artifactAnalysis ?? primaryAnalysis);
|
|
const analysisOk = analysis?.ok === true;
|
|
const analysisFailure = analysisOk
|
|
? null
|
|
: {
|
|
reason: analysis === null ? "analyzer-output-not-json" : "analyzer-reported-not-ok",
|
|
exitCode: result.exitCode,
|
|
timedOut: result.timedOut,
|
|
parsedJson: analysis !== null,
|
|
recoveredFromArtifacts: artifactAnalysis !== null,
|
|
stdoutBytes: result.stdout.length,
|
|
stderrBytes: result.stderr.length,
|
|
stdoutTail: result.stdout.trim().slice(-1200),
|
|
stderrTail: result.stderr.trim().slice(-1200),
|
|
workspace: spec.workspace,
|
|
observerId: webObserveIdFromOptions(options),
|
|
valuesRedacted: true,
|
|
};
|
|
const analysisArtifactDir = options.stateDir ? join(options.stateDir, "analysis") : "";
|
|
const failureAnalysis = analysis ?? (analysisFailure ? {
|
|
ok: false,
|
|
command: "web-probe-observe analyze",
|
|
stateDir: options.stateDir,
|
|
reportJsonPath: analysisArtifactDir ? join(analysisArtifactDir, "report.json") : null,
|
|
reportMdPath: analysisArtifactDir ? join(analysisArtifactDir, "report.md") : null,
|
|
analyzer: {
|
|
exitCode: result.exitCode,
|
|
timedOut: result.timedOut,
|
|
stdoutBytes: result.stdout.length,
|
|
stderrBytes: result.stderr.length,
|
|
stdoutPath: analysisArtifactDir ? join(analysisArtifactDir, "analyzer-stdout.json") : null,
|
|
stderrPath: analysisArtifactDir ? join(analysisArtifactDir, "analyzer-stderr.log") : null,
|
|
stdoutTail: result.stdout.trim().slice(-1200),
|
|
stderrTail: result.stderr.trim().slice(-1200),
|
|
recoveredFrom: "analyzer-timeout-failure-contract",
|
|
valuesRedacted: true,
|
|
},
|
|
findings: [{
|
|
kind: result.timedOut ? "web-probe-analyzer-timeout" : "web-probe-analyzer-failed",
|
|
severity: "red",
|
|
count: 1,
|
|
summary: result.timedOut
|
|
? "observe analyze timed out before producing a compact report; inspect analyzer stdout/stderr artifacts under the observer analysis directory"
|
|
: "observe analyze failed before producing a compact report; inspect analyzer stdout/stderr artifacts under the observer analysis directory",
|
|
}],
|
|
next: {
|
|
collectAnalyzerStdout: options.stateDir ? `bun scripts/cli.ts hwlab nodes web-probe observe collect --node ${options.node} --lane ${options.lane} --state-dir ${options.stateDir} --file analysis/analyzer-stdout.json` : null,
|
|
collectAnalyzerStderr: options.stateDir ? `bun scripts/cli.ts hwlab nodes web-probe observe collect --node ${options.node} --lane ${options.lane} --state-dir ${options.stateDir} --file analysis/analyzer-stderr.log` : null,
|
|
valuesRedacted: true,
|
|
},
|
|
valuesRedacted: true,
|
|
} : null);
|
|
return withWebObserveAnalyzeRendered({
|
|
ok: analysisOk,
|
|
status: analysisOk ? "analyzed" : "blocked",
|
|
command: webObserveCommandLabel("analyze", options),
|
|
id: webObserveIdFromOptions(options),
|
|
node: options.node,
|
|
lane: options.lane,
|
|
workspace: spec.workspace,
|
|
analysis: failureAnalysis,
|
|
failure: analysisFailure,
|
|
alertThresholds,
|
|
result: analysis === null ? compactCommandResultWithStdoutTail(result) : compactCommandResult(result),
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
|
|
function recoverWebObserveAnalyzeFromArtifacts(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec, result: { exitCode: number; timedOut: boolean }): Record<string, unknown> | null {
|
|
const recoverScript = [
|
|
"set -eu",
|
|
nodeWebObserveResolveStateDirShell(options),
|
|
"analysis_dir=\"$state_dir/analysis\"",
|
|
"analysis_stdout=\"$analysis_dir/analyzer-stdout.json\"",
|
|
"analysis_stderr=\"$analysis_dir/analyzer-stderr.log\"",
|
|
"report_json=\"$analysis_dir/report.json\"",
|
|
"report_md=\"$analysis_dir/report.md\"",
|
|
`transport_exit=${shellQuote(String(result.exitCode))}`,
|
|
`transport_timed_out=${shellQuote(result.timedOut ? "true" : "false")}`,
|
|
"node - \"$state_dir\" \"$analysis_stdout\" \"$analysis_stderr\" \"$report_json\" \"$report_md\" \"$transport_exit\" \"$transport_timed_out\" <<'UNIDESK_WEB_OBSERVE_RECOVER_ANALYZE_ARTIFACT'",
|
|
"const fs = require('fs');",
|
|
"const crypto = require('crypto');",
|
|
"const [stateDir, stdoutPath, stderrPath, reportJsonPath, reportMdPath, transportExitRaw, transportTimedOutRaw] = process.argv.slice(2);",
|
|
"const readText = (path) => { try { return fs.readFileSync(path, 'utf8'); } catch { return ''; } };",
|
|
"const readJson = (path) => { const text = readText(path); if (!text.trim()) return null; try { return JSON.parse(text); } catch { return null; } };",
|
|
"const statSize = (path) => { try { return fs.statSync(path).size; } catch { return 0; } };",
|
|
"const sha256 = (path) => { const text = readText(path); return text ? 'sha256:' + crypto.createHash('sha256').update(text).digest('hex') : null; };",
|
|
"const objectOrNull = (value) => value && typeof value === 'object' && !Array.isArray(value) ? value : null;",
|
|
"const arr = (value) => Array.isArray(value) ? value : [];",
|
|
"const clip = (value, limit = 160) => value === null || value === undefined ? null : String(value).slice(0, limit);",
|
|
"const numberish = (...values) => { for (const value of values) { const n = Number(value); if (Number.isFinite(n)) return value; } return null; };",
|
|
"const slimFinding = (item) => { const v = objectOrNull(item) || {}; return { kind: clip(v.kind ?? v.id ?? v.code, 48), code: clip(v.code ?? v.id ?? v.kind, 48), severity: clip(v.severity ?? v.level, 24), level: clip(v.level ?? v.severity, 24), count: v.count ?? v.sampleCount ?? null, sampleCount: v.sampleCount ?? v.count ?? null, summary: clip(v.summary ?? v.message, 180), message: clip(v.message ?? v.summary, 180) }; };",
|
|
"const slimRound = (item) => { const v = objectOrNull(item) || {}; return { promptIndex: v.promptIndex ?? null, promptTextHash: clip(v.promptTextHash, 80), sampleCount: v.sampleCount ?? null, firstSeq: v.firstSeq ?? null, lastSeq: v.lastSeq ?? null, lastTotalElapsedSeconds: v.lastTotalElapsedSeconds ?? null, lastRecentUpdateSeconds: v.lastRecentUpdateSeconds ?? null, loadingSamples: v.loadingSamples ?? null, maxLoadingCount: v.maxLoadingCount ?? null, loadingOwnerCount: v.loadingOwnerCount ?? null, diagnosticSamples: v.diagnosticSamples ?? null, terminalSamples: v.terminalSamples ?? null, finalTextSamples: v.finalTextSamples ?? null, turnTimingTotalElapsedZeroResetCount: v.turnTimingTotalElapsedZeroResetCount ?? null, turnTimingTotalElapsedForwardJumpCount: v.turnTimingTotalElapsedForwardJumpCount ?? null, turnTimingTotalElapsedForwardJumpMaxSeconds: v.turnTimingTotalElapsedForwardJumpMaxSeconds ?? null, turnTimingRecentUpdateJumpCount: v.turnTimingRecentUpdateJumpCount ?? null, turnTimingRecentUpdateMaxIncreaseSeconds: v.turnTimingRecentUpdateMaxIncreaseSeconds ?? null }; };",
|
|
"const slimTurnColumn = (item) => { const v = objectOrNull(item) || {}; return { label: clip(v.label, 24), source: clip(v.source, 48), pageRole: clip(v.pageRole, 24), pageId: clip(v.pageId, 32), pageEpoch: v.pageEpoch ?? null, promptIndex: v.promptIndex ?? null, lastPromptIndex: v.lastPromptIndex ?? null, firstSeq: v.firstSeq ?? null, lastSeq: v.lastSeq ?? null, traceId: clip(v.traceId, 64), messageId: clip(v.messageId, 64) }; };",
|
|
"const slimGroup = (item) => { const v = objectOrNull(item) || {}; return { count: v.count ?? null, method: clip(v.method, 12), status: v.status ?? null, path: clip(v.path ?? v.urlPath ?? v.route, 96), firstAt: v.firstAt ?? null, lastAt: v.lastAt ?? null, text: clip(v.text ?? v.preview, 180), failureKinds: arr(v.failureKinds).slice(0, 4).map((x) => clip(x, 48)), traceIds: arr(v.traceIds).slice(0, 3).map((x) => clip(x, 64)) }; };",
|
|
"const slimSlowSample = (item) => { const v = objectOrNull(item) || {}; return { ts: v.ts ?? null, seq: v.seq ?? null, path: clip(v.path ?? v.rawPath, 96), initiatorType: clip(v.initiatorType, 24), durationMs: v.durationMs ?? null, requestToResponseStartMs: v.requestToResponseStartMs ?? v.streamOpenMs ?? null, responseTransferMs: v.responseTransferMs ?? null, timingStatus: clip(v.timingStatus, 16), nextHopProtocol: clip(v.nextHopProtocol, 24), serverTimingNames: arr(v.serverTimingNames).slice(0, 4).map((x) => clip(x, 32)), otelTraceId: clip(v.otelTraceId, 32) }; };",
|
|
"const slimSlowApi = (item) => { const v = objectOrNull(item) || {}; return { path: clip(v.path ?? v.route, 96), route: clip(v.route ?? v.path, 96), sampleCount: v.sampleCount ?? null, p95Ms: v.p95Ms ?? v.p95 ?? null, maxMs: v.maxMs ?? v.max ?? null, budgetMs: v.budgetMs ?? null, overBudgetCount: v.overBudgetCount ?? null, overFiveSecondCount: v.overFiveSecondCount ?? null, slowSamples: arr(v.slowSamples).slice(0, 3).map(slimSlowSample) }; };",
|
|
"const compactLoading = (value) => { const v = objectOrNull(value); if (!v) return null; const s = objectOrNull(v.summary) || {}; return { summary: s, longestSegments: arr(v.longestSegments ?? v.segments).slice(0, 8), owners: arr(v.owners).slice(0, 8), timeline: arr(v.timeline).slice(-12) }; };",
|
|
"const compactSessionRailTitles = (value) => { const v = objectOrNull(value); if (!v) return null; return { summary: objectOrNull(v.summary) || {}, samples: arr(v.samples).slice(0, 8), examples: arr(v.examples).slice(0, 8) }; };",
|
|
"const compactTraceOrder = (value) => { const v = objectOrNull(value); if (!v) return null; const s = objectOrNull(v.summary) || {}; return { summary: { sampleCount: v.sampleCount ?? s.sampleCount ?? null, traceRowCount: v.traceRowCount ?? s.traceRowCount ?? null, orderAnomalyCount: v.orderAnomalyCount ?? s.orderAnomalyCount ?? arr(v.orderAnomalies).length, completionNotLastCount: v.completionNotLastCount ?? s.completionNotLastCount ?? arr(v.completionNotLast).length }, orderAnomalies: arr(v.orderAnomalies).slice(0, 8) }; };",
|
|
"const compactMetrics = (value) => { const v = objectOrNull(value) || {}; const s = objectOrNull(v.summary) || {}; return { sampleCount: numberish(v.sampleCount, s.sampleCount), rounds: arr(v.rounds ?? v.roundItems).slice(-8).map(slimRound), roundItems: arr(v.roundItems ?? v.rounds).slice(-8).map(slimRound), turnColumns: arr(v.turnColumns).slice(-12).map(slimTurnColumn), turnTimingRows: numberish(v.turnTimingRows, s.turnTimingRows), turnTimingNonMonotonicCount: numberish(v.turnTimingNonMonotonicCount, s.turnTimingNonMonotonicCount), turnTimingTotalElapsedDecreaseCount: numberish(v.turnTimingTotalElapsedDecreaseCount, s.turnTimingTotalElapsedDecreaseCount), turnTimingTotalElapsedZeroResetCount: numberish(v.turnTimingTotalElapsedZeroResetCount, s.turnTimingTotalElapsedZeroResetCount), turnTimingTotalElapsedForwardJumpCount: numberish(v.turnTimingTotalElapsedForwardJumpCount, s.turnTimingTotalElapsedForwardJumpCount), turnTimingTerminalElapsedGrowthCount: numberish(v.turnTimingTerminalElapsedGrowthCount, s.turnTimingTerminalElapsedGrowthCount), turnTimingRecentUpdateJumpCount: numberish(v.turnTimingRecentUpdateJumpCount, v.turnTimingRecentUpdateSawtoothJumpCount, s.turnTimingRecentUpdateJumpCount, s.turnTimingRecentUpdateSawtoothJumpCount), turnTimingRecentUpdateMaxIncreaseSeconds: numberish(v.turnTimingRecentUpdateMaxIncreaseSeconds, s.turnTimingRecentUpdateMaxIncreaseSeconds), loading: compactLoading(v.loading), traceOrder: compactTraceOrder(v.traceOrder), sessionRailTitles: compactSessionRailTitles(v.sessionRailTitles), promptSegments: numberish(v.promptSegments, s.promptSegments), loadingSampleCount: numberish(v.loadingSampleCount, s.loadingSampleCount), loadingMaxCount: numberish(v.loadingMaxCount, s.loadingMaxCount), loadingMaxOwnerCount: numberish(v.loadingMaxOwnerCount, s.loadingMaxOwnerCount), loadingOwnerCount: numberish(v.loadingOwnerCount, s.loadingOwnerCount), loadingLongestContinuousSeconds: numberish(v.loadingLongestContinuousSeconds, s.loadingLongestContinuousSeconds), loadingCurrentContinuousSeconds: numberish(v.loadingCurrentContinuousSeconds, s.loadingCurrentContinuousSeconds), loadingOverFiveSecondSegmentCount: numberish(v.loadingOverFiveSecondSegmentCount, s.loadingOverFiveSecondSegmentCount) }; };",
|
|
"const stdoutJson = readJson(stdoutPath);",
|
|
"const reportJson = readJson(reportJsonPath);",
|
|
"const source = objectOrNull(stdoutJson) || objectOrNull(reportJson);",
|
|
"if (!source) { console.log(JSON.stringify({ ok: false, command: 'web-probe-observe analyze', stateDir, error: 'web-probe-analyzer-artifacts-missing', analyzer: { recoveredFrom: 'analysis-artifact-after-transport-timeout', stdoutPath, stderrPath, reportJsonPath, reportMdPath, stdoutBytes: statSize(stdoutPath), stderrBytes: statSize(stderrPath), reportJsonBytes: statSize(reportJsonPath), reportMdBytes: statSize(reportMdPath), transportExitCode: Number(transportExitRaw), transportTimedOut: transportTimedOutRaw === 'true', valuesRedacted: true }, valuesRedacted: true })); process.exit(0); }",
|
|
"const recent = objectOrNull(source.windows?.recent) || {};",
|
|
"const srcMetrics = objectOrNull(source.sampleMetrics) || objectOrNull(recent.sampleMetrics) || {};",
|
|
"const pagePerformance = objectOrNull(source.pagePerformance) || objectOrNull(recent.pagePerformance) || {};",
|
|
"const runtimeAlerts = objectOrNull(source.runtimeAlerts) || objectOrNull(recent.runtimeAlerts) || {};",
|
|
"const promptNetwork = objectOrNull(source.promptNetwork) || objectOrNull(recent.promptNetwork) || {};",
|
|
"const archiveSummary = objectOrNull(source.archiveSummary) || {};",
|
|
"const archiveSampleMetrics = objectOrNull(archiveSummary.sampleMetrics) || objectOrNull(source.sampleMetrics?.summary) || objectOrNull(srcMetrics.summary) || {};",
|
|
"const slowApis = arr(source.pagePerformanceSlowApi).length > 0 ? arr(source.pagePerformanceSlowApi) : arr(pagePerformance.sameOriginApiByPath).filter((item) => Number(item?.overBudgetCount ?? item?.overFiveSecondCount ?? 0) > 0);",
|
|
"const compact = { ok: source.ok === true, command: source.command ?? 'web-probe-observe analyze', stateDir: source.stateDir ?? stateDir, jsonlScope: source.jsonlScope ?? null, alertThresholds: source.alertThresholds ?? null, counts: source.counts ?? null, archiveSummary: { ...archiveSummary, sampleMetrics: archiveSampleMetrics, pagePerformance: objectOrNull(archiveSummary.pagePerformance) || objectOrNull(pagePerformance.summary) || {}, runtimeAlerts: objectOrNull(archiveSummary.runtimeAlerts) || objectOrNull(runtimeAlerts.summary) || {}, redFindings: arr(archiveSummary.redFindings).slice(0, 12).map(slimFinding) }, analysisWindow: source.analysisWindow ?? objectOrNull(recent.summary), sampleMetrics: compactMetrics(srcMetrics), pageProvenance: objectOrNull(source.pageProvenance?.summary) || source.pageProvenance ?? null, pagePerformance: objectOrNull(pagePerformance.summary) || pagePerformance, promptNetwork: objectOrNull(promptNetwork.summary) || promptNetwork, runtimeAlerts: objectOrNull(runtimeAlerts.summary) || runtimeAlerts, runnerErrors: arr(source.runnerErrors).slice(-8), commandFailures: arr(source.commandFailures).slice(-8), httpErrorGroups: arr(source.httpErrorGroups ?? runtimeAlerts.networkHttpErrorsByPath).slice(0, 8).map(slimGroup), requestFailedGroups: arr(source.requestFailedGroups ?? runtimeAlerts.networkRequestFailedByPath).slice(0, 8).map(slimGroup), domDiagnosticGroups: arr(source.domDiagnosticGroups ?? runtimeAlerts.domDiagnosticsByText).slice(0, 5).map(slimGroup), domDiagnosticSamples: arr(source.domDiagnosticSamples ?? runtimeAlerts.domDiagnostics).slice(0, 8).map(slimGroup), consoleAlertGroups: arr(source.consoleAlertGroups ?? runtimeAlerts.consoleAlertsByPath).slice(0, 8).map(slimGroup), consoleAlertSamples: arr(source.consoleAlertSamples ?? runtimeAlerts.consoleAlerts).slice(0, 8).map(slimGroup), turnTimingRecentUpdateJumps: arr(source.turnTimingRecentUpdateJumps ?? srcMetrics.turnTimingRecentUpdateSawtoothJumps).slice(0, 8), turnTimingElapsedZeroResets: arr(source.turnTimingElapsedZeroResets ?? srcMetrics.turnTimingElapsedZeroResets).slice(0, 8), turnTimingTotalElapsedForwardJumps: arr(source.turnTimingTotalElapsedForwardJumps ?? srcMetrics.turnTimingTotalElapsedForwardJumps).slice(0, 8), pagePerformanceSlowApi: slowApis.slice(0, 8).map(slimSlowApi), archivePagePerformanceSlowApi: arr(source.archivePagePerformanceSlowApi).slice(0, 8).map(slimSlowApi), pagePerformancePartialApi: arr(source.pagePerformancePartialApi).slice(0, 8), pagePerformanceSseStreams: arr(source.pagePerformanceSseStreams).slice(0, 8), findings: arr(source.findings).slice(0, 12).map(slimFinding), archiveRedFindings: arr(source.archiveRedFindings ?? archiveSummary.redFindings).slice(0, 12).map(slimFinding), reportJsonPath: source.reportJsonPath ?? reportJsonPath, reportJsonSha256: source.reportJsonSha256 ?? sha256(reportJsonPath), reportMdPath: source.reportMdPath ?? reportMdPath, reportMdSha256: source.reportMdSha256 ?? sha256(reportMdPath), analyzer: { ...(objectOrNull(source.analyzer) || {}), recoveredFrom: 'analysis-artifact-after-transport-timeout', stdoutPath, stderrPath, stdoutBytes: statSize(stdoutPath), stderrBytes: statSize(stderrPath), reportJsonBytes: statSize(reportJsonPath), reportMdBytes: statSize(reportMdPath), transportExitCode: Number(transportExitRaw), transportTimedOut: transportTimedOutRaw === 'true', valuesRedacted: true }, valuesRedacted: true };",
|
|
"console.log(JSON.stringify(compact));",
|
|
"UNIDESK_WEB_OBSERVE_RECOVER_ANALYZE_ARTIFACT",
|
|
].join("\n");
|
|
const recovered = parseJsonObject(runTransWorkspaceStdinScript(options.node, spec.workspace, recoverScript, Math.min(options.commandTimeoutSeconds, 30)).stdout);
|
|
return recovered;
|
|
}
|
|
|
|
function recoverWebObserveAnalyzeTurnDetails(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec, analysis: Record<string, unknown> | null): Record<string, unknown> | null {
|
|
if (analysis === null || typeof analysis !== "object") return analysis;
|
|
const sampleMetrics = record(analysis.sampleMetrics);
|
|
const hasRounds = Array.isArray(sampleMetrics?.rounds) && sampleMetrics.rounds.length > 0;
|
|
const hasColumns = Array.isArray(sampleMetrics?.turnColumns) && sampleMetrics.turnColumns.length > 0;
|
|
const columnsHavePageIdentity = hasColumns && (sampleMetrics?.turnColumns as unknown[]).some((item) => {
|
|
const column = record(item);
|
|
return typeof column?.pageRole === "string" || typeof column?.pageId === "string";
|
|
});
|
|
if (hasRounds && hasColumns && columnsHavePageIdentity) return analysis;
|
|
const reportJsonPath = typeof analysis.reportJsonPath === "string" ? analysis.reportJsonPath : "";
|
|
if (!reportJsonPath) return analysis;
|
|
const archiveMetrics = record(record(analysis.archiveSummary)?.sampleMetrics);
|
|
const archiveHasTurnEvidence = Number(archiveMetrics?.rounds ?? 0) > 0 || Number(archiveMetrics?.turnColumns ?? 0) > 0 || Number(archiveMetrics?.turnTimingRows ?? 0) > 0;
|
|
if (!archiveHasTurnEvidence) return analysis;
|
|
const recoverScript = [
|
|
"set -eu",
|
|
`report_json=${shellQuote(reportJsonPath)}`,
|
|
"node - \"$report_json\" <<'UNIDESK_WEB_OBSERVE_RECOVER_TURN_DETAILS'",
|
|
"const fs = require('fs');",
|
|
"const reportJsonPath = process.argv[2];",
|
|
"const readJson = (path) => { try { return JSON.parse(fs.readFileSync(path, 'utf8')); } catch { return null; } };",
|
|
"const report = readJson(reportJsonPath) || {};",
|
|
"const stdoutJson = readJson(String(reportJsonPath).replace(/\\/analysis\\/report\\.json$/u, '/analysis/analyzer-stdout.json')) || {};",
|
|
"const objectOrNull = (value) => value && typeof value === 'object' && !Array.isArray(value) ? value : null;",
|
|
"const firstNonEmptyArray = (...values) => { for (const value of values) if (Array.isArray(value) && value.length > 0) return value; return []; };",
|
|
"const clip = (value, limit = 160) => value === null || value === undefined ? null : String(value).slice(0, limit);",
|
|
"const slimRound = (item) => { const v = objectOrNull(item) || {}; return { promptIndex: v.promptIndex ?? null, sampleCount: v.sampleCount ?? null, loadingSamples: v.loadingSamples ?? null, maxLoadingCount: v.maxLoadingCount ?? null, loadingOwnerCount: v.loadingOwnerCount ?? null, lastTotalElapsedSeconds: v.lastTotalElapsedSeconds ?? null, lastRecentUpdateSeconds: v.lastRecentUpdateSeconds ?? null, diagnosticSamples: v.diagnosticSamples ?? null, terminalSamples: v.terminalSamples ?? null, turnTimingTotalElapsedForwardJumpCount: v.turnTimingTotalElapsedForwardJumpCount ?? null, turnTimingTotalElapsedForwardJumpMaxSeconds: v.turnTimingTotalElapsedForwardJumpMaxSeconds ?? null, promptTextHash: clip(v.promptTextHash, 80) }; };",
|
|
"const slimTurnColumn = (item) => { const v = objectOrNull(item) || {}; return { label: clip(v.label, 24), pageRole: clip(v.pageRole, 24), pageId: clip(v.pageId, 32), pageEpoch: v.pageEpoch ?? null, promptIndex: v.promptIndex ?? null, lastPromptIndex: v.lastPromptIndex ?? null, traceId: clip(v.traceId, 48), firstSeq: v.firstSeq ?? null, lastSeq: v.lastSeq ?? null, source: clip(v.source, 48) }; };",
|
|
"const rounds = firstNonEmptyArray(stdoutJson.sampleMetrics?.rounds, stdoutJson.sampleMetrics?.roundItems, report.windows?.recent?.sampleMetrics?.rounds, report.sampleMetrics?.rounds).slice(-8).map(slimRound);",
|
|
"const turnColumns = firstNonEmptyArray(stdoutJson.sampleMetrics?.turnColumns, report.windows?.recent?.sampleMetrics?.turnColumns, report.sampleMetrics?.turnColumns).slice(-12).map(slimTurnColumn);",
|
|
"console.log(JSON.stringify({ ok: true, rounds, turnColumns }));",
|
|
"UNIDESK_WEB_OBSERVE_RECOVER_TURN_DETAILS",
|
|
].join("\n");
|
|
const recovered = parseJsonObject(runTransWorkspaceStdinScript(options.node, spec.workspace, recoverScript, Math.min(options.commandTimeoutSeconds, 30)).stdout);
|
|
const recoveredRounds = Array.isArray(recovered.rounds) ? recovered.rounds : [];
|
|
const recoveredColumns = Array.isArray(recovered.turnColumns) ? recovered.turnColumns : [];
|
|
if (recoveredRounds.length === 0 && recoveredColumns.length === 0) return analysis;
|
|
const nextMetrics = { ...(sampleMetrics ?? {}) };
|
|
if (!hasRounds && recoveredRounds.length > 0) {
|
|
nextMetrics.roundItems = recoveredRounds;
|
|
nextMetrics.rounds = recoveredRounds;
|
|
}
|
|
if (!hasColumns && recoveredColumns.length > 0) nextMetrics.turnColumns = recoveredColumns;
|
|
return { ...analysis, sampleMetrics: nextMetrics, recoveredTurnDetails: { rounds: recoveredRounds.length, turnColumns: recoveredColumns.length, source: "analysis-report", valuesRedacted: true } };
|
|
}
|
|
|
|
function withWebObserveAnalyzeRendered(value: Record<string, unknown>): RenderedCliResult {
|
|
return {
|
|
ok: value.ok !== false,
|
|
command: typeof value.command === "string" ? value.command : "hwlab nodes web-probe observe analyze",
|
|
contentType: "text/plain",
|
|
renderedText: renderWebObserveAnalyzeTable(value),
|
|
};
|
|
}
|
|
|
|
function renderWebObserveAnalyzeTable(value: Record<string, unknown>): string {
|
|
const analysis = record(value.analysis) ?? value;
|
|
const analyzer = record(analysis?.analyzer);
|
|
const result = record(value.result);
|
|
const counts = record(analysis?.counts);
|
|
const jsonlScope = record(analysis?.jsonlScope);
|
|
const analysisWindow = record(analysis?.analysisWindow);
|
|
const archiveSummary = record(analysis?.archiveSummary);
|
|
const nestedSampleMetrics = record(analysis?.sampleMetrics);
|
|
const directSampleMetrics = record(value.sampleMetrics);
|
|
const hasRenderableTurnMetrics = (item: Record<string, unknown> | null): boolean => {
|
|
if (!item) return false;
|
|
return (Array.isArray(item.roundItems) && item.roundItems.length > 0)
|
|
|| (Array.isArray(item.rounds) && item.rounds.length > 0)
|
|
|| (Array.isArray(item.turnColumns) && item.turnColumns.length > 0);
|
|
};
|
|
const sampleMetrics = hasRenderableTurnMetrics(nestedSampleMetrics)
|
|
? nestedSampleMetrics
|
|
: hasRenderableTurnMetrics(directSampleMetrics)
|
|
? directSampleMetrics
|
|
: nestedSampleMetrics ?? directSampleMetrics;
|
|
const archiveMetrics = record(archiveSummary?.sampleMetrics);
|
|
const archivePagePerformance = record(archiveSummary?.pagePerformance);
|
|
const runtimeAlerts = record(analysis?.runtimeAlerts);
|
|
const pagePerformance = record(analysis?.pagePerformance);
|
|
const pagePerformanceSummary = record(pagePerformance?.summary);
|
|
const alertThresholds = record(analysis?.alertThresholds ?? pagePerformanceSummary?.alertThresholds ?? value.alertThresholds);
|
|
const budgetLabel = (rawValue: unknown): string => {
|
|
const parsed = Number(rawValue);
|
|
if (!Number.isFinite(parsed) || parsed <= 0) return "unconfigured";
|
|
const ms = parsed;
|
|
return ms % 1000 === 0 ? `${Math.round(ms / 1000)}s` : `${ms}ms`;
|
|
};
|
|
const sameOriginApiBudgetLabel = budgetLabel(alertThresholds?.sameOriginApiSlowMs ?? pagePerformanceSummary?.budgetMs);
|
|
const partialApiBudgetLabel = budgetLabel(alertThresholds?.partialApiSlowMs);
|
|
const streamOpenBudgetLabel = budgetLabel(alertThresholds?.longLivedStreamOpenSlowMs);
|
|
const loadingBudgetLabel = budgetLabel(alertThresholds?.visibleLoadingSlowMs);
|
|
const pageLabel = (item: Record<string, unknown>): string => {
|
|
const role = webObserveText(item.pageRole);
|
|
const pageId = webObserveShort(webObserveText(item.pageId), 18);
|
|
const epoch = webObserveText(item.pageEpoch);
|
|
if (role === "-" && pageId === "-") return "-";
|
|
return epoch === "-" ? `${role}/${pageId}` : `${role}/${pageId}#${epoch}`;
|
|
};
|
|
const promptNetwork = record(analysis?.promptNetwork);
|
|
const loading = record(sampleMetrics?.loading);
|
|
const loadingSummary = record(loading?.summary);
|
|
const traceOrder = record(sampleMetrics?.traceOrder);
|
|
const traceOrderSummary = record(traceOrder?.summary);
|
|
const traceOrderAnomalies = Array.isArray(traceOrder?.orderAnomalies) ? traceOrder.orderAnomalies.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const loadingSegments = Array.isArray(loading?.longestSegments) ? loading.longestSegments.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const loadingOwners = Array.isArray(loading?.owners) ? loading.owners.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const sessionRailTitles = record(sampleMetrics?.sessionRailTitles);
|
|
const sessionRailTitleSummary = record(sessionRailTitles?.summary);
|
|
const sessionRailTitleSamples = Array.isArray(sessionRailTitles?.samples) ? sessionRailTitles.samples.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const sessionRailTitleExamples = Array.isArray(sessionRailTitles?.examples) ? sessionRailTitles.examples.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const roundSource = Array.isArray(sampleMetrics?.roundItems) && sampleMetrics.roundItems.length > 0
|
|
? sampleMetrics.roundItems
|
|
: Array.isArray(sampleMetrics?.rounds)
|
|
? sampleMetrics.rounds
|
|
: sampleMetrics?.roundItems ?? sampleMetrics?.rounds;
|
|
const rounds = webObserveArray(roundSource).map(record).filter((item): item is Record<string, unknown> => item !== null).slice(-8);
|
|
const turnColumns = webObserveArray(sampleMetrics?.turnColumns).map(record).filter((item): item is Record<string, unknown> => item !== null).slice(-12);
|
|
const roundCount = Array.isArray(sampleMetrics?.rounds) ? sampleMetrics.rounds.length : sampleMetrics?.rounds;
|
|
const turnColumnCount = Array.isArray(sampleMetrics?.turnColumns) ? sampleMetrics.turnColumns.length : sampleMetrics?.turnColumns;
|
|
const slowApis = Array.isArray(analysis?.pagePerformanceSlowApi) ? analysis.pagePerformanceSlowApi.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const archiveSlowApis = Array.isArray(analysis?.archivePagePerformanceSlowApi) ? analysis.archivePagePerformanceSlowApi.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const partialApis = Array.isArray(analysis?.pagePerformancePartialApi) ? analysis.pagePerformancePartialApi.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const sseStreams = Array.isArray(analysis?.pagePerformanceSseStreams) ? analysis.pagePerformanceSseStreams.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const findings = Array.isArray(analysis?.findings) ? analysis.findings.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const archiveRedFindings = Array.isArray(analysis?.archiveRedFindings)
|
|
? analysis.archiveRedFindings.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8)
|
|
: Array.isArray(archiveSummary?.redFindings)
|
|
? archiveSummary.redFindings.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8)
|
|
: [];
|
|
const httpErrorGroups = Array.isArray(analysis?.httpErrorGroups) ? analysis.httpErrorGroups.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const requestFailedGroups = Array.isArray(analysis?.requestFailedGroups) ? analysis.requestFailedGroups.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const domDiagnostics = Array.isArray(analysis?.domDiagnosticGroups) ? analysis.domDiagnosticGroups.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 5) : [];
|
|
const domDiagnosticSamples = Array.isArray(analysis?.domDiagnosticSamples) ? analysis.domDiagnosticSamples.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const consoleAlertGroups = Array.isArray(analysis?.consoleAlertGroups) ? analysis.consoleAlertGroups.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const consoleAlertSamples = Array.isArray(analysis?.consoleAlertSamples) ? analysis.consoleAlertSamples.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const runnerErrors = Array.isArray(analysis?.runnerErrors) ? analysis.runnerErrors.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(-8) : [];
|
|
const commandFailures = Array.isArray(analysis?.commandFailures) ? analysis.commandFailures.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(-8) : [];
|
|
const recentUpdateJumps = Array.isArray(analysis?.turnTimingRecentUpdateJumps) ? analysis.turnTimingRecentUpdateJumps.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const elapsedZeroResets = Array.isArray(analysis?.turnTimingElapsedZeroResets) ? analysis.turnTimingElapsedZeroResets.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const elapsedForwardJumps = Array.isArray(analysis?.turnTimingTotalElapsedForwardJumps) ? analysis.turnTimingTotalElapsedForwardJumps.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const domDiagnosticRows = domDiagnostics.length > 0
|
|
? domDiagnostics.map((item) => [
|
|
webObserveText(item.count),
|
|
webObserveShort(webObserveText(item.firstAt), 24),
|
|
webObserveShort(webObserveText(item.lastAt), 24),
|
|
webObserveShort(webObserveText(item.text), 96),
|
|
])
|
|
: domDiagnosticSamples.map((item) => [
|
|
webObserveText(item.seq),
|
|
webObserveShort(webObserveText(item.ts), 24),
|
|
webObserveShort([
|
|
webObserveText(item.source),
|
|
webObserveText(item.diagnosticCode),
|
|
item.traceId ? `trace=${webObserveText(item.traceId)}` : "",
|
|
item.httpStatus !== undefined && item.httpStatus !== null ? `http=${webObserveText(item.httpStatus)}` : "",
|
|
item.idleSeconds !== undefined && item.idleSeconds !== null ? `idle=${webObserveText(item.idleSeconds)}` : "",
|
|
item.waitingFor ? `waitingFor=${webObserveText(item.waitingFor)}` : "",
|
|
item.lastEventLabel ? `lastEvent=${webObserveText(item.lastEventLabel)}` : "",
|
|
].filter((part) => part !== "" && part !== "-").join(" "), 64),
|
|
webObserveShort(webObserveText(item.text), 96),
|
|
]);
|
|
const consoleAlertRows = consoleAlertGroups.length > 0
|
|
? consoleAlertGroups.map((item) => [
|
|
webObserveText(item.count),
|
|
webObserveShort(webObserveText(item.type), 14),
|
|
webObserveText(item.status),
|
|
webObserveShort(webObserveText(item.path), 52),
|
|
webObserveShort(webObserveText(item.lastAt ?? item.firstAt), 24),
|
|
webObserveShort(webObserveArray(item.traceIds).join(",") || "-", 28),
|
|
])
|
|
: consoleAlertSamples.map((item) => [
|
|
webObserveShort(webObserveText(item.ts), 24),
|
|
webObserveShort(webObserveText(item.type), 14),
|
|
webObserveText(item.status),
|
|
webObserveShort(webObserveText(item.path), 52),
|
|
webObserveShort(webObserveText(item.traceId), 28),
|
|
webObserveShort(webObserveText(item.text), 72),
|
|
]);
|
|
const recentUpdateJumpRows = recentUpdateJumps.map((item) => [
|
|
webObserveShort(webObserveText(item.columnLabel), 14),
|
|
pageLabel(item),
|
|
webObserveText(item.promptIndex),
|
|
webObserveText(item.fromSeq),
|
|
webObserveText(item.toSeq),
|
|
`${webObserveText(item.fromValue)}->${webObserveText(item.toValue)}`,
|
|
webObserveText(item.delta),
|
|
webObserveText(item.sampleDeltaSeconds),
|
|
webObserveText(item.allowedIncreaseSeconds),
|
|
webObserveShort(webObserveText(item.traceId), 24),
|
|
]);
|
|
const elapsedZeroResetRows = elapsedZeroResets.map((item) => [
|
|
webObserveShort(webObserveText(item.columnLabel), 14),
|
|
pageLabel(item),
|
|
webObserveText(item.promptIndex),
|
|
webObserveText(item.fromSeq),
|
|
webObserveText(item.toSeq),
|
|
`${webObserveText(item.fromValue)}->${webObserveText(item.toValue)}`,
|
|
webObserveText(item.delta),
|
|
webObserveShort(webObserveText(item.traceId), 24),
|
|
]);
|
|
const elapsedForwardJumpRows = elapsedForwardJumps.map((item) => [
|
|
webObserveShort(webObserveText(item.columnLabel), 14),
|
|
pageLabel(item),
|
|
webObserveText(item.promptIndex),
|
|
webObserveText(item.fromSeq),
|
|
webObserveText(item.toSeq),
|
|
`${webObserveText(item.fromValue)}->${webObserveText(item.toValue)}`,
|
|
webObserveText(item.delta),
|
|
webObserveText(item.sampleDeltaSeconds),
|
|
webObserveText(item.allowedIncreaseSeconds),
|
|
webObserveShort(webObserveText(item.traceId), 24),
|
|
]);
|
|
const httpErrorRows = httpErrorGroups.map((item) => [
|
|
webObserveText(item.count),
|
|
webObserveText(item.method),
|
|
webObserveText(item.status),
|
|
webObserveShort(webObserveText(item.path), 52),
|
|
webObserveShort(webObserveText(item.lastAt ?? item.firstAt), 24),
|
|
webObserveShort(webObserveArray(item.failureKinds).join(",") || "-", 40),
|
|
]);
|
|
const requestFailedRows = requestFailedGroups.map((item) => [
|
|
webObserveText(item.count),
|
|
webObserveText(item.method),
|
|
webObserveText(item.status),
|
|
webObserveShort(webObserveText(item.path), 52),
|
|
webObserveShort(webObserveText(item.lastAt ?? item.firstAt), 24),
|
|
webObserveShort(webObserveArray(item.failureKinds).join(",") || "-", 40),
|
|
]);
|
|
const lines = [
|
|
`hwlab nodes web-probe observe analyze (${webObserveText(value.status)})`,
|
|
"",
|
|
...(value.ok === false ? [
|
|
"Blocked detail:",
|
|
webObserveTable(["EXIT", "TIMEOUT", "ANALYZER", "STDOUT", "STDERR"], [[
|
|
webObserveText(result.exitCode),
|
|
webObserveText(result.timedOut),
|
|
webObserveShort([
|
|
analyzer?.recoveredFrom ? `source=${webObserveText(analyzer.recoveredFrom)}` : "",
|
|
analyzer?.exitCode !== undefined && analyzer?.exitCode !== null ? `exit=${webObserveText(analyzer.exitCode)}` : "",
|
|
analyzer?.stdoutBytes !== undefined && analyzer?.stdoutBytes !== null ? `stdout=${webObserveText(analyzer.stdoutBytes)}B` : "",
|
|
analyzer?.stderrBytes !== undefined && analyzer?.stderrBytes !== null ? `stderr=${webObserveText(analyzer.stderrBytes)}B` : "",
|
|
].filter((part) => part !== "").join(" "), 120),
|
|
webObserveShort(webObserveText(result.stdoutTail), 120),
|
|
webObserveShort(webObserveText(analyzer?.stderrTail ?? result.stderr), 500),
|
|
]]),
|
|
"Analyzer artifacts:",
|
|
webObserveTable(["KIND", "PATH"], [
|
|
["stdout", webObserveShort(webObserveText(analyzer?.stdoutPath), 96)],
|
|
["stderr", webObserveShort(webObserveText(analyzer?.stderrPath), 96)],
|
|
]),
|
|
"",
|
|
] : []),
|
|
webObserveTable(["ID", "NODE", "LANE", "SCOPE", "SAMPLES", "CONTROL", "NETWORK", "CONSOLE", "ARTIFACTS"], [[
|
|
webObserveText(value.id),
|
|
webObserveText(value.node),
|
|
webObserveText(value.lane),
|
|
webObserveShort([webObserveText(jsonlScope?.mode ?? "current"), webObserveText(record(jsonlScope?.focus)?.mode), jsonlScope?.archivePrefix ? webObserveText(jsonlScope.archivePrefix) : ""].filter((part) => part && part !== "-").join(":"), 36),
|
|
webObserveText(counts?.samples),
|
|
webObserveText(counts?.control),
|
|
webObserveText(counts?.network),
|
|
webObserveText(counts?.console),
|
|
webObserveText(counts?.artifacts),
|
|
]]),
|
|
"",
|
|
"Analysis window:",
|
|
webObserveTable(["WINDOW", "FROM", "TO", "SAMPLES", "NETWORK", "CONSOLE", "ARCHIVE_SAMPLES", "ARCHIVE_RED"], [[
|
|
webObserveText(analysisWindow?.name ?? "recent-5m"),
|
|
webObserveShort(webObserveText(analysisWindow?.fromAt), 24),
|
|
webObserveShort(webObserveText(analysisWindow?.toAt), 24),
|
|
webObserveText(analysisWindow?.samples),
|
|
webObserveText(analysisWindow?.network),
|
|
webObserveText(analysisWindow?.console),
|
|
webObserveText(record(archiveSummary?.sampleMetrics)?.sampleCount ?? counts?.samples),
|
|
webObserveText(archiveSummary?.redFindingCount),
|
|
]]),
|
|
"",
|
|
"Reports:",
|
|
webObserveTable(["KIND", "PATH", "SHA256"], [
|
|
["json", webObserveShort(webObserveText(analysis?.reportJsonPath), 96), webObserveShort(webObserveText(analysis?.reportJsonSha256), 24)],
|
|
["md", webObserveShort(webObserveText(analysis?.reportMdPath), 96), webObserveShort(webObserveText(analysis?.reportMdSha256), 24)],
|
|
]),
|
|
"",
|
|
"Turn timing:",
|
|
webObserveTable(["ROUNDS", "TURNS", "ROWS", "NON_MONO", "ELAPSED_DEC", "ZERO_RESET", "ELAPSED_JUMP", "TERMINAL_GROWTH", "RECENT_JUMP", "MAX_RECENT_STEP"], [[
|
|
webObserveText(roundCount),
|
|
webObserveText(turnColumnCount),
|
|
webObserveText(sampleMetrics?.turnTimingRows),
|
|
webObserveText(sampleMetrics?.turnTimingNonMonotonicCount),
|
|
webObserveText(sampleMetrics?.turnTimingTotalElapsedDecreaseCount),
|
|
webObserveText(sampleMetrics?.turnTimingTotalElapsedZeroResetCount ?? elapsedZeroResets.length),
|
|
webObserveText(sampleMetrics?.turnTimingTotalElapsedForwardJumpCount ?? elapsedForwardJumps.length),
|
|
webObserveText(sampleMetrics?.turnTimingTerminalElapsedGrowthCount),
|
|
webObserveText(sampleMetrics?.turnTimingRecentUpdateJumpCount ?? sampleMetrics?.turnTimingRecentUpdateSawtoothJumpCount),
|
|
webObserveText(sampleMetrics?.turnTimingRecentUpdateMaxIncreaseSeconds),
|
|
]]),
|
|
"",
|
|
"Trace row order:",
|
|
webObserveTable(["SAMPLES", "TRACE_ROWS", "ORDER_ANOMALY", "COMPLETION_NOT_LAST"], [[
|
|
webObserveText(traceOrderSummary?.sampleCount),
|
|
webObserveText(traceOrderSummary?.traceRowCount),
|
|
webObserveText(traceOrderSummary?.orderAnomalyCount),
|
|
webObserveText(traceOrderSummary?.completionNotLastCount),
|
|
]]),
|
|
"",
|
|
"Trace row order anomalies:",
|
|
traceOrderAnomalies.length === 0
|
|
? "-"
|
|
: webObserveTable(["SEQ", "PAGE", "TRACE", "ROWS", "REASONS", "TOTAL", "CLOCK", "PREVIEW"], traceOrderAnomalies.map((item) => [
|
|
webObserveText(item.sampleIndex ?? item.seq),
|
|
pageLabel(item),
|
|
webObserveShort(webObserveText(item.traceId), 24),
|
|
`${webObserveText(item.previousRowIndex)}->${webObserveText(item.currentRowIndex)}`,
|
|
webObserveShort(webObserveArray(item.reasons).map(webObserveText).join(",") || "-", 48),
|
|
`${webObserveText(item.previousTotalSeconds)}->${webObserveText(item.currentTotalSeconds)}`,
|
|
`${webObserveText(item.previousClockSeconds)}->${webObserveText(item.currentClockSeconds)}`,
|
|
webObserveShort(`${webObserveText(item.previousPreview)} / ${webObserveText(item.currentPreview)}`, 96),
|
|
])),
|
|
"",
|
|
"Archive turn timing:",
|
|
webObserveTable(["SAMPLES", "ROUNDS", "TURNS", "ROWS", "NON_MONO", "ELAPSED_DEC", "ZERO_RESET", "ELAPSED_JUMP", "TERMINAL_GROWTH", "RECENT_JUMP", "MAX_RECENT_STEP"], [[
|
|
webObserveText(archiveMetrics?.sampleCount),
|
|
webObserveText(archiveMetrics?.rounds),
|
|
webObserveText(archiveMetrics?.turnColumns),
|
|
webObserveText(archiveMetrics?.turnTimingRows),
|
|
webObserveText(archiveMetrics?.turnTimingNonMonotonicCount),
|
|
webObserveText(archiveMetrics?.turnTimingTotalElapsedDecreaseCount),
|
|
webObserveText(archiveMetrics?.turnTimingTotalElapsedZeroResetCount),
|
|
webObserveText(archiveMetrics?.turnTimingTotalElapsedForwardJumpCount),
|
|
webObserveText(archiveMetrics?.turnTimingTerminalElapsedGrowthCount),
|
|
webObserveText(archiveMetrics?.turnTimingRecentUpdateJumpCount ?? archiveMetrics?.turnTimingRecentUpdateSawtoothJumpCount),
|
|
webObserveText(archiveMetrics?.turnTimingRecentUpdateMaxIncreaseSeconds),
|
|
]]),
|
|
"",
|
|
"Rounds:",
|
|
webObserveTable(["ROUND", "SAMPLES", "LOADING", "MAX_LOAD", "TOTAL_LAST", "RECENT_LAST", "CARD_DIAG", "TERMINAL", "PROMPT_HASH"], rounds.length > 0 ? rounds.map((item) => [
|
|
webObserveText(item.promptIndex),
|
|
webObserveText(item.sampleCount),
|
|
webObserveText(item.loadingSamples),
|
|
webObserveText(item.maxLoadingCount),
|
|
webObserveText(item.lastTotalElapsedSeconds),
|
|
webObserveText(item.lastRecentUpdateSeconds),
|
|
webObserveText(item.diagnosticSamples),
|
|
webObserveText(item.terminalSamples),
|
|
webObserveShort(webObserveText(item.promptTextHash), 24),
|
|
]) : [["-", "-", "-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Loading visibility:",
|
|
webObserveTable(["SAMPLES", "LOADING_SAMPLES", "MAX_COUNT", "MAX_OWNERS", "OWNERS", "LONGEST_S", "CURRENT_S", `OVER_${loadingBudgetLabel}`], [[
|
|
webObserveText(loadingSummary?.sampleCount ?? sampleMetrics?.sampleCount),
|
|
webObserveText(loadingSummary?.loadingSampleCount ?? sampleMetrics?.loadingSampleCount),
|
|
webObserveText(loadingSummary?.maxSimultaneousCount ?? sampleMetrics?.loadingMaxCount),
|
|
webObserveText(loadingSummary?.maxSimultaneousOwnerCount ?? sampleMetrics?.loadingMaxOwnerCount),
|
|
webObserveText(loadingSummary?.ownerCount ?? sampleMetrics?.loadingOwnerCount),
|
|
webObserveText(loadingSummary?.longestContinuousSeconds ?? sampleMetrics?.loadingLongestContinuousSeconds),
|
|
webObserveText(loadingSummary?.currentContinuousSeconds ?? sampleMetrics?.loadingCurrentContinuousSeconds),
|
|
webObserveText(loadingSummary?.overBudgetSegmentCount ?? loadingSummary?.overFiveSecondSegmentCount ?? sampleMetrics?.loadingOverFiveSecondSegmentCount),
|
|
]]),
|
|
"",
|
|
"Session rail titles:",
|
|
webObserveTable(["SAMPLES", "VISIBLE", "MAJORITY", "MAX_RATIO", "MAX_VISIBLE", "MAX_FALLBACK"], [[
|
|
webObserveText(sessionRailTitleSummary?.sampleCount ?? sampleMetrics?.sessionRailSampleCount ?? archiveMetrics?.sessionRailSampleCount),
|
|
webObserveText(sessionRailTitleSummary?.visibleSampleCount ?? sampleMetrics?.sessionRailVisibleSampleCount ?? archiveMetrics?.sessionRailVisibleSampleCount),
|
|
webObserveText(sessionRailTitleSummary?.majorityFallbackSampleCount ?? sampleMetrics?.sessionRailFallbackMajoritySampleCount ?? archiveMetrics?.sessionRailFallbackMajoritySampleCount),
|
|
webObserveText(sessionRailTitleSummary?.maxFallbackRatio ?? sampleMetrics?.sessionRailFallbackMaxRatio ?? archiveMetrics?.sessionRailFallbackMaxRatio),
|
|
webObserveText(sessionRailTitleSummary?.maxVisibleCount ?? sampleMetrics?.sessionRailFallbackMaxVisibleCount ?? archiveMetrics?.sessionRailFallbackMaxVisibleCount),
|
|
webObserveText(sessionRailTitleSummary?.maxFallbackTitleCount ?? sampleMetrics?.sessionRailFallbackMaxCount ?? archiveMetrics?.sessionRailFallbackMaxCount),
|
|
]]),
|
|
sessionRailTitleSamples.length === 0 ? "Session rail fallback samples:\n-" : webObserveTable(["SEQ", "ROLE", "VISIBLE", "FALLBACK", "RATIO", "EXAMPLE"], sessionRailTitleSamples.map((item) => [
|
|
webObserveText(item.seq),
|
|
webObserveText(item.pageRole),
|
|
webObserveText(item.visibleCount),
|
|
webObserveText(item.fallbackTitleCount),
|
|
webObserveText(item.fallbackTitleRatio),
|
|
webObserveShort(webObserveText((Array.isArray(item.examples) ? record(item.examples[0])?.titlePreview : null) ?? ""), 48),
|
|
])),
|
|
sessionRailTitleExamples.length === 0 ? "Session rail fallback examples:\n-" : webObserveTable(["SEQ", "ROLE", "ACTIVE", "SESSION", "TITLE"], sessionRailTitleExamples.map((item) => [
|
|
webObserveText(item.firstSeq),
|
|
webObserveText(item.pageRole),
|
|
webObserveText(item.active),
|
|
webObserveText(item.sessionIdPrefix),
|
|
webObserveShort(webObserveText(item.titlePreview ?? item.titleHash), 64),
|
|
])),
|
|
"",
|
|
"Loading owners:",
|
|
webObserveTable(["SAMPLES", "OCCUR", "MAX", "LONGEST_S", "KIND", "TRACE", "MESSAGE", "SESSION", "OWNER"], loadingOwners.length > 0 ? loadingOwners.map((item) => [
|
|
webObserveText(item.sampleCount),
|
|
webObserveText(item.occurrenceCount),
|
|
webObserveText(item.maxSimultaneousCount),
|
|
webObserveText(item.longestContinuousSeconds),
|
|
webObserveShort(webObserveText(item.ownerKind), 16),
|
|
webObserveShort(webObserveText(item.ownerTraceId), 24),
|
|
webObserveShort(webObserveText(item.ownerMessageId), 24),
|
|
webObserveShort(webObserveText(item.ownerSessionId), 24),
|
|
webObserveShort(webObserveText(item.ownerLabel ?? item.ownerKey), 64),
|
|
]) : [["-", "-", "-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Loading segments:",
|
|
webObserveTable(["OBSERVED_S", "UPPER_S", "SAMPLES", "MAX", "OWNERS", "SEQ", "LAST", "STATUS"], loadingSegments.length > 0 ? loadingSegments.map((item) => [
|
|
webObserveText(item.durationSeconds),
|
|
webObserveText(item.upperBoundSeconds ?? item.durationSeconds),
|
|
webObserveText(item.sampleCount),
|
|
webObserveText(item.maxCount),
|
|
webObserveText(item.ownerCount),
|
|
`${webObserveText(item.firstSeq)}..${webObserveText(item.lastSeq)}`,
|
|
webObserveShort(webObserveText(item.lastAt), 24),
|
|
item.ongoing === true ? "ongoing" : webObserveShort(webObserveText(item.endedAt), 24),
|
|
]) : [["-", "-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Turn columns:",
|
|
webObserveTable(["TURN", "PAGE", "PROMPT", "TRACE", "FIRST_SEQ", "LAST_SEQ", "SOURCE"], turnColumns.length > 0 ? turnColumns.map((item) => [
|
|
webObserveText(item.label),
|
|
pageLabel(item),
|
|
webObserveText(item.promptIndex ?? item.lastPromptIndex),
|
|
webObserveShort(webObserveText(item.traceId), 28),
|
|
webObserveText(item.firstSeq),
|
|
webObserveText(item.lastSeq),
|
|
webObserveShort(webObserveText(item.source), 24),
|
|
]) : [["-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Recent update jumps:",
|
|
webObserveTable(["TURN", "PAGE", "PROMPT", "FROM_SEQ", "TO_SEQ", "VALUE", "DELTA", "SAMPLE_S", "ALLOWED", "TRACE"], recentUpdateJumpRows.length > 0 ? recentUpdateJumpRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Total elapsed zero resets:",
|
|
webObserveTable(["TURN", "PAGE", "PROMPT", "FROM_SEQ", "TO_SEQ", "VALUE", "DELTA", "TRACE"], elapsedZeroResetRows.length > 0 ? elapsedZeroResetRows : [["-", "-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Total elapsed forward jumps:",
|
|
webObserveTable(["TURN", "PAGE", "PROMPT", "FROM_SEQ", "TO_SEQ", "VALUE", "DELTA", "SAMPLE_S", "ALLOWED", "TRACE"], elapsedForwardJumpRows.length > 0 ? elapsedForwardJumpRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Runtime alerts:",
|
|
webObserveTable(["HTTP_ERR", "REQUEST_FAILED", "DOM_DIAG", "CONSOLE_ALERT", "PROMPT_SEGMENTS", "PROMPT_NETWORK"], [[
|
|
webObserveText(runtimeAlerts?.httpErrorCount),
|
|
webObserveText(runtimeAlerts?.requestFailedCount),
|
|
webObserveText(runtimeAlerts?.domDiagnosticSampleCount),
|
|
webObserveText(runtimeAlerts?.consoleAlertCount),
|
|
webObserveText(sampleMetrics?.promptSegments),
|
|
webObserveText(promptNetwork?.promptSegments),
|
|
]]),
|
|
"",
|
|
"Runner errors:",
|
|
webObserveTable(["TS", "TYPE", "RETRY", "EXH", "ATTEMPTS", "LAST_KIND", "READY", "MESSAGE"], runnerErrors.length > 0 ? runnerErrors.map((item) => [
|
|
webObserveShort(webObserveText(item.ts), 24),
|
|
webObserveShort(webObserveText(item.type), 18),
|
|
webObserveText(item.retry),
|
|
webObserveText(item.retryExhausted),
|
|
webObserveText(item.attemptCount),
|
|
webObserveShort(webObserveText(item.lastFailureKind), 24),
|
|
webObserveShort(webObserveText(item.lastReadinessReason), 24),
|
|
webObserveShort(webObserveText(item.message || item.lastError), 96),
|
|
]) : [["-", "-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Command failures:",
|
|
webObserveTable(["TS", "TYPE", "COMMAND", "DURATION", "SAMPLE", "PATH", "MESSAGE"], commandFailures.length > 0 ? commandFailures.map((item) => [
|
|
webObserveShort(webObserveText(item.ts), 24),
|
|
webObserveShort(webObserveText(item.type), 18),
|
|
webObserveShort(webObserveText(item.commandId), 28),
|
|
webObserveText(item.durationMs),
|
|
webObserveText(item.failureSampleOk === true ? item.sampleSeq : "-"),
|
|
webObserveShort([webObserveText(item.beforePath), webObserveText(item.afterPath)].filter((part) => part !== "-" && part !== "").join("->") || "-", 52),
|
|
webObserveShort(webObserveText(item.message || item.failureKind || item.name), 96),
|
|
]) : [["-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"HTTP error groups:",
|
|
webObserveTable(["COUNT", "METHOD", "STATUS", "PATH", "LAST", "FAILURE"], httpErrorRows.length > 0 ? httpErrorRows : [["-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Request failed groups:",
|
|
webObserveTable(["COUNT", "METHOD", "STATUS", "PATH", "LAST", "FAILURE"], requestFailedRows.length > 0 ? requestFailedRows : [["-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
`Slow API (>${sameOriginApiBudgetLabel}):`,
|
|
webObserveTable(["PATH", "SAMPLES", "P95", "MAX", "OVER_BUDGET"], slowApis.length > 0 ? slowApis.map((item) => [
|
|
webObserveShort(webObserveText(item.path ?? item.route), 52),
|
|
webObserveText(item.sampleCount),
|
|
webObserveText(item.p95Ms ?? item.p95),
|
|
webObserveText(item.maxMs ?? item.max),
|
|
webObserveText(item.overBudgetCount ?? item.overFiveSecondCount),
|
|
]) : [["-", "-", "-", "-", "-"]]),
|
|
"",
|
|
`Slow API samples (>${sameOriginApiBudgetLabel}):`,
|
|
webObserveTable(["TS", "SEQ", "PATH", "DURATION", "REQ_WAIT", "RESP_XFER", "TIMING", "INIT", "PROTO", "SERVER", "OTEL"], (() => {
|
|
const rows = slowApis.flatMap((item) => (Array.isArray(item.slowSamples) ? item.slowSamples : []).map((sample) => ({ ...sample, path: sample.path ?? item.path ?? item.route }))).slice(0, 8).map((sample) => [
|
|
webObserveShort(webObserveText(sample.ts), 24),
|
|
webObserveText(sample.seq),
|
|
webObserveShort(webObserveText(sample.path), 44),
|
|
webObserveText(sample.durationMs),
|
|
webObserveText(sample.requestToResponseStartMs ?? sample.streamOpenMs),
|
|
webObserveText(sample.responseTransferMs),
|
|
webObserveShort(webObserveText(sample.timingStatus), 12),
|
|
webObserveShort(webObserveText(sample.initiatorType), 12),
|
|
webObserveShort(webObserveText(sample.nextHopProtocol), 12),
|
|
webObserveShort(Array.isArray(sample.serverTimingNames) ? sample.serverTimingNames.join(",") : "-", 24),
|
|
webObserveShort(webObserveText(sample.otelTraceId), 16),
|
|
]);
|
|
return rows.length > 0 ? rows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]];
|
|
})()),
|
|
"",
|
|
`Partial API timing (>${partialApiBudgetLabel}, not counted as completed slow API):`,
|
|
webObserveTable(["PATH", "SAMPLES", "COMPLETE", "PARTIAL", "PARTIAL_OVER_BUDGET"], partialApis.length > 0 ? partialApis.map((item) => [
|
|
webObserveShort(webObserveText(item.path ?? item.route), 52),
|
|
webObserveText(item.sampleCount),
|
|
webObserveText(item.completeTimingSampleCount),
|
|
webObserveText(item.partialTimingSampleCount),
|
|
webObserveText(item.partialOverBudgetCount ?? item.partialOverFiveSecondCount),
|
|
]) : [["-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Partial API samples:",
|
|
webObserveTable(["TS", "SEQ", "PATH", "DURATION", "TIMING", "INIT", "PROTO"], (() => {
|
|
const rows = partialApis.flatMap((item) => (Array.isArray(item.partialSamples) ? item.partialSamples : []).map((sample) => ({ ...sample, path: sample.path ?? item.path ?? item.route }))).slice(0, 8).map((sample) => [
|
|
webObserveShort(webObserveText(sample.ts), 24),
|
|
webObserveText(sample.seq),
|
|
webObserveShort(webObserveText(sample.path), 44),
|
|
webObserveText(sample.durationMs),
|
|
webObserveShort(webObserveText(sample.timingStatus), 12),
|
|
webObserveShort(webObserveText(sample.initiatorType), 12),
|
|
webObserveShort(webObserveText(sample.nextHopProtocol), 12),
|
|
]);
|
|
return rows.length > 0 ? rows : [["-", "-", "-", "-", "-", "-", "-"]];
|
|
})()),
|
|
"",
|
|
"Long-lived streams (SSE):",
|
|
webObserveTable(["PATH", "SAMPLES", "OPEN_P95", "OPEN_MAX", `OPEN>${streamOpenBudgetLabel}`, "LIFE>5S"], sseStreams.length > 0 ? sseStreams.map((item) => [
|
|
webObserveShort(webObserveText(item.path ?? item.route), 52),
|
|
webObserveText(item.sampleCount),
|
|
webObserveText(item.streamOpenP95Ms),
|
|
webObserveText(item.streamOpenMaxMs),
|
|
webObserveText(item.streamOpenOverBudgetCount ?? item.streamOpenOverFiveSecondCount),
|
|
webObserveText(item.streamLifetimeOverFiveSecondCount),
|
|
]) : [["-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Archive page performance:",
|
|
webObserveTable(["SAME_ORIGIN_API", "SLOW_PATHS", "SLOW_SAMPLES", "WORST_P95"], [[
|
|
webObserveText(archivePagePerformance?.sameOriginApiPathCount ?? archivePagePerformance?.sameOriginApiPaths),
|
|
webObserveText(archivePagePerformance?.slowPathCount),
|
|
webObserveText(archivePagePerformance?.slowSampleCount),
|
|
webObserveText(archivePagePerformance?.worstP95Ms),
|
|
]]),
|
|
"",
|
|
`Archive slow API (>${sameOriginApiBudgetLabel}):`,
|
|
webObserveTable(["PATH", "SAMPLES", "P95", "MAX", "OVER_BUDGET"], archiveSlowApis.length > 0 ? archiveSlowApis.map((item) => [
|
|
webObserveShort(webObserveText(item.path ?? item.route), 52),
|
|
webObserveText(item.sampleCount),
|
|
webObserveText(item.p95Ms ?? item.p95),
|
|
webObserveText(item.maxMs ?? item.max),
|
|
webObserveText(item.overBudgetCount ?? item.overFiveSecondCount),
|
|
]) : [["-", "-", "-", "-", "-"]]),
|
|
"",
|
|
`Archive slow API samples (>${sameOriginApiBudgetLabel}):`,
|
|
webObserveTable(["TS", "SEQ", "PATH", "DURATION", "REQ_WAIT", "TIMING", "OTEL"], (() => {
|
|
const rows = archiveSlowApis.flatMap((item) => (Array.isArray(item.slowSamples) ? item.slowSamples : []).map((sample) => ({ ...sample, path: sample.path ?? item.path ?? item.route }))).slice(0, 8).map((sample) => [
|
|
webObserveShort(webObserveText(sample.ts), 24),
|
|
webObserveText(sample.seq),
|
|
webObserveShort(webObserveText(sample.path), 44),
|
|
webObserveText(sample.durationMs),
|
|
webObserveText(sample.requestToResponseStartMs ?? sample.streamOpenMs),
|
|
webObserveShort(webObserveText(sample.timingStatus), 12),
|
|
webObserveShort(webObserveText(sample.otelTraceId), 16),
|
|
]);
|
|
return rows.length > 0 ? rows : [["-", "-", "-", "-", "-", "-", "-"]];
|
|
})()),
|
|
"",
|
|
"DOM diagnostics:",
|
|
webObserveTable(domDiagnostics.length > 0 ? ["COUNT", "FIRST", "LAST", "TEXT"] : ["SEQ", "TS", "META", "TEXT"], domDiagnosticRows.length > 0 ? domDiagnosticRows : [["-", "-", "-", "-"]]),
|
|
"",
|
|
"Console alerts:",
|
|
webObserveTable(consoleAlertGroups.length > 0 ? ["COUNT", "TYPE", "STATUS", "PATH", "LAST", "TRACES"] : ["TS", "TYPE", "STATUS", "PATH", "TRACE", "TEXT"], consoleAlertRows.length > 0 ? consoleAlertRows : [["-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Findings:",
|
|
webObserveTable(["KIND", "SEVERITY", "COUNT", "SUMMARY"], findings.length > 0 ? findings.map((item) => [
|
|
webObserveShort(webObserveText(item.kind ?? item.code), 32),
|
|
webObserveText(item.severity ?? item.level),
|
|
webObserveText(item.count ?? item.sampleCount),
|
|
webObserveShort(webObserveText(item.summary ?? item.message), 96),
|
|
]) : [["-", "-", "-", "-"]]),
|
|
"",
|
|
"Archive red findings:",
|
|
webObserveTable(["KIND", "COUNT", "SUMMARY"], archiveRedFindings.length > 0 ? archiveRedFindings.map((item) => [
|
|
webObserveShort(webObserveText(item.kind ?? item.code), 36),
|
|
webObserveText(item.count ?? item.sampleCount),
|
|
webObserveShort(webObserveText(item.summary ?? item.message), 110),
|
|
]) : [["-", "-", "-"]]),
|
|
];
|
|
if (pagePerformance !== null && Object.keys(pagePerformance).length > 0) {
|
|
lines.push("", "Page performance:", ` ${webObserveShort(JSON.stringify(pagePerformance), 180)}`);
|
|
}
|
|
lines.push(
|
|
"",
|
|
"Next:",
|
|
` cat ${webObserveText(analysis?.reportMdPath)}`,
|
|
` cat ${webObserveText(analysis?.reportJsonPath)}`,
|
|
"",
|
|
"Disclosure:",
|
|
" default view is a bounded report summary; use report.md/report.json paths for full offline analysis.",
|
|
);
|
|
return lines.join("\n");
|
|
}
|
|
|
|
function nodeWebObserveResolveStateDirShell(options: Pick<NodeWebProbeObserveOptions, "stateDir" | "jobId" | "node" | "lane">): string {
|
|
if (options.stateDir !== null) {
|
|
return [
|
|
`state_dir=${shellQuote(options.stateDir)}`,
|
|
"test -d \"$state_dir\" || { printf '{\"ok\":false,\"error\":\"state-dir-missing\",\"stateDir\":\"%s\"}\\n' \"$state_dir\"; exit 2; }",
|
|
].join("\n");
|
|
}
|
|
if (options.jobId === null) throw new Error("web-probe observe requires --state-dir or --job-id");
|
|
const root = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}`;
|
|
return [
|
|
`observe_root=${shellQuote(root)}`,
|
|
`job_id=${shellQuote(options.jobId)}`,
|
|
"state_dir=$(find \"$observe_root\" -type d -name \"*_${job_id}\" 2>/dev/null | sort | tail -n 1 || true)",
|
|
"test -n \"$state_dir\" || { printf '{\"ok\":false,\"error\":\"job-id-not-found\",\"jobId\":\"%s\",\"root\":\"%s\"}\\n' \"$job_id\" \"$observe_root\"; exit 2; }",
|
|
].join("\n");
|
|
}
|
|
|
|
function webObserveIndexPath(): string {
|
|
return rootPath(".state/web-observe/index.json");
|
|
}
|
|
|
|
function readWebObserveIndex(): Record<string, WebObserveIndexEntry> {
|
|
const path = webObserveIndexPath();
|
|
if (!existsSync(path)) return {};
|
|
const parsed = JSON.parse(readFileSync(path, "utf8")) as unknown;
|
|
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error(`web-probe observe index is invalid: ${path}`);
|
|
const entries: Record<string, WebObserveIndexEntry> = {};
|
|
for (const [id, value] of Object.entries(parsed)) {
|
|
if (!isSafeWebObserveJobId(id)) continue;
|
|
if (typeof value !== "object" || value === null || Array.isArray(value)) continue;
|
|
const recordValue = value as Record<string, unknown>;
|
|
const stateDir = typeof recordValue.stateDir === "string" ? recordValue.stateDir : "";
|
|
const node = typeof recordValue.node === "string" ? recordValue.node : "";
|
|
const lane = typeof recordValue.lane === "string" ? recordValue.lane : "";
|
|
const workspace = typeof recordValue.workspace === "string" ? recordValue.workspace : "";
|
|
if (!isSafeWebObserveStateDir(stateDir) || node.length === 0 || lane.length === 0 || workspace.length === 0) continue;
|
|
entries[id] = {
|
|
id,
|
|
node,
|
|
lane,
|
|
workspace,
|
|
stateDir,
|
|
url: typeof recordValue.url === "string" ? recordValue.url : "",
|
|
targetPath: typeof recordValue.targetPath === "string" ? recordValue.targetPath : "",
|
|
status: typeof recordValue.status === "string" ? recordValue.status : null,
|
|
pid: typeof recordValue.pid === "number" ? recordValue.pid : null,
|
|
startedAt: typeof recordValue.startedAt === "string" ? recordValue.startedAt : null,
|
|
updatedAt: typeof recordValue.updatedAt === "string" ? recordValue.updatedAt : "",
|
|
};
|
|
}
|
|
return entries;
|
|
}
|
|
|
|
function readWebObserveIndexEntry(id: string): WebObserveIndexEntry | null {
|
|
return readWebObserveIndex()[id] ?? null;
|
|
}
|
|
|
|
function discoverWebObserveIndexEntry(id: string): WebObserveIndexEntry | null {
|
|
const matches: WebObserveIndexEntry[] = [];
|
|
const errors: string[] = [];
|
|
for (const lane of hwlabRuntimeLaneIds()) {
|
|
for (const node of hwlabRuntimeNodeIds()) {
|
|
let spec: HwlabRuntimeLaneSpec;
|
|
try {
|
|
spec = hwlabRuntimeLaneSpecForNode(lane, node);
|
|
} catch {
|
|
continue;
|
|
}
|
|
try {
|
|
const entry = discoverWebObserveIndexEntryOnTarget(id, node, lane, spec);
|
|
if (entry !== null) matches.push(entry);
|
|
} catch (error) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
errors.push(`${node}/${lane}: ${message.slice(0, 240)}`);
|
|
}
|
|
}
|
|
}
|
|
if (matches.length > 1) {
|
|
const candidates = matches.map((entry) => `${entry.node}/${entry.lane}:${entry.stateDir}`).join(", ");
|
|
throw new Error(`web-probe observe observer-id-ambiguous: ${id}; candidates=${candidates}`);
|
|
}
|
|
if (matches.length === 0) {
|
|
if (errors.length > 0) {
|
|
throw new Error(`web-probe observe observer-id-unknown: ${id}; discoveryErrors=${errors.join(" | ")}`);
|
|
}
|
|
return null;
|
|
}
|
|
upsertWebObserveIndexEntry(matches[0]!);
|
|
return matches[0]!;
|
|
}
|
|
|
|
function discoverWebObserveIndexEntryOnTarget(id: string, node: string, lane: HwlabRuntimeLane, spec: HwlabRuntimeLaneSpec): WebObserveIndexEntry | null {
|
|
const observeRoot = `.state/web-observe/${safeWebObserveSegment(node)}/${safeWebObserveSegment(lane)}`;
|
|
const script = [
|
|
"set -eu",
|
|
`observe_root=${shellQuote(observeRoot)}`,
|
|
`job_id=${shellQuote(id)}`,
|
|
"node - \"$observe_root\" \"$job_id\" <<'NODE'",
|
|
"const fs = require('fs');",
|
|
"const path = require('path');",
|
|
"const [root, jobId] = process.argv.slice(2);",
|
|
"const matches = [];",
|
|
"function readJson(file) { try { return JSON.parse(fs.readFileSync(file, 'utf8')); } catch { return null; } }",
|
|
"function walk(dir, depth) {",
|
|
" if (depth > 10 || matches.length > 1) return;",
|
|
" let entries = [];",
|
|
" try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }",
|
|
" for (const entry of entries) {",
|
|
" if (!entry.isDirectory()) continue;",
|
|
" const full = path.join(dir, entry.name);",
|
|
" if (entry.name.endsWith('_' + jobId)) matches.push(full);",
|
|
" else walk(full, depth + 1);",
|
|
" if (matches.length > 1) return;",
|
|
" }",
|
|
"}",
|
|
"walk(root, 0);",
|
|
"if (matches.length === 0) { console.log(JSON.stringify({ ok: true, found: false, root, jobId })); process.exit(0); }",
|
|
"if (matches.length > 1) { console.log(JSON.stringify({ ok: false, found: true, ambiguous: true, root, jobId, matches })); process.exit(3); }",
|
|
"const stateDir = matches[0];",
|
|
"const manifest = readJson(path.join(stateDir, 'manifest.json')) || {};",
|
|
"const heartbeat = readJson(path.join(stateDir, 'heartbeat.json')) || {};",
|
|
"let pid = null;",
|
|
"try { const raw = fs.readFileSync(path.join(stateDir, 'pid'), 'utf8').trim(); const value = Number(raw); if (Number.isFinite(value)) pid = value; } catch {}",
|
|
"console.log(JSON.stringify({",
|
|
" ok: true,",
|
|
" found: true,",
|
|
" ambiguous: false,",
|
|
" entry: {",
|
|
" id: jobId,",
|
|
" stateDir,",
|
|
" url: typeof manifest.baseUrl === 'string' ? manifest.baseUrl : '',",
|
|
" targetPath: typeof manifest.targetPath === 'string' ? manifest.targetPath : '',",
|
|
" status: typeof manifest.status === 'string' ? manifest.status : null,",
|
|
" pid,",
|
|
" startedAt: typeof manifest.startedAt === 'string' ? manifest.startedAt : null,",
|
|
" updatedAt: typeof heartbeat.updatedAt === 'string' ? heartbeat.updatedAt : new Date().toISOString()",
|
|
" }",
|
|
"}));",
|
|
"NODE",
|
|
].join("\n");
|
|
const result = runTransWorkspaceStdinScript(node, spec.workspace, script, 55);
|
|
const payload = parseJsonObject(result.stdout);
|
|
if (result.exitCode !== 0 || result.timedOut || payload?.ok !== true) {
|
|
const reason = result.timedOut ? "timeout" : payload?.ambiguous === true ? "ambiguous" : `exit=${result.exitCode}`;
|
|
throw new Error(`observer discovery failed (${reason}): ${result.stderr.trim().slice(-240) || result.stdout.trim().slice(-240)}`);
|
|
}
|
|
if (payload.found !== true) return null;
|
|
const entry = record(payload.entry);
|
|
const stateDir = typeof entry.stateDir === "string" ? entry.stateDir : "";
|
|
if (!isSafeWebObserveStateDir(stateDir)) throw new Error(`observer discovery returned unsafe stateDir: ${stateDir}`);
|
|
return {
|
|
id,
|
|
node,
|
|
lane,
|
|
workspace: spec.workspace,
|
|
stateDir,
|
|
url: typeof entry.url === "string" ? entry.url : "",
|
|
targetPath: typeof entry.targetPath === "string" ? entry.targetPath : "",
|
|
status: typeof entry.status === "string" ? entry.status : null,
|
|
pid: typeof entry.pid === "number" ? entry.pid : null,
|
|
startedAt: typeof entry.startedAt === "string" ? entry.startedAt : null,
|
|
updatedAt: typeof entry.updatedAt === "string" ? entry.updatedAt : new Date().toISOString(),
|
|
};
|
|
}
|
|
|
|
function upsertWebObserveIndexEntry(entry: WebObserveIndexEntry): Record<string, unknown> {
|
|
const path = webObserveIndexPath();
|
|
try {
|
|
const current = readWebObserveIndex();
|
|
mkdirSync(dirname(path), { recursive: true });
|
|
const next = { ...current, [entry.id]: entry };
|
|
const tmp = `${path}.tmp-${process.pid}`;
|
|
writeFileSync(tmp, `${JSON.stringify(next, null, 2)}\n`, { mode: 0o600 });
|
|
renameSync(tmp, path);
|
|
return {
|
|
ok: true,
|
|
id: entry.id,
|
|
path,
|
|
statusCommand: `bun scripts/cli.ts hwlab nodes web-probe observe status ${entry.id}`,
|
|
valuesRedacted: true,
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
id: entry.id,
|
|
path,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
fallback: `bun scripts/cli.ts hwlab nodes web-probe observe status --node ${entry.node} --lane ${entry.lane} --state-dir ${entry.stateDir}`,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
}
|
|
|
|
function webObserveIdFromOptions(options: Pick<NodeWebProbeObserveOptions, "id" | "jobId">): string | null {
|
|
return options.id ?? options.jobId;
|
|
}
|
|
|
|
function webObserveIdFromStatus(status: Record<string, unknown> | null, options: Pick<NodeWebProbeObserveOptions, "id" | "jobId">): string | null {
|
|
const manifest = record(status?.manifest);
|
|
const heartbeat = record(status?.heartbeat);
|
|
const manifestJobId = typeof manifest.jobId === "string" ? manifest.jobId : null;
|
|
const heartbeatJobId = typeof heartbeat.jobId === "string" ? heartbeat.jobId : null;
|
|
return manifestJobId ?? heartbeatJobId ?? webObserveIdFromOptions(options);
|
|
}
|
|
|
|
function webObserveIndexEntryFromOptions(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec, id: string, status: Record<string, unknown>): WebObserveIndexEntry {
|
|
const manifest = record(status.manifest);
|
|
const heartbeat = record(status.heartbeat);
|
|
return {
|
|
id,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
workspace: spec.workspace,
|
|
stateDir: options.stateDir ?? "",
|
|
url: typeof manifest.baseUrl === "string" ? manifest.baseUrl : options.url,
|
|
targetPath: typeof manifest.targetPath === "string" ? manifest.targetPath : options.targetPath,
|
|
status: typeof manifest.status === "string" ? manifest.status : null,
|
|
pid: typeof status.pid === "number" ? status.pid : null,
|
|
startedAt: typeof manifest.startedAt === "string" ? manifest.startedAt : null,
|
|
updatedAt: typeof heartbeat.updatedAt === "string" ? heartbeat.updatedAt : new Date().toISOString(),
|
|
};
|
|
}
|
|
|
|
function webObserveCommandLabel(action: NodeWebProbeObserveAction, options: Pick<NodeWebProbeObserveOptions, "id" | "jobId" | "node" | "lane">): string {
|
|
const id = webObserveIdFromOptions(options);
|
|
return id === null
|
|
? `hwlab nodes web-probe observe ${action} --node ${options.node} --lane ${options.lane}`
|
|
: `hwlab nodes web-probe observe ${action} ${id}`;
|
|
}
|
|
|
|
function webObserveNextCommands(id: string): Record<string, string> {
|
|
return {
|
|
status: `bun scripts/cli.ts hwlab nodes web-probe observe status ${id}`,
|
|
command: `bun scripts/cli.ts hwlab nodes web-probe observe command ${id} --type mark --label checkpoint`,
|
|
stop: `bun scripts/cli.ts hwlab nodes web-probe observe stop ${id}`,
|
|
analyze: `bun scripts/cli.ts hwlab nodes web-probe observe analyze ${id}`,
|
|
};
|
|
}
|
|
|
|
function webObserveCell(value: unknown, maxLength = 96): string {
|
|
if (value === null || value === undefined) return "-";
|
|
const text = typeof value === "string" ? value : String(value);
|
|
const compact = text.replace(/\s+/gu, " ").trim();
|
|
if (compact.length === 0) return "-";
|
|
if (compact.length <= maxLength) return compact;
|
|
return `${compact.slice(0, Math.max(1, maxLength - 1))}...`;
|
|
}
|
|
|
|
function webObserveTable(headers: string[], rows: unknown[][]): string {
|
|
const stringRows = rows.map((row) => row.map((value) => webObserveCell(value)));
|
|
const widths = headers.map((header, index) => Math.max(header.length, ...stringRows.map((row) => row[index]?.length ?? 0)));
|
|
const renderRow = (row: string[]) => row.map((cell, index) => cell.padEnd(widths[index] ?? cell.length)).join(" ").trimEnd();
|
|
return [renderRow(headers), ...stringRows.map(renderRow)].join("\n");
|
|
}
|
|
|
|
function webObserveArray(value: unknown): unknown[] {
|
|
return Array.isArray(value) ? value : [];
|
|
}
|
|
|
|
function withWebObserveRendered(result: Record<string, unknown>, renderedText: string): Record<string, unknown> {
|
|
return {
|
|
...result,
|
|
renderedText,
|
|
contentType: "text/plain",
|
|
};
|
|
}
|
|
|
|
function renderWebProbeRunResult(result: Record<string, unknown>): Record<string, unknown> {
|
|
const summary = record(result.summary);
|
|
const probe = record(result.probe);
|
|
const trace = record(probe.trace);
|
|
const session = record(probe.session);
|
|
const artifacts = record(probe.artifacts);
|
|
const reportLoad = record(result.reportLoad);
|
|
const commandTimeout = record(result.commandTimeout);
|
|
const job = record(result.job);
|
|
const start = record(result.start);
|
|
const commandResult = record(result.result ?? result.statusResult);
|
|
const reportPath = reportLoad.path ?? summary.reportPath ?? artifacts.reportPath ?? start.reportPath ?? "-";
|
|
const reportSource = reportLoad.source ?? (result.mode === "async" ? "-" : "stdout");
|
|
const blockedRows = result.ok === true ? [] : [
|
|
"",
|
|
webObserveTable(
|
|
["BLOCKED_FIELD", "VALUE"],
|
|
[
|
|
["failureKind", result.failureKind ?? summary.failureKind ?? "-"],
|
|
["failedCondition", summary.failedCondition ?? result.degradedReason ?? "-"],
|
|
["nextAction", summary.nextAction ?? "-"],
|
|
["exitCode", commandResult.exitCode ?? "-"],
|
|
["timedOut", commandResult.timedOut ?? commandTimeout.timedOut ?? "-"],
|
|
],
|
|
),
|
|
];
|
|
const renderedText = [
|
|
webObserveTable(
|
|
["WEB_PROBE_RUN", "NODE", "LANE", "STATUS", "OK", "DEGRADED", "URL"],
|
|
[[result.command ?? "web-probe run", result.node, result.lane, result.status, result.ok, result.degradedReason ?? "-", result.url]],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["FIELD", "VALUE"],
|
|
[
|
|
["mode", result.mode ?? (result.reportLoad ? "async" : "direct")],
|
|
["traceId", summary.traceId ?? trace.traceId ?? "-"],
|
|
["sessionId", summary.sessionId ?? session.sessionId ?? "-"],
|
|
["conversationId", summary.conversationId ?? session.conversationId ?? "-"],
|
|
["agentStatus", summary.agentStatus ?? trace.finalAgentStatus ?? "-"],
|
|
["traceStatus", summary.traceStatus ?? trace.finalTraceStatus ?? "-"],
|
|
["messageCount", summary.messageCount ?? session.messageCount ?? "-"],
|
|
["reportSource", reportSource],
|
|
["reportPath", reportPath],
|
|
["reportSha256", summary.reportSha256 ?? artifacts.reportSha256 ?? "-"],
|
|
["timeout", `${commandTimeout.seconds ?? "-"}s timedOut=${commandTimeout.timedOut ?? "-"}`],
|
|
["job", job.jobId ?? "-"],
|
|
],
|
|
),
|
|
...blockedRows,
|
|
"",
|
|
"NEXT",
|
|
` report: ${reportPath}`,
|
|
` rerun: ${result.command ?? `hwlab nodes web-probe run --node ${result.node ?? "-"} --lane ${result.lane ?? "-"}`}`,
|
|
].join("\n");
|
|
return withWebObserveRendered(result, renderedText);
|
|
}
|
|
|
|
function renderWebObserveStartResult(result: Record<string, unknown>): Record<string, unknown> {
|
|
const observer = record(result.observer);
|
|
const heartbeat = record(observer.heartbeat);
|
|
const manifest = record(observer.manifest);
|
|
const commandResult = record(result.result);
|
|
const credential = record(result.credential);
|
|
const id = result.id ?? observer.id ?? observer.jobId ?? manifest.jobId ?? "-";
|
|
const status = result.status ?? manifest.status ?? heartbeat.status ?? "-";
|
|
const stateDir = observer.stateDir ?? manifest.stateDir ?? "-";
|
|
const blockedRows = result.ok === true ? [] : [
|
|
"",
|
|
"Blocked detail:",
|
|
webObserveTable(
|
|
["EXIT", "TIMEOUT", "STDOUT", "STDERR"],
|
|
[[
|
|
webObserveText(commandResult.exitCode),
|
|
webObserveText(commandResult.timedOut),
|
|
webObserveShort(webObserveText(commandResult.stdoutTail ?? commandResult.stdout), 160),
|
|
webObserveShort(webObserveText(commandResult.stderrTail ?? commandResult.stderr), 300),
|
|
]],
|
|
),
|
|
];
|
|
const renderedText = [
|
|
webObserveTable(
|
|
["OBSERVER", "NODE", "LANE", "STATUS", "PID", "SAMPLE", "UPDATED"],
|
|
[[
|
|
id,
|
|
result.node,
|
|
result.lane,
|
|
status,
|
|
observer.pid ?? heartbeat.pid,
|
|
heartbeat.sampleSeq,
|
|
heartbeat.updatedAt,
|
|
]],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["URL", "TARGET_PATH", "STATE_DIR"],
|
|
[[result.url, result.targetPath, webObserveShort(webObserveText(stateDir), 96)]],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["CREDENTIAL", "SOURCE", "VALUES"],
|
|
[[
|
|
credential.username ?? "-",
|
|
credential.sourceRef ?? "-",
|
|
credential.valuesRedacted === true ? "redacted" : "unknown",
|
|
]],
|
|
),
|
|
...blockedRows,
|
|
"",
|
|
"NEXT",
|
|
` status: bun scripts/cli.ts hwlab nodes web-probe observe status ${id}`,
|
|
` analyze: bun scripts/cli.ts hwlab nodes web-probe observe analyze ${id}`,
|
|
` command: bun scripts/cli.ts hwlab nodes web-probe observe command ${id} --type mark --label checkpoint`,
|
|
` stop: bun scripts/cli.ts hwlab nodes web-probe observe stop ${id}`,
|
|
].join("\n");
|
|
return withWebObserveRendered(result, renderedText);
|
|
}
|
|
|
|
function renderWebObserveStatusResult(result: Record<string, unknown>): Record<string, unknown> {
|
|
const observer = record(result.observer);
|
|
const commandResult = record(result.result);
|
|
const manifest = record(observer.manifest);
|
|
const heartbeat = record(observer.heartbeat);
|
|
const tails = record(observer.tails);
|
|
const samples = webObserveArray(tails.samples);
|
|
const network = webObserveArray(tails.network);
|
|
const control = webObserveArray(tails.control);
|
|
const artifacts = webObserveArray(tails.artifacts);
|
|
const lastSample = record(samples[samples.length - 1]);
|
|
const failedNetwork = network
|
|
.map((item) => record(item))
|
|
.filter((item) => item.type === "requestfailed" || typeof item.status === "number" && item.status >= 500);
|
|
const id = result.id ?? observer.id ?? manifest.jobId ?? heartbeat.jobId ?? "-";
|
|
const resultSection = result.ok === true ? [] : [
|
|
"",
|
|
webObserveTable(
|
|
["RESULT", "VALUE"],
|
|
[
|
|
["exitCode", commandResult.exitCode ?? "-"],
|
|
["timedOut", commandResult.timedOut ?? "-"],
|
|
["stdoutTail", commandResult.stdoutTail ?? commandResult.stdout ?? "-"],
|
|
["stderrTail", commandResult.stderrTail ?? commandResult.stderr ?? "-"],
|
|
],
|
|
),
|
|
];
|
|
const renderedText = [
|
|
webObserveTable(
|
|
["OBSERVER", "NODE", "LANE", "STATUS", "ALIVE", "PID", "SAMPLE", "COMMAND", "UPDATED"],
|
|
[[id, result.node, result.lane, manifest.status ?? heartbeat.status ?? result.status, observer.processAlive, observer.pid, heartbeat.sampleSeq, heartbeat.commandSeq, heartbeat.updatedAt]],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["URL", "ROUTE_SESSION", "ACTIVE_SESSION", "MESSAGES", "TRACE_ROWS"],
|
|
[[heartbeat.currentUrl, lastSample.routeSessionId, lastSample.activeSessionId, lastSample.messageCount, lastSample.traceRowCount]],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["TAIL", "COUNT", "DETAIL"],
|
|
[
|
|
["control", control.length, record(control[control.length - 1]).type ?? "-"],
|
|
["samples", samples.length, record(samples[samples.length - 1]).ts ?? "-"],
|
|
["network", network.length, failedNetwork.length === 0 ? "no recent failed/5xx tail" : `${failedNetwork.length} recent failed/5xx`],
|
|
["artifacts", artifacts.length, record(artifacts[artifacts.length - 1]).sha256 ?? "-"],
|
|
],
|
|
),
|
|
...resultSection,
|
|
"",
|
|
"NEXT",
|
|
` status: bun scripts/cli.ts hwlab nodes web-probe observe status ${id}`,
|
|
` analyze: bun scripts/cli.ts hwlab nodes web-probe observe analyze ${id}`,
|
|
` command: bun scripts/cli.ts hwlab nodes web-probe observe command ${id} --type mark --label checkpoint`,
|
|
].join("\n");
|
|
return withWebObserveRendered(result, renderedText);
|
|
}
|
|
|
|
function renderWebObserveCommandResult(result: Record<string, unknown>): Record<string, unknown> {
|
|
const observerCommand = record(result.observerCommand);
|
|
const observer = record(result.observer);
|
|
const id = result.id ?? "-";
|
|
const renderedText = [
|
|
webObserveTable(
|
|
["OBSERVER", "NODE", "LANE", "COMMAND_ID", "TYPE", "STATUS", "DETAIL"],
|
|
[[id, result.node, result.lane, result.commandId, observerCommand.type, result.status, observer.queued === true ? "queued" : observer.phase ?? observer.status ?? "-"]],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["INPUT", "VALUE"],
|
|
[
|
|
["label", observerCommand.label ?? "-"],
|
|
["path", observerCommand.path ?? "-"],
|
|
["provider", observerCommand.provider ?? "-"],
|
|
["sessionId", observerCommand.sessionId ?? "-"],
|
|
["text", observerCommand.textHash === null || observerCommand.textHash === undefined ? "-" : `${observerCommand.textBytes ?? "-"}B ${observerCommand.textHash}`],
|
|
],
|
|
),
|
|
"",
|
|
"NEXT",
|
|
` status: bun scripts/cli.ts hwlab nodes web-probe observe status ${id}`,
|
|
` analyze: bun scripts/cli.ts hwlab nodes web-probe observe analyze ${id}`,
|
|
].join("\n");
|
|
return withWebObserveRendered(result, renderedText);
|
|
}
|
|
|
|
function renderWebObserveAnalyzeResult(result: Record<string, unknown>): Record<string, unknown> {
|
|
const analysis = record(result.analysis);
|
|
const failure = record(result.failure);
|
|
const counts = record(analysis.counts);
|
|
const sampleMetrics = record(analysis.sampleMetrics);
|
|
const runtimeAlerts = record(analysis.runtimeAlerts);
|
|
const pagePerformance = record(analysis.pagePerformance);
|
|
const rounds = webObserveArray(sampleMetrics.rounds).slice(-10).map((item) => record(item));
|
|
const turnColumns = webObserveArray(sampleMetrics.turnColumns).slice(-12).map((item) => record(item));
|
|
const domDiagnosticGroups = webObserveArray(runtimeAlerts.domDiagnosticsByFingerprint).slice(0, 10).map((item) => record(item));
|
|
const domDiagnostics = webObserveArray(runtimeAlerts.domDiagnostics).slice(-12).map((item) => record(item));
|
|
const jsonlReadIssues = webObserveArray(analysis.jsonlReadIssues).slice(0, 8).map((item) => record(item));
|
|
const slowApis = webObserveArray(analysis.pagePerformanceSlowApi).slice(0, 8).map((item) => record(item));
|
|
const sseStreams = webObserveArray(analysis.pagePerformanceSseStreams).slice(0, 8).map((item) => record(item));
|
|
const findings = webObserveArray(analysis.findings).slice(0, 8).map((item) => record(item));
|
|
const id = result.id ?? "-";
|
|
const blockerRows = String(result.status ?? "") === "blocked"
|
|
? [
|
|
["reason", failure.reason],
|
|
["exitCode", failure.exitCode],
|
|
["timedOut", failure.timedOut],
|
|
["parsedJson", failure.parsedJson],
|
|
["stdoutBytes", failure.stdoutBytes],
|
|
["stderrBytes", failure.stderrBytes],
|
|
["stdoutTail", String(failure.stdoutTail ?? "").replace(/\s+/g, " ").slice(0, 220)],
|
|
["stderrTail", String(failure.stderrTail ?? "").replace(/\s+/g, " ").slice(0, 220)],
|
|
["workspace", failure.workspace],
|
|
]
|
|
: [];
|
|
const renderedText = [
|
|
webObserveTable(
|
|
["OBSERVER", "NODE", "LANE", "STATUS", "REPORT_JSON", "REPORT_MD"],
|
|
[[id, result.node, result.lane, result.status, analysis.reportJsonSha256, analysis.reportMdSha256]],
|
|
),
|
|
...(blockerRows.length > 0
|
|
? ["", webObserveTable(["ANALYZE_BLOCKER", "VALUE"], blockerRows)]
|
|
: []),
|
|
"",
|
|
webObserveTable(
|
|
["SAMPLES", "CONTROL", "NETWORK", "CONSOLE", "ARTIFACTS", "ROUNDS", "ROWS"],
|
|
[[counts.samples, counts.control, counts.network, counts.console, counts.artifacts, rounds.length || sampleMetrics.rounds, sampleMetrics.turnTimingRows]],
|
|
),
|
|
"",
|
|
jsonlReadIssues.length === 0
|
|
? "JSONL_READ_ISSUES\n-"
|
|
: webObserveTable(
|
|
["FILE", "CODE", "MESSAGE"],
|
|
jsonlReadIssues.map((item) => [item.file, item.code, item.message]),
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["TURN_TIMING", "VALUE"],
|
|
[
|
|
["nonMonotonic", sampleMetrics.turnTimingNonMonotonicCount],
|
|
["elapsedDecrease", sampleMetrics.turnTimingTotalElapsedDecreaseCount],
|
|
["elapsedForwardJump", sampleMetrics.turnTimingTotalElapsedForwardJumpCount],
|
|
["elapsedForwardJumpMaxSec", sampleMetrics.turnTimingTotalElapsedForwardJumpMaxSeconds],
|
|
["recentJump", sampleMetrics.turnTimingRecentUpdateJumpCount],
|
|
["maxRecentStepSec", sampleMetrics.turnTimingRecentUpdateMaxIncreaseSeconds],
|
|
],
|
|
),
|
|
"",
|
|
rounds.length === 0
|
|
? "ROUNDS\n-"
|
|
: webObserveTable(
|
|
["ROUND", "COMMAND", "SAMPLES", "TOTAL_MAX", "TOTAL_LAST", "RECENT_MAX", "RECENT_LAST", "CARD_DIAG", "TERM", "NONMONO", "ELAPSED_JUMP", "JUMP"],
|
|
rounds.map((item) => [
|
|
item.promptIndex,
|
|
item.promptCommandId,
|
|
item.sampleCount,
|
|
item.maxTotalElapsedSeconds,
|
|
item.lastTotalElapsedSeconds,
|
|
item.maxRecentUpdateSeconds,
|
|
item.lastRecentUpdateSeconds,
|
|
item.diagnosticSamples,
|
|
item.terminalSamples,
|
|
item.turnTimingNonMonotonicCount,
|
|
item.turnTimingTotalElapsedForwardJumpCount,
|
|
item.turnTimingRecentUpdateJumpCount,
|
|
]),
|
|
),
|
|
"",
|
|
turnColumns.length === 0
|
|
? "TURN_COLUMNS\n-"
|
|
: webObserveTable(
|
|
["TURN", "PROMPT", "TRACE", "MESSAGE", "FIRST_SEQ", "LAST_SEQ", "LAST_TS"],
|
|
turnColumns.map((item) => [item.label ?? item.id, item.lastPromptIndex ?? item.promptIndex, item.traceId, item.messageId, item.firstSeq, item.lastSeq, item.lastTs]),
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["ALERT", "COUNT"],
|
|
[
|
|
["httpError", runtimeAlerts.httpErrorCount],
|
|
["requestFailed", runtimeAlerts.requestFailedCount],
|
|
["domDiagnosticSamples", runtimeAlerts.domDiagnosticSampleCount],
|
|
["domDiagnosticGroups", runtimeAlerts.domDiagnosticGroupCount],
|
|
["consoleAlerts", runtimeAlerts.consoleAlertCount],
|
|
["executionErrors", runtimeAlerts.executionErrorCount],
|
|
],
|
|
),
|
|
"",
|
|
domDiagnosticGroups.length === 0
|
|
? "DOM_DIAGNOSTIC_GROUPS\n-"
|
|
: webObserveTable(
|
|
["CODE", "COUNT", "FIRST_SEQ", "LAST_SEQ", "FIRST", "LAST", "PROMPTS", "TRACE", "PREVIEW"],
|
|
domDiagnosticGroups.map((item) => [
|
|
item.diagnosticCode,
|
|
item.count,
|
|
item.firstSeq,
|
|
item.lastSeq,
|
|
item.firstAt,
|
|
item.lastAt,
|
|
webObserveArray(item.promptIndexes).join(",") || "-",
|
|
item.traceId,
|
|
String(item.preview ?? "").replace(/\s+/g, " ").slice(0, 140),
|
|
]),
|
|
),
|
|
"",
|
|
domDiagnostics.length === 0
|
|
? "DOM_DIAGNOSTICS\n-"
|
|
: webObserveTable(
|
|
["SEQ", "TS", "PROMPT", "CODE", "TRACE", "HTTP", "IDLE", "WAITING_FOR", "LAST_EVENT", "PREVIEW"],
|
|
domDiagnostics.map((item) => [
|
|
item.seq,
|
|
item.ts,
|
|
item.promptIndex,
|
|
item.diagnosticCode,
|
|
item.traceId,
|
|
item.httpStatus,
|
|
item.idleSeconds,
|
|
item.waitingFor,
|
|
item.lastEventLabel,
|
|
String(item.preview ?? "").replace(/\s+/g, " ").slice(0, 180),
|
|
]),
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["PERF", "VALUE"],
|
|
[
|
|
["sameOriginApiPaths", pagePerformance.sameOriginApiPathCount],
|
|
["longLivedStreamPaths", pagePerformance.longLivedStreamPathCount],
|
|
["streamOpenOverBudgetSamples", pagePerformance.longLivedStreamOpenOverBudgetSampleCount ?? pagePerformance.longLivedStreamOpenOverFiveSecondSampleCount],
|
|
["slowPathCount", pagePerformance.slowPathCount],
|
|
["slowSampleCount", pagePerformance.slowSampleCount],
|
|
["worstP95Ms", pagePerformance.worstP95Ms],
|
|
],
|
|
),
|
|
"",
|
|
slowApis.length === 0
|
|
? "SLOW_API\n-"
|
|
: webObserveTable(
|
|
["SLOW_API", "P50", "P75", "P95", "OVER_BUDGET", "COUNT"],
|
|
slowApis.map((item) => [item.path, item.p50Ms, item.p75Ms, item.p95Ms, item.overBudgetCount ?? item.overFiveSecondCount, item.sampleCount]),
|
|
),
|
|
"",
|
|
sseStreams.length === 0
|
|
? "SSE_STREAMS\n-"
|
|
: webObserveTable(
|
|
["SSE_STREAM", "OPEN_P95", "OPEN_MAX", "OPEN_OVER_BUDGET", "LIFE>5S", "COUNT"],
|
|
sseStreams.map((item) => [item.path, item.streamOpenP95Ms, item.streamOpenMaxMs, item.streamOpenOverBudgetCount ?? item.streamOpenOverFiveSecondCount, item.streamLifetimeOverFiveSecondCount, item.sampleCount]),
|
|
),
|
|
"",
|
|
findings.length === 0
|
|
? "FINDINGS\n-"
|
|
: webObserveTable(
|
|
["FINDING", "SEVERITY", "COUNT", "SUMMARY"],
|
|
findings.map((item) => [item.id, item.severity, item.count, item.summary]),
|
|
),
|
|
"",
|
|
"REPORTS",
|
|
` json: ${analysis.reportJsonPath ?? "-"}`,
|
|
` md: ${analysis.reportMdPath ?? "-"}`,
|
|
"",
|
|
"NEXT",
|
|
` status: bun scripts/cli.ts hwlab nodes web-probe observe status ${id}`,
|
|
].join("\n");
|
|
return withWebObserveRendered(result, renderedText);
|
|
}
|
|
|
|
function renderWebProbeScriptResult(result: Record<string, unknown>): Record<string, unknown> {
|
|
const summary = record(result.summary);
|
|
const issueEvidence = record(result.issueEvidence);
|
|
const probe = record(result.probe);
|
|
const script = record(probe.script);
|
|
const scriptResult = record(script.result);
|
|
const reportLoad = record(result.reportLoad);
|
|
const commandResult = record(result.result);
|
|
const recoveredArtifacts = record(result.recoveredArtifacts);
|
|
const recoveredArtifactSummary = record(recoveredArtifacts.artifacts);
|
|
const recoveredItems = webObserveArray(recoveredArtifactSummary.items).slice(-8).map((item) => record(item));
|
|
const steps = webObserveArray(probe.steps).slice(-5).map((item) => record(item));
|
|
const resultRows = webProbeScriptRecordRows(scriptResult, 12);
|
|
const evidenceRows = webProbeScriptRecordRows(issueEvidence, 12);
|
|
const summaryRows = [
|
|
["ok", result.ok],
|
|
["status", result.status],
|
|
["degradedReason", result.degradedReason ?? "-"],
|
|
["failureKind", result.failureKind ?? "-"],
|
|
["scriptSource", result.scriptSource ?? "-"],
|
|
["reportSource", reportLoad.source ?? summary.recoveredFrom ?? "-"],
|
|
["reportPath", reportLoad.path ?? probe.reportPath ?? "-"],
|
|
["reportSha256", probe.reportSha256 ?? "-"],
|
|
["scriptSha256", probe.scriptSha256 ?? "-"],
|
|
["transportTimedOut", summary.transportTimedOut ?? commandResult.timedOut ?? "-"],
|
|
["exitCode", commandResult.exitCode ?? "-"],
|
|
];
|
|
const renderedText = [
|
|
webObserveTable(
|
|
["WEB_PROBE_SCRIPT", "NODE", "LANE", "STATUS", "OK", "URL"],
|
|
[[result.command ?? "web-probe script", result.node, result.lane, result.status, result.ok, result.url]],
|
|
),
|
|
"",
|
|
webObserveTable(["FIELD", "VALUE"], summaryRows),
|
|
"",
|
|
resultRows.length === 0
|
|
? "SCRIPT_RESULT\n-"
|
|
: webObserveTable(["KEY", "VALUE"], resultRows),
|
|
"",
|
|
evidenceRows.length === 0
|
|
? "ISSUE_EVIDENCE\n-"
|
|
: webObserveTable(["KEY", "VALUE"], evidenceRows),
|
|
"",
|
|
steps.length === 0
|
|
? "STEPS\n-"
|
|
: webObserveTable(
|
|
["STEP", "OK", "DETAIL"],
|
|
steps.map((item) => [
|
|
item.name ?? item.step ?? item.label ?? "-",
|
|
item.ok ?? item.status ?? "-",
|
|
webProbeScriptPreview(item.result ?? item.data ?? item.error ?? item.message ?? item),
|
|
]),
|
|
),
|
|
"",
|
|
recoveredItems.length === 0
|
|
? "ARTIFACTS\n-"
|
|
: webObserveTable(
|
|
["KIND", "BYTES", "PATH"],
|
|
recoveredItems.map((item) => [item.kind, item.byteCount, item.path]),
|
|
),
|
|
"",
|
|
"NEXT",
|
|
` report: ${reportLoad.path ?? probe.reportPath ?? "-"}`,
|
|
` rerun: ${result.command ?? `hwlab nodes web-probe script --node ${result.node ?? "-"} --lane ${result.lane ?? "-"}`}`,
|
|
].join("\n");
|
|
return withWebObserveRendered(result, renderedText);
|
|
}
|
|
|
|
function webProbeScriptRecordRows(value: Record<string, unknown>, limit: number): unknown[][] {
|
|
return Object.entries(value)
|
|
.slice(0, limit)
|
|
.map(([key, nested]) => [key, webProbeScriptPreview(nested)]);
|
|
}
|
|
|
|
function webProbeScriptPreview(value: unknown): string {
|
|
if (value === null || value === undefined) return "-";
|
|
if (typeof value === "string") return value.replace(/\s+/gu, " ").trim();
|
|
if (typeof value === "number" || typeof value === "boolean") return String(value);
|
|
if (Array.isArray(value)) {
|
|
const preview = value.slice(0, 3).map((item) => webProbeScriptPreview(item)).join(", ");
|
|
return `[${value.length}] ${preview}`;
|
|
}
|
|
if (typeof value === "object") {
|
|
return JSON.stringify(value).replace(/\s+/gu, " ").slice(0, 240);
|
|
}
|
|
return String(value);
|
|
}
|
|
|
|
function withWebObserveShortcuts(value: Record<string, unknown> | null, id: string | null): Record<string, unknown> | null {
|
|
if (value === null || id === null) return value;
|
|
return {
|
|
...value,
|
|
id,
|
|
next: webObserveNextCommands(id),
|
|
};
|
|
}
|
|
|
|
function nodeWebObserveStatusNodeScript(tailLines: number, node: string, lane: string): string {
|
|
return `node -e ${shellQuote(`
|
|
const fs=require('fs'),path=require('path');
|
|
const dir=process.env.state_dir||process.argv[1];
|
|
const node=process.argv[2];
|
|
const lane=process.argv[3];
|
|
const tailN=Math.min(${tailLines},3);
|
|
const readJson=(name)=>{try{return JSON.parse(fs.readFileSync(path.join(dir,name),'utf8'))}catch{return null}};
|
|
const tailJsonl=(name)=>{try{const file=path.join(dir,name); const st=fs.statSync(file); const maxBytes=Math.min(st.size,8*1024*1024); const fd=fs.openSync(file,'r'); try{const buf=Buffer.alloc(maxBytes); fs.readSync(fd,buf,0,maxBytes,st.size-maxBytes); const lines=buf.toString('utf8').split(/\\r?\\n/).filter(Boolean); if(st.size>maxBytes&&lines.length>0) lines.shift(); return lines.slice(-tailN).map(line=>{try{return JSON.parse(line)}catch{return {parseError:true, rawTail:line.slice(-500)}}});}finally{fs.closeSync(fd)}}catch{return []}};
|
|
const short=(value)=>String(value||'').slice(0,160);
|
|
const compactManifest=(item)=>item?{jobId:item.jobId,status:item.status,specRef:item.specRef,baseUrl:item.baseUrl,targetPath:item.targetPath,network:item.network,pageAuthority:item.pageAuthority,sampling:item.sampling,safety:item.safety,startedAt:item.startedAt,completedAt:item.completedAt,error:item.error?{message:short(item.error.message),auth:item.error.auth?{lastRetryLabel:item.error.auth.lastRetryLabel||null,retryExhausted:item.error.auth.retryExhausted===true,lastError:short(item.error.auth.lastError||'')}:null}:null}:null;
|
|
const compactAuth=(auth)=>auth?{phase:auth.phase||null,lastRetryLabel:auth.lastRetryLabel||null,retryAttempt:auth.retryAttempt??null,retryMaxAttempts:auth.retryMaxAttempts??null,retryDelayMs:auth.retryDelayMs??null,lastStatus:auth.lastStatus??null,lastStatusText:auth.lastStatusText||null,retryable:auth.retryable??null,cookiePresent:auth.cookiePresent??null,retryExhausted:auth.retryExhausted===true,lastError:short(auth.lastError||'')}:null;
|
|
const compactHeartbeat=(item)=>item?{ok:item.ok,jobId:item.jobId,pid:item.pid,stateDir:item.stateDir,status:item.status,pageId:item.pageId,baseUrl:item.baseUrl,currentUrl:item.currentUrl,sampleSeq:item.sampleSeq,commandSeq:item.commandSeq,activeCommandId:item.activeCommandId,auth:compactAuth(item.auth),updatedAt:item.updatedAt,uptimeMs:item.uptimeMs,error:item.error?{message:short(item.error.message),auth:compactAuth(item.error.auth)}:null}:null;
|
|
const retryLabel=(detail)=>detail&&detail.auth?detail.auth.lastRetryLabel||'':detail&&detail.result?detail.result.lastRetryLabel||'':detail&&detail.error&&detail.error.auth?detail.error.auth.lastRetryLabel||'':'';
|
|
const detailText=(detail)=>detail&&detail.error?short((detail.error.message||'')+(detail.error.auth&&detail.error.auth.lastError?' '+detail.error.auth.lastError:'')):detail&&detail.result?short([detail.result.statusText,detail.result.retryExhausted?'retry-exhausted':''].filter(Boolean).join(' ')):'';
|
|
const compactControl=(item)=>({ts:item.ts,seq:item.seq,phase:item.phase,type:item.type,commandId:item.commandId,durationMs:item.detail&&item.detail.durationMs||null,retry:retryLabel(item.detail),detail:detailText(item.detail)});
|
|
const compactSample=(item)=>({seq:item.seq,ts:item.ts,path:item.path,routeSessionId:item.routeSessionId||null,activeSessionId:item.activeSessionId||null,messageCount:Array.isArray(item.messages)?item.messages.length:0,traceRowCount:Array.isArray(item.traceRows)?item.traceRows.length:0,loadingCount:Array.isArray(item.loadings)?item.loadings.length:0,loadingOwners:Array.isArray(item.loadings)?Array.from(new Set(item.loadings.map((loading)=>loading.ownerLabel||loading.ownerKind||loading.ownerKey||'loading'))).slice(0,4):[],sessionRail:item.sessionRail?{visibleCount:item.sessionRail.visibleCount??null,fallbackTitleCount:item.sessionRail.fallbackTitleCount??null,fallbackTitleRatio:item.sessionRail.fallbackTitleRatio??null,fallbackItems:Array.isArray(item.sessionRail.fallbackItems)?item.sessionRail.fallbackItems.slice(0,4).map((fallback)=>({sessionIdPrefix:fallback.sessionIdPrefix??null,titlePreview:fallback.titlePreview??null,active:fallback.active===true})):[]}:null});
|
|
const compactNetwork=(item)=>({ts:item.ts,type:item.type,method:item.method,url:short(item.url),status:item.status||null,failure:item.failure?short(item.failure):null});
|
|
const pidText=fs.existsSync(path.join(dir,'pid'))?fs.readFileSync(path.join(dir,'pid'),'utf8').trim():null;
|
|
let alive=false; if(pidText){try{process.kill(Number(pidText),0); alive=true}catch{}}
|
|
console.log(JSON.stringify({ok:true,command:'web-probe-observe status',stateDir:dir,pid:pidText?Number(pidText):null,processAlive:alive,manifest:compactManifest(readJson('manifest.json')),heartbeat:compactHeartbeat(readJson('heartbeat.json')),tails:{control:tailJsonl('control.jsonl').map(compactControl),samples:tailJsonl('samples.jsonl').map(compactSample),network:tailJsonl('network.jsonl').map(compactNetwork)},next:{command:'bun scripts/cli.ts hwlab nodes web-probe observe command --node '+node+' --lane '+lane+' --state-dir '+dir+' --type mark --label checkpoint',stop:'bun scripts/cli.ts hwlab nodes web-probe observe stop --node '+node+' --lane '+lane+' --state-dir '+dir,analyze:'bun scripts/cli.ts hwlab nodes web-probe observe analyze --node '+node+' --lane '+lane+' --state-dir '+dir},valuesRedacted:true}));
|
|
`)} "$state_dir" ${shellQuote(node)} ${shellQuote(lane)}`;
|
|
}
|
|
|
|
function nodeWebObserveCollectNodeScript(maxFiles: number, collectFile: string | null, collectFinding: string | null, collectGrep: string | null): string {
|
|
return `node -e ${shellQuote(`
|
|
const fs=require('fs'),path=require('path'),crypto=require('crypto');
|
|
const dir=process.argv[1]; const findingFilter=process.argv[3]||''; let selected=(process.argv[2]||'') || (findingFilter ? 'analysis/report.json' : ''); const grepText=process.argv[4]||''; const maxFiles=${maxFiles}; const maxReadBytes=64*1024; const maxJsonlTailBytes=1024*1024; const maxTextPreviewBytes=4096; const maxGrepContextLines=24; const maxGrepLineText=180;
|
|
const shaFile=(file)=>'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex');
|
|
const shaText=(value)=>'sha256:'+crypto.createHash('sha256').update(String(value||'')).digest('hex');
|
|
const safeRel=(value)=>Boolean(value)&&!path.isAbsolute(value)&&!value.includes('..')&&!value.includes('\\\\')&&value.split('/').every((part)=>part&&part!=='.'&&part!=='..');
|
|
const short=(value,max=180)=>value===undefined||value===null?undefined:String(value).replace(/\\s+/g,' ').slice(0,max);
|
|
const grepLines=(text,pattern)=>{
|
|
if(!pattern) return null;
|
|
const needle=String(pattern).toLowerCase();
|
|
const lines=String(text||'').split(/\\r?\\n/);
|
|
const contexts=[]; let matchCount=0; const emitted=new Set();
|
|
for(let index=0; index<lines.length; index++){
|
|
if(!lines[index].toLowerCase().includes(needle)) continue;
|
|
matchCount++;
|
|
for(let offset=-2; offset<=2; offset++){
|
|
const lineIndex=index+offset;
|
|
if(lineIndex<0||lineIndex>=lines.length) continue;
|
|
if(emitted.has(lineIndex)) continue;
|
|
emitted.add(lineIndex);
|
|
contexts.push({lineNo:lineIndex+1,match:offset===0,text:short(lines[lineIndex],maxGrepLineText)});
|
|
if(contexts.length>=maxGrepContextLines) break;
|
|
}
|
|
if(contexts.length>=maxGrepContextLines) break;
|
|
}
|
|
return {patternSha256:shaText(pattern),matchCount,returnedLineCount:contexts.length,truncated:matchCount>0&&contexts.length>=maxGrepContextLines,lines:contexts,valuesRedacted:true};
|
|
};
|
|
const slimObject=(value,maxKeys=24)=>{
|
|
if(!value||typeof value!=='object'||Array.isArray(value)) return value??null;
|
|
const out={};
|
|
for(const key of Object.keys(value).slice(0,maxKeys)){
|
|
const item=value[key];
|
|
if(item===null||item===undefined||typeof item==='number'||typeof item==='boolean'||typeof item==='string') out[key]=short(item,160);
|
|
else if(Array.isArray(item)) out[key]={arrayLength:item.length};
|
|
else out[key]={keys:Object.keys(item).slice(0,12)};
|
|
}
|
|
return out;
|
|
};
|
|
const sensitiveKey=(key)=>/token|secret|password|passwd|authorization|cookie|api[-_]?key|session/i.test(String(key||''));
|
|
const slimJsonValue=(value,depth=0)=>{
|
|
if(value===null||value===undefined) return value??null;
|
|
if(typeof value==='number'||typeof value==='boolean') return value;
|
|
if(typeof value==='string') return short(value,360);
|
|
if(depth>=4) return Array.isArray(value)?{arrayLength:value.length}:{keys:Object.keys(value||{}).slice(0,20)};
|
|
if(Array.isArray(value)) return value.slice(0,12).map((item)=>slimJsonValue(item,depth+1)).concat(value.length>12?[{omitted:value.length-12}]:[]);
|
|
if(typeof value==='object'){
|
|
const out={};
|
|
const keys=Object.keys(value).slice(0,40);
|
|
for(const key of keys) out[key]=sensitiveKey(key)?'[redacted]':slimJsonValue(value[key],depth+1);
|
|
const omitted=Object.keys(value).length-keys.length;
|
|
if(omitted>0) out.__omittedKeys=omitted;
|
|
return out;
|
|
}
|
|
return short(value,180);
|
|
};
|
|
const slimFinding=(item)=>{
|
|
if(!item||typeof item!=='object') return null;
|
|
const samples=Array.isArray(item.samples)?item.samples.length:null;
|
|
const groups=Array.isArray(item.groups)?item.groups.length:null;
|
|
return {
|
|
kind:short(item.kind||item.code||item.id,80),
|
|
severity:short(item.severity||item.level,24),
|
|
count:item.count??item.sampleCount??null,
|
|
summary:short(item.summary||item.message,240),
|
|
sampleRows:samples,
|
|
groupRows:groups
|
|
};
|
|
};
|
|
const firstArray=(...values)=>values.find((value)=>Array.isArray(value))||[];
|
|
const compactMetric=(value)=>value===undefined||value===null?undefined:(typeof value==='object'?short(JSON.stringify(slimObject(value,8)),160):short(value,160));
|
|
const compactProjectionPair=(messageCount,traceCount)=>messageCount===undefined&&traceCount===undefined?undefined:'msg='+String(messageCount??'-')+' trace='+String(traceCount??'-');
|
|
const firstNestedRef=(value)=>value&&typeof value==='object'?(value.control&&typeof value.control==='object'?value.control:(value.observer&&typeof value.observer==='object'?value.observer:null)):null;
|
|
const compactTraceList=(value)=>{
|
|
if(Array.isArray(value)) return value.length>0?value.slice(0,4).map((item)=>String(item)).join(','):undefined;
|
|
if(value===undefined||value===null||value==='') return undefined;
|
|
return String(value);
|
|
};
|
|
const traceIdsFromRows=(rows)=>{
|
|
if(!Array.isArray(rows)) return undefined;
|
|
const ids=[];
|
|
for(const row of rows){
|
|
if(!row||typeof row!=='object') continue;
|
|
const value=row.traceId??row.businessTraceId??row.id;
|
|
if(value&&!ids.includes(String(value))) ids.push(String(value));
|
|
if(ids.length>=4) break;
|
|
}
|
|
return ids.length>0?ids:undefined;
|
|
};
|
|
const messageStatusesFromRows=(rows)=>{
|
|
if(!Array.isArray(rows)) return undefined;
|
|
const values=[];
|
|
for(const row of rows){
|
|
if(!row||typeof row!=='object') continue;
|
|
const status=row.status??row.dataStatus??row.messageStatus;
|
|
const trace=row.traceId??row.businessTraceId;
|
|
const text=[status,trace].filter(Boolean).join(':');
|
|
if(text&&!values.includes(text)) values.push(String(text));
|
|
if(values.length>=4) break;
|
|
}
|
|
return values.length>0?values:undefined;
|
|
};
|
|
const slimFindingSample=(value)=>{
|
|
if(!value||typeof value!=='object') return null;
|
|
const refValue=firstNestedRef(value)||{};
|
|
const controlCount=compactProjectionPair(value.controlMessageCount,value.controlTraceRowCount)??value.controlValue??value.beforeMessageCount??value.beforeValue;
|
|
const observerCount=compactProjectionPair(value.observerMessageCount,value.observerTraceRowCount)??value.observerValue??value.afterMessageCount??value.afterValue;
|
|
const trace=compactTraceList(value.traceIds)??compactTraceList(value.missingTraceIdsInObserver)??compactTraceList(value.controlTraceIds)??compactTraceList(value.observerTraceIds)??value.traceId??refValue.traceId;
|
|
return {
|
|
seq:value.seq??value.sampleSeq??value.fromSeq??value.controlSeq??refValue.seq??null,
|
|
ts:short(value.ts??value.sampleTs??value.fromTs??value.controlTs??refValue.ts,32),
|
|
diffKind:short(value.diffKind??value.kind??value.type??value.metric??value.anomaly,48),
|
|
fromValue:value.fromValue??value.beforeValue??null,
|
|
toValue:value.toValue??value.afterValue??null,
|
|
fromDigest:short(value.fromDigest,80),
|
|
toDigest:short(value.toDigest,80),
|
|
previousSeq:value.previousSeq??null,
|
|
previousTs:short(value.previousTs,32),
|
|
delta:value.delta??null,
|
|
sampleDeltaSeconds:value.sampleDeltaSeconds??null,
|
|
allowedIncreaseSeconds:value.allowedIncreaseSeconds??null,
|
|
excessiveIncreaseSeconds:value.excessiveIncreaseSeconds??null,
|
|
promptIndex:value.promptIndex??null,
|
|
event:short(value.event??value.anomaly??(value.steerUsed===true?'steer':Array.isArray(value.submitModes)?value.submitModes.join(','):undefined),48),
|
|
expectedPattern:short(value.expectedPattern,96),
|
|
control:compactMetric(controlCount??(value.fromMessageCount!==undefined||value.fromTraceRowCount!==undefined?compactProjectionPair(value.fromMessageCount,value.fromTraceRowCount):undefined)),
|
|
observer:compactMetric(observerCount??(value.toMessageCount!==undefined||value.toTraceRowCount!==undefined?compactProjectionPair(value.toMessageCount,value.toTraceRowCount):undefined)),
|
|
sessionId:short(value.sessionId??value.routeSessionId??value.activeSessionId??value.controlSessionId??value.observerSessionId??refValue.routeSessionId??refValue.activeSessionId,64),
|
|
path:short(value.path??value.urlPath??value.url??value.controlPath??value.observerPath??refValue.url,120),
|
|
trace:short(trace,120),
|
|
detail:short(value.detail??(value.durationText!==undefined||value.activityText!==undefined||value.textPreview!==undefined?['duration='+String(value.durationText??'-'),'activity='+String(value.activityText??'-'),'status='+String(value.status??'-'),'preview='+String(value.textPreview??'-')].join(' '):undefined)??(Array.isArray(value.submitModes)||Array.isArray(value.responseStatuses)?['modes='+String(Array.isArray(value.submitModes)?value.submitModes.join(','):'-'),'statuses='+String(Array.isArray(value.responseStatuses)?value.responseStatuses.join(','):'-'),'failure='+String(value.failureKind??'-')].join(' '):undefined)??(value.fromDigest||value.toDigest?'digest '+String(value.fromDigest??'-')+' -> '+String(value.toDigest??'-'):undefined)??(value.messageTextDigestDiff?'messageDigest '+String(value.controlMessageDigest??'-')+' != '+String(value.observerMessageDigest??'-'):undefined)??value.summary??value.message??value.preview??value.expectedPattern,180),
|
|
valuesRedacted:true
|
|
};
|
|
};
|
|
const slimFindingDetail=(parsed,id)=>{
|
|
if(!id||!parsed||typeof parsed!=='object'||!Array.isArray(parsed.findings)) return null;
|
|
const finding=parsed.findings.find((item)=>item&&typeof item==='object'&&String(item.id??item.kind??item.code??'')===id);
|
|
if(!finding) return {id,found:false,samples:[],valuesRedacted:true};
|
|
const sampleSource=Array.isArray(finding.samples)?'samples':Array.isArray(finding.groups)?'groups':Array.isArray(finding.examples)?'examples':Array.isArray(finding.rows)?'rows':Array.isArray(finding.rounds)?'rounds':'none';
|
|
const samples=firstArray(finding.samples,finding.groups,finding.examples,finding.rows,finding.rounds).slice(0,4).map(slimFindingSample).filter(Boolean);
|
|
return {id:short(finding.id??finding.kind??finding.code,80),found:true,severity:short(finding.severity??finding.level,24),count:finding.count??finding.sampleCount??samples.length,summary:short(finding.summary??finding.message,180),sampleSource,samples,valuesRedacted:true};
|
|
};
|
|
const slimRunnerError=(item)=>item&&typeof item==='object'?{ts:short(item.ts||item.at,32),type:short(item.type,48),retry:item.retry||item.lastRetryLabel||null,retryExhausted:item.retryExhausted===true,attemptCount:item.attemptCount??(Array.isArray(item.attempts)?item.attempts.length:null),lastFailureKind:short(item.lastFailureKind||item.failureKind,80),lastReadinessReason:short(item.lastReadinessReason,80),message:short(item.message||item.lastError,240)}:null;
|
|
const collectFallbackCandidates=(requested)=>{
|
|
const candidates=[];
|
|
if(findingFilter&&requested!=='analysis/report.json') candidates.push('analysis/report.json');
|
|
if(requested==='analysis/report.json'||findingFilter||requested.startsWith('analysis/')){
|
|
try{
|
|
const analysisDir=path.join(dir,'analysis');
|
|
const names=fs.readdirSync(analysisDir).filter((name)=>name.endsWith('.json')).map((name)=>({name,mtime:fs.statSync(path.join(analysisDir,name)).mtimeMs})).sort((a,b)=>b.mtime-a.mtime);
|
|
for(const item of names) candidates.push('analysis/'+item.name);
|
|
}catch{}
|
|
}
|
|
return [...new Set(candidates)].filter((item)=>item&&item!==requested&&safeRel(item));
|
|
};
|
|
const slimJsonl=(value)=>{
|
|
if(!value||typeof value!=='object'||Array.isArray(value)) return {value:short(value)};
|
|
const readiness=value.readiness&&typeof value.readiness==='object'?value.readiness:null;
|
|
const snapshot=value.snapshot&&typeof value.snapshot==='object'?value.snapshot:null;
|
|
const error=value.error&&typeof value.error==='object'?value.error:null;
|
|
return {
|
|
ts:short(value.ts||value.timestamp||value.at,32),
|
|
seq:value.seq,
|
|
type:short(value.type,48),
|
|
phase:short(value.phase,32),
|
|
commandId:short(value.commandId,40),
|
|
commandType:short(value.commandType||value.command?.type,40),
|
|
pageRole:short(value.pageRole,20),
|
|
path:short(value.path||value.routePath||snapshot?.path,80),
|
|
routeSessionId:short(value.routeSessionId||snapshot?.routeSessionId,48),
|
|
activeSessionId:short(value.activeSessionId||snapshot?.activeSessionId,48),
|
|
messageCount:value.messageCount??(Array.isArray(value.messages)?value.messages.length:undefined)??snapshot?.messageCount,
|
|
messageStatuses:short(compactTraceList(messageStatusesFromRows(value.messages)),160),
|
|
traceIds:short(compactTraceList(value.traceIds??traceIdsFromRows(value.traceRows)),160),
|
|
traceEventCount:value.traceEventCount??snapshot?.traceEventCount,
|
|
traceRowCount:value.traceRowCount??(Array.isArray(value.traceRows)?value.traceRows.length:undefined)??snapshot?.traceRowCount,
|
|
loadingCount:value.loadingCount??(Array.isArray(value.loadings)?value.loadings.length:undefined)??snapshot?.loadingCount,
|
|
method:short(value.method,12),
|
|
status:value.status,
|
|
url:short(value.url,120),
|
|
failure:short(value.failure||value.failureText||value.errorText,120),
|
|
readiness:readiness?{ok:readiness.ok,reason:short(readiness.reason,80),path:short(readiness.path,80),domReady:readiness.domReady,workbenchShellVisible:readiness.workbenchShellVisible,sessionCreateVisible:readiness.sessionCreateVisible,commandInputVisible:readiness.commandInputVisible}:undefined,
|
|
message:short(value.message||error?.message||value.text,180)
|
|
};
|
|
};
|
|
if(selected){
|
|
if(!safeRel(selected)){console.log(JSON.stringify({ok:false,command:'web-probe-observe collect',stateDir:dir,mode:'file',reason:'unsafe-file',file:selected,valuesRedacted:true},null,2)); process.exit(2);}
|
|
let file=path.join(dir,selected); let relative=path.relative(dir,file);
|
|
if(relative.startsWith('..')||path.isAbsolute(relative)){console.log(JSON.stringify({ok:false,command:'web-probe-observe collect',stateDir:dir,mode:'file',reason:'file-outside-state-dir',file:selected,valuesRedacted:true},null,2)); process.exit(2);}
|
|
if(!fs.existsSync(file)||!fs.statSync(file).isFile()){
|
|
const fallback=collectFallbackCandidates(selected).find((item)=>{const candidate=path.join(dir,item); return fs.existsSync(candidate)&&fs.statSync(candidate).isFile();});
|
|
if(fallback){selected=fallback; file=path.join(dir,selected); relative=path.relative(dir,file);}
|
|
}
|
|
if(!fs.existsSync(file)||!fs.statSync(file).isFile()){console.log(JSON.stringify({ok:false,command:'web-probe-observe collect',stateDir:dir,mode:'file',reason:'file-not-found',file:selected,fallbackCandidates:collectFallbackCandidates(selected).slice(0,8),valuesRedacted:true},null,2)); process.exit(1);}
|
|
const st=fs.statSync(file); const raw=fs.readFileSync(file); const truncated=raw.length>maxReadBytes; const isJsonl=selected.endsWith('.jsonl'); const isJson=selected.endsWith('.json');
|
|
const tailReadBytes=isJsonl?maxJsonlTailBytes:maxReadBytes;
|
|
const text=(isJsonl?raw.slice(Math.max(0,raw.length-tailReadBytes)):raw.slice(0,Math.min(raw.length,maxReadBytes))).toString('utf8');
|
|
const fullTextForGrep=grepText?raw.toString('utf8'):'';
|
|
const grep=grepText?grepLines(fullTextForGrep,grepText):null;
|
|
const lines=text.split(/\\r?\\n/).filter(Boolean);
|
|
if(isJsonl&&truncated&&lines.length>0) lines.shift();
|
|
let jsonSummary=null; let jsonContent=null;
|
|
if(isJson){try{const parsed=JSON.parse(raw.toString('utf8')); const inferredFindingFilter=findingFilter||(grepText&&Array.isArray(parsed&&parsed.findings)&&parsed.findings.some((item)=>item&&typeof item==='object'&&String(item.id??item.kind??item.code??'')===grepText)?grepText:null); jsonContent=raw.length<=4096?slimJsonValue(parsed):null; jsonSummary={topLevelKeys:parsed&&typeof parsed==='object'&&!Array.isArray(parsed)?Object.keys(parsed).slice(0,24):[],counts:slimObject(parsed&&parsed.counts,16),sampleMetrics:slimObject(parsed&&parsed.sampleMetrics,16),runtimeAlerts:slimObject(parsed&&parsed.runtimeAlerts,16),pagePerformance:slimObject(parsed&&parsed.pagePerformance,16),runnerErrors:Array.isArray(parsed&&parsed.runnerErrors)?parsed.runnerErrors.slice(-4).map(slimRunnerError).filter(Boolean):[],findings:Array.isArray(parsed&&parsed.findings)?parsed.findings.slice(0,6).map(slimFinding).filter(Boolean):[],findingDetail:slimFindingDetail(parsed,inferredFindingFilter)}}catch(error){jsonSummary={parseError:String(error&&error.message||error).slice(0,160)}}}
|
|
const jsonlTail=isJsonl&&!grep?lines.slice(-8).map((line,index)=>{try{return slimJsonl(JSON.parse(line))}catch(error){return {parseError:true,index,lineTail:line.slice(-240),error:String(error&&error.message||error).slice(0,160)}}}):null;
|
|
console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',stateDir:dir,mode:'file',file:{path:file,relative,byteCount:st.size,sha256:shaFile(file),truncated,content:isJsonl||isJson||jsonSummary||grep?undefined:text.slice(0,maxTextPreviewBytes),contentTruncated:!isJsonl&&!isJson&&!jsonSummary&&!grep&&text.length>maxTextPreviewBytes,jsonlTail,jsonSummary,jsonContent,grep,lineCount:lines.length},valuesRedacted:true}));
|
|
process.exit(0);
|
|
}
|
|
const out=[]; const walk=(p)=>{for(const ent of fs.readdirSync(p,{withFileTypes:true})){const full=path.join(p,ent.name); if(ent.isDirectory()) walk(full); else out.push(full); if(out.length>=maxFiles) return;}};
|
|
walk(dir);
|
|
const selectedFiles=out.slice(0,maxFiles);
|
|
const files=selectedFiles.slice(0,24).map(file=>{const st=fs.statSync(file); return {relative:path.relative(dir,file),byteCount:st.size,sha256:shaFile(file)}});
|
|
const totalBytes=selectedFiles.reduce((sum,file)=>sum+fs.statSync(file).size,0);
|
|
console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',stateDir:dir,fileCount:selectedFiles.length,listedFileCount:files.length,omittedFileCount:Math.max(0,selectedFiles.length-files.length),totalBytes,files,valuesRedacted:true}));
|
|
`)} "$state_dir" ${shellQuote(collectFile ?? "")} ${shellQuote(collectFinding ?? "")} ${shellQuote(collectGrep ?? "")}`;
|
|
}
|
|
|
|
function nodeWebObserveWaitCommandShell(commandId: string, waitMs: number): string {
|
|
if (waitMs <= 0) {
|
|
return [
|
|
`printf '{"ok":true,"queued":true,"commandId":%s,"stateDir":%s}\\n' ${shellQuote(JSON.stringify(commandId))} "$(node -e "console.log(JSON.stringify(process.argv[1]))" "$state_dir")"`,
|
|
].join("\n");
|
|
}
|
|
const waitSeconds = Math.max(1, Math.ceil(waitMs / 1000));
|
|
return [
|
|
`command_id=${shellQuote(commandId)}`,
|
|
`deadline=$(( $(date +%s) + ${waitSeconds} ))`,
|
|
"while [ \"$(date +%s)\" -le \"$deadline\" ]; do",
|
|
" if [ -f \"$state_dir/commands/done/${command_id}.json\" ]; then cat \"$state_dir/commands/done/${command_id}.json\"; exit 0; fi",
|
|
" if [ -f \"$state_dir/commands/failed/${command_id}.json\" ]; then cat \"$state_dir/commands/failed/${command_id}.json\"; exit 2; fi",
|
|
" sleep 1",
|
|
"done",
|
|
"printf '{\"ok\":true,\"queued\":true,\"waitTimedOut\":true,\"commandId\":\"%s\",\"stateDir\":\"%s\"}\\n' \"$command_id\" \"$state_dir\"",
|
|
].join("\n");
|
|
}
|
|
|
|
function commandSummaryForOutput(payload: Record<string, unknown>): Record<string, unknown> {
|
|
const text = typeof payload.text === "string" ? payload.text : null;
|
|
return {
|
|
id: payload.id,
|
|
type: payload.type,
|
|
path: payload.path ?? null,
|
|
label: payload.label ?? null,
|
|
sessionId: payload.sessionId ?? null,
|
|
provider: payload.provider ?? null,
|
|
textHash: text === null ? null : `sha256:${createHash("sha256").update(text).digest("hex")}`,
|
|
textBytes: text === null ? null : Buffer.byteLength(text),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function isSafeWebObserveStateDir(value: string): boolean {
|
|
return value.length > 0
|
|
&& !value.includes("\0")
|
|
&& !value.includes("..")
|
|
&& /^\.state\/web-observe\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\/\d{4}\/\d{2}\/\d{2}\/[A-Za-z0-9_.TZ-]+_[A-Za-z0-9_.-]+_webobs-[A-Za-z0-9_.-]+$/u.test(value);
|
|
}
|
|
|
|
function isSafeWebObserveCollectFile(value: string): boolean {
|
|
return value.length > 0
|
|
&& value.length <= 240
|
|
&& !value.includes("\0")
|
|
&& !value.includes("..")
|
|
&& !value.startsWith("/")
|
|
&& !value.includes("\\")
|
|
&& value.split("/").every((part) => part.length > 0 && part !== "." && part !== "..")
|
|
&& /^[A-Za-z0-9_.\/-]+$/u.test(value);
|
|
}
|
|
|
|
function isSafeWebObserveTraceId(value: string): boolean {
|
|
return /^trc_[A-Za-z0-9_-]{6,120}$/u.test(value);
|
|
}
|
|
|
|
function isSafeWebObserveFindingId(value: string): boolean {
|
|
return value.length > 0
|
|
&& value.length <= 120
|
|
&& !value.includes("\0")
|
|
&& /^[A-Za-z0-9_.:-]+$/u.test(value);
|
|
}
|
|
|
|
function isSafeWebObserveArchivePrefix(value: string): boolean {
|
|
return value.length > 0
|
|
&& value.length <= 120
|
|
&& !value.includes("\0")
|
|
&& !value.includes("..")
|
|
&& /^[A-Za-z0-9_.-]+$/u.test(value);
|
|
}
|
|
|
|
function isSafeWebObserveJobId(value: string): boolean {
|
|
return /^webobs-[A-Za-z0-9_.-]+$/u.test(value);
|
|
}
|
|
|
|
function safeWebObserveSegment(value: string): string {
|
|
return value.replace(/[^A-Za-z0-9_.-]+/gu, "-").replace(/^-+|-+$/gu, "") || "item";
|
|
}
|
|
|
|
function safeWebObserveTargetSegment(value: string): string {
|
|
const segment = value.replace(/^https?:\/\//u, "").replace(/[^A-Za-z0-9_.-]+/gu, "-").replace(/^-+|-+$/gu, "");
|
|
return segment.slice(0, 48) || "workbench";
|
|
}
|
|
|
|
function runNodeWebProbeScript(
|
|
options: NodeWebProbeScriptOptions,
|
|
spec: HwlabRuntimeLaneSpec,
|
|
secretSpec: RuntimeSecretSpec,
|
|
material: BootstrapAdminPasswordMaterial,
|
|
credential: Record<string, unknown>,
|
|
): Record<string, unknown> {
|
|
const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode);
|
|
const script = nodeWebProbeScriptRemoteShell(options, secretSpec, material.password ?? "", webProbeProxy);
|
|
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
|
const commandTimedOut = result.timedOut || result.exitCode === 124;
|
|
const stdoutReport = parseJsonObject(result.stdout);
|
|
const runPaths = webProbeScriptRunPathsFromStderr(result.stderr);
|
|
const recoveredReport = stdoutReport === null ? readNodeWebProbeScriptReport(options, spec, runPaths.reportPath) : null;
|
|
const recoveredArtifacts = stdoutReport === null || commandTimedOut ? readNodeWebProbeScriptArtifacts(options, spec, runPaths.runDir) : null;
|
|
const parsedReport = stdoutReport ?? recoveredReport?.report ?? null;
|
|
const report = compactWebProbeScriptResult(parsedReport);
|
|
const passed = result.exitCode === 0 && report?.ok === true;
|
|
const summary = nullableRecord(report?.summary);
|
|
const stdoutBytes = Buffer.byteLength(result.stdout, "utf8");
|
|
const outputFailureKind = parsedReport === null
|
|
? commandTimedOut
|
|
? "web-probe-command-timeout"
|
|
: stdoutBytes > 64 * 1024
|
|
? "web-probe-output-too-large"
|
|
: "web-probe-report-parse-failed"
|
|
: null;
|
|
const degradedReason = commandTimedOut
|
|
? "web-probe-command-timeout"
|
|
: typeof summary?.degradedReason === "string"
|
|
? summary.degradedReason
|
|
: typeof report?.failureKind === "string"
|
|
? report.failureKind
|
|
: outputFailureKind
|
|
? outputFailureKind
|
|
: commandTimedOut
|
|
? "web-probe-command-timeout"
|
|
: null;
|
|
const failureKind = commandTimedOut
|
|
? "web-probe-command-timeout"
|
|
: typeof summary?.failureKind === "string"
|
|
? summary.failureKind
|
|
: typeof report?.failureKind === "string"
|
|
? report.failureKind
|
|
: outputFailureKind;
|
|
const effectiveSummary = summary !== null ? {
|
|
...summary,
|
|
transportTimedOut: commandTimedOut,
|
|
recoveredFrom: stdoutReport !== null ? "stdout" : recoveredReport?.source ?? null,
|
|
} : (outputFailureKind === null ? null : {
|
|
ok: false,
|
|
status: "blocked",
|
|
degradedReason: outputFailureKind,
|
|
failureKind: outputFailureKind,
|
|
failedCondition: outputFailureKind === "web-probe-output-too-large"
|
|
? "remote web-probe stdout exceeded the bounded summary parser input"
|
|
: outputFailureKind === "web-probe-command-timeout"
|
|
? "remote web-probe command timed out before stdout returned a parseable report"
|
|
: "remote web-probe stdout did not contain a parseable JSON report",
|
|
runDir: runPaths.runDir,
|
|
reportPath: runPaths.reportPath,
|
|
reportLoad: recoveredReport === null ? null : {
|
|
source: recoveredReport.source,
|
|
path: recoveredReport.path,
|
|
degradedReason: recoveredReport.degradedReason,
|
|
result: recoveredReport.result === null ? null : compactCommandResultRedacted(recoveredReport.result, [material.password ?? ""]),
|
|
},
|
|
screenshots: recoveredArtifacts?.screenshots ?? [],
|
|
artifacts: recoveredArtifacts?.artifacts ?? null,
|
|
stdoutBytes,
|
|
exitCode: result.exitCode,
|
|
stderrTail: result.stderr.trim().slice(-2000),
|
|
valuesRedacted: true,
|
|
});
|
|
const issueEvidence = nullableRecord(report?.issueEvidence) ?? nullableRecord(effectiveSummary?.issueEvidence);
|
|
const compactResult = compactCommandResultRedacted(result, [material.password ?? ""]);
|
|
if (outputFailureKind !== null) {
|
|
compactResult.stdoutTail = redactKnownSecrets(result.stdout.slice(-2000), [material.password ?? ""]);
|
|
compactResult.stderrTail = redactKnownSecrets(result.stderr.slice(-2000), [material.password ?? ""]);
|
|
}
|
|
return renderWebProbeScriptResult({
|
|
ok: passed,
|
|
status: passed ? "pass" : "blocked",
|
|
command: `hwlab nodes web-probe script --node ${options.node} --lane ${options.lane}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
workspace: spec.workspace,
|
|
url: options.url,
|
|
network: webProbeProxy.summary,
|
|
credential,
|
|
scriptSource: options.scriptSource,
|
|
degradedReason,
|
|
failureKind,
|
|
summary: effectiveSummary,
|
|
issueEvidence,
|
|
probe: report,
|
|
reportLoad: stdoutReport !== null ? { source: "stdout", path: report?.reportPath ?? null, degradedReason: null } : recoveredReport === null ? null : {
|
|
source: recoveredReport.source,
|
|
path: recoveredReport.path,
|
|
degradedReason: recoveredReport.degradedReason,
|
|
result: recoveredReport.result === null ? null : compactCommandResultRedacted(recoveredReport.result, [material.password ?? ""]),
|
|
},
|
|
recoveredArtifacts: recoveredArtifacts === null ? null : {
|
|
source: recoveredArtifacts.source,
|
|
degradedReason: recoveredArtifacts.degradedReason,
|
|
result: recoveredArtifacts.result === null ? null : compactCommandResultRedacted(recoveredArtifacts.result, [material.password ?? ""]),
|
|
artifacts: recoveredArtifacts.artifacts,
|
|
},
|
|
result: compactResult,
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
|
|
function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secretSpec: RuntimeSecretSpec, password: string, webProbeProxy: NodeWebProbeHostProxyEnv): string {
|
|
const userScriptB64 = Buffer.from(options.scriptText, "utf8").toString("base64");
|
|
const runnerB64 = Buffer.from(nodeWebProbeScriptRunnerSource(), "utf8").toString("base64");
|
|
return [
|
|
"set -eu",
|
|
"run_root=.state/web-probe-script",
|
|
"mkdir -p \"$run_root\"",
|
|
"run_dir=$(mktemp -d \"$run_root/run.XXXXXX\")",
|
|
"report_file=\"$run_dir/web-probe-script-report.json\"",
|
|
"chmod 700 \"$run_dir\"",
|
|
"printf 'UNIDESK_WEB_PROBE_RUN_DIR=%s\\n' \"$run_dir\" >&2",
|
|
"printf 'UNIDESK_WEB_PROBE_REPORT_PATH=%s\\n' \"$report_file\" >&2",
|
|
"user_script=\"$run_dir/user-script.mjs\"",
|
|
"runner=\"$run_dir/runner.mjs\"",
|
|
`node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$user_script" ${shellQuote(userScriptB64)}`,
|
|
`node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$runner" ${shellQuote(runnerB64)}`,
|
|
[
|
|
...webProbeProxy.envAssignments,
|
|
`HWLAB_WEB_BASE_URL=${shellQuote(options.url)}`,
|
|
`HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`,
|
|
`HWLAB_WEB_PASS=${shellQuote(password)}`,
|
|
"UNIDESK_WEB_PROBE_RUN_DIR=\"$run_dir\"",
|
|
"UNIDESK_WEB_PROBE_USER_SCRIPT=\"$user_script\"",
|
|
`UNIDESK_WEB_PROBE_TIMEOUT_MS=${shellQuote(String(options.timeoutMs))}`,
|
|
`UNIDESK_WEB_PROBE_VIEWPORT=${shellQuote(options.viewport)}`,
|
|
`UNIDESK_WEB_PROBE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`,
|
|
"node \"$runner\"",
|
|
].join(" "),
|
|
].join("\n");
|
|
}
|
|
|
|
function webProbeScriptRunPathsFromStderr(stderr: string): { runDir: string | null; reportPath: string | null } {
|
|
const runDir = lineValueFromText(stderr, "UNIDESK_WEB_PROBE_RUN_DIR");
|
|
const markedReportPath = lineValueFromText(stderr, "UNIDESK_WEB_PROBE_REPORT_PATH");
|
|
const reportPath = markedReportPath ?? (runDir === null ? null : `${runDir.replace(/\/$/u, "")}/web-probe-script-report.json`);
|
|
return {
|
|
runDir: isSafeWebProbeScriptRunDir(runDir) ? runDir : null,
|
|
reportPath: isSafeWebProbeScriptReportPath(reportPath) ? reportPath : null,
|
|
};
|
|
}
|
|
|
|
function lineValueFromText(text: string, name: string): string | null {
|
|
const pattern = new RegExp(`^${name}=([^\\r\\n]+)$`, "mu");
|
|
const match = text.match(pattern);
|
|
return match ? match[1].trim() : null;
|
|
}
|
|
|
|
function readNodeWebProbeScriptReport(
|
|
options: NodeWebProbeScriptOptions,
|
|
spec: HwlabRuntimeLaneSpec,
|
|
reportPath: string | null,
|
|
): { source: string; report: Record<string, unknown> | null; result: CommandResult | null; degradedReason: string | null; path: string | null } | null {
|
|
if (!reportPath) return null;
|
|
if (!isSafeWebProbeScriptReportPath(reportPath)) return { source: "unsafe-path", report: null, result: null, degradedReason: "web-probe-report-path-invalid", path: reportPath };
|
|
const script = [
|
|
"set -eu",
|
|
`test -f ${shellQuote(reportPath)}`,
|
|
`node - ${shellQuote(reportPath)} <<'NODE'`,
|
|
"const fs = require('fs');",
|
|
"const reportPath = process.argv[2];",
|
|
"const report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));",
|
|
"function rec(value) { return value && typeof value === 'object' && !Array.isArray(value) ? value : {}; }",
|
|
"function compact(value, depth = 0) {",
|
|
" if (value === null || value === undefined) return value ?? null;",
|
|
" if (typeof value === 'string') return value.replace(/\\s+/gu, ' ').trim().slice(0, 120);",
|
|
" if (typeof value === 'number' || typeof value === 'boolean') return value;",
|
|
" if (depth >= 3) return '[max-depth]';",
|
|
" if (Array.isArray(value)) return value.slice(0, 4).map((item) => compact(item, depth + 1));",
|
|
" if (typeof value === 'object') {",
|
|
" const out = {};",
|
|
" for (const [key, nested] of Object.entries(value).slice(0, 8)) out[key] = compact(nested, depth + 1);",
|
|
" return out;",
|
|
" }",
|
|
" return String(value).slice(0, 240);",
|
|
"}",
|
|
"function compactSummary(value) {",
|
|
" const summary = rec(value);",
|
|
" return {",
|
|
" ok: summary.ok === true,",
|
|
" status: typeof summary.status === 'string' ? summary.status : null,",
|
|
" degradedReason: typeof summary.degradedReason === 'string' ? summary.degradedReason : null,",
|
|
" failureKind: typeof summary.failureKind === 'string' ? summary.failureKind : null,",
|
|
" failedCondition: typeof summary.failedCondition === 'string' ? compact(summary.failedCondition) : null,",
|
|
" nextAction: typeof summary.nextAction === 'string' ? compact(summary.nextAction) : null,",
|
|
" baseUrl: typeof summary.baseUrl === 'string' ? summary.baseUrl : null,",
|
|
" finalUrl: typeof summary.finalUrl === 'string' ? summary.finalUrl : null,",
|
|
" lastUrl: typeof summary.lastUrl === 'string' ? summary.lastUrl : null,",
|
|
" scriptSha256: typeof summary.scriptSha256 === 'string' ? summary.scriptSha256 : null,",
|
|
" runDir: typeof summary.runDir === 'string' ? summary.runDir : null,",
|
|
" reportPath: typeof summary.reportPath === 'string' ? summary.reportPath : null,",
|
|
" reportSha256: typeof summary.reportSha256 === 'string' ? summary.reportSha256 : null,",
|
|
" lastScreenshot: compact(summary.lastScreenshot),",
|
|
" screenshots: compact(summary.screenshots),",
|
|
" apiMatrix: compact(summary.apiMatrix),",
|
|
" stepCount: typeof summary.stepCount === 'number' ? summary.stepCount : null,",
|
|
" lastStep: compact(summary.lastStep),",
|
|
" issueEvidence: compact(summary.issueEvidence),",
|
|
" valuesRedacted: true,",
|
|
" };",
|
|
"}",
|
|
"const scriptBlock = rec(report.script);",
|
|
"const compactReport = {",
|
|
" ok: report.ok === true,",
|
|
" status: typeof report.status === 'string' ? report.status : null,",
|
|
" summary: compactSummary(report.summary),",
|
|
" issueEvidence: compact(report.issueEvidence ?? rec(report.summary).issueEvidence),",
|
|
" baseUrl: typeof report.baseUrl === 'string' ? report.baseUrl : null,",
|
|
" finalUrl: typeof report.finalUrl === 'string' ? report.finalUrl : null,",
|
|
" lastUrl: typeof report.lastUrl === 'string' ? report.lastUrl : null,",
|
|
" scriptSha256: typeof report.scriptSha256 === 'string' ? report.scriptSha256 : null,",
|
|
" runDir: typeof report.runDir === 'string' ? report.runDir : null,",
|
|
" reportPath: typeof report.reportPath === 'string' ? report.reportPath : reportPath,",
|
|
" reportSha256: typeof report.reportSha256 === 'string' ? report.reportSha256 : null,",
|
|
" auth: compact(report.auth),",
|
|
" script: { ok: scriptBlock.ok === true, result: compact(scriptBlock.result), stepCount: Array.isArray(scriptBlock.steps) ? scriptBlock.steps.length : null },",
|
|
" steps: Array.isArray(report.steps) ? report.steps.slice(-5).map((item) => compact(item)) : [],",
|
|
" failureKind: typeof report.failureKind === 'string' ? report.failureKind : null,",
|
|
" guidance: typeof report.guidance === 'string' ? report.guidance : null,",
|
|
" lastScreenshot: compact(report.lastScreenshot),",
|
|
" readiness: compact(report.readiness),",
|
|
" artifacts: compact(report.artifacts),",
|
|
" error: typeof report.error === 'string' ? report.error : null,",
|
|
" errorMessage: typeof report.errorMessage === 'string' ? report.errorMessage : null,",
|
|
" safety: compact(report.safety),",
|
|
" valuesRedacted: true,",
|
|
"};",
|
|
"console.log(JSON.stringify(compactReport));",
|
|
"NODE",
|
|
].join("\n");
|
|
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, 55);
|
|
if (result.exitCode !== 0 || result.timedOut) {
|
|
return { source: "report-file", report: null, result, degradedReason: result.timedOut ? "web-probe-command-timeout" : "web-probe-report-read-failed", path: reportPath };
|
|
}
|
|
const report = parseJsonObject(result.stdout);
|
|
return {
|
|
source: "report-file",
|
|
report,
|
|
result,
|
|
degradedReason: report === null ? "web-probe-report-parse-failed" : null,
|
|
path: reportPath,
|
|
};
|
|
}
|
|
|
|
function readNodeWebProbeScriptArtifacts(
|
|
options: NodeWebProbeScriptOptions,
|
|
spec: HwlabRuntimeLaneSpec,
|
|
runDir: string | null,
|
|
): { source: string; artifacts: Record<string, unknown> | null; screenshots: Record<string, unknown>[]; result: CommandResult | null; degradedReason: string | null } | null {
|
|
if (!runDir) return null;
|
|
if (!isSafeWebProbeScriptRunDir(runDir)) return { source: "unsafe-path", artifacts: null, screenshots: [], result: null, degradedReason: "web-probe-run-dir-invalid" };
|
|
const script = [
|
|
"set -eu",
|
|
`test -d ${shellQuote(runDir)}`,
|
|
`find ${shellQuote(runDir)} -maxdepth 1 -type f \\( -name '*.png' -o -name '*.json' \\) -printf '%p\\t%s\\n' | tail -n 30`,
|
|
].join("\n");
|
|
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, 55);
|
|
if (result.exitCode !== 0 || result.timedOut) {
|
|
return { source: "run-dir", artifacts: null, screenshots: [], result, degradedReason: result.timedOut ? "web-probe-command-timeout" : "web-probe-artifact-list-failed" };
|
|
}
|
|
const items = result.stdout.split(/\r?\n/u)
|
|
.map((line) => line.trim())
|
|
.filter(Boolean)
|
|
.map((line) => {
|
|
const [path, sizeText] = line.split("\t");
|
|
const byteCount = Number(sizeText);
|
|
return {
|
|
kind: path.endsWith(".png") ? "screenshot" : "json",
|
|
path,
|
|
byteCount: Number.isFinite(byteCount) ? byteCount : null,
|
|
};
|
|
})
|
|
.filter((item) => isSafeWebProbeScriptArtifactPath(String(item.path)));
|
|
const screenshots = items.filter((item) => item.kind === "screenshot");
|
|
return {
|
|
source: "run-dir",
|
|
artifacts: { runDir, count: items.length, items },
|
|
screenshots,
|
|
result,
|
|
degradedReason: null,
|
|
};
|
|
}
|
|
|
|
function isSafeWebProbeScriptRunDir(value: string | null): value is string {
|
|
return typeof value === "string"
|
|
&& value.length > 0
|
|
&& !value.includes("\0")
|
|
&& !value.includes("..")
|
|
&& /\.state\/web-probe-script\/run\.[A-Za-z0-9]+$/u.test(value);
|
|
}
|
|
|
|
function isSafeWebProbeScriptReportPath(value: string | null): value is string {
|
|
return typeof value === "string"
|
|
&& isSafeWebProbeScriptArtifactPath(value)
|
|
&& value.endsWith("/web-probe-script-report.json");
|
|
}
|
|
|
|
function isSafeWebProbeScriptArtifactPath(value: string): boolean {
|
|
return typeof value === "string"
|
|
&& value.length > 0
|
|
&& !value.includes("\0")
|
|
&& !value.includes("..")
|
|
&& /\.state\/web-probe-script\/run\.[A-Za-z0-9]+\/[^/]+[.](?:png|json)$/u.test(value);
|
|
}
|
|
|
|
|
|
|
|
function parseSecretOptions(args: string[]): NodeSecretOptions {
|
|
const [actionRaw] = args;
|
|
if (actionRaw !== "status" && actionRaw !== "ensure" && actionRaw !== "cleanup-owned-postgres" && actionRaw !== "cleanup-obsolete") {
|
|
throw new Error("secret usage: status|ensure --node NODE --lane vNN --name hwlab-vNN-openfga|hwlab-vNN-master-server-admin-api-key|hwlab-vNN-bootstrap-admin|hwlab-cloud-api-vNN-db|hwlab-vNN-code-agent-provider [--dry-run|--confirm] | cleanup-owned-postgres --node NODE --lane vNN [--dry-run|--confirm] | cleanup-obsolete --node NODE --lane vNN --name hwpod-vNN-db [--dry-run|--confirm]");
|
|
}
|
|
const node = requiredOption(args, "--node");
|
|
assertNodeId(node);
|
|
const lane = requiredOption(args, "--lane");
|
|
assertLane(lane);
|
|
const spec = runtimeSecretSpec({ node, lane });
|
|
const name = optionValue(args, "--name") ?? spec.openFgaSecret;
|
|
const key = optionValue(args, "--key");
|
|
const confirm = args.includes("--confirm");
|
|
const explicitDryRun = args.includes("--dry-run");
|
|
if (confirm && explicitDryRun) throw new Error("secret accepts only one of --confirm or --dry-run");
|
|
if (actionRaw === "cleanup-owned-postgres") {
|
|
if (lane === "v02") throw new Error("secret cleanup-owned-postgres is only for v0.3+ lanes after migration to G14 platform Postgres");
|
|
if (key !== undefined) throw new Error("secret cleanup-owned-postgres does not accept --key");
|
|
if (name !== spec.openFgaSecret && name !== spec.postgresSecret) throw new Error(`secret cleanup-owned-postgres for --lane ${lane} targets ${spec.postgresSecret}; omit --name or pass --name ${spec.postgresSecret}`);
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane,
|
|
name: spec.postgresSecret,
|
|
preset: "owned-postgres-cleanup",
|
|
confirm,
|
|
dryRun: explicitDryRun || !confirm,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
|
};
|
|
}
|
|
if (actionRaw === "cleanup-obsolete") {
|
|
if (lane === "v02") throw new Error("secret cleanup-obsolete is only for v0.3+ lanes after deprecated v0.3 HWPOD DB SecretRef cleanup");
|
|
if (key !== undefined) throw new Error("secret cleanup-obsolete does not accept --key");
|
|
if (name !== spec.obsoleteHwpodDbSecret) throw new Error(`secret cleanup-obsolete for --lane ${lane} currently only targets obsolete ${spec.obsoleteHwpodDbSecret}`);
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane,
|
|
name,
|
|
preset: "obsolete-secret-cleanup",
|
|
confirm,
|
|
dryRun: explicitDryRun || !confirm,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
|
};
|
|
}
|
|
if (name === spec.masterAdminApiKeySecret) {
|
|
if (key !== undefined && key !== MASTER_ADMIN_API_KEY_KEY) throw new Error(`secret ${name} supports only key ${MASTER_ADMIN_API_KEY_KEY}`);
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane,
|
|
name,
|
|
key: key ?? MASTER_ADMIN_API_KEY_KEY,
|
|
preset: "master-server-admin-api-key",
|
|
confirm,
|
|
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
|
};
|
|
}
|
|
if (name === spec.bootstrapAdminSecret) {
|
|
if (key !== undefined && key !== BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY) throw new Error(`secret ${name} supports only key ${BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY}`);
|
|
const force = args.includes("--force");
|
|
if (force && actionRaw !== "ensure") throw new Error("secret --force is only supported with ensure");
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane,
|
|
name,
|
|
key: key ?? BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY,
|
|
preset: "bootstrap-admin",
|
|
confirm,
|
|
force,
|
|
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
|
};
|
|
}
|
|
if (name === spec.codeAgentProviderSecret) {
|
|
if (key !== undefined && key !== CODE_AGENT_PROVIDER_OPENAI_KEY && key !== CODE_AGENT_PROVIDER_OPENCODE_KEY) {
|
|
throw new Error(`secret ${name} supports keys ${CODE_AGENT_PROVIDER_OPENAI_KEY} and ${CODE_AGENT_PROVIDER_OPENCODE_KEY}`);
|
|
}
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane,
|
|
name,
|
|
key,
|
|
preset: "code-agent-provider",
|
|
confirm,
|
|
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
|
};
|
|
}
|
|
if (name === spec.cloudApiDbSecret) {
|
|
if (key !== undefined && key !== spec.cloudApiDbKey) throw new Error(`secret ${name} supports only key ${spec.cloudApiDbKey}`);
|
|
if (actionRaw === "ensure" && spec.platformDb && spec.externalPostgres === undefined) {
|
|
throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`);
|
|
}
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane,
|
|
name,
|
|
key: key ?? spec.cloudApiDbKey,
|
|
preset: "cloud-api-db",
|
|
confirm,
|
|
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 240, 900),
|
|
};
|
|
}
|
|
if (name !== spec.openFgaSecret) {
|
|
throw new Error(`secret status/ensure supports --name ${spec.openFgaSecret}, ${spec.masterAdminApiKeySecret}, ${spec.bootstrapAdminSecret}, ${spec.cloudApiDbSecret}, or ${spec.codeAgentProviderSecret} for --lane ${lane}; obsolete ${spec.obsoleteHwpodDbSecret} must use cleanup-obsolete`);
|
|
}
|
|
if (key !== undefined && key !== OPENFGA_AUTHN_KEY && key !== OPENFGA_DATASTORE_URI_KEY && key !== OPENFGA_POSTGRES_PASSWORD_KEY) {
|
|
throw new Error(`secret ${name} supports keys ${OPENFGA_AUTHN_KEY}, ${OPENFGA_DATASTORE_URI_KEY}, and ${OPENFGA_POSTGRES_PASSWORD_KEY}`);
|
|
}
|
|
if (actionRaw === "ensure" && spec.platformDb && spec.externalPostgres === undefined) {
|
|
throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`);
|
|
}
|
|
return {
|
|
action: actionRaw,
|
|
node,
|
|
lane,
|
|
name,
|
|
key,
|
|
preset: "openfga",
|
|
confirm,
|
|
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
|
};
|
|
}
|
|
|
|
function runtimeSecretSpec(input: { node: string; lane: string }): RuntimeSecretSpec {
|
|
const namespace = `hwlab-${input.lane}`;
|
|
const runtimeLaneSpec = isHwlabRuntimeLane(input.lane) ? hwlabRuntimeLaneSpecForNode(input.lane, input.node) : undefined;
|
|
const externalPostgres = runtimeLaneSpec?.externalPostgres;
|
|
const postgresStore = runtimeLaneSpec?.runtimeStore?.postgres;
|
|
const bootstrapAdmin = runtimeLaneSpec?.bootstrapAdmin;
|
|
const platformDb = externalPostgres !== undefined || postgresStore?.mode === "platform-service" || (postgresStore?.mode === undefined && /^v0*[3-9]\d*$/.test(input.lane));
|
|
const localPostgresService = postgresStore?.serviceName ?? `${namespace}-postgres`;
|
|
const platformPostgresService = externalPostgres?.serviceName ?? postgresStore?.serviceName ?? "g14-platform-postgres";
|
|
const legacyPostgresHost = `${localPostgresService}.${namespace}.svc.cluster.local`;
|
|
const platformPostgresHost = `${platformPostgresService}.${namespace}.svc.cluster.local`;
|
|
const platformPostgresEndpointSlice = `${platformPostgresService}-host`;
|
|
const openFgaDbName = externalPostgres?.database ?? postgresStore?.openfga?.database ?? (platformDb ? `openfga_${input.lane}` : "hwlab_openfga");
|
|
const openFgaDbUser = externalPostgres?.openfga.role ?? postgresStore?.openfga?.role ?? (platformDb ? `openfga_${input.lane}_app` : "hwlab_openfga");
|
|
const cloudApiDbName = externalPostgres?.database ?? postgresStore?.cloudApi?.database ?? `hwlab_${input.lane}`;
|
|
const cloudApiDbUser = externalPostgres?.cloudApi.role ?? postgresStore?.cloudApi?.role ?? (platformDb ? `hwlab_${input.lane}_app` : `hwlab_${input.lane}`);
|
|
return {
|
|
node: input.node,
|
|
lane: input.lane,
|
|
namespace,
|
|
platformDb,
|
|
...(runtimeLaneSpec === undefined ? {} : { runtimeLaneSpec }),
|
|
...(externalPostgres === undefined ? {} : { externalPostgres }),
|
|
platformPostgresService,
|
|
platformPostgresEndpointAddress: externalPostgres?.endpointAddress,
|
|
platformPostgresEndpointSlice,
|
|
postgresSecret: postgresStore?.secretName ?? `${namespace}-postgres`,
|
|
postgresStatefulSet: postgresStore?.statefulSet ?? postgresStore?.secretName ?? `${namespace}-postgres`,
|
|
postgresAdminUser: postgresStore?.adminUser ?? `hwlab_${input.lane}`,
|
|
openFgaSecret: externalPostgres?.openfga.secretName ?? postgresStore?.openfga?.secretName ?? `${namespace}-openfga`,
|
|
openFgaDbName,
|
|
openFgaDbUser,
|
|
openFgaDbHost: platformDb ? platformPostgresHost : legacyPostgresHost,
|
|
masterAdminApiKeySecret: `${namespace}-master-server-admin-api-key`,
|
|
bootstrapAdminSecret: bootstrapAdmin?.secretName ?? `${namespace}-bootstrap-admin`,
|
|
bootstrapAdminPasswordHashKey: bootstrapAdmin?.secretKey ?? BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY,
|
|
bootstrapAdminUsername: bootstrapAdmin?.username ?? "admin",
|
|
bootstrapAdminDisplayName: bootstrapAdmin?.displayName ?? `HWLAB ${input.lane} Admin`,
|
|
...(bootstrapAdmin?.passwordSourceRef === undefined ? {} : { bootstrapAdminPasswordSourceRef: bootstrapAdmin.passwordSourceRef }),
|
|
...(bootstrapAdmin?.passwordSourceKey === undefined ? {} : { bootstrapAdminPasswordSourceKey: bootstrapAdmin.passwordSourceKey }),
|
|
...(bootstrapAdmin?.passwordHashTransform === undefined ? {} : { bootstrapAdminPasswordHashTransform: bootstrapAdmin.passwordHashTransform }),
|
|
bootstrapAdminSourceNamespace: BOOTSTRAP_ADMIN_SOURCE_NAMESPACE,
|
|
bootstrapAdminSourceSecret: BOOTSTRAP_ADMIN_SOURCE_SECRET,
|
|
cloudApiDbSecret: externalPostgres?.cloudApi.secretName ?? postgresStore?.cloudApi?.secretName ?? `hwlab-cloud-api-${input.lane}-db`,
|
|
cloudApiDbKey: externalPostgres?.cloudApi.secretKey ?? postgresStore?.cloudApi?.secretKey ?? CLOUD_API_DB_KEY,
|
|
cloudApiDbName,
|
|
cloudApiDbUser,
|
|
cloudApiDbHost: platformDb ? platformPostgresHost : legacyPostgresHost,
|
|
cloudApiDeployment: "hwlab-cloud-api",
|
|
obsoleteHwpodDbSecret: `hwpod-${input.lane}-db`,
|
|
obsoleteHwpodDbName: `hwpod_${input.lane}`,
|
|
obsoleteHwpodDbUser: `hwpod_${input.lane}_app`,
|
|
codeAgentProviderSecret: `${namespace}-code-agent-provider`,
|
|
codeAgentProviderSourceNamespace: CODE_AGENT_PROVIDER_SOURCE_NAMESPACE,
|
|
codeAgentProviderSourceSecret: CODE_AGENT_PROVIDER_SOURCE_SECRET,
|
|
fieldManager: `unidesk-hwlab-node-${input.lane}-secret`,
|
|
};
|
|
}
|
|
|
|
function runNodeSecret(options: NodeSecretOptions): Record<string, unknown> {
|
|
const spec = runtimeSecretSpec(options);
|
|
if (options.preset === "obsolete-secret-cleanup") return runObsoleteSecretCleanup(options, spec);
|
|
if (spec.externalPostgres !== undefined && options.action === "ensure" && (options.preset === "cloud-api-db" || options.preset === "openfga")) {
|
|
return runExternalPostgresSecretEnsure(options, spec);
|
|
}
|
|
const bootstrapAdminMaterial = options.preset === "bootstrap-admin" ? readBootstrapAdminSecretMaterial(spec) : null;
|
|
const input = options.preset === "master-server-admin-api-key" && options.action === "ensure" && !options.dryRun
|
|
? readMasterAdminApiKey(spec).key
|
|
: options.preset === "bootstrap-admin" && options.action === "ensure" && !options.dryRun && bootstrapAdminMaterial?.ok === true
|
|
? bootstrapAdminMaterial.passwordHash ?? ""
|
|
: "";
|
|
const script = options.preset === "openfga"
|
|
? spec.platformDb ? platformDbSecretStatusScript(options, spec) : openFgaSecretScript(options, spec)
|
|
: options.preset === "master-server-admin-api-key"
|
|
? masterAdminApiKeySecretScript(options, spec)
|
|
: options.preset === "bootstrap-admin"
|
|
? bootstrapAdminSecretScript(options, spec, bootstrapAdminMaterial)
|
|
: options.preset === "cloud-api-db"
|
|
? spec.platformDb ? platformDbSecretStatusScript(options, spec) : cloudApiDbSecretScript(options, spec)
|
|
: options.preset === "owned-postgres-cleanup"
|
|
? ownedPostgresCleanupScript(options, spec)
|
|
: codeAgentProviderSecretScript(options, spec);
|
|
const result = runTransScript(options.node, script, input, options.timeoutSeconds);
|
|
const status = secretStatusFromText(statusText(result), result.exitCode === 0, result.exitCode, result.stderr, spec);
|
|
const dryRunOk = options.action === "ensure" && options.dryRun && result.exitCode === 0;
|
|
const cleanupDryRunOk = options.action === "cleanup-owned-postgres" && options.dryRun && result.exitCode === 0;
|
|
const obsoleteCleanupDryRunOk = options.action === "cleanup-obsolete" && options.dryRun && status.ok === true;
|
|
const ok = dryRunOk || cleanupDryRunOk || obsoleteCleanupDryRunOk ? true : status.ok === true;
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes secret ${options.action}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
namespace: spec.namespace,
|
|
secret: options.name,
|
|
key: options.key ?? null,
|
|
preset: options.preset,
|
|
mode: options.action === "status" ? "status" : options.dryRun ? "dry-run" : options.action === "cleanup-owned-postgres" || options.action === "cleanup-obsolete" ? "confirmed-delete" : "confirmed-ensure",
|
|
status,
|
|
mutation: status.mutation === true,
|
|
result: compactCommandResult(result),
|
|
valuesRedacted: true,
|
|
next: ok && options.action === "status" ? undefined : nextSecretCommand(options, spec),
|
|
};
|
|
}
|
|
|
|
function nextSecretCommand(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record<string, string> {
|
|
if (options.action === "cleanup-owned-postgres") {
|
|
return { ensure: `bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node ${options.node} --lane ${options.lane} --confirm` };
|
|
}
|
|
if (options.action === "cleanup-obsolete") {
|
|
return { cleanup: `bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node ${options.node} --lane ${options.lane} --name ${options.name} --confirm` };
|
|
}
|
|
if (spec.externalPostgres !== undefined && (options.preset === "cloud-api-db" || options.preset === "openfga")) {
|
|
return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` };
|
|
}
|
|
if (spec.platformDb && (options.preset === "cloud-api-db" || options.preset === "openfga")) {
|
|
return {
|
|
status: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""}`,
|
|
controlPlaneStatus: `bun scripts/cli.ts hwlab nodes control-plane status --node ${options.node} --lane ${options.lane}`,
|
|
};
|
|
}
|
|
return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` };
|
|
}
|
|
|
|
function runExternalPostgresSecretEnsure(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record<string, unknown> {
|
|
const runtimeLaneSpec = spec.runtimeLaneSpec;
|
|
if (runtimeLaneSpec === undefined) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes secret ${options.action}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
namespace: spec.namespace,
|
|
secret: options.name,
|
|
key: options.key ?? null,
|
|
preset: options.preset,
|
|
mode: options.dryRun ? "dry-run" : "confirmed-ensure",
|
|
mutation: false,
|
|
degradedReason: "external-postgres-runtime-lane-spec-missing",
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
const sync = syncNodeExternalPostgresSecrets(runtimeLaneSpec, options.dryRun, options.timeoutSeconds);
|
|
const shouldReadStatus = sync !== null && sync.ok === true;
|
|
const statusResult = shouldReadStatus ? runTransScript(options.node, platformDbSecretStatusScript({ ...options, action: "status", dryRun: true }, spec), "", options.timeoutSeconds) : null;
|
|
const status = statusResult === null
|
|
? null
|
|
: secretStatusFromText(statusText(statusResult), statusResult.exitCode === 0, statusResult.exitCode, statusResult.stderr, spec);
|
|
const ok = sync !== null && sync.ok === true && (options.dryRun || status?.ok === true);
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes secret ${options.action}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
namespace: spec.namespace,
|
|
secret: options.name,
|
|
key: options.key ?? null,
|
|
preset: options.preset,
|
|
mode: options.dryRun ? "dry-run" : "confirmed-ensure",
|
|
status: status ?? sync,
|
|
externalPostgresSecretSync: sync,
|
|
mutation: sync?.mutation === true,
|
|
result: statusResult === null ? null : compactCommandResult(statusResult),
|
|
valuesRedacted: true,
|
|
next: ok ? undefined : nextSecretCommand(options, spec),
|
|
};
|
|
}
|
|
|
|
function publicExposureSummary(exposure: HwlabRuntimePublicExposureSpec): Record<string, unknown> {
|
|
return {
|
|
mode: exposure.mode,
|
|
publicBaseUrl: exposure.publicBaseUrl,
|
|
hostname: exposure.hostname,
|
|
expectedA: exposure.expectedA,
|
|
frpc: {
|
|
serverAddr: exposure.serverAddr,
|
|
serverPort: exposure.serverPort,
|
|
tokenSourceRef: exposure.tokenSourceRef,
|
|
tokenSourceKey: exposure.tokenSourceKey,
|
|
secretName: exposure.secretName,
|
|
secretKey: exposure.secretKey,
|
|
tokenKey: exposure.tokenKey,
|
|
webProxy: exposure.webProxy,
|
|
apiProxy: exposure.apiProxy,
|
|
},
|
|
caddy: {
|
|
route: exposure.caddyRoute,
|
|
configPath: exposure.caddyConfigPath,
|
|
serviceName: exposure.caddyServiceName,
|
|
tls: exposure.caddyTls,
|
|
responseHeaderTimeoutSeconds: exposure.responseHeaderTimeoutSeconds,
|
|
},
|
|
};
|
|
}
|
|
|
|
function runNodePublicExposure(options: NodePublicExposureOptions): Record<string, unknown> {
|
|
const exposure = options.spec.publicExposure;
|
|
if (exposure === null || !exposure.enabled) {
|
|
return {
|
|
ok: false,
|
|
command: "hwlab nodes control-plane public-exposure",
|
|
node: options.node,
|
|
lane: options.lane,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
error: "publicExposure is not configured for this node/lane target",
|
|
mutation: false,
|
|
};
|
|
}
|
|
const source = readPublicExposureTokenSource(exposure);
|
|
if (!source.ok) {
|
|
return {
|
|
ok: false,
|
|
command: "hwlab nodes control-plane public-exposure",
|
|
node: options.node,
|
|
lane: options.lane,
|
|
mode: options.dryRun ? "dry-run" : "confirmed",
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
publicExposure: publicExposureSummary(exposure),
|
|
source,
|
|
mutation: false,
|
|
valuesRedacted: true,
|
|
next: { fixSecretSource: `create .state/secrets/${exposure.tokenSourceRef} with ${exposure.tokenSourceKey}=<redacted>` },
|
|
};
|
|
}
|
|
const secretApplyResult = runTransScript(options.node, publicExposureSecretScript(options, exposure), source.value ?? "", options.timeoutSeconds);
|
|
let secretFields = keyValueLinesFromText(statusText(secretApplyResult));
|
|
let secretResult = secretApplyResult;
|
|
if (!options.dryRun && secretApplyResult.exitCode === 0 && secretFields.afterSecretExists === "yes") {
|
|
const restartResult = runPublicExposureFrpcRecreate(options);
|
|
secretFields = { ...secretFields, ...keyValueLinesFromText(statusText(restartResult)) };
|
|
secretResult = combinePublicExposureCommandResults(secretApplyResult, restartResult);
|
|
}
|
|
const caddyResult = runTransHostScript(exposure.caddyRoute, publicExposureCaddyScript(options, exposure), "", options.timeoutSeconds);
|
|
const caddyFields = keyValueLinesFromText(statusText(caddyResult));
|
|
const secretStatus = publicExposureSecretStatus(secretFields, secretResult);
|
|
const caddyStatus = publicExposureCaddyStatus(caddyFields, caddyResult);
|
|
const ok = secretStatus.ok === true && caddyStatus.ok === true;
|
|
return {
|
|
ok,
|
|
command: "hwlab nodes control-plane public-exposure",
|
|
node: options.node,
|
|
lane: options.lane,
|
|
mode: options.dryRun ? "dry-run" : "confirmed",
|
|
mutation: !options.dryRun && (secretFields.mutation === "true" || caddyFields.mutation === "true"),
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
publicExposure: publicExposureSummary(exposure),
|
|
source: {
|
|
ok: source.ok,
|
|
path: source.path,
|
|
key: exposure.tokenSourceKey,
|
|
bytes: source.value?.length ?? 0,
|
|
fingerprint: source.fingerprint,
|
|
valueRedacted: true,
|
|
},
|
|
secret: secretStatus,
|
|
caddy: caddyStatus,
|
|
valuesRedacted: true,
|
|
next: ok
|
|
? {
|
|
triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${options.node} --lane ${options.lane} --confirm`,
|
|
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${options.node} --lane ${options.lane}`,
|
|
}
|
|
: { retry: `bun scripts/cli.ts hwlab nodes control-plane public-exposure --node ${options.node} --lane ${options.lane} --confirm` },
|
|
};
|
|
}
|
|
|
|
function readPublicExposureTokenSource(exposure: HwlabRuntimePublicExposureSpec): { ok: boolean; path: string; checkedPaths: string[]; key: string; value: string | null; fingerprint: string | null; error?: string } {
|
|
const checkedPaths = publicExposureTokenSourcePaths(exposure);
|
|
const path = checkedPaths.find((candidate) => existsSync(candidate)) ?? checkedPaths[0] ?? join(repoRoot, ".state", "secrets", exposure.tokenSourceRef);
|
|
if (!existsSync(path)) return { ok: false, path, checkedPaths, key: exposure.tokenSourceKey, value: null, fingerprint: null, error: "secret-source-missing" };
|
|
const values = parseEnvFile(readFileSync(path, "utf8"));
|
|
const value = values[exposure.tokenSourceKey];
|
|
if (value === undefined || value.length === 0) return { ok: false, path, checkedPaths, key: exposure.tokenSourceKey, value: null, fingerprint: null, error: "secret-key-missing" };
|
|
return {
|
|
ok: true,
|
|
path,
|
|
checkedPaths,
|
|
key: exposure.tokenSourceKey,
|
|
value,
|
|
fingerprint: `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`,
|
|
};
|
|
}
|
|
|
|
function publicExposureTokenSourcePaths(exposure: HwlabRuntimePublicExposureSpec): string[] {
|
|
const paths = [join(repoRoot, ".state", "secrets", exposure.tokenSourceRef)];
|
|
const marker = "/.worktree/";
|
|
const index = repoRoot.indexOf(marker);
|
|
if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", exposure.tokenSourceRef));
|
|
return [...new Set(paths)];
|
|
}
|
|
|
|
function publicExposureSecretStatus(fields: Record<string, string>, result: CommandResult): Record<string, unknown> {
|
|
const dryRun = fields.dryRun === "true";
|
|
return {
|
|
ok: result.exitCode === 0 && (dryRun || (
|
|
fields.afterSecretExists === "yes"
|
|
&& fields.strategyPatchExitCode === "0"
|
|
&& fields.rolloutRestartExitCode === "0"
|
|
&& fields.rolloutStatusExitCode === "0"
|
|
)),
|
|
dryRun,
|
|
mutation: fields.mutation === "true",
|
|
namespace: fields.namespace || null,
|
|
secret: fields.secret || null,
|
|
deployment: fields.deployment || null,
|
|
beforeExists: fields.beforeSecretExists === "yes",
|
|
afterExists: fields.afterSecretExists === "yes",
|
|
tokenBytes: numericField(fields.tokenBytes),
|
|
applyExitCode: numericField(fields.applyExitCode),
|
|
restartMode: fields.restartMode || null,
|
|
strategyPatchExitCode: numericField(fields.strategyPatchExitCode),
|
|
rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode),
|
|
scaleDownExitCode: numericField(fields.scaleDownExitCode),
|
|
scaleDownWaitExitCode: numericField(fields.scaleDownWaitExitCode),
|
|
scaleDownWaitPodCount: numericField(fields.scaleDownWaitPodCount),
|
|
scaleUpExitCode: numericField(fields.scaleUpExitCode),
|
|
rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode),
|
|
readyReplicas: numericField(fields.readyReplicas),
|
|
availableReplicas: numericField(fields.availableReplicas),
|
|
desiredReplicas: numericField(fields.desiredReplicas),
|
|
strategyPatchErrorPreview: fields.strategyPatchErrorPreview || null,
|
|
restartErrorPreview: fields.restartErrorPreview || null,
|
|
scaleDownErrorPreview: fields.scaleDownErrorPreview || null,
|
|
scaleUpErrorPreview: fields.scaleUpErrorPreview || null,
|
|
readyErrorPreview: fields.readyErrorPreview || null,
|
|
exitCode: result.exitCode,
|
|
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000),
|
|
};
|
|
}
|
|
|
|
function publicExposureCaddyStatus(fields: Record<string, string>, result: CommandResult): Record<string, unknown> {
|
|
const dryRun = fields.dryRun === "true";
|
|
return {
|
|
ok: result.exitCode === 0
|
|
&& fields.afterBlockPresent === "yes"
|
|
&& fields.validateExitCode === "0"
|
|
&& (dryRun || (
|
|
fields.active === "active"
|
|
&& fields.localWebStatus === "200"
|
|
&& fields.localApiStatus === "200"
|
|
)),
|
|
dryRun,
|
|
mutation: fields.mutation === "true",
|
|
hostname: fields.hostname || null,
|
|
configPath: fields.configPath || null,
|
|
beforeBlockPresent: fields.beforeBlockPresent === "yes",
|
|
afterBlockPresent: fields.afterBlockPresent === "yes",
|
|
staleBlocksRemoved: numericField(fields.staleBlocksRemoved),
|
|
pythonExitCode: numericField(fields.pythonExitCode),
|
|
validateMode: fields.validateMode || null,
|
|
validateExitCode: numericField(fields.validateExitCode),
|
|
reloadExitCode: numericField(fields.reloadExitCode),
|
|
active: fields.active || null,
|
|
localWebStatus: fields.localWebStatus || null,
|
|
localApiStatus: fields.localApiStatus || null,
|
|
validateErrorPreview: fields.validateErrorPreview || null,
|
|
localWebErrorPreview: fields.localWebErrorPreview || null,
|
|
localApiErrorPreview: fields.localApiErrorPreview || null,
|
|
exitCode: result.exitCode,
|
|
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000),
|
|
};
|
|
}
|
|
|
|
function publicExposureSecretScript(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec): string {
|
|
const deployment = `${options.spec.runtimeNamespace}-frpc`;
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(options.spec.runtimeNamespace)}`,
|
|
`secret=${shellQuote(exposure.secretName)}`,
|
|
`deployment=${shellQuote(deployment)}`,
|
|
`token_key=${shellQuote(exposure.tokenKey)}`,
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
"token=$(cat)",
|
|
"before_secret_exists=no",
|
|
"kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && before_secret_exists=yes",
|
|
"token_bytes=$(printf '%s' \"$token\" | wc -c | tr -d ' ')",
|
|
"apply_exit=",
|
|
"restart_mode=recreate-rollout-restart",
|
|
"strategy_patch_exit=",
|
|
"rollout_restart_exit=",
|
|
"scale_down_exit=",
|
|
"scale_down_wait_exit=",
|
|
"scale_down_wait_pod_count=",
|
|
"scale_up_exit=",
|
|
"rollout_status_exit=",
|
|
"ready_replicas=",
|
|
"available_replicas=",
|
|
"desired_replicas=",
|
|
"mutation=false",
|
|
"if [ \"$dry_run\" = false ]; then",
|
|
" tmp=$(mktemp /tmp/hwlab-public-frpc-secret.XXXXXX.yaml)",
|
|
" token_b64=$(printf '%s' \"$token\" | base64 | tr -d '\\n')",
|
|
" cat >\"$tmp\" <<EOF",
|
|
"apiVersion: v1",
|
|
"kind: Secret",
|
|
"metadata:",
|
|
" name: $secret",
|
|
" namespace: $namespace",
|
|
" labels:",
|
|
" app.kubernetes.io/part-of: hwlab",
|
|
" app.kubernetes.io/managed-by: unidesk",
|
|
"type: Opaque",
|
|
"data:",
|
|
" $token_key: $token_b64",
|
|
"EOF",
|
|
" kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-node-public-exposure -f \"$tmp\" >/tmp/hwlab-public-frpc-secret.apply.out 2>/tmp/hwlab-public-frpc-secret.apply.err",
|
|
" apply_exit=$?",
|
|
" rm -f \"$tmp\"",
|
|
" if [ \"$apply_exit\" = 0 ]; then mutation=true; fi",
|
|
"fi",
|
|
"after_secret_exists=no",
|
|
"kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && after_secret_exists=yes",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$secret\"",
|
|
"printf 'deployment\\t%s\\n' \"$deployment\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"",
|
|
"printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"",
|
|
"printf 'tokenBytes\\t%s\\n' \"$token_bytes\"",
|
|
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
|
|
"printf 'restartMode\\t%s\\n' \"$restart_mode\"",
|
|
"printf 'strategyPatchExitCode\\t%s\\n' \"$strategy_patch_exit\"",
|
|
"printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"",
|
|
"printf 'scaleDownExitCode\\t%s\\n' \"$scale_down_exit\"",
|
|
"printf 'scaleDownWaitExitCode\\t%s\\n' \"$scale_down_wait_exit\"",
|
|
"printf 'scaleDownWaitPodCount\\t%s\\n' \"$scale_down_wait_pod_count\"",
|
|
"printf 'scaleUpExitCode\\t%s\\n' \"$scale_up_exit\"",
|
|
"printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"",
|
|
"printf 'readyReplicas\\t%s\\n' \"$ready_replicas\"",
|
|
"printf 'availableReplicas\\t%s\\n' \"$available_replicas\"",
|
|
"printf 'desiredReplicas\\t%s\\n' \"$desired_replicas\"",
|
|
"if [ \"$dry_run\" = true ]; then exit 0; fi",
|
|
"[ \"$after_secret_exists\" = yes ] && [ \"$apply_exit\" = 0 ]",
|
|
].join("\n");
|
|
}
|
|
|
|
function runPublicExposureFrpcRecreate(options: NodePublicExposureOptions): CommandResult {
|
|
const namespace = options.spec.runtimeNamespace;
|
|
const deployment = `${namespace}-frpc`;
|
|
const fields: Record<string, string> = {
|
|
deployment,
|
|
restartMode: "recreate-rollout-restart",
|
|
strategyPatchExitCode: "",
|
|
rolloutRestartExitCode: "",
|
|
scaleDownExitCode: "",
|
|
scaleDownWaitExitCode: "",
|
|
scaleDownWaitPodCount: "",
|
|
scaleUpExitCode: "",
|
|
rolloutStatusExitCode: "",
|
|
readyReplicas: "",
|
|
availableReplicas: "",
|
|
desiredReplicas: "",
|
|
strategyPatchErrorPreview: "",
|
|
restartErrorPreview: "",
|
|
scaleDownErrorPreview: "",
|
|
scaleUpErrorPreview: "",
|
|
readyErrorPreview: "",
|
|
};
|
|
const stdoutParts: string[] = [];
|
|
const stderrParts: string[] = [];
|
|
const addResult = (result: CommandResult): void => {
|
|
if (result.stdout.trim()) stdoutParts.push(result.stdout.trim());
|
|
if (result.stderr.trim()) stderrParts.push(result.stderr.trim());
|
|
};
|
|
const shortTimeout = Math.min(Math.max(20, options.timeoutSeconds), 55);
|
|
const strategyPatch = runTransScript(options.node, [
|
|
"set +e",
|
|
`namespace=${shellQuote(namespace)}`,
|
|
`deployment=${shellQuote(deployment)}`,
|
|
"kubectl -n \"$namespace\" patch deploy/\"$deployment\" --type=merge -p '{\"spec\":{\"strategy\":{\"type\":\"Recreate\",\"rollingUpdate\":null}}}' >/tmp/hwlab-public-frpc-strategy-patch.out 2>/tmp/hwlab-public-frpc-strategy-patch.err",
|
|
"code=$?",
|
|
"printf 'strategyPatchExitCode\\t%s\\n' \"$code\"",
|
|
"printf 'strategyPatchErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-strategy-patch.err ] && tr '\\n' ';' </tmp/hwlab-public-frpc-strategy-patch.err | cut -c1-1000 || true)\"",
|
|
"exit \"$code\"",
|
|
].join("\n"), "", shortTimeout);
|
|
addResult(strategyPatch);
|
|
Object.assign(fields, keyValueLinesFromText(statusText(strategyPatch)));
|
|
|
|
if (strategyPatch.exitCode === 0) {
|
|
const restart = runTransScript(options.node, [
|
|
"set +e",
|
|
`namespace=${shellQuote(namespace)}`,
|
|
`deployment=${shellQuote(deployment)}`,
|
|
"kubectl -n \"$namespace\" rollout restart deploy/\"$deployment\" >/tmp/hwlab-public-frpc-restart.out 2>/tmp/hwlab-public-frpc-restart.err",
|
|
"code=$?",
|
|
"printf 'rolloutRestartExitCode\\t%s\\n' \"$code\"",
|
|
"printf 'restartErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-restart.err ] && tr '\\n' ';' </tmp/hwlab-public-frpc-restart.err | cut -c1-1000 || true)\"",
|
|
"exit \"$code\"",
|
|
].join("\n"), "", shortTimeout);
|
|
addResult(restart);
|
|
Object.assign(fields, keyValueLinesFromText(statusText(restart)));
|
|
}
|
|
|
|
if (fields.rolloutRestartExitCode === "0") {
|
|
const deadline = Date.now() + 55_000;
|
|
while (Date.now() <= deadline) {
|
|
const probe = runTransScript(options.node, [
|
|
"set +e",
|
|
`namespace=${shellQuote(namespace)}`,
|
|
`deployment=${shellQuote(deployment)}`,
|
|
"kubectl -n \"$namespace\" get deploy \"$deployment\" -o jsonpath='{.spec.replicas}{\"\\t\"}{.status.readyReplicas}{\"\\t\"}{.status.availableReplicas}' >/tmp/hwlab-public-frpc-ready.out 2>/tmp/hwlab-public-frpc-ready.err",
|
|
"code=$?",
|
|
"values=$(cat /tmp/hwlab-public-frpc-ready.out 2>/dev/null || true)",
|
|
"desired=$(printf '%s' \"$values\" | cut -f1)",
|
|
"ready=$(printf '%s' \"$values\" | cut -f2)",
|
|
"available=$(printf '%s' \"$values\" | cut -f3)",
|
|
"printf 'desiredReplicas\\t%s\\n' \"$desired\"",
|
|
"printf 'readyReplicas\\t%s\\n' \"$ready\"",
|
|
"printf 'availableReplicas\\t%s\\n' \"$available\"",
|
|
"printf 'readyErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-ready.err ] && tr '\\n' ';' </tmp/hwlab-public-frpc-ready.err | cut -c1-1000 || true)\"",
|
|
"[ \"$code\" = 0 ] && [ \"$desired\" = 1 ] && [ \"$ready\" = 1 ] && [ \"$available\" = 1 ]",
|
|
].join("\n"), "", shortTimeout);
|
|
addResult(probe);
|
|
Object.assign(fields, keyValueLinesFromText(statusText(probe)));
|
|
if (probe.exitCode === 0) {
|
|
fields.rolloutStatusExitCode = "0";
|
|
break;
|
|
}
|
|
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
|
|
}
|
|
}
|
|
if (fields.rolloutStatusExitCode === "") fields.rolloutStatusExitCode = "1";
|
|
|
|
const ok = fields.strategyPatchExitCode === "0"
|
|
&& fields.rolloutRestartExitCode === "0"
|
|
&& fields.rolloutStatusExitCode === "0";
|
|
const stdout = Object.entries(fields).map(([key, value]) => `${key}\t${value}`).join("\n") + "\n";
|
|
return {
|
|
command: [transPath(), `${options.node}:k3s`, "sh", "--", "<public-exposure-frpc-recreate>"],
|
|
cwd: repoRoot,
|
|
exitCode: ok ? 0 : 1,
|
|
stdout: [stdout.trim(), ...stdoutParts].filter(Boolean).join("\n") + "\n",
|
|
stderr: stderrParts.join("\n"),
|
|
signal: null,
|
|
timedOut: false,
|
|
};
|
|
}
|
|
|
|
function combinePublicExposureCommandResults(first: CommandResult, second: CommandResult): CommandResult {
|
|
return {
|
|
command: [...first.command, "&&", ...second.command],
|
|
cwd: repoRoot,
|
|
exitCode: first.exitCode === 0 && second.exitCode === 0 ? 0 : second.exitCode ?? first.exitCode,
|
|
stdout: [first.stdout.trim(), second.stdout.trim()].filter(Boolean).join("\n") + "\n",
|
|
stderr: [first.stderr.trim(), second.stderr.trim()].filter(Boolean).join("\n"),
|
|
signal: second.signal ?? first.signal,
|
|
timedOut: first.timedOut || second.timedOut,
|
|
};
|
|
}
|
|
|
|
function publicExposureCaddyScript(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec): string {
|
|
const blockB64 = Buffer.from(publicExposureCaddyBlock(exposure), "utf8").toString("base64");
|
|
const marker = `unidesk managed ${exposure.hostname}`;
|
|
return [
|
|
"set +e",
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
`hostname=${shellQuote(exposure.hostname)}`,
|
|
`config_path=${shellQuote(exposure.caddyConfigPath)}`,
|
|
`service=${shellQuote(exposure.caddyServiceName)}`,
|
|
`marker=${shellQuote(marker)}`,
|
|
`block_b64=${shellQuote(blockB64)}`,
|
|
`expected_a=${shellQuote(exposure.expectedA)}`,
|
|
`web_port=${shellQuote(String(exposure.webProxy.remotePort))}`,
|
|
`api_port=${shellQuote(String(exposure.apiProxy.remotePort))}`,
|
|
"rm -f /tmp/hwlab-public-caddy-validate.out /tmp/hwlab-public-caddy-validate.err /tmp/hwlab-public-caddy-python.err /tmp/hwlab-public-caddy-web.err /tmp/hwlab-public-caddy-api.err",
|
|
"tmp=$(mktemp -d)",
|
|
"block=\"$tmp/block\"",
|
|
"next=\"$tmp/Caddyfile\"",
|
|
"printf '%s' \"$block_b64\" | base64 -d >\"$block\"",
|
|
"before_present=no",
|
|
"if [ -f \"$config_path\" ] && grep -Fq \"# BEGIN $marker\" \"$config_path\"; then before_present=yes; fi",
|
|
"if [ -f \"$config_path\" ]; then cp \"$config_path\" \"$next\"; else : >\"$next\"; fi",
|
|
"stale_blocks_removed=$(python3 - \"$next\" \"$block\" \"$marker\" \"$web_port\" \"$api_port\" 2>/tmp/hwlab-public-caddy-python.err <<'PY'",
|
|
"import pathlib, re, sys",
|
|
"config = pathlib.Path(sys.argv[1])",
|
|
"block = pathlib.Path(sys.argv[2]).read_text(encoding='utf-8')",
|
|
"marker = sys.argv[3]",
|
|
"web_port = sys.argv[4]",
|
|
"api_port = sys.argv[5]",
|
|
"current_name = marker[len('unidesk managed '):] if marker.startswith('unidesk managed ') else marker",
|
|
"text = config.read_text(encoding='utf-8') if config.exists() else ''",
|
|
"begin = f'# BEGIN {marker}'",
|
|
"end = f'# END {marker}'",
|
|
"managed = f'{begin}\\n{block.rstrip()}\\n{end}\\n'",
|
|
"stale_removed = [0]",
|
|
"pattern = re.compile(r'(?ms)^# BEGIN unidesk managed (?P<name>[^\\n]+)\\n(?P<body>.*?)\\n# END unidesk managed (?P=name)\\n*')",
|
|
"def keep(match):",
|
|
" name = match.group('name')",
|
|
" body = match.group('body')",
|
|
" if name != current_name and (f'127.0.0.1:{web_port}' in body or f'127.0.0.1:{api_port}' in body):",
|
|
" stale_removed[0] += 1",
|
|
" return ''",
|
|
" return match.group(0)",
|
|
"text = pattern.sub(keep, text)",
|
|
"if begin in text and end in text:",
|
|
" start = text.index(begin)",
|
|
" stop = text.index(end, start) + len(end)",
|
|
" text = text[:start].rstrip() + '\\n\\n' + managed.rstrip() + '\\n' + text[stop:].lstrip('\\n')",
|
|
"else:",
|
|
" text = text.rstrip() + '\\n\\n' + managed",
|
|
"config.write_text(text, encoding='utf-8')",
|
|
"print(stale_removed[0])",
|
|
"PY",
|
|
")",
|
|
"python_exit=$?",
|
|
"validate_exit=1",
|
|
"validate_mode=validate",
|
|
"if [ \"$python_exit\" != 0 ]; then",
|
|
" validate_mode=python",
|
|
" validate_exit=$python_exit",
|
|
"else",
|
|
" if [ \"$dry_run\" = true ]; then",
|
|
" validate_mode=adapt",
|
|
" caddy adapt --config \"$next\" --adapter caddyfile >/tmp/hwlab-public-caddy-validate.out 2>/tmp/hwlab-public-caddy-validate.err",
|
|
" else",
|
|
" sudo caddy validate --config \"$next\" --adapter caddyfile >/tmp/hwlab-public-caddy-validate.out 2>/tmp/hwlab-public-caddy-validate.err",
|
|
" fi",
|
|
" validate_exit=$?",
|
|
"fi",
|
|
"reload_exit=",
|
|
"mutation=false",
|
|
"if [ \"$dry_run\" = false ] && [ \"$validate_exit\" = 0 ]; then",
|
|
" sudo install -m 0644 \"$next\" \"$config_path\" >/tmp/hwlab-public-caddy-install.out 2>/tmp/hwlab-public-caddy-install.err",
|
|
" install_exit=$?",
|
|
" if [ \"$install_exit\" = 0 ]; then",
|
|
" mutation=true",
|
|
" sudo systemctl reload \"$service\" >/tmp/hwlab-public-caddy-reload.out 2>/tmp/hwlab-public-caddy-reload.err || sudo systemctl restart \"$service\" >>/tmp/hwlab-public-caddy-reload.out 2>>/tmp/hwlab-public-caddy-reload.err",
|
|
" reload_exit=$?",
|
|
" else",
|
|
" reload_exit=$install_exit",
|
|
" fi",
|
|
"fi",
|
|
"active=$(systemctl is-active \"$service\" 2>/dev/null || true)",
|
|
"after_present=no",
|
|
"if [ \"$dry_run\" = true ]; then grep -Fq \"# BEGIN $marker\" \"$next\" && after_present=yes; else grep -Fq \"# BEGIN $marker\" \"$config_path\" && after_present=yes; fi",
|
|
"local_web_status=",
|
|
"local_api_status=",
|
|
"if [ \"$dry_run\" = false ]; then",
|
|
" local_web_status=$(curl -kfsS --max-time 15 --resolve \"$hostname:443:127.0.0.1\" \"https://$hostname/\" -o /dev/null -w '%{http_code}' 2>/tmp/hwlab-public-caddy-web.err || true)",
|
|
" local_api_status=$(curl -kfsS --max-time 15 --resolve \"$hostname:443:127.0.0.1\" \"https://$hostname/health/live\" -o /dev/null -w '%{http_code}' 2>/tmp/hwlab-public-caddy-api.err || true)",
|
|
"fi",
|
|
"validate_err_preview=$(cat /tmp/hwlab-public-caddy-python.err /tmp/hwlab-public-caddy-validate.err 2>/dev/null | tr '\\n' ';' | cut -c1-1000 || true)",
|
|
"local_web_err_preview=$([ -f /tmp/hwlab-public-caddy-web.err ] && tr '\\n' ';' </tmp/hwlab-public-caddy-web.err 2>/dev/null | cut -c1-1000 || true)",
|
|
"local_api_err_preview=$([ -f /tmp/hwlab-public-caddy-api.err ] && tr '\\n' ';' </tmp/hwlab-public-caddy-api.err 2>/dev/null | cut -c1-1000 || true)",
|
|
"printf 'hostname\\t%s\\n' \"$hostname\"",
|
|
"printf 'expectedA\\t%s\\n' \"$expected_a\"",
|
|
"printf 'configPath\\t%s\\n' \"$config_path\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeBlockPresent\\t%s\\n' \"$before_present\"",
|
|
"printf 'afterBlockPresent\\t%s\\n' \"$after_present\"",
|
|
"printf 'staleBlocksRemoved\\t%s\\n' \"$stale_blocks_removed\"",
|
|
"printf 'pythonExitCode\\t%s\\n' \"$python_exit\"",
|
|
"printf 'validateMode\\t%s\\n' \"$validate_mode\"",
|
|
"printf 'validateExitCode\\t%s\\n' \"$validate_exit\"",
|
|
"printf 'reloadExitCode\\t%s\\n' \"$reload_exit\"",
|
|
"printf 'active\\t%s\\n' \"$active\"",
|
|
"printf 'localWebStatus\\t%s\\n' \"$local_web_status\"",
|
|
"printf 'localApiStatus\\t%s\\n' \"$local_api_status\"",
|
|
"printf 'validateErrorPreview\\t%s\\n' \"$validate_err_preview\"",
|
|
"printf 'localWebErrorPreview\\t%s\\n' \"$local_web_err_preview\"",
|
|
"printf 'localApiErrorPreview\\t%s\\n' \"$local_api_err_preview\"",
|
|
"rm -rf \"$tmp\"",
|
|
"[ \"$validate_exit\" = 0 ] && [ \"$after_present\" = yes ]",
|
|
].join("\n");
|
|
}
|
|
|
|
function publicExposureCaddyBlock(exposure: HwlabRuntimePublicExposureSpec): string {
|
|
const tlsLines = exposure.caddyTls === "internal" ? " tls internal\n" : "";
|
|
return `${exposure.hostname} {
|
|
${tlsLines} @api path /health* /auth* /v1* /json-rpc* /openapi* /docs* /swagger*
|
|
reverse_proxy @api 127.0.0.1:${exposure.apiProxy.remotePort} {
|
|
transport http {
|
|
response_header_timeout ${exposure.responseHeaderTimeoutSeconds}s
|
|
}
|
|
}
|
|
reverse_proxy 127.0.0.1:${exposure.webProxy.remotePort} {
|
|
transport http {
|
|
response_header_timeout ${exposure.responseHeaderTimeoutSeconds}s
|
|
}
|
|
}
|
|
}`;
|
|
}
|
|
|
|
function runTransScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult {
|
|
return runCommand([transPath(), `${node}:k3s`, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
|
|
}
|
|
|
|
function runTransHostScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult {
|
|
return runCommand([transPath(), node, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
|
|
}
|
|
|
|
function runTransWorkspaceStdinScript(node: string, workspace: string, script: string, timeoutSeconds: number): CommandResult {
|
|
return runCommand([transPath(), `${node}:${workspace}`, "sh"], repoRoot, { input: script, timeoutMs: timeoutSeconds * 1000 });
|
|
}
|
|
|
|
function runObsoleteSecretCleanup(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record<string, unknown> {
|
|
const kubernetesResult = runTransScript(options.node, obsoleteSecretCleanupScript(options, spec), "", options.timeoutSeconds);
|
|
const kubernetesStatus = secretStatusFromText(statusText(kubernetesResult), kubernetesResult.exitCode === 0, kubernetesResult.exitCode, kubernetesResult.stderr, spec);
|
|
const hostOptions = { ...options, dryRun: options.dryRun || kubernetesStatus.ok !== true };
|
|
const platformDbResult = runTransHostScript(options.node, obsoletePlatformDbCleanupScript(hostOptions, spec), "", options.timeoutSeconds);
|
|
const platformDbStatus = obsoletePlatformDbStatusFromText(statusText(platformDbResult), platformDbResult.exitCode === 0, platformDbResult.exitCode, platformDbResult.stderr, spec);
|
|
const ok = kubernetesStatus.ok === true && platformDbStatus.ok === true && (options.dryRun || hostOptions.dryRun === false);
|
|
const mutation = kubernetesStatus.mutation === true || platformDbStatus.mutation === true;
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes secret ${options.action}`,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
namespace: spec.namespace,
|
|
secret: options.name,
|
|
key: null,
|
|
preset: options.preset,
|
|
mode: options.dryRun ? "dry-run" : "confirmed-delete",
|
|
status: {
|
|
ok,
|
|
preset: "obsolete-hwpod-db-cleanup",
|
|
dryRun: options.dryRun,
|
|
mutation,
|
|
kubernetesSecret: kubernetesStatus,
|
|
platformDatabase: platformDbStatus,
|
|
hostMutationSkipped: !options.dryRun && hostOptions.dryRun,
|
|
summary: ok
|
|
? `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, and ${spec.obsoleteHwpodDbUser} are absent or ready to remove`
|
|
: `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, or ${spec.obsoleteHwpodDbUser} still needs cleanup`,
|
|
},
|
|
mutation,
|
|
result: {
|
|
kubernetesSecret: compactCommandResult(kubernetesResult),
|
|
platformDatabase: compactCommandResult(platformDbResult),
|
|
},
|
|
valuesRedacted: true,
|
|
next: ok ? undefined : nextSecretCommand(options, spec),
|
|
};
|
|
}
|
|
|
|
function runNodeEndpointBridge(options: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
if (options.dryRun && options.confirm) throw new Error("control-plane allow-endpoint-bridge accepts only one of --dry-run or --confirm");
|
|
const dryRun = options.dryRun || !options.confirm;
|
|
const result = runTransScript(options.node, endpointBridgeScript({ lane: options.lane, dryRun }), "", options.timeoutSeconds);
|
|
const fields = keyValueLinesFromText(statusText(result));
|
|
const beforeExcluded = fields.beforeEndpointResourcesExcluded === "yes";
|
|
const beforeIgnored = fields.beforeEndpointsIgnoreUpdates === "yes" || fields.beforeEndpointSliceIgnoreUpdates === "yes";
|
|
const afterExcluded = fields.afterEndpointResourcesExcluded === "yes";
|
|
const afterIgnored = fields.afterEndpointsIgnoreUpdates === "yes" || fields.afterEndpointSliceIgnoreUpdates === "yes";
|
|
const beforeExtraEndpointSlices = splitWhitespaceField(fields.beforeExtraEndpointSliceNames);
|
|
const afterExtraEndpointSlices = splitWhitespaceField(fields.afterExtraEndpointSliceNames);
|
|
const beforeLegacyEndpoints = fields.beforeLegacyEndpointsExists === "yes";
|
|
const afterLegacyEndpoints = fields.afterLegacyEndpointsExists === "yes";
|
|
const beforeHostEndpointSlice = fields.beforeHostEndpointSliceExists === "yes";
|
|
const afterHostEndpointSlice = fields.afterHostEndpointSliceExists === "yes";
|
|
const bridgeReady = !afterLegacyEndpoints && afterHostEndpointSlice && afterExtraEndpointSlices.length === 0;
|
|
const ok = result.exitCode === 0 && !afterExcluded && !afterIgnored && bridgeReady;
|
|
return {
|
|
ok: dryRun ? result.exitCode === 0 : ok,
|
|
command: "hwlab nodes control-plane allow-endpoint-bridge",
|
|
node: options.node,
|
|
lane: options.lane,
|
|
namespace: "argocd",
|
|
application: fields.application || `hwlab-node-${options.lane}`,
|
|
mode: dryRun ? "dry-run" : "confirmed-control-plane-update",
|
|
status: {
|
|
action: fields.action || null,
|
|
dryRun,
|
|
mutation: fields.mutation === "true",
|
|
before: {
|
|
endpointResourcesExcluded: beforeExcluded,
|
|
endpointsIgnoreUpdates: fields.beforeEndpointsIgnoreUpdates === "yes",
|
|
endpointSliceIgnoreUpdates: fields.beforeEndpointSliceIgnoreUpdates === "yes",
|
|
legacyEndpointsExist: beforeLegacyEndpoints,
|
|
hostEndpointSliceExists: beforeHostEndpointSlice,
|
|
extraEndpointSlices: beforeExtraEndpointSlices,
|
|
},
|
|
after: {
|
|
endpointResourcesExcluded: afterExcluded,
|
|
endpointsIgnoreUpdates: fields.afterEndpointsIgnoreUpdates === "yes",
|
|
endpointSliceIgnoreUpdates: fields.afterEndpointSliceIgnoreUpdates === "yes",
|
|
legacyEndpointsExist: afterLegacyEndpoints,
|
|
hostEndpointSliceExists: afterHostEndpointSlice,
|
|
extraEndpointSlices: afterExtraEndpointSlices,
|
|
},
|
|
runtimeNamespace: fields.runtimeNamespace || `hwlab-${options.lane}`,
|
|
platformService: fields.platformService || "g14-platform-postgres",
|
|
hostEndpointSlice: fields.hostEndpointSlice || "g14-platform-postgres-host",
|
|
patchExitCode: numericField(fields.patchExitCode),
|
|
rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode),
|
|
rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode),
|
|
deleteLegacyEndpointsExitCode: numericField(fields.deleteLegacyEndpointsExitCode),
|
|
deleteExtraEndpointSlicesExitCode: numericField(fields.deleteExtraEndpointSlicesExitCode),
|
|
refreshExitCode: numericField(fields.refreshExitCode),
|
|
exitCode: result.exitCode,
|
|
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000),
|
|
summary: !afterExcluded && !afterIgnored && bridgeReady
|
|
? "Argo tracks HWLAB external Postgres EndpointSlice and no legacy Endpoints remain"
|
|
: "Argo endpoint bridge is not in final Service plus EndpointSlice shape",
|
|
},
|
|
result: compactCommandResult(result),
|
|
};
|
|
}
|
|
|
|
function endpointBridgeScript(options: { lane: HwlabRuntimeLane; dryRun: boolean }): string {
|
|
const application = `hwlab-node-${options.lane}`;
|
|
const runtimeNamespace = `hwlab-${options.lane}`;
|
|
return [
|
|
"set +e",
|
|
"namespace=argocd",
|
|
`runtime_namespace=${shellQuote(runtimeNamespace)}`,
|
|
"configmap=argocd-cm",
|
|
`application=${shellQuote(application)}`,
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
"platform_service=g14-platform-postgres",
|
|
"host_endpointslice=g14-platform-postgres-host",
|
|
"preset=endpoint-bridge-resource-tracking",
|
|
"cm_data() { kubectl -n \"$namespace\" get configmap \"$configmap\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }",
|
|
"cm_has_key() { value=$(cm_data \"$1\"); [ -n \"$value\" ] && [ \"$value\" != \"<no value>\" ] && printf yes || printf no; }",
|
|
"endpoint_resources_excluded() { exclusions=$(cm_data resource.exclusions); printf '%s' \"$exclusions\" | grep -Eq '(^|[[:space:]])(Endpoints|EndpointSlice)([[:space:]]|$)' && printf yes || printf no; }",
|
|
"resource_exists() { kubectl -n \"$runtime_namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"extra_endpoint_slices() { kubectl -n \"$runtime_namespace\" get endpointslice -l \"kubernetes.io/service-name=$platform_service\" -o name 2>/dev/null | sed \"/\\/$host_endpointslice$/d\" | tr '\\n' ' ' | sed 's/[[:space:]]*$//'; }",
|
|
"wait_runtime_bridge_clean() {",
|
|
" for _ in $(seq 1 30); do",
|
|
" current_legacy=$(resource_exists endpoints \"$platform_service\")",
|
|
" current_extra=$(extra_endpoint_slices)",
|
|
" current_host=$(resource_exists endpointslice \"$host_endpointslice\")",
|
|
" if [ \"$current_legacy\" != yes ] && [ -z \"$current_extra\" ] && [ \"$current_host\" = yes ]; then return 0; fi",
|
|
" sleep 2",
|
|
" done",
|
|
" return 1",
|
|
"}",
|
|
"before_endpoint_resources_excluded=$(endpoint_resources_excluded)",
|
|
"before_endpoints_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.Endpoints')",
|
|
"before_endpoint_slice_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice')",
|
|
"before_legacy_endpoints_exists=$(resource_exists endpoints \"$platform_service\")",
|
|
"before_host_endpointslice_exists=$(resource_exists endpointslice \"$host_endpointslice\")",
|
|
"before_extra_endpoint_slice_names=$(extra_endpoint_slices)",
|
|
"needs_argo_update=false",
|
|
"if [ \"$before_endpoint_resources_excluded\" = yes ] || [ \"$before_endpoints_ignore_updates\" = yes ] || [ \"$before_endpoint_slice_ignore_updates\" = yes ]; then needs_argo_update=true; fi",
|
|
"needs_runtime_cleanup=false",
|
|
"if [ \"$before_legacy_endpoints_exists\" = yes ] || [ -n \"$before_extra_endpoint_slice_names\" ]; then needs_runtime_cleanup=true; fi",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"patch_exit=",
|
|
"rollout_restart_exit=",
|
|
"rollout_status_exit=",
|
|
"delete_legacy_endpoints_exit=",
|
|
"delete_extra_endpointslices_exit=",
|
|
"refresh_exit=",
|
|
"if [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$needs_argo_update\" = true ] && [ \"$needs_runtime_cleanup\" = true ]; then action=would-remove-old-endpoint-exclusions-and-legacy-endpoints",
|
|
" elif [ \"$needs_argo_update\" = true ]; then action=would-remove-old-endpoint-exclusions",
|
|
" elif [ \"$needs_runtime_cleanup\" = true ]; then action=would-remove-legacy-endpoints",
|
|
" else action=kept; fi",
|
|
"else",
|
|
" if [ \"$needs_argo_update\" = true ]; then",
|
|
" patch_file=$(mktemp /tmp/hwlab-argocd-endpoint-bridge.XXXXXX.json)",
|
|
" python3 - <<'PY' >\"$patch_file\"",
|
|
"import json",
|
|
"desired = '''### Internal Kubernetes resources excluded to reduce watch volume",
|
|
"- apiGroups:",
|
|
" - coordination.k8s.io",
|
|
" kinds:",
|
|
" - Lease",
|
|
"### Internal Kubernetes Authz/Authn resources excluded to reduce watched events",
|
|
"- apiGroups:",
|
|
" - authentication.k8s.io",
|
|
" - authorization.k8s.io",
|
|
" kinds:",
|
|
" - SelfSubjectReview",
|
|
" - TokenReview",
|
|
" - LocalSubjectAccessReview",
|
|
" - SelfSubjectAccessReview",
|
|
" - SelfSubjectRulesReview",
|
|
" - SubjectAccessReview",
|
|
"### Intermediate Certificate Request excluded to reduce watched events",
|
|
"- apiGroups:",
|
|
" - certificates.k8s.io",
|
|
" kinds:",
|
|
" - CertificateSigningRequest",
|
|
"- apiGroups:",
|
|
" - cert-manager.io",
|
|
" kinds:",
|
|
" - CertificateRequest",
|
|
"### Cilium internal resources excluded to reduce UI clutter",
|
|
"- apiGroups:",
|
|
" - cilium.io",
|
|
" kinds:",
|
|
" - CiliumIdentity",
|
|
" - CiliumEndpoint",
|
|
" - CiliumEndpointSlice",
|
|
"### Kyverno intermediate and reporting resources excluded to reduce watched events",
|
|
"- apiGroups:",
|
|
" - kyverno.io",
|
|
" - reports.kyverno.io",
|
|
" - wgpolicyk8s.io",
|
|
" kinds:",
|
|
" - PolicyReport",
|
|
" - ClusterPolicyReport",
|
|
" - EphemeralReport",
|
|
" - ClusterEphemeralReport",
|
|
" - AdmissionReport",
|
|
" - ClusterAdmissionReport",
|
|
" - BackgroundScanReport",
|
|
" - ClusterBackgroundScanReport",
|
|
" - UpdateRequest",
|
|
"'''",
|
|
"print(json.dumps({",
|
|
" 'data': {",
|
|
" 'resource.exclusions': desired,",
|
|
" 'resource.customizations.ignoreResourceUpdates.Endpoints': None,",
|
|
" 'resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice': None,",
|
|
" }",
|
|
"}))",
|
|
"PY",
|
|
" kubectl -n \"$namespace\" patch configmap \"$configmap\" --type merge --patch-file \"$patch_file\" >/tmp/hwlab-argocd-endpoint-bridge-patch.out 2>/tmp/hwlab-argocd-endpoint-bridge-patch.err",
|
|
" patch_exit=$?",
|
|
" rm -f \"$patch_file\"",
|
|
" if [ \"$patch_exit\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" rollout restart statefulset/argocd-application-controller >/tmp/hwlab-argocd-endpoint-bridge-rollout-restart.out 2>/tmp/hwlab-argocd-endpoint-bridge-rollout-restart.err",
|
|
" rollout_restart_exit=$?",
|
|
" if [ \"$rollout_restart_exit\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" rollout status statefulset/argocd-application-controller --timeout=180s >/tmp/hwlab-argocd-endpoint-bridge-rollout-status.out 2>/tmp/hwlab-argocd-endpoint-bridge-rollout-status.err",
|
|
" rollout_status_exit=$?",
|
|
" fi",
|
|
" fi",
|
|
" fi",
|
|
" if [ -n \"$patch_exit\" ] && [ \"$patch_exit\" != 0 ]; then action=patch-failed",
|
|
" elif [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed",
|
|
" elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed",
|
|
" else",
|
|
" if [ \"$needs_runtime_cleanup\" = true ]; then",
|
|
" kubectl -n \"$runtime_namespace\" delete endpoints \"$platform_service\" --ignore-not-found=true >/tmp/hwlab-platform-postgres-endpoints-delete.out 2>/tmp/hwlab-platform-postgres-endpoints-delete.err",
|
|
" delete_legacy_endpoints_exit=$?",
|
|
" wait_runtime_bridge_clean",
|
|
" remaining_extra=$(extra_endpoint_slices)",
|
|
" if [ -n \"$remaining_extra\" ]; then",
|
|
" kubectl -n \"$runtime_namespace\" delete $remaining_extra --ignore-not-found=true >/tmp/hwlab-platform-postgres-endpointslices-delete.out 2>/tmp/hwlab-platform-postgres-endpointslices-delete.err",
|
|
" delete_extra_endpointslices_exit=$?",
|
|
" wait_runtime_bridge_clean",
|
|
" fi",
|
|
" fi",
|
|
" if [ \"$needs_argo_update\" = true ] || [ \"$needs_runtime_cleanup\" = true ]; then",
|
|
" kubectl -n \"$namespace\" annotate application \"$application\" argocd.argoproj.io/refresh=hard --overwrite >/tmp/hwlab-argocd-endpoint-bridge-refresh.out 2>/tmp/hwlab-argocd-endpoint-bridge-refresh.err",
|
|
" refresh_exit=$?",
|
|
" fi",
|
|
" if [ -n \"$delete_legacy_endpoints_exit\" ] && [ \"$delete_legacy_endpoints_exit\" != 0 ]; then action=delete-legacy-endpoints-failed",
|
|
" elif [ -n \"$delete_extra_endpointslices_exit\" ] && [ \"$delete_extra_endpointslices_exit\" != 0 ]; then action=delete-extra-endpointslices-failed",
|
|
" elif [ -n \"$refresh_exit\" ] && [ \"$refresh_exit\" != 0 ]; then action=refresh-failed",
|
|
" elif [ \"$needs_argo_update\" = true ] && [ \"$needs_runtime_cleanup\" = true ]; then action=removed-old-endpoint-exclusions-and-legacy-endpoints; mutation=true",
|
|
" elif [ \"$needs_argo_update\" = true ]; then action=removed-old-endpoint-exclusions; mutation=true",
|
|
" elif [ \"$needs_runtime_cleanup\" = true ]; then action=removed-legacy-endpoints; mutation=true",
|
|
" else action=kept; fi",
|
|
" fi",
|
|
"fi",
|
|
"after_endpoint_resources_excluded=$(endpoint_resources_excluded)",
|
|
"after_endpoints_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.Endpoints')",
|
|
"after_endpoint_slice_ignore_updates=$(cm_has_key 'resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice')",
|
|
"after_legacy_endpoints_exists=$(resource_exists endpoints \"$platform_service\")",
|
|
"after_host_endpointslice_exists=$(resource_exists endpointslice \"$host_endpointslice\")",
|
|
"after_extra_endpoint_slice_names=$(extra_endpoint_slices)",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'runtimeNamespace\\t%s\\n' \"$runtime_namespace\"",
|
|
"printf 'configMap\\t%s\\n' \"$configmap\"",
|
|
"printf 'application\\t%s\\n' \"$application\"",
|
|
"printf 'platformService\\t%s\\n' \"$platform_service\"",
|
|
"printf 'hostEndpointSlice\\t%s\\n' \"$host_endpointslice\"",
|
|
"printf 'preset\\t%s\\n' \"$preset\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeEndpointResourcesExcluded\\t%s\\n' \"$before_endpoint_resources_excluded\"",
|
|
"printf 'beforeEndpointsIgnoreUpdates\\t%s\\n' \"$before_endpoints_ignore_updates\"",
|
|
"printf 'beforeEndpointSliceIgnoreUpdates\\t%s\\n' \"$before_endpoint_slice_ignore_updates\"",
|
|
"printf 'beforeLegacyEndpointsExists\\t%s\\n' \"$before_legacy_endpoints_exists\"",
|
|
"printf 'beforeHostEndpointSliceExists\\t%s\\n' \"$before_host_endpointslice_exists\"",
|
|
"printf 'beforeExtraEndpointSliceNames\\t%s\\n' \"$before_extra_endpoint_slice_names\"",
|
|
"printf 'afterEndpointResourcesExcluded\\t%s\\n' \"$after_endpoint_resources_excluded\"",
|
|
"printf 'afterEndpointsIgnoreUpdates\\t%s\\n' \"$after_endpoints_ignore_updates\"",
|
|
"printf 'afterEndpointSliceIgnoreUpdates\\t%s\\n' \"$after_endpoint_slice_ignore_updates\"",
|
|
"printf 'afterLegacyEndpointsExists\\t%s\\n' \"$after_legacy_endpoints_exists\"",
|
|
"printf 'afterHostEndpointSliceExists\\t%s\\n' \"$after_host_endpointslice_exists\"",
|
|
"printf 'afterExtraEndpointSliceNames\\t%s\\n' \"$after_extra_endpoint_slice_names\"",
|
|
"printf 'patchExitCode\\t%s\\n' \"$patch_exit\"",
|
|
"printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"",
|
|
"printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"",
|
|
"printf 'deleteLegacyEndpointsExitCode\\t%s\\n' \"$delete_legacy_endpoints_exit\"",
|
|
"printf 'deleteExtraEndpointSlicesExitCode\\t%s\\n' \"$delete_extra_endpointslices_exit\"",
|
|
"printf 'refreshExitCode\\t%s\\n' \"$refresh_exit\"",
|
|
"if [ \"$dry_run\" != true ] && { [ \"$after_endpoint_resources_excluded\" = yes ] || [ \"$after_endpoints_ignore_updates\" = yes ] || [ \"$after_endpoint_slice_ignore_updates\" = yes ]; }; then exit 46; fi",
|
|
"if [ \"$dry_run\" != true ] && { [ \"$after_legacy_endpoints_exists\" = yes ] || [ -n \"$after_extra_endpoint_slice_names\" ] || [ \"$after_host_endpointslice_exists\" != yes ]; }; then exit 47; fi",
|
|
"if [ -n \"$patch_exit\" ] && [ \"$patch_exit\" != 0 ]; then exit \"$patch_exit\"; fi",
|
|
"if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi",
|
|
"if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi",
|
|
"if [ -n \"$delete_legacy_endpoints_exit\" ] && [ \"$delete_legacy_endpoints_exit\" != 0 ]; then exit \"$delete_legacy_endpoints_exit\"; fi",
|
|
"if [ -n \"$delete_extra_endpointslices_exit\" ] && [ \"$delete_extra_endpointslices_exit\" != 0 ]; then exit \"$delete_extra_endpointslices_exit\"; fi",
|
|
"if [ -n \"$refresh_exit\" ] && [ \"$refresh_exit\" != 0 ]; then exit \"$refresh_exit\"; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function ownedPostgresCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
|
|
const pvc = `data-${spec.postgresSecret}-0`;
|
|
const platformService = spec.platformPostgresService;
|
|
const postgresService = spec.postgresSecret;
|
|
const postgresConfigMap = `${spec.postgresSecret}-init`;
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(spec.namespace)}`,
|
|
`postgres_secret=${shellQuote(spec.postgresSecret)}`,
|
|
`postgres_statefulset=${shellQuote(spec.postgresStatefulSet)}`,
|
|
`postgres_service=${shellQuote(postgresService)}`,
|
|
`postgres_configmap=${shellQuote(postgresConfigMap)}`,
|
|
`pvc=${shellQuote(pvc)}`,
|
|
`platform_service=${shellQuote(platformService)}`,
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
"preset=owned-postgres-cleanup",
|
|
"exists_flag() { kind=\"$1\"; item=\"$2\"; kubectl -n \"$namespace\" get \"$kind\" \"$item\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"pv_name() { kubectl -n \"$namespace\" get pvc \"$pvc\" -o jsonpath='{.spec.volumeName}' 2>/dev/null; }",
|
|
"phase_of_pvc() { kubectl -n \"$namespace\" get pvc \"$pvc\" -o jsonpath='{.status.phase}' 2>/dev/null; }",
|
|
"before_secret_exists=$(exists_flag secret \"$postgres_secret\")",
|
|
"before_pvc_exists=$(exists_flag pvc \"$pvc\")",
|
|
"before_pvc_phase=$(phase_of_pvc)",
|
|
"before_pv=$(pv_name)",
|
|
"before_statefulset_exists=$(exists_flag statefulset \"$postgres_statefulset\")",
|
|
"before_service_exists=$(exists_flag service \"$postgres_service\")",
|
|
"before_configmap_exists=$(exists_flag configmap \"$postgres_configmap\")",
|
|
"platform_service_exists=$(exists_flag service \"$platform_service\")",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"delete_statefulset_exit=",
|
|
"delete_service_exit=",
|
|
"delete_configmap_exit=",
|
|
"delete_secret_exit=",
|
|
"delete_pvc_exit=",
|
|
"before_any_owned=false",
|
|
"for flag in \"$before_statefulset_exists\" \"$before_service_exists\" \"$before_configmap_exists\" \"$before_secret_exists\" \"$before_pvc_exists\"; do",
|
|
" if [ \"$flag\" = yes ]; then before_any_owned=true; fi",
|
|
"done",
|
|
"if [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$before_any_owned\" = true ]; then action=would-delete; else action=already-absent; fi",
|
|
"else",
|
|
" kubectl -n \"$namespace\" delete statefulset \"$postgres_statefulset\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-statefulset-delete.out 2>/tmp/hwlab-owned-postgres-statefulset-delete.err",
|
|
" delete_statefulset_exit=$?",
|
|
" kubectl -n \"$namespace\" delete service \"$postgres_service\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-service-delete.out 2>/tmp/hwlab-owned-postgres-service-delete.err",
|
|
" delete_service_exit=$?",
|
|
" kubectl -n \"$namespace\" delete configmap \"$postgres_configmap\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-configmap-delete.out 2>/tmp/hwlab-owned-postgres-configmap-delete.err",
|
|
" delete_configmap_exit=$?",
|
|
" kubectl -n \"$namespace\" delete secret \"$postgres_secret\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-secret-delete.out 2>/tmp/hwlab-owned-postgres-secret-delete.err",
|
|
" delete_secret_exit=$?",
|
|
" kubectl -n \"$namespace\" delete pvc \"$pvc\" --ignore-not-found=true >/tmp/hwlab-owned-postgres-pvc-delete.out 2>/tmp/hwlab-owned-postgres-pvc-delete.err",
|
|
" delete_pvc_exit=$?",
|
|
" for _ in $(seq 1 30); do",
|
|
" current_statefulset=$(exists_flag statefulset \"$postgres_statefulset\")",
|
|
" current_service=$(exists_flag service \"$postgres_service\")",
|
|
" current_configmap=$(exists_flag configmap \"$postgres_configmap\")",
|
|
" current_secret=$(exists_flag secret \"$postgres_secret\")",
|
|
" current_pvc=$(exists_flag pvc \"$pvc\")",
|
|
" if [ \"$current_statefulset\" != yes ] && [ \"$current_service\" != yes ] && [ \"$current_configmap\" != yes ] && [ \"$current_secret\" != yes ] && [ \"$current_pvc\" != yes ]; then break; fi",
|
|
" sleep 2",
|
|
" done",
|
|
" if [ \"$delete_statefulset_exit\" -eq 0 ] && [ \"$delete_service_exit\" -eq 0 ] && [ \"$delete_configmap_exit\" -eq 0 ] && [ \"$delete_secret_exit\" -eq 0 ] && [ \"$delete_pvc_exit\" -eq 0 ]; then",
|
|
" if [ \"$before_any_owned\" = true ]; then action=deleted; mutation=true; else action=already-absent; fi",
|
|
" else",
|
|
" action=delete-failed",
|
|
" fi",
|
|
"fi",
|
|
"after_secret_exists=$(exists_flag secret \"$postgres_secret\")",
|
|
"after_pvc_exists=$(exists_flag pvc \"$pvc\")",
|
|
"after_pvc_phase=$(phase_of_pvc)",
|
|
"after_pv=$(pv_name)",
|
|
"after_statefulset_exists=$(exists_flag statefulset \"$postgres_statefulset\")",
|
|
"after_service_exists=$(exists_flag service \"$postgres_service\")",
|
|
"after_configmap_exists=$(exists_flag configmap \"$postgres_configmap\")",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$postgres_secret\"",
|
|
"printf 'statefulSet\\t%s\\n' \"$postgres_statefulset\"",
|
|
"printf 'service\\t%s\\n' \"$postgres_service\"",
|
|
"printf 'configMap\\t%s\\n' \"$postgres_configmap\"",
|
|
"printf 'pvc\\t%s\\n' \"$pvc\"",
|
|
"printf 'preset\\t%s\\n' \"$preset\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"",
|
|
"printf 'beforePvcExists\\t%s\\n' \"$before_pvc_exists\"",
|
|
"printf 'beforePvcPhase\\t%s\\n' \"$before_pvc_phase\"",
|
|
"printf 'beforePersistentVolume\\t%s\\n' \"$before_pv\"",
|
|
"printf 'beforeStatefulSetExists\\t%s\\n' \"$before_statefulset_exists\"",
|
|
"printf 'beforeServiceExists\\t%s\\n' \"$before_service_exists\"",
|
|
"printf 'beforeConfigMapExists\\t%s\\n' \"$before_configmap_exists\"",
|
|
"printf 'platformServiceExists\\t%s\\n' \"$platform_service_exists\"",
|
|
"printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"",
|
|
"printf 'afterPvcExists\\t%s\\n' \"$after_pvc_exists\"",
|
|
"printf 'afterPvcPhase\\t%s\\n' \"$after_pvc_phase\"",
|
|
"printf 'afterPersistentVolume\\t%s\\n' \"$after_pv\"",
|
|
"printf 'afterStatefulSetExists\\t%s\\n' \"$after_statefulset_exists\"",
|
|
"printf 'afterServiceExists\\t%s\\n' \"$after_service_exists\"",
|
|
"printf 'afterConfigMapExists\\t%s\\n' \"$after_configmap_exists\"",
|
|
"printf 'deleteStatefulSetExitCode\\t%s\\n' \"$delete_statefulset_exit\"",
|
|
"printf 'deleteServiceExitCode\\t%s\\n' \"$delete_service_exit\"",
|
|
"printf 'deleteConfigMapExitCode\\t%s\\n' \"$delete_configmap_exit\"",
|
|
"printf 'deleteSecretExitCode\\t%s\\n' \"$delete_secret_exit\"",
|
|
"printf 'deletePvcExitCode\\t%s\\n' \"$delete_pvc_exit\"",
|
|
"if [ \"$platform_service_exists\" != yes ]; then exit 44; fi",
|
|
"if [ \"$after_statefulset_exists\" = yes ] || [ \"$after_service_exists\" = yes ] || [ \"$after_configmap_exists\" = yes ] || [ \"$after_secret_exists\" = yes ] || [ \"$after_pvc_exists\" = yes ]; then exit 45; fi",
|
|
"if [ -n \"$delete_statefulset_exit\" ] && [ \"$delete_statefulset_exit\" != 0 ]; then exit \"$delete_statefulset_exit\"; fi",
|
|
"if [ -n \"$delete_service_exit\" ] && [ \"$delete_service_exit\" != 0 ]; then exit \"$delete_service_exit\"; fi",
|
|
"if [ -n \"$delete_configmap_exit\" ] && [ \"$delete_configmap_exit\" != 0 ]; then exit \"$delete_configmap_exit\"; fi",
|
|
"if [ -n \"$delete_secret_exit\" ] && [ \"$delete_secret_exit\" != 0 ]; then exit \"$delete_secret_exit\"; fi",
|
|
"if [ -n \"$delete_pvc_exit\" ] && [ \"$delete_pvc_exit\" != 0 ]; then exit \"$delete_pvc_exit\"; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function obsoleteSecretCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(spec.namespace)}`,
|
|
`secret=${shellQuote(options.name)}`,
|
|
`expected_secret=${shellQuote(spec.obsoleteHwpodDbSecret)}`,
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
"preset=obsolete-secret-cleanup",
|
|
"exists_flag() { kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"workload_refs=$(kubectl -n \"$namespace\" get deploy,statefulset,daemonset,job,cronjob -o yaml 2>/dev/null | grep -n -C 2 \"$secret\" || true)",
|
|
"workload_refs_present=$([ -n \"$workload_refs\" ] && printf yes || printf no)",
|
|
"workload_refs_preview=$(printf '%s' \"$workload_refs\" | sed -n '1,20p' | tr '\\n' ';' | cut -c1-1000)",
|
|
"before_exists=$(exists_flag)",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"delete_secret_exit=",
|
|
"if [ \"$secret\" != \"$expected_secret\" ]; then action=unsupported-secret; fi",
|
|
"if [ \"$action\" = observed ]; then",
|
|
" if [ \"$workload_refs_present\" = yes ]; then",
|
|
" action=blocked-referenced",
|
|
" elif [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$before_exists\" = yes ]; then action=would-delete; else action=already-absent; fi",
|
|
" else",
|
|
" kubectl -n \"$namespace\" delete secret \"$secret\" --ignore-not-found=true >/tmp/hwlab-obsolete-secret-delete.out 2>/tmp/hwlab-obsolete-secret-delete.err",
|
|
" delete_secret_exit=$?",
|
|
" for _ in $(seq 1 15); do",
|
|
" current_exists=$(exists_flag)",
|
|
" if [ \"$current_exists\" != yes ]; then break; fi",
|
|
" sleep 1",
|
|
" done",
|
|
" if [ \"$delete_secret_exit\" -eq 0 ]; then",
|
|
" if [ \"$before_exists\" = yes ]; then action=deleted; mutation=true; else action=already-absent; fi",
|
|
" else",
|
|
" action=delete-failed",
|
|
" fi",
|
|
" fi",
|
|
"fi",
|
|
"after_exists=$(exists_flag)",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$secret\"",
|
|
"printf 'preset\\t%s\\n' \"$preset\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeSecretExists\\t%s\\n' \"$before_exists\"",
|
|
"printf 'afterSecretExists\\t%s\\n' \"$after_exists\"",
|
|
"printf 'workloadRefsPresent\\t%s\\n' \"$workload_refs_present\"",
|
|
"printf 'workloadRefsPreview\\t%s\\n' \"$workload_refs_preview\"",
|
|
"printf 'deleteSecretExitCode\\t%s\\n' \"$delete_secret_exit\"",
|
|
"if [ \"$action\" = unsupported-secret ]; then exit 43; fi",
|
|
"if [ \"$workload_refs_present\" = yes ]; then exit 46; fi",
|
|
"if [ \"$dry_run\" != true ] && [ \"$after_exists\" = yes ]; then exit 47; fi",
|
|
"if [ -n \"$delete_secret_exit\" ] && [ \"$delete_secret_exit\" != 0 ]; then exit \"$delete_secret_exit\"; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function obsoletePlatformDbCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
|
|
return [
|
|
"set +e",
|
|
`db_name=${shellQuote(spec.obsoleteHwpodDbName)}`,
|
|
`db_user=${shellQuote(spec.obsoleteHwpodDbUser)}`,
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
"preset=obsolete-platform-db-cleanup",
|
|
"database_exists_flag() {",
|
|
" output=$(sudo -u postgres psql -d postgres -Atqc \"select exists(select 1 from pg_database where datname='$db_name');\" 2>/tmp/hwlab-obsolete-platform-db-probe.err)",
|
|
" code=$?",
|
|
" if [ \"$code\" -ne 0 ]; then printf unknown; return \"$code\"; fi",
|
|
" [ \"$output\" = t ] && printf yes || printf no",
|
|
"}",
|
|
"role_exists_flag() {",
|
|
" output=$(sudo -u postgres psql -d postgres -Atqc \"select exists(select 1 from pg_roles where rolname='$db_user');\" 2>/tmp/hwlab-obsolete-platform-role-probe.err)",
|
|
" code=$?",
|
|
" if [ \"$code\" -ne 0 ]; then printf unknown; return \"$code\"; fi",
|
|
" [ \"$output\" = t ] && printf yes || printf no",
|
|
"}",
|
|
"before_database_exists=$(database_exists_flag)",
|
|
"before_database_probe_exit=$?",
|
|
"before_role_exists=$(role_exists_flag)",
|
|
"before_role_probe_exit=$?",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"drop_database_exit=",
|
|
"drop_role_exit=",
|
|
"before_any=false",
|
|
"if [ \"$before_database_exists\" = yes ] || [ \"$before_role_exists\" = yes ]; then before_any=true; fi",
|
|
"if [ \"$before_database_exists\" = unknown ] || [ \"$before_role_exists\" = unknown ]; then",
|
|
" action=probe-failed",
|
|
"elif [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$before_any\" = true ]; then action=would-drop; else action=already-absent; fi",
|
|
"else",
|
|
" if [ \"$before_database_exists\" = yes ]; then",
|
|
" sudo -u postgres psql -v ON_ERROR_STOP=1 -d postgres -v db_name=\"$db_name\" >/tmp/hwlab-obsolete-platform-db-drop.out 2>/tmp/hwlab-obsolete-platform-db-drop.err <<'SQL'",
|
|
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = :'db_name' AND pid <> pg_backend_pid();",
|
|
"DROP DATABASE IF EXISTS :\"db_name\";",
|
|
"SQL",
|
|
" drop_database_exit=$?",
|
|
" else",
|
|
" drop_database_exit=0",
|
|
" fi",
|
|
" if [ \"$drop_database_exit\" -eq 0 ] && [ \"$before_role_exists\" = yes ]; then",
|
|
" sudo -u postgres psql -v ON_ERROR_STOP=1 -d postgres -v db_user=\"$db_user\" >/tmp/hwlab-obsolete-platform-role-drop.out 2>/tmp/hwlab-obsolete-platform-role-drop.err <<'SQL'",
|
|
"DROP ROLE IF EXISTS :\"db_user\";",
|
|
"SQL",
|
|
" drop_role_exit=$?",
|
|
" elif [ \"$drop_database_exit\" -eq 0 ]; then",
|
|
" drop_role_exit=0",
|
|
" else",
|
|
" drop_role_exit=",
|
|
" fi",
|
|
" if [ \"$drop_database_exit\" = 0 ] && [ \"$drop_role_exit\" = 0 ]; then",
|
|
" if [ \"$before_any\" = true ]; then action=dropped; mutation=true; else action=already-absent; fi",
|
|
" else",
|
|
" action=drop-failed",
|
|
" fi",
|
|
"fi",
|
|
"after_database_exists=$(database_exists_flag)",
|
|
"after_database_probe_exit=$?",
|
|
"after_role_exists=$(role_exists_flag)",
|
|
"after_role_probe_exit=$?",
|
|
"printf 'database\\t%s\\n' \"$db_name\"",
|
|
"printf 'role\\t%s\\n' \"$db_user\"",
|
|
"printf 'preset\\t%s\\n' \"$preset\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeDatabaseExists\\t%s\\n' \"$before_database_exists\"",
|
|
"printf 'beforeRoleExists\\t%s\\n' \"$before_role_exists\"",
|
|
"printf 'afterDatabaseExists\\t%s\\n' \"$after_database_exists\"",
|
|
"printf 'afterRoleExists\\t%s\\n' \"$after_role_exists\"",
|
|
"printf 'beforeDatabaseProbeExitCode\\t%s\\n' \"$before_database_probe_exit\"",
|
|
"printf 'beforeRoleProbeExitCode\\t%s\\n' \"$before_role_probe_exit\"",
|
|
"printf 'afterDatabaseProbeExitCode\\t%s\\n' \"$after_database_probe_exit\"",
|
|
"printf 'afterRoleProbeExitCode\\t%s\\n' \"$after_role_probe_exit\"",
|
|
"printf 'dropDatabaseExitCode\\t%s\\n' \"$drop_database_exit\"",
|
|
"printf 'dropRoleExitCode\\t%s\\n' \"$drop_role_exit\"",
|
|
"if [ \"$before_database_exists\" = unknown ] || [ \"$before_role_exists\" = unknown ] || [ \"$after_database_exists\" = unknown ] || [ \"$after_role_exists\" = unknown ]; then exit 49; fi",
|
|
"if [ \"$dry_run\" != true ] && { [ \"$after_database_exists\" = yes ] || [ \"$after_role_exists\" = yes ]; }; then exit 50; fi",
|
|
"if [ -n \"$drop_database_exit\" ] && [ \"$drop_database_exit\" != 0 ]; then exit \"$drop_database_exit\"; fi",
|
|
"if [ -n \"$drop_role_exit\" ] && [ \"$drop_role_exit\" != 0 ]; then exit \"$drop_role_exit\"; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function platformDbSecretStatusScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
|
|
const isOpenFga = options.preset === "openfga";
|
|
const platformEndpointSlice = spec.platformPostgresEndpointSlice;
|
|
const expectedUriHost = spec.platformPostgresEndpointAddress ?? (isOpenFga ? spec.openFgaDbHost : spec.cloudApiDbHost);
|
|
const databaseUrlKey = isOpenFga ? spec.externalPostgres?.openfga.secretKey ?? OPENFGA_DATASTORE_URI_KEY : spec.cloudApiDbKey;
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(spec.namespace)}`,
|
|
`name=${shellQuote(isOpenFga ? spec.openFgaSecret : spec.cloudApiDbSecret)}`,
|
|
`database_url_key=${shellQuote(databaseUrlKey)}`,
|
|
`authn_key=${shellQuote(OPENFGA_AUTHN_KEY)}`,
|
|
`postgres_password_key=${shellQuote(OPENFGA_POSTGRES_PASSWORD_KEY)}`,
|
|
`legacy_postgres_secret=${shellQuote(spec.postgresSecret)}`,
|
|
`platform_service=${shellQuote(spec.platformPostgresService)}`,
|
|
`platform_endpointslice=${shellQuote(platformEndpointSlice)}`,
|
|
`platform_host=${shellQuote(spec.platformPostgresService)}`,
|
|
`platform_host_fqdn=${shellQuote(spec.openFgaDbHost)}`,
|
|
`platform_endpoint_address=${shellQuote(spec.platformPostgresEndpointAddress ?? "")}`,
|
|
`db_name=${shellQuote(isOpenFga ? spec.openFgaDbName : spec.cloudApiDbName)}`,
|
|
`db_user=${shellQuote(isOpenFga ? spec.openFgaDbUser : spec.cloudApiDbUser)}`,
|
|
`db_host=${shellQuote(expectedUriHost)}`,
|
|
`selected_key=${shellQuote(options.key ?? "")}`,
|
|
`preset=${shellQuote(options.preset)}`,
|
|
"dry_run=true",
|
|
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$1\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"resource_exists_flag() { kubectl -n \"$namespace\" get \"$1\" \"$2\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"endpointslice_exists_flag() { kubectl -n \"$namespace\" get endpointslice \"$1\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }",
|
|
"decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }",
|
|
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
|
|
"uri_has_platform_host=no",
|
|
"uri_has_db_name=no",
|
|
"uri_has_db_user=no",
|
|
"uri_matches_expected() {",
|
|
" uri=$1",
|
|
" uri_has_platform_host=no",
|
|
" uri_has_db_name=no",
|
|
" uri_has_db_user=no",
|
|
" case \"$uri\" in *\"@$platform_host:\"*|*\"@$platform_host/\"*|*\"@$platform_host_fqdn:\"*|*\"@$platform_host_fqdn/\"*|*\"@$db_host:\"*|*\"@$db_host/\"*) uri_has_platform_host=yes ;; esac",
|
|
" if [ -n \"$platform_endpoint_address\" ]; then",
|
|
" case \"$uri\" in *\"@$platform_endpoint_address:\"*|*\"@$platform_endpoint_address/\"*) uri_has_platform_host=yes ;; esac",
|
|
" fi",
|
|
" case \"$uri\" in *\"/$db_name\"|*\"/$db_name?\"*|*\"/$db_name?\"*) uri_has_db_name=yes ;; esac",
|
|
" case \"$uri\" in postgres://$db_user:*|postgresql://$db_user:*) uri_has_db_user=yes ;; esac",
|
|
"}",
|
|
"exists=$(secret_exists_flag \"$name\")",
|
|
"legacy_postgres_exists=$(secret_exists_flag \"$legacy_postgres_secret\")",
|
|
"uri_b64=$(secret_b64_key \"$name\" \"$database_url_key\")",
|
|
"uri_present=$([ -n \"$uri_b64\" ] && printf yes || printf no)",
|
|
"uri_bytes=$(decoded_length \"$uri_b64\")",
|
|
"uri_value=$(decoded_value \"$uri_b64\")",
|
|
"authn_b64=$(secret_b64_key \"$name\" \"$authn_key\")",
|
|
"authn_present=$([ -n \"$authn_b64\" ] && printf yes || printf no)",
|
|
"authn_bytes=$(decoded_length \"$authn_b64\")",
|
|
"pg_password_b64=$(secret_b64_key \"$name\" \"$postgres_password_key\")",
|
|
"pg_password_present=$([ -n \"$pg_password_b64\" ] && printf yes || printf no)",
|
|
"pg_password_bytes=$(decoded_length \"$pg_password_b64\")",
|
|
"platform_service_exists=$(resource_exists_flag service \"$platform_service\")",
|
|
"platform_endpoints_exists=$(resource_exists_flag endpoints \"$platform_service\")",
|
|
"platform_endpointslice_exists=$(endpointslice_exists_flag \"$platform_endpointslice\")",
|
|
"uri_matches_expected \"$uri_value\"",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$name\"",
|
|
"printf 'key\\t%s\\n' \"$database_url_key\"",
|
|
"printf 'preset\\t%s\\n' \"$preset\"",
|
|
"printf 'action\\tobserved\\n'",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\tfalse\\n'",
|
|
"printf 'platformDbMode\\ttrue\\n'",
|
|
"printf 'afterExists\\t%s\\n' \"$exists\"",
|
|
"printf 'afterDatabaseUrlPresent\\t%s\\n' \"$uri_present\"",
|
|
"printf 'afterDatabaseUrlBytes\\t%s\\n' \"$uri_bytes\"",
|
|
"printf 'afterDatastoreUriPresent\\t%s\\n' \"$uri_present\"",
|
|
"printf 'afterDatastoreUriBytes\\t%s\\n' \"$uri_bytes\"",
|
|
"printf 'afterAuthnPresent\\t%s\\n' \"$authn_present\"",
|
|
"printf 'afterAuthnBytes\\t%s\\n' \"$authn_bytes\"",
|
|
"printf 'afterPostgresPasswordPresent\\t%s\\n' \"$pg_password_present\"",
|
|
"printf 'afterPostgresPasswordBytes\\t%s\\n' \"$pg_password_bytes\"",
|
|
"printf 'legacyPostgresSecret\\t%s\\n' \"$legacy_postgres_secret\"",
|
|
"printf 'legacyPostgresSecretExists\\t%s\\n' \"$legacy_postgres_exists\"",
|
|
"printf 'afterPostgresSecretExists\\t%s\\n' \"$legacy_postgres_exists\"",
|
|
"printf 'platformService\\t%s\\n' \"$platform_service\"",
|
|
"printf 'platformServiceExists\\t%s\\n' \"$platform_service_exists\"",
|
|
"printf 'platformEndpointsExists\\t%s\\n' \"$platform_endpoints_exists\"",
|
|
"printf 'platformEndpointSlice\\t%s\\n' \"$platform_endpointslice\"",
|
|
"printf 'platformEndpointSliceExists\\t%s\\n' \"$platform_endpointslice_exists\"",
|
|
"printf 'platformEndpointAddress\\t%s\\n' \"$platform_endpoint_address\"",
|
|
"printf 'dbName\\t%s\\n' \"$db_name\"",
|
|
"printf 'dbUser\\t%s\\n' \"$db_user\"",
|
|
"printf 'dbHost\\t%s\\n' \"$db_host\"",
|
|
"printf 'dbHostMatchesPlatform\\t%s\\n' \"$uri_has_platform_host\"",
|
|
"printf 'dbNameMatchesExpected\\t%s\\n' \"$uri_has_db_name\"",
|
|
"printf 'dbUserMatchesExpected\\t%s\\n' \"$uri_has_db_user\"",
|
|
"uri_value=",
|
|
"if [ \"$platform_service_exists\" != yes ]; then exit 44; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function openFgaSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(spec.namespace)}`,
|
|
`openfga_secret=${shellQuote(spec.openFgaSecret)}`,
|
|
`postgres_secret=${shellQuote(spec.postgresSecret)}`,
|
|
`postgres_statefulset=${shellQuote(spec.postgresStatefulSet)}`,
|
|
`postgres_admin_user=${shellQuote(spec.postgresAdminUser)}`,
|
|
`selected_key=${shellQuote(options.key ?? "")}`,
|
|
`authn_key=${shellQuote(OPENFGA_AUTHN_KEY)}`,
|
|
`datastore_uri_key=${shellQuote(OPENFGA_DATASTORE_URI_KEY)}`,
|
|
`postgres_password_key=${shellQuote(OPENFGA_POSTGRES_PASSWORD_KEY)}`,
|
|
`db_name=${shellQuote(spec.openFgaDbName)}`,
|
|
`db_user=${shellQuote(spec.openFgaDbUser)}`,
|
|
`db_host=${shellQuote(spec.openFgaDbHost)}`,
|
|
`action_request=${shellQuote(options.action)}`,
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
`field_manager=${shellQuote(spec.fieldManager)}`,
|
|
"preset=openfga",
|
|
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$1\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }",
|
|
"decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }",
|
|
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
|
|
"parse_database_url() {",
|
|
" uri=$1",
|
|
" uri_host= uri_user= uri_database= uri_sslmode= uri_password_present=no",
|
|
" if [ -n \"$uri\" ]; then",
|
|
" rest=${uri#*://}",
|
|
" rest_no_query=${rest%%\\?*}",
|
|
" auth=${rest_no_query%@*}",
|
|
" host_path=${rest_no_query#*@}",
|
|
" if [ \"$host_path\" != \"$rest_no_query\" ]; then",
|
|
" uri_user=${auth%%:*}",
|
|
" if [ \"$auth\" != \"$uri_user\" ]; then uri_password_present=yes; fi",
|
|
" else",
|
|
" host_path=$rest_no_query",
|
|
" fi",
|
|
" host_port=${host_path%%/*}",
|
|
" uri_host=${host_port%%:*}",
|
|
" uri_database=${host_path#*/}",
|
|
" if [ \"$uri_database\" = \"$host_path\" ]; then uri_database=; fi",
|
|
" uri_database=${uri_database%%\\?*}",
|
|
" case \"$uri\" in",
|
|
" *sslmode=*) uri_sslmode=${uri#*sslmode=}; uri_sslmode=${uri_sslmode%%\\&*} ;;",
|
|
" esac",
|
|
" fi",
|
|
"}",
|
|
"psql_scalar() { kubectl -n \"$namespace\" exec \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -U \"$postgres_admin_user\" -d postgres -tAc \"$1\" 2>/dev/null | tr -d '[:space:]'; }",
|
|
"probe_db() {",
|
|
" role_result=unknown",
|
|
" database_result=unknown",
|
|
" probe_exit=missing-postgres-admin-secret",
|
|
" if [ -n \"$postgres_admin_password\" ]; then",
|
|
" role_result=$(psql_scalar \"select exists(select 1 from pg_roles where rolname='$db_user');\")",
|
|
" role_exit=$?",
|
|
" database_result=$(psql_scalar \"select exists(select 1 from pg_database where datname='$db_name');\")",
|
|
" database_exit=$?",
|
|
" if [ \"$role_exit\" -eq 0 ] && [ \"$database_exit\" -eq 0 ]; then probe_exit=0; else probe_exit=$role_exit/$database_exit; fi",
|
|
" fi",
|
|
"}",
|
|
"before_exists=$(secret_exists_flag \"$openfga_secret\")",
|
|
"before_postgres_exists=$(secret_exists_flag \"$postgres_secret\")",
|
|
"before_authn_b64=$(secret_b64_key \"$openfga_secret\" \"$authn_key\")",
|
|
"before_uri_b64=$(secret_b64_key \"$openfga_secret\" \"$datastore_uri_key\")",
|
|
"before_pg_password_b64=$(secret_b64_key \"$openfga_secret\" \"$postgres_password_key\")",
|
|
"postgres_admin_b64=$(secret_b64_key \"$postgres_secret\" POSTGRES_PASSWORD)",
|
|
"before_authn_present=$([ -n \"$before_authn_b64\" ] && printf yes || printf no)",
|
|
"before_uri_present=$([ -n \"$before_uri_b64\" ] && printf yes || printf no)",
|
|
"before_pg_password_present=$([ -n \"$before_pg_password_b64\" ] && printf yes || printf no)",
|
|
"before_authn_bytes=$(decoded_length \"$before_authn_b64\")",
|
|
"before_uri_bytes=$(decoded_length \"$before_uri_b64\")",
|
|
"before_pg_password_bytes=$(decoded_length \"$before_pg_password_b64\")",
|
|
"authn_value=$(decoded_value \"$before_authn_b64\")",
|
|
"datastore_uri=$(decoded_value \"$before_uri_b64\")",
|
|
"parse_database_url \"$datastore_uri\"",
|
|
"before_uri_host=$uri_host",
|
|
"before_uri_user=$uri_user",
|
|
"before_uri_database=$uri_database",
|
|
"before_uri_sslmode=$uri_sslmode",
|
|
"before_uri_password_present=$uri_password_present",
|
|
"pg_password=$(decoded_value \"$before_pg_password_b64\")",
|
|
"postgres_admin_password=$(decoded_value \"$postgres_admin_b64\")",
|
|
"probe_db",
|
|
"db_role_exists_before=$role_result",
|
|
"db_database_exists_before=$database_result",
|
|
"db_probe_exit_before=$probe_exit",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"postgres_secret_exit=",
|
|
"postgres_rollout_exit=",
|
|
"apply_exit=",
|
|
"db_ensure_exit=",
|
|
"rollout_restart_exit=",
|
|
"rollout_status_exit=",
|
|
"openfga_deployment=hwlab-openfga",
|
|
"openfga_image=",
|
|
"migrate_job=",
|
|
"migrate_apply_exit=",
|
|
"migrate_wait_exit=",
|
|
"if [ \"$action_request\" = ensure ]; then",
|
|
" missing_secret=false",
|
|
" [ \"$before_authn_present\" = yes ] && [ \"$before_authn_bytes\" -gt 0 ] || missing_secret=true",
|
|
" [ \"$before_uri_present\" = yes ] && [ \"$before_uri_bytes\" -gt 0 ] || missing_secret=true",
|
|
" [ \"$before_pg_password_present\" = yes ] && [ \"$before_pg_password_bytes\" -gt 0 ] || missing_secret=true",
|
|
" expected_datastore_uri=",
|
|
" if [ -n \"$pg_password\" ]; then",
|
|
" expected_datastore_uri=\"postgres://$db_user:$pg_password@$db_host:5432/$db_name?sslmode=disable\"",
|
|
" [ \"$datastore_uri\" = \"$expected_datastore_uri\" ] || missing_secret=true",
|
|
" fi",
|
|
" missing_db=false",
|
|
" [ \"$db_role_exists_before\" = t ] || missing_db=true",
|
|
" [ \"$db_database_exists_before\" = t ] || missing_db=true",
|
|
" if [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$before_postgres_exists\" != yes ] || [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then action=would-ensure; else action=kept; fi",
|
|
" elif ! command -v openssl >/dev/null 2>&1; then",
|
|
" action=openssl-missing; apply_exit=127",
|
|
" else",
|
|
" [ -n \"$postgres_admin_password\" ] || postgres_admin_password=$(openssl rand -hex 24)",
|
|
" kubectl -n \"$namespace\" create secret generic \"$postgres_secret\" --from-literal=\"POSTGRES_PASSWORD=$postgres_admin_password\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
|
|
" postgres_secret_exit=$?",
|
|
" if [ \"$postgres_secret_exit\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" rollout status \"statefulset/$postgres_statefulset\" --timeout=120s >/tmp/hwlab-postgres-rollout.out 2>/tmp/hwlab-postgres-rollout.err",
|
|
" postgres_rollout_exit=$?",
|
|
" fi",
|
|
" if [ \"$postgres_secret_exit\" -ne 0 ]; then action=postgres-secret-failed; apply_exit=$postgres_secret_exit",
|
|
" elif [ \"$postgres_rollout_exit\" -ne 0 ]; then action=postgres-rollout-failed; apply_exit=$postgres_rollout_exit",
|
|
" else",
|
|
" [ -n \"$authn_value\" ] || authn_value=$(openssl rand -base64 48)",
|
|
" [ -n \"$pg_password\" ] || pg_password=$(openssl rand -hex 24)",
|
|
" datastore_uri=\"postgres://$db_user:$pg_password@$db_host:5432/$db_name?sslmode=disable\"",
|
|
" kubectl -n \"$namespace\" create secret generic \"$openfga_secret\" --from-literal=\"$authn_key=$authn_value\" --from-literal=\"$datastore_uri_key=$datastore_uri\" --from-literal=\"$postgres_password_key=$pg_password\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
|
|
" apply_exit=$?",
|
|
" if [ \"$apply_exit\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" exec -i \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -v ON_ERROR_STOP=1 -U \"$postgres_admin_user\" -d postgres -v db_name=\"$db_name\" -v db_user=\"$db_user\" -v db_pass=\"$pg_password\" >/tmp/hwlab-openfga-psql.out 2>/tmp/hwlab-openfga-psql.err <<'SQL'",
|
|
"SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'db_user', :'db_pass')",
|
|
"WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'db_user')",
|
|
"\\gexec",
|
|
"ALTER ROLE :\"db_user\" LOGIN PASSWORD :'db_pass';",
|
|
"SELECT format('CREATE DATABASE %I OWNER %I', :'db_name', :'db_user')",
|
|
"WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'db_name')",
|
|
"\\gexec",
|
|
"ALTER DATABASE :\"db_name\" OWNER TO :\"db_user\";",
|
|
"SQL",
|
|
" db_ensure_exit=$?",
|
|
" if [ \"$db_ensure_exit\" -eq 0 ]; then",
|
|
" openfga_image=$(kubectl -n \"$namespace\" get deployment \"$openfga_deployment\" -o 'jsonpath={.spec.template.spec.containers[0].image}' 2>/tmp/hwlab-openfga-image.err)",
|
|
" migrate_job=\"$openfga_deployment-migrate-unidesk-$(date +%s)\"",
|
|
" tmp=$(mktemp /tmp/hwlab-openfga-migrate.XXXXXX.yaml)",
|
|
" cat >\"$tmp\" <<EOF_JOB",
|
|
"apiVersion: batch/v1",
|
|
"kind: Job",
|
|
"metadata:",
|
|
" name: $migrate_job",
|
|
" namespace: $namespace",
|
|
" labels:",
|
|
" app.kubernetes.io/part-of: hwlab",
|
|
" app.kubernetes.io/managed-by: unidesk",
|
|
"spec:",
|
|
" backoffLimit: 0",
|
|
" ttlSecondsAfterFinished: 300",
|
|
" template:",
|
|
" spec:",
|
|
" restartPolicy: Never",
|
|
" containers:",
|
|
" - name: openfga-migrate",
|
|
" image: $openfga_image",
|
|
" imagePullPolicy: IfNotPresent",
|
|
" args: [\"migrate\"]",
|
|
" env:",
|
|
" - name: OPENFGA_DATASTORE_ENGINE",
|
|
" value: postgres",
|
|
" - name: OPENFGA_DATASTORE_URI",
|
|
" valueFrom:",
|
|
" secretKeyRef:",
|
|
" name: $openfga_secret",
|
|
" key: $datastore_uri_key",
|
|
"EOF_JOB",
|
|
" kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f \"$tmp\" >/tmp/hwlab-openfga-migrate-apply.out 2>/tmp/hwlab-openfga-migrate-apply.err",
|
|
" migrate_apply_exit=$?",
|
|
" rm -f \"$tmp\"",
|
|
" if [ \"$migrate_apply_exit\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" wait --for=condition=complete \"job/$migrate_job\" --timeout=25s >/tmp/hwlab-openfga-migrate-wait.out 2>/tmp/hwlab-openfga-migrate-wait.err",
|
|
" migrate_wait_exit=$?",
|
|
" fi",
|
|
" if [ \"$migrate_apply_exit\" != 0 ]; then action=migrate-apply-failed",
|
|
" elif [ \"$migrate_wait_exit\" != 0 ]; then action=migrate-wait-failed",
|
|
" else",
|
|
" kubectl -n \"$namespace\" rollout restart \"deployment/$openfga_deployment\" >/tmp/hwlab-openfga-rollout-restart.out 2>/tmp/hwlab-openfga-rollout-restart.err",
|
|
" rollout_restart_exit=$?",
|
|
" if [ \"$rollout_restart_exit\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" rollout status \"deployment/$openfga_deployment\" --timeout=20s >/tmp/hwlab-openfga-rollout-status.out 2>/tmp/hwlab-openfga-rollout-status.err",
|
|
" rollout_status_exit=$?",
|
|
" fi",
|
|
" if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed",
|
|
" elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed",
|
|
" else action=ensured; mutation=true; fi",
|
|
" fi",
|
|
" else action=db-ensure-failed; fi",
|
|
" else action=apply-failed; fi",
|
|
" fi",
|
|
" fi",
|
|
"fi",
|
|
"after_exists=$(secret_exists_flag \"$openfga_secret\")",
|
|
"after_postgres_exists=$(secret_exists_flag \"$postgres_secret\")",
|
|
"after_authn_b64=$(secret_b64_key \"$openfga_secret\" \"$authn_key\")",
|
|
"after_uri_b64=$(secret_b64_key \"$openfga_secret\" \"$datastore_uri_key\")",
|
|
"after_pg_password_b64=$(secret_b64_key \"$openfga_secret\" \"$postgres_password_key\")",
|
|
"after_authn_present=$([ -n \"$after_authn_b64\" ] && printf yes || printf no)",
|
|
"after_uri_present=$([ -n \"$after_uri_b64\" ] && printf yes || printf no)",
|
|
"after_pg_password_present=$([ -n \"$after_pg_password_b64\" ] && printf yes || printf no)",
|
|
"after_authn_bytes=$(decoded_length \"$after_authn_b64\")",
|
|
"after_uri_bytes=$(decoded_length \"$after_uri_b64\")",
|
|
"after_pg_password_bytes=$(decoded_length \"$after_pg_password_b64\")",
|
|
"after_datastore_uri=$(decoded_value \"$after_uri_b64\")",
|
|
"parse_database_url \"$after_datastore_uri\"",
|
|
"after_uri_host=$uri_host",
|
|
"after_uri_user=$uri_user",
|
|
"after_uri_database=$uri_database",
|
|
"after_uri_sslmode=$uri_sslmode",
|
|
"after_uri_password_present=$uri_password_present",
|
|
"expected_uri_prefix=\"postgres://$db_user:\"",
|
|
"expected_uri_suffix=\"@$db_host:5432/$db_name?sslmode=disable\"",
|
|
"case \"$datastore_uri\" in \"$expected_uri_prefix\"*\"$expected_uri_suffix\") before_uri_matches_expected=yes ;; *) before_uri_matches_expected=no ;; esac",
|
|
"case \"$after_datastore_uri\" in \"$expected_uri_prefix\"*\"$expected_uri_suffix\") after_uri_matches_expected=yes ;; *) after_uri_matches_expected=no ;; esac",
|
|
"probe_db",
|
|
"db_role_exists_after=$role_result",
|
|
"db_database_exists_after=$database_result",
|
|
"db_probe_exit_after=$probe_exit",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$openfga_secret\"",
|
|
"printf 'key\\t%s\\n' \"$selected_key\"",
|
|
"printf 'preset\\t%s\\n' \"$preset\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeExists\\t%s\\n' \"$before_exists\"",
|
|
"printf 'beforePostgresSecretExists\\t%s\\n' \"$before_postgres_exists\"",
|
|
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
|
|
"printf 'afterPostgresSecretExists\\t%s\\n' \"$after_postgres_exists\"",
|
|
"printf 'afterAuthnPresent\\t%s\\n' \"$after_authn_present\"",
|
|
"printf 'afterAuthnBytes\\t%s\\n' \"$after_authn_bytes\"",
|
|
"printf 'afterDatastoreUriPresent\\t%s\\n' \"$after_uri_present\"",
|
|
"printf 'afterDatastoreUriBytes\\t%s\\n' \"$after_uri_bytes\"",
|
|
"printf 'beforeDatastoreUriHost\\t%s\\n' \"$before_uri_host\"",
|
|
"printf 'beforeDatastoreUriUser\\t%s\\n' \"$before_uri_user\"",
|
|
"printf 'beforeDatastoreUriDatabase\\t%s\\n' \"$before_uri_database\"",
|
|
"printf 'beforeDatastoreUriSslmode\\t%s\\n' \"$before_uri_sslmode\"",
|
|
"printf 'beforeDatastoreUriPasswordPresent\\t%s\\n' \"$before_uri_password_present\"",
|
|
"printf 'beforeDatastoreUriMatchesExpected\\t%s\\n' \"$before_uri_matches_expected\"",
|
|
"printf 'afterDatastoreUriHost\\t%s\\n' \"$after_uri_host\"",
|
|
"printf 'afterDatastoreUriUser\\t%s\\n' \"$after_uri_user\"",
|
|
"printf 'afterDatastoreUriDatabase\\t%s\\n' \"$after_uri_database\"",
|
|
"printf 'afterDatastoreUriSslmode\\t%s\\n' \"$after_uri_sslmode\"",
|
|
"printf 'afterDatastoreUriPasswordPresent\\t%s\\n' \"$after_uri_password_present\"",
|
|
"printf 'afterDatastoreUriMatchesExpected\\t%s\\n' \"$after_uri_matches_expected\"",
|
|
"printf 'afterPostgresPasswordPresent\\t%s\\n' \"$after_pg_password_present\"",
|
|
"printf 'afterPostgresPasswordBytes\\t%s\\n' \"$after_pg_password_bytes\"",
|
|
"printf 'dbRoleExistsAfter\\t%s\\n' \"$db_role_exists_after\"",
|
|
"printf 'dbDatabaseExistsAfter\\t%s\\n' \"$db_database_exists_after\"",
|
|
"printf 'dbProbeExitCodeAfter\\t%s\\n' \"$db_probe_exit_after\"",
|
|
"printf 'postgresSecretExitCode\\t%s\\n' \"$postgres_secret_exit\"",
|
|
"printf 'postgresRolloutExitCode\\t%s\\n' \"$postgres_rollout_exit\"",
|
|
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
|
|
"printf 'dbEnsureExitCode\\t%s\\n' \"$db_ensure_exit\"",
|
|
"printf 'openfgaImage\\t%s\\n' \"$openfga_image\"",
|
|
"printf 'migrateJob\\t%s\\n' \"$migrate_job\"",
|
|
"printf 'migrateApplyExitCode\\t%s\\n' \"$migrate_apply_exit\"",
|
|
"printf 'migrateWaitExitCode\\t%s\\n' \"$migrate_wait_exit\"",
|
|
"printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"",
|
|
"printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"",
|
|
"authn_value= datastore_uri= after_datastore_uri= pg_password= postgres_admin_password=",
|
|
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
|
|
"if [ -n \"$db_ensure_exit\" ] && [ \"$db_ensure_exit\" != 0 ]; then exit \"$db_ensure_exit\"; fi",
|
|
"if [ -n \"$migrate_apply_exit\" ] && [ \"$migrate_apply_exit\" != 0 ]; then exit \"$migrate_apply_exit\"; fi",
|
|
"if [ -n \"$migrate_wait_exit\" ] && [ \"$migrate_wait_exit\" != 0 ]; then exit \"$migrate_wait_exit\"; fi",
|
|
"if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi",
|
|
"if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function masterAdminApiKeySecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(spec.namespace)}`,
|
|
`name=${shellQuote(spec.masterAdminApiKeySecret)}`,
|
|
`api_key_name=${shellQuote(MASTER_ADMIN_API_KEY_KEY)}`,
|
|
`action_request=${shellQuote(options.action)}`,
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
`field_manager=${shellQuote(spec.fieldManager)}`,
|
|
`cloud_api_deployment=${shellQuote(spec.cloudApiDeployment)}`,
|
|
"preset=master-server-admin-api-key",
|
|
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$name\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$name\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }",
|
|
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
|
|
"decoded_prefix() { if [ -n \"$1\" ]; then value=$(printf '%s' \"$1\" | base64 -d 2>/dev/null || true); printf '%s' \"$value\" | cut -c1-12; value=; fi; }",
|
|
"before_exists=$(secret_exists_flag)",
|
|
"before_api_key_b64=$(secret_b64_key \"$api_key_name\")",
|
|
"before_api_key_present=$([ -n \"$before_api_key_b64\" ] && printf yes || printf no)",
|
|
"before_api_key_bytes=$(decoded_length \"$before_api_key_b64\")",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"apply_exit=",
|
|
"rollout_restart_exit=",
|
|
"rollout_status_exit=",
|
|
"if [ \"$action_request\" = ensure ]; then",
|
|
" missing_secret=false",
|
|
" [ \"$before_api_key_present\" = yes ] && [ \"$before_api_key_bytes\" -gt 0 ] || missing_secret=true",
|
|
" if [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$missing_secret\" = true ]; then action=would-ensure; else action=kept; fi",
|
|
" else",
|
|
" api_key=$(cat)",
|
|
" case \"$api_key\" in hwl_live_*) ;; *) action=api-key-invalid; apply_exit=43 ;; esac",
|
|
" if [ -z \"$apply_exit\" ]; then",
|
|
" kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$api_key_name=$api_key\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
|
|
" apply_exit=$?",
|
|
" if [ \"$apply_exit\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" rollout restart \"deployment/$cloud_api_deployment\" >/tmp/hwlab-master-admin-api-key-rollout-restart.out 2>/tmp/hwlab-master-admin-api-key-rollout-restart.err",
|
|
" rollout_restart_exit=$?",
|
|
" if [ \"$rollout_restart_exit\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" rollout status \"deployment/$cloud_api_deployment\" --timeout=180s >/tmp/hwlab-master-admin-api-key-rollout-status.out 2>/tmp/hwlab-master-admin-api-key-rollout-status.err",
|
|
" rollout_status_exit=$?",
|
|
" fi",
|
|
" if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed",
|
|
" elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed",
|
|
" else action=ensured; mutation=true; fi",
|
|
" else action=apply-failed; fi",
|
|
" fi",
|
|
" api_key=",
|
|
" fi",
|
|
"fi",
|
|
"after_exists=$(secret_exists_flag)",
|
|
"after_api_key_b64=$(secret_b64_key \"$api_key_name\")",
|
|
"after_api_key_present=$([ -n \"$after_api_key_b64\" ] && printf yes || printf no)",
|
|
"after_api_key_bytes=$(decoded_length \"$after_api_key_b64\")",
|
|
"after_api_key_prefix=$(decoded_prefix \"$after_api_key_b64\")",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$name\"",
|
|
"printf 'key\\t%s\\n' \"$api_key_name\"",
|
|
"printf 'preset\\t%s\\n' \"$preset\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
|
|
"printf 'afterApiKeyPresent\\t%s\\n' \"$after_api_key_present\"",
|
|
"printf 'afterApiKeyBytes\\t%s\\n' \"$after_api_key_bytes\"",
|
|
"printf 'afterApiKeyPrefix\\t%s\\n' \"$after_api_key_prefix\"",
|
|
"printf 'cloudApiDeployment\\t%s\\n' \"$cloud_api_deployment\"",
|
|
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
|
|
"printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"",
|
|
"printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"",
|
|
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
|
|
"if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi",
|
|
"if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function bootstrapAdminSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec, material: BootstrapAdminSecretMaterial | null): string {
|
|
const yamlSourceEnabled = spec.bootstrapAdminPasswordSourceRef !== undefined && spec.bootstrapAdminPasswordSourceKey !== undefined && spec.bootstrapAdminPasswordHashTransform !== undefined;
|
|
if (!yamlSourceEnabled) return legacyBootstrapAdminSecretScript(options, spec);
|
|
const materialOk = material?.ok === true;
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(spec.namespace)}`,
|
|
`name=${shellQuote(spec.bootstrapAdminSecret)}`,
|
|
`password_hash_key=${shellQuote(spec.bootstrapAdminPasswordHashKey)}`,
|
|
`username=${shellQuote(spec.bootstrapAdminUsername)}`,
|
|
`display_name=${shellQuote(spec.bootstrapAdminDisplayName)}`,
|
|
`source_ref=${shellQuote(spec.bootstrapAdminPasswordSourceRef ?? "")}`,
|
|
`source_key=${shellQuote(spec.bootstrapAdminPasswordSourceKey ?? "")}`,
|
|
`source_path=${shellQuote(material?.sourcePath === null || material?.sourcePath === undefined ? "" : displayRepoPath(material.sourcePath))}`,
|
|
`source_present=${shellQuote(material?.sourcePresent === true ? "yes" : "no")}`,
|
|
`source_fingerprint=${shellQuote(material?.sourceFingerprint ?? "")}`,
|
|
`source_error=${shellQuote(material?.error ?? "")}`,
|
|
`transform=${shellQuote(spec.bootstrapAdminPasswordHashTransform ?? "")}`,
|
|
`material_ok=${shellQuote(materialOk ? "true" : "false")}`,
|
|
`force_sync=${shellQuote(options.force === true ? "true" : "false")}`,
|
|
`cloud_api_deployment=${shellQuote(spec.cloudApiDeployment)}`,
|
|
`action_request=${shellQuote(options.action)}`,
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
`field_manager=${shellQuote(spec.fieldManager)}`,
|
|
"preset=bootstrap-admin",
|
|
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$name\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$name\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }",
|
|
"secret_annotation() { kubectl -n \"$namespace\" get secret \"$name\" -o \"go-template={{ with .metadata.annotations }}{{ index . \\\"$1\\\" }}{{ end }}\" 2>/dev/null || true; }",
|
|
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
|
|
"before_exists=$(secret_exists_flag)",
|
|
"before_hash_b64=$(secret_b64_key \"$password_hash_key\")",
|
|
"before_source_ref=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-source-ref)",
|
|
"before_source_key=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-source-key)",
|
|
"before_source_fingerprint=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-source-fingerprint)",
|
|
"before_username=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-username)",
|
|
"before_hash_present=$([ -n \"$before_hash_b64\" ] && printf yes || printf no)",
|
|
"before_hash_bytes=$(decoded_length \"$before_hash_b64\")",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"apply_exit=",
|
|
"rollout_restart_exit=",
|
|
"rollout_status_exit=",
|
|
"if [ \"$action_request\" = ensure ]; then",
|
|
" needs_sync=false",
|
|
" [ \"$before_exists\" = yes ] && [ \"$before_hash_bytes\" -gt 0 ] || needs_sync=true",
|
|
" [ \"$before_source_ref\" = \"$source_ref\" ] && [ \"$before_source_key\" = \"$source_key\" ] && [ \"$before_source_fingerprint\" = \"$source_fingerprint\" ] && [ \"$before_username\" = \"$username\" ] || needs_sync=true",
|
|
" [ \"$force_sync\" = true ] && needs_sync=true",
|
|
" if [ \"$material_ok\" != true ]; then",
|
|
" action=${source_error:-secret-source-invalid}",
|
|
" apply_exit=44",
|
|
" elif [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$needs_sync\" = true ]; then action=would-sync-from-yaml-source; else action=kept; fi",
|
|
" elif [ \"$needs_sync\" = false ]; then",
|
|
" action=kept",
|
|
" else",
|
|
" password_hash=$(cat)",
|
|
" case \"$password_hash\" in sha256:*:*) ;; *) action=password-hash-invalid; apply_exit=45 ;; esac",
|
|
" if [ -z \"$apply_exit\" ]; then",
|
|
" cat <<EOF_SECRET | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
|
|
"apiVersion: v1",
|
|
"kind: Secret",
|
|
"metadata:",
|
|
" name: $name",
|
|
" namespace: $namespace",
|
|
" annotations:",
|
|
" hwlab.pikastech.local/bootstrap-admin-username: \"$username\"",
|
|
" hwlab.pikastech.local/bootstrap-admin-display-name: \"$display_name\"",
|
|
" hwlab.pikastech.local/bootstrap-admin-source-ref: \"$source_ref\"",
|
|
" hwlab.pikastech.local/bootstrap-admin-source-key: \"$source_key\"",
|
|
" hwlab.pikastech.local/bootstrap-admin-source-fingerprint: \"$source_fingerprint\"",
|
|
" hwlab.pikastech.local/bootstrap-admin-password-transform: \"$transform\"",
|
|
" labels:",
|
|
" app.kubernetes.io/part-of: hwlab",
|
|
" hwlab.pikastech.local/secret-preset: bootstrap-admin",
|
|
"type: Opaque",
|
|
"stringData:",
|
|
" $password_hash_key: \"$password_hash\"",
|
|
"EOF_SECRET",
|
|
" apply_exit=$?",
|
|
" fi",
|
|
" if [ \"$apply_exit\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" rollout restart \"deployment/$cloud_api_deployment\" >/tmp/hwlab-bootstrap-admin-rollout-restart.out 2>/tmp/hwlab-bootstrap-admin-rollout-restart.err",
|
|
" rollout_restart_exit=$?",
|
|
" if [ \"$rollout_restart_exit\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" rollout status \"deployment/$cloud_api_deployment\" --timeout=180s >/tmp/hwlab-bootstrap-admin-rollout-status.out 2>/tmp/hwlab-bootstrap-admin-rollout-status.err",
|
|
" rollout_status_exit=$?",
|
|
" fi",
|
|
" if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed",
|
|
" elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed",
|
|
" else action=synced-from-yaml-source; mutation=true; fi",
|
|
" else action=apply-failed; fi",
|
|
" password_hash=",
|
|
" fi",
|
|
"fi",
|
|
"after_exists=$(secret_exists_flag)",
|
|
"after_hash_b64=$(secret_b64_key \"$password_hash_key\")",
|
|
"after_source_ref=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-source-ref)",
|
|
"after_source_key=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-source-key)",
|
|
"after_source_fingerprint=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-source-fingerprint)",
|
|
"after_username=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-username)",
|
|
"after_hash_present=$([ -n \"$after_hash_b64\" ] && printf yes || printf no)",
|
|
"after_hash_bytes=$(decoded_length \"$after_hash_b64\")",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$name\"",
|
|
"printf 'key\\t%s\\n' \"$password_hash_key\"",
|
|
"printf 'preset\\t%s\\n' \"$preset\"",
|
|
"printf 'username\\t%s\\n' \"$username\"",
|
|
"printf 'displayName\\t%s\\n' \"$display_name\"",
|
|
"printf 'sourceRef\\t%s\\n' \"$source_ref\"",
|
|
"printf 'sourceKey\\t%s\\n' \"$source_key\"",
|
|
"printf 'sourcePath\\t%s\\n' \"$source_path\"",
|
|
"printf 'sourceExists\\t%s\\n' \"$source_present\"",
|
|
"printf 'sourceFingerprint\\t%s\\n' \"$source_fingerprint\"",
|
|
"printf 'passwordHashTransform\\t%s\\n' \"$transform\"",
|
|
"printf 'forceSync\\t%s\\n' \"$force_sync\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeExists\\t%s\\n' \"$before_exists\"",
|
|
"printf 'beforePasswordHashPresent\\t%s\\n' \"$before_hash_present\"",
|
|
"printf 'beforePasswordHashBytes\\t%s\\n' \"$before_hash_bytes\"",
|
|
"printf 'beforeSourceRef\\t%s\\n' \"$before_source_ref\"",
|
|
"printf 'beforeSourceKey\\t%s\\n' \"$before_source_key\"",
|
|
"printf 'beforeSourceFingerprint\\t%s\\n' \"$before_source_fingerprint\"",
|
|
"printf 'beforeUsername\\t%s\\n' \"$before_username\"",
|
|
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
|
|
"printf 'afterPasswordHashPresent\\t%s\\n' \"$after_hash_present\"",
|
|
"printf 'afterPasswordHashBytes\\t%s\\n' \"$after_hash_bytes\"",
|
|
"printf 'afterSourceRef\\t%s\\n' \"$after_source_ref\"",
|
|
"printf 'afterSourceKey\\t%s\\n' \"$after_source_key\"",
|
|
"printf 'afterSourceFingerprint\\t%s\\n' \"$after_source_fingerprint\"",
|
|
"printf 'afterUsername\\t%s\\n' \"$after_username\"",
|
|
"printf 'cloudApiDeployment\\t%s\\n' \"$cloud_api_deployment\"",
|
|
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
|
|
"printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"",
|
|
"printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"",
|
|
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
|
|
"if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi",
|
|
"if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function legacyBootstrapAdminSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(spec.namespace)}`,
|
|
`name=${shellQuote(spec.bootstrapAdminSecret)}`,
|
|
`source_namespace=${shellQuote(spec.bootstrapAdminSourceNamespace)}`,
|
|
`source_name=${shellQuote(spec.bootstrapAdminSourceSecret)}`,
|
|
`password_hash_key=${shellQuote(spec.bootstrapAdminPasswordHashKey)}`,
|
|
`cloud_api_deployment=${shellQuote(spec.cloudApiDeployment)}`,
|
|
`action_request=${shellQuote(options.action)}`,
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
`field_manager=${shellQuote(spec.fieldManager)}`,
|
|
"preset=bootstrap-admin",
|
|
"secret_exists_flag() { kubectl -n \"$1\" get secret \"$2\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"secret_b64_key() { kubectl -n \"$1\" get secret \"$2\" -o \"go-template={{ index .data \\\"$3\\\" }}\" 2>/dev/null || true; }",
|
|
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
|
|
"before_exists=$(secret_exists_flag \"$namespace\" \"$name\")",
|
|
"before_hash_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$password_hash_key\")",
|
|
"source_exists=$(secret_exists_flag \"$source_namespace\" \"$source_name\")",
|
|
"source_hash_b64=$(secret_b64_key \"$source_namespace\" \"$source_name\" \"$password_hash_key\")",
|
|
"before_hash_present=$([ -n \"$before_hash_b64\" ] && printf yes || printf no)",
|
|
"source_hash_present=$([ -n \"$source_hash_b64\" ] && printf yes || printf no)",
|
|
"before_hash_bytes=$(decoded_length \"$before_hash_b64\")",
|
|
"source_hash_bytes=$(decoded_length \"$source_hash_b64\")",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"apply_exit=",
|
|
"rollout_restart_exit=",
|
|
"rollout_status_exit=",
|
|
"if [ \"$action_request\" = ensure ]; then",
|
|
" missing_target=false",
|
|
" [ \"$before_exists\" = yes ] && [ \"$before_hash_bytes\" -gt 0 ] || missing_target=true",
|
|
" missing_source=false",
|
|
" [ \"$source_exists\" = yes ] && [ \"$source_hash_bytes\" -gt 0 ] || missing_source=true",
|
|
" if [ \"$missing_source\" = true ]; then",
|
|
" action=source-missing-password-hash",
|
|
" apply_exit=44",
|
|
" elif [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$missing_target\" = true ]; then action=would-copy-from-source; else action=kept; fi",
|
|
" elif [ \"$missing_target\" = false ]; then",
|
|
" action=kept",
|
|
" else",
|
|
" cat <<EOF_SECRET | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
|
|
"apiVersion: v1",
|
|
"kind: Secret",
|
|
"metadata:",
|
|
" name: $name",
|
|
" namespace: $namespace",
|
|
" labels:",
|
|
" app.kubernetes.io/part-of: hwlab",
|
|
" hwlab.pikastech.local/secret-preset: bootstrap-admin",
|
|
"type: Opaque",
|
|
"data:",
|
|
" password-hash: $source_hash_b64",
|
|
"EOF_SECRET",
|
|
" apply_exit=$?",
|
|
" if [ \"$apply_exit\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" rollout restart \"deployment/$cloud_api_deployment\" >/tmp/hwlab-bootstrap-admin-rollout-restart.out 2>/tmp/hwlab-bootstrap-admin-rollout-restart.err",
|
|
" rollout_restart_exit=$?",
|
|
" if [ \"$rollout_restart_exit\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" rollout status \"deployment/$cloud_api_deployment\" --timeout=180s >/tmp/hwlab-bootstrap-admin-rollout-status.out 2>/tmp/hwlab-bootstrap-admin-rollout-status.err",
|
|
" rollout_status_exit=$?",
|
|
" fi",
|
|
" if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed",
|
|
" elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed",
|
|
" else action=copied-from-source; mutation=true; fi",
|
|
" else action=apply-failed; fi",
|
|
" fi",
|
|
"fi",
|
|
"after_exists=$(secret_exists_flag \"$namespace\" \"$name\")",
|
|
"after_hash_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$password_hash_key\")",
|
|
"after_hash_present=$([ -n \"$after_hash_b64\" ] && printf yes || printf no)",
|
|
"after_hash_bytes=$(decoded_length \"$after_hash_b64\")",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$name\"",
|
|
"printf 'key\\t%s\\n' \"$password_hash_key\"",
|
|
"printf 'preset\\t%s\\n' \"$preset\"",
|
|
"printf 'sourceNamespace\\t%s\\n' \"$source_namespace\"",
|
|
"printf 'sourceSecret\\t%s\\n' \"$source_name\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeExists\\t%s\\n' \"$before_exists\"",
|
|
"printf 'beforePasswordHashPresent\\t%s\\n' \"$before_hash_present\"",
|
|
"printf 'beforePasswordHashBytes\\t%s\\n' \"$before_hash_bytes\"",
|
|
"printf 'sourceExists\\t%s\\n' \"$source_exists\"",
|
|
"printf 'sourcePasswordHashPresent\\t%s\\n' \"$source_hash_present\"",
|
|
"printf 'sourcePasswordHashBytes\\t%s\\n' \"$source_hash_bytes\"",
|
|
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
|
|
"printf 'afterPasswordHashPresent\\t%s\\n' \"$after_hash_present\"",
|
|
"printf 'afterPasswordHashBytes\\t%s\\n' \"$after_hash_bytes\"",
|
|
"printf 'cloudApiDeployment\\t%s\\n' \"$cloud_api_deployment\"",
|
|
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
|
|
"printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"",
|
|
"printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"",
|
|
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
|
|
"if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi",
|
|
"if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function codeAgentProviderSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(spec.namespace)}`,
|
|
`name=${shellQuote(spec.codeAgentProviderSecret)}`,
|
|
`source_namespace=${shellQuote(spec.codeAgentProviderSourceNamespace)}`,
|
|
`source_name=${shellQuote(spec.codeAgentProviderSourceSecret)}`,
|
|
`openai_key=${shellQuote(CODE_AGENT_PROVIDER_OPENAI_KEY)}`,
|
|
`opencode_key=${shellQuote(CODE_AGENT_PROVIDER_OPENCODE_KEY)}`,
|
|
`selected_key=${shellQuote(options.key ?? "")}`,
|
|
`action_request=${shellQuote(options.action)}`,
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
`field_manager=${shellQuote(spec.fieldManager)}`,
|
|
"preset=code-agent-provider",
|
|
"secret_exists_flag() { kubectl -n \"$1\" get secret \"$2\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"secret_b64_key() { kubectl -n \"$1\" get secret \"$2\" -o \"go-template={{ index .data \\\"$3\\\" }}\" 2>/dev/null || true; }",
|
|
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
|
|
"before_exists=$(secret_exists_flag \"$namespace\" \"$name\")",
|
|
"before_openai_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$openai_key\")",
|
|
"before_opencode_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$opencode_key\")",
|
|
"source_exists=$(secret_exists_flag \"$source_namespace\" \"$source_name\")",
|
|
"source_openai_b64=$(secret_b64_key \"$source_namespace\" \"$source_name\" \"$openai_key\")",
|
|
"source_opencode_b64=$(secret_b64_key \"$source_namespace\" \"$source_name\" \"$opencode_key\")",
|
|
"before_openai_present=$([ -n \"$before_openai_b64\" ] && printf yes || printf no)",
|
|
"before_opencode_present=$([ -n \"$before_opencode_b64\" ] && printf yes || printf no)",
|
|
"source_openai_present=$([ -n \"$source_openai_b64\" ] && printf yes || printf no)",
|
|
"source_opencode_present=$([ -n \"$source_opencode_b64\" ] && printf yes || printf no)",
|
|
"before_openai_bytes=$(decoded_length \"$before_openai_b64\")",
|
|
"before_opencode_bytes=$(decoded_length \"$before_opencode_b64\")",
|
|
"source_openai_bytes=$(decoded_length \"$source_openai_b64\")",
|
|
"source_opencode_bytes=$(decoded_length \"$source_opencode_b64\")",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"apply_exit=",
|
|
"if [ \"$action_request\" = ensure ]; then",
|
|
" missing_target=false",
|
|
" [ \"$before_exists\" = yes ] && { [ \"$before_openai_bytes\" -gt 0 ] || [ \"$before_opencode_bytes\" -gt 0 ]; } || missing_target=true",
|
|
" missing_source=false",
|
|
" [ \"$source_exists\" = yes ] && { [ \"$source_openai_bytes\" -gt 0 ] || [ \"$source_opencode_bytes\" -gt 0 ]; } || missing_source=true",
|
|
" if [ \"$missing_source\" = true ]; then",
|
|
" action=source-missing-provider-key",
|
|
" apply_exit=44",
|
|
" elif [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$missing_target\" = true ]; then action=would-copy-from-source; else action=kept; fi",
|
|
" elif [ \"$missing_target\" = false ]; then",
|
|
" action=kept",
|
|
" else",
|
|
" cat <<EOF_SECRET | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
|
|
"apiVersion: v1",
|
|
"kind: Secret",
|
|
"metadata:",
|
|
" name: $name",
|
|
" namespace: $namespace",
|
|
" labels:",
|
|
" app.kubernetes.io/part-of: hwlab",
|
|
" hwlab.pikastech.local/secret-preset: code-agent-provider",
|
|
"type: Opaque",
|
|
"data:",
|
|
" openai-api-key: $source_openai_b64",
|
|
" opencode-api-key: $source_opencode_b64",
|
|
"EOF_SECRET",
|
|
" apply_exit=$?",
|
|
" if [ \"$apply_exit\" -eq 0 ]; then action=copied-from-source; mutation=true; else action=apply-failed; fi",
|
|
" fi",
|
|
"fi",
|
|
"after_exists=$(secret_exists_flag \"$namespace\" \"$name\")",
|
|
"after_openai_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$openai_key\")",
|
|
"after_opencode_b64=$(secret_b64_key \"$namespace\" \"$name\" \"$opencode_key\")",
|
|
"after_openai_present=$([ -n \"$after_openai_b64\" ] && printf yes || printf no)",
|
|
"after_opencode_present=$([ -n \"$after_opencode_b64\" ] && printf yes || printf no)",
|
|
"after_openai_bytes=$(decoded_length \"$after_openai_b64\")",
|
|
"after_opencode_bytes=$(decoded_length \"$after_opencode_b64\")",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$name\"",
|
|
"printf 'preset\\t%s\\n' \"$preset\"",
|
|
"printf 'sourceNamespace\\t%s\\n' \"$source_namespace\"",
|
|
"printf 'sourceSecret\\t%s\\n' \"$source_name\"",
|
|
"printf 'selectedKey\\t%s\\n' \"$selected_key\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeExists\\t%s\\n' \"$before_exists\"",
|
|
"printf 'beforeOpenaiPresent\\t%s\\n' \"$before_openai_present\"",
|
|
"printf 'beforeOpenaiBytes\\t%s\\n' \"$before_openai_bytes\"",
|
|
"printf 'beforeOpencodePresent\\t%s\\n' \"$before_opencode_present\"",
|
|
"printf 'beforeOpencodeBytes\\t%s\\n' \"$before_opencode_bytes\"",
|
|
"printf 'sourceExists\\t%s\\n' \"$source_exists\"",
|
|
"printf 'sourceOpenaiPresent\\t%s\\n' \"$source_openai_present\"",
|
|
"printf 'sourceOpenaiBytes\\t%s\\n' \"$source_openai_bytes\"",
|
|
"printf 'sourceOpencodePresent\\t%s\\n' \"$source_opencode_present\"",
|
|
"printf 'sourceOpencodeBytes\\t%s\\n' \"$source_opencode_bytes\"",
|
|
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
|
|
"printf 'afterOpenaiPresent\\t%s\\n' \"$after_openai_present\"",
|
|
"printf 'afterOpenaiBytes\\t%s\\n' \"$after_openai_bytes\"",
|
|
"printf 'afterOpencodePresent\\t%s\\n' \"$after_opencode_present\"",
|
|
"printf 'afterOpencodeBytes\\t%s\\n' \"$after_opencode_bytes\"",
|
|
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
|
|
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function cloudApiDbSecretScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(spec.namespace)}`,
|
|
`name=${shellQuote(spec.cloudApiDbSecret)}`,
|
|
`database_url_key=${shellQuote(spec.cloudApiDbKey)}`,
|
|
`postgres_secret=${shellQuote(spec.postgresSecret)}`,
|
|
`postgres_statefulset=${shellQuote(spec.postgresStatefulSet)}`,
|
|
`postgres_admin_user=${shellQuote(spec.postgresAdminUser)}`,
|
|
`db_name=${shellQuote(spec.cloudApiDbName)}`,
|
|
`db_user=${shellQuote(spec.cloudApiDbUser)}`,
|
|
`db_host=${shellQuote(spec.cloudApiDbHost)}`,
|
|
`cloud_api_deployment=${shellQuote(spec.cloudApiDeployment)}`,
|
|
"db_consumer_deployments=\"hwlab-cloud-api hwlab-user-billing hwlab-workbench-runtime\"",
|
|
`action_request=${shellQuote(options.action)}`,
|
|
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
|
|
`field_manager=${shellQuote(spec.fieldManager)}`,
|
|
"preset=cloud-api-db",
|
|
"secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$1\" >/dev/null 2>&1 && printf yes || printf no; }",
|
|
"secret_b64_key() { kubectl -n \"$namespace\" get secret \"$1\" -o \"go-template={{ index .data \\\"$2\\\" }}\" 2>/dev/null || true; }",
|
|
"decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }",
|
|
"decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }",
|
|
"psql_scalar() { kubectl -n \"$namespace\" exec \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -U \"$postgres_admin_user\" -d postgres -tAc \"$1\" 2>/dev/null | tr -d '[:space:]'; }",
|
|
"probe_db() {",
|
|
" role_result=unknown",
|
|
" database_result=unknown",
|
|
" probe_exit=missing-postgres-admin-secret",
|
|
" if [ -n \"$postgres_admin_password\" ]; then",
|
|
" role_result=$(psql_scalar \"select exists(select 1 from pg_roles where rolname='$db_user');\")",
|
|
" role_exit=$?",
|
|
" database_result=$(psql_scalar \"select exists(select 1 from pg_database where datname='$db_name');\")",
|
|
" database_exit=$?",
|
|
" if [ \"$role_exit\" -eq 0 ] && [ \"$database_exit\" -eq 0 ]; then probe_exit=0; else probe_exit=$role_exit/$database_exit; fi",
|
|
" fi",
|
|
"}",
|
|
"before_exists=$(secret_exists_flag \"$name\")",
|
|
"before_postgres_exists=$(secret_exists_flag \"$postgres_secret\")",
|
|
"before_url_b64=$(secret_b64_key \"$name\" \"$database_url_key\")",
|
|
"before_url_present=$([ -n \"$before_url_b64\" ] && printf yes || printf no)",
|
|
"before_url_bytes=$(decoded_length \"$before_url_b64\")",
|
|
"database_url=$(decoded_value \"$before_url_b64\")",
|
|
"parse_database_url \"$database_url\"",
|
|
"before_url_host=$uri_host",
|
|
"before_url_user=$uri_user",
|
|
"before_url_database=$uri_database",
|
|
"before_url_sslmode=$uri_sslmode",
|
|
"before_url_password_present=$uri_password_present",
|
|
"postgres_admin_b64=$(secret_b64_key \"$postgres_secret\" POSTGRES_PASSWORD)",
|
|
"postgres_admin_present=$([ -n \"$postgres_admin_b64\" ] && printf yes || printf no)",
|
|
"postgres_admin_password=$(decoded_value \"$postgres_admin_b64\")",
|
|
"probe_db",
|
|
"db_role_exists_before=$role_result",
|
|
"db_database_exists_before=$database_result",
|
|
"db_probe_exit_before=$probe_exit",
|
|
"action=observed",
|
|
"mutation=false",
|
|
"apply_exit=",
|
|
"db_ensure_exit=",
|
|
"rollout_restart_exit=",
|
|
"rollout_status_exit=",
|
|
"if [ \"$action_request\" = ensure ]; then",
|
|
" missing_secret=false",
|
|
" [ \"$before_url_present\" = yes ] && [ \"$before_url_bytes\" -gt 0 ] || missing_secret=true",
|
|
" expected_database_url=",
|
|
" if [ -n \"$postgres_admin_password\" ]; then",
|
|
" expected_database_url=\"postgres://$db_user:$postgres_admin_password@$db_host:5432/$db_name?sslmode=disable\"",
|
|
" [ \"$database_url\" = \"$expected_database_url\" ] || missing_secret=true",
|
|
" fi",
|
|
" missing_db=false",
|
|
" [ \"$db_role_exists_before\" = t ] || missing_db=true",
|
|
" [ \"$db_database_exists_before\" = t ] || missing_db=true",
|
|
" if [ \"$dry_run\" = true ]; then",
|
|
" if [ \"$before_postgres_exists\" != yes ] || [ \"$postgres_admin_present\" != yes ] || [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then action=would-ensure; else action=kept; fi",
|
|
" elif [ \"$before_postgres_exists\" != yes ] || [ \"$postgres_admin_present\" != yes ] || [ -z \"$postgres_admin_password\" ]; then",
|
|
" action=postgres-admin-secret-missing",
|
|
" apply_exit=44",
|
|
" elif [ \"$missing_secret\" = false ] && [ \"$missing_db\" = false ]; then",
|
|
" action=kept",
|
|
" else",
|
|
" database_url=\"postgres://$db_user:$postgres_admin_password@$db_host:5432/$db_name?sslmode=disable\"",
|
|
" kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$database_url_key=$database_url\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -",
|
|
" apply_exit=$?",
|
|
" if [ \"$apply_exit\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" exec -i \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -v ON_ERROR_STOP=1 -U \"$postgres_admin_user\" -d postgres -v db_name=\"$db_name\" -v db_user=\"$db_user\" -v db_pass=\"$postgres_admin_password\" >/tmp/hwlab-cloud-api-db-psql.out 2>/tmp/hwlab-cloud-api-db-psql.err <<'SQL'",
|
|
"SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'db_user', :'db_pass')",
|
|
"WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'db_user')",
|
|
"\\gexec",
|
|
"ALTER ROLE :\"db_user\" LOGIN PASSWORD :'db_pass';",
|
|
"SELECT format('CREATE DATABASE %I OWNER %I', :'db_name', :'db_user')",
|
|
"WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'db_name')",
|
|
"\\gexec",
|
|
"ALTER DATABASE :\"db_name\" OWNER TO :\"db_user\";",
|
|
"SQL",
|
|
" db_ensure_exit=$?",
|
|
" if [ \"$db_ensure_exit\" -eq 0 ]; then",
|
|
" if [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then",
|
|
" rollout_restart_exit=0",
|
|
" for deployment in $db_consumer_deployments; do",
|
|
" kubectl -n \"$namespace\" rollout restart \"deployment/$deployment\" >/tmp/hwlab-db-consumer-rollout-restart-$deployment.out 2>/tmp/hwlab-db-consumer-rollout-restart-$deployment.err",
|
|
" rc=$?",
|
|
" if [ \"$rc\" -ne 0 ]; then rollout_restart_exit=$rc; break; fi",
|
|
" done",
|
|
" if [ \"$rollout_restart_exit\" -eq 0 ]; then",
|
|
" rollout_status_exit=0",
|
|
" for deployment in $db_consumer_deployments; do",
|
|
" kubectl -n \"$namespace\" rollout status \"deployment/$deployment\" --timeout=180s >/tmp/hwlab-db-consumer-rollout-status-$deployment.out 2>/tmp/hwlab-db-consumer-rollout-status-$deployment.err",
|
|
" rc=$?",
|
|
" if [ \"$rc\" -ne 0 ]; then rollout_status_exit=$rc; break; fi",
|
|
" done",
|
|
" fi",
|
|
" fi",
|
|
" if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then action=rollout-restart-failed",
|
|
" elif [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then action=rollout-status-failed",
|
|
" else action=ensured; mutation=true; fi",
|
|
" else action=db-ensure-failed; fi",
|
|
" else action=apply-failed; fi",
|
|
" fi",
|
|
"fi",
|
|
"after_exists=$(secret_exists_flag \"$name\")",
|
|
"after_url_b64=$(secret_b64_key \"$name\" \"$database_url_key\")",
|
|
"after_url_present=$([ -n \"$after_url_b64\" ] && printf yes || printf no)",
|
|
"after_url_bytes=$(decoded_length \"$after_url_b64\")",
|
|
"after_database_url=$(decoded_value \"$after_url_b64\")",
|
|
"parse_database_url \"$after_database_url\"",
|
|
"after_url_host=$uri_host",
|
|
"after_url_user=$uri_user",
|
|
"after_url_database=$uri_database",
|
|
"after_url_sslmode=$uri_sslmode",
|
|
"after_url_password_present=$uri_password_present",
|
|
"expected_url_prefix=\"postgres://$db_user:\"",
|
|
"expected_url_suffix=\"@$db_host:5432/$db_name?sslmode=disable\"",
|
|
"case \"$database_url\" in \"$expected_url_prefix\"*\"$expected_url_suffix\") before_url_matches_expected=yes ;; *) before_url_matches_expected=no ;; esac",
|
|
"case \"$after_database_url\" in \"$expected_url_prefix\"*\"$expected_url_suffix\") after_url_matches_expected=yes ;; *) after_url_matches_expected=no ;; esac",
|
|
"probe_db",
|
|
"db_role_exists_after=$role_result",
|
|
"db_database_exists_after=$database_result",
|
|
"db_probe_exit_after=$probe_exit",
|
|
"printf 'namespace\\t%s\\n' \"$namespace\"",
|
|
"printf 'secret\\t%s\\n' \"$name\"",
|
|
"printf 'key\\t%s\\n' \"$database_url_key\"",
|
|
"printf 'preset\\t%s\\n' \"$preset\"",
|
|
"printf 'action\\t%s\\n' \"$action\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'mutation\\t%s\\n' \"$mutation\"",
|
|
"printf 'beforeExists\\t%s\\n' \"$before_exists\"",
|
|
"printf 'beforePostgresSecretExists\\t%s\\n' \"$before_postgres_exists\"",
|
|
"printf 'beforeDatabaseUrlPresent\\t%s\\n' \"$before_url_present\"",
|
|
"printf 'beforeDatabaseUrlBytes\\t%s\\n' \"$before_url_bytes\"",
|
|
"printf 'beforeDatabaseUrlHost\\t%s\\n' \"$before_url_host\"",
|
|
"printf 'beforeDatabaseUrlUser\\t%s\\n' \"$before_url_user\"",
|
|
"printf 'beforeDatabaseUrlDatabase\\t%s\\n' \"$before_url_database\"",
|
|
"printf 'beforeDatabaseUrlSslmode\\t%s\\n' \"$before_url_sslmode\"",
|
|
"printf 'beforeDatabaseUrlPasswordPresent\\t%s\\n' \"$before_url_password_present\"",
|
|
"printf 'beforeDatabaseUrlMatchesExpected\\t%s\\n' \"$before_url_matches_expected\"",
|
|
"printf 'afterExists\\t%s\\n' \"$after_exists\"",
|
|
"printf 'afterDatabaseUrlPresent\\t%s\\n' \"$after_url_present\"",
|
|
"printf 'afterDatabaseUrlBytes\\t%s\\n' \"$after_url_bytes\"",
|
|
"printf 'afterDatabaseUrlHost\\t%s\\n' \"$after_url_host\"",
|
|
"printf 'afterDatabaseUrlUser\\t%s\\n' \"$after_url_user\"",
|
|
"printf 'afterDatabaseUrlDatabase\\t%s\\n' \"$after_url_database\"",
|
|
"printf 'afterDatabaseUrlSslmode\\t%s\\n' \"$after_url_sslmode\"",
|
|
"printf 'afterDatabaseUrlPasswordPresent\\t%s\\n' \"$after_url_password_present\"",
|
|
"printf 'afterDatabaseUrlMatchesExpected\\t%s\\n' \"$after_url_matches_expected\"",
|
|
"printf 'postgresAdminSecretPresent\\t%s\\n' \"$postgres_admin_present\"",
|
|
"printf 'postgresSecret\\t%s\\n' \"$postgres_secret\"",
|
|
"printf 'dbName\\t%s\\n' \"$db_name\"",
|
|
"printf 'dbUser\\t%s\\n' \"$db_user\"",
|
|
"printf 'dbHost\\t%s\\n' \"$db_host\"",
|
|
"printf 'dbRoleExistsBefore\\t%s\\n' \"$db_role_exists_before\"",
|
|
"printf 'dbDatabaseExistsBefore\\t%s\\n' \"$db_database_exists_before\"",
|
|
"printf 'dbProbeExitCodeBefore\\t%s\\n' \"$db_probe_exit_before\"",
|
|
"printf 'dbRoleExistsAfter\\t%s\\n' \"$db_role_exists_after\"",
|
|
"printf 'dbDatabaseExistsAfter\\t%s\\n' \"$db_database_exists_after\"",
|
|
"printf 'dbProbeExitCodeAfter\\t%s\\n' \"$db_probe_exit_after\"",
|
|
"printf 'cloudApiDeployment\\t%s\\n' \"$cloud_api_deployment\"",
|
|
"printf 'dbConsumerDeployments\\t%s\\n' \"$db_consumer_deployments\"",
|
|
"printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
|
|
"printf 'dbEnsureExitCode\\t%s\\n' \"$db_ensure_exit\"",
|
|
"printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"",
|
|
"printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"",
|
|
"database_url= after_database_url= postgres_admin_password=",
|
|
"if [ -n \"$apply_exit\" ] && [ \"$apply_exit\" != 0 ]; then exit \"$apply_exit\"; fi",
|
|
"if [ -n \"$db_ensure_exit\" ] && [ \"$db_ensure_exit\" != 0 ]; then exit \"$db_ensure_exit\"; fi",
|
|
"if [ -n \"$rollout_restart_exit\" ] && [ \"$rollout_restart_exit\" != 0 ]; then exit \"$rollout_restart_exit\"; fi",
|
|
"if [ -n \"$rollout_status_exit\" ] && [ \"$rollout_status_exit\" != 0 ]; then exit \"$rollout_status_exit\"; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function secretStatusFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string, spec: RuntimeSecretSpec): Record<string, unknown> {
|
|
const fields = keyValueLinesFromText(text);
|
|
if (fields.preset === "owned-postgres-cleanup") {
|
|
const absent = fields.afterStatefulSetExists !== "yes" &&
|
|
fields.afterServiceExists !== "yes" &&
|
|
fields.afterConfigMapExists !== "yes" &&
|
|
fields.afterSecretExists !== "yes" &&
|
|
fields.afterPvcExists !== "yes";
|
|
const platformServiceReady = fields.platformServiceExists === "yes";
|
|
return {
|
|
ok: commandOk && absent && platformServiceReady,
|
|
namespace: fields.namespace || spec.namespace,
|
|
secret: fields.secret || spec.postgresSecret,
|
|
statefulSet: fields.statefulSet || spec.postgresStatefulSet,
|
|
service: fields.service || spec.postgresSecret,
|
|
configMap: fields.configMap || `${spec.postgresSecret}-init`,
|
|
pvc: fields.pvc || `data-${spec.postgresSecret}-0`,
|
|
preset: "owned-postgres-cleanup",
|
|
action: fields.action || null,
|
|
dryRun: fields.dryRun === "true",
|
|
mutation: fields.mutation === "true",
|
|
before: {
|
|
statefulSetExists: fields.beforeStatefulSetExists === "yes",
|
|
serviceExists: fields.beforeServiceExists === "yes",
|
|
configMapExists: fields.beforeConfigMapExists === "yes",
|
|
secretExists: fields.beforeSecretExists === "yes",
|
|
pvcExists: fields.beforePvcExists === "yes",
|
|
pvcPhase: fields.beforePvcPhase || null,
|
|
persistentVolume: fields.beforePersistentVolume || null,
|
|
},
|
|
after: {
|
|
statefulSetExists: fields.afterStatefulSetExists === "yes",
|
|
serviceExists: fields.afterServiceExists === "yes",
|
|
configMapExists: fields.afterConfigMapExists === "yes",
|
|
secretExists: fields.afterSecretExists === "yes",
|
|
pvcExists: fields.afterPvcExists === "yes",
|
|
pvcPhase: fields.afterPvcPhase || null,
|
|
persistentVolume: fields.afterPersistentVolume || null,
|
|
},
|
|
platformService: {
|
|
name: "g14-platform-postgres",
|
|
exists: platformServiceReady,
|
|
},
|
|
deleteStatefulSetExitCode: numericField(fields.deleteStatefulSetExitCode),
|
|
deleteServiceExitCode: numericField(fields.deleteServiceExitCode),
|
|
deleteConfigMapExitCode: numericField(fields.deleteConfigMapExitCode),
|
|
deleteSecretExitCode: numericField(fields.deleteSecretExitCode),
|
|
deletePvcExitCode: numericField(fields.deletePvcExitCode),
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
valuesRedacted: true,
|
|
summary: absent
|
|
? `${fields.statefulSet || spec.postgresStatefulSet}, ${fields.service || spec.postgresSecret}, ${fields.configMap || `${spec.postgresSecret}-init`}, ${fields.secret || spec.postgresSecret}, and ${fields.pvc || `data-${spec.postgresSecret}-0`} absent`
|
|
: `owned Postgres resources still exist in ${fields.namespace || spec.namespace}`,
|
|
};
|
|
}
|
|
if (fields.preset === "obsolete-secret-cleanup") {
|
|
const absent = fields.afterSecretExists !== "yes";
|
|
const refsAbsent = fields.workloadRefsPresent !== "yes";
|
|
const dryRun = fields.dryRun === "true";
|
|
return {
|
|
ok: commandOk && refsAbsent && (dryRun || absent),
|
|
namespace: fields.namespace || spec.namespace,
|
|
secret: fields.secret || spec.obsoleteHwpodDbSecret,
|
|
preset: "obsolete-secret-cleanup",
|
|
action: fields.action || null,
|
|
dryRun,
|
|
mutation: fields.mutation === "true",
|
|
before: {
|
|
secretExists: fields.beforeSecretExists === "yes",
|
|
},
|
|
after: {
|
|
secretExists: fields.afterSecretExists === "yes",
|
|
},
|
|
workloadRefs: {
|
|
present: fields.workloadRefsPresent === "yes",
|
|
preview: fields.workloadRefsPreview || "",
|
|
},
|
|
deleteSecretExitCode: numericField(fields.deleteSecretExitCode),
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
valuesRedacted: true,
|
|
summary: refsAbsent && (dryRun || absent)
|
|
? `${fields.secret || spec.obsoleteHwpodDbSecret} is unreferenced${dryRun ? "" : " and absent"}`
|
|
: `${fields.secret || spec.obsoleteHwpodDbSecret} still present or referenced`,
|
|
};
|
|
}
|
|
if (fields.preset === "master-server-admin-api-key") {
|
|
const afterBytes = numericField(fields.afterApiKeyBytes);
|
|
const healthy = fields.afterExists === "yes" && fields.afterApiKeyPresent === "yes" && typeof afterBytes === "number" && afterBytes > 0;
|
|
return {
|
|
ok: commandOk && healthy,
|
|
namespace: fields.namespace || spec.namespace,
|
|
secret: fields.secret || spec.masterAdminApiKeySecret,
|
|
preset: "master-server-admin-api-key",
|
|
action: fields.action || null,
|
|
dryRun: fields.dryRun === "true",
|
|
mutation: fields.mutation === "true",
|
|
after: { exists: fields.afterExists === "yes", apiKey: { keyPresent: fields.afterApiKeyPresent === "yes", valueBytes: afterBytes, keyPrefix: fields.afterApiKeyPrefix || null } },
|
|
cloudApiDeployment: fields.cloudApiDeployment || spec.cloudApiDeployment,
|
|
dbConsumerDeployments: fields.dbConsumerDeployments ? fields.dbConsumerDeployments.split(/\s+/u).filter(Boolean) : [spec.cloudApiDeployment],
|
|
applyExitCode: numericField(fields.applyExitCode),
|
|
rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode),
|
|
rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode),
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
valuesRedacted: true,
|
|
summary: healthy ? `${fields.secret || spec.masterAdminApiKeySecret}/${MASTER_ADMIN_API_KEY_KEY} exists` : `${fields.secret || spec.masterAdminApiKeySecret}/${MASTER_ADMIN_API_KEY_KEY} missing`,
|
|
};
|
|
}
|
|
if (fields.preset === "bootstrap-admin") {
|
|
const beforeHashBytes = numericField(fields.beforePasswordHashBytes);
|
|
const sourceHashBytes = numericField(fields.sourcePasswordHashBytes);
|
|
const afterHashBytes = numericField(fields.afterPasswordHashBytes);
|
|
const yamlSourceMode = typeof fields.sourceRef === "string" && fields.sourceRef.length > 0;
|
|
const targetHashReady = fields.afterExists === "yes" &&
|
|
fields.afterPasswordHashPresent === "yes" &&
|
|
typeof afterHashBytes === "number" && afterHashBytes > 0;
|
|
const yamlSourceReady = !yamlSourceMode || (
|
|
fields.sourceExists === "yes" &&
|
|
typeof fields.sourceFingerprint === "string" &&
|
|
fields.sourceFingerprint.length > 0 &&
|
|
fields.afterSourceRef === fields.sourceRef &&
|
|
fields.afterSourceKey === fields.sourceKey &&
|
|
fields.afterSourceFingerprint === fields.sourceFingerprint &&
|
|
fields.afterUsername === fields.username
|
|
);
|
|
const healthy = targetHashReady && yamlSourceReady;
|
|
return {
|
|
ok: commandOk && healthy,
|
|
namespace: fields.namespace || spec.namespace,
|
|
secret: fields.secret || spec.bootstrapAdminSecret,
|
|
key: fields.key || spec.bootstrapAdminPasswordHashKey,
|
|
preset: "bootstrap-admin",
|
|
account: {
|
|
username: fields.username || spec.bootstrapAdminUsername,
|
|
displayName: fields.displayName || spec.bootstrapAdminDisplayName,
|
|
},
|
|
source: yamlSourceMode
|
|
? {
|
|
sourceRef: fields.sourceRef,
|
|
sourceKey: fields.sourceKey || null,
|
|
sourcePath: fields.sourcePath || null,
|
|
exists: fields.sourceExists === "yes",
|
|
fingerprint: fields.sourceFingerprint || null,
|
|
passwordHashTransform: fields.passwordHashTransform || spec.bootstrapAdminPasswordHashTransform || null,
|
|
valuesRedacted: true,
|
|
}
|
|
: {
|
|
namespace: fields.sourceNamespace || spec.bootstrapAdminSourceNamespace,
|
|
secret: fields.sourceSecret || spec.bootstrapAdminSourceSecret,
|
|
exists: fields.sourceExists === "yes",
|
|
passwordHash: { keyPresent: fields.sourcePasswordHashPresent === "yes", valueBytes: sourceHashBytes },
|
|
},
|
|
action: fields.action || null,
|
|
dryRun: fields.dryRun === "true",
|
|
forceSync: fields.forceSync === "true",
|
|
mutation: fields.mutation === "true",
|
|
before: {
|
|
exists: fields.beforeExists === "yes",
|
|
passwordHash: { keyPresent: fields.beforePasswordHashPresent === "yes", valueBytes: beforeHashBytes },
|
|
...(yamlSourceMode ? {
|
|
sourceRef: fields.beforeSourceRef || null,
|
|
sourceKey: fields.beforeSourceKey || null,
|
|
sourceFingerprint: fields.beforeSourceFingerprint || null,
|
|
username: fields.beforeUsername || null,
|
|
} : {}),
|
|
},
|
|
after: {
|
|
exists: fields.afterExists === "yes",
|
|
passwordHash: { keyPresent: fields.afterPasswordHashPresent === "yes", valueBytes: afterHashBytes },
|
|
...(yamlSourceMode ? {
|
|
sourceRef: fields.afterSourceRef || null,
|
|
sourceKey: fields.afterSourceKey || null,
|
|
sourceFingerprint: fields.afterSourceFingerprint || null,
|
|
username: fields.afterUsername || null,
|
|
} : {}),
|
|
},
|
|
cloudApiDeployment: fields.cloudApiDeployment || spec.cloudApiDeployment,
|
|
applyExitCode: numericField(fields.applyExitCode),
|
|
rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode),
|
|
rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode),
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
valuesRedacted: true,
|
|
summary: healthy ? `${fields.secret || spec.bootstrapAdminSecret}/${spec.bootstrapAdminPasswordHashKey} exists` : `${fields.secret || spec.bootstrapAdminSecret}/${spec.bootstrapAdminPasswordHashKey} missing`,
|
|
};
|
|
}
|
|
if (fields.preset === "code-agent-provider") {
|
|
const beforeOpenaiBytes = numericField(fields.beforeOpenaiBytes);
|
|
const beforeOpencodeBytes = numericField(fields.beforeOpencodeBytes);
|
|
const sourceOpenaiBytes = numericField(fields.sourceOpenaiBytes);
|
|
const sourceOpencodeBytes = numericField(fields.sourceOpencodeBytes);
|
|
const afterOpenaiBytes = numericField(fields.afterOpenaiBytes);
|
|
const afterOpencodeBytes = numericField(fields.afterOpencodeBytes);
|
|
const openaiReady = fields.afterOpenaiPresent === "yes" && typeof afterOpenaiBytes === "number" && afterOpenaiBytes > 0;
|
|
const opencodeReady = fields.afterOpencodePresent === "yes" && typeof afterOpencodeBytes === "number" && afterOpencodeBytes > 0;
|
|
const healthy = fields.afterExists === "yes" && (openaiReady || opencodeReady);
|
|
return {
|
|
ok: commandOk && healthy,
|
|
namespace: fields.namespace || spec.namespace,
|
|
secret: fields.secret || spec.codeAgentProviderSecret,
|
|
preset: "code-agent-provider",
|
|
source: {
|
|
namespace: fields.sourceNamespace || spec.codeAgentProviderSourceNamespace,
|
|
secret: fields.sourceSecret || spec.codeAgentProviderSourceSecret,
|
|
exists: fields.sourceExists === "yes",
|
|
openaiApiKey: { keyPresent: fields.sourceOpenaiPresent === "yes", valueBytes: sourceOpenaiBytes },
|
|
opencodeApiKey: { keyPresent: fields.sourceOpencodePresent === "yes", valueBytes: sourceOpencodeBytes },
|
|
},
|
|
selectedKey: fields.selectedKey || null,
|
|
action: fields.action || null,
|
|
dryRun: fields.dryRun === "true",
|
|
mutation: fields.mutation === "true",
|
|
before: {
|
|
exists: fields.beforeExists === "yes",
|
|
openaiApiKey: { keyPresent: fields.beforeOpenaiPresent === "yes", valueBytes: beforeOpenaiBytes },
|
|
opencodeApiKey: { keyPresent: fields.beforeOpencodePresent === "yes", valueBytes: beforeOpencodeBytes },
|
|
},
|
|
after: {
|
|
exists: fields.afterExists === "yes",
|
|
openaiApiKey: { keyPresent: fields.afterOpenaiPresent === "yes", valueBytes: afterOpenaiBytes },
|
|
opencodeApiKey: { keyPresent: fields.afterOpencodePresent === "yes", valueBytes: afterOpencodeBytes },
|
|
requiredAnyProviderKeyPresent: openaiReady || opencodeReady,
|
|
},
|
|
applyExitCode: numericField(fields.applyExitCode),
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
valuesRedacted: true,
|
|
summary: healthy ? `${fields.secret || spec.codeAgentProviderSecret} has a usable provider key` : `${fields.secret || spec.codeAgentProviderSecret} missing provider keys`,
|
|
};
|
|
}
|
|
if (fields.preset === "cloud-api-db") {
|
|
const beforeUrlBytes = numericField(fields.beforeDatabaseUrlBytes);
|
|
const afterUrlBytes = numericField(fields.afterDatabaseUrlBytes);
|
|
if (fields.platformDbMode === "true") {
|
|
const keysHealthy = fields.afterExists === "yes" &&
|
|
fields.afterDatabaseUrlPresent === "yes" &&
|
|
typeof afterUrlBytes === "number" && afterUrlBytes > 0;
|
|
const platformBridgeHealthy = fields.platformServiceExists === "yes" &&
|
|
fields.platformEndpointsExists !== "yes" &&
|
|
fields.platformEndpointSliceExists === "yes";
|
|
const uriHealthy = fields.dbHostMatchesPlatform === "yes" &&
|
|
fields.dbNameMatchesExpected === "yes" &&
|
|
fields.dbUserMatchesExpected === "yes";
|
|
const healthy = keysHealthy && platformBridgeHealthy && uriHealthy;
|
|
return {
|
|
ok: commandOk && healthy,
|
|
namespace: fields.namespace || spec.namespace,
|
|
secret: fields.secret || spec.cloudApiDbSecret,
|
|
key: fields.key || spec.cloudApiDbKey,
|
|
preset: "cloud-api-db",
|
|
action: fields.action || null,
|
|
dryRun: fields.dryRun === "true",
|
|
mutation: fields.mutation === "true",
|
|
platformDbMode: true,
|
|
after: {
|
|
exists: fields.afterExists === "yes",
|
|
databaseUrl: { keyPresent: fields.afterDatabaseUrlPresent === "yes", valueBytes: afterUrlBytes },
|
|
},
|
|
legacyPostgresSecret: {
|
|
name: fields.legacyPostgresSecret || spec.postgresSecret,
|
|
exists: fields.legacyPostgresSecretExists === "yes",
|
|
},
|
|
platformService: {
|
|
name: fields.platformService || spec.platformPostgresService,
|
|
exists: fields.platformServiceExists === "yes",
|
|
endpointsExist: fields.platformEndpointsExists === "yes",
|
|
legacyEndpointsAbsent: fields.platformEndpointsExists !== "yes",
|
|
endpointSlice: fields.platformEndpointSlice || `${spec.platformPostgresService}-host`,
|
|
endpointSliceExists: fields.platformEndpointSliceExists === "yes",
|
|
},
|
|
dbName: fields.dbName || spec.cloudApiDbName,
|
|
dbUser: fields.dbUser || spec.cloudApiDbUser,
|
|
dbHost: fields.dbHost || spec.cloudApiDbHost,
|
|
dbHostMatchesPlatform: fields.dbHostMatchesPlatform === "yes",
|
|
dbNameMatchesExpected: fields.dbNameMatchesExpected === "yes",
|
|
dbUserMatchesExpected: fields.dbUserMatchesExpected === "yes",
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
valuesRedacted: true,
|
|
summary: healthy
|
|
? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} points to ${fields.platformService || spec.platformPostgresService}`
|
|
: `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} is not aligned to platform DB`,
|
|
};
|
|
}
|
|
const keysHealthy = fields.afterExists === "yes" &&
|
|
fields.afterDatabaseUrlPresent === "yes" &&
|
|
typeof afterUrlBytes === "number" && afterUrlBytes > 0;
|
|
const databaseHealthy = fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t";
|
|
const expectedDbHost = fields.dbHost || spec.cloudApiDbHost;
|
|
const expectedDbUser = fields.dbUser || spec.cloudApiDbUser;
|
|
const expectedDbName = fields.dbName || spec.cloudApiDbName;
|
|
const actualUrlAligned = fields.afterDatabaseUrlMatchesExpected === "yes";
|
|
const healthy = keysHealthy && databaseHealthy && actualUrlAligned;
|
|
return {
|
|
ok: commandOk && healthy,
|
|
namespace: fields.namespace || spec.namespace,
|
|
secret: fields.secret || spec.cloudApiDbSecret,
|
|
key: fields.key || spec.cloudApiDbKey,
|
|
preset: "cloud-api-db",
|
|
action: fields.action || null,
|
|
dryRun: fields.dryRun === "true",
|
|
mutation: fields.mutation === "true",
|
|
before: {
|
|
exists: fields.beforeExists === "yes",
|
|
postgresSecretExists: fields.beforePostgresSecretExists === "yes",
|
|
databaseUrl: {
|
|
keyPresent: fields.beforeDatabaseUrlPresent === "yes",
|
|
valueBytes: beforeUrlBytes,
|
|
host: fields.beforeDatabaseUrlHost || null,
|
|
user: fields.beforeDatabaseUrlUser || null,
|
|
database: fields.beforeDatabaseUrlDatabase || null,
|
|
sslmode: fields.beforeDatabaseUrlSslmode || null,
|
|
passwordPresent: fields.beforeDatabaseUrlPasswordPresent === "yes",
|
|
alignedToExpected: fields.beforeDatabaseUrlMatchesExpected === "yes",
|
|
},
|
|
database: {
|
|
roleExists: fields.dbRoleExistsBefore || "unknown",
|
|
databaseExists: fields.dbDatabaseExistsBefore || "unknown",
|
|
probeExitCode: fields.dbProbeExitCodeBefore || null,
|
|
},
|
|
},
|
|
after: {
|
|
exists: fields.afterExists === "yes",
|
|
databaseUrl: {
|
|
keyPresent: fields.afterDatabaseUrlPresent === "yes",
|
|
valueBytes: afterUrlBytes,
|
|
host: fields.afterDatabaseUrlHost || null,
|
|
user: fields.afterDatabaseUrlUser || null,
|
|
database: fields.afterDatabaseUrlDatabase || null,
|
|
sslmode: fields.afterDatabaseUrlSslmode || null,
|
|
passwordPresent: fields.afterDatabaseUrlPasswordPresent === "yes",
|
|
alignedToExpected: actualUrlAligned,
|
|
},
|
|
database: {
|
|
roleExists: fields.dbRoleExistsAfter || "unknown",
|
|
databaseExists: fields.dbDatabaseExistsAfter || "unknown",
|
|
probeExitCode: fields.dbProbeExitCodeAfter || null,
|
|
},
|
|
},
|
|
postgresAdminSecretPresent: fields.postgresAdminSecretPresent === "yes",
|
|
postgresSecret: fields.postgresSecret || spec.postgresSecret,
|
|
dbName: expectedDbName,
|
|
dbUser: expectedDbUser,
|
|
dbHost: expectedDbHost,
|
|
expectedDatabaseUrl: {
|
|
host: expectedDbHost,
|
|
user: expectedDbUser,
|
|
database: expectedDbName,
|
|
sslmode: "disable",
|
|
},
|
|
databaseUrlDrift: !actualUrlAligned,
|
|
...(!actualUrlAligned ? { degradedReason: "cloud-api-db-secret-drift" } : {}),
|
|
cloudApiDeployment: fields.cloudApiDeployment || spec.cloudApiDeployment,
|
|
applyExitCode: numericField(fields.applyExitCode),
|
|
dbEnsureExitCode: numericField(fields.dbEnsureExitCode),
|
|
rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode),
|
|
rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode),
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
valuesRedacted: true,
|
|
summary: healthy
|
|
? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} points to expected local database`
|
|
: !actualUrlAligned
|
|
? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} points to ${fields.afterDatabaseUrlHost || "-"} ${fields.afterDatabaseUrlDatabase || "-"} as ${fields.afterDatabaseUrlUser || "-"}, expected ${expectedDbHost} ${expectedDbName} as ${expectedDbUser}`
|
|
: `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} or runtime database missing`,
|
|
};
|
|
}
|
|
const afterAuthnBytes = numericField(fields.afterAuthnBytes);
|
|
const afterUriBytes = numericField(fields.afterDatastoreUriBytes);
|
|
const afterPasswordBytes = numericField(fields.afterPostgresPasswordBytes);
|
|
if (fields.platformDbMode === "true") {
|
|
const keysHealthy = fields.afterExists === "yes" &&
|
|
fields.afterAuthnPresent === "yes" &&
|
|
fields.afterDatastoreUriPresent === "yes" &&
|
|
typeof afterAuthnBytes === "number" && afterAuthnBytes > 0 &&
|
|
typeof afterUriBytes === "number" && afterUriBytes > 0;
|
|
const platformBridgeHealthy = fields.platformServiceExists === "yes" &&
|
|
fields.platformEndpointsExists !== "yes" &&
|
|
fields.platformEndpointSliceExists === "yes";
|
|
const uriHealthy = fields.dbHostMatchesPlatform === "yes" &&
|
|
fields.dbNameMatchesExpected === "yes" &&
|
|
fields.dbUserMatchesExpected === "yes";
|
|
const healthy = keysHealthy && platformBridgeHealthy && uriHealthy;
|
|
return {
|
|
ok: commandOk && healthy,
|
|
namespace: fields.namespace || spec.namespace,
|
|
secret: fields.secret || spec.openFgaSecret,
|
|
preset: fields.preset || "openfga",
|
|
action: fields.action || null,
|
|
dryRun: fields.dryRun === "true",
|
|
mutation: fields.mutation === "true",
|
|
platformDbMode: true,
|
|
after: {
|
|
exists: fields.afterExists === "yes",
|
|
authnPresharedKey: { keyPresent: fields.afterAuthnPresent === "yes", valueBytes: afterAuthnBytes },
|
|
datastoreUri: { keyPresent: fields.afterDatastoreUriPresent === "yes", valueBytes: afterUriBytes },
|
|
postgresPassword: { keyPresent: fields.afterPostgresPasswordPresent === "yes", valueBytes: afterPasswordBytes },
|
|
},
|
|
legacyPostgresSecret: {
|
|
name: fields.legacyPostgresSecret || spec.postgresSecret,
|
|
exists: fields.legacyPostgresSecretExists === "yes",
|
|
},
|
|
platformService: {
|
|
name: fields.platformService || spec.platformPostgresService,
|
|
exists: fields.platformServiceExists === "yes",
|
|
endpointsExist: fields.platformEndpointsExists === "yes",
|
|
legacyEndpointsAbsent: fields.platformEndpointsExists !== "yes",
|
|
endpointSlice: fields.platformEndpointSlice || `${spec.platformPostgresService}-host`,
|
|
endpointSliceExists: fields.platformEndpointSliceExists === "yes",
|
|
},
|
|
dbName: fields.dbName || spec.openFgaDbName,
|
|
dbUser: fields.dbUser || spec.openFgaDbUser,
|
|
dbHost: fields.dbHost || spec.openFgaDbHost,
|
|
dbHostMatchesPlatform: fields.dbHostMatchesPlatform === "yes",
|
|
dbNameMatchesExpected: fields.dbNameMatchesExpected === "yes",
|
|
dbUserMatchesExpected: fields.dbUserMatchesExpected === "yes",
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
valuesRedacted: true,
|
|
summary: healthy
|
|
? `${fields.secret || spec.openFgaSecret} datastore-uri points to ${fields.platformService || spec.platformPostgresService}`
|
|
: `${fields.secret || spec.openFgaSecret} datastore-uri is not aligned to platform DB`,
|
|
};
|
|
}
|
|
const keysHealthy = fields.afterExists === "yes" &&
|
|
fields.afterPostgresSecretExists === "yes" &&
|
|
fields.afterAuthnPresent === "yes" &&
|
|
fields.afterDatastoreUriPresent === "yes" &&
|
|
fields.afterPostgresPasswordPresent === "yes" &&
|
|
typeof afterAuthnBytes === "number" && afterAuthnBytes > 0 &&
|
|
typeof afterUriBytes === "number" && afterUriBytes > 0 &&
|
|
typeof afterPasswordBytes === "number" && afterPasswordBytes > 0;
|
|
const databaseHealthy = fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t";
|
|
const expectedOpenFgaHost = fields.dbHost || spec.openFgaDbHost;
|
|
const expectedOpenFgaUser = fields.dbUser || spec.openFgaDbUser;
|
|
const expectedOpenFgaDbName = fields.dbName || spec.openFgaDbName;
|
|
const datastoreUriAligned = fields.afterDatastoreUriMatchesExpected === "yes";
|
|
const healthy = keysHealthy && databaseHealthy && datastoreUriAligned;
|
|
return {
|
|
ok: commandOk && healthy,
|
|
namespace: fields.namespace || spec.namespace,
|
|
secret: fields.secret || spec.openFgaSecret,
|
|
preset: fields.preset || "openfga",
|
|
action: fields.action || null,
|
|
dryRun: fields.dryRun === "true",
|
|
mutation: fields.mutation === "true",
|
|
after: {
|
|
exists: fields.afterExists === "yes",
|
|
postgresSecretExists: fields.afterPostgresSecretExists === "yes",
|
|
authnPresharedKey: { keyPresent: fields.afterAuthnPresent === "yes", valueBytes: afterAuthnBytes },
|
|
datastoreUri: {
|
|
keyPresent: fields.afterDatastoreUriPresent === "yes",
|
|
valueBytes: afterUriBytes,
|
|
host: fields.afterDatastoreUriHost || null,
|
|
user: fields.afterDatastoreUriUser || null,
|
|
database: fields.afterDatastoreUriDatabase || null,
|
|
sslmode: fields.afterDatastoreUriSslmode || null,
|
|
passwordPresent: fields.afterDatastoreUriPasswordPresent === "yes",
|
|
alignedToExpected: datastoreUriAligned,
|
|
},
|
|
postgresPassword: { keyPresent: fields.afterPostgresPasswordPresent === "yes", valueBytes: afterPasswordBytes },
|
|
database: { roleExists: fields.dbRoleExistsAfter || "unknown", databaseExists: fields.dbDatabaseExistsAfter || "unknown", probeExitCode: fields.dbProbeExitCodeAfter || null },
|
|
},
|
|
expectedDatastoreUri: {
|
|
host: expectedOpenFgaHost,
|
|
user: expectedOpenFgaUser,
|
|
database: expectedOpenFgaDbName,
|
|
sslmode: "disable",
|
|
},
|
|
datastoreUriDrift: !datastoreUriAligned,
|
|
...(!datastoreUriAligned ? { degradedReason: "openfga-datastore-uri-drift" } : {}),
|
|
postgresSecretExitCode: numericField(fields.postgresSecretExitCode),
|
|
postgresRolloutExitCode: numericField(fields.postgresRolloutExitCode),
|
|
applyExitCode: numericField(fields.applyExitCode),
|
|
dbEnsureExitCode: numericField(fields.dbEnsureExitCode),
|
|
openfgaImage: fields.openfgaImage || null,
|
|
migrateJob: fields.migrateJob || null,
|
|
migrateApplyExitCode: numericField(fields.migrateApplyExitCode),
|
|
migrateWaitExitCode: numericField(fields.migrateWaitExitCode),
|
|
rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode),
|
|
rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode),
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
valuesRedacted: true,
|
|
summary: healthy
|
|
? `${fields.secret || spec.openFgaSecret} datastore-uri points to expected local database`
|
|
: !datastoreUriAligned
|
|
? `${fields.secret || spec.openFgaSecret} datastore-uri points to ${fields.afterDatastoreUriHost || "-"} ${fields.afterDatastoreUriDatabase || "-"} as ${fields.afterDatastoreUriUser || "-"}, expected ${expectedOpenFgaHost} ${expectedOpenFgaDbName} as ${expectedOpenFgaUser}`
|
|
: `${fields.secret || spec.openFgaSecret} keys or Postgres database missing`,
|
|
};
|
|
}
|
|
|
|
function obsoletePlatformDbStatusFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string, spec: RuntimeSecretSpec): Record<string, unknown> {
|
|
const fields = keyValueLinesFromText(text);
|
|
const dryRun = fields.dryRun === "true";
|
|
const databaseAbsent = fields.afterDatabaseExists !== "yes" && fields.afterDatabaseExists !== "unknown";
|
|
const roleAbsent = fields.afterRoleExists !== "yes" && fields.afterRoleExists !== "unknown";
|
|
const probesOk = fields.beforeDatabaseExists !== "unknown" &&
|
|
fields.beforeRoleExists !== "unknown" &&
|
|
fields.afterDatabaseExists !== "unknown" &&
|
|
fields.afterRoleExists !== "unknown";
|
|
return {
|
|
ok: commandOk && probesOk && (dryRun || (databaseAbsent && roleAbsent)),
|
|
database: fields.database || spec.obsoleteHwpodDbName,
|
|
role: fields.role || spec.obsoleteHwpodDbUser,
|
|
preset: "obsolete-platform-db-cleanup",
|
|
action: fields.action || null,
|
|
dryRun,
|
|
mutation: fields.mutation === "true",
|
|
before: {
|
|
databaseExists: fields.beforeDatabaseExists === "yes",
|
|
roleExists: fields.beforeRoleExists === "yes",
|
|
},
|
|
after: {
|
|
databaseExists: fields.afterDatabaseExists === "yes",
|
|
roleExists: fields.afterRoleExists === "yes",
|
|
},
|
|
beforeProbeExitCode: {
|
|
database: numericField(fields.beforeDatabaseProbeExitCode),
|
|
role: numericField(fields.beforeRoleProbeExitCode),
|
|
},
|
|
afterProbeExitCode: {
|
|
database: numericField(fields.afterDatabaseProbeExitCode),
|
|
role: numericField(fields.afterRoleProbeExitCode),
|
|
},
|
|
dropDatabaseExitCode: numericField(fields.dropDatabaseExitCode),
|
|
dropRoleExitCode: numericField(fields.dropRoleExitCode),
|
|
exitCode,
|
|
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
|
valuesRedacted: true,
|
|
summary: probesOk && (dryRun || (databaseAbsent && roleAbsent))
|
|
? `${fields.database || spec.obsoleteHwpodDbName} and ${fields.role || spec.obsoleteHwpodDbUser} are ${dryRun ? "observable" : "absent"}`
|
|
: `${fields.database || spec.obsoleteHwpodDbName} or ${fields.role || spec.obsoleteHwpodDbUser} still present or unobservable`,
|
|
};
|
|
}
|
|
|
|
export function nodeSecretStatusFromTextForTest(text: string, commandOk: boolean, exitCode: number | null, stderr: string, node = "G14", lane = "v03"): Record<string, unknown> {
|
|
return secretStatusFromText(text, commandOk, exitCode, stderr, runtimeSecretSpec({ node, lane }));
|
|
}
|
|
|
|
function masterAdminApiKeyEnvPath(spec: RuntimeSecretSpec): string {
|
|
return `/root/.config/hwlab-${spec.lane}/master-server-admin-api-key.env`;
|
|
}
|
|
|
|
function readMasterAdminApiKey(spec: RuntimeSecretSpec): { key: string; source: string } {
|
|
const source = masterAdminApiKeyEnvPath(spec);
|
|
if (!existsSync(source)) throw new Error(`HWLAB_API_KEY source missing: ${source}`);
|
|
const content = readFileSync(source, "utf8");
|
|
const match = content.match(/^HWLAB_API_KEY=(.+)$/m);
|
|
const raw = (match?.[1] ?? "").trim().replace(/^['"]|['"]$/g, "");
|
|
if (!raw.startsWith("hwl_live_")) throw new Error(`HWLAB_API_KEY source invalid: ${source}`);
|
|
return { key: raw, source };
|
|
}
|
|
|
|
function optionValue(args: string[], name: string): string | undefined {
|
|
const index = args.indexOf(name);
|
|
if (index === -1) return undefined;
|
|
const value = args[index + 1];
|
|
if (!value || value.startsWith("--")) throw new Error(`${name} requires a value`);
|
|
return value;
|
|
}
|
|
|
|
function requiredOption(args: string[], name: string): string {
|
|
const value = optionValue(args, name);
|
|
if (value === undefined) throw new Error(`${name} is required`);
|
|
return value;
|
|
}
|
|
|
|
function stripOption(args: string[], name: string): string[] {
|
|
return stripOptions(args, [name]);
|
|
}
|
|
|
|
function stripOptions(args: string[], names: readonly string[]): string[] {
|
|
const remove = new Set(names);
|
|
const without: string[] = [];
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index] ?? "";
|
|
if (remove.has(arg)) {
|
|
if (arg !== "--confirm" && arg !== "--dry-run" && arg !== "--wait") index += 1;
|
|
continue;
|
|
}
|
|
without.push(arg);
|
|
}
|
|
return without;
|
|
}
|
|
|
|
function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number {
|
|
const raw = optionValue(args, name);
|
|
if (raw === undefined) return defaultValue;
|
|
const value = Number(raw);
|
|
if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`);
|
|
return Math.min(value, maxValue);
|
|
}
|
|
|
|
function assertLane(value: string): void {
|
|
if (!/^v[0-9]{2,}$/u.test(value)) throw new Error(`--lane must look like v03/v04, got ${value}`);
|
|
}
|
|
|
|
function assertNodeId(value: string): void {
|
|
if (!/^[A-Za-z0-9_-]+$/u.test(value)) throw new Error(`--node must be a simple node id, got ${value}`);
|
|
}
|
|
|
|
function record(value: unknown): Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
|
}
|
|
|
|
function nullableRecord(value: unknown): Record<string, unknown> | null {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
|
|
}
|
|
|
|
function stringValue(value: unknown, path: string): string {
|
|
if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string`);
|
|
return value;
|
|
}
|
|
|
|
function optionalStringValue(value: unknown, path: string): string | null {
|
|
if (value === undefined || value === null) return null;
|
|
if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string when set`);
|
|
return value;
|
|
}
|
|
|
|
function positiveIntegerValue(value: unknown, path: string): number {
|
|
if (!Number.isInteger(value) || value <= 0) throw new Error(`${path} must be a positive integer`);
|
|
return value;
|
|
}
|
|
|
|
function parseJsonObject(text: string): Record<string, unknown> {
|
|
const trimmed = text.trim();
|
|
if (trimmed.length === 0) return {};
|
|
try {
|
|
return record(JSON.parse(trimmed) as unknown);
|
|
} catch {
|
|
const start = trimmed.indexOf("{");
|
|
const end = trimmed.lastIndexOf("}");
|
|
if (start >= 0 && end > start) {
|
|
try {
|
|
return record(JSON.parse(trimmed.slice(start, end + 1)) as unknown);
|
|
} catch {}
|
|
}
|
|
}
|
|
return {};
|
|
}
|
|
|
|
function shellQuote(value: string): string {
|
|
return `'${value.replace(/'/gu, `'"'"'`)}'`;
|
|
}
|
|
|
|
function statusText(result: CommandResult): string {
|
|
return result.stdout || result.stderr;
|
|
}
|
|
|
|
function keyValueLinesFromText(text: string): Record<string, string> {
|
|
const fields: Record<string, string> = {};
|
|
for (const line of text.split(/\r?\n/u)) {
|
|
const index = line.indexOf("\t");
|
|
if (index <= 0) continue;
|
|
fields[line.slice(0, index)] = line.slice(index + 1);
|
|
}
|
|
return fields;
|
|
}
|
|
|
|
function numericField(value: string | undefined): number | null {
|
|
if (value === undefined || value === "") return null;
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
}
|
|
|
|
function commaListField(value: string | undefined): string[] {
|
|
if (!value) return [];
|
|
return value.split(",").map((item) => item.trim()).filter(Boolean);
|
|
}
|
|
|
|
function splitWhitespaceField(value: string | undefined): string[] {
|
|
if (!value) return [];
|
|
return value.split(/\s+/u).filter(Boolean);
|
|
}
|
|
|
|
function compactCommandResult(result: CommandResult): Record<string, unknown> {
|
|
return {
|
|
command: compactCommand(result.command),
|
|
exitCode: result.exitCode,
|
|
stdoutBytes: result.stdout.length,
|
|
stdoutTail: result.exitCode === 0 && !result.timedOut ? "" : result.stdout.trim().slice(-2000),
|
|
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000),
|
|
timedOut: result.timedOut,
|
|
};
|
|
}
|
|
|
|
function compactCommandResultWithStdoutTail(result: CommandResult): Record<string, unknown> {
|
|
const compact = compactCommandResult(result);
|
|
compact.stdoutTail = result.stdout.trim().slice(-2000);
|
|
return compact;
|
|
}
|
|
|
|
function compactCommandResultRedacted(result: CommandResult, secrets: string[]): Record<string, unknown> {
|
|
const compact = compactCommandResult(result);
|
|
if (typeof compact.stderr === "string" && compact.stderr.length > 0) {
|
|
compact.stderr = redactKnownSecrets(compact.stderr, secrets);
|
|
}
|
|
return compact;
|
|
}
|
|
|
|
function redactKnownSecrets(text: string, secrets: string[]): string {
|
|
let next = text;
|
|
for (const secret of secrets.filter((item) => item.length > 0)) {
|
|
next = next.split(secret).join("<redacted>");
|
|
}
|
|
return next;
|
|
}
|
|
|
|
function parseJsonObject(text: string): Record<string, unknown> | null {
|
|
const trimmed = text.trim();
|
|
if (trimmed.length === 0) return null;
|
|
try {
|
|
const parsed = JSON.parse(trimmed) as unknown;
|
|
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record<string, unknown> : null;
|
|
} catch {
|
|
const objectText = firstJsonObjectText(trimmed);
|
|
if (objectText) {
|
|
try {
|
|
const parsed = JSON.parse(objectText) as unknown;
|
|
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record<string, unknown> : null;
|
|
} catch {}
|
|
}
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function firstJsonObjectText(text: string): string | null {
|
|
const start = text.indexOf("{");
|
|
if (start < 0) return null;
|
|
let depth = 0;
|
|
let inString = false;
|
|
let escaped = false;
|
|
for (let index = start; index < text.length; index += 1) {
|
|
const char = text[index];
|
|
if (inString) {
|
|
if (escaped) {
|
|
escaped = false;
|
|
} else if (char === "\\") {
|
|
escaped = true;
|
|
} else if (char === "\"") {
|
|
inString = false;
|
|
}
|
|
continue;
|
|
}
|
|
if (char === "\"") {
|
|
inString = true;
|
|
continue;
|
|
}
|
|
if (char === "{") depth += 1;
|
|
else if (char === "}") {
|
|
depth -= 1;
|
|
if (depth === 0) return text.slice(start, index + 1);
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function compactCommand(command: string[]): string[] {
|
|
const scriptIndex = command.indexOf("--");
|
|
if (scriptIndex >= 0 && scriptIndex + 1 < command.length) return [...command.slice(0, scriptIndex + 1), "<script omitted>"];
|
|
return command;
|
|
}
|