import { createHash, randomUUID } from "node:crypto"; import { closeSync, existsSync, mkdirSync, mkdtempSync, openSync, readFileSync, renameSync, rmSync, truncateSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { spawn } from "node:child_process"; import type { LinuxInstallTarget, ProviderAppsConfig } from "../../src/components/provider-apps/config"; import { runCommand, type CommandResult } from "./command"; const repositoryRoot = resolve(import.meta.dir, "../.."); const transPath = "/root/.local/bin/trans"; function option(args: string[], name: string): string | null { const index = args.indexOf(name); if (index < 0) return null; const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`); return value; } function selectedTarget(config: ProviderAppsConfig, args: string[]): [string, LinuxInstallTarget] { const id = option(args, "--target"); if (id === null) throw new Error("provider-apps linux requires --target "); const target = config.install.linux.targets[id]; if (target === undefined) throw new Error(`unknown Linux Provider target: ${id}`); return [id, target]; } function now(): string { return new Date().toISOString(); } function linuxJobRoot(config: ProviderAppsConfig, jobId: string): string { if (!/^[a-f0-9]{12}$/u.test(jobId)) throw new Error("provider-apps linux job id is invalid"); return resolve(repositoryRoot, config.runtime.stateDir, "linux-jobs", jobId); } function writeJobStatus(root: string, value: Record): void { mkdirSync(root, { recursive: true }); const path = join(root, "status.json"); const temporary = `${path}.tmp-${process.pid}`; writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, "utf8"); renameSync(temporary, path); } function readJobStatus(config: ProviderAppsConfig, jobId: string): Record { const path = join(linuxJobRoot(config, jobId), "status.json"); if (!existsSync(path)) throw new Error(`unknown provider-apps linux job: ${jobId}`); return JSON.parse(readFileSync(path, "utf8")) as Record; } function readRunnerPid(root: string): number | null { try { const pid = Number.parseInt(readFileSync(join(root, "runner.pid"), "utf8").trim(), 10); return Number.isInteger(pid) && pid > 0 ? pid : null; } catch { return null; } } function jobIdOption(args: string[]): string { const value = option(args, "--id") ?? option(args, "--job-id"); if (value === null) throw new Error("provider-apps linux job command requires --id "); return value; } function processRunning(pid: unknown): boolean { if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) return false; try { process.kill(Number(pid), 0); return true; } catch { return false; } } function parseEnv(path: string): Record { if (!existsSync(path)) throw new Error(`credential sourceRef is missing: ${path}`); const result: Record = {}; for (const line of readFileSync(path, "utf8").split(/\r?\n/u)) { const trimmed = line.trim(); if (trimmed.length === 0 || trimmed.startsWith("#")) continue; const match = /^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$/u.exec(trimmed); if (match === null) continue; let value = match[2] ?? ""; if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) value = value.slice(1, -1); result[match[1] ?? ""] = value; } return result; } function fingerprint(value: string): string { return `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 12)}`; } function fileSha256(path: string): string { const result = checked(runCommand(["sha256sum", path], repositoryRoot, { timeoutMs: 120_000 }), "hash benchmark file"); const digest = result.stdout.trim().split(/\s+/u)[0] ?? ""; if (!/^[0-9a-f]{64}$/u.test(digest)) throw new Error(`invalid SHA-256 output for ${path}`); return digest; } function shQuote(value: string): string { return `'${value.replaceAll("'", `'"'"'`)}'`; } function psQuote(value: string): string { return `'${value.replaceAll("'", "''")}'`; } function providerPorts(server: string): { host: string; control: number; data: number } { const parsed = new URL(server); const control = parsed.port.length > 0 ? Number(parsed.port) : parsed.protocol === "wss:" ? 443 : 80; const data = control === 8081 ? 8082 : control === 18082 ? 18084 : control + 2; if (parsed.hostname.length === 0 || !Number.isInteger(control) || !Number.isInteger(data) || data > 65535) { throw new Error(`Linux Provider server cannot derive TCP endpoints: ${server}`); } return { host: parsed.hostname, control, data }; } function checked(result: CommandResult, operation: string): CommandResult { if (result.exitCode !== 0) { const detail = result.stderr.trim() || result.stdout.trim() || `exitCode=${result.exitCode}`; throw new Error(`${operation} failed: ${detail}`); } return result; } function transferFailureDetail(result: CommandResult): string { return [result.stdout.trim(), result.stderr.trim()].filter((value) => value.length > 0).join("\n"); } function trans(route: string, operation: string[], timeoutMs = 60_000): CommandResult { return runCommand([transPath, route, ...operation], repositoryRoot, { timeoutMs }); } function bootstrapPowerShell(target: LinuxInstallTarget, remoteCommand: string, timeoutMs = 60_000): CommandResult { const source = [ "$ErrorActionPreference='Stop'", `$key=${psQuote(target.bootstrap.identityFile)}`, `$guest=${psQuote(target.bootstrap.guest)}`, `$remote=${psQuote(remoteCommand)}`, "& ssh.exe -i $key -o BatchMode=yes -o ConnectTimeout=8 $guest $remote", "$code=$LASTEXITCODE", "if ($code -ne 0) { throw \"remote SSH command failed with exit code $code\" }", "exit $code", ].join("; "); return trans(target.bootstrap.route, ["ps", source], timeoutMs); } function plan(config: ProviderAppsConfig, id: string, target: LinuxInstallTarget): Record { const values = parseEnv(target.credential.sourceRef.path); const token = values[target.credential.sourceRef.key] ?? ""; return { ok: token.length > 0, operation: "provider-apps.linux.plan", target: id, bootstrap: { mode: target.bootstrap.mode, route: target.bootstrap.route, guest: target.bootstrap.guest }, providerId: target.providerId, server: target.server, user: target.user, home: target.home, configPath: `${target.home}/.unidesk/provider.yaml`, credential: { sourceRef: target.credential.sourceRef.path, key: target.credential.sourceRef.key, present: token.length > 0, fingerprint: token.length > 0 ? fingerprint(token) : null, valuesPrinted: false, }, installer: `${config.publicExposure.publicBaseUrl}${config.install.routes.linuxInstaller}`, oneLine: `curl -fsSL ${config.publicExposure.publicBaseUrl}${config.install.routes.linuxInstaller} | bash`, service: target.systemdUnit, network: { bypassKubernetesServiceNat: target.network.bypassKubernetesServiceNat, systemdUnit: target.network.systemdUnit, endpoints: providerPorts(target.server), }, benchmark: target.benchmark, }; } function installNetworkBypass(target: LinuxInstallTarget): void { const helperPath = `/usr/local/lib/unideskcprovider/${target.providerId}-network-bypass.sh`; if (!target.network.bypassKubernetesServiceNat) { const script = [ "set -eu", `sudo systemctl disable --now ${shQuote(target.network.systemdUnit)} >/dev/null 2>&1 || true`, `sudo rm -f ${shQuote(`/etc/systemd/system/${target.network.systemdUnit}`)} ${shQuote(helperPath)}`, "sudo systemctl daemon-reload", ].join("; "); checked(bootstrapPowerShell(target, script), "remove Linux Provider network bypass"); return; } const endpoint = providerPorts(target.server); if (!/^\d{1,3}(?:\.\d{1,3}){3}$/u.test(endpoint.host)) { throw new Error(`Kubernetes Service NAT bypass requires an IPv4 Provider host: ${endpoint.host}`); } const helper = `#!/bin/sh set -eu action=\${1:-start} uid=$(id -u ${shQuote(target.user)}) for port in ${endpoint.control} ${endpoint.data}; do if [ "$action" = start ]; then iptables -t nat -C OUTPUT -d ${endpoint.host}/32 -p tcp --dport "$port" -m owner --uid-owner "$uid" -j RETURN 2>/dev/null || \\ iptables -t nat -I OUTPUT 1 -d ${endpoint.host}/32 -p tcp --dport "$port" -m owner --uid-owner "$uid" -j RETURN else while iptables -t nat -C OUTPUT -d ${endpoint.host}/32 -p tcp --dport "$port" -m owner --uid-owner "$uid" -j RETURN 2>/dev/null; do iptables -t nat -D OUTPUT -d ${endpoint.host}/32 -p tcp --dport "$port" -m owner --uid-owner "$uid" -j RETURN done fi done `; const unit = `[Unit] Description=UniDesk Native Provider Kubernetes Service NAT bypass After=network-online.target k3s-agent.service Before=${target.systemdUnit} Wants=network-online.target [Service] Type=oneshot RemainAfterExit=yes ExecStart=${helperPath} start ExecStop=${helperPath} stop [Install] WantedBy=multi-user.target `; const helperBase64 = Buffer.from(helper, "utf8").toString("base64"); const unitBase64 = Buffer.from(unit, "utf8").toString("base64"); const script = [ "set -eu", `sudo systemctl stop ${shQuote(target.network.systemdUnit)} >/dev/null 2>&1 || true`, `printf %s ${shQuote(helperBase64)} | base64 -d | sudo install -D -m 0755 /dev/stdin ${shQuote(helperPath)}`, `printf %s ${shQuote(unitBase64)} | base64 -d | sudo install -D -m 0644 /dev/stdin ${shQuote(`/etc/systemd/system/${target.network.systemdUnit}`)}`, "sudo systemctl daemon-reload", `sudo systemctl enable --now ${shQuote(target.network.systemdUnit)}`, ].join("; "); checked(bootstrapPowerShell(target, script), "install Linux Provider network bypass"); } function applyNow(config: ProviderAppsConfig, id: string, target: LinuxInstallTarget, args: string[]): Record { if (!args.includes("--confirm")) throw new Error("provider-apps linux apply requires --confirm"); const values = parseEnv(target.credential.sourceRef.path); const token = values[target.credential.sourceRef.key] ?? ""; if (token.length === 0) throw new Error(`${target.credential.sourceRef.path}.${target.credential.sourceRef.key} is empty`); const local = mkdtempSync(join(tmpdir(), "unideskcprovider-linux-")); const remote = `/tmp/unideskcprovider-linux-${process.pid}-${Date.now()}`; const windowsStage = `${target.bootstrap.stagingDirectory}\\${process.pid}-${Date.now()}`; const localConfig = join(local, "provider.yaml"); const localToken = join(local, target.credential.targetFile); const configYaml = [ `id: ${target.providerId}`, `server: ${target.server}`, `credentialFile: ${target.credential.targetFile}`, `pythonRuntime: ${config.install.defaults.pythonRuntime}`, `autoStart: ${String(config.install.defaults.autoStart)}`, "", ].join("\n"); try { installNetworkBypass(target); writeFileSync(localConfig, configYaml, { encoding: "utf8", mode: 0o600 }); writeFileSync(localToken, `${token}\n`, { encoding: "utf8", mode: 0o600 }); checked(trans(target.bootstrap.route, ["ps", `New-Item -ItemType Directory -Force -Path ${psQuote(windowsStage)} | Out-Null`]), "prepare Windows SSH staging"); checked(trans(target.bootstrap.route, ["upload", localConfig, `${windowsStage}\\provider.yaml`, "--allow-text-transfer"], 120_000), "upload provider.yaml to SSH bridge"); checked(trans(target.bootstrap.route, ["upload", localToken, `${windowsStage}\\${target.credential.targetFile}`, "--allow-text-transfer"], 120_000), "upload Provider credential to SSH bridge"); const scpSource = [ "$ErrorActionPreference='Stop'", `$key=${psQuote(target.bootstrap.identityFile)}`, `$guest=${psQuote(target.bootstrap.guest)}`, `$stage=${psQuote(windowsStage)}`, `$remote=${psQuote(remote)}`, "& ssh.exe -i $key -o BatchMode=yes -o ConnectTimeout=8 $guest \"install -d -m 700 $remote\"", "if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }", "& scp.exe -i $key -o BatchMode=yes (Join-Path $stage 'provider.yaml') \"${guest}:$remote/provider.yaml\"", "if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }", `& scp.exe -i $key -o BatchMode=yes (Join-Path $stage ${psQuote(target.credential.targetFile)}) ` + `\"\${guest}:$remote/${target.credential.targetFile}\"`, "exit $LASTEXITCODE", ].join("; "); checked(trans(target.bootstrap.route, ["ps", scpSource], 180_000), "copy configuration through Windows SSH bridge"); const script = [ "set -eu", `sudo install -d -m 700 -o ${shQuote(target.user)} -g ${shQuote(target.user)} ${shQuote(`${target.home}/.unidesk`)}`, `sudo install -m 600 -o ${shQuote(target.user)} -g ${shQuote(target.user)} ${shQuote(`${remote}/provider.yaml`)} ${shQuote(`${target.home}/.unidesk/provider.yaml`)}`, `sudo install -m 600 -o ${shQuote(target.user)} -g ${shQuote(target.user)} ${shQuote(`${remote}/${target.credential.targetFile}`)} ${shQuote(`${target.home}/.unidesk/${target.credential.targetFile}`)}`, `rm -rf ${shQuote(remote)}`, `sudo -u ${shQuote(target.user)} -H bash -lc ${shQuote(`curl -fsSL ${config.publicExposure.publicBaseUrl}${config.install.routes.linuxInstaller} | bash`)}`, ].join("; "); const installed = checked(bootstrapPowerShell(target, script, 300_000), "install Linux Provider"); if (!installed.stdout.includes("OK operation=unideskcprovider.install")) { throw new Error(`install Linux Provider returned no completion marker: ${installed.stdout.trim() || "empty stdout"}`); } return { ...plan(config, id, target), ok: true, operation: "provider-apps.linux.apply", mutation: true, installerOutput: installed.stdout.trim(), }; } finally { rmSync(local, { recursive: true, force: true }); bootstrapPowerShell(target, `rm -rf ${shQuote(remote)}`); trans(target.bootstrap.route, ["ps", `Remove-Item -Recurse -Force -ErrorAction SilentlyContinue ${psQuote(windowsStage)}`]); } } function submitApply(config: ProviderAppsConfig, id: string, args: string[]): Record { if (!args.includes("--confirm")) throw new Error("provider-apps linux apply requires --confirm"); const jobId = randomUUID().replaceAll("-", "").slice(0, 12); const root = linuxJobRoot(config, jobId); mkdirSync(root, { recursive: true }); const createdAt = now(); writeJobStatus(root, { jobId, operation: "provider-apps.linux.apply", target: id, status: "queued", createdAt, }); const stdout = openSync(join(root, "stdout.log"), "a"); const stderr = openSync(join(root, "stderr.log"), "a"); const child = spawn(process.execPath, [ resolve(repositoryRoot, "scripts/cli.ts"), "provider-apps", "linux", "apply-worker", "--target", id, "--job-id", jobId, "--confirm", ], { cwd: repositoryRoot, detached: true, stdio: ["ignore", stdout, stderr], env: { ...process.env, UNIDESK_PROVIDER_APPS_LINUX_WORKER: "1" }, }); if (child.pid === undefined) { closeSync(stdout); closeSync(stderr); writeJobStatus(root, { jobId, operation: "provider-apps.linux.apply", target: id, status: "failed", error: "Linux Provider apply worker did not return a PID", createdAt, finishedAt: now(), }); throw new Error("Linux Provider apply worker did not return a PID"); } writeFileSync(join(root, "runner.pid"), `${child.pid}\n`, "utf8"); child.unref(); closeSync(stdout); closeSync(stderr); return { ok: true, operation: "provider-apps.linux.apply", mode: "async-job", mutation: true, target: id, jobId, status: "queued", pid: child.pid, createdAt, next: { status: `bun scripts/cli.ts provider-apps linux apply-status --target ${id} --id ${jobId}`, logs: `bun scripts/cli.ts provider-apps linux apply-logs --target ${id} --id ${jobId}`, }, }; } function applyWorker(config: ProviderAppsConfig, id: string, target: LinuxInstallTarget, args: string[]): Record { if (process.env.UNIDESK_PROVIDER_APPS_LINUX_WORKER !== "1") throw new Error("provider-apps linux apply-worker is internal"); const jobId = jobIdOption(args); const root = linuxJobRoot(config, jobId); const queued = readJobStatus(config, jobId); const running = { ...queued, status: "running", pid: process.pid, startedAt: now(), updatedAt: now() }; writeJobStatus(root, running); try { const result = applyNow(config, id, target, args); const finished = { ...running, status: "succeeded", result, finishedAt: now(), updatedAt: now() }; writeJobStatus(root, finished); return { ok: true, operation: "provider-apps.linux.apply-worker", jobId, status: "succeeded" }; } catch (error) { const message = error instanceof Error ? error.message : String(error); writeJobStatus(root, { ...running, status: "failed", error: message, finishedAt: now(), updatedAt: now() }); return { ok: false, operation: "provider-apps.linux.apply-worker", jobId, status: "failed", error: message }; } } function applyStatus(config: ProviderAppsConfig, args: string[]): Record { const jobId = jobIdOption(args); const root = linuxJobRoot(config, jobId); const status = readJobStatus(config, jobId); const recordedStatus = String(status.status ?? "unknown"); const runnerPid = typeof status.pid === "number" ? status.pid : readRunnerPid(root); const runnerRunning = processRunning(runnerPid); const workerExitedEarly = (recordedStatus === "queued" || recordedStatus === "running") && !runnerRunning; return { ...status, ...(workerExitedEarly ? { status: "failed", recordedStatus, error: "Linux Provider apply worker exited before recording a terminal status; inspect apply-logs", } : {}), ok: recordedStatus !== "failed" && !workerExitedEarly, operation: "provider-apps.linux.apply-status", runnerPid, running: recordedStatus === "running" && runnerRunning, }; } function applyLogs(config: ProviderAppsConfig, args: string[]): Record { const jobId = jobIdOption(args); const root = linuxJobRoot(config, jobId); readJobStatus(config, jobId); const tail = (name: string): string => { const path = join(root, name); if (!existsSync(path)) return ""; const contents = readFileSync(path); return contents.subarray(Math.max(0, contents.length - config.runtime.maxLogTailBytes)).toString("utf8"); }; return { ok: true, operation: "provider-apps.linux.apply-logs", jobId, stdoutTail: tail("stdout.log"), stderrTail: tail("stderr.log"), }; } function status(id: string, target: LinuxInstallTarget): Record { const script = [ "set -eu", `printf 'serviceActive=%s\\n' "$(systemctl is-active ${shQuote(target.systemdUnit)} 2>/dev/null || true)"`, `printf 'serviceEnabled=%s\\n' "$(systemctl is-enabled ${shQuote(target.systemdUnit)} 2>/dev/null || true)"`, `printf 'networkServiceActive=%s\\n' "$(systemctl is-active ${shQuote(target.network.systemdUnit)} 2>/dev/null || true)"`, `printf 'networkServiceEnabled=%s\\n' "$(systemctl is-enabled ${shQuote(target.network.systemdUnit)} 2>/dev/null || true)"`, `test -f ${shQuote(`${target.home}/.unidesk/provider.yaml`)} && echo configPresent=true || echo configPresent=false`, `test -s ${shQuote(`${target.home}/.unidesk/${target.credential.targetFile}`)} && echo credentialPresent=true || echo credentialPresent=false`, `systemctl show ${shQuote(target.systemdUnit)} -p MainPID -p NRestarts --value 2>/dev/null | sed -n '1,2p'`, ].join("; "); let result = trans(id, ["argv", "bash", "-lc", script]); if (result.exitCode !== 0) result = checked(bootstrapPowerShell(target, script), "read Linux Provider status"); return { ok: result.stdout.includes("serviceActive=active"), operation: "provider-apps.linux.status", target: id, output: result.stdout.trim() }; } function logs(id: string, target: LinuxInstallTarget): Record { const script = `journalctl -u ${shQuote(target.systemdUnit)} --no-pager -n ${target.logsTailLines}`; let result = trans(id, ["argv", "bash", "-lc", script]); if (result.exitCode !== 0) result = checked(bootstrapPowerShell(target, script), "read Linux Provider logs"); return { ok: true, operation: "provider-apps.linux.logs", target: id, tail: result.stdout.trim() }; } async function transAsync( route: string, operation: string[], timeoutMs: number, streamProgress = false, ): Promise<{ exitCode: number; stdout: string; stderr: string; elapsedMs: number }> { const started = performance.now(); const child = Bun.spawn([transPath, route, ...operation], { cwd: repositoryRoot, env: { ...process.env, UNIDESK_TRAN_RUNTIME_TIMEOUT_MS: String(timeoutMs) }, stdout: "pipe", stderr: "pipe", }); const timer = setTimeout(() => child.kill(), timeoutMs); const stderrPromise = streamProgress ? (async () => { const reader = child.stderr.getReader(); const decoder = new TextDecoder(); let result = ""; while (true) { const next = await reader.read(); if (next.done) break; const text = decoder.decode(next.value, { stream: true }); result += text; process.stderr.write(text); } result += decoder.decode(); return result; })() : new Response(child.stderr).text(); const [exitCode, stdout, stderr] = await Promise.all([ child.exited, new Response(child.stdout).text(), stderrPromise, ]); clearTimeout(timer); return { exitCode, stdout, stderr, elapsedMs: Math.round(performance.now() - started) }; } async function benchmark(id: string, target: LinuxInstallTarget, args: string[]): Promise> { if (!args.includes("--confirm")) throw new Error("provider-apps linux benchmark requires --confirm"); const spec = target.benchmark; const transferTimeoutMs = spec.transferTimeoutSeconds * 1000; const runId = randomUUID(); const commandSamples: Array<{ round: number; lane: number; elapsedMs: number }> = []; for (let round = 0; round < spec.commandRounds; round += 1) { const results = await Promise.all(Array.from({ length: spec.concurrency }, async (_, lane) => { const marker = `${runId}-${round}-${lane}`; const result = await transAsync(id, ["argv", "bash", "-lc", `printf '%s\\n' ${shQuote(marker)}`], 60_000); if (result.exitCode !== 0 || result.stdout.trim() !== marker) throw new Error(`command lane failed round=${round} lane=${lane}: ${result.stderr || result.stdout}`); return { round, lane, elapsedMs: result.elapsedMs }; })); commandSamples.push(...results); } const local = mkdtempSync(join(tmpdir(), "unideskcprovider-benchmark-")); const payload = join(local, "payload.bin"); writeFileSync(payload, "", { mode: 0o600 }); truncateSync(payload, spec.transferBytes); const payloadSha = fileSha256(payload); const transfers: Array> = []; const remoteFiles = new Set(); try { for (let round = 0; round < spec.transferRounds; round += 1) { const remote = `/tmp/unideskcprovider-benchmark-${runId}-${round}.bin`; remoteFiles.add(remote); process.stderr.write(`${JSON.stringify({ event: "provider-apps.linux.benchmark.stage", stage: "upload", round, bytes: spec.transferBytes })}\n`); const upload = await transAsync(id, ["upload", payload, remote, "--runtime-timeout-ms", String(transferTimeoutMs)], transferTimeoutMs, true); if (upload.exitCode !== 0) throw new Error(`upload failed round=${round}: ${transferFailureDetail(upload)}`); const localCopy = join(local, `download-${round}.bin`); process.stderr.write(`${JSON.stringify({ event: "provider-apps.linux.benchmark.stage", stage: "download", round, bytes: spec.transferBytes })}\n`); const download = await transAsync(id, ["download", remote, localCopy, "--runtime-timeout-ms", String(transferTimeoutMs)], transferTimeoutMs, true); if (download.exitCode !== 0) throw new Error(`download failed round=${round}: ${transferFailureDetail(download)}`); const sha = fileSha256(localCopy); if (sha !== payloadSha) throw new Error(`download SHA mismatch round=${round}`); rmSync(localCopy, { force: true }); await transAsync(id, ["argv", "rm", "-f", remote], 60_000); remoteFiles.delete(remote); transfers.push({ round, uploadElapsedMs: upload.elapsedMs, uploadBytesPerSecond: Math.round((spec.transferBytes * 1000) / upload.elapsedMs), downloadElapsedMs: download.elapsedMs, downloadBytesPerSecond: Math.round((spec.transferBytes * 1000) / download.elapsedMs), }); } } finally { for (const remote of remoteFiles) await transAsync(id, ["argv", "rm", "-f", remote], 60_000); rmSync(local, { recursive: true, force: true }); } checked(bootstrapPowerShell(target, `sudo systemctl restart ${shQuote(target.systemdUnit)}`), "restart Linux Provider"); const recoveryStarted = Date.now(); let recovered = false; while (Date.now() - recoveryStarted < spec.readyTimeoutSeconds * 1000) { const probe = trans(id, ["argv", "hostname"], 15_000); if (probe.exitCode === 0) { recovered = true; break; } await Bun.sleep(1_000); } if (!recovered) throw new Error(`Provider did not recover within ${spec.readyTimeoutSeconds}s`); const poolProbes = await Promise.all(Array.from({ length: spec.poolProbeConcurrency }, async (_, lane) => { const marker = `${runId}-pool-${lane}`; const result = await transAsync(id, ["argv", "bash", "-lc", `printf '%s\\n' ${shQuote(marker)}`], 60_000); if (result.exitCode !== 0 || result.stdout.trim() !== marker) throw new Error(`pool probe failed lane=${lane}: ${result.stderr || result.stdout}`); return { lane, elapsedMs: result.elapsedMs }; })); const elapsedMs = Date.now() - recoveryStarted; return { ok: true, operation: "provider-apps.linux.benchmark", target: id, runId, command: { concurrency: spec.concurrency, rounds: spec.commandRounds, samples: commandSamples }, transfer: { bytesPerFile: spec.transferBytes, concurrency: spec.concurrency, rounds: spec.transferRounds, sha256: payloadSha, samples: transfers }, recovery: { elapsedMs, timeoutSeconds: spec.readyTimeoutSeconds }, pool: { concurrency: spec.poolProbeConcurrency, readyClaims: poolProbes.length, samples: poolProbes }, }; } export async function runProviderAppsLinuxCommand(config: ProviderAppsConfig, args: string[]): Promise> { if (args.length === 0 || args[0] === "help" || args[0] === "--help") { return { ok: true, operation: "provider-apps.linux.help", usage: [ "bun scripts/cli.ts provider-apps linux plan --target ", "bun scripts/cli.ts provider-apps linux apply --target --confirm", "bun scripts/cli.ts provider-apps linux apply-status --target --id ", "bun scripts/cli.ts provider-apps linux apply-logs --target --id ", "bun scripts/cli.ts provider-apps linux status --target ", "bun scripts/cli.ts provider-apps linux logs --target ", "bun scripts/cli.ts provider-apps linux benchmark --target --confirm", ], }; } const [id, target] = selectedTarget(config, args); if (args[0] === "plan") return plan(config, id, target); if (args[0] === "apply") return submitApply(config, id, args); if (args[0] === "apply-worker") return applyWorker(config, id, target, args); if (args[0] === "apply-status") return applyStatus(config, args); if (args[0] === "apply-logs") return applyLogs(config, args); if (args[0] === "status") return status(id, target); if (args[0] === "logs") return logs(id, target); if (args[0] === "benchmark") return await benchmark(id, target, args); throw new Error(`unsupported provider-apps linux action: ${args[0]}`); }