feat: 增加YAML受控swap扩容

This commit is contained in:
Codex
2026-07-16 13:26:11 +02:00
parent dcdca121e2
commit d5833500fd
9 changed files with 686 additions and 7 deletions
+7 -1
View File
@@ -1,6 +1,6 @@
---
name: unidesk-ops
description: UniDesk 手动运维 CLI — `server``gc`PK01 `platform-db postgres`、platform-infra egress proxy 和 k3s dependency proxy benchmark 运维。覆盖主 server 启停、健康检查、swap、日志、Docker 镜像清理、磁盘 GC、服务重建/重启、PK01 host PostgreSQL、D601/D518 proxyserver 视角流量测速和 k3s 真实依赖拉取 benchmark。用户提到 server start、server status、server swap、server rebuild、server restart、gc、磁盘清理、platform-db、PK01 PostgreSQL、egress-proxy traffic、proxy 测速、k3s benchmark、apk/npm/go/git mirror 拉取测速时使用。
description: UniDesk 手动运维 CLI — `server``gc`YAML 声明的 `platform-db postgres` 与 host swap、platform-infra egress proxy 和 k3s dependency proxy benchmark 运维。覆盖主 server 启停、健康检查、swap、日志、Docker 镜像清理、磁盘 GC、服务重建/重启、host PostgreSQL、D601/D518 proxyserver 视角流量测速和 k3s 真实依赖拉取 benchmark。用户提到 server start、server status、server swap、host swap 扩容、server rebuild、server restart、gc、磁盘清理、platform-db、PostgreSQL、egress-proxy traffic、proxy 测速、k3s benchmark、apk/npm/go/git mirror 拉取测速时使用。
---
# UniDesk Ops
@@ -16,6 +16,7 @@ bun scripts/cli.ts server stop
bun scripts/cli.ts server logs --tail-bytes 12000
bun scripts/cli.ts gc plan
bun scripts/cli.ts platform-db postgres status
bun scripts/cli.ts platform-db postgres swap plan --config config/platform-db/postgres-nc01.yaml
bun scripts/cli.ts platform-infra egress-proxy traffic --target D601 --sample-seconds 15
bun scripts/cli.ts platform-infra egress-proxy k3s-build-benchmark --targets D601,D518 --profile real-deps-500m --dry-run
```
@@ -82,6 +83,11 @@ D601 若需要让 `sub2api-egress-proxy` 绕开 pod overlay,可在 YAML 中显
- backend-core 运行面恢复 healthy 后,除非用户明确要求,不主动 rebuild/restart/替换 backend-core。
- Master server 不作为通用构建机;Docker/Rust/Go/前端高 CPU 构建必须走批准的 CI/运行面。
- GC 默认 plan 只读,真实删除必须显式 `run --confirm` 并遵循 allowlist/retention。
- 已启用的远端 host swap 扩容使用 `platform-db postgres swap plan|apply --confirm|status`
- 总容量、主文件、扩展文件、`swappiness` 和 sysctl 文件只来自目标 YAML 的 `node.swap`
- 只新增缺失容量,不 `swapoff` 既有区域,不操作业务进程;
- `apply --confirm` 只提交远端异步 job,随后用返回的 `--job-id` 查询;
- swap 始终不计入 WebProbe 的真实 `MemAvailable` 门槛。
- Secret、database URL 和 API key 不打印完整值。
- k3s proxy benchmark 是运行面验收,不是源码层测试;长任务必须 start/status/logs/traffic 短轮询,结束或不达标时必须 cleanup。
@@ -1,5 +1,11 @@
# Platform Infra Operations
PK01 PostgreSQL, platform-infra egress proxy and k3s dependency benchmark work must use controlled UniDesk CLI commands.
Host PostgreSQL、对应节点的 swap 扩容、platform-infra egress proxy k3s dependency benchmark 必须使用受控 UniDesk CLI
已存在 swap 的节点不得直接 `swapoff` 后原地扩容。使用
`platform-db postgres swap plan|apply --confirm|status`,从目标 PostgreSQL YAML 的
`node.swap` 读取总容量、主文件、扩展文件、`swappiness` 和 sysctl 文件;命令保留既有
swap 在线,仅增加缺失容量。swap 只作为 OOM 安全垫,不计入 WebProbe 的
`MemAvailable` 启动资格。
For k3s dependency proxy benchmarks, use real remote dependencies and short status/log/traffic polling. Clean up benchmark Jobs when complete or clearly underperforming.
+4 -1
View File
@@ -28,8 +28,11 @@ node:
requireSwap: true
swap:
ensure: true
size: 12GiB
size: 64GiB
path: /swapfile
expansionPath: /swapfile-unidesk
swappiness: 10
sysctlFile: /etc/sysctl.d/90-unidesk-swap.conf
postgres:
package:
@@ -0,0 +1,27 @@
# R1.2.7 任务报告
## 结果
- NC01 swap 已从 12 GiB 扩展为声明容量 64 GiB:保留 `/swapfile` 12 GiB 在线,新增 `/swapfile-unidesk` 52 GiB,未执行 `swapoff`,未停止、重启或修改 Webterm。
- 唯一配置真相为 `config/platform-db/postgres-nc01.yaml#node.swap`,声明总容量、主文件、扩展文件、`swappiness=10``/etc/sysctl.d/90-unidesk-swap.conf`
- 新增通用 `platform-db postgres swap plan|apply --confirm|status [--job-id]`:一次 plan 披露磁盘余量、缺失容量、fstab、声明/内核容量、物理内存和 WebProbe swap 排除语义;apply 只提交远端异步 job。
- 执行 job `swap-20260716132336-3785364` 成功。声明容量为 68,719,476,736 bytes,内核 swap 为 68,719,468,544 bytes,可用 swap 约 54 GiB;主文件与扩展文件均为 0600、active 且写入 fstab,运行态和持久化 `swappiness` 均为 10`policySync.inSync=true`
- PostgreSQL 原入口在扩容后仍为 healthy,TLS 连接、服务、角色、数据库和备份状态未受影响。
## 物理内存与受控 GC
- 扩容后 `gc remote NC01 memory-distribution` 仍只以 `/proc/meminfo``MemAvailable` 判定,swap 未计入 4 GiB WebProbe 门槛。
- 一次 YAML 白名单 cgroup 回收 job `nc01-memory-pressure-1784201061-3790243` 成功执行两项,k3s 与 kubepods cgroup 合计下降约 692 MiB;并发工作负载回填导致 job 窗口 `MemAvailable` 从 3,848,298,496 降至 3,758,592,000 bytes,未稳定达到 4 GiB,因此停止重复 GC。
- 当前主要真实内存仍为 k3s-server、Kafka、Argo CD、Tempo 与持续增长的 AgentRun runner 子进程泄露。AgentRun zombie 从 130 增至 188,说明仅增加 `tini` 尚未覆盖 Bun 直接子进程回收;继续在既有 AgentRun runtime reliability 任务中修复,不以 swap 掩盖泄露。
## 验证
- `bun --check scripts/src/platform-db-host-swap.ts`
- `bun --check scripts/src/platform-db.ts`
- `bun scripts/cli.ts check --syntax-only`11/11 通过。
- `bun scripts/cli.ts platform-db postgres swap plan --config config/platform-db/postgres-nc01.yaml`
- `bun scripts/cli.ts platform-db postgres swap status --config config/platform-db/postgres-nc01.yaml`
- `bun scripts/cli.ts platform-db postgres status --config config/platform-db/postgres-nc01.yaml`
- `git diff --check`
未运行 Vitest,未启动 WebProbe,未触碰 PK01 配置或运行面。
@@ -34,6 +34,9 @@
#### R1.2.6 [completed]
回滚错误的 [UniDesk PR #2302](https://github.com/pikasTech/unidesk/pull/2302):恢复 WebProbe 只以真实 `/proc/meminfo` `MemAvailable` 超过 4 GiB 作为人工启动资格、sentinel 在低于 4 GiB 时跳过本轮的 YAML-first 门禁;不得在 2 GiB 错误基线上继续修补,不启动 WebProbe且不触碰 Webterm,完成任务后将详细报告写入[任务报告](./details/web-observe-runtime-reliability/R1.2.6_Task_Report.md)。
#### R1.2.7 [completed]
完成 [UniDesk #1710](https://github.com/pikasTech/unidesk/issues/1710) 的 NC01 64 GiB swap 安全垫与受控扩容能力:以 `config/platform-db/postgres-nc01.yaml` 为唯一真相,补齐已启用 swapfile 的安全扩容 plan/status/apply 与可见错误;运行面扩容不得触碰 Webterm,不得把 swap 计入 WebProbe 4 GiB 物理内存门槛,最终经 Git、PR 与自动链收敛,并继续评估业务侧真实内存释放,完成任务后将详细报告写入[任务报告](./details/web-observe-runtime-reliability/R1.2.7_Task_Report.md)。
### R1.3
测试与验收:https://github.com/pikasTech/unidesk/issues/1842;覆盖生命周期、stale/protected 分类和失败路径,提交独立 PR;合并后只等待自动 CI/CD 发布链,使用正式入口验证 observer/Chrome 退出、内存压力缓解且健康运行面不受扰动,完成任务后将详细报告写入[任务报告](./details/web-observe-runtime-reliability/R1.3_Task_Report.md)。
+1
View File
@@ -128,6 +128,7 @@
- Master server is a consumer/control host, not the runtime location. Do not deploy Sub2API, PostgreSQL, Redis, or heavy validation loops on master server.
- Sub2API active/standby placement is selected by YAML, not by ad hoc runtime patches. A standby target must render without a local PostgreSQL StatefulSet, keep the Sub2API app and local Redis cache scaled to zero, use only ephemeral Redis storage if Redis is later activated, and omit public exposure, HTTPS egress proxy, and account sentinel resources unless YAML explicitly promotes that target. An externally backed active target connects directly to the YAML-declared external PostgreSQL endpoint with `sslmode=require`, keeps durable app state outside the runtime node, and uses local Redis only as ephemeral cache. Host-Docker active targets such as PK01 are still Sub2API platform targets; `codex-pool plan|sync|validate` has a host-Docker adapter for PK01, while sentinel image/report/probe and parts of trace remain target-capability-specific and must not be treated as PK01 runtime health failures when they require k3s resources. Multiple externally backed active targets may coexist when YAML declares distinct target ids, host routes, public URLs, FRP remote ports or local edge bindings, and Secret sources; target-scoped operations must use `--target <id>` and must not treat one target's URL or Secret as a fallback for another. Promotion or failback must be applied by editing `config/platform-infra/sub2api.yaml` and running the same `platform-infra sub2api --target <id>` CLI path.
- External platform PostgreSQL endpoints for Sub2API are produced by the platform DB YAML and its `platform-db postgres` CLI. Cross-node Sub2API consumers connect directly to that endpoint; the master server is not a PostgreSQL data-plane relay. DNS aliases are optional when the exported `DATABASE_URL` uses a reachable IP with `sslmode=require`; current PK01-specific rules live in `docs/reference/pk01.md`.
- Host PostgreSQL 节点的 swap 容量由对应 `config/platform-db/*.yaml#node.swap` 唯一拥有。已有 swap 在线时使用 `platform-db postgres swap plan|apply --confirm|status` 追加 YAML 声明的缺失容量;不得先 `swapoff` 现有区域,不得通过全量 PostgreSQL apply 顺带扩容。状态必须分别披露声明容量、内核可用容量、fstab、`swappiness` 和 `swapExcludedFromWebProbeAvailability=true`。
- Remote k3s targets that share the already-initialized PK01 Sub2API database should use YAML-controlled `target.runtime.autoSetup=false` and a rendered runtime config initContainer instead of rerunning Sub2API auto setup. The app's migration check path is latency-sensitive because it checks many migration rows one by one; repeated remote auto setup against PK01 can time out without indicating DB corruption. Required PostgreSQL `pg_hba` entries and host firewall forward allowances must be declared in YAML and applied through the controlled platform DB/server paths rather than by changing DB passwords, connection strings, or data.
- Sub2API account sentinel, public exposure, and HTTPS egress proxy are target-scoped YAML decisions. The active target may run them when YAML enables them; the standby G14 target must stay deployed but inactive until YAML promotion. `sentinel.enabledOnTargets` is the authority for where Codex-pool sentinel image, CronJob, Secret and state resources are expected; disabled targets should report sentinel validation as skipped instead of failing on missing runtime sentinel objects. Do not create a second sentinel, FRP client, public management surface, or edge proxy by hand; enable or move those resources only through the target YAML and the `platform-infra sub2api` / `codex-pool --target` CLI paths.
+1 -1
View File
@@ -78,7 +78,7 @@ export function rootHelp(): unknown {
{ command: "platform-infra sub2api|langbot|n8n|webterm|wechat-archive ...", description: "Deploy platform-infra services such as Sub2API, LangBot, n8n and Web Terminal, manage YAML-controlled public Caddy/FRP exposure and WeChat archive workflows, and inspect status/logs without printing secrets." },
{ command: "pikaoa attachments-backup ...", description: "管理 PikaOA 生产附件离机备份控制面;开发与生产运行面由 PaC 自动交付。" },
{ command: "secrets plan|sync|status", description: "Plan, push and inspect YAML-declared local secret source keys to Kubernetes Secrets without printing or reverse-engineering values." },
{ command: "platform-db postgres plan|status|export-secrets|apply", description: "Manage YAML-declared host-native PostgreSQL 16 on PK01 for Sub2API/platform state, with PostgreSQL native TLS and redacted secret exports." },
{ command: "platform-db postgres plan|status|export-secrets|apply|swap", description: "Manage YAML-declared host-native PostgreSQL and its node swap capacity through bounded plans, asynchronous mutation, status, TLS and redacted secret output." },
{ command: "hwlab cd audit --env dev | hwlab cd status --env dev | hwlab cd apply --env dev --dry-run", description: "Archived HWLAB DEV CD diagnostics only; current HWLAB rollout uses NC01 GitOps." },
{ command: "code-agent-sandbox", description: "Independent Code Agent Sandbox service skeleton for adapter, mode, and credential-boundary diagnostics." },
{ command: "schedule list|get|runs|run|retry-run|delete", description: "Manage backend-core scheduled tasks and run history; schedule run <id> supports --wait-ms N and retry-run reuses the failed run's schedule." },
+592
View File
@@ -0,0 +1,592 @@
import type { UniDeskConfig } from "./config";
import { parseJsonOutput } from "./platform-infra-ops-library";
import { runSshCommandCapture, type SshCaptureResult } from "./ssh";
export interface HostSwapDeclaration {
configPath: string;
nodeId: string;
route: string;
minDiskFreeGiB: number;
swap: {
ensure: boolean;
size: string;
path: string;
expansionPath: string;
swappiness: number;
sysctlFile: string;
};
}
interface HostSwapOptions {
configPath: string;
confirm: boolean;
full: boolean;
raw: boolean;
jobId: string | null;
}
type HostSwapDeclarationLoader = (configPath: string) => HostSwapDeclaration;
const defaultConfigPath = "config/platform-db/postgres-pk01.yaml";
export async function runPlatformDbHostSwapCommand(
config: UniDeskConfig,
action: string,
args: string[],
loadDeclaration: HostSwapDeclarationLoader,
): Promise<Record<string, unknown>> {
const options = parseOptions(args);
const declaration = loadDeclaration(options.configPath);
if (!declaration.swap.ensure) {
return {
ok: false,
error: "host-swap-management-disabled-by-yaml",
mutation: false,
configSource: `${declaration.configPath}#node.swap.ensure`,
};
}
if (action === "plan") return await plan(config, declaration, options);
if (action === "status") return await status(config, declaration, options);
if (action === "apply") return await apply(config, declaration, options);
return { ok: false, error: "unsupported-platform-db-swap-command", action, help: swapHelp(declaration.configPath) };
}
export function swapHelp(configPath = defaultConfigPath): Record<string, unknown> {
return {
command: "platform-db postgres swap plan|status|apply",
output: "json",
usage: [
`bun scripts/cli.ts platform-db postgres swap plan --config ${configPath}`,
`bun scripts/cli.ts platform-db postgres swap status --config ${configPath}`,
`bun scripts/cli.ts platform-db postgres swap apply --config ${configPath} --confirm`,
`bun scripts/cli.ts platform-db postgres swap status --config ${configPath} --job-id <id>`,
],
safety: {
sourceOfTruth: `${configPath}#node.swap`,
asynchronousMutation: true,
expansion: "keep every active swap area online and add only the YAML-declared missing capacity",
webProbe: "swap remains excluded from the WebProbe MemAvailable admission threshold",
},
};
}
function parseOptions(args: string[]): HostSwapOptions {
const options: HostSwapOptions = { configPath: defaultConfigPath, confirm: false, full: false, raw: false, jobId: null };
for (let index = 0; index < args.length; index += 1) {
const arg = args[index];
if (arg === "--config") {
const value = args[index + 1];
if (value === undefined || value.trim().length === 0) throw new Error("--config requires a non-empty path");
options.configPath = value;
index += 1;
} else if (arg === "--job-id") {
const value = args[index + 1];
if (value === undefined || !/^[A-Za-z0-9_.-]+$/u.test(value)) throw new Error("--job-id requires a safe non-empty identifier");
options.jobId = value;
index += 1;
} else if (arg === "--confirm") {
options.confirm = true;
} else if (arg === "--full") {
options.full = true;
} else if (arg === "--raw") {
options.raw = true;
options.full = true;
} else {
throw new Error(`unsupported platform-db postgres swap option: ${arg}`);
}
}
return options;
}
async function plan(config: UniDeskConfig, declaration: HostSwapDeclaration, options: HostSwapOptions): Promise<Record<string, unknown>> {
if (options.jobId !== null || options.confirm) throw new Error("swap plan does not accept --job-id or --confirm");
const observed = await collect(config, declaration);
const desiredBytes = parseByteSize(declaration.swap.size);
const facts = observed.parsed;
const declaredCapacity = numberValue(facts?.declaredCapacityBytes);
const diskAvailable = numberValue(facts?.diskAvailableBytes);
const reserveBytes = declaration.minDiskFreeGiB * 1024 ** 3;
const additionalDiskBytes = declaredCapacity === null ? desiredBytes : Math.max(0, desiredBytes - declaredCapacity);
const diskReady = diskAvailable !== null && diskAvailable >= additionalDiskBytes + reserveBytes;
const inSync = isInSync(facts, declaration, desiredBytes);
return {
ok: observed.capture.exitCode === 0 && facts !== null && (inSync || diskReady),
action: "platform-db-postgres-swap-plan",
mutation: false,
target: targetSummary(declaration),
desired: desiredSummary(declaration, desiredBytes),
observed: compactFacts(facts),
plan: {
action: inSync ? "none" : "add-declared-swap-capacity",
inSync,
diskReady,
additionalDiskBytes,
reserveBytes,
swapExcludedFromWebProbeAvailability: true,
steps: inSync ? [] : [
"keep-all-current-swap-active",
"allocate-and-enable-declared-extension",
"persist-fstab-and-swappiness",
"verify-declared-capacity",
],
},
remote: compactCapture(observed.capture, options.full),
...(options.raw ? { raw: observed.capture } : {}),
next: inSync
? { status: statusCommand(declaration) }
: {
apply: `bun scripts/cli.ts platform-db postgres swap apply --config ${declaration.configPath} --confirm`,
status: statusCommand(declaration),
},
};
}
async function status(config: UniDeskConfig, declaration: HostSwapDeclaration, options: HostSwapOptions): Promise<Record<string, unknown>> {
if (options.confirm) throw new Error("swap status does not accept --confirm");
if (options.jobId !== null) return await jobStatus(config, declaration, options.jobId, options.full, options.raw);
const observed = await collect(config, declaration);
const desiredBytes = parseByteSize(declaration.swap.size);
const inSync = isInSync(observed.parsed, declaration, desiredBytes);
return {
ok: observed.capture.exitCode === 0 && observed.parsed !== null,
action: "platform-db-postgres-swap-status",
mutation: false,
target: targetSummary(declaration),
desired: desiredSummary(declaration, desiredBytes),
observed: compactFacts(observed.parsed),
policySync: { inSync, swapExcludedFromWebProbeAvailability: true },
remote: compactCapture(observed.capture, options.full),
...(options.raw ? { raw: observed.capture } : {}),
next: inSync ? {} : { plan: `bun scripts/cli.ts platform-db postgres swap plan --config ${declaration.configPath}` },
};
}
async function apply(config: UniDeskConfig, declaration: HostSwapDeclaration, options: HostSwapOptions): Promise<Record<string, unknown>> {
if (options.jobId !== null) throw new Error("swap apply does not accept --job-id");
if (!options.confirm) return await plan(config, declaration, { ...options, confirm: false });
const desiredBytes = parseByteSize(declaration.swap.size);
const payload = {
path: declaration.swap.path,
expansionPath: declaration.swap.expansionPath,
desiredBytes,
minDiskFreeBytes: declaration.minDiskFreeGiB * 1024 ** 3,
swappiness: declaration.swap.swappiness,
sysctlFile: declaration.swap.sysctlFile,
};
const capture = await runSshCommandCapture(config, declaration.route, ["sh"], startJobScript(payload));
const parsed = parseJsonOutput(capture.stdout);
const jobId = stringValue(parsed?.jobId);
const ok = capture.exitCode === 0 && parsed?.ok === true && jobId !== null;
return {
ok,
action: "platform-db-postgres-swap-apply",
mutation: ok,
target: targetSummary(declaration),
desired: desiredSummary(declaration, desiredBytes),
remoteJob: parsed,
remote: compactCapture(capture, options.full || !ok),
...(options.raw ? { raw: capture } : {}),
next: jobId === null
? { plan: `bun scripts/cli.ts platform-db postgres swap plan --config ${declaration.configPath}` }
: { status: `${statusCommand(declaration)} --job-id ${jobId}` },
};
}
async function collect(config: UniDeskConfig, declaration: HostSwapDeclaration): Promise<{ capture: SshCaptureResult; parsed: Record<string, unknown> | null }> {
const encoded = Buffer.from(JSON.stringify({
path: declaration.swap.path,
expansionPath: declaration.swap.expansionPath,
sysctlFile: declaration.swap.sysctlFile,
}), "utf8").toString("base64");
const capture = await runSshCommandCapture(config, declaration.route, ["sh"], factsScript(encoded));
return { capture, parsed: parseJsonOutput(capture.stdout) };
}
async function jobStatus(
config: UniDeskConfig,
declaration: HostSwapDeclaration,
jobId: string,
full: boolean,
raw: boolean,
): Promise<Record<string, unknown>> {
const capture = await runSshCommandCapture(config, declaration.route, ["sh"], `
set -eu
sudo -n sh <<'ROOT'
job_dir="/root/.unidesk/platform-db/swap-jobs/${jobId}"
if [ ! -d "$job_dir" ]; then
printf '{"ok":false,"status":"missing","jobId":"${jobId}"}\\n'
exit 0
fi
state='{}'
result='null'
[ ! -f "$job_dir/state.json" ] || state="$(cat "$job_dir/state.json")"
[ ! -f "$job_dir/result.json" ] || result="$(cat "$job_dir/result.json")"
python3 - "$state" "$result" <<'PY'
import json, sys
state=json.loads(sys.argv[1])
result=json.loads(sys.argv[2])
print(json.dumps({"ok": True, "job": state, "result": result}, separators=(",", ":")))
PY
ROOT
`);
const parsed = parseJsonOutput(capture.stdout);
const statusValue = stringValue(recordValue(parsed?.job)?.status);
return {
ok: capture.exitCode === 0 && parsed?.ok === true,
action: "platform-db-postgres-swap-job-status",
mutation: false,
target: targetSummary(declaration),
jobId,
job: parsed?.job ?? null,
result: parsed?.result ?? null,
terminal: statusValue === "succeeded" || statusValue === "failed",
remote: compactCapture(capture, full || capture.exitCode !== 0),
...(raw ? { raw: capture } : {}),
next: statusValue === "succeeded"
? { status: statusCommand(declaration) }
: statusValue === "failed"
? { plan: `bun scripts/cli.ts platform-db postgres swap plan --config ${declaration.configPath}` }
: { status: `${statusCommand(declaration)} --job-id ${jobId}` },
};
}
function startJobScript(payload: Record<string, unknown>): string {
const encoded = Buffer.from(JSON.stringify(payload), "utf8").toString("base64");
return `
set -eu
if ! command -v sudo >/dev/null 2>&1 || ! sudo -n true >/dev/null 2>&1; then
printf '{"ok":false,"error":"sudo-noninteractive-unavailable"}\\n'
exit 1
fi
sudo -n sh <<'ROOT'
set -eu
umask 077
base="/root/.unidesk/platform-db/swap-jobs"
mkdir -p "$base"
job_id="swap-$(date +%Y%m%d%H%M%S)-$$"
job_dir="$base/$job_id"
mkdir "$job_dir"
printf '%s' '${encoded}' | base64 -d > "$job_dir/payload.json"
cat > "$job_dir/run.sh" <<'RUNNER'
${remoteRunner()}
RUNNER
chmod 700 "$job_dir/run.sh"
printf '{"status":"queued","stage":"created","jobId":"%s"}\\n' "$job_id" > "$job_dir/state.json"
nohup "$job_dir/run.sh" "$job_dir" > "$job_dir/run.log" 2>&1 &
printf '{"ok":true,"jobId":"%s","status":"queued"}\\n' "$job_id"
ROOT
`;
}
function remoteRunner(): string {
return String.raw`#!/bin/sh
set -eu
job_dir="$1"
payload="$job_dir/payload.json"
state="$job_dir/state.json"
result="$job_dir/result.json"
json_get() {
python3 - "$payload" "$1" <<'PY'
import json, sys
value=json.load(open(sys.argv[1], encoding="utf-8"))
for part in sys.argv[2].split("."):
value=value[part]
print(value)
PY
}
write_state() {
status="$1"
stage="$2"
exit_code=""
if [ "$#" -ge 3 ]; then exit_code="$3"; fi
python3 - "$state" "$status" "$stage" "$exit_code" <<'PY'
import datetime, json, os, sys
path,status,stage,exit_code=sys.argv[1:5]
try:
current=json.load(open(path, encoding="utf-8"))
except Exception:
current={}
current.update({"status":status,"stage":stage,"updatedAt":datetime.datetime.now(datetime.timezone.utc).isoformat()})
if status == "running" and not current.get("startedAt"):
current["startedAt"]=current["updatedAt"]
if status in {"succeeded","failed"}:
current["finishedAt"]=current["updatedAt"]
current["exitCode"]=int(exit_code or ("0" if status == "succeeded" else "1"))
tmp=path+".tmp"
json.dump(current, open(tmp,"w"), separators=(",",":"))
os.replace(tmp,path)
PY
}
fail() {
rc="$1"
stage="$2"
write_state failed "$stage" "$rc"
exit "$rc"
}
run_stage() {
stage="$1"
shift
write_state running "$stage"
"$@" || fail "$?" "$stage"
}
PATH_DECLARED="$(json_get path)"
EXPANSION_PATH="$(json_get expansionPath)"
DESIRED_BYTES="$(json_get desiredBytes)"
MIN_DISK_FREE_BYTES="$(json_get minDiskFreeBytes)"
SWAPPINESS="$(json_get swappiness)"
SYSCTL_FILE="$(json_get sysctlFile)"
write_state running preflight
case "$PATH_DECLARED" in /*) ;; *) fail 2 invalid-swap-path ;; esac
case "$EXPANSION_PATH" in /*) ;; *) fail 2 invalid-expansion-path ;; esac
case "$SYSCTL_FILE" in /etc/sysctl.d/*.conf) ;; *) fail 2 invalid-sysctl-path ;; esac
if [ -L "$PATH_DECLARED" ] || [ -L "$EXPANSION_PATH" ]; then fail 2 swap-path-symlink; fi
available_bytes="$(df -B1 --output=avail "$PATH_DECLARED" 2>/dev/null | tail -1 | tr -d ' ' || df -B1 --output=avail / | tail -1 | tr -d ' ')"
declared_capacity_bytes=0
if awk -v path="$PATH_DECLARED" 'NR>1 && $1==path {found=1} END {exit found?0:1}' /proc/swaps && [ -f "$PATH_DECLARED" ]; then
declared_capacity_bytes="$((declared_capacity_bytes + $(stat -c %s "$PATH_DECLARED")))"
fi
if awk -v path="$EXPANSION_PATH" 'NR>1 && $1==path {found=1} END {exit found?0:1}' /proc/swaps && [ -f "$EXPANSION_PATH" ]; then
declared_capacity_bytes="$((declared_capacity_bytes + $(stat -c %s "$EXPANSION_PATH")))"
fi
missing_bytes="$((DESIRED_BYTES - declared_capacity_bytes))"
if [ "$missing_bytes" -le 0 ]; then
write_state running persist-policy
else
required_bytes="$((missing_bytes + MIN_DISK_FREE_BYTES))"
if [ "$available_bytes" -lt "$required_bytes" ]; then fail 4 insufficient-disk-for-safe-expansion; fi
if [ -e "$EXPANSION_PATH" ]; then fail 3 expansion-path-already-exists; fi
write_state running allocate-extension
if ! fallocate -l "$missing_bytes" "$EXPANSION_PATH"; then
rm -f "$EXPANSION_PATH"
fail 5 allocate-extension
fi
chmod 600 "$EXPANSION_PATH"
if ! mkswap "$EXPANSION_PATH"; then
rm -f "$EXPANSION_PATH"
fail 6 format-extension
fi
if ! swapon "$EXPANSION_PATH"; then
rm -f "$EXPANSION_PATH"
fail 7 enable-extension
fi
fi
write_state running persist-policy
touch /etc/fstab
awk -v path="$PATH_DECLARED" '$1==path && $3=="swap" {found=1} END {exit found?0:1}' /etc/fstab || printf '%s none swap sw 0 0\n' "$PATH_DECLARED" >> /etc/fstab
if awk -v path="$EXPANSION_PATH" 'NR>1 && $1==path {found=1} END {exit found?0:1}' /proc/swaps; then
awk -v path="$EXPANSION_PATH" '$1==path && $3=="swap" {found=1} END {exit found?0:1}' /etc/fstab || printf '%s none swap sw 0 0\n' "$EXPANSION_PATH" >> /etc/fstab
fi
tmp_sysctl="$SYSCTL_FILE.tmp"
printf 'vm.swappiness = %s\n' "$SWAPPINESS" > "$tmp_sysctl"
chmod 644 "$tmp_sysctl"
mv "$tmp_sysctl" "$SYSCTL_FILE"
sysctl -q -w "vm.swappiness=$SWAPPINESS"
write_state running verify
if ! python3 - "$PATH_DECLARED" "$EXPANSION_PATH" "$DESIRED_BYTES" "$SWAPPINESS" "$SYSCTL_FILE" > "$result" <<'PY'
import json, os, stat, sys
path,expansion_path,desired_raw,swappiness_raw,sysctl_file=sys.argv[1:6]
desired=int(desired_raw)
swappiness=int(swappiness_raw)
swaps=[]
with open("/proc/swaps", encoding="utf-8") as handle:
for line in list(handle)[1:]:
fields=line.split()
if len(fields) >= 5:
swaps.append({"path":fields[0],"sizeBytes":int(fields[2])*1024,"usedBytes":int(fields[3])*1024,"priority":int(fields[4])})
configured=next((item for item in swaps if item["path"] == path), None)
extension=next((item for item in swaps if item["path"] == expansion_path), None)
runtime_swappiness=int(open("/proc/sys/vm/swappiness", encoding="utf-8").read().strip())
declared_capacity=sum(os.stat(item).st_size for item in (path, expansion_path) if os.path.isfile(item) and any(swap["path"] == item for swap in swaps))
result={
"ok": configured is not None and declared_capacity >= desired and runtime_swappiness == swappiness,
"path": path,
"desiredBytes": desired,
"configured": configured,
"extension": extension,
"totalSwapBytes": sum(item["sizeBytes"] for item in swaps),
"declaredCapacityBytes": declared_capacity,
"runtimeSwappiness": runtime_swappiness,
"sysctlFile": sysctl_file,
"sysctlFileExists": os.path.isfile(sysctl_file),
"mode": oct(stat.S_IMODE(os.stat(path).st_mode))[2:] if os.path.isfile(path) else None,
}
print(json.dumps(result, separators=(",",":")))
raise SystemExit(0 if result["ok"] else 8)
PY
then
fail 8 verify
fi
write_state succeeded complete 0
`;
}
function factsScript(encoded: string): string {
return String.raw`
set -eu
payload="$(mktemp)"
trap 'rm -f "$payload"' EXIT
printf '%s' '${encoded}' | base64 -d > "$payload"
python3 - "$payload" <<'PY'
import json, os, stat, sys
spec=json.load(open(sys.argv[1], encoding="utf-8"))
path=spec["path"]
expansion_path=spec["expansionPath"]
sysctl_file=spec["sysctlFile"]
swaps=[]
with open("/proc/swaps", encoding="utf-8") as handle:
for line in list(handle)[1:]:
fields=line.split()
if len(fields) >= 5:
swaps.append({"path":fields[0],"sizeBytes":int(fields[2])*1024,"usedBytes":int(fields[3])*1024,"priority":int(fields[4])})
mem={}
with open("/proc/meminfo", encoding="utf-8") as handle:
for line in handle:
if ":" not in line: continue
key,value=line.split(":",1)
fields=value.split()
if fields and fields[0].isdigit(): mem[key]=int(fields[0])*1024
disk=os.statvfs(path if os.path.exists(path) else os.path.dirname(path) or "/")
fstab=False
expansion_fstab=False
try:
with open("/etc/fstab", encoding="utf-8") as handle:
for line in handle:
fields=line.split()
if len(fields) >= 3 and fields[0] == path and fields[2] == "swap": fstab=True
if len(fields) >= 3 and fields[0] == expansion_path and fields[2] == "swap": expansion_fstab=True
except OSError:
pass
sysctl_persisted=None
try:
for line in open(sysctl_file, encoding="utf-8"):
if line.strip().startswith("vm.swappiness"):
sysctl_persisted=int(line.split("=",1)[1].strip())
except OSError:
pass
result={
"ok": True,
"path": path,
"configuredPathExists": os.path.isfile(path),
"configuredPathSizeBytes": os.stat(path).st_size if os.path.isfile(path) else None,
"configuredPathMode": oct(stat.S_IMODE(os.stat(path).st_mode))[2:] if os.path.isfile(path) else None,
"configuredPathActive": any(item["path"] == path for item in swaps),
"expansionPath": expansion_path,
"expansionPathExists": os.path.isfile(expansion_path),
"expansionPathSizeBytes": os.stat(expansion_path).st_size if os.path.isfile(expansion_path) else None,
"expansionPathMode": oct(stat.S_IMODE(os.stat(expansion_path).st_mode))[2:] if os.path.isfile(expansion_path) else None,
"expansionPathActive": any(item["path"] == expansion_path for item in swaps),
"activeSwaps": swaps,
"declaredCapacityBytes": sum(os.stat(item).st_size for item in (path, expansion_path) if os.path.isfile(item) and any(swap["path"] == item for swap in swaps)),
"swapTotalBytes": mem.get("SwapTotal",0),
"swapFreeBytes": mem.get("SwapFree",0),
"memAvailableBytes": mem.get("MemAvailable"),
"diskAvailableBytes": disk.f_bavail * disk.f_frsize,
"fstabPersisted": fstab,
"expansionFstabPersisted": expansion_fstab,
"runtimeSwappiness": int(open("/proc/sys/vm/swappiness", encoding="utf-8").read().strip()),
"persistedSwappiness": sysctl_persisted,
"sysctlFile": sysctl_file,
"swapExcludedFromWebProbeAvailability": True,
}
print(json.dumps(result, separators=(",",":")))
PY
`;
}
function parseByteSize(value: string): number {
const match = value.trim().match(/^([1-9][0-9]*)([KMGT]i?B?|B)$/iu);
if (match === null) throw new Error(`node.swap.size must use a positive size such as 64GiB; received ${value}`);
const unit = match[2].toUpperCase().replace("IB", "").replace("B", "");
const powers: Record<string, number> = { "": 0, K: 1, M: 2, G: 3, T: 4 };
return Number(match[1]) * 1024 ** powers[unit];
}
function isInSync(facts: Record<string, unknown> | null, declaration: HostSwapDeclaration, desiredBytes: number): boolean {
return facts !== null
&& facts.configuredPathActive === true
&& numberValue(facts.declaredCapacityBytes) !== null
&& (numberValue(facts.declaredCapacityBytes) ?? 0) >= desiredBytes
&& facts.configuredPathMode === "600"
&& facts.fstabPersisted === true
&& (facts.expansionPathActive !== true || (facts.expansionPathMode === "600" && facts.expansionFstabPersisted === true))
&& numberValue(facts.runtimeSwappiness) === declaration.swap.swappiness
&& numberValue(facts.persistedSwappiness) === declaration.swap.swappiness;
}
function desiredSummary(declaration: HostSwapDeclaration, desiredBytes: number): Record<string, unknown> {
return {
configSource: `${declaration.configPath}#node.swap`,
ensure: declaration.swap.ensure,
path: declaration.swap.path,
expansionPath: declaration.swap.expansionPath,
size: declaration.swap.size,
sizeBytes: desiredBytes,
swappiness: declaration.swap.swappiness,
sysctlFile: declaration.swap.sysctlFile,
};
}
function compactFacts(facts: Record<string, unknown> | null): Record<string, unknown> | null {
if (facts === null) return null;
return {
configuredPathExists: facts.configuredPathExists ?? null,
configuredPathSizeBytes: facts.configuredPathSizeBytes ?? null,
configuredPathMode: facts.configuredPathMode ?? null,
configuredPathActive: facts.configuredPathActive ?? null,
expansionPath: facts.expansionPath ?? null,
expansionPathExists: facts.expansionPathExists ?? null,
expansionPathSizeBytes: facts.expansionPathSizeBytes ?? null,
expansionPathMode: facts.expansionPathMode ?? null,
expansionPathActive: facts.expansionPathActive ?? null,
activeSwaps: facts.activeSwaps ?? [],
declaredCapacityBytes: facts.declaredCapacityBytes ?? null,
swapTotalBytes: facts.swapTotalBytes ?? null,
swapFreeBytes: facts.swapFreeBytes ?? null,
memAvailableBytes: facts.memAvailableBytes ?? null,
diskAvailableBytes: facts.diskAvailableBytes ?? null,
fstabPersisted: facts.fstabPersisted ?? null,
expansionFstabPersisted: facts.expansionFstabPersisted ?? null,
runtimeSwappiness: facts.runtimeSwappiness ?? null,
persistedSwappiness: facts.persistedSwappiness ?? null,
sysctlFile: facts.sysctlFile ?? null,
swapExcludedFromWebProbeAvailability: true,
};
}
function targetSummary(declaration: HostSwapDeclaration): Record<string, unknown> {
return { node: declaration.nodeId, route: declaration.route, config: declaration.configPath };
}
function statusCommand(declaration: HostSwapDeclaration): string {
return `bun scripts/cli.ts platform-db postgres swap status --config ${declaration.configPath}`;
}
function compactCapture(capture: SshCaptureResult, full: boolean): Record<string, unknown> {
return {
exitCode: capture.exitCode,
stdoutBytes: Buffer.byteLength(capture.stdout, "utf8"),
stderrBytes: Buffer.byteLength(capture.stderr, "utf8"),
stdoutTail: full || capture.exitCode !== 0 ? capture.stdout.slice(-8000) : "",
stderrTail: full || capture.exitCode !== 0 ? capture.stderr.slice(-4000) : "",
};
}
function numberValue(value: unknown): number | null {
return typeof value === "number" && Number.isFinite(value) ? value : null;
}
function stringValue(value: unknown): string | null {
return typeof value === "string" && value.length > 0 ? value : null;
}
function recordValue(value: unknown): Record<string, unknown> | null {
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
}
+44 -3
View File
@@ -13,6 +13,7 @@ import {
type LogicalDumpConfig,
type LogicalDumpRuntimeFact,
} from "./platform-db-postgres-backups";
import { runPlatformDbHostSwapCommand, swapHelp, type HostSwapDeclaration } from "./platform-db-host-swap";
const defaultConfigPath = "config/platform-db/postgres-pk01.yaml";
const managedHbaStart = "# BEGIN unidesk managed pk01-platform-postgres";
@@ -61,6 +62,9 @@ interface PostgresHostConfig {
ensure: boolean;
size: string;
path: string;
expansionPath?: string;
swappiness?: number;
sysctlFile?: string;
};
};
postgres: {
@@ -310,17 +314,19 @@ export function platformDbHelp(): unknown {
"bun scripts/cli.ts platform-db postgres monitor plan --config config/platform-db/postgres-nc01.yaml",
"bun scripts/cli.ts platform-db postgres monitor status --config config/platform-db/postgres-nc01.yaml",
"bun scripts/cli.ts platform-db postgres monitor reset --config config/platform-db/postgres-nc01.yaml --confirm",
"bun scripts/cli.ts platform-db postgres swap plan|status|apply --config config/platform-db/postgres-nc01.yaml",
],
description: "Manage YAML-declared host-native PostgreSQL for UniDesk platform state. The PK01 declaration uses PostgreSQL 16 for Sub2API and does not deploy k3s.",
description: "Manage YAML-declared host-native PostgreSQL and its host swap capacity for UniDesk platform state.",
safety: {
configTruth: defaultConfigPath,
defaultMutation: false,
exportSecrets: "export-secrets only materializes YAML-declared local Secret source/export files; it does not touch the remote PostgreSQL service.",
disclosure: "plan, status and export-secrets use bounded summaries by default; --full and --raw explicitly disclose the complete structured result without printing Secret values.",
apply: "apply --confirm returns an async UniDesk job; the job uses short trans calls and a remote PK01 job instead of keeping one long SSH session open.",
apply: "apply --confirm returns an async UniDesk job; the job uses short trans calls and a remote target job instead of keeping one long SSH session open.",
monitorReset: "monitor reset is restricted by owning YAML to the dedicated web_probe_monitor database and requires --confirm; it cannot accept a database name or arbitrary SQL.",
swap: swapHelp(),
redaction: "Passwords and full DATABASE_URL values are never printed; output is limited to key names, presence, fingerprints and state.",
noK3s: "PK01 PostgreSQL is host-systemd, not Docker or k3s.",
noK3s: "The selected PostgreSQL declaration is host-systemd, not Docker or k3s.",
},
};
}
@@ -328,6 +334,10 @@ export function platformDbHelp(): unknown {
export async function runPlatformDbCommand(config: UniDeskConfig, args: string[]): Promise<Record<string, unknown>> {
const [target, action = "plan"] = args;
if (target !== "postgres") return unsupported(args);
if (action === "swap") {
const swapAction = args[2] ?? "plan";
return await runPlatformDbHostSwapCommand(config, swapAction, args.slice(3), (configPath) => hostSwapDeclaration(readPostgresHostConfig(configPath)));
}
if (action === "monitor") {
const monitorAction = args[2] ?? "plan";
const options = parseOptions(args.slice(3));
@@ -747,6 +757,9 @@ function readPostgresHostConfig(pathArg: string): PostgresHostConfig {
ensure: booleanField(swap, "ensure", `${configPath}.node.swap`),
size: stringField(swap, "size", `${configPath}.node.swap`),
path: absolutePathField(swap, "path", `${configPath}.node.swap`),
...(swap.expansionPath === undefined ? {} : { expansionPath: absolutePathField(swap, "expansionPath", `${configPath}.node.swap`) }),
...(swap.swappiness === undefined ? {} : { swappiness: integerField(swap, "swappiness", `${configPath}.node.swap`) }),
...(swap.sysctlFile === undefined ? {} : { sysctlFile: absolutePathField(swap, "sysctlFile", `${configPath}.node.swap`) }),
},
},
postgres: {
@@ -822,6 +835,34 @@ function readPostgresHostConfig(pathArg: string): PostgresHostConfig {
};
}
function hostSwapDeclaration(pg: PostgresHostConfig): HostSwapDeclaration {
const swappiness = pg.node.swap.swappiness;
const sysctlFile = pg.node.swap.sysctlFile;
const expansionPath = pg.node.swap.expansionPath;
if (swappiness === undefined) throw new Error(`${pg.configPath}.node.swap.swappiness is required for controlled swap operations`);
if (swappiness < 0 || swappiness > 200) throw new Error(`${pg.configPath}.node.swap.swappiness must be between 0 and 200`);
if (sysctlFile === undefined) throw new Error(`${pg.configPath}.node.swap.sysctlFile is required for controlled swap operations`);
if (expansionPath === undefined) throw new Error(`${pg.configPath}.node.swap.expansionPath is required for controlled swap operations`);
if (expansionPath === pg.node.swap.path) throw new Error(`${pg.configPath}.node.swap.expansionPath must differ from path`);
for (const [field, value] of [["path", pg.node.swap.path], ["expansionPath", expansionPath], ["sysctlFile", sysctlFile]] as const) {
if (!/^\/[A-Za-z0-9._/-]+$/u.test(value)) throw new Error(`${pg.configPath}.node.swap.${field} must be a safe absolute host path`);
}
return {
configPath: pg.configPath,
nodeId: pg.node.id,
route: pg.node.route,
minDiskFreeGiB: pg.node.requiredHostFacts.minDiskFreeGiB,
swap: {
ensure: pg.node.swap.ensure,
size: pg.node.swap.size,
path: pg.node.swap.path,
expansionPath,
swappiness,
sysctlFile,
},
};
}
export function readPostgresLogicalDumps(pathArg: string): LogicalDumpConfig[] {
return readPostgresHostConfig(pathArg).backup.logicalDumps;
}