diff --git a/deploy/deploy.yaml b/deploy/deploy.yaml index 53c21b34..c9fc7d33 100644 --- a/deploy/deploy.yaml +++ b/deploy/deploy.yaml @@ -562,6 +562,11 @@ lanes: services: - serviceId: hwlab-cloud-api replicas: 2 + shutdownDrain: + enabled: true + path: /internal/workbench/drain + sleepSeconds: 3 + timeoutMs: 2500 env: HWLAB_CLOUD_DB_URL: secretRef:hwlab-cloud-api-v03-db/database-url HWLAB_CLOUD_DB_CONTRACT: v03-redacted-presence-only diff --git a/internal/cloud/server.ts b/internal/cloud/server.ts index f20dee58..60ff9ddb 100644 --- a/internal/cloud/server.ts +++ b/internal/cloud/server.ts @@ -69,7 +69,7 @@ import { handleCodeAgentTraceHttp, startAgentRunProjectionResume } from "./server-code-agent-http.ts"; -import { handleWorkbenchReadModelHttp, handleWorkbenchRealtimeHttp } from "./server-workbench-http.ts"; +import { drainWorkbenchRealtimeConnections, handleWorkbenchReadModelHttp, handleWorkbenchRealtimeHttp } from "./server-workbench-http.ts"; import { startWorkbenchEmptySessionGc } from "./workbench-empty-session-gc.ts"; import { handleM3IoControlHttp } from "./server-m3-http.ts"; import { handleSkillsHttp } from "./server-skills-http.ts"; @@ -661,6 +661,25 @@ async function handleRestAdapter(request, response, url, options) { return; } + if (url.pathname === "/internal/workbench/drain") { + if (request.method !== "POST") { + sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "POST required." } }); + return; + } + if (!isAuthorizedWorkbenchDrainRequest(request, options.env ?? process.env)) { + sendJson(response, 403, { ok: false, error: { code: "workbench_drain_forbidden", message: "Workbench drain endpoint requires pod-local preStop authorization." } }); + return; + } + const result = await drainWorkbenchRealtimeConnections({ + reason: "k8s_prestop", + signal: "SIGTERM", + timeoutMs: parsePositiveInteger((options.env ?? process.env).HWLAB_WORKBENCH_SSE_DRAIN_TIMEOUT_MS, 2500), + env: options.env ?? process.env + }); + sendJson(response, 200, { ok: true, route: "/internal/workbench/drain", result }); + return; + } + if (url.pathname === "/v1/workbench/events") { await handleWorkbenchRealtimeHttp(request, response, url, options); return; @@ -2040,6 +2059,12 @@ function safeOpaqueId(value) { return /^[A-Za-z0-9_.:-]{3,180}$/u.test(text) ? text : null; } +function isAuthorizedWorkbenchDrainRequest(request, env = process.env) { + const expected = String(env.HOSTNAME ?? "").trim(); + if (!expected) return false; + return getHeader(request, "x-hwlab-drain-hostname") === expected; +} + function parsePositiveInteger(value, fallback) { const parsed = Number.parseInt(value ?? "", 10); return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; diff --git a/scripts/gitops-render.mjs b/scripts/gitops-render.mjs index 5c1de6bd..66b72b6f 100644 --- a/scripts/gitops-render.mjs +++ b/scripts/gitops-render.mjs @@ -537,6 +537,29 @@ function applyServiceHealthProbe(container, healthProbe) { } } +function applyServiceShutdownDrainLifecycle(container, shutdownDrain) { + if (!container || !shutdownDrain || shutdownDrain.enabled !== true) return; + const pathValue = typeof shutdownDrain.path === "string" && /^\/[A-Za-z0-9/_:.-]+$/u.test(shutdownDrain.path) ? shutdownDrain.path : "/internal/workbench/drain"; + const sleepSeconds = boundedInteger(shutdownDrain.sleepSeconds, 3, 0, 30); + const timeoutMs = boundedInteger(shutdownDrain.timeoutMs, 2500, 100, 30000); + container.env ??= []; + upsertEnv(container.env, "HWLAB_WORKBENCH_SSE_DRAIN_TIMEOUT_MS", String(timeoutMs)); + const command = [ + 'port="${PORT:-${HWLAB_PORT:-${HWLAB_CLOUD_API_PORT:-6667}}}"', + `url="http://127.0.0.1:\${port}${pathValue}"`, + '/usr/local/bin/bun -e \'const [url, hostname] = process.argv.slice(2); await fetch(url, { method: "POST", headers: { "x-hwlab-drain-hostname": hostname || "" } }).catch(() => {});\' "$url" "${HOSTNAME:-}" >/dev/null 2>&1 || true', + `sleep ${sleepSeconds}` + ].join("; "); + container.lifecycle ??= {}; + container.lifecycle.preStop = { exec: { command: ["sh", "-c", command] } }; +} + +function boundedInteger(value, fallback, minimum, maximum) { + const parsed = Number(value); + if (!Number.isInteger(parsed)) return fallback; + return Math.max(minimum, Math.min(maximum, parsed)); +} + function normalizeProbeTiming(value) { if (!value || typeof value !== "object" || Array.isArray(value)) return {}; const result = {}; @@ -1338,6 +1361,7 @@ function transformWorkloads({ workloads, deploy, catalog, source, sourceBranch = if (serviceId === "hwlab-cloud-api") { if (runtimeLane) { removeV02RepoOwnedCodeAgentPodConfig(item.spec, podTemplate.spec); + applyServiceShutdownDrainLifecycle(container, deployService.shutdownDrain); upsertEnv(container.env, "HWLAB_M3_IO_CONTROL_ENABLED", "false"); upsertEnv(container.env, "HWLAB_CODE_AGENT_AGENTRUN_REPO_URL", gitReadUrl ?? defaultRuntimeLaneGitReadUrl); upsertEnv(container.env, "HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID", runtimeNodeIdForProfile(deploy, profile));