fix(v03): drain workbench sse before pod termination (#2003)

deploy/deploy.yaml

@@ -562,6 +562,11 @@ lanes:
     services:
       - serviceId: hwlab-cloud-api
         replicas: 2
+        shutdownDrain:
+          enabled: true
+          path: /internal/workbench/drain
+          sleepSeconds: 3
+          timeoutMs: 2500
         env:
           HWLAB_CLOUD_DB_URL: secretRef:hwlab-cloud-api-v03-db/database-url
           HWLAB_CLOUD_DB_CONTRACT: v03-redacted-presence-only

internal/cloud/server.ts

@@ -69,7 +69,7 @@ import {
   handleCodeAgentTraceHttp,
   startAgentRunProjectionResume
 } from "./server-code-agent-http.ts";
-import { handleWorkbenchReadModelHttp, handleWorkbenchRealtimeHttp } from "./server-workbench-http.ts";
+import { drainWorkbenchRealtimeConnections, handleWorkbenchReadModelHttp, handleWorkbenchRealtimeHttp } from "./server-workbench-http.ts";
 import { startWorkbenchEmptySessionGc } from "./workbench-empty-session-gc.ts";
 import { handleM3IoControlHttp } from "./server-m3-http.ts";
 import { handleSkillsHttp } from "./server-skills-http.ts";
@@ -661,6 +661,25 @@ async function handleRestAdapter(request, response, url, options) {
     return;
   }
 
+  if (url.pathname === "/internal/workbench/drain") {
+    if (request.method !== "POST") {
+      sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "POST required." } });
+      return;
+    }
+    if (!isAuthorizedWorkbenchDrainRequest(request, options.env ?? process.env)) {
+      sendJson(response, 403, { ok: false, error: { code: "workbench_drain_forbidden", message: "Workbench drain endpoint requires pod-local preStop authorization." } });
+      return;
+    }
+    const result = await drainWorkbenchRealtimeConnections({
+      reason: "k8s_prestop",
+      signal: "SIGTERM",
+      timeoutMs: parsePositiveInteger((options.env ?? process.env).HWLAB_WORKBENCH_SSE_DRAIN_TIMEOUT_MS, 2500),
+      env: options.env ?? process.env
+    });
+    sendJson(response, 200, { ok: true, route: "/internal/workbench/drain", result });
+    return;
+  }
+
   if (url.pathname === "/v1/workbench/events") {
     await handleWorkbenchRealtimeHttp(request, response, url, options);
     return;
@@ -2040,6 +2059,12 @@ function safeOpaqueId(value) {
   return /^[A-Za-z0-9_.:-]{3,180}$/u.test(text) ? text : null;
 }
 
+function isAuthorizedWorkbenchDrainRequest(request, env = process.env) {
+  const expected = String(env.HOSTNAME ?? "").trim();
+  if (!expected) return false;
+  return getHeader(request, "x-hwlab-drain-hostname") === expected;
+}
+
 function parsePositiveInteger(value, fallback) {
   const parsed = Number.parseInt(value ?? "", 10);
   return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;

scripts/gitops-render.mjs

@@ -537,6 +537,29 @@ function applyServiceHealthProbe(container, healthProbe) {
   }
 }
 
+function applyServiceShutdownDrainLifecycle(container, shutdownDrain) {
+  if (!container || !shutdownDrain || shutdownDrain.enabled !== true) return;
+  const pathValue = typeof shutdownDrain.path === "string" && /^\/[A-Za-z0-9/_:.-]+$/u.test(shutdownDrain.path) ? shutdownDrain.path : "/internal/workbench/drain";
+  const sleepSeconds = boundedInteger(shutdownDrain.sleepSeconds, 3, 0, 30);
+  const timeoutMs = boundedInteger(shutdownDrain.timeoutMs, 2500, 100, 30000);
+  container.env ??= [];
+  upsertEnv(container.env, "HWLAB_WORKBENCH_SSE_DRAIN_TIMEOUT_MS", String(timeoutMs));
+  const command = [
+    'port="${PORT:-${HWLAB_PORT:-${HWLAB_CLOUD_API_PORT:-6667}}}"',
+    `url="http://127.0.0.1:\${port}${pathValue}"`,
+    '/usr/local/bin/bun -e \'const [url, hostname] = process.argv.slice(2); await fetch(url, { method: "POST", headers: { "x-hwlab-drain-hostname": hostname || "" } }).catch(() => {});\' "$url" "${HOSTNAME:-}" >/dev/null 2>&1 || true',
+    `sleep ${sleepSeconds}`
+  ].join("; ");
+  container.lifecycle ??= {};
+  container.lifecycle.preStop = { exec: { command: ["sh", "-c", command] } };
+}
+
+function boundedInteger(value, fallback, minimum, maximum) {
+  const parsed = Number(value);
+  if (!Number.isInteger(parsed)) return fallback;
+  return Math.max(minimum, Math.min(maximum, parsed));
+}
+
 function normalizeProbeTiming(value) {
   if (!value || typeof value !== "object" || Array.isArray(value)) return {};
   const result = {};
@@ -1338,6 +1361,7 @@ function transformWorkloads({ workloads, deploy, catalog, source, sourceBranch =
       if (serviceId === "hwlab-cloud-api") {
         if (runtimeLane) {
           removeV02RepoOwnedCodeAgentPodConfig(item.spec, podTemplate.spec);
+          applyServiceShutdownDrainLifecycle(container, deployService.shutdownDrain);
           upsertEnv(container.env, "HWLAB_M3_IO_CONTROL_ENABLED", "false");
           upsertEnv(container.env, "HWLAB_CODE_AGENT_AGENTRUN_REPO_URL", gitReadUrl ?? defaultRuntimeLaneGitReadUrl);
           upsertEnv(container.env, "HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID", runtimeNodeIdForProfile(deploy, profile));