diff --git a/deploy/deploy.yaml b/deploy/deploy.yaml index c4b7c753..c4b8a753 100644 --- a/deploy/deploy.yaml +++ b/deploy/deploy.yaml @@ -521,19 +521,9 @@ lanes: TZ: Asia/Shanghai HWLAB_METRICS_NAMESPACE: hwlab-v03 HWLAB_OBSERVABILITY_CONFIG_REVISION: PJ2026-01060505-20260619-long-trace-timeout-mitigation - HWLAB_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS: "30000" - HWLAB_CODE_AGENT_AGENTRUN_NO_RESPONSE_TIMEOUT_MS: "600000" HWLAB_CODE_AGENT_AGENTRUN_DISPATCH_RETRY_MAX: "5" HWLAB_CODE_AGENT_AGENTRUN_DISPATCH_RETRY_BASE_MS: "1000" HWLAB_CODE_AGENT_AGENTRUN_DISPATCH_RETRY_MAX_DELAY_MS: "30000" - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_POLL_INTERVAL_MS: "1000" - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_ENABLED: "1" - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INITIAL_DELAY_MS: "5000" - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INTERVAL_MS: "60000" - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS: "30000" - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE: "100" - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_JITTER_MS: "15000" - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_FAILURE_BACKOFF_MS: "120000" observable: true hwlab-workbench-runtime: runtimeKind: go-service @@ -845,6 +835,13 @@ lanes: HWLAB_KAFKA_EVENT_TOPIC: hwlab.event.v1 HWLAB_KAFKA_CLIENT_ID: hwlab-v03-cloud-api HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID: hwlab-v03-agentrun-event-bridge + HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS: "2000" + HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS: "250" + HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE: "100" + HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS: "30000" + HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS: "10000" + HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS: "1000" + HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" HWLAB_WORKBENCH_EMPTY_SESSION_GC_ENABLED: "1" HWLAB_WORKBENCH_EMPTY_SESSION_TTL_MS: "600000" HWLAB_WORKBENCH_EMPTY_SESSION_GC_INTERVAL_MS: "60000" diff --git a/internal/agent/agentrun-dispatch.mjs b/internal/agent/agentrun-dispatch.mjs index d68af0c1..f08612b1 100644 --- a/internal/agent/agentrun-dispatch.mjs +++ b/internal/agent/agentrun-dispatch.mjs @@ -214,20 +214,21 @@ export function createHwlabAgentRunDispatchAssembly(options = {}) { providerProfile: backendProfile, dispatchSource: "hwlab-code-agent" }, + dispatch: { + kind: "kubernetes-job", + input: { + idempotencyKey: options.runnerJobIdempotencyKey ?? `hwlab:${traceId}:runner-job`, + ...(options.attemptId ? { attemptId: nonEmptyString(options.attemptId, "attemptId") } : {}), + transientEnv + } + }, idempotencyKey: options.commandIdempotencyKey ?? `hwlab:${traceId}:turn` }; - const runnerJobPayload = { - idempotencyKey: options.runnerJobIdempotencyKey ?? `hwlab:${traceId}:runner-job`, - ...(options.attemptId ? { attemptId: nonEmptyString(options.attemptId, "attemptId") } : {}), - transientEnv - }; - return { managerUrl, runPayload, commandPayload, - runnerJobPayload, boundaries: { valuesPrinted: false, githubCredentialSource: includeGithubToolCredential ? "toolCredentials" : null, @@ -249,18 +250,19 @@ export async function dispatchHwlabAgentRun(options = {}) { const run = await postJson(fetchImpl, managerUrl, "/api/v1/runs", assembly.runPayload); const runId = nonEmptyString(run.id, "run.id"); const command = await postJson(fetchImpl, managerUrl, `/api/v1/runs/${encodeURIComponent(runId)}/commands`, assembly.commandPayload); - const commandId = nonEmptyString(command.id, "command.id"); - const runnerJobPayload = { ...assembly.runnerJobPayload, commandId }; - const runnerJob = await postJson(fetchImpl, managerUrl, `/api/v1/runs/${encodeURIComponent(runId)}/runner-jobs`, runnerJobPayload); + nonEmptyString(command.id, "command.id"); + const dispatchIntent = command.dispatchIntent; + if (!dispatchIntent || dispatchIntent.durable !== true || !optionalString(dispatchIntent.id)) { + throw new Error("AgentRun command response requires a durable dispatchIntent"); + } return { managerUrl, run, command, - runnerJob, + dispatchIntent, requests: { runPayload: assembly.runPayload, - commandPayload: assembly.commandPayload, - runnerJobPayload + commandPayload: assembly.commandPayload }, boundaries: assembly.boundaries }; diff --git a/internal/agent/agentrun-dispatch.test.mjs b/internal/agent/agentrun-dispatch.test.mjs index 85ed6721..deac7702 100644 --- a/internal/agent/agentrun-dispatch.test.mjs +++ b/internal/agent/agentrun-dispatch.test.mjs @@ -61,7 +61,7 @@ test("HWLAB AgentRun assembly grants GitHub and UniDesk SSH through toolCredenti assert.equal(tools.some((item) => item.tool === "github" && item.projection.envName === "GH_TOKEN"), true); assert.equal(tools.some((item) => item.tool === "unidesk-ssh" && item.projection.envName === "UNIDESK_SSH_CLIENT_TOKEN"), true); - const transientEnv = assembly.runnerJobPayload.transientEnv; + const transientEnv = assembly.commandPayload.dispatch.input.transientEnv; assert.equal(transientEnv.find((item) => item.name === DEFAULT_UNIDESK_MAIN_SERVER_ENV)?.value, "https://unidesk.example.test"); assert.equal(transientEnv.some((item) => item.name === "UNIDESK_SSH_CLIENT_TOKEN"), false); assert.equal(assembly.boundaries.unideskSshCredentialSource, "toolCredentials"); @@ -112,7 +112,7 @@ test("HWLAB AgentRun assembly does not duplicate explicit UniDesk main-server en unideskMainServerIp: "https://unidesk.example.test", transientEnv: [{ name: DEFAULT_UNIDESK_MAIN_SERVER_ENV, value: "https://explicit.example.test", sensitive: false }] }); - const entries = assembly.runnerJobPayload.transientEnv.filter((item) => item.name === DEFAULT_UNIDESK_MAIN_SERVER_ENV); + const entries = assembly.commandPayload.dispatch.input.transientEnv.filter((item) => item.name === DEFAULT_UNIDESK_MAIN_SERVER_ENV); assert.equal(entries.length, 1); assert.equal(entries[0].value, "https://explicit.example.test"); }); @@ -128,14 +128,13 @@ test("HWLAB AgentRun assembly requires full commit and main server when UniDesk ); }); -test("dispatchHwlabAgentRun posts run command and runner job with commandId", async () => { +test("dispatchHwlabAgentRun atomically requests durable dispatch with the command", async () => { const requests = []; const fetchImpl = async (url, init) => { const request = { url, method: init.method, body: JSON.parse(init.body) }; requests.push(request); if (url.endsWith("/api/v1/runs")) return jsonResponse({ id: "run_hwlab_001" }); - if (url.endsWith("/api/v1/runs/run_hwlab_001/commands")) return jsonResponse({ id: "cmd_hwlab_001" }); - if (url.endsWith("/api/v1/runs/run_hwlab_001/runner-jobs")) return jsonResponse({ id: "job_hwlab_001", jobName: "agentrun-v01-runner-selftest" }); + if (url.endsWith("/api/v1/runs/run_hwlab_001/commands")) return jsonResponse({ id: "cmd_hwlab_001", dispatchIntent: { id: "dispatch_hwlab_001", state: "pending", runnerJobId: "rjob_hwlab_001", durable: true } }); return jsonResponse({ error: "unexpected path" }, { status: 404 }); }; const result = await dispatchHwlabAgentRun({ @@ -148,21 +147,19 @@ test("dispatchHwlabAgentRun posts run command and runner job with commandId", as assert.equal(result.run.id, "run_hwlab_001"); assert.equal(result.command.id, "cmd_hwlab_001"); - assert.equal(result.runnerJob.jobName, "agentrun-v01-runner-selftest"); + assert.equal(result.dispatchIntent.id, "dispatch_hwlab_001"); assert.deepEqual(requests.map((item) => new URL(item.url).pathname), [ "/api/v1/runs", - "/api/v1/runs/run_hwlab_001/commands", - "/api/v1/runs/run_hwlab_001/runner-jobs" + "/api/v1/runs/run_hwlab_001/commands" ]); - assert.equal(requests[2].body.commandId, "cmd_hwlab_001"); - assert.equal(requests[2].body.transientEnv.some((item) => item.name === "UNIDESK_SSH_CLIENT_TOKEN"), false); + assert.equal(requests[1].body.dispatch.kind, "kubernetes-job"); + assert.equal(requests[1].body.dispatch.input.transientEnv.some((item) => item.name === "UNIDESK_SSH_CLIENT_TOKEN"), false); }); test("dispatchHwlabAgentRun unwraps AgentRun manager response envelopes", async () => { const fetchImpl = async (url, init) => { if (url.endsWith("/api/v1/runs")) return jsonResponse({ ok: true, data: { id: "run_hwlab_envelope" } }); - if (url.endsWith("/api/v1/runs/run_hwlab_envelope/commands")) return jsonResponse({ ok: true, data: { id: "cmd_hwlab_envelope" } }); - if (url.endsWith("/api/v1/runs/run_hwlab_envelope/runner-jobs")) return jsonResponse({ ok: true, data: { id: "job_hwlab_envelope", jobName: "agentrun-v01-runner-envelope" } }); + if (url.endsWith("/api/v1/runs/run_hwlab_envelope/commands")) return jsonResponse({ ok: true, data: { id: "cmd_hwlab_envelope", dispatchIntent: { id: "dispatch_hwlab_envelope", state: "pending", runnerJobId: "rjob_hwlab_envelope", durable: true } } }); return jsonResponse({ ok: false, failureKind: "unexpected-path" }, { status: 404 }); }; @@ -176,7 +173,7 @@ test("dispatchHwlabAgentRun unwraps AgentRun manager response envelopes", async assert.equal(result.run.id, "run_hwlab_envelope"); assert.equal(result.command.id, "cmd_hwlab_envelope"); - assert.equal(result.runnerJob.jobName, "agentrun-v01-runner-envelope"); + assert.equal(result.dispatchIntent.id, "dispatch_hwlab_envelope"); }); function jsonResponse(body, { status = 200 } = {}) { diff --git a/internal/cloud/backend-performance.test.ts b/internal/cloud/backend-performance.test.ts index 0ee1bf49..fb20b6cc 100644 --- a/internal/cloud/backend-performance.test.ts +++ b/internal/cloud/backend-performance.test.ts @@ -6,18 +6,18 @@ import { createServer as createHttpServer } from "node:http"; import { test } from "bun:test"; import { createBackendPerformanceStore, backendPerformanceRouteTemplate, withBackendPerformanceContext } from "./backend-performance.ts"; -import { submitAgentRunChatTurn, syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts"; +import { submitAgentRunChatTurn } from "./code-agent-agentrun-adapter.ts"; import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; import { createCloudApiServer } from "./server.ts"; -import { createCodeAgentChatResultStore } from "./server-code-agent-http.ts"; test("backend performance store templates routes and appends Cloud API metrics to loopback metrics", async () => { - assert.equal(backendPerformanceRouteTemplate("/v1/agent/traces/trc_secret_0123456789abcdef?full=1"), "/v1/agent/traces/:id"); + assert.equal(backendPerformanceRouteTemplate("/v1/agent/traces/trc_secret_0123456789abcdef?full=1"), "/v1/agent/traces/:traceId"); assert.equal(backendPerformanceRouteTemplate("/api/v1/runs/run_0123456789abcdef/commands/cmd_0123456789abcdef/result"), "/api/v1/runs/:id/commands/:id/result"); assert.equal(backendPerformanceRouteTemplate("/v1/admin/billing/users/12345/credits/adjust"), "/v1/admin/billing/users/:id/credits/adjust"); const backendPerformanceStore = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } }); - const server = createCloudApiServer({ backendPerformanceStore, env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } }); + const accessController = { required: true, async authenticate() { return { ok: true, actor: { id: "usr_backend_metrics", role: "admin" }, session: { id: "web_backend_metrics" } }; } }; + const server = createCloudApiServer({ backendPerformanceStore, accessController, env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); try { const address = server.address(); @@ -37,7 +37,6 @@ test("backend performance store templates routes and appends Cloud API metrics t await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); } }); - test("backend performance store exposes low-cardinality request evidence for RUM alignment", () => { const store = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } }); const context = store.createManualContext({ route: "/v1/workbench/sessions/ses_secret/messages", method: "GET" }); @@ -79,10 +78,9 @@ test("backend performance records Code Agent trace handler phases without high-c const runId = "run_backend_phase"; const commandId = "cmd_backend_phase"; const traceStore = createCodeAgentTraceStore(); - const codeAgentChatResults = createCodeAgentChatResultStore(); traceStore.append(traceId, { type: "message", role: "assistant", label: "assistant:start", sourceSeq: 1, commandId }); traceStore.append(traceId, { type: "result", status: "completed", terminal: true, label: "result:completed", sourceSeq: 2, commandId }); - codeAgentChatResults.set(traceId, { + const projectedResult = { status: "completed", conversationId: "cnv_backend_phase", sessionId: "ses_backend_phase", @@ -107,9 +105,16 @@ test("backend performance records Code Agent trace handler phases without high-c terminalStatus: "completed", eventCount: 2, updatedAt: "2026-06-18T00:00:00.000Z" - } - }); + }, + runnerTrace: traceStore.snapshot(traceId) + }; const backendPerformanceStore = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } }); + const workbenchRuntime = { + async queryAgentTraceEvents({ traceId: requestedTraceId }: { traceId: string }) { + assert.equal(requestedTraceId, traceId); + return { events: traceStore.snapshot(traceId).events }; + } + }; const accessController = { required: true, async authenticate() { @@ -123,11 +128,11 @@ test("backend performance records Code Agent trace handler phases without high-c projectId: "prj_hwpod_workbench", ownerUserId: "usr_backend_phase", ownerRole: "user", - session: {} + session: { traceResults: { [traceId]: projectedResult } } }; } }; - const server = createCloudApiServer({ backendPerformanceStore, traceStore, codeAgentChatResults, accessController, env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } }); + const server = createCloudApiServer({ backendPerformanceStore, traceStore, workbenchRuntime, accessController, env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); try { const address = server.address(); @@ -145,8 +150,8 @@ test("backend performance records Code Agent trace handler phases without high-c const metrics = await fetch(`${baseUrl}/v1/web-performance/metrics`); const text = await metrics.text(); assert.equal(metrics.status, 200); - assert.match(text, /hwlab_cloud_api_request_phase_duration_seconds_count\{[^}]*route="\/v1\/agent\/traces\/:id"[^}]*phase="trace_project_check"[^}]*\} 1/u); - assert.match(text, /hwlab_cloud_api_request_phase_duration_seconds_count\{[^}]*route="\/v1\/agent\/traces\/:id"[^}]*phase="trace_paginate"[^}]*\} 1/u); + assert.match(text, /hwlab_cloud_api_request_phase_duration_seconds_count\{[^}]*route="\/v1\/agent\/traces\/:traceId"[^}]*phase="trace_project_check"[^}]*\} 1/u); + assert.match(text, /hwlab_cloud_api_request_phase_duration_seconds_count\{[^}]*route="\/v1\/agent\/traces\/:traceId"[^}]*phase="trace_paginate"[^}]*\} 1/u); assert.doesNotMatch(text, /trc_backend_phase_trace|run_backend_phase|cmd_backend_phase|ses_backend_phase|cnv_backend_phase/u); } finally { await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); @@ -167,8 +172,7 @@ test("AgentRun adapter records upstream timing through backend performance conte }; if (request.method === "POST" && url.pathname === "/api/v1/sessions") return send({ ok: true }); if (request.method === "POST" && url.pathname === "/api/v1/runs") return send({ id: "run_backend_perf", status: "pending", backendProfile: "codex", sessionRef: body.sessionRef, resourceBundleRef: body.resourceBundleRef }); - if (request.method === "POST" && url.pathname === "/api/v1/runs/run_backend_perf/commands") return send({ id: "cmd_backend_perf", runId: "run_backend_perf", state: "pending", type: "turn", seq: 1 }); - if (request.method === "POST" && url.pathname === "/api/v1/runs/run_backend_perf/runner-jobs") return send({ action: "create-kubernetes-job", runId: "run_backend_perf", commandId: body.commandId, attemptId: "attempt_backend_perf", runnerId: "runner_backend_perf", namespace: "agentrun-v01", jobName: "agentrun-v01-runner-backend-perf" }); + if (request.method === "POST" && url.pathname === "/api/v1/runs/run_backend_perf/commands") return send({ id: "cmd_backend_perf", runId: "run_backend_perf", state: "pending", type: "turn", seq: 1, dispatchIntent: { id: "dispatch_backend_perf", state: "pending", runnerJobId: "rjob_backend_perf", attemptCount: 0, durable: true } }); response.writeHead(404, { "content-type": "application/json" }); response.end(`${JSON.stringify({ ok: false, message: `unexpected ${request.method} ${url.pathname}` })}\n`); }); @@ -215,11 +219,11 @@ test("AgentRun adapter records upstream timing through backend performance conte })); assert.equal(result.status, "running"); - assert.ok(calls.some((call) => call.path === "/api/v1/runs/run_backend_perf/runner-jobs")); + assert.ok(!calls.some((call) => call.path === "/api/v1/runs/run_backend_perf/runner-jobs")); const text = store.metricsText(); assert.match(text, /hwlab_cloud_api_upstream_duration_seconds_count\{[^}]*component="agentrun"[^}]*route="\/api\/v1\/runs"[^}]*\} 1/u); assert.match(text, /hwlab_cloud_api_upstream_duration_seconds_count\{[^}]*component="agentrun"[^}]*route="\/api\/v1\/runs\/:id\/commands"[^}]*\} 1/u); - assert.match(text, /hwlab_cloud_api_upstream_duration_seconds_count\{[^}]*component="agentrun"[^}]*route="\/api\/v1\/runs\/:id\/runner-jobs"[^}]*\} 1/u); + assert.doesNotMatch(text, /runner-jobs/u); assert.doesNotMatch(text, /run_backend_perf|cmd_backend_perf|ses_backend_perf|cnv_backend_perf/u); } finally { await new Promise((resolve, reject) => agentRunServer.close((error) => error ? reject(error) : resolve())); @@ -289,72 +293,3 @@ test("backend performance reports zero projection lag for caught-up terminal tra assert.ok(stuckLine?.endsWith(" 0")); assert.ok(terminalDelayBucketLine?.endsWith(" 1")); }); - -test("AgentRun read-side sync advances event cursor without forcing result aggregation before terminal evidence", async () => { - const calls: string[] = []; - const agentRunServer = createHttpServer(async (request, response) => { - const url = new URL(request.url || "/", "http://127.0.0.1"); - calls.push(`${request.method} ${url.pathname}`); - const send = (data: Record) => { - response.writeHead(200, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: true, data })}\n`); - }; - if (request.method === "GET" && url.pathname === "/api/v1/runs/run_read_side/events") { - return send({ - items: [ - { seq: 11, type: "assistant_message", runId: "run_read_side", payload: { commandId: "cmd_read_side", text: "working" }, createdAt: "2026-06-19T02:00:01.000Z" }, - { seq: 12, type: "tool_call", runId: "run_read_side", payload: { commandId: "cmd_read_side", toolName: "shell" }, createdAt: "2026-06-19T02:00:02.000Z" }, - ], - }); - } - if (request.method === "GET" && url.pathname.endsWith("/result")) return send({ terminalStatus: "completed", text: "should not be fetched" }); - response.writeHead(404, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: false, message: `unexpected ${request.method} ${url.pathname}` })}\n`); - }); - await new Promise((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve)); - try { - const address = agentRunServer.address(); - assert.ok(address && typeof address === "object"); - const store = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03", HWLAB_RUNTIME_LANE: "v03", HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601" } }); - const traceStore = createCodeAgentTraceStore(); - const synced = await syncAgentRunChatResult({ - traceId: "trc_read_side_cursor", - currentResult: { - status: "running", - traceId: "trc_read_side_cursor", - agentRun: { - adapter: "agentrun-v01", - runId: "run_read_side", - commandId: "cmd_read_side", - traceId: "trc_read_side_cursor", - providerTrace: { traceId: "trc_read_side_cursor" }, - lastSeq: 10, - managerUrl: `http://127.0.0.1:${address.port}`, - backendProfile: "codex", - }, - }, - options: { - backendPerformanceStore: store, - env: { - HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1", - HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601", - HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS: "2500", - }, - }, - traceStore, - forceResultSync: false, - }); - - assert.equal(synced.resultSynced, false); - assert.equal(synced.eventsRefreshed, true); - assert.equal(synced.result.agentRun.lastSeq, 12); - assert.ok(calls.some((call) => call === "GET /api/v1/runs/run_read_side/events")); - assert.ok(!calls.some((call) => call.endsWith("/result"))); - const text = store.metricsText(); - assert.match(text, /hwlab_workbench_projection_cursor_advance_total\{[^}]*status="advanced"[^}]*\} 2/u); - assert.match(text, /hwlab_workbench_projector_events_processed_total\{[^}]*event_kind="assistant_message"[^}]*\} 1/u); - assert.match(text, /hwlab_workbench_projector_batch_duration_seconds_count\{[^}]*phase="agentrun_events_fetch"[^}]*status="ok"[^}]*\} 1/u); - } finally { - await new Promise((resolve, reject) => agentRunServer.close((error) => error ? reject(error) : resolve())); - } -}); diff --git a/internal/cloud/backend-performance.ts b/internal/cloud/backend-performance.ts index 07b7703e..17b708cc 100644 --- a/internal/cloud/backend-performance.ts +++ b/internal/cloud/backend-performance.ts @@ -582,19 +582,19 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp "# HELP hwlab_agentrun_result_timeout_total AgentRun result timeout count by refresh budget.", "# TYPE hwlab_agentrun_result_timeout_total counter", ...renderCounters("hwlab_agentrun_result_timeout_total", agentRunResultTimeoutSeries), - "# HELP hwlab_workbench_projector_batch_duration_seconds Workbench projector/finalizer batch phase duration in seconds.", + "# HELP hwlab_workbench_projector_batch_duration_seconds Workbench Kafka projector batch phase duration in seconds.", "# TYPE hwlab_workbench_projector_batch_duration_seconds histogram", ...renderHistogram("hwlab_workbench_projector_batch_duration_seconds", projectorBatchDurationSeries, PROJECTOR_BATCH_BUCKETS_SECONDS), - "# HELP hwlab_workbench_projector_candidates_total Workbench projector/finalizer candidate count.", + "# HELP hwlab_workbench_projector_candidates_total Workbench Kafka projector candidate count.", "# TYPE hwlab_workbench_projector_candidates_total counter", ...renderCounters("hwlab_workbench_projector_candidates_total", projectorCandidatesSeries), - "# HELP hwlab_workbench_projector_events_processed_total Workbench projector/finalizer processed event count.", + "# HELP hwlab_workbench_projector_events_processed_total Workbench Kafka projector processed event count.", "# TYPE hwlab_workbench_projector_events_processed_total counter", ...renderCounters("hwlab_workbench_projector_events_processed_total", projectorEventsProcessedSeries), - "# HELP hwlab_workbench_projector_errors_total Workbench projector/finalizer error count.", + "# HELP hwlab_workbench_projector_errors_total Workbench Kafka projector error count.", "# TYPE hwlab_workbench_projector_errors_total counter", ...renderCounters("hwlab_workbench_projector_errors_total", projectorErrorsSeries), - "# HELP hwlab_workbench_projector_last_success_unixtime Last successful Workbench projector/finalizer sync unix time.", + "# HELP hwlab_workbench_projector_last_success_unixtime Last successful Workbench Kafka projector transaction unix time.", "# TYPE hwlab_workbench_projector_last_success_unixtime gauge", ...renderGauges("hwlab_workbench_projector_last_success_unixtime", projectorLastSuccessSeries), "" diff --git a/internal/cloud/bun-server.ts b/internal/cloud/bun-server.ts index 871e5b18..78430ebc 100644 --- a/internal/cloud/bun-server.ts +++ b/internal/cloud/bun-server.ts @@ -11,6 +11,12 @@ export async function createCloudApiBunServer(options: any = {}) { installPostgresTransientUnhandledRejectionBoundary({ env, logger }); const hwpodNodeWsRegistry = options.hwpodNodeWsRegistry || createHwpodNodeWsRegistry({ ...options, env }); const innerServer = createCloudApiServer({ ...options, env, hwpodNodeWsRegistry }); + try { + await innerServer.hwlabStartupReady; + } catch (error) { + await innerServer.hwlabAbortStartup?.(); + throw error; + } await new Promise((resolve) => innerServer.listen(0, "127.0.0.1", resolve)); const innerAddress = innerServer.address(); const innerPort = typeof innerAddress === "object" && innerAddress ? innerAddress.port : 0; @@ -19,7 +25,7 @@ export async function createCloudApiBunServer(options: any = {}) { const host = options.host ?? "0.0.0.0"; const port = options.port ?? 0; const idleTimeout = parsePositiveInteger(options.idleTimeout ?? env.HWLAB_CLOUD_API_IDLE_TIMEOUT_SECONDS, 120); - const server = Bun.serve({ + const server = (options.bunServe ?? Bun.serve)({ hostname: host, port, idleTimeout, diff --git a/internal/cloud/code-agent-agentrun-adapter.ts b/internal/cloud/code-agent-agentrun-adapter.ts index 0658f7dc..268c495d 100644 --- a/internal/cloud/code-agent-agentrun-adapter.ts +++ b/internal/cloud/code-agent-agentrun-adapter.ts @@ -1,5 +1,5 @@ // SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-17-r0; draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. -// Responsibility: AgentRun v0.1 adapter, incremental Workbench projection cursor sync, and upstream timing projection for Cloud API observability. +// Responsibility: AgentRun command admission and control adapter; Kafka owns event/result projection. import { randomUUID } from "node:crypto"; @@ -13,32 +13,12 @@ import { safeTraceId, text } from "./server-http-utils.ts"; -import { currentBackendPerformanceContext } from "./backend-performance.ts"; import { DEFAULT_AGENTRUN_MGR_URL, agentRunJson, isAgentRunManagerInternalServiceHost, resolveAgentRunManagerUrl } from "./code-agent-agentrun-runtime.ts"; -import { - agentRunErrorWithPolling, - agentRunEventCommandId, - agentRunMetricStatusForError, - agentRunPollingRetryableError, - agentRunPollingRootCause, - agentRunProjectionPollCount, - agentRunProjectionPollingState, - agentRunTerminalConsistencyGap, - agentRunTerminalResultRefreshSatisfied, - agentRunTerminalStatusFromEvents, - agentRunTraceCursorSeq, - emitAgentRunProjectionPollingSpan, - fetchAgentRunEventsForTrace, - measuredAgentRunEventsFetch, - measuredAgentRunResultFetch, - nonNegativeNumber, - recordAgentRunEventsProjectionMetrics -} from "./code-agent-agentrun-projection-polling.ts"; import { agentRunAlreadyTerminalCancelPayload, agentRunCancelBlockedPayload, @@ -65,7 +45,6 @@ import { agentRunPromptMetadataFromText, messageAuthorityTextValue } from "./code-agent-agentrun-prompt.ts"; -import { agentRunResultSyncDeferred, buildAgentRunProjectionEventsFetchPlan, buildWorkbenchProjectionStateUpdate } from "./workbench-projection-cursor.ts"; import { codeAgentOtelTraceFields, codeAgentOtelTraceContext, emitCodeAgentOtelSpan } from "./otel-trace.ts"; import * as agentRunResult from "./code-agent-agentrun-result.ts"; @@ -700,333 +679,11 @@ function sleepAgentRunDispatchRetry(ms) { return new Promise((resolve) => setTimeout(resolve, Math.max(0, Number(ms) || 0))); } -export async function syncAgentRunChatResult({ traceId, currentResult = null, options = {}, traceStore = defaultCodeAgentTraceStore, appendResultEvent = true, refreshEvents = true, forceResultSync = false, retryParams = null }) { - const initial = currentResult ?? await loadPersistedAgentRunResult(traceId, options); - if (isAgentRunUserCancelSealed(initial)) { - const runnerTrace = initial.runnerTrace ?? traceStore.snapshot(traceId); - return { result: initial, runnerTrace, found: true, eventsRefreshed: false, resultSynced: false, terminalRefreshSkipped: true, localCancelSealed: true }; - } - const mapped = initial ? await resolveAgentRunTraceCommandMapping({ traceId, mapped: initial, options }) : initial; - if (isAgentRunUserCancelSealed(mapped)) { - const runnerTrace = mapped.runnerTrace ?? traceStore.snapshot(traceId); - return { result: mapped, runnerTrace, found: true, eventsRefreshed: false, resultSynced: false, terminalRefreshSkipped: true, localCancelSealed: true }; - } - const env = options.env ?? process.env; - const performanceStore = options.backendPerformanceStore ?? currentBackendPerformanceContext(); - if (!forceResultSync && agentRunTerminalResultRefreshSatisfied(mapped, traceId)) { - const runnerTrace = mapped.runnerTrace ?? traceStore.snapshot(traceId); - const polling = agentRunProjectionPollingState({ env, terminalFromEvents: "completed", rootCause: "terminal_result_sealed", terminalConsistencyGap: 0 }); - emitAgentRunProjectionPollingSpan({ traceId, env, agentRun: mapped?.agentRun, polling, terminalRefreshSkipped: true }); - return { result: mapped, runnerTrace, found: true, eventsRefreshed: false, resultSynced: false, terminalRefreshSkipped: true, polling }; - } - if (!mapped?.agentRun?.runId || !mapped?.agentRun?.commandId) return { result: currentResult, runnerTrace: traceStore.snapshot(traceId), found: Boolean(currentResult), eventsRefreshed: false, resultSynced: false }; - const fetchImpl = options.fetchImpl ?? globalThis.fetch; - const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl); - const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000); - const projectionState = await loadWorkbenchProjectionStateForAgentRun(options.runtimeStore, { traceId, agentRun: mapped.agentRun }); - const projectionRetryParams = retryParams ?? projectionState?.retryParams ?? null; - const projectionPollCount = agentRunProjectionPollCount(projectionState); - const fetchPlan = buildAgentRunProjectionEventsFetchPlan({ - projectionState, - agentRun: mapped.agentRun, - pageLimit: env.HWLAB_CODE_AGENT_AGENTRUN_EVENTS_PAGE_LIMIT - }); - const previousLastSeq = Number(fetchPlan.afterSeq ?? mapped.agentRun.lastSeq ?? 0); - let eventsResponse = null; - try { - eventsResponse = refreshEvents ? await measuredAgentRunEventsFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapping: { ...mapped.agentRun, lastSeq: fetchPlan.afterSeq, afterSeqOverride: fetchPlan.afterSeq, eventsPageLimit: fetchPlan.limit, traceSummary: mapped.traceSummary, pollCount: projectionPollCount, backoffMs: nonNegativeNumber(projectionState?.backoffMs), rootCause: projectionState?.rootCause ?? null, terminalConsistencyGap: nonNegativeNumber(projectionState?.terminalConsistencyGap) } }) : null; - } catch (error) { - const polling = agentRunProjectionPollingState({ projectionState, env, error, rootCause: agentRunPollingRootCause(error), terminalConsistencyGap: nonNegativeNumber(projectionState?.terminalConsistencyGap) }); - await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, { - currentState: projectionState, - result: mapped, - agentRun: { ...mapped.agentRun, lastSeq: previousLastSeq }, - eventsResponse: null, - resultSyncState: projectionState?.resultSyncState ?? null, - projectionStatus: "degraded", - projectionHealth: "degraded", - retryParams: projectionRetryParams, - polling, - error, - now: options.now - }); - throw agentRunErrorWithPolling(error, polling); - } - if (eventsResponse) appendAgentRunEventsToTrace(traceStore, traceId, eventsResponse.events, mapped.agentRun); - const nextLastSeq = eventsResponse ? agentRunTraceCursorSeq(eventsResponse, mapped.agentRun.lastSeq) : mapped.agentRun.lastSeq; - const backendProfile = resolveAgentRunBackendProfile(env, { - providerProfile: firstNonEmpty( - projectionRetryParams?.providerProfile, - projectionRetryParams?.codeAgentProviderProfile, - projectionRetryParams?.backendProfile, - mapped.agentRun.backendProfile - ) - }); - recordAgentRunEventsProjectionMetrics(performanceStore, eventsResponse, previousLastSeq, nextLastSeq); - const terminalFromEvents = agentRunTerminalStatusFromEvents(eventsResponse?.events); - const cursorAdvance = Math.max(0, Number(nextLastSeq ?? 0) - Number(previousLastSeq ?? 0)); - const terminalConsistencyGap = agentRunTerminalConsistencyGap({ mapped, eventsResponse }); - const projectionPolling = agentRunProjectionPollingState({ - projectionState, - env, - pollCount: projectionPollCount, - cursorAdvance, - terminalFromEvents, - rootCause: terminalFromEvents ? "terminal_from_events" : cursorAdvance > 0 ? "cursor_advanced" : "projection_no_progress", - terminalConsistencyGap - }); - await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, { - currentState: projectionState, - result: mapped, - agentRun: { ...mapped.agentRun, lastSeq: nextLastSeq }, - eventsResponse, - resultSyncState: terminalFromEvents ? "pending" : null, - projectionStatus: terminalFromEvents ? "terminal" : null, - polling: projectionPolling, - retryParams: projectionRetryParams, - now: options.now - }); - const deferResultSync = terminalFromEvents ? false : agentRunResultSyncDeferred({ forceResultSync, options, env }); - if (!forceResultSync && (deferResultSync || !agentRunCommandResultSyncRequired(mapped, eventsResponse))) { - const nextMapping = withAgentRunRunnerJobCount({ - ...mapped.agentRun, - lastSeq: nextLastSeq, - status: mapped.agentRun.status ?? "running", - runStatus: mapped.agentRun.runStatus ?? null, - commandState: mapped.agentRun.commandState ?? null, - terminalStatus: null, - resultSyncState: terminalFromEvents ? "pending" : mapped.agentRun.resultSyncState ?? null, - updatedAt: nowIso(options.now), - valuesPrinted: false - }); - const payload = decorateAgentRunRunningResult({ base: { ...mapped, status: "running", agentRun: nextMapping, updatedAt: nowIso(options.now) }, mapping: nextMapping, traceStore, traceId }); - options.codeAgentChatResults?.set?.(traceId, payload); - return { result: payload, runnerTrace: payload.runnerTrace ?? traceStore.snapshot(traceId), found: true, eventsRefreshed: Boolean(eventsResponse), resultSynced: false, polling: projectionPolling }; - } - let result; - try { - result = await measuredAgentRunResultFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapped, eventsResponse }); - } catch (error) { - const retryable = agentRunPollingRetryableError(error); - const polling = agentRunProjectionPollingState({ projectionState, env, pollCount: projectionPollCount, cursorAdvance, terminalFromEvents, error, rootCause: agentRunPollingRootCause(error), terminalConsistencyGap }); - await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, { - currentState: projectionState, - result: mapped, - agentRun: { ...mapped.agentRun, lastSeq: nextLastSeq }, - eventsResponse, - resultSyncState: retryable ? "pending" : agentRunMetricStatusForError(error) === "timeout" ? "timed_out" : "failed", - projectionStatus: retryable ? "degraded" : terminalFromEvents ? "terminal" : null, - projectionHealth: "degraded", - retryParams: projectionRetryParams, - polling, - error, - now: options.now - }); - throw agentRunErrorWithPolling(error, polling); - } - const nextMapping = withAgentRunRunnerJobCount({ - ...mapped.agentRun, - ...agentRunResultRefs(result), - lastSeq: nextLastSeq, - status: result?.status ?? mapped.agentRun.status ?? "running", - runStatus: result?.runStatus ?? mapped.agentRun.runStatus ?? null, - commandState: result?.commandState ?? mapped.agentRun.commandState ?? null, - terminalStatus: result?.terminalStatus ?? mapped.agentRun.terminalStatus ?? null, - updatedAt: nowIso(options.now), - valuesPrinted: false - }); - const freshSessionRetry = await retryAgentRunTurnAfterFreshSessionFailure({ - params: projectionRetryParams, - options, - traceId, - traceStore, - mapped, - failedResult: result, - failedMapping: nextMapping, - env, - managerUrl, - fetchImpl, - timeoutMs, - backendProfile - }); - if (freshSessionRetry) { - options.codeAgentChatResults?.set?.(traceId, freshSessionRetry.payload); - await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, { - currentState: projectionState, - result: freshSessionRetry.payload, - agentRun: freshSessionRetry.mapping, - eventsResponse: null, - resultSyncState: null, - projectionStatus: "projecting", - retryParams: projectionRetryParams, - now: options.now - }); - const retrySync = await syncAgentRunChatResult({ - traceId, - currentResult: freshSessionRetry.payload, - options, - traceStore, - appendResultEvent, - refreshEvents: true, - forceResultSync, - retryParams: projectionRetryParams - }); - if (retrySync?.result) { - return { - ...retrySync, - found: true, - eventsRefreshed: Boolean(eventsResponse) || Boolean(retrySync.eventsRefreshed), - freshSessionRetried: true - }; - } - return { result: freshSessionRetry.payload, runnerTrace: freshSessionRetry.payload.runnerTrace ?? traceStore.snapshot(traceId), found: true, eventsRefreshed: Boolean(eventsResponse), resultSynced: false, freshSessionRetried: true }; - } - const base = { ...mapped, agentRun: nextMapping, updatedAt: nowIso(options.now) }; - const payload = agentRunResultToCodeAgentPayload({ base, result, traceStore, traceId, appendResultEvent }); - options.codeAgentChatResults?.set?.(traceId, payload); - const resultPolling = agentRunProjectionPollingState({ - projectionState, - env, - pollCount: projectionPollCount, - cursorAdvance, - terminalFromEvents: isAgentRunTerminalStatus(payload?.status ?? nextMapping.terminalStatus ?? nextMapping.commandState) ? (nextMapping.terminalStatus ?? payload?.status) : terminalFromEvents, - rootCause: isAgentRunTerminalStatus(payload?.status ?? nextMapping.terminalStatus ?? nextMapping.commandState) ? "terminal_result_synced" : "result_synced", - terminalConsistencyGap: agentRunTerminalConsistencyGap({ mapped: payload, eventsResponse, result }) - }); - await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, { - currentState: projectionState, - result: payload, - agentRun: nextMapping, - eventsResponse, - resultSyncState: "synced", - projectionStatus: isAgentRunTerminalStatus(payload?.status) ? "terminal" : null, - polling: resultPolling, - retryParams: projectionRetryParams, - now: options.now - }); - return { result: payload, runnerTrace: payload.runnerTrace ?? traceStore.snapshot(traceId), found: true, eventsRefreshed: Boolean(eventsResponse), resultSynced: true, polling: resultPolling }; -} - -async function loadWorkbenchProjectionStateForAgentRun(runtimeStore, { traceId, agentRun = {} } = {}) { - if (!runtimeStore || typeof runtimeStore.getWorkbenchProjectionState !== "function") return null; - try { - const result = await runtimeStore.getWorkbenchProjectionState({ traceId }); - const state = result?.projectionState ?? null; - if (!state) return null; - if (text(state.runId ?? state.sourceRunId) !== text(agentRun.runId)) return null; - if (text(state.commandId ?? state.sourceCommandId) !== text(agentRun.commandId)) return null; - return state; - } catch { - return null; - } -} - -async function writeWorkbenchProjectionStateForAgentRun(runtimeStore, params = {}) { - if (!runtimeStore || typeof runtimeStore.writeWorkbenchProjectionState !== "function") return null; - const projectionState = buildWorkbenchProjectionStateUpdate(params); - if (!projectionState.traceId || !projectionState.runId || !projectionState.commandId) return null; - return await runtimeStore.writeWorkbenchProjectionState({ projectionState }); -} - -function agentRunCommandResultSyncRequired(mapped = {}, eventsResponse = null) { - const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : {}; - if (agentRunTerminalResultRefreshSatisfied(mapped, mapped?.traceId)) return true; - if (isAgentRunTerminalStatus(agentRun.terminalStatus ?? agentRun.commandState)) return true; - return Boolean(agentRunTerminalStatusFromEvents(eventsResponse?.events)); -} - -function isAgentRunTerminalStatus(value) { - const status = normalizeAgentRunStatus(value); - return status === "timeout" || TERMINAL_RUN_STATUSES.has(status); -} - function normalizeAgentRunStatus(value) { const status = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); return status === "cancelled" ? "canceled" : status; } -function isAgentRunUserCancelSealed(payload = null) { - if (!payload || typeof payload !== "object") return false; - const status = normalizeAgentRunStatus(payload.status); - const cancelStatus = normalizeAgentRunStatus(payload.cancelStatus ?? payload.cancelDisposition?.status); - const terminalStatus = normalizeAgentRunStatus(payload.agentRun?.terminalStatus ?? payload.agentRun?.commandState ?? payload.agentRun?.status); - return payload.canceled === true || status === "canceled" || cancelStatus === "canceled" || terminalStatus === "canceled"; -} - -async function resolveAgentRunTraceCommandMapping({ traceId, mapped, options = {} }) { - const safeId = safeTraceId(traceId); - const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : null; - if (!safeId || !agentRun?.runId) return mapped; - const agentRunTraceId = safeTraceId(agentRun.traceId); - const providerTraceId = safeTraceId(agentRun.providerTrace?.traceId); - const traceSummaryCommandId = text(mapped?.traceSummary?.agentRun?.commandId); - const commandId = text(agentRun.commandId); - const traceFieldsMatch = agentRunTraceId === safeId && providerTraceId === safeId && (!traceSummaryCommandId || traceSummaryCommandId === commandId); - if (traceFieldsMatch && (mapped.status === "running" || agentRunTerminalResultRefreshSatisfied(mapped, safeId))) return mapped; - const env = options.env ?? process.env; - const fetchImpl = options.fetchImpl ?? globalThis.fetch; - const managerUrl = resolveAgentRunManagerUrl(env, agentRun.managerUrl); - const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000); - const command = await findAgentRunCommandForTrace({ fetchImpl, managerUrl, timeoutMs, env, runId: agentRun.runId, traceId: safeId }); - if (!command?.id) { - throw Object.assign(new Error(`AgentRun command registry has no command for ${safeId} in run ${agentRun.runId}`), { - code: "agentrun_trace_command_not_found", - statusCode: 404, - traceId: safeId, - runId: agentRun.runId - }); - } - if (command.id === commandId) { - return { - ...mapped, - finalResponse: null, - traceSummary: null, - agentRun: { - ...agentRun, - traceId: safeId, - lastSeq: 0, - providerTrace: null, - commandState: command.state ?? agentRun.commandState ?? null, - updatedAt: nowIso(options.now), - valuesPrinted: false - } - }; - } - return { - ...mapped, - finalResponse: null, - traceSummary: null, - agentRun: { - ...agentRun, - commandId: command.id, - traceId: safeId, - lastSeq: 0, - providerTrace: null, - commandState: command.state ?? agentRun.commandState ?? null, - updatedAt: nowIso(options.now), - valuesPrinted: false - } - }; -} - -export async function refreshAgentRunTrace({ traceId, result = null, options = {}, traceStore = defaultCodeAgentTraceStore }) { - const initial = result ?? await loadPersistedAgentRunResult(traceId, options); - const mapped = initial ? await resolveAgentRunTraceCommandMapping({ traceId, mapped: initial, options }) : initial; - if (!mapped?.agentRun?.runId) return traceStore.snapshot(traceId); - const env = options.env ?? process.env; - const fetchImpl = options.fetchImpl ?? globalThis.fetch; - const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl); - const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000); - const eventsResponse = await fetchAgentRunEventsForTrace({ fetchImpl, managerUrl, timeoutMs, env, mapping: { ...mapped.agentRun, traceSummary: mapped.traceSummary } }); - const events = eventsResponse.events; - appendAgentRunEventsToTrace(traceStore, traceId, events, mapped.agentRun); - const lastSeq = agentRunTraceCursorSeq(eventsResponse, mapped.agentRun.lastSeq); - if (lastSeq !== Number(mapped.agentRun.lastSeq ?? 0)) { - options.codeAgentChatResults?.set?.(traceId, { ...mapped, agentRun: { ...mapped.agentRun, lastSeq, updatedAt: nowIso(options.now) } }); - } - return traceStore.snapshot(traceId); -} - export async function cancelAgentRunChatTurn({ traceId, currentResult = null, options = {}, traceStore = defaultCodeAgentTraceStore }) { let mapped = currentResult ?? await loadPersistedAgentRunResult(traceId, options); if (!mapped?.agentRun?.commandId) return null; @@ -1043,12 +700,6 @@ export async function cancelAgentRunChatTurn({ traceId, currentResult = null, op }; const fetchImpl = options.fetchImpl ?? globalThis.fetch; const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl); - try { - const synced = await syncAgentRunChatResult({ traceId, currentResult: mapped, options: cancelOptions, traceStore, forceResultSync: true }); - mapped = synced?.result ?? synced ?? mapped; - } catch { - // Cancel must still expose the command cancel disposition; result refresh failures are reported by the normal trace/result paths. - } const terminalStatus = agentRunCancelTerminalStatus(mapped); if (terminalStatus) return agentRunAlreadyTerminalCancelPayload({ traceId, payload: mapped, terminalStatus, options: cancelOptions, traceStore }); const cancelPath = `/api/v1/commands/${encodeURIComponent(mapped.agentRun.commandId)}/cancel`; @@ -1155,7 +806,7 @@ export async function steerAgentRunChatTurn({ traceId, currentResult = null, par accepted: true, status: "running", shortConnection: true, - controlSemantics: "steer-active-turn-and-poll-target-trace", + controlSemantics: "steer-active-turn-and-project-sse", route: "/v1/agent/chat/steer", traceId, targetTraceId: traceId, @@ -1163,8 +814,8 @@ export async function steerAgentRunChatTurn({ traceId, currentResult = null, par conversationId: safeConversationId(mapped.conversationId ?? params.conversationId) || null, sessionId: safeSessionId(mapped.sessionId ?? params.sessionId) || null, threadId: safeOpaqueId(mapped.threadId ?? params.threadId) || null, - resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`, traceUrl: `/v1/agent/traces/${encodeURIComponent(traceId)}`, + workbenchEventsUrl: `/v1/workbench/events?sessionId=${encodeURIComponent(safeSessionId(mapped.sessionId ?? params.sessionId) || "")}&traceId=${encodeURIComponent(traceId)}`, agentRun: { ...agentRun, steerCommandId, @@ -1243,18 +894,6 @@ function agentRunSeedFromSession(session, traceId) { return topLevelAgentRun?.runId ? topLevelAgentRun : null; } -async function findAgentRunCommandForTrace({ fetchImpl, managerUrl, timeoutMs, env, runId, traceId }) { - if (!runId || !safeTraceId(traceId)) return null; - const commands = await agentRunJson(fetchImpl, managerUrl, `/api/v1/runs/${encodeURIComponent(runId)}/commands?afterSeq=0&limit=100`, { method: "GET", timeoutMs, env }); - return (Array.isArray(commands?.items) ? commands.items : []).find((item) => agentRunCommandMatchesTrace(item, traceId)) ?? null; -} - -function agentRunCommandMatchesTrace(command, traceId) { - const payload = command?.payload && typeof command.payload === "object" ? command.payload : {}; - return command?.idempotencyKey === traceId || payload.traceId === traceId; -} - - function withoutUserBillingReservation(value = {}) { if (!value || typeof value !== "object") return value; const { userBillingReservation, ...rest } = value; @@ -1353,126 +992,6 @@ function newSessionIdAfterEviction(baseSessionId, traceId) { return baseSessionId + "-reset-" + profile; } -async function retryAgentRunTurnAfterFreshSessionFailure({ - params = null, - options = {}, - traceId, - traceStore = defaultCodeAgentTraceStore, - mapped = {}, - failedResult = {}, - failedMapping = {}, - env = process.env, - managerUrl, - fetchImpl, - timeoutMs, - backendProfile -} = {}) { - const failure = agentRunFreshSessionFailure(failedResult); - if (!failure.requiresFreshSession) return null; - if (mapped?.agentRun?.freshSessionRetryOf || failedMapping?.freshSessionRetryOf) return null; - const promptText = messageAuthorityTextValue(params?.message ?? params?.prompt ?? params?.text ?? ""); - if (!promptText) { - traceStore.append(traceId, agentRunTraceEvent({ - type: "backend", - status: "failed", - label: "agentrun:session-reset:skipped", - errorCode: "agentrun_retry_prompt_missing", - message: "AgentRun session storage was evicted, but hwlab-cloud-api cannot retry the current turn because the original prompt text is unavailable in this sync path.", - runId: failedMapping?.runId ?? mapped?.agentRun?.runId ?? null, - commandId: failedMapping?.commandId ?? mapped?.agentRun?.commandId ?? null, - terminal: true, - valuesPrinted: false - }, failedMapping || mapped?.agentRun || backendProfile)); - return null; - } - const baseSessionId = scopedAgentRunSessionIdForParams(params, traceId); - const sessionId = newSessionIdAfterEviction(baseSessionId, traceId); - const retryParams = paramsWithAgentRunSessionThread(params, null); - const startedAt = nowIso(options.now); - const previous = { - runId: failedMapping?.runId ?? mapped?.agentRun?.runId ?? null, - commandId: failedMapping?.commandId ?? mapped?.agentRun?.commandId ?? null, - sessionId: failedMapping?.sessionId ?? mapped?.agentRun?.sessionId ?? baseSessionId, - threadId: failedMapping?.threadId ?? mapped?.agentRun?.threadId ?? mapped?.threadId ?? null, - failureKind: failure.code, - failureMessage: failure.message, - valuesPrinted: false - }; - traceStore.append(traceId, agentRunTraceEvent({ - type: "backend", - status: "running", - label: "agentrun:session-reset:retry-current-turn", - message: "AgentRun session storage/thread resume failed; hwlab-cloud-api is retrying the same user turn with a fresh AgentRun session and threadId=null.", - runId: previous.runId, - commandId: previous.commandId, - sessionId, - previousSessionId: previous.sessionId, - previousThreadId: previous.threadId, - errorCode: failure.code, - backendProfile, - waitingFor: "agentrun-run-create", - valuesPrinted: false - }, failedMapping || mapped?.agentRun || backendProfile)); - await ensureAgentRunSessionPersistent({ fetchImpl, managerUrl, sessionId, env, traceId, backendProfile, traceStore }); - const toolCapabilities = await resolveToolCapabilities({ params: retryParams, options }); - const runInput = buildAgentRunCreateRunInput({ params: retryParams, env, traceId, backendProfile, sessionId, toolCapabilities }); - const run = await agentRunDispatchJson({ fetchImpl, managerUrl, path: "/api/v1/runs", method: "POST", body: runInput, timeoutMs, env, traceStore, traceId, backendProfile, stage: "run-create-after-session-reset" }); - const runId = requiredString(run?.id, "run.id"); - traceStore.append(traceId, agentRunTraceEvent({ - type: "backend", - status: "running", - label: "agentrun:run:created-after-session-reset", - message: "AgentRun fresh-session retry run " + runId + " created for the current user turn.", - runId, - backendProfile, - waitingFor: "agentrun-command-create", - valuesPrinted: false - }, backendProfile)); - const dispatchInput = buildAgentRunDurableDispatchInput({ env, traceId, backendProfile }); - const commandInput = buildAgentRunCommandInput({ params: retryParams, traceId, backendProfile, sessionId, dispatchInput }); - const command = await agentRunDispatchJson({ fetchImpl, managerUrl, path: "/api/v1/runs/" + encodeURIComponent(runId) + "/commands", method: "POST", body: commandInput, timeoutMs, env, traceStore, traceId, backendProfile, runId, stage: "command-create-after-session-reset" }); - const commandId = requiredString(command?.id, "command.id"); - const dispatchIntent = requireAgentRunDurableDispatchIntent(command?.dispatchIntent); - traceStore.append(traceId, agentRunTraceEvent({ - type: "backend", - status: "running", - label: "agentrun:command:created-after-session-reset", - message: "AgentRun fresh-session retry command " + commandId + " and durable dispatch intent " + dispatchIntent.id + " were admitted atomically.", - runId, - commandId, - backendProfile, - dispatchIntentId: dispatchIntent.id, - runnerJobId: dispatchIntent.runnerJobId, - waitingFor: "agentrun-result", - valuesPrinted: false - }, backendProfile)); - const mapping = { - ...agentRunMapping({ env, managerUrl, backendProfile, run, command, dispatchIntent, traceId, startedAt, params: retryParams }), - freshSessionRetryAttempt: 1, - freshSessionRetryOf: previous, - status: `dispatch-intent-${dispatchIntent.state}`, - valuesPrinted: false - }; - traceStore.append(traceId, agentRunTraceEvent({ - type: "backend", - status: "running", - label: "agentrun:dispatch-intent:admitted-after-session-reset", - message: "AgentRun owns the durable fresh-session runner dispatch; HWLAB has no process-local runner Job creation window.", - runId: mapping.runId, - commandId: mapping.commandId, - backendProfile, - dispatchIntentId: dispatchIntent.id, - dispatchIntentState: dispatchIntent.state, - runnerJobId: dispatchIntent.runnerJobId, - waitingFor: "agentrun-result", - valuesPrinted: false - }, mapping)); - const base = initialAgentRunChatResult({ params: retryParams, options, traceId }); - const payload = decorateAgentRunRunningResult({ base: { ...base, freshSessionRetryOf: previous }, mapping, traceStore, traceId }); - return { payload, mapping }; -} - - function buildAgentRunCreateRunInput({ params, env, traceId, backendProfile, sessionId, toolCapabilities = null }) { const commitId = requireAgentRunSourceCommit(env); const runtimeAuthority = requireAgentRunRuntimeAuthority(env); @@ -1536,7 +1055,6 @@ function buildAgentRunCreateRunInput({ params, env, traceId, backendProfile, ses kind: "hwlab-cloud-api", traceId, ...codeAgentOtelTraceFields(traceId, env), - resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`, traceUrl: `/v1/agent/traces/${encodeURIComponent(traceId)}`, valuesPrinted: false } diff --git a/internal/cloud/code-agent-agentrun-cancel.test.ts b/internal/cloud/code-agent-agentrun-cancel.test.ts index 9796756c..6a55cb14 100644 --- a/internal/cloud/code-agent-agentrun-cancel.test.ts +++ b/internal/cloud/code-agent-agentrun-cancel.test.ts @@ -4,7 +4,7 @@ import assert from "node:assert/strict"; import { test } from "bun:test"; -import { cancelAgentRunChatTurn, syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts"; +import { cancelAgentRunChatTurn } from "./code-agent-agentrun-adapter.ts"; import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; test("AgentRun cancel downstream failure returns visible blocked disposition instead of throwing", async () => { @@ -54,44 +54,3 @@ test("AgentRun cancel downstream failure returns visible blocked disposition ins assert.equal(codeAgentChatResults.get(traceId)?.cancelDisposition?.code, "command_not_cancelable"); assert.ok(traceStore.snapshot(traceId).events.some((event) => event.label === "agentrun:cancel:blocked" && event.status === "blocked")); }); - -test("AgentRun result sync preserves a user-canceled payload instead of accepting late completed result", async () => { - const traceId = "trc_cancel_sealed_result"; - const traceStore = createCodeAgentTraceStore(); - const currentResult = { - traceId, - status: "canceled", - canceled: true, - cancelStatus: "canceled", - cancelDisposition: { status: "canceled", forwarded: true, terminal: true, traceId, valuesPrinted: false }, - finalResponse: { text: "hwlab-user-cancel", status: "canceled", traceId, valuesPrinted: false }, - agentRun: { - runId: "run_cancel_sealed_result", - commandId: "cmd_cancel_sealed_result", - managerUrl: "http://127.0.0.1:65535", - status: "cancelled", - commandState: "cancelled", - terminalStatus: "cancelled", - traceId, - valuesPrinted: false - }, - valuesPrinted: false - }; - - const synced = await syncAgentRunChatResult({ - traceId, - currentResult, - traceStore, - options: { - env: { HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1" }, - fetchImpl() { - throw new Error("late AgentRun result must not be fetched after HWLAB user cancel is sealed"); - } - } - }); - - assert.equal(synced.localCancelSealed, true); - assert.equal(synced.resultSynced, false); - assert.equal(synced.result.status, "canceled"); - assert.equal(synced.result.finalResponse.text, "hwlab-user-cancel"); -}); diff --git a/internal/cloud/code-agent-agentrun-durable-admission.test.ts b/internal/cloud/code-agent-agentrun-durable-admission.test.ts index 017fbd16..4fb1b7a3 100644 --- a/internal/cloud/code-agent-agentrun-durable-admission.test.ts +++ b/internal/cloud/code-agent-agentrun-durable-admission.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { test } from "bun:test"; -import { submitAgentRunChatTurn, syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts"; +import { submitAgentRunChatTurn } from "./code-agent-agentrun-adapter.ts"; import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; const sourceCommit = "0123456789abcdef0123456789abcdef01234567"; @@ -154,70 +154,6 @@ for (const terminal of [ }); } -test("gpt.pika session-eviction retry uses the same durable dispatch admission", async () => { - const traceId = "trc_gpt_pika_durable_retry"; - const calls = []; - const traceStore = createCodeAgentTraceStore(); - const synced = await syncAgentRunChatResult({ - traceId, - traceStore, - refreshEvents: false, - forceResultSync: true, - retryParams: { - ownerUserId: "usr_gpt_pika_retry", - projectId: "pikasTech/HWLAB", - conversationId: "cnv_gpt_pika_retry", - sessionId: "ses_gpt_pika_retry", - threadId: "thread_evicted", - message: "retry the evicted gpt.pika turn", - providerProfile: "gpt.pika" - }, - currentResult: { - traceId, - status: "running", - conversationId: "cnv_gpt_pika_retry", - sessionId: "ses_gpt_pika_retry", - threadId: "thread_evicted", - providerProfile: "gpt.pika", - agentRun: { - adapter: "agentrun-v01", - runId: "run_gpt_pika_evicted", - commandId: "cmd_gpt_pika_evicted", - sessionId: "ses_agentrun_gpt_pika_retry", - threadId: "thread_evicted", - backendProfile: "gpt.pika", - managerUrl: "http://127.0.0.1:65535", - status: "running", - traceId, - lastSeq: 0, - providerTrace: { traceId, valuesPrinted: false }, - valuesPrinted: false - }, - valuesPrinted: false - }, - options: { - env: agentRunTestEnv(), - fetchImpl: createAgentRunRetryFetch(calls, traceId) - } - }); - - assert.equal(synced.freshSessionRetried, true); - assert.equal(synced.result.status, "completed"); - assert.equal(synced.result.agentRun.runId, "run_gpt_pika_retry"); - assert.equal(synced.result.agentRun.commandId, "cmd_gpt_pika_retry"); - assert.equal(synced.result.agentRun.freshSessionRetryOf.runId, "run_gpt_pika_evicted"); - const retryCommand = calls.find((call) => call.path === "/api/v1/runs/run_gpt_pika_retry/commands"); - assert.ok(retryCommand); - assert.equal(retryCommand.body.idempotencyKey, traceId); - assert.equal(retryCommand.body.payload.threadId, null); - assert.equal(retryCommand.body.dispatch.kind, "kubernetes-job"); - assert.equal(Object.hasOwn(retryCommand.body.dispatch.input, "commandId"), false); - assert.equal(retryCommand.body.dispatch.input.transientEnv.every((entry) => entry.sensitive === false), true); - assert.equal(JSON.stringify(retryCommand.body).includes("HWLAB_API_KEY"), false); - assert.equal(calls.some((call) => call.path.endsWith("/runner-jobs")), false); - assert.ok(traceStore.snapshot(traceId).events.some((event) => event.label === "agentrun:dispatch-intent:admitted-after-session-reset")); -}); - function agentRunTestEnv() { return { AGENTRUN_MGR_URL: "http://127.0.0.1:65535", @@ -273,89 +209,3 @@ function createAgentRunFetch(calls, { dispatchIntent }) { throw new Error(`unexpected AgentRun call ${method} ${parsed.pathname}`); }; } - -function createAgentRunRetryFetch(calls, traceId) { - return async (url, init = {}) => { - const parsed = new URL(url); - const method = init.method ?? "GET"; - const body = init.body ? JSON.parse(String(init.body)) : null; - calls.push({ method, path: parsed.pathname, body }); - const send = (data) => new Response(JSON.stringify({ ok: true, data }), { - status: 200, - headers: { "content-type": "application/json" } - }); - if (method === "GET" && parsed.pathname === "/api/v1/runs/run_gpt_pika_evicted/commands/cmd_gpt_pika_evicted/result") { - return send({ - runId: "run_gpt_pika_evicted", - commandId: "cmd_gpt_pika_evicted", - status: "failed", - runStatus: "failed", - commandState: "failed", - terminalStatus: "failed", - failureKind: "session-store-evicted", - failureMessage: "session storage was evicted while resuming the thread", - completed: false, - lastSeq: 1 - }); - } - if (method === "POST" && parsed.pathname === "/api/v1/sessions") { - return send({ session: { id: body.sessionId, metadata: {} } }); - } - if (method === "POST" && parsed.pathname === "/api/v1/runs") { - return send({ - id: "run_gpt_pika_retry", - status: "pending", - sessionRef: body.sessionRef, - backendProfile: body.backendProfile - }); - } - if (method === "POST" && parsed.pathname === "/api/v1/runs/run_gpt_pika_retry/commands") { - return send({ - id: "cmd_gpt_pika_retry", - runId: "run_gpt_pika_retry", - state: "pending", - type: "turn", - seq: 1, - dispatchIntent: { - id: "dispatch_gpt_pika_retry", - state: "pending", - runnerJobId: "rjob_gpt_pika_retry", - attemptCount: 0, - durable: true - } - }); - } - if (method === "GET" && parsed.pathname === "/api/v1/runs/run_gpt_pika_retry/events") { - return send({ items: [] }); - } - if (method === "GET" && parsed.pathname === "/api/v1/runs/run_gpt_pika_retry/commands") { - return send({ items: [{ - id: "cmd_gpt_pika_retry", - runId: "run_gpt_pika_retry", - state: "completed", - idempotencyKey: traceId, - payload: { traceId } - }] }); - } - if (method === "GET" && parsed.pathname === "/api/v1/runs/run_gpt_pika_retry/commands/cmd_gpt_pika_retry/result") { - return send({ - runId: "run_gpt_pika_retry", - commandId: "cmd_gpt_pika_retry", - status: "completed", - runStatus: "completed", - commandState: "completed", - terminalStatus: "completed", - completed: true, - reply: "GPT_PIKA_RETRY_OK", - lastSeq: 2, - sessionRef: { - sessionId: "ses_agentrun_gpt_pika_retry-reset-trcgptpikadu", - conversationId: "cnv_gpt_pika_retry", - threadId: "thread_gpt_pika_retry" - }, - traceId - }); - } - throw new Error(`unexpected AgentRun retry call ${method} ${parsed.pathname}`); - }; -} diff --git a/internal/cloud/code-agent-agentrun-projection-polling.ts b/internal/cloud/code-agent-agentrun-projection-polling.ts deleted file mode 100644 index 546f82d1..00000000 --- a/internal/cloud/code-agent-agentrun-projection-polling.ts +++ /dev/null @@ -1,350 +0,0 @@ -// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. -// Responsibility: AgentRun projection polling/backoff I/O, durable polling state, and projection_sync observability attributes. - -import { agentRunJson } from "./code-agent-agentrun-runtime.ts"; -import { emitCodeAgentOtelSpan } from "./otel-trace.ts"; -import { parsePositiveInteger, safeTraceId, text } from "./server-http-utils.ts"; - -const DEFAULT_AGENTRUN_PROJECTION_SYNC_BACKOFF_INITIAL_MS = 1_000; -const DEFAULT_AGENTRUN_PROJECTION_SYNC_BACKOFF_MAX_MS = 30_000; -const AGENTRUN_TERMINAL_STATUSES = new Set(["completed", "failed", "blocked", "cancelled", "canceled", "timeout"]); - -export function agentRunProjectionPollCount(projectionState = null) { - return nonNegativeNumber(projectionState?.pollCount) + 1; -} - -export function agentRunProjectionPollingState({ projectionState = null, env = process.env, pollCount = null, cursorAdvance = 0, terminalFromEvents = null, error = null, rootCause = null, terminalConsistencyGap = 0 } = {}) { - const retryable = agentRunPollingRetryableError(error); - const noProgress = !error && !terminalFromEvents && Number(cursorAdvance ?? 0) <= 0; - const previousNoProgress = nonNegativeNumber(projectionState?.noProgressPollCount); - const noProgressPollCount = retryable || noProgress ? previousNoProgress + 1 : 0; - const backoffReason = retryable ? "provider_retry_or_rate_limit" : noProgress ? "projection_no_progress" : null; - const backoffMs = backoffReason ? agentRunProjectionBackoffMs(noProgressPollCount, env) : 0; - return { - pollCount: Number.isInteger(Number(pollCount)) && Number(pollCount) > 0 ? Math.floor(Number(pollCount)) : agentRunProjectionPollCount(projectionState), - noProgressPollCount, - backoffMs, - backoffReason, - rootCause: boundedAgentRunRootCause(rootCause ?? agentRunPollingRootCause(error) ?? backoffReason ?? "projection_sync"), - terminalConsistencyGap: nonNegativeNumber(terminalConsistencyGap), - nextRetryAt: backoffMs > 0 ? new Date(Date.now() + backoffMs).toISOString() : null, - valuesPrinted: false - }; -} - -export function agentRunPollingRetryableError(error = null) { - const statusCode = Number(error?.statusCode ?? error?.agentRunError?.statusCode ?? 0); - if ([408, 425, 429, 500, 502, 503, 504].includes(statusCode)) return true; - const code = String(error?.code ?? error?.agentRunError?.failureKind ?? "").toLowerCase(); - return error?.retryable === true || code === "agentrun_timeout" || /rate.?limit|retry|timeout|temporar|unavailable|overload/u.test(code); -} - -export function agentRunPollingRootCause(error = null) { - if (!error) return null; - const statusCode = Number(error?.statusCode ?? error?.agentRunError?.statusCode ?? 0); - if (statusCode === 429) return "provider_rate_limited"; - if ([408, 425, 500, 502, 503, 504].includes(statusCode)) return `provider_retry_http_${statusCode}`; - return error?.code ?? error?.agentRunError?.failureKind ?? error?.name ?? "agentrun_sync_failed"; -} - -export function agentRunErrorWithPolling(error, polling = null) { - if (!error || !polling) return error; - error.polling = polling; - error.backoffMs = polling.backoffMs; - error.nextRetryAt = polling.nextRetryAt; - error.rootCause = polling.rootCause; - return error; -} - -export function agentRunTerminalConsistencyGap({ mapped = {}, eventsResponse = null, result = null } = {}) { - const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : {}; - const terminal = agentRunPollingTerminalStatus(result?.terminalStatus ?? result?.commandState ?? result?.status ?? mapped?.status ?? agentRun.terminalStatus ?? agentRun.commandState) || Boolean(agentRunTerminalStatusFromEvents(eventsResponse?.events)); - if (!terminal) return 0; - const latest = Math.max( - nonNegativeNumber(eventsResponse?.maxSeq), - nonNegativeNumber(result?.eventCount), - nonNegativeNumber(result?.lastSeq), - nonNegativeNumber(agentRun.lastSeq) - ); - const observed = Math.max( - nonNegativeNumber(eventsResponse?.traceLastSeq), - nonNegativeNumber(result?.lastSeq), - nonNegativeNumber(agentRun.lastSeq) - ); - return Math.max(0, latest - observed); -} - -export function agentRunTerminalResultRefreshSatisfied(mapped = {}, traceId = null) { - const safeId = safeTraceId(traceId ?? mapped?.traceId); - const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : null; - const traceSummary = mapped?.traceSummary && typeof mapped.traceSummary === "object" ? mapped.traceSummary : null; - const finalResponse = mapped?.finalResponse && typeof mapped.finalResponse === "object" ? mapped.finalResponse : null; - const providerTrace = agentRun?.providerTrace && typeof agentRun.providerTrace === "object" - ? agentRun.providerTrace - : mapped?.providerTrace && typeof mapped.providerTrace === "object" - ? mapped.providerTrace - : null; - const commandId = text(agentRun?.commandId); - const runId = text(agentRun?.runId); - if (!safeId || !runId || !commandId) return false; - if (!agentRunPollingTerminalStatus(mapped?.status ?? agentRun?.terminalStatus ?? agentRun?.commandState ?? agentRun?.status)) return false; - if (safeTraceId(mapped?.traceId ?? agentRun?.traceId) !== safeId) return false; - if (safeTraceId(providerTrace?.traceId) !== safeId) return false; - const finalTraceId = safeTraceId(finalResponse?.traceId ?? safeId); - if (finalTraceId && finalTraceId !== safeId) return false; - if (!String(finalResponse?.text ?? "").trim()) return false; - if (traceSummary?.source !== "agentrun-command-result") return false; - if (safeTraceId(traceSummary.traceId ?? safeId) !== safeId) return false; - const summaryAgentRun = traceSummary.agentRun && typeof traceSummary.agentRun === "object" ? traceSummary.agentRun : null; - if (text(summaryAgentRun?.runId) !== runId) return false; - if (text(summaryAgentRun?.commandId) !== commandId) return false; - return true; -} - -export function emitAgentRunProjectionPollingSpan({ traceId, env = process.env, agentRun = {}, polling = {}, terminalRefreshSkipped = false } = {}) { - void emitCodeAgentOtelSpan("projection_sync", traceId, env, { - attributes: { - runId: agentRun?.runId ?? null, - commandId: agentRun?.commandId ?? null, - terminalRefreshSkipped: terminalRefreshSkipped === true, - pollCount: nonNegativeNumber(polling.pollCount), - backoffMs: nonNegativeNumber(polling.backoffMs), - rootCause: boundedAgentRunRootCause(polling.rootCause), - terminalConsistencyGap: nonNegativeNumber(polling.terminalConsistencyGap) - } - }); -} - -export async function fetchAgentRunEventsForTrace({ fetchImpl, managerUrl, timeoutMs, env, mapping = {} }) { - const runId = requiredString(mapping.runId, "runId"); - const currentCommandId = typeof mapping.commandId === "string" ? mapping.commandId : ""; - const replayWindow = agentRunTraceReplayWindow(mapping); - const afterSeqOverride = Number(mapping.afterSeqOverride); - const afterSeq = Number.isFinite(afterSeqOverride) && afterSeqOverride >= 0 ? Math.floor(afterSeqOverride) : replayWindow.afterSeq; - const endSeq = Number.isFinite(afterSeqOverride) && afterSeqOverride >= 0 ? 0 : replayWindow.endSeq; - const limit = parsePositiveInteger(mapping.eventsPageLimit ?? env?.HWLAB_CODE_AGENT_AGENTRUN_EVENTS_PAGE_LIMIT, 500); - const path = `/api/v1/runs/${encodeURIComponent(runId)}/events?afterSeq=${encodeURIComponent(String(afterSeq))}&limit=${encodeURIComponent(String(limit))}`; - const response = await agentRunJson(fetchImpl, managerUrl, path, { method: "GET", timeoutMs, env }); - const rawEvents = Array.isArray(response?.items) ? response.items : []; - const events = rawEvents.filter((event) => agentRunEventBelongsToTrace(event, { currentCommandId, afterSeq, endSeq })); - const maxSeq = Math.max(afterSeq, ...rawEvents.map((event) => Number(event?.seq ?? 0)).filter(Number.isFinite)); - const traceLastSeq = Math.max(afterSeq, ...events.map((event) => Number(event?.seq ?? 0)).filter(Number.isFinite)); - void emitCodeAgentOtelSpan("projection_sync", mapping.traceId ?? mapping.hwlabTraceId ?? mapping.businessTraceId, env, { - attributes: { - runId, - commandId: currentCommandId || null, - afterSeq, - endSeq, - limit, - rawEventCount: rawEvents.length, - eventCount: events.length, - commandFiltered: Boolean(currentCommandId), - maxSeq, - traceLastSeq, - terminalFromRawEvents: agentRunTerminalStatusFromEvents(rawEvents), - terminalFromEvents: agentRunTerminalStatusFromEvents(events), - pollCount: nonNegativeNumber(mapping.pollCount), - backoffMs: nonNegativeNumber(mapping.backoffMs), - rootCause: boundedAgentRunRootCause(mapping.rootCause), - terminalConsistencyGap: nonNegativeNumber(mapping.terminalConsistencyGap) - } - }); - return { - events, - afterSeq, - endSeq, - rawEventCount: rawEvents.length, - commandFiltered: Boolean(currentCommandId), - maxSeq, - traceLastSeq - }; -} - -export async function measuredAgentRunEventsFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapping = {} }) { - const startedAt = nowMs(); - try { - const response = await fetchAgentRunEventsForTrace({ fetchImpl, managerUrl, timeoutMs, env, mapping }); - performanceStore?.recordWorkbenchProjectorBatch?.({ phase: "agentrun_events_fetch", status: "ok", durationMs: nowMs() - startedAt }); - return response; - } catch (error) { - performanceStore?.recordWorkbenchProjectorBatch?.({ phase: "agentrun_events_fetch", status: agentRunMetricStatusForError(error), durationMs: nowMs() - startedAt }); - performanceStore?.recordWorkbenchProjectorError?.({ reason: error?.code ?? "agentrun_events_fetch_failed" }); - throw error; - } -} - -export async function measuredAgentRunResultFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapped = {}, eventsResponse = null }) { - const startedAt = nowMs(); - const path = `/api/v1/runs/${encodeURIComponent(mapped.agentRun.runId)}/commands/${encodeURIComponent(mapped.agentRun.commandId)}/result`; - try { - const result = await agentRunJson(fetchImpl, managerUrl, path, { method: "GET", timeoutMs, env }); - const eventCount = agentRunResultEventCount(result, eventsResponse); - performanceStore?.recordAgentRunResult?.({ - durationMs: nowMs() - startedAt, - eventCount, - eventsScanned: eventCount, - pagesScanned: estimateAgentRunResultPages(eventCount), - status: result?.terminalStatus ?? result?.status ?? "ok", - budget: timeoutMs - }); - return result; - } catch (error) { - const status = agentRunMetricStatusForError(error); - const eventCount = agentRunResultEventCount(null, eventsResponse); - performanceStore?.recordAgentRunResult?.({ - durationMs: nowMs() - startedAt, - eventCount, - eventsScanned: eventCount, - pagesScanned: estimateAgentRunResultPages(eventCount), - status, - budget: timeoutMs, - timedOut: status === "timeout" - }); - performanceStore?.recordWorkbenchProjectorError?.({ reason: error?.code ?? "agentrun_result_failed" }); - throw error; - } -} - -export function recordAgentRunEventsProjectionMetrics(performanceStore, eventsResponse = null, previousLastSeq = 0, nextLastSeq = 0) { - if (!performanceStore || !eventsResponse) return; - const cursorAdvance = Math.max(0, Number(nextLastSeq ?? 0) - Number(previousLastSeq ?? 0)); - if (cursorAdvance > 0) { - performanceStore.recordWorkbenchProjectionCursorAdvance?.({ - status: "advanced", - reason: eventsResponse.commandFiltered ? "command_filtered_events" : "run_events", - projectionStatus: "projecting", - source: "agentrun_v01", - count: cursorAdvance - }); - } - const counts = new Map(); - for (const event of eventsResponse.events ?? []) { - const kind = String(event?.type ?? event?.payload?.phase ?? "unknown").trim() || "unknown"; - counts.set(kind, (counts.get(kind) ?? 0) + 1); - } - if (counts.size === 0 && Number(eventsResponse.rawEventCount ?? 0) > 0) counts.set("filtered", Number(eventsResponse.rawEventCount)); - for (const [eventKind, count] of counts.entries()) { - performanceStore.recordWorkbenchProjectorEvents?.({ eventKind, status: "ok", count }); - } -} - -export function agentRunMetricStatusForError(error) { - return error?.code === "agentrun_timeout" || error?.name === "AbortError" || Number(error?.statusCode) === 504 ? "timeout" : "error"; -} - -export function agentRunTraceCursorSeq(eventsResponse = {}, previousLastSeq = 0) { - const afterSeq = Number(eventsResponse.afterSeq ?? 0); - const endSeq = Number(eventsResponse.endSeq ?? 0); - const traceLastSeq = Number(eventsResponse.traceLastSeq ?? 0); - if (Number.isFinite(traceLastSeq) && traceLastSeq > afterSeq) return Math.floor(traceLastSeq); - if (Number.isFinite(endSeq) && endSeq > 0) return Math.floor(endSeq); - if (eventsResponse.commandFiltered === true) return Math.max(Number(previousLastSeq ?? 0), afterSeq); - return Math.max(Number(previousLastSeq ?? 0), Number(eventsResponse.maxSeq ?? 0)); -} - -export function agentRunTerminalStatusFromEvents(events = []) { - if (!Array.isArray(events)) return null; - for (const event of [...events].reverse()) { - const payload = event?.payload && typeof event.payload === "object" ? event.payload : {}; - const phase = payload.phase ?? event?.phase; - const candidates = [ - event?.type === "terminal_status" ? payload.terminalStatus ?? event?.terminalStatus ?? payload.status ?? event?.status : null, - phase === "command-terminal" ? payload.terminalStatus ?? event?.terminalStatus ?? payload.status ?? event?.status : null, - phase === "turn:completed" || phase === "turn/steer:completed" ? "completed" : null, - event?.terminal === true ? payload.terminalStatus ?? event?.terminalStatus ?? payload.status ?? event?.status ?? "completed" : null, - payload.terminal === true ? payload.terminalStatus ?? event?.terminalStatus ?? payload.status ?? event?.status ?? "completed" : null - ]; - for (const value of candidates) { - const status = normalizeAgentRunStatus(value); - if (agentRunPollingTerminalStatus(status)) return status; - } - } - return null; -} - -export function agentRunEventCommandId(event) { - const payload = event?.payload && typeof event.payload === "object" ? event.payload : {}; - if (typeof event?.commandId === "string") return event.commandId; - return typeof payload.commandId === "string" ? payload.commandId : ""; -} - -export function nonNegativeNumber(value) { - const number = Number(value ?? 0); - return Number.isFinite(number) && number > 0 ? Math.floor(number) : 0; -} - -function agentRunProjectionBackoffMs(attempt, env = process.env) { - const initialMs = parsePositiveInteger(env?.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_BACKOFF_INITIAL_MS, DEFAULT_AGENTRUN_PROJECTION_SYNC_BACKOFF_INITIAL_MS); - const maxMs = parsePositiveInteger(env?.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_BACKOFF_MAX_MS, DEFAULT_AGENTRUN_PROJECTION_SYNC_BACKOFF_MAX_MS); - const exponent = Math.max(0, Math.min(10, nonNegativeNumber(attempt) - 1)); - return Math.min(maxMs, initialMs * 2 ** exponent); -} - -function boundedAgentRunRootCause(value) { - const raw = String(value ?? "").trim().replace(/[^a-z0-9_.:-]+/giu, "_").replace(/^_+|_+$/gu, "").toLowerCase(); - return raw ? raw.slice(0, 120) : null; -} - -function agentRunResultEventCount(result = null, eventsResponse = null) { - const candidates = [ - result?.traceSummary?.sourceEventCount, - result?.traceSummary?.eventCount, - result?.sourceEventCount, - result?.eventCount, - result?.providerTrace?.eventCount, - eventsResponse?.maxSeq, - eventsResponse?.rawEventCount, - eventsResponse?.events?.length - ]; - for (const value of candidates) { - const count = Number(value); - if (Number.isFinite(count) && count >= 0) return Math.floor(count); - } - return 0; -} - -function estimateAgentRunResultPages(eventCount) { - const count = Number(eventCount); - if (!Number.isFinite(count) || count <= 0) return 0; - return Math.max(1, Math.ceil(count / 500)); -} - -function agentRunTraceReplayWindow(mapping = {}) { - const eventStartSeq = Number(mapping.eventStartSeq ?? mapping.commandStartSeq ?? mapping.startSeq ?? 0); - const summary = mapping.traceSummary && typeof mapping.traceSummary === "object" ? mapping.traceSummary : null; - const summaryAgentRun = summary?.agentRun && typeof summary.agentRun === "object" ? summary.agentRun : null; - const summaryLastSeq = Number(summaryAgentRun?.lastSeq ?? summary?.lastSeq ?? 0); - const currentLastSeq = Number(mapping.lastSeq ?? 0); - const endSeq = Number.isFinite(summaryLastSeq) && summaryLastSeq > 0 ? Math.floor(summaryLastSeq) : 0; - if (Number.isFinite(eventStartSeq) && eventStartSeq > 0) return { afterSeq: Math.max(0, Math.floor(eventStartSeq) - 1), endSeq }; - if (endSeq > 0) return { afterSeq: Math.max(0, endSeq - 500), endSeq }; - if (Number.isFinite(currentLastSeq) && currentLastSeq > 0) return { afterSeq: Math.floor(currentLastSeq), endSeq: 0 }; - return { afterSeq: 0, endSeq: 0 }; -} - -function agentRunEventBelongsToTrace(event, { currentCommandId = "", afterSeq = 0, endSeq = 0 } = {}) { - const eventCommandId = agentRunEventCommandId(event); - const seq = Number(event?.seq ?? 0); - if (currentCommandId && eventCommandId && eventCommandId !== currentCommandId) return false; - if (endSeq > 0 && Number.isFinite(seq)) return seq > afterSeq && seq <= endSeq; - return true; -} - -function agentRunPollingTerminalStatus(value) { - return AGENTRUN_TERMINAL_STATUSES.has(normalizeAgentRunStatus(value)); -} - -function normalizeAgentRunStatus(value) { - const status = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); - return status === "cancelled" ? "canceled" : status; -} - -function requiredString(value, fieldName) { - if (typeof value === "string" && value.trim()) return value.trim(); - throw Object.assign(new Error(`AgentRun response missing ${fieldName}`), { code: "agentrun_response_invalid", statusCode: 502 }); -} - -function nowMs() { - if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now(); - return Date.now(); -} diff --git a/internal/cloud/code-agent-agentrun-result.ts b/internal/cloud/code-agent-agentrun-result.ts index b629af2c..f4924d13 100644 --- a/internal/cloud/code-agent-agentrun-result.ts +++ b/internal/cloud/code-agent-agentrun-result.ts @@ -1,5 +1,5 @@ // SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-17-r0; draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. -// Responsibility: AgentRun command-result/event mapping, terminal projection, and trace observability helpers. +// Responsibility: AgentRun command-result/event mapping and trace observability helpers; Kafka owns persistence. import { randomUUID } from "node:crypto"; @@ -20,25 +20,6 @@ import { isAgentRunManagerInternalServiceHost, resolveAgentRunManagerUrl } from "./code-agent-agentrun-runtime.ts"; -import { - agentRunErrorWithPolling, - agentRunEventCommandId, - agentRunMetricStatusForError, - agentRunPollingRetryableError, - agentRunPollingRootCause, - agentRunProjectionPollCount, - agentRunProjectionPollingState, - agentRunTerminalConsistencyGap, - agentRunTerminalResultRefreshSatisfied, - agentRunTerminalStatusFromEvents, - agentRunTraceCursorSeq, - emitAgentRunProjectionPollingSpan, - fetchAgentRunEventsForTrace, - measuredAgentRunEventsFetch, - measuredAgentRunResultFetch, - nonNegativeNumber, - recordAgentRunEventsProjectionMetrics -} from "./code-agent-agentrun-projection-polling.ts"; import { agentRunAlreadyTerminalCancelPayload, agentRunCancelBlockedPayload, @@ -65,7 +46,6 @@ import { agentRunPromptMetadataFromText, messageAuthorityTextValue } from "./code-agent-agentrun-prompt.ts"; -import { agentRunResultSyncDeferred, buildAgentRunProjectionEventsFetchPlan, buildWorkbenchProjectionStateUpdate } from "./workbench-projection-cursor.ts"; import { codeAgentOtelTraceFields, codeAgentOtelTraceContext, emitCodeAgentOtelSpan } from "./otel-trace.ts"; const ADAPTER_ID = "agentrun-v01"; @@ -313,7 +293,7 @@ export function agentRunResultToCodeAgentPayload({ base, result, traceStore, tra status: "completed", label: "agentrun:result:completed", createdAt: terminalEventCreatedAt, - message: "AgentRun result is ready for HWLAB short-connection polling.", + message: "AgentRun result is committed to the Workbench projection event stream.", runId: base.agentRun.runId, commandId: base.agentRun.commandId, terminal: true, @@ -574,6 +554,11 @@ export function appendAgentRunEventsToTrace(traceStore, traceId, events, mapping if (normalized) traceStore.append(traceId, normalized, agentRunTraceMeta({}, {})); } } +function agentRunEventCommandId(event) { + const payload = event?.payload && typeof event.payload === "object" ? event.payload : {}; + if (typeof event?.commandId === "string") return event.commandId; + return typeof payload.commandId === "string" ? payload.commandId : ""; +} export function isForeignAgentRunCommandEvent(event, mapping = {}) { const currentCommandId = typeof mapping.commandId === "string" ? mapping.commandId : ""; diff --git a/internal/cloud/code-agent-trace-store.test.ts b/internal/cloud/code-agent-trace-store.test.ts index 920b75dd..6359f5ce 100644 --- a/internal/cloud/code-agent-trace-store.test.ts +++ b/internal/cloud/code-agent-trace-store.test.ts @@ -45,9 +45,8 @@ test("code agent trace store dedupes repeated AgentRun source events", () => { assert.deepEqual(snapshot.events.map((item) => item.label), ["agentrun:backend:resource-bundle-materialized", "agentrun:assistant:message"]); }); -test("durable code agent trace store projects appended events without counting assistant deltas", async () => { +test("durable code agent trace store persists diagnostics without counting assistant deltas", async () => { const writes = []; - const workbenchWrites = []; const baseStore = createCodeAgentTraceStore({ maxEvents: 20 }); const traceStore = createDurableCodeAgentTraceStore({ traceStore: baseStore, @@ -56,10 +55,6 @@ test("durable code agent trace store projects appended events without counting a writes.push({ params, requestMeta }); return { written: true }; } - }, - workbenchEventWriter(event, requestMeta) { - workbenchWrites.push({ event, requestMeta }); - return { written: true }; } }); const traceId = "trc_trace-store-durable-projection"; @@ -76,10 +71,6 @@ test("durable code agent trace store projects appended events without counting a assert.equal(writes[0].params.event.traceId, traceId); assert.equal(writes[0].params.event.seq, 1); assert.equal(writes[0].requestMeta.agentSessionId, "ses_trace_store_durable"); - assert.equal(workbenchWrites.length, 1); - assert.equal(workbenchWrites[0].event.traceId, traceId); - assert.equal(workbenchWrites[0].requestMeta.sessionId, "ses_trace_store_durable"); - assert.equal(workbenchWrites[0].requestMeta.agentSessionId, "ses_trace_store_durable"); }); test("code agent trace store retains six thousand regular events", () => { diff --git a/internal/cloud/code-agent-trace-store.ts b/internal/cloud/code-agent-trace-store.ts index 59943573..b8c63464 100644 --- a/internal/cloud/code-agent-trace-store.ts +++ b/internal/cloud/code-agent-trace-store.ts @@ -228,12 +228,12 @@ export function createCodeAgentTraceStore(options = {}) { export const defaultCodeAgentTraceStore = createCodeAgentTraceStore(); -export function createDurableCodeAgentTraceStore({ traceStore = defaultCodeAgentTraceStore, traceEventStore = null, workbenchEventWriter = null } = {}) { +export function createDurableCodeAgentTraceStore({ traceStore = defaultCodeAgentTraceStore, traceEventStore = null } = {}) { if (!traceEventStore || typeof traceEventStore.writeAgentTraceEvent !== "function") return traceStore; function append(traceId, event = {}, meta = {}) { const normalized = traceStore.append(traceId, event, meta); - if (normalized?.traceId) persistTraceEvent(traceEventStore, normalized, meta, workbenchEventWriter); + if (normalized?.traceId) persistTraceEvent(traceEventStore, normalized, meta); return normalized; } @@ -247,11 +247,11 @@ export function createDurableCodeAgentTraceStore({ traceStore = defaultCodeAgent }; } -function persistTraceEvent(traceEventStore, event, meta = {}, workbenchEventWriter = null) { - void persistTraceEventAsync(traceEventStore, event, meta, workbenchEventWriter); +function persistTraceEvent(traceEventStore, event, meta = {}) { + void persistTraceEventAsync(traceEventStore, event, meta); } -async function persistTraceEventAsync(traceEventStore, event, meta = {}, workbenchEventWriter = null) { +async function persistTraceEventAsync(traceEventStore, event, meta = {}) { try { const resolvedAgentSessionId = event.agentSessionId ?? event.sessionId ?? meta.agentSessionId ?? meta.sessionId; const resolvedSessionId = event.sessionId ?? meta.sessionId ?? resolvedAgentSessionId; @@ -265,13 +265,6 @@ async function persistTraceEventAsync(traceEventStore, event, meta = {}, workben } catch (error) { console.error(`[workbench-projection] durable trace event write failed: traceId=${event.traceId} sourceEventId=${event.sourceEventId ?? "none"} error=${error?.message ?? error}`); } - if (typeof workbenchEventWriter === "function") { - try { - await workbenchEventWriter(event, projectionContext); - } catch (error) { - console.error(`[workbench-projection] workbench event writer failed: traceId=${event.traceId} error=${error?.message ?? error}`); - } - } } catch (error) { // Trace capture must not fail the user turn, but projection write errors must be visible for diagnostics. console.error(`[workbench-projection] persistTraceEvent sync error: traceId=${event.traceId} error=${error?.message ?? error}`); diff --git a/internal/cloud/health-contract.ts b/internal/cloud/health-contract.ts index d3a0e7b3..d9920550 100644 --- a/internal/cloud/health-contract.ts +++ b/internal/cloud/health-contract.ts @@ -13,10 +13,11 @@ import { export const CLOUD_API_HEALTH_CONTRACT_VERSION = "v3"; -export function buildCloudApiReadiness({ db, codeAgent, runtime } = {}) { +export function buildCloudApiReadiness({ db, codeAgent, runtime, kafkaProjector } = {}) { const dbReady = isDbReady(db); const codeAgentReady = isCodeAgentReady(codeAgent); const runtimeReady = isRuntimeReady(runtime); + const kafkaProjectorReady = !kafkaProjector?.started || kafkaProjector.status === "ready"; const blockers = []; let runtimeReadinessBlocker = null; @@ -32,6 +33,7 @@ export function buildCloudApiReadiness({ db, codeAgent, runtime } = {}) { runtimeReadinessBlocker = runtimeBlocker(runtime); blockers.push(runtimeReadinessBlocker); } + if (!kafkaProjectorReady) blockers.push(kafkaProjectorBlocker(kafkaProjector)); const provider = buildProviderReadiness(codeAgent); const dbDurable = buildRuntimeDurabilityReadiness({ db, @@ -59,6 +61,7 @@ export function buildCloudApiReadiness({ db, codeAgent, runtime } = {}) { db: dbReady ? "ready" : "blocked", codeAgent: codeAgentReady ? "ready" : "blocked", runtime: runtimeReady ? "ready" : dbReady ? "blocked" : "waiting_for_db", + kafkaProjector: kafkaProjectorReady ? (kafkaProjector?.started ? "ready" : "disabled") : "blocked", provider: provider.status, dbDurable: dbDurable.status, sessionRunner: sessionRunner.status, @@ -73,6 +76,24 @@ export function buildCloudApiReadiness({ db, codeAgent, runtime } = {}) { }; } +function kafkaProjectorBlocker(projector) { + return { + code: projector?.errorCode ?? "hwlab_kafka_projector_not_ready", + type: "runtime_blocker", + scope: "agentrun-kafka-projector", + status: "open", + sourceIssue: "pikasTech/HWLAB#2464", + summary: projector?.message ?? `AgentRun Kafka projector is ${projector?.status ?? "not ready"}`, + evidence: { + status: projector?.status ?? "unknown", + groupId: projector?.groupId ?? null, + agentRunTopic: projector?.agentRunTopic ?? null, + hwlabTopic: projector?.hwlabTopic ?? null, + valuesRedacted: true + } + }; +} + function isDbReady(db) { return Boolean(db?.ready && db?.connected && db?.liveConnected !== false && db?.liveDbEvidence); } diff --git a/internal/cloud/kafka-event-bridge.test.ts b/internal/cloud/kafka-event-bridge.test.ts index c8c7a8b3..8b4aa3e3 100644 --- a/internal/cloud/kafka-event-bridge.test.ts +++ b/internal/cloud/kafka-event-bridge.test.ts @@ -4,7 +4,47 @@ import assert from "node:assert/strict"; import { test } from "bun:test"; -import { projectAgentRunKafkaEventToHwlabEvent } from "./kafka-event-bridge.ts"; +import { createCloudApiBunServer } from "./bun-server.ts"; +import { buildCloudApiReadiness } from "./health-contract.ts"; +import { decodeCanonicalAgentRunKafkaMessage, kafkaEventBridgeConfig, liveKafkaOtelSpanAttributes, projectAgentRunKafkaEventToHwlabEvent, publishAgentRunKafkaMessageLive, relayHwlabKafkaOutboxOnce, shouldEmitLiveKafkaOtelSpan, startHwlabKafkaEventBridge } from "./kafka-event-bridge.ts"; + +const PROJECTOR_ENV = Object.freeze({ + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "true", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "true", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true", + HWLAB_KAFKA_ENABLED: "true", + HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED: "true", + HWLAB_KAFKA_BOOTSTRAP_SERVERS: "kafka.test:9092", + HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC: "agentrun.event.v1", + HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1", + HWLAB_KAFKA_CLIENT_ID: "hwlab-projector-test", + HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID: "hwlab-projector-test-v1", + HWLAB_KAFKA_PROJECTOR_GROUP_ID: "hwlab-projector-test-v1", + HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS: "10", + HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS: "60000", + HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE: "1", + HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS: "30000", + HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS: "10000", + HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS: "1000" +}); + +const LIVE_ENV = Object.freeze({ + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false", + HWLAB_KAFKA_ENABLED: "true", + HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED: "true", + HWLAB_KAFKA_BOOTSTRAP_SERVERS: "kafka.test:9092", + HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC: "agentrun.event.v1", + HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1", + HWLAB_KAFKA_CLIENT_ID: "hwlab-live-test", + HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID: "hwlab-live-bridge-fixed", + HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID: "hwlab-live-fanout-fixed" +}); test("projects AgentRun assistant_message Kafka event into HWLAB trace event", () => { const projected = projectAgentRunKafkaEventToHwlabEvent({ @@ -39,7 +79,10 @@ test("projects AgentRun assistant_message Kafka event into HWLAB trace event", ( assert.equal(projected.eventType, "hwlab.trace.event.projected"); assert.equal(projected.source, "hwlab-test"); assert.equal(projected.traceId, "trc_kafka_projection"); + assert.equal(projected.hwlabSessionId, "ses_kafka_projection"); assert.equal(projected.sessionId, "ses_kafka_projection"); + assert.equal(projected.runId, "run_kafka_projection"); + assert.equal(projected.commandId, "cmd_kafka_projection"); assert.equal(projected.context.runId, "run_kafka_projection"); assert.equal(projected.context.commandId, "cmd_kafka_projection"); assert.equal(projected.context.sourceSeq, 7); @@ -48,12 +91,30 @@ test("projects AgentRun assistant_message Kafka event into HWLAB trace event", ( assert.equal(projected.event.label, "agentrun:assistant:message"); assert.equal(projected.event.status, "running"); assert.equal(projected.event.text, "partial answer"); + assert.equal(projected.event.assistantText, "partial answer"); + assert.equal(projected.event.finalResponse, null); assert.equal(projected.event.terminal, false); assert.equal(projected.sourceEvent.topic, "agentrun.event.v1"); assert.equal(projected.sourceEvent.offset, "42"); assert.equal(projected.valuesPrinted, false); }); +test("projects final AgentRun assistant text without sealing before terminal_status", () => { + const projected = projectAgentRunKafkaEventToHwlabEvent({ + schema: "agentrun.event.v1", + eventType: "agentrun.run.event", + run: { runId: "run_final", sessionId: "ses_final" }, + command: { commandId: "cmd_final" }, + event: { seq: 9, type: "assistant_message", payload: { traceId: "trc_final", text: "durable final answer", final: true, replyAuthority: true } } + }, { source: "hwlab-test" }); + + assert.equal(projected.event.assistantText, "durable final answer"); + assert.equal(projected.event.status, "running"); + assert.equal(projected.event.terminal, false); + assert.equal(projected.event.finalResponse.text, "durable final answer"); + assert.equal(projected.event.finalResponse.traceId, "trc_final"); +}); + test("projects AgentRun terminal_status Kafka event into terminal HWLAB event", () => { const projected = projectAgentRunKafkaEventToHwlabEvent({ schema: "agentrun.event.v1", @@ -80,6 +141,202 @@ test("projects AgentRun terminal_status Kafka event into terminal HWLAB event", assert.equal(projected.event.message, "AgentRun completed"); }); +test("live Kafka mode publishes canonical AgentRun input directly and fans one broker event to all SSE subscribers", async () => { + const consumerConfigs = []; + const subscriptions = []; + const runOptions = []; + const runOrder = []; + const produced = []; + const otelSpans = []; + const consumers = [0, 1].map((index) => ({ + async connect() {}, + async subscribe(input) { subscriptions[index] = input; }, + async run(input) { runOptions[index] = input; runOrder.push(index); }, + async stop() {}, + async disconnect() {} + })); + const producer = { + async connect() {}, + async send(input) { produced.push(input); return [{ topicName: input.topic, partition: 4, baseOffset: "91" }]; }, + async disconnect() {} + }; + let consumerIndex = 0; + const kafkaFactory = () => ({ + consumer(config) { + consumerConfigs.push(config); + return consumers[consumerIndex++]; + }, + producer() { return producer; } + }); + const forbiddenRuntimeStore = new Proxy({}, { + get() { throw new Error("live mode must not touch the runtime store"); } + }); + const bridge = startHwlabKafkaEventBridge({ + env: LIVE_ENV, + runtimeStore: forbiddenRuntimeStore, + kafkaFactory, + logger: null, + otelSpanEmitter(name, traceId, env, options) { otelSpans.push({ name, traceId, env, options }); } + }); + await bridge.ready; + + assert.deepEqual(consumerConfigs.map((item) => item.groupId), ["hwlab-live-bridge-fixed", "hwlab-live-fanout-fixed"]); + assert.deepEqual(runOrder, [1, 0], "fanout group must be ready before direct publish can create a new HWLAB event"); + assert.deepEqual(subscriptions, [ + { topic: "agentrun.event.v1", fromBeginning: false }, + { topic: "hwlab.event.v1", fromBeginning: false } + ]); + const message = canonicalMessage({ offset: "70" }); + await runOptions[0].eachMessage({ topic: "agentrun.event.v1", partition: 0, message }); + assert.equal(produced.length, 1); + assert.equal(produced[0].topic, "hwlab.event.v1"); + const envelope = JSON.parse(produced[0].messages[0].value); + assert.equal(envelope.schema, "hwlab.event.v1"); + assert.equal(envelope.traceId, "trc_projector_test"); + assert.equal(envelope.hwlabSessionId, "ses_projector_test"); + assert.equal(envelope.runId, "run_projector_test"); + assert.equal(envelope.commandId, "cmd_projector_test"); + assert.equal(envelope.event.assistantText, "projected answer"); + assert.equal(otelSpans[0].name, "hwlab.kafka.live.direct_publish"); + assert.equal(otelSpans[0].traceId, "trc_projector_test"); + assert.deepEqual(otelSpans[0].options.attributes, { + businessTraceId: "trc_projector_test", + hwlabSessionId: "ses_projector_test", + runId: "run_projector_test", + commandId: "cmd_projector_test", + topic: "hwlab.event.v1", + partition: 4, + offset: "91", + sourceTopic: "agentrun.event.v1", + sourcePartition: 0, + sourceOffset: "70", + eventType: "assistant_message", + terminal: false, + valuesRedacted: true + }); + + const first = []; + const second = []; + const firstTransports = []; + bridge.subscribeLiveHwlabEvents((value, transport) => { first.push(value); firstTransports.push(transport); }); + bridge.subscribeLiveHwlabEvents((value) => second.push(value)); + await runOptions[1].eachBatch({ + batch: { topic: "hwlab.event.v1", partition: 0, highWatermark: "1", messages: [{ offset: "0", value: Buffer.from(JSON.stringify(envelope)) }] }, + resolveOffset() {}, + async heartbeat() {}, + isRunning: () => true, + isStale: () => false + }); + assert.equal(bridge.liveSubscriberCount(), 2); + assert.deepEqual(first, [envelope]); + assert.deepEqual(second, [envelope]); + assert.deepEqual(firstTransports, [{ topic: "hwlab.event.v1", partition: 0, offset: "0" }]); + assert.equal(otelSpans[1].name, "hwlab.kafka.live.fanout_receive"); + assert.equal(otelSpans[1].options.attributes.topic, "hwlab.event.v1"); + assert.equal(otelSpans[1].options.attributes.partition, 0); + assert.equal(otelSpans[1].options.attributes.offset, "0"); + assert.equal(consumerConfigs.length, 2, "browser subscribers must not allocate Kafka consumer groups"); + assert.equal((await bridge.status()).consumerLag.total, 0); + await bridge.stop(); +}); + +test("live Kafka publisher has no PG inbox, facts, checkpoint, or outbox dependency", async () => { + const sent = []; + const runtimeStore = new Proxy({}, { get() { throw new Error("unexpected DB access"); } }); + void runtimeStore; + const result = await publishAgentRunKafkaMessageLive({ topic: "agentrun.event.v1", partition: 3, message: canonicalMessage({ offset: "71" }) }, { + producer: { async send(input) { sent.push(input); } }, + config: { clientId: "hwlab-live-test", hwlabTopic: "hwlab.event.v1" }, + logger: null + }); + assert.equal(result.published, true); + assert.equal(sent.length, 1); + assert.equal(JSON.parse(sent[0].messages[0].value).sourceEvent.offset, "71"); +}); + +test("live Kafka OTel sampling is deterministic and bounded without payload attributes", () => { + const envelope = (agentRunEventType, sourceSeq, event = {}) => ({ + schema: "hwlab.event.v1", + eventId: `hwlab:evt_otel_${sourceSeq}`, + sourceEventId: `evt_otel_${sourceSeq}`, + traceId: "trc_live_otel", + hwlabSessionId: "ses_live_otel", + runId: "run_live_otel", + commandId: "cmd_live_otel", + context: { agentRunEventType, sourceSeq, valuesRedacted: true }, + event: { type: "backend", eventType: "backend", terminal: false, message: "must-not-leak", payload: { password: "must-not-leak" }, ...event } + }); + + assert.equal(shouldEmitLiveKafkaOtelSpan(envelope("assistant_message", 1, { type: "assistant" })), true); + assert.equal(shouldEmitLiveKafkaOtelSpan(envelope("tool_call", 2, { type: "tool" })), true); + assert.equal(shouldEmitLiveKafkaOtelSpan(envelope("terminal_status", 3, { type: "result", terminal: true })), true); + const decisions = Array.from({ length: 64 }, (_, index) => shouldEmitLiveKafkaOtelSpan(envelope("command_output", index + 1, { type: "output" }))); + assert.deepEqual(decisions, Array.from({ length: 64 }, (_, index) => (index + 1) % 32 === 0)); + assert.deepEqual(decisions, Array.from({ length: 64 }, (_, index) => shouldEmitLiveKafkaOtelSpan(envelope("command_output", index + 1, { type: "output" })))); + + const attributes = liveKafkaOtelSpanAttributes(envelope("assistant_message", 1, { type: "assistant" }), { + topic: "hwlab.event.v1", + partition: 2, + offset: "14" + }); + assert.deepEqual(attributes, { + businessTraceId: "trc_live_otel", + hwlabSessionId: "ses_live_otel", + runId: "run_live_otel", + commandId: "cmd_live_otel", + topic: "hwlab.event.v1", + partition: 2, + offset: "14", + sourceTopic: null, + sourcePartition: null, + sourceOffset: null, + eventType: "assistant_message", + terminal: false, + valuesRedacted: true + }); + assert.equal(JSON.stringify(attributes).includes("must-not-leak"), false); +}); + +test("composable Kafka capabilities reject consumer group collisions", () => { + assert.throws( + () => kafkaEventBridgeConfig({ + ...LIVE_ENV, + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "true", + HWLAB_KAFKA_PROJECTOR_GROUP_ID: LIVE_ENV.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID, + HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS: "1000" + }), + (error) => error?.code === "hwlab_kafka_consumer_groups_not_distinct" + ); +}); + +test("projection realtime subscribes to projection commits without enabling Kafka ingest or relay", async () => { + let subscribeCount = 0; + let unsubscribeCount = 0; + const bridge = startHwlabKafkaEventBridge({ + env: { + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true" + }, + runtimeStore: { + async assertWorkbenchTransactionalRealtimeReady() { return { ready: true }; }, + async subscribeWorkbenchProjectionCommits() { + subscribeCount += 1; + return () => { unsubscribeCount += 1; }; + } + }, + kafkaFactory() { throw new Error("projection realtime must not create a Kafka client"); }, + logger: null + }); + + await bridge.ready; + assert.equal(subscribeCount, 1); + await bridge.stop(); + assert.equal(unsubscribeCount, 1); +}); + test("projects AgentRun run-created envelope even when no source event is present", () => { const projected = projectAgentRunKafkaEventToHwlabEvent({ schema: "agentrun.event.v1", @@ -99,3 +356,308 @@ test("projects AgentRun run-created envelope even when no source event is presen assert.equal(projected.event.label, "agentrun:event:agentrun.run.created"); assert.equal(projected.event.status, "running"); }); + +test("accepts canonical runner-job-created payload with redacted tool credential references", () => { + const message = canonicalMessage({ offset: "60" }); + const envelope = JSON.parse(message.value.toString("utf8")); + envelope.event.type = "backend_status"; + envelope.event.payload = { + traceId: envelope.traceId, + hwlabSessionId: envelope.hwlabSessionId, + commandId: envelope.commandId, + phase: "runner-job-created", + toolCredentials: [{ + tool: "unidesk-ssh", + purpose: "ssh-passthrough", + name: "agentrun-v02-tool-unidesk-ssh", + namespace: "agentrun-v02", + keys: ["UNIDESK_SSH_CLIENT_TOKEN"], + projection: { kind: "env", envName: "UNIDESK_SSH_CLIENT_TOKEN" }, + valuesPrinted: false + }], + valuesPrinted: false + }; + message.value = Buffer.from(JSON.stringify(envelope)); + + const decoded = decodeCanonicalAgentRunKafkaMessage({ topic: "agentrun.event.v1", partition: 0, message }); + assert.equal(decoded.ok, true); + const projected = projectAgentRunKafkaEventToHwlabEvent(decoded.event, { source: "hwlab-test" }); + assert.equal(projected.event.label, "agentrun:backend:runner-job-created"); +}); + +test("Kafka projector commits next offset only after durable projection", async () => { + const calls = []; + let commitInput = null; + const harness = kafkaProjectorHarness({ + async commitAgentRunKafkaProjection(input) { + commitInput = input; + calls.push("db"); + return { projectedSeq: 1 }; + } + }, calls); + const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null }); + await bridge.ready; + await harness.runBatch(canonicalMessage({ offset: "41" })); + + assert.ok(calls.indexOf("heartbeat") < calls.indexOf("db")); + assert.ok(calls.indexOf("db") < calls.indexOf("resolve:41")); + assert.equal(commitInput.partitionKey, "ses_projector_test"); + assert.deepEqual(calls.slice(-3), ["resolve:41", "commit:42", "heartbeat"]); + await bridge.stop(); +}); + +test("Kafka projector leaves offset uncommitted when durable projection throws", async () => { + const calls = []; + const expected = Object.assign(new Error("database unavailable"), { code: "db_unavailable" }); + const harness = kafkaProjectorHarness({ + async commitAgentRunKafkaProjection() { + calls.push("db"); + throw expected; + } + }, calls); + const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null }); + await bridge.ready; + + await assert.rejects(harness.runBatch(canonicalMessage({ offset: "42" })), expected); + assert.equal(calls.some((entry) => entry.startsWith("commit:")), false); + assert.equal(calls.some((entry) => entry.startsWith("resolve:")), false); + await bridge.stop(); +}); + +test("Kafka projector signals committed projection but does not commit a stale generation offset", async () => { + const calls = []; + let signal = null; + let harness; + harness = kafkaProjectorHarness({ + async commitAgentRunKafkaProjection() { + calls.push("db"); + harness.setStale(true); + return { projectedSeq: 1 }; + } + }, calls); + const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null }); + bridge.subscribeProjectionCommits((value) => { signal = value; }); + await bridge.ready; + + await harness.runBatch(canonicalMessage({ offset: "43" })); + + assert.equal(calls.includes("resolve:43"), false); + assert.equal(calls.includes("commit:44"), false); + assert.equal(signal.sessionId, "ses_projector_test"); + assert.equal(signal.traceId, "trc_projector_test"); + await bridge.stop(); +}); + +test("Kafka projector commits a suppressed post-seal offset without publishing a projection signal", async () => { + const calls = []; + let signal = null; + const harness = kafkaProjectorHarness({ + async commitAgentRunKafkaProjection() { + calls.push("db"); + return { projectedSeq: 9, projectionWritten: false, suppressedAfterSeal: true }; + } + }, calls); + const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null }); + bridge.subscribeProjectionCommits((value) => { signal = value; }); + await bridge.ready; + + await harness.runBatch(canonicalMessage({ offset: "44" })); + + assert.equal(signal, null); + assert.deepEqual(calls.slice(-3), ["resolve:44", "commit:45", "heartbeat"]); + await bridge.stop(); +}); + +test("Kafka projector persists malformed and non-HWLAB events before committing offsets", async () => { + const calls = []; + const harness = kafkaProjectorHarness({}, calls); + const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null }); + await bridge.ready; + await harness.runBatch({ offset: "50", value: Buffer.from("{}") }); + await harness.runBatch(canonicalMessage({ offset: "51", traceId: null, hwlabSessionId: null })); + + assert.deepEqual(calls.filter((entry) => ["dlq", "ignored", "commit:51", "commit:52"].includes(entry)), ["dlq", "commit:51", "ignored", "commit:52"]); + await bridge.stop(); +}); + +test("Kafka projector exposes startup failure as rejected readiness and blocked status", async () => { + const expected = Object.assign(new Error("broker unavailable"), { code: "kafka_unavailable" }); + const harness = kafkaProjectorHarness({}, []); + harness.kafkaFactory = () => ({ + consumer: () => harness.consumer, + producer: () => ({ ...harness.producer, connect: async () => { throw expected; } }) + }); + const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null }); + + await assert.rejects(bridge.ready, expected); + assert.equal((await bridge.status()).status, "blocked"); + assert.equal((await bridge.status()).errorCode, "kafka_unavailable"); + await bridge.stop(); +}); + +test("cloud API Bun entry does not listen when Kafka projector startup fails", async () => { + const expected = Object.assign(new Error("broker unavailable before listen"), { code: "kafka_startup_blocked" }); + const harness = kafkaProjectorHarness({}, []); + let bunServeCalls = 0; + const kafkaFactory = () => ({ + consumer: () => harness.consumer, + producer: () => ({ ...harness.producer, connect: async () => { throw expected; } }) + }); + + await assert.rejects(createCloudApiBunServer({ + env: { ...PROJECTOR_ENV }, + runtimeStore: harness.runtimeStore, + kafkaFactory, + accessController: { async ensureBootstrap() {}, async authenticate() { return { ok: false }; } }, + bunServe() { bunServeCalls += 1; throw new Error("Bun.serve must not run"); }, + installSignalHandlers: false, + logger: { log() {}, info() {}, warn() {}, error() {} } + }), expected); + + assert.equal(bunServeCalls, 0); +}); + +test("cloud readiness is blocked while the Kafka projector is blocked", () => { + const readiness = buildCloudApiReadiness({ + db: {}, + codeAgent: {}, + runtime: {}, + kafkaProjector: { started: true, status: "blocked", errorCode: "kafka_startup_blocked", message: "broker unavailable" } + }); + + assert.equal(readiness.components.kafkaProjector, "blocked"); + assert.equal(readiness.blockerCodes.includes("kafka_startup_blocked"), true); + assert.equal(readiness.ready, false); +}); + +test("HWLAB Kafka relay claims just in time and forwards fencing token", async () => { + const calls = []; + const pending = [ + { outboxSeq: 1, eventId: "evt-relay-1", topic: "hwlab.event.v1", partitionKey: "trc-relay", payload: { eventId: "evt-relay-1" }, headers: {}, attemptCount: 2, leaseOwner: "relay-owner", leaseExpiresAt: "2026-07-10T10:02:30.000Z" }, + { outboxSeq: 2, eventId: "evt-relay-2", topic: "hwlab.event.v1", partitionKey: "trc-relay", payload: { eventId: "evt-relay-2" }, headers: {}, attemptCount: 1, leaseOwner: "relay-owner", leaseExpiresAt: "2026-07-10T10:02:30.000Z" } + ]; + const runtimeStore = { + async claimHwlabKafkaOutbox(params) { calls.push(`claim:${params.limit}`); return pending.length ? [pending.shift()] : []; }, + async completeHwlabKafkaOutbox(item) { calls.push(`complete:${item.outboxSeq}:${item.attemptCount}:${item.leaseOwner}`); } + }; + const producer = { async send(record) { calls.push(`send:${record.messages[0].value.includes("evt-relay-1") ? 1 : 2}`); } }; + const config = { relay: { batchSize: 2, leaseMs: 30000, retryBackoffMs: 1000 } }; + + const result = await relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner: "relay-owner", logger: null }); + + assert.equal(result.deliveredCount, 2); + assert.deepEqual(calls, ["claim:1", "send:1", "complete:1:2:relay-owner", "claim:1", "send:2", "complete:2:1:relay-owner"]); +}); + +test("HWLAB Kafka relay retries a failed send and clears backlog on the next cycle", async () => { + const calls = []; + let pending = true; + let attemptCount = 0; + const runtimeStore = { + async claimHwlabKafkaOutbox() { + if (!pending) return []; + attemptCount += 1; + return [{ outboxSeq: 7, eventId: "evt-relay-retry", topic: "hwlab.event.v1", partitionKey: "trc-relay-retry", payload: { eventId: "evt-relay-retry" }, headers: {}, attemptCount, leaseOwner: "relay-retry-owner", leaseExpiresAt: `2026-07-10T10:0${attemptCount}:30.000Z` }]; + }, + async completeHwlabKafkaOutbox(item) { + calls.push(`complete:${item.attemptCount}`); + pending = false; + }, + async retryHwlabKafkaOutbox(item, error, retryAt) { + calls.push(`retry:${item.attemptCount}:${error.message}:${retryAt}`); + }, + async hwlabKafkaProjectorStatus() { + return { backlogCount: pending ? 1 : 0, retryCount: pending && attemptCount > 0 ? 1 : 0, failedInboxCount: 0, dlqCount: 0 }; + } + }; + let sendCount = 0; + const producer = { + async send() { + sendCount += 1; + calls.push(`send:${sendCount}`); + if (sendCount === 1) throw new Error("broker unavailable"); + } + }; + const config = { relay: { batchSize: 1, leaseMs: 30000, sendTimeoutMs: 10000, retryBackoffMs: 1000 } }; + + const failedCycle = await relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner: "relay-retry-owner", now: () => "2026-07-10T10:00:00.000Z", logger: null }); + assert.deepEqual(failedCycle, { claimedCount: 1, deliveredCount: 0, retryCount: 1, valuesPrinted: false }); + assert.deepEqual(await runtimeStore.hwlabKafkaProjectorStatus(), { backlogCount: 1, retryCount: 1, failedInboxCount: 0, dlqCount: 0 }); + + const recoveredCycle = await relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner: "relay-retry-owner", now: () => "2026-07-10T10:00:02.000Z", logger: null }); + assert.deepEqual(recoveredCycle, { claimedCount: 1, deliveredCount: 1, retryCount: 0, valuesPrinted: false }); + assert.deepEqual(await runtimeStore.hwlabKafkaProjectorStatus(), { backlogCount: 0, retryCount: 0, failedInboxCount: 0, dlqCount: 0 }); + assert.deepEqual(calls, [ + "send:1", + "retry:1:broker unavailable:2026-07-10T10:00:01.000Z", + "send:2", + "complete:2" + ]); +}); + +function kafkaProjectorHarness(overrides = {}, calls = []) { + let runOptions = null; + let stale = false; + const consumer = { + async connect() {}, + async subscribe() {}, + async run(options) { runOptions = options; }, + async commitOffsets(offsets) { calls.push(`commit:${offsets[0].offset}`); }, + async stop() {}, + async disconnect() {} + }; + const producer = { async connect() {}, async send() {}, async disconnect() {} }; + const runtimeStore = { + async assertWorkbenchTransactionalRealtimeReady() { return { ready: true }; }, + async subscribeWorkbenchProjectionCommits() { return () => {}; }, + async commitAgentRunKafkaProjection() { return { projectedSeq: 1 }; }, + async recordFailedAgentRunKafkaMessage() { calls.push("dlq"); }, + async recordIgnoredAgentRunKafkaMessage() { calls.push("ignored"); return { ignored: true }; }, + async claimHwlabKafkaOutbox() { return []; }, + async completeHwlabKafkaOutbox() {}, + async retryHwlabKafkaOutbox() {}, + async hwlabKafkaProjectorStatus() { return { backlogCount: 0 }; }, + ...overrides + }; + return { + consumer, + producer, + runtimeStore, + kafkaFactory: () => ({ consumer: () => consumer, producer: () => producer }), + setStale(value) { stale = value === true; }, + async runBatch(message) { + assert.ok(runOptions?.eachBatch); + await runOptions.eachBatch({ + batch: { topic: "agentrun.event.v1", partition: 2, messages: [message] }, + resolveOffset(offset) { calls.push(`resolve:${offset}`); }, + async heartbeat() { calls.push("heartbeat"); }, + isRunning: () => true, + isStale: () => stale + }); + } + }; +} + +function canonicalMessage({ offset, traceId = "trc_projector_test", hwlabSessionId = "ses_projector_test" }) { + const eventId = `evt_projector_${offset}`; + const runId = "run_projector_test"; + return { + offset, + key: Buffer.from(runId), + value: Buffer.from(JSON.stringify({ + schema: "agentrun.event.v1", + eventType: "agentrun.event.committed", + source: "agentrun-v02", + eventId, + sourceSeq: Number(offset) + 1, + traceId, + hwlabSessionId, + runId, + commandId: "cmd_projector_test", + run: { runId, status: "running", valuesPrinted: false }, + command: { commandId: "cmd_projector_test", runId, seq: 1, type: "prompt", state: "running", valuesPrinted: false }, + event: { id: eventId, runId, seq: Number(offset) + 1, type: "assistant_message", payload: { traceId, hwlabSessionId, commandId: "cmd_projector_test", text: "projected answer" }, createdAt: "2026-07-10T10:00:00.000Z" }, + valuesPrinted: false + })) + }; +} diff --git a/internal/cloud/kafka-event-bridge.ts b/internal/cloud/kafka-event-bridge.ts index 4e2d7069..f61412ca 100644 --- a/internal/cloud/kafka-event-bridge.ts +++ b/internal/cloud/kafka-event-bridge.ts @@ -4,64 +4,191 @@ import { createHash, randomUUID } from "node:crypto"; import { Kafka, logLevel } from "kafkajs"; +import { emitCodeAgentOtelSpan } from "./otel-trace.ts"; +import { buildWorkbenchProjectionEventFacts } from "./workbench-projection-writer.ts"; +import { + WORKBENCH_REALTIME_CAPABILITY_ENVS, + workbenchRealtimeCapabilities +} from "./workbench-realtime-capabilities.ts"; const TRUE_VALUES = new Set(["1", "true", "yes", "on"]); const DEFAULT_AGENTRUN_EVENT_TOPIC = "agentrun.event.v1"; const DEFAULT_HWLAB_EVENT_TOPIC = "hwlab.event.v1"; const DEFAULT_STDIO_TOPIC = "codex-stdio.raw.v1"; const DEFAULT_CLIENT_ID = "hwlab-v03-cloud-api"; -const DEFAULT_GROUP_ID = "hwlab-v03-agentrun-event-bridge"; const DEFAULT_QUERY_TIMEOUT_MS = 5000; const DEFAULT_QUERY_LIMIT = 50; +const LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS = 32; +const REQUIRED_KAFKA_ENV = Object.freeze([ + "HWLAB_KAFKA_BOOTSTRAP_SERVERS", + "HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC", + "HWLAB_KAFKA_EVENT_TOPIC", + "HWLAB_KAFKA_CLIENT_ID" +]); export function kafkaEventBridgeConfig(env = process.env) { - if (!truthy(env.HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED ?? env.HWLAB_KAFKA_AGENTRUN_CONSUME_ENABLED ?? env.HWLAB_KAFKA_ENABLED)) return null; + const featureEnvPresent = Object.values(WORKBENCH_REALTIME_CAPABILITY_ENVS).some((name) => stringValue(env[name])); + const legacyKafkaEnabled = truthy(env.HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED ?? env.HWLAB_KAFKA_AGENTRUN_CONSUME_ENABLED ?? env.HWLAB_KAFKA_ENABLED); + if (!featureEnvPresent && !legacyKafkaEnabled) return null; + const capabilities = workbenchRealtimeCapabilities(env); + const kafkaEnabled = capabilities.directPublish || capabilities.liveKafkaSse || capabilities.transactionalProjector || capabilities.projectionOutboxRelay; + const transactionalEnabled = capabilities.transactionalProjector || capabilities.projectionOutboxRelay || capabilities.projectionRealtime; + if (!kafkaEnabled && !transactionalEnabled) return null; + const required = [...REQUIRED_KAFKA_ENV]; + if (!kafkaEnabled) required.length = 0; + if (capabilities.directPublish) required.push("HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID"); + if (capabilities.liveKafkaSse) required.push("HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID"); + if (capabilities.transactionalProjector) required.push("HWLAB_KAFKA_PROJECTOR_GROUP_ID", "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS"); + if (capabilities.projectionOutboxRelay) required.push( + "HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS", + "HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE", + "HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS", + "HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS", + "HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS" + ); + const missing = required.filter((name) => !stringValue(env[name])); + if (missing.length > 0) { + const error = new Error(`HWLAB Kafka capability configuration is incomplete: ${missing.join(", ")}`); + error.code = "hwlab_kafka_realtime_config_incomplete"; + error.missing = missing; + throw error; + } const brokers = csv(env.HWLAB_KAFKA_BOOTSTRAP_SERVERS); - if (brokers.length === 0) return null; - const agentRunTopic = stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC) || DEFAULT_AGENTRUN_EVENT_TOPIC; - const hwlabTopic = stringValue(env.HWLAB_KAFKA_EVENT_TOPIC) || DEFAULT_HWLAB_EVENT_TOPIC; - const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID) || DEFAULT_CLIENT_ID; - const groupId = stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID ?? env.HWLAB_KAFKA_AGENTRUN_CONSUMER_GROUP_ID) || DEFAULT_GROUP_ID; - return { brokers, agentRunTopic, hwlabTopic, clientId, groupId }; + const agentRunTopic = stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC); + const hwlabTopic = stringValue(env.HWLAB_KAFKA_EVENT_TOPIC); + const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID); + const relay = capabilities.projectionOutboxRelay ? { + intervalMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS, "HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS"), + batchSize: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE, "HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE"), + leaseMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS, "HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS"), + sendTimeoutMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS, "HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS"), + retryBackoffMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS, "HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS") + } : null; + if (relay && relay.sendTimeoutMs + 5000 >= relay.leaseMs) { + const error = new Error("HWLAB Kafka relay lease must exceed producer send timeout by at least 5000ms."); + error.code = "hwlab_kafka_relay_lease_invalid"; + throw error; + } + const enabledConsumerGroups = [ + capabilities.directPublish ? stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID) : null, + capabilities.transactionalProjector ? stringValue(env.HWLAB_KAFKA_PROJECTOR_GROUP_ID) : null, + capabilities.liveKafkaSse ? stringValue(env.HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID) : null + ].filter(Boolean); + if (new Set(enabledConsumerGroups).size !== enabledConsumerGroups.length) { + const error = new Error("HWLAB Kafka direct publisher, projector, and live SSE consumer groups must be distinct when enabled together."); + error.code = "hwlab_kafka_consumer_groups_not_distinct"; + error.groupIds = enabledConsumerGroups; + throw error; + } + return { + capabilities, + brokers, + agentRunTopic, + hwlabTopic, + clientId, + directPublishGroupId: stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID), + projectorGroupId: stringValue(env.HWLAB_KAFKA_PROJECTOR_GROUP_ID), + hwlabEventGroupId: stringValue(env.HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID), + projectorHeartbeatIntervalMs: capabilities.transactionalProjector ? strictPositiveInteger(env.HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS, "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS") : null, + relay + }; } -export function startHwlabKafkaEventBridge({ env = process.env, logger = console, kafkaFactory = defaultKafkaFactory } = {}) { +export function startHwlabKafkaEventBridge({ env = process.env, logger = console, kafkaFactory = defaultKafkaFactory, runtimeStore = null, now = () => new Date().toISOString(), otelSpanEmitter = emitCodeAgentOtelSpan } = {}) { const config = kafkaEventBridgeConfig(env); - if (!config) return { started: false, reason: "disabled_or_unconfigured", stop() {}, valuesPrinted: false }; + if (!config) return { started: false, reason: "disabled_or_unconfigured", stop() {}, subscribeProjectionCommits() { return () => {}; }, valuesPrinted: false }; + const components = []; + if (config.capabilities.directPublish || config.capabilities.liveKafkaSse) { + components.push(startLiveHwlabKafkaEventBridge({ config, env, logger, kafkaFactory, otelSpanEmitter })); + } + if (config.capabilities.transactionalProjector || config.capabilities.projectionOutboxRelay || config.capabilities.projectionRealtime) { + components.push(startTransactionalHwlabKafkaEventBridge({ config, logger, kafkaFactory, runtimeStore, now })); + } + return components.length === 1 ? components[0] : combineKafkaEventBridgeComponents(config, components); +} + +function startTransactionalHwlabKafkaEventBridge({ config, logger = console, kafkaFactory = defaultKafkaFactory, runtimeStore = null, now = () => new Date().toISOString() } = {}) { + requireKafkaProjectorStore(runtimeStore, config.capabilities); let stopped = false; let consumer = null; let producer = null; + let relayTimer = null; + let relayRunning = false; + let stopProjectionNotifications = null; + const projectionCommitListeners = new Set(); + const publishProjectionCommit = (signal = {}) => { + for (const listener of [...projectionCommitListeners]) { + try { listener({ sessionId: signal.sessionId ?? null, traceId: signal.traceId ?? null, outboxSeq: signal.outboxSeq ?? null, recovery: signal.recovery === true, valuesRedacted: true }); } catch {} + } + }; + const relayOwner = `${config.clientId}:outbox-relay:${randomUUID()}`; + let startupError = null; + let startupReady = false; const ready = (async () => { - const kafka = kafkaFactory(config); - consumer = kafka.consumer({ groupId: config.groupId, allowAutoTopicCreation: false }); - producer = kafka.producer({ allowAutoTopicCreation: false }); - await producer.connect(); - await consumer.connect(); - await consumer.subscribe({ topic: config.agentRunTopic, fromBeginning: false }); - await consumer.run({ - eachMessage: async ({ topic, partition, message }) => { - if (stopped) return; - const projected = projectAgentRunKafkaMessageToHwlabEvent({ topic, partition, message }, { source: config.clientId }); - if (!projected) return; - await producer.send({ - topic: config.hwlabTopic, - messages: [{ - key: kafkaMessageKey(projected), - value: JSON.stringify(projected), - headers: kafkaHeaders(projected) - }] - }); + await runtimeStore.assertWorkbenchTransactionalRealtimeReady(); + const kafka = config.capabilities.transactionalProjector || config.capabilities.projectionOutboxRelay ? kafkaFactory(config) : null; + if (config.capabilities.transactionalProjector) consumer = kafka.consumer({ groupId: config.projectorGroupId, allowAutoTopicCreation: false }); + if (config.capabilities.projectionOutboxRelay) producer = kafka.producer({ allowAutoTopicCreation: false }); + if (producer) await producer.connect(); + if (consumer) await consumer.connect(); + if (config.capabilities.projectionRealtime && typeof runtimeStore.subscribeWorkbenchProjectionCommits === "function") { + stopProjectionNotifications = await runtimeStore.subscribeWorkbenchProjectionCommits(publishProjectionCommit); + } + if (consumer) await consumer.subscribe({ topic: config.agentRunTopic, fromBeginning: false }); + if (consumer) await consumer.run({ + autoCommit: false, + eachBatchAutoResolve: false, + eachBatch: async ({ batch, resolveOffset, heartbeat, isRunning, isStale }) => { + for (const message of batch.messages) { + if (stopped || !isRunning() || isStale()) break; + await heartbeat(); + const heartbeatLoop = startKafkaHeartbeatLoop(heartbeat, config.projectorHeartbeatIntervalMs); + let projection; + try { + projection = await projectAgentRunKafkaMessage({ topic: batch.topic, partition: batch.partition, message }, { runtimeStore, config, logger }); + } finally { + await heartbeatLoop.stop(); + } + if (projection?.projected) publishProjectionCommit(projection); + if (heartbeatLoop.error) throw heartbeatLoop.error; + if (stopped || !isRunning() || isStale()) { + logWarn(logger, "hwlab-kafka-offset-commit-skipped", { topic: batch.topic, partition: batch.partition, offset: message.offset, reason: isStale() ? "stale_generation" : "consumer_stopped", valuesPrinted: false }); + break; + } + resolveOffset(message.offset); + await consumer.commitOffsets([{ topic: batch.topic, partition: batch.partition, offset: nextKafkaOffset(message.offset) }]); + await heartbeat(); + } } }); + const runRelay = async () => { + if (!config.capabilities.projectionOutboxRelay) return; + if (stopped || relayRunning) return; + relayRunning = true; + try { + await relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner: relayOwner, now, logger }); + } finally { + relayRunning = false; + } + }; + if (config.capabilities.projectionOutboxRelay) { + relayTimer = setInterval(() => { void runRelay().catch((error) => logWarn(logger, "hwlab-kafka-outbox-relay-failed", { message: errorMessage(error), valuesPrinted: false })); }, config.relay.intervalMs); + relayTimer.unref?.(); + await runRelay(); + } + startupReady = true; logInfo(logger, "hwlab-kafka-event-bridge-started", { agentRunTopic: config.agentRunTopic, hwlabTopic: config.hwlabTopic, clientId: config.clientId, - groupId: config.groupId, + projectorGroupId: config.projectorGroupId, + capabilities: config.capabilities, valuesPrinted: false }); - })().catch((error) => { + })(); + void ready.catch((error) => { + startupError = error; logWarn(logger, "hwlab-kafka-event-bridge-start-failed", { message: errorMessage(error), agentRunTopic: config.agentRunTopic, @@ -76,12 +203,386 @@ export function startHwlabKafkaEventBridge({ env = process.env, logger = console ready, async stop() { stopped = true; + if (relayTimer) clearInterval(relayTimer); + if (typeof stopProjectionNotifications === "function") { + try { await stopProjectionNotifications(); } catch {} + } + if (consumer?.stop) await consumer.stop().catch(() => undefined); await Promise.allSettled([consumer?.disconnect?.(), producer?.disconnect?.()]); }, + async status() { + if (startupError) return { status: "blocked", errorCode: startupError.code ?? "hwlab_kafka_projector_start_failed", message: errorMessage(startupError), valuesRedacted: true }; + if (!startupReady) return { status: "initializing", valuesRedacted: true }; + const transactionalReadiness = typeof runtimeStore.workbenchTransactionalRealtimeReadiness === "function" + ? await runtimeStore.workbenchTransactionalRealtimeReadiness() + : { ready: true, valuesRedacted: true }; + const projectorStatus = config.capabilities.transactionalProjector || config.capabilities.projectionOutboxRelay + ? await runtimeStore.hwlabKafkaProjectorStatus() + : {}; + return { status: transactionalReadiness.ready === false ? "blocked" : "ready", capabilities: config.capabilities, transactionalReadiness, ...projectorStatus, valuesRedacted: true }; + }, + startupStatus() { + if (startupError) return { status: "blocked", errorCode: startupError.code ?? "hwlab_kafka_projector_start_failed", message: errorMessage(startupError), valuesRedacted: true }; + return { status: startupReady ? "ready" : "initializing", valuesRedacted: true }; + }, + subscribeProjectionCommits(listener) { + if (typeof listener !== "function") return () => {}; + projectionCommitListeners.add(listener); + return () => projectionCommitListeners.delete(listener); + }, valuesPrinted: false }; } +function startLiveHwlabKafkaEventBridge({ config, env = process.env, logger = console, kafkaFactory = defaultKafkaFactory, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) { + let stopped = false; + let bridgeConsumer = null; + let fanoutConsumer = null; + let producer = null; + let startupError = null; + let startupReady = false; + const subscribers = new Set(); + const lagByPartition = new Map(); + const publishLiveEnvelope = (envelope, transport) => { + for (const listener of [...subscribers]) { + try { + listener(envelope, transport); + } catch (error) { + logWarn(logger, "hwlab-live-kafka-subscriber-failed", { message: errorMessage(error), valuesPrinted: false }); + } + } + }; + const ready = (async () => { + const kafka = kafkaFactory(config); + if (config.capabilities.directPublish) { + bridgeConsumer = kafka.consumer({ groupId: config.directPublishGroupId, allowAutoTopicCreation: false }); + producer = kafka.producer({ allowAutoTopicCreation: false }); + } + if (config.capabilities.liveKafkaSse) fanoutConsumer = kafka.consumer({ groupId: config.hwlabEventGroupId, allowAutoTopicCreation: false }); + if (producer) await producer.connect(); + if (bridgeConsumer) await bridgeConsumer.connect(); + if (fanoutConsumer) await fanoutConsumer.connect(); + if (bridgeConsumer) await bridgeConsumer.subscribe({ topic: config.agentRunTopic, fromBeginning: false }); + if (fanoutConsumer) await fanoutConsumer.subscribe({ topic: config.hwlabTopic, fromBeginning: false }); + if (fanoutConsumer) await fanoutConsumer.run({ + eachBatchAutoResolve: false, + eachBatch: async ({ batch, resolveOffset, heartbeat, isRunning, isStale }) => { + let lastOffset = null; + for (const message of batch.messages) { + if (stopped || !isRunning() || isStale()) break; + const envelope = parseJson(message?.value ? Buffer.from(message.value).toString("utf8") : ""); + if (!envelope || envelope.schema !== "hwlab.event.v1") { + logWarn(logger, "hwlab-live-kafka-envelope-ignored", { reason: "schema-invalid", valuesPrinted: false }); + } else { + const transport = { topic: batch.topic, partition: batch.partition, offset: message.offset }; + emitLiveKafkaOtelSpan("hwlab.kafka.live.fanout_receive", envelope, transport, { env, otelSpanEmitter }); + publishLiveEnvelope(envelope, transport); + } + resolveOffset(message.offset); + lastOffset = message.offset; + await heartbeat(); + } + if (lastOffset !== null) lagByPartition.set(batch.partition, kafkaPartitionLag(batch.highWatermark, lastOffset)); + } + }); + if (bridgeConsumer) await bridgeConsumer.run({ + eachMessage: async ({ topic, partition, message }) => { + if (stopped) return; + await publishAgentRunKafkaMessageLive({ topic, partition, message }, { producer, config, env, logger, otelSpanEmitter }); + } + }); + startupReady = true; + logInfo(logger, "hwlab-live-kafka-event-bridge-started", { + capabilities: config.capabilities, + agentRunTopic: config.agentRunTopic, + hwlabTopic: config.hwlabTopic, + clientId: config.clientId, + directPublishGroupId: config.directPublishGroupId, + hwlabEventGroupId: config.hwlabEventGroupId, + valuesPrinted: false + }); + })(); + void ready.catch((error) => { + startupError = error; + logWarn(logger, "hwlab-live-kafka-event-bridge-start-failed", { + message: errorMessage(error), + agentRunTopic: config.agentRunTopic, + hwlabTopic: config.hwlabTopic, + valuesPrinted: false + }); + }); + return { + started: true, + ...config, + ready, + async stop() { + stopped = true; + subscribers.clear(); + await Promise.allSettled([bridgeConsumer?.stop?.(), fanoutConsumer?.stop?.()]); + await Promise.allSettled([bridgeConsumer?.disconnect?.(), fanoutConsumer?.disconnect?.(), producer?.disconnect?.()]); + }, + async status() { + if (startupError) return { status: "blocked", capabilities: config.capabilities, errorCode: startupError.code ?? "hwlab_live_kafka_start_failed", message: errorMessage(startupError), valuesRedacted: true }; + return { status: startupReady ? "ready" : "initializing", capabilities: config.capabilities, subscriberCount: subscribers.size, consumerLag: liveKafkaLagSummary(lagByPartition), lossPossible: true, valuesRedacted: true }; + }, + startupStatus() { + if (startupError) return { status: "blocked", capabilities: config.capabilities, errorCode: startupError.code ?? "hwlab_live_kafka_start_failed", message: errorMessage(startupError), valuesRedacted: true }; + return { status: startupReady ? "ready" : "initializing", capabilities: config.capabilities, valuesRedacted: true }; + }, + subscribeProjectionCommits() { return () => {}; }, + subscribeLiveHwlabEvents(listener) { + if (typeof listener !== "function") return () => {}; + subscribers.add(listener); + return () => subscribers.delete(listener); + }, + liveSubscriberCount() { return subscribers.size; }, + valuesPrinted: false + }; +} + +function kafkaPartitionLag(highWatermark, lastOffset) { + try { + const lag = BigInt(String(highWatermark ?? "0")) - (BigInt(String(lastOffset ?? "0")) + 1n); + return Number(lag > 0n ? lag : 0n); + } catch { + return null; + } +} + +function liveKafkaLagSummary(lagByPartition) { + const partitions = [...lagByPartition.entries()].map(([partition, lag]) => ({ partition, lag })).sort((left, right) => left.partition - right.partition); + const known = partitions.map((item) => item.lag).filter((lag) => Number.isFinite(lag)); + return { + known: known.length > 0, + total: known.length > 0 ? known.reduce((sum, lag) => sum + lag, 0) : null, + partitions, + valuesRedacted: true + }; +} + +function combineKafkaEventBridgeComponents(config, components) { + const ready = Promise.all(components.map((component) => component.ready)); + const componentStartup = () => components.map((component) => component.startupStatus?.() ?? { status: "initializing" }); + return { + started: true, + ...config, + ready, + async stop() { await Promise.allSettled(components.map((component) => component.stop())); }, + async status() { + const statuses = await Promise.all(components.map((component) => component.status())); + const blocked = statuses.find((status) => status.status === "blocked"); + const initializing = statuses.some((status) => status.status !== "ready"); + return { + status: blocked ? "blocked" : initializing ? "initializing" : "ready", + capabilities: config.capabilities, + components: statuses, + errorCode: blocked?.errorCode ?? null, + message: blocked?.message ?? null, + valuesRedacted: true + }; + }, + startupStatus() { + const statuses = componentStartup(); + const blocked = statuses.find((status) => status.status === "blocked"); + return { + status: blocked ? "blocked" : statuses.every((status) => status.status === "ready") ? "ready" : "initializing", + capabilities: config.capabilities, + components: statuses, + errorCode: blocked?.errorCode ?? null, + message: blocked?.message ?? null, + valuesRedacted: true + }; + }, + subscribeProjectionCommits(listener) { + const stops = components.map((component) => component.subscribeProjectionCommits?.(listener)).filter((stop) => typeof stop === "function"); + return () => stops.forEach((stop) => stop()); + }, + subscribeLiveHwlabEvents(listener) { + const owner = components.find((component) => typeof component.subscribeLiveHwlabEvents === "function"); + return owner?.subscribeLiveHwlabEvents(listener) ?? (() => {}); + }, + liveSubscriberCount() { + const owner = components.find((component) => typeof component.liveSubscriberCount === "function"); + return owner?.liveSubscriberCount() ?? 0; + }, + valuesPrinted: false + }; +} + +export async function publishAgentRunKafkaMessageLive(kafkaMessage = {}, { producer, config, env = process.env, logger = console, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) { + const transport = kafkaTransport(kafkaMessage); + const decoded = decodeCanonicalAgentRunKafkaMessage(kafkaMessage); + if (!decoded.ok) { + logWarn(logger, "hwlab-live-kafka-message-invalid", { topic: transport.sourceTopic, partition: transport.sourcePartition, offset: transport.sourceOffset, errorCode: decoded.error.code, valuesPrinted: false }); + return { handled: true, invalid: true, published: false, valuesPrinted: false }; + } + if (!stringValue(decoded.event.traceId) || !stringValue(decoded.event.hwlabSessionId)) { + return { handled: true, ignored: true, published: false, valuesPrinted: false }; + } + const projected = projectAgentRunKafkaEventToHwlabEvent(decoded.event, { + source: config.clientId, + sourceTopic: transport.sourceTopic, + sourcePartition: transport.sourcePartition, + sourceOffset: transport.sourceOffset, + sourceKey: transport.sourceKey, + inputSha256: transport.inputSha256 + }); + const publishMetadata = await producer.send({ + topic: config.hwlabTopic, + messages: [{ key: kafkaMessageKey(projected), value: JSON.stringify(projected), headers: kafkaHeaders(projected) }] + }); + const publishedRecord = Array.isArray(publishMetadata) ? publishMetadata[0] : null; + emitLiveKafkaOtelSpan("hwlab.kafka.live.direct_publish", projected, { + topic: config.hwlabTopic, + partition: publishedRecord?.partition, + offset: publishedRecord?.baseOffset ?? publishedRecord?.offset, + sourceTopic: transport.sourceTopic, + sourcePartition: transport.sourcePartition, + sourceOffset: transport.sourceOffset + }, { env, otelSpanEmitter }); + return { handled: true, published: true, envelope: projected, sessionId: projected.sessionId, traceId: projected.traceId, valuesPrinted: false }; +} + +export function shouldEmitLiveKafkaOtelSpan(envelope = {}) { + const event = objectValue(envelope.event); + const context = objectValue(envelope.context); + const eventType = firstText(context.agentRunEventType, event.eventType, event.type, envelope.eventType)?.toLowerCase(); + if (event.terminal === true || eventType === "terminal" || eventType === "terminal_status") return true; + if (["assistant", "assistant_message", "tool", "tool_call"].includes(eventType)) return true; + if (eventType !== "command_output" && event.type !== "output") return true; + const sourceSeq = integerValue(context.sourceSeq ?? event.sourceSeq); + if (Number.isInteger(sourceSeq) && sourceSeq > 0) return sourceSeq % LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS === 0; + const identity = firstText(envelope.sourceEventId, envelope.eventId, envelope.traceId, envelope.hwlabSessionId, envelope.sessionId); + if (!identity) return false; + return createHash("sha256").update(identity).digest().readUInt32BE(0) % LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS === 0; +} + +export function liveKafkaOtelSpanAttributes(envelope = {}, transport = {}) { + const event = objectValue(envelope.event); + const context = objectValue(envelope.context); + const sourceEvent = objectValue(envelope.sourceEvent); + return { + businessTraceId: firstText(envelope.traceId, event.traceId), + hwlabSessionId: firstText(envelope.hwlabSessionId, envelope.sessionId, event.sessionId), + runId: firstText(envelope.runId, context.runId, event.runId), + commandId: firstText(envelope.commandId, context.commandId, event.commandId), + topic: firstText(transport.topic, sourceEvent.topic), + partition: integerValue(transport.partition ?? sourceEvent.partition), + offset: firstText(transport.offset, sourceEvent.offset), + sourceTopic: firstText(transport.sourceTopic), + sourcePartition: integerValue(transport.sourcePartition), + sourceOffset: firstText(transport.sourceOffset), + eventType: firstText(context.agentRunEventType, event.eventType, event.type, envelope.eventType), + terminal: event.terminal === true, + valuesRedacted: true + }; +} + +export function emitLiveKafkaOtelSpan(name, envelope = {}, transport = {}, { env = process.env, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) { + if (typeof otelSpanEmitter !== "function" || !shouldEmitLiveKafkaOtelSpan(envelope)) return { emitted: false, valuesRedacted: true }; + const attributes = liveKafkaOtelSpanAttributes(envelope, transport); + if (!attributes.businessTraceId) return { emitted: false, valuesRedacted: true }; + try { + const pending = otelSpanEmitter(name, attributes.businessTraceId, env, { attributes }); + void Promise.resolve(pending).catch(() => undefined); + } catch { + // OTel is best effort and must never change the live Kafka delivery path. + } + return { emitted: true, valuesRedacted: true }; +} + +export async function projectAgentRunKafkaMessage(kafkaMessage = {}, { runtimeStore, config, logger = console } = {}) { + const transport = kafkaTransport(kafkaMessage); + const decoded = decodeCanonicalAgentRunKafkaMessage(kafkaMessage); + if (!decoded.ok) { + await runtimeStore.recordFailedAgentRunKafkaMessage({ transport, sourceEventId: decoded.sourceEventId, envelope: decoded.envelope, error: decoded.error }); + logWarn(logger, "hwlab-kafka-message-dlq", { topic: transport.sourceTopic, partition: transport.sourcePartition, offset: transport.sourceOffset, errorCode: decoded.error.code, valuesPrinted: false }); + return { handled: true, invalid: true, valuesPrinted: false }; + } + if (!stringValue(decoded.event.traceId) || !stringValue(decoded.event.hwlabSessionId)) { + const ignored = await runtimeStore.recordIgnoredAgentRunKafkaMessage({ transport, envelope: decoded.event, reason: "hwlab-correlation-absent" }); + return { handled: true, ignored: true, duplicate: ignored?.duplicate === true, valuesPrinted: false }; + } + const projected = projectAgentRunKafkaEventToHwlabEvent(decoded.event, { + source: config.clientId, + sourceTopic: transport.sourceTopic, + sourcePartition: transport.sourcePartition, + sourceOffset: transport.sourceOffset, + sourceKey: transport.sourceKey, + inputSha256: transport.inputSha256 + }); + const result = await runtimeStore.commitAgentRunKafkaProjection({ + transport, + canonicalEvent: decoded.event, + projectedEvent: projected.event, + requestMeta: { traceId: projected.traceId, sessionId: projected.sessionId, agentSessionId: projected.sessionId, valuesPrinted: false }, + factsFactory: ({ projectedSeq, previousCheckpoint, projectedAt }) => buildWorkbenchProjectionEventFacts({ event: projected.event, requestMeta: { traceId: projected.traceId, sessionId: projected.sessionId }, previousCheckpoint, projectedSeq, projectedAt }), + hwlabEvent: projected, + hwlabTopic: config.hwlabTopic, + partitionKey: kafkaMessageKey(projected), + headers: kafkaHeaders(projected) + }); + if (result.conflict) logWarn(logger, "hwlab-kafka-message-conflict", { topic: transport.sourceTopic, partition: transport.sourcePartition, offset: transport.sourceOffset, errorCode: result.diagnostic?.code, valuesPrinted: false }); + return { + handled: true, + projected: !result.duplicate && !result.conflict && result.projectionWritten !== false && result.suppressedAfterSeal !== true, + duplicate: result.duplicate === true, + conflict: result.conflict === true, + suppressedAfterSeal: result.suppressedAfterSeal === true, + projectedSeq: result.projectedSeq, + sessionId: projected.sessionId, + traceId: projected.traceId, + valuesPrinted: false + }; +} + +export async function relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner, now = () => new Date().toISOString(), logger = console } = {}) { + let claimedCount = 0; + let deliveredCount = 0; + let retryCount = 0; + for (let index = 0; index < config.relay.batchSize; index += 1) { + const item = (await runtimeStore.claimHwlabKafkaOutbox({ owner, leaseMs: config.relay.leaseMs, limit: 1 }))[0]; + if (!item) break; + claimedCount += 1; + try { + await producer.send({ topic: item.topic, timeout: config.relay.sendTimeoutMs, messages: [{ key: item.partitionKey, value: JSON.stringify(item.payload), headers: item.headers }] }); + await runtimeStore.completeHwlabKafkaOutbox(item); + deliveredCount += 1; + } catch (error) { + const retryAt = new Date(Date.parse(now()) + config.relay.retryBackoffMs).toISOString(); + await runtimeStore.retryHwlabKafkaOutbox(item, error, retryAt); + retryCount += 1; + logWarn(logger, "hwlab-kafka-outbox-publish-retry", { outboxSeq: item.outboxSeq, eventId: item.eventId, retryAt, message: errorMessage(error), valuesPrinted: false }); + break; + } + } + return { claimedCount, deliveredCount, retryCount, valuesPrinted: false }; +} + +export function startKafkaHeartbeatLoop(heartbeat, intervalMs) { + let stopped = false; + let running = null; + let heartbeatError = null; + const pulse = () => { + if (stopped || running || typeof heartbeat !== "function") return; + running = Promise.resolve() + .then(() => heartbeat()) + .catch((error) => { heartbeatError ??= error; }) + .finally(() => { running = null; }); + }; + pulse(); + const timer = setInterval(pulse, intervalMs); + timer.unref?.(); + return { + get error() { return heartbeatError; }, + async stop() { + stopped = true; + clearInterval(timer); + if (running) await running; + } + }; +} + export function projectAgentRunKafkaMessageToHwlabEvent(kafkaMessage = {}, options = {}) { const valueText = kafkaMessage?.message?.value ? Buffer.from(kafkaMessage.message.value).toString("utf8") : ""; const input = parseJson(valueText); @@ -106,7 +607,8 @@ export function projectAgentRunKafkaEventToHwlabEvent(input = {}, options = {}) const context = objectValue(input.context ?? payload.context); const sourceSeq = integerValue(sourceEvent.seq ?? input.seq ?? input.sourceSeq); const traceId = firstText(input.traceId, hwlab.traceId, context.traceId, sourceEvent.traceId, payload.traceId, payload.hwlabTraceId, payload.businessTraceId, payload.metadata?.traceId, command.traceId, run.traceId); - const sessionId = firstText(input.sessionId, hwlab.sessionId, context.sessionId, run.sessionId, payload.sessionId, payload.hwlabSessionId); + const sourceEventId = firstText(input.eventId, sourceEvent.id); + const sessionId = firstText(input.hwlabSessionId, payload.hwlabSessionId, hwlab.sessionId, context.hwlabSessionId, input.sessionId, context.sessionId, run.hwlabSessionId, run.sessionId, payload.sessionId); const runId = firstText(run.runId, sourceEvent.runId, payload.runId, input.runId); const commandId = firstText(command.commandId, sourceEvent.commandId, payload.commandId, input.commandId); const event = mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run, command, traceId, sessionId, sourceSeq, runId, commandId }); @@ -114,10 +616,15 @@ export function projectAgentRunKafkaEventToHwlabEvent(input = {}, options = {}) return { schema: "hwlab.event.v1", eventType: "hwlab.trace.event.projected", + eventId: sourceEventId ? `hwlab:${sourceEventId}` : null, + sourceEventId, source: stringValue(options.source) || DEFAULT_CLIENT_ID, producedAt, traceId, + hwlabSessionId: sessionId, sessionId, + runId, + commandId, context: { conversationId: firstText(input.conversationId, hwlab.conversationId, context.conversationId, run.conversationId, payload.conversationId), threadId: firstText(input.threadId, hwlab.threadId, context.threadId, run.threadId, payload.threadId), @@ -129,7 +636,7 @@ export function projectAgentRunKafkaEventToHwlabEvent(input = {}, options = {}) agentRunEventType: firstText(sourceEvent.type, payload.type), valuesRedacted: true }, - event, + event: { ...event, sourceEventId }, sourceEvent: { schema: firstText(input.schema), topic: stringValue(options.sourceTopic), @@ -147,6 +654,86 @@ export function projectAgentRunKafkaEventToHwlabEvent(input = {}, options = {}) }; } +export function decodeCanonicalAgentRunKafkaMessage(kafkaMessage = {}) { + const valueText = kafkaMessage?.message?.value ? Buffer.from(kafkaMessage.message.value).toString("utf8") : ""; + const envelope = parseJson(valueText); + const sourceEventId = firstText(envelope?.eventId, envelope?.event?.id); + try { + assertCanonicalAgentRunEvent(envelope); + return { ok: true, event: envelope, sourceEventId, inputSha256: sha256(valueText), valuesPrinted: false }; + } catch (error) { + return { ok: false, envelope, sourceEventId, inputSha256: sha256(valueText), error: { code: error?.code ?? "workbench_kafka_message_invalid", message: errorMessage(error), valuesRedacted: true }, valuesPrinted: false }; + } +} + +function assertCanonicalAgentRunEvent(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) throw contractError("workbench_kafka_json_invalid", "Kafka value must be a JSON object."); + assertClosedObject(value, ["schema", "eventType", "source", "eventId", "outboxSeq", "sourceSeq", "committedAt", "traceId", "sessionId", "hwlabSessionId", "runId", "commandId", "run", "command", "event", "valuesPrinted"], "AgentRun envelope"); + if (value.schema !== "agentrun.event.v1" || value.eventType !== "agentrun.event.committed") throw contractError("workbench_kafka_schema_invalid", "Kafka value is not a canonical AgentRun committed event."); + for (const [name, field] of [["eventId", value.eventId], ["runId", value.runId]]) { + if (!stringValue(field)) throw contractError("workbench_kafka_schema_invalid", `Canonical AgentRun ${name} is required.`); + } + for (const [name, field] of [["traceId", value.traceId], ["hwlabSessionId", value.hwlabSessionId]]) { + if (field !== null && field !== undefined && !stringValue(field)) throw contractError("workbench_kafka_schema_invalid", `Canonical AgentRun ${name} must be a string or null.`); + } + if (!Number.isInteger(Number(value.sourceSeq)) || Number(value.sourceSeq) <= 0) throw contractError("workbench_kafka_schema_invalid", "Canonical AgentRun sourceSeq must be a positive integer."); + if (value.valuesPrinted !== false) throw contractError("workbench_kafka_redaction_invalid", "Canonical AgentRun event must declare valuesPrinted=false."); + assertClosedObject(value.run, ["runId", "status", "terminalStatus", "failureKind", "tenantId", "projectId", "providerId", "backendProfile", "sessionId", "hwlabSessionId", "conversationId", "threadId", "valuesPrinted"], "AgentRun run"); + if (value.command !== null && value.command !== undefined) assertClosedObject(value.command, ["commandId", "runId", "seq", "type", "state", "payloadHash", "valuesPrinted"], "AgentRun command"); + assertClosedObject(value.event, ["id", "runId", "seq", "type", "payload", "createdAt"], "AgentRun event"); + if (value.event.id !== value.eventId || Number(value.event.seq) !== Number(value.sourceSeq)) throw contractError("workbench_kafka_identity_invalid", "Canonical AgentRun event identity does not match the envelope."); + if (!value.event.payload || typeof value.event.payload !== "object" || Array.isArray(value.event.payload)) throw contractError("workbench_kafka_schema_invalid", "Canonical AgentRun event payload must be an object."); + if (containsSecretLikeKey(value)) throw contractError("workbench_kafka_redaction_invalid", "Canonical AgentRun event contains a secret-like field."); + return value; +} + +function assertClosedObject(value, allowed, label) { + if (!value || typeof value !== "object" || Array.isArray(value)) throw contractError("workbench_kafka_schema_invalid", `${label} must be an object.`); + const unknown = Object.keys(value).filter((key) => !allowed.includes(key)); + if (unknown.length > 0) throw contractError("workbench_kafka_schema_invalid", `${label} contains unsupported fields: ${unknown.join(", ")}`); +} + +function containsSecretLikeKey(value, depth = 0) { + if (depth > 12 || value === null || value === undefined) return false; + if (Array.isArray(value)) return value.some((item) => containsSecretLikeKey(item, depth + 1)); + if (typeof value !== "object") return false; + return Object.entries(value).some(([key, entry]) => { + const normalized = key.replace(/[^a-z0-9]/giu, "").toLowerCase(); + if (/^(apikey|accesstoken|refreshtoken|password|authorization|credentialvalue|secretvalue|privatekey|clientsecret|bearertoken)$/u.test(normalized)) return !redactedSecretMarker(entry); + if (typeof entry === "string" && /^(?:https?|postgres(?:ql)?):\/\/[^/@\s]+:[^/@\s]+@/iu.test(entry)) return true; + return containsSecretLikeKey(entry, depth + 1); + }); +} + +function redactedSecretMarker(value) { + if (value === null || value === undefined) return true; + const normalized = stringValue(value)?.toLowerCase(); + return normalized === "redacted" || normalized === "[redacted]" || normalized === "present" || normalized === "missing"; +} + +function kafkaTransport(kafkaMessage = {}) { + const valueText = kafkaMessage?.message?.value ? Buffer.from(kafkaMessage.message.value).toString("utf8") : ""; + return { + sourceTopic: stringValue(kafkaMessage.topic), + sourcePartition: integerValue(kafkaMessage.partition), + sourceOffset: stringValue(kafkaMessage.message?.offset), + sourceKey: kafkaMessage.message?.key ? Buffer.from(kafkaMessage.message.key).toString("utf8") : null, + inputSha256: sha256(valueText), + valuesPrinted: false + }; +} + +function requireKafkaProjectorStore(runtimeStore, capabilities = {}) { + const required = []; + if (capabilities.transactionalProjector) required.push("commitAgentRunKafkaProjection", "recordFailedAgentRunKafkaMessage", "recordIgnoredAgentRunKafkaMessage"); + if (capabilities.projectionOutboxRelay) required.push("claimHwlabKafkaOutbox", "completeHwlabKafkaOutbox", "retryHwlabKafkaOutbox"); + if (capabilities.projectionRealtime) required.push("subscribeWorkbenchProjectionCommits"); + if (capabilities.transactionalProjector || capabilities.projectionOutboxRelay) required.push("hwlabKafkaProjectorStatus"); + if (capabilities.transactionalProjector || capabilities.projectionOutboxRelay || capabilities.projectionRealtime) required.push("assertWorkbenchTransactionalRealtimeReady"); + const missing = required.filter((name) => typeof runtimeStore?.[name] !== "function"); + if (missing.length > 0) throw contractError("hwlab_kafka_projector_store_invalid", `Kafka durable capabilities require a runtime store: ${missing.join(", ")}`); +} + export async function queryKafkaEventStream({ env = process.env, stream = "hwlab", topic = null, traceId = null, sessionId = null, runId = null, commandId = null, limit = DEFAULT_QUERY_LIMIT, timeoutMs = DEFAULT_QUERY_TIMEOUT_MS, fromBeginning = true, kafkaFactory = defaultKafkaFactory } = {}) { const brokers = csv(env.HWLAB_KAFKA_BOOTSTRAP_SERVERS); if (brokers.length === 0) throw new Error("HWLAB_KAFKA_BOOTSTRAP_SERVERS is required for Kafka event queries."); @@ -214,35 +801,43 @@ export async function openKafkaEventStream({ env = process.env, stream = "hwlab" const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID) || DEFAULT_CLIENT_ID; const resolvedTopic = stringValue(topic) || kafkaTopicForStream(stream, env); const filters = compactObject({ traceId, sessionId, runId, commandId }); - const kafka = kafkaFactory({ brokers, clientId: `${clientId}-sse` }); - const consumer = kafka.consumer({ groupId: `${clientId}-sse-${Date.now()}-${randomUUID().slice(0, 8)}`, allowAutoTopicCreation: false }); + const kafka = kafkaFactory({ brokers, clientId: `${clientId}-debug-sse` }); + const groupId = `${clientId}-debug-sse-${Date.now()}-${randomUUID().slice(0, 8)}`; + const consumer = kafka.consumer({ groupId, allowAutoTopicCreation: false }); let running = false; let stopped = false; await consumer.connect(); await consumer.subscribe({ topic: resolvedTopic, fromBeginning: fromBeginning === true }); running = true; - consumer.run({ - eachMessage: async ({ topic: messageTopic, partition, message }) => { - if (stopped) return; - const valueText = message.value ? Buffer.from(message.value).toString("utf8") : ""; - const value = parseJson(valueText); - if (!value || !eventMatchesFilters(value, filters)) return; - await onEvent({ - topic: messageTopic, - partition, - offset: message.offset, - key: message.key ? Buffer.from(message.key).toString("utf8") : null, - timestamp: message.timestamp ?? null, - value - }); - } - }).catch((error) => { - if (!stopped && typeof onError === "function") onError(error); - }); + try { + await consumer.run({ + eachMessage: async ({ topic: messageTopic, partition, message }) => { + if (stopped) return; + const valueText = message.value ? Buffer.from(message.value).toString("utf8") : ""; + const value = parseJson(valueText); + if (!value || !eventMatchesFilters(value, filters)) return; + await onEvent({ + topic: messageTopic, + partition, + offset: message.offset, + key: message.key ? Buffer.from(message.key).toString("utf8") : null, + timestamp: message.timestamp ?? null, + value + }); + } + }); + } catch (error) { + if (typeof onError === "function") onError(error); + stopped = true; + if (running) await consumer.stop().catch(() => undefined); + await boundedDisconnect(consumer); + throw error; + } return { ok: true, stream, topic: resolvedTopic, + groupId, filters, async stop() { stopped = true; @@ -269,8 +864,23 @@ function mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run, }; const type = firstText(sourceEvent.type, payload.type, input.eventType) || "event"; if (type === "assistant_message") { - const terminal = payload.replyAuthority === true || payload.final === true; - return { ...base, type: "assistant", eventType: "assistant", status: terminal ? "completed" : "running", label: "agentrun:assistant:message", message: textPayload(payload, "assistant message"), text: textPayload(payload, ""), itemId: firstText(payload.itemId), replyAuthority: payload.replyAuthority === true, final: payload.final === true, terminal }; + const authoritative = payload.replyAuthority === true || payload.final === true; + const assistantText = textPayload(payload, "assistant message"); + return { + ...base, + type: "assistant", + eventType: "assistant", + status: "running", + label: "agentrun:assistant:message", + message: assistantText, + text: assistantText, + assistantText, + finalResponse: authoritative ? { text: assistantText, status: "completed", traceId, source: "agentrun.kafka", valuesPrinted: false } : null, + itemId: firstText(payload.itemId), + replyAuthority: payload.replyAuthority === true, + final: payload.final === true, + terminal: false + }; } if (type === "backend_status") { const phase = firstText(payload.phase) || "status"; @@ -281,6 +891,24 @@ function mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run, const normalized = terminalStatus === "cancelled" ? "canceled" : terminalStatus; return { ...base, type: "result", eventType: "terminal", status: normalized, label: `agentrun:terminal:${terminalStatus}`, errorCode: firstText(payload.failureKind, payload.errorCode, sourceEvent.failureKind, sourceEvent.errorCode), failureKind: firstText(payload.failureKind, sourceEvent.failureKind), message: textPayload({ ...sourceEvent, ...payload }, `AgentRun terminal status ${terminalStatus}`), terminal: true }; } + if (type === "tool_call") { + const toolName = firstText(payload.toolName, payload.name, payload.method) || "tool"; + return { + ...base, + type: "tool", + eventType: "tool", + status: firstText(payload.status) || "running", + label: `agentrun:tool:${toolName}`, + message: textPayload(payload, `AgentRun tool ${toolName}`), + toolName, + method: firstText(payload.method), + itemId: firstText(payload.itemId), + exitCode: integerValue(payload.exitCode), + durationMs: integerValue(payload.durationMs), + outputSummary: firstText(payload.outputSummary, payload.summary?.text), + terminal: false + }; + } if (type === "command_output") { const stream = payload.stream === "stderr" ? "stderr" : "stdout"; return { ...base, type: "output", eventType: "status", status: "running", label: `agentrun:output:${stream}`, stream, message: textPayload(payload, ""), outputTruncated: payload.outputTruncated === true }; @@ -349,7 +977,7 @@ async function boundedDisconnect(consumer, timeoutMs = 2000) { } function kafkaMessageKey(projected) { - return firstText(projected.traceId, projected.sessionId, projected.context?.commandId, projected.context?.runId) || "hwlab-event"; + return firstText(projected.sessionId, projected.traceId, projected.context?.commandId, projected.context?.runId) || "hwlab-event"; } function kafkaHeaders(projected) { @@ -408,6 +1036,26 @@ function integerValue(value) { return Number.isFinite(parsed) ? Math.floor(parsed) : null; } +function nextKafkaOffset(value) { + try { + return (BigInt(String(value)) + 1n).toString(); + } catch { + throw contractError("workbench_kafka_offset_invalid", "Kafka source offset must be an integer string."); + } +} + +function strictPositiveInteger(value, name) { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) throw contractError("hwlab_kafka_projector_config_invalid", `${name} must be a positive integer.`); + return parsed; +} + +function contractError(code, message) { + const error = new Error(message); + error.code = code; + return error; +} + function timestampValue(value) { const text = stringValue(value); if (!text) return null; diff --git a/internal/cloud/kafka-shadow-producer.ts b/internal/cloud/kafka-shadow-producer.ts deleted file mode 100644 index d5ef390a..00000000 --- a/internal/cloud/kafka-shadow-producer.ts +++ /dev/null @@ -1,185 +0,0 @@ -// SPEC: PJ2026-010205 HWLAB接入 draft-2026-06-25-p0-session-warm-runner-contract. -// Responsibility: best-effort Kafka shadow production for HWLAB -> AgentRun command admission. - -import { createHash } from "node:crypto"; - -import { Kafka, logLevel } from "kafkajs"; - -const TRUE_VALUES = new Set(["1", "true", "yes", "on"]); -const warnedCodes = new Set(); -let producerState = null; - -export function publishHwlabAgentRunCommandShadow({ params = {}, payload = {}, traceId, lifecycle = {}, env = process.env } = {}) { - const config = kafkaShadowConfig(env); - if (!config) return; - const message = buildHwlabCommandShadowMessage({ params, payload, traceId, lifecycle, config }); - void producerForConfig(config) - .then((producer) => producer.send({ topic: config.topic, messages: [message] })) - .catch((error) => warnShadowProducer("hwlab-kafka-shadow-produce-failed", { - message: error instanceof Error ? error.message : String(error), - topic: config.topic, - clientId: config.clientId - })); -} - -function kafkaShadowConfig(env) { - if (!truthy(env.HWLAB_KAFKA_SHADOW_PRODUCE_ENABLED)) return null; - if (truthy(env.HWLAB_KAFKA_SHADOW_CONSUME_ENABLED)) { - warnOnce("hwlab-kafka-shadow-consume-ignored", { - message: "HWLAB Kafka consumer cutover is disabled in this stage; producer continues in shadow mode.", - valuesPrinted: false - }); - } - const brokers = csv(env.HWLAB_KAFKA_BOOTSTRAP_SERVERS); - const topic = stringValue(env.HWLAB_KAFKA_COMMAND_TOPIC); - const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID); - const missing = []; - if (brokers.length === 0) missing.push("HWLAB_KAFKA_BOOTSTRAP_SERVERS"); - if (!topic) missing.push("HWLAB_KAFKA_COMMAND_TOPIC"); - if (!clientId) missing.push("HWLAB_KAFKA_CLIENT_ID"); - if (missing.length > 0) { - warnOnce("hwlab-kafka-shadow-config-missing", { missing, valuesPrinted: false }); - return null; - } - return { brokers, topic, clientId, key: `${clientId}|${topic}|${brokers.join(",")}` }; -} - -function buildHwlabCommandShadowMessage({ params, payload, traceId, lifecycle, config }) { - const agentRun = objectValue(payload.agentRun); - const prompt = promptEvidence(params); - const event = { - schema: "hwlab.agentrun.command.shadow.v1", - eventType: "hwlab.command.admitted", - source: config.clientId, - producedAt: new Date().toISOString(), - mode: "shadow-produce-only", - traceId: safeText(traceId ?? payload.traceId ?? params.traceId), - hwlab: { - sessionId: safeText(params.sessionId ?? payload.sessionId ?? agentRun.hwlabSessionId), - conversationId: safeText(params.conversationId ?? payload.conversationId ?? agentRun.conversationId), - turnId: safeText(lifecycle.turnId ?? payload.turnId), - userMessageId: safeText(lifecycle.userMessageId ?? payload.userMessageId), - assistantMessageId: safeText(lifecycle.assistantMessageId ?? payload.assistantMessageId), - projectId: safeText(params.projectId ?? payload.projectId), - providerProfile: safeText(params.providerProfile ?? agentRun.backendProfile) - }, - agentRun: { - adapter: safeText(agentRun.adapter), - runId: safeText(agentRun.runId), - commandId: safeText(agentRun.commandId), - attemptId: safeText(agentRun.attemptId), - runnerId: safeText(agentRun.runnerId), - jobName: safeText(agentRun.jobName), - namespace: safeText(agentRun.namespace), - backendProfile: safeText(agentRun.backendProfile), - status: safeText(agentRun.status), - commandState: safeText(agentRun.commandState), - terminalStatus: safeText(agentRun.terminalStatus) - }, - prompt, - diagnostics: { - payloadSha256: jsonSha256({ traceId: traceId ?? null, agentRun: eventKeyParts(agentRun), prompt }), - payloadBytes: jsonBytes({ traceId: traceId ?? null, agentRun: eventKeyParts(agentRun), prompt }), - valuesPrinted: false, - shadowConsumeEnabled: false - }, - valuesPrinted: false - }; - return { - key: safeText(agentRun.commandId ?? traceId ?? params.sessionId) || config.clientId, - value: JSON.stringify(event), - headers: { - "x-shadow-mode": "produce-only", - "x-values-printed": "false", - "x-trace-id": event.traceId ?? "" - } - }; -} - -async function producerForConfig(config) { - if (producerState?.key === config.key && producerState.producer) return producerState.producer; - if (producerState?.key === config.key && producerState.connecting) return producerState.connecting; - const kafka = new Kafka({ clientId: config.clientId, brokers: config.brokers, logLevel: logLevel.NOTHING }); - const producer = kafka.producer({ allowAutoTopicCreation: false }); - const state = { - key: config.key, - producer, - connecting: producer.connect().then(() => { - state.connecting = null; - return producer; - }).catch((error) => { - if (producerState === state) producerState = null; - throw error; - }) - }; - producerState = state; - return state.connecting; -} - -function promptEvidence(params) { - const text = firstString(params.prompt, params.message, params.input, objectValue(params.userMessage).text, objectValue(params.userMessage).content); - if (!text) return { present: false, bytes: 0, sha256: null, valuesPrinted: false }; - return { - present: true, - bytes: Buffer.byteLength(text, "utf8"), - sha256: createHash("sha256").update(text).digest("hex"), - valuesPrinted: false - }; -} - -function eventKeyParts(agentRun) { - return { - runId: safeText(agentRun.runId), - commandId: safeText(agentRun.commandId), - attemptId: safeText(agentRun.attemptId), - backendProfile: safeText(agentRun.backendProfile) - }; -} - -function truthy(value) { - return TRUE_VALUES.has(String(value ?? "").trim().toLowerCase()); -} - -function csv(value) { - return String(value ?? "").split(",").map((entry) => entry.trim()).filter(Boolean); -} - -function stringValue(value) { - const text = String(value ?? "").trim(); - return text.length > 0 ? text : null; -} - -function safeText(value, max = 240) { - const text = stringValue(value); - if (!text) return null; - return text.length > max ? `${text.slice(0, max)}...` : text; -} - -function firstString(...values) { - for (const value of values) { - if (typeof value === "string" && value.trim().length > 0) return value; - } - return null; -} - -function objectValue(value) { - return value && typeof value === "object" && !Array.isArray(value) ? value : {}; -} - -function jsonSha256(value) { - return createHash("sha256").update(JSON.stringify(value)).digest("hex"); -} - -function jsonBytes(value) { - return Buffer.byteLength(JSON.stringify(value), "utf8"); -} - -function warnOnce(code, details) { - if (warnedCodes.has(code)) return; - warnedCodes.add(code); - warnShadowProducer(code, details); -} - -function warnShadowProducer(code, details = {}) { - console.warn(JSON.stringify({ code, component: "hwlab-kafka-shadow-producer", ...details, valuesPrinted: false })); -} diff --git a/internal/cloud/server-agent-chat-agentrun-backoff.test.ts b/internal/cloud/server-agent-chat-agentrun-backoff.test.ts deleted file mode 100644 index 06ae85eb..00000000 --- a/internal/cloud/server-agent-chat-agentrun-backoff.test.ts +++ /dev/null @@ -1,130 +0,0 @@ -// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. -// Responsibility: Focused AgentRun projection polling/backoff regressions for Cloud API chat result sync. - -import assert from "node:assert/strict"; -import { createServer as createHttpServer } from "node:http"; -import { test } from "bun:test"; - -import { syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts"; -import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; - -test("AgentRun sync stops upstream polling when terminal command result is already sealed (#1445)", async () => { - const traceId = "trc_1445_sealed_terminal"; - const runId = "run_1445_sealed_terminal"; - const commandId = "cmd_1445_sealed_terminal"; - const traceStore = createCodeAgentTraceStore(); - let fetched = false; - const synced = await syncAgentRunChatResult({ - traceId, - currentResult: { - ok: true, - status: "completed", - traceId, - finalResponse: { text: "sealed final response", traceId }, - traceSummary: { source: "agentrun-command-result", traceId, agentRun: { runId, commandId, lastSeq: 17 } }, - agentRun: { - adapter: "agentrun-v01", - runId, - commandId, - traceId, - status: "completed", - runStatus: "completed", - commandState: "completed", - terminalStatus: "completed", - providerTrace: { traceId, runId, commandId, valuesPrinted: false }, - lastSeq: 17, - managerUrl: "http://127.0.0.1:1", - valuesPrinted: false - }, - valuesPrinted: false - }, - options: { - fetchImpl: async () => { - fetched = true; - throw new Error("sealed terminal result should not fetch AgentRun"); - }, - env: { HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1" } - }, - traceStore - }); - - assert.equal(fetched, false); - assert.equal(synced.terminalRefreshSkipped, true); - assert.equal(synced.resultSynced, false); - assert.equal(synced.polling.rootCause, "terminal_result_sealed"); - assert.equal(synced.polling.backoffMs, 0); - assert.equal(synced.polling.terminalConsistencyGap, 0); -}); - -test("AgentRun events rate limit writes durable projection backoff (#1445)", async () => { - const traceId = "trc_1445_events_rate_limit"; - const runId = "run_1445_events_rate_limit"; - const commandId = "cmd_1445_events_rate_limit"; - const calls = []; - const writes = []; - const runtimeStore = { - async getWorkbenchProjectionState() { - return { projectionState: { traceId, sourceRunId: runId, sourceCommandId: commandId, lastSourceSeq: 40, pollCount: 2, noProgressPollCount: 1, resultSyncState: "not_started", createdAt: "2026-07-02T00:00:00.000Z" } }; - }, - async writeWorkbenchProjectionState(input = {}) { - writes.push(input.projectionState); - return { written: true, projectionState: input.projectionState }; - } - }; - const agentRunServer = createHttpServer(async (request, response) => { - const url = new URL(request.url || "/", "http://127.0.0.1"); - calls.push(`${request.method} ${url.pathname}${url.search}`); - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/events`) { - response.writeHead(429, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: false, failureKind: "provider_rate_limited", message: "provider rate limited; retry later" })}\n`); - return; - } - response.writeHead(404, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: false, message: `unexpected ${request.method} ${url.pathname}` })}\n`); - }); - await new Promise((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve)); - const agentRunPort = agentRunServer.address().port; - const traceStore = createCodeAgentTraceStore(); - try { - await assert.rejects( - syncAgentRunChatResult({ - traceId, - currentResult: { - status: "running", - traceId, - agentRun: { adapter: "agentrun-v01", runId, commandId, traceId, providerTrace: { traceId }, lastSeq: 40, managerUrl: `http://127.0.0.1:${agentRunPort}`, valuesPrinted: false }, - valuesPrinted: false - }, - options: { - runtimeStore, - env: { - HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_BACKOFF_INITIAL_MS: "1000", - HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS: "2500" - } - }, - traceStore - }), - (error) => { - assert.equal(error.statusCode, 429); - assert.equal(error.polling.rootCause, "provider_rate_limited"); - assert.equal(error.polling.backoffReason, "provider_retry_or_rate_limit"); - assert.equal(error.polling.backoffMs, 2000); - assert.ok(error.polling.nextRetryAt); - return true; - } - ); - - assert.equal(calls.length, 1); - assert.match(calls[0], /afterSeq=40/u); - assert.equal(writes.length, 1); - assert.equal(writes[0].projectionStatus, "degraded"); - assert.equal(writes[0].projectionHealth, "degraded"); - assert.equal(writes[0].backoffMs, 2000); - assert.equal(writes[0].backoffReason, "provider_retry_or_rate_limit"); - assert.equal(writes[0].rootCause, "provider_rate_limited"); - assert.ok(writes[0].nextRetryAt); - } finally { - await new Promise((resolve, reject) => agentRunServer.close((error) => (error ? reject(error) : resolve()))); - } -}); diff --git a/internal/cloud/server-agent-chat-lifecycle.test.ts b/internal/cloud/server-agent-chat-lifecycle.test.ts index a59d1828..55120ee5 100644 --- a/internal/cloud/server-agent-chat-lifecycle.test.ts +++ b/internal/cloud/server-agent-chat-lifecycle.test.ts @@ -13,7 +13,7 @@ import { createCloudRuntimeStore } from "../db/runtime-store.ts"; import { validateCodeAgentChatSchema } from "./code-agent-chat.ts"; import { createCodexStdioSessionManager } from "./codex-stdio-session.ts"; import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; -import { submitAgentRunChatTurn, syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts"; +import { submitAgentRunChatTurn } from "./code-agent-agentrun-adapter.ts"; import { codexStdioChatFixture, codexStdioReadyFixture, diff --git a/internal/cloud/server-agent-chat-session.test.ts b/internal/cloud/server-agent-chat-session.test.ts index 965bcb5c..9bfc2cad 100644 --- a/internal/cloud/server-agent-chat-session.test.ts +++ b/internal/cloud/server-agent-chat-session.test.ts @@ -1,5 +1,5 @@ // SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. -// Responsibility: Cloud API AgentRun session/profile continuity and durable projection resume regression tests. +// Responsibility: Cloud API AgentRun session/profile continuity regression tests. import assert from "node:assert/strict"; import { createServer as createHttpServer } from "node:http"; @@ -13,7 +13,7 @@ import { createCloudRuntimeStore } from "../db/runtime-store.ts"; import { validateCodeAgentChatSchema } from "./code-agent-chat.ts"; import { createCodexStdioSessionManager } from "./codex-stdio-session.ts"; import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; -import { submitAgentRunChatTurn, syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts"; +import { submitAgentRunChatTurn } from "./code-agent-agentrun-adapter.ts"; import { codexStdioChatFixture, codexStdioReadyFixture, @@ -1356,391 +1356,3 @@ test("cloud api turn status reuses request session lookup without loading persis }); } }); - -test("cloud api resumes AgentRun projection from durable state after restart", async () => { - const calls = []; - const durableEvents = []; - const projectionStates = new Map(); - const ownerWrites = []; - const traceId = "trc_projection_resume_restart"; - const runId = "run_projection_resume_restart"; - const commandId = "cmd_projection_resume_restart"; - const sessionId = "ses_projection_resume_restart"; - const conversationId = "cnv_projection_resume_restart"; - const finalText = "OK from resumed AgentRun projection."; - - const agentRunServer = createHttpServer(async (request, response) => { - const url = new URL(request.url || "/", "http://127.0.0.1"); - calls.push({ method: request.method, path: url.pathname, search: url.search, afterSeq: url.searchParams.get("afterSeq") }); - const send = (body, status = 200) => { - response.writeHead(status, { "content-type": "application/json" }); - response.end(`${JSON.stringify(body)}\n`); - }; - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/events`) { - assert.equal(url.searchParams.get("afterSeq"), "21"); - return send({ items: [ - { id: "evt_projection_resume_22", runId, seq: 22, type: "assistant_message", payload: { commandId, text: finalText }, createdAt: "2026-06-18T16:31:11.000Z" }, - { id: "evt_projection_resume_23", runId, seq: 23, type: "terminal_status", payload: { commandId, terminalStatus: "completed" }, createdAt: "2026-06-18T16:31:12.000Z" } - ] }); - } - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/commands/${commandId}/result`) { - return send({ - runId, - commandId, - status: "completed", - runStatus: "completed", - commandState: "completed", - terminalStatus: "completed", - completed: true, - reply: finalText, - lastSeq: 23, - eventCount: 23, - sessionRef: { sessionId, conversationId, threadId: "thread-projection-resume-restart" } - }); - } - return send({ ok: false, message: `unexpected ${request.method} ${url.pathname}` }, 404); - }); - await new Promise((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve)); - const agentRunPort = agentRunServer.address().port; - - const session = testAgentSessionRecord({ - sessionId, - conversationId, - projectId: "prj_hwpod_workbench", - status: "running", - traceId, - updatedAt: "2026-06-18T16:31:04.000Z", - session: { - traceResults: { - [traceId]: { - traceId, - status: "running", - sessionId, - conversationId, - updatedAt: "2026-06-18T16:31:04.000Z", - agentRun: { - adapter: "agentrun-v01", - managerUrl: `http://127.0.0.1:${agentRunPort}`, - runId, - commandId, - traceId, - lastSeq: 0, - status: "running", - commandState: "running", - backendProfile: "codex", - valuesPrinted: false - }, - valuesRedacted: true - } - } - } - }); - - durableEvents.push({ - traceId, - seq: 27, - source: "agentrun", - sourceSeq: 21, - type: "backend", - status: "running", - label: "agentrun:backend:mcpServer/startupStatus/updated", - commandId, - runId, - createdAt: "2026-06-18T16:31:04.200Z", - valuesPrinted: false - }); - projectionStates.set(traceId, { - traceId, - sessionId, - conversationId, - threadId: "thread-projection-resume-restart", - ownerUserId: TEST_AGENT_ACTOR.id, - ownerRole: TEST_AGENT_ACTOR.role, - runId, - commandId, - sourceRunId: runId, - sourceCommandId: commandId, - managerUrl: `http://127.0.0.1:${agentRunPort}`, - backendProfile: "codex", - lastSourceSeq: 21, - lastAgentRunSeq: 21, - lastProjectedSeq: 27, - sourceLatestSeq: 21, - upstreamLatestSeq: 21, - projectionStatus: "projecting", - projectionHealth: "healthy", - resultSyncState: "not_started", - createdAt: "2026-06-18T16:31:04.000Z", - updatedAt: "2026-06-18T16:31:04.000Z", - valuesPrinted: false - }); - - const runtimeStore = { - async queryAgentTraceEvents(params = {}) { - return { events: durableEvents.filter((event) => event.traceId === params.traceId), count: durableEvents.length }; - }, - async writeAgentTraceEvent(params = {}) { - const event = params.event ?? params.traceEvent ?? params; - durableEvents.push(event); - return { written: true, traceEvent: event }; - }, - async getWorkbenchProjectionState(params = {}) { - const projectionState = projectionStates.get(params.traceId) ?? null; - return { projectionState, found: Boolean(projectionState) }; - }, - async queryWorkbenchProjectionStates() { - return { states: [...projectionStates.values()], count: projectionStates.size }; - }, - async writeWorkbenchProjectionState(params = {}) { - const projectionState = params.projectionState ?? params.state ?? params; - projectionStates.set(projectionState.traceId, projectionState); - return { written: true, projectionState }; - } - }; - const accessController = { - required: false, - store: { - async listAgentSessionsForUser() { return [session]; }, - async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; } - }, - async authenticate() { return { ok: true, actor: TEST_AGENT_ACTOR, session: TEST_AUTH_SESSION }; }, - async recordAgentSessionOwner(input = {}) { - ownerWrites.push(input); - return { ...session, status: input.status ?? session.status, session: input.session ?? session.session }; - } - }; - - const server = createCloudApiServer({ - accessController, - runtimeStore, - codeAgentChatResults: new Map(), - env: { - HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", - HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_ENABLED: "1", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INITIAL_DELAY_MS: "1", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INTERVAL_MS: "0", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS: "0", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE: "10" - } - }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - for (let index = 0; index < 80 && projectionStates.get(traceId)?.resultSyncState !== "synced"; index += 1) await delay(10); - assert.ok(calls.some((call) => call.path === `/api/v1/runs/${runId}/events` && call.afterSeq === "21")); - assert.equal(calls.some((call) => call.path === `/api/v1/runs/${runId}/commands/${commandId}/result`), true); - assert.equal(projectionStates.get(traceId)?.lastSourceSeq, 23); - assert.equal(projectionStates.get(traceId)?.resultSyncState, "synced"); - assert.equal(ownerWrites.some((write) => write.status === "completed"), true); - assert.ok(durableEvents.some((event) => event.sourceSeq === 22)); - } finally { - await new Promise((resolve, reject) => { - server.close((error) => (error ? reject(error) : resolve())); - }); - await new Promise((resolve, reject) => { - agentRunServer.close((error) => (error ? reject(error) : resolve())); - }); - } -}); - -test("cloud api projection resume retries evicted AgentRun session with stored prompt", async () => { - const calls = []; - const durableEvents = []; - const projectionStates = new Map(); - const traceId = "trc_projection_resume_evicted_retry"; - const runId = "run_projection_resume_evicted"; - const commandId = "cmd_projection_resume_evicted"; - const retryRunId = "run_projection_resume_evicted_retry"; - const retryCommandId = "cmd_projection_resume_evicted_retry"; - const sessionId = "ses_projection_resume_evicted"; - const conversationId = "cnv_projection_resume_evicted"; - const promptText = "resume projection evicted session with original prompt"; - - const agentRunServer = createHttpServer(async (request, response) => { - const url = new URL(request.url || "/", "http://127.0.0.1"); - const chunks = []; - for await (const chunk of request) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); - const body = chunks.length ? JSON.parse(Buffer.concat(chunks).toString("utf8")) : null; - calls.push({ method: request.method, path: url.pathname, search: url.search, afterSeq: url.searchParams.get("afterSeq"), body }); - const send = (data, status = 200) => { - response.writeHead(status, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: status < 400, data })}\n`); - }; - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/events`) { - return send({ items: [ - { id: "evt_projection_evicted_1", runId, seq: 1, type: "error", payload: { commandId, failureKind: "session-store-evicted", message: "codex app-server thread/resume reported no rollout found for PVC-backed session; session storage was likely evicted" }, createdAt: "2026-06-19T12:01:00.000Z" }, - { id: "evt_projection_evicted_2", runId, seq: 2, type: "terminal_status", payload: { commandId, terminalStatus: "failed", failureKind: "session-store-evicted" }, createdAt: "2026-06-19T12:01:01.000Z" } - ] }); - } - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/commands/${commandId}/result`) { - return send({ - runId, - commandId, - status: "failed", - runStatus: "failed", - commandState: "failed", - terminalStatus: "failed", - failureKind: "session-store-evicted", - failureMessage: "codex app-server thread/resume reported no rollout found for PVC-backed session; session storage was likely evicted", - completed: false, - lastSeq: 2, - eventCount: 2, - sessionRef: { sessionId: "ses_agentrun_projection_resume_evicted", conversationId, threadId: "thread_projection_evicted" } - }); - } - if (request.method === "POST" && url.pathname === "/api/v1/sessions") { - assert.match(body.sessionId, /-reset-/u); - return send({ sessionId: body.sessionId, backendProfile: body.backendProfile }); - } - if (request.method === "POST" && url.pathname === "/api/v1/runs") { - assert.match(body.sessionRef?.sessionId, /-reset-/u); - assert.equal(body.sessionRef?.threadId, undefined); - return send({ id: retryRunId, status: "pending", backendProfile: "dsflash-go", sessionRef: body.sessionRef, resourceBundleRef: body.resourceBundleRef }); - } - if (request.method === "POST" && url.pathname === `/api/v1/runs/${retryRunId}/commands`) { - assert.equal(body.type, "turn"); - assert.equal(body.payload.prompt, promptText); - assert.equal(body.payload.message, promptText); - assert.equal(body.payload.threadId, null); - assert.match(body.payload.sessionId, /-reset-/u); - assert.equal(body.payload.providerProfile, "dsflash-go"); - assert.equal(body.dispatch?.kind, "kubernetes-job"); - assert.equal(body.dispatch?.input?.commandId, undefined); - return send({ - id: retryCommandId, - runId: retryRunId, - state: "queued", - dispatchIntent: { - id: "dispatch_projection_resume_evicted_retry", - state: "pending", - runnerJobId: "rjob_projection_resume_evicted_retry", - attemptCount: 0, - durable: true - } - }); - } - return send({ message: `unexpected ${request.method} ${url.pathname}` }, 500); - }); - await new Promise((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve)); - const agentRunPort = agentRunServer.address().port; - - projectionStates.set(traceId, { - traceId, - sessionId, - conversationId, - threadId: "thread_projection_evicted", - ownerUserId: TEST_AGENT_ACTOR.id, - ownerRole: TEST_AGENT_ACTOR.role, - runId, - commandId, - sourceRunId: runId, - sourceCommandId: commandId, - managerUrl: `http://127.0.0.1:${agentRunPort}`, - backendProfile: "dsflash-go", - providerId: "JD01", - lastSourceSeq: 0, - lastAgentRunSeq: 0, - lastProjectedSeq: 0, - sourceLatestSeq: 0, - upstreamLatestSeq: 0, - projectionStatus: "projecting", - projectionHealth: "healthy", - resultSyncState: "not_started", - retryParams: { - message: promptText, - prompt: promptText, - text: promptText, - sessionId, - conversationId, - threadId: "thread_projection_evicted", - ownerUserId: TEST_AGENT_ACTOR.id, - ownerRole: TEST_AGENT_ACTOR.role, - providerProfile: "dsflash-go", - codeAgentProviderProfile: "dsflash-go", - backendProfile: "dsflash-go", - valuesPrinted: false - }, - createdAt: "2026-06-19T12:00:00.000Z", - updatedAt: "2026-06-19T12:00:00.000Z", - valuesPrinted: false - }); - - const runtimeStore = { - async queryAgentTraceEvents(params = {}) { - return { events: durableEvents.filter((event) => event.traceId === params.traceId), count: durableEvents.length }; - }, - async writeAgentTraceEvent(params = {}) { - const event = params.event ?? params.traceEvent ?? params; - durableEvents.push(event); - return { written: true, traceEvent: event }; - }, - async getWorkbenchProjectionState(params = {}) { - const projectionState = projectionStates.get(params.traceId) ?? null; - return { projectionState, found: Boolean(projectionState) }; - }, - async queryWorkbenchProjectionStates() { - return { states: [...projectionStates.values()], count: projectionStates.size }; - }, - async writeWorkbenchProjectionState(params = {}) { - const projectionState = params.projectionState ?? params.state ?? params; - projectionStates.set(projectionState.traceId, projectionState); - return { written: true, projectionState }; - } - }; - const accessController = { - required: false, - store: { - async listAgentSessionsForUser() { return []; }, - async getAgentSessionByTraceId() { return null; } - }, - async authenticate() { return { ok: true, actor: TEST_AGENT_ACTOR, session: TEST_AUTH_SESSION }; } - }; - - const server = createCloudApiServer({ - accessController, - runtimeStore, - codeAgentChatResults: new Map(), - env: { - HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", - HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1", - HWLAB_CODE_AGENT_AGENTRUN_SOURCE_COMMIT: "a".repeat(40), - HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "JD01", - HWLAB_RUNTIME_NAMESPACE: "hwlab-v03", - HWLAB_RUNTIME_LANE: "v03", - HWLAB_RUNTIME_API_URL: "http://hwlab-cloud-api.hwlab-v03.svc.cluster.local:6667", - HWLAB_RUNTIME_WEB_URL: "http://hwlab-cloud-web.hwlab-v03.svc.cluster.local:8080", - HWLAB_RUNTIME_ENDPOINT_LOCKED: "1", - HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME: "1", - HWLAB_CODE_AGENT_AGENTRUN_BACKEND_PROFILE: "dsflash-go", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_ENABLED: "1", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INITIAL_DELAY_MS: "1", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INTERVAL_MS: "0", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS: "0", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE: "10" - } - }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - for (let index = 0; index < 100 && projectionStates.get(traceId)?.runId !== retryRunId; index += 1) await delay(10); - assert.equal(calls.some((call) => call.path === `/api/v1/runs/${runId}/commands/${commandId}/result`), true); - assert.equal(calls.some((call) => call.path === `/api/v1/runs/${retryRunId}/commands`), true); - const state = projectionStates.get(traceId); - assert.equal(state.runId, retryRunId); - assert.equal(state.commandId, retryCommandId); - assert.equal(state.sourceRunId, retryRunId); - assert.equal(state.sourceCommandId, retryCommandId); - assert.equal(state.projectionStatus, "projecting"); - assert.equal(state.retryParams.message, promptText); - assert.equal(state.retryParams.providerProfile, "dsflash-go"); - assert.equal(calls.some((call) => call.path.endsWith("/runner-jobs")), false); - } finally { - await new Promise((resolve, reject) => { - server.close((error) => (error ? reject(error) : resolve())); - }); - await new Promise((resolve, reject) => { - agentRunServer.close((error) => (error ? reject(error) : resolve())); - }); - } -}); diff --git a/internal/cloud/server-agent-chat.test.ts b/internal/cloud/server-agent-chat.test.ts index e19f3aad..db888cdf 100644 --- a/internal/cloud/server-agent-chat.test.ts +++ b/internal/cloud/server-agent-chat.test.ts @@ -13,7 +13,8 @@ import { createCloudRuntimeStore } from "../db/runtime-store.ts"; import { validateCodeAgentChatSchema } from "./code-agent-chat.ts"; import { createCodexStdioSessionManager } from "./codex-stdio-session.ts"; import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; -import { submitAgentRunChatTurn, syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts"; +import { submitAgentRunChatTurn } from "./code-agent-agentrun-adapter.ts"; +import { buildWorkbenchProjectionEventFacts } from "./workbench-projection-writer.ts"; import { codexStdioChatFixture, codexStdioReadyFixture, @@ -31,6 +32,60 @@ import { const TEST_AGENT_ACTOR = { id: "usr_agent_owner", username: "agent-owner", displayName: "Agent Owner", role: "user", status: "active" }; const TEST_AUTH_SESSION = { id: "auth_ses_agent_owner" }; +async function waitForTraceEvent(traceStore, traceId, predicate, message = "expected trace event") { + for (let index = 0; index < 100; index += 1) { + const event = traceStore.snapshot(traceId).events.find(predicate); + if (event) return event; + await delay(5); + } + assert.fail(message); +} + +async function commitTestKafkaTerminal(runtimeStore, { + traceId, + sessionId, + runId = null, + commandId = null, + status = "completed", + text, + failureKind = null, + projectedSeq = 1, + sourceSeq = 1, + createdAt = "2026-07-10T10:00:00.000Z" +}) { + const built = buildWorkbenchProjectionEventFacts({ + projectedSeq, + projectedAt: createdAt, + event: { + traceId, + sessionId, + runId, + commandId, + sourceEventId: `evt_${traceId.slice(4)}_${sourceSeq}`, + sourceSeq, + type: "assistant", + eventType: "assistant", + label: "agentrun:kafka:assistant-final", + status, + terminal: true, + final: true, + replyAuthority: true, + text, + failureKind, + errorCode: failureKind, + createdAt + } + }); + assert.equal(built.written, true); + await runtimeStore.writeWorkbenchFacts({ facts: built.facts }, { + source: "test-kafka-projector", + traceId, + sessionId, + valuesPrinted: false + }); + return built; +} + function testAgentSessionRecord(input = {}) { const sessionId = input.sessionId ?? input.id; return { @@ -111,7 +166,7 @@ test("cloud api trace resource paginates events by sinceSeq", async () => { const traceStore = createCodeAgentTraceStore(); const traceId = "trc_trace_pagination"; for (let index = 0; index < 5; index += 1) { - traceStore.append(traceId, { type: "trace", status: "observed", label: `event:${index}` }); + traceStore.append(traceId, { type: "trace", status: index === 4 ? "completed" : "observed", terminal: index === 4, label: `event:${index}` }); } const server = createCloudApiServer({ traceStore, @@ -148,6 +203,11 @@ test("cloud api trace resource paginates events by sinceSeq", async () => { assert.equal(finalPage.status, 200); const finalBody = await finalPage.json(); assert.deepEqual(finalBody.events.map((event) => event.label), ["event:4"]); + assert.equal(finalBody.events[0].terminal, false); + assert.equal(finalBody.events[0].diagnosticOnly, true); + assert.equal(finalBody.events[0].authority, "trace-store-diagnostics"); + assert.equal(finalBody.terminal, false); + assert.equal(finalBody.terminalAuthority, "workbench-projection"); assert.equal(finalBody.hasMore, false); assert.equal(finalBody.truncated, false); assert.equal(finalBody.fullTraceLoaded, true); @@ -158,6 +218,116 @@ test("cloud api trace resource paginates events by sinceSeq", async () => { } }); +test("cloud api trace pagination assigns a collision-free response cursor across diagnostic and projected seq", async () => { + const traceStore = createCodeAgentTraceStore(); + const runtimeStore = createCloudRuntimeStore(); + const traceId = "trc_trace_pagination_collision"; + const sessionId = "ses_trace_pagination_collision"; + traceStore.append(traceId, { seq: 1, type: "diagnostic", status: "running", label: "diagnostic:seq-1", terminal: false }); + const built = buildWorkbenchProjectionEventFacts({ + projectedSeq: 1, + projectedAt: "2026-07-10T10:00:00.000Z", + event: { + traceId, + sessionId, + sourceEventId: "evt_trace_pagination_collision_1", + sourceSeq: 1, + type: "backend", + label: "projection:seq-1", + status: "running", + terminal: false, + createdAt: "2026-07-10T10:00:00.000Z" + } + }); + await runtimeStore.writeWorkbenchFacts({ facts: built.facts }, { traceId, sessionId, valuesPrinted: false }); + const server = createCloudApiServer({ + traceStore, + runtimeStore, + workbenchRuntime: runtimeStore, + accessController: { + required: false, + async authenticate() { return { ok: true, actor: TEST_AGENT_ACTOR, session: TEST_AUTH_SESSION }; } + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const first = await fetch(`http://127.0.0.1:${port}/v1/agent/traces/${traceId}?limit=1`); + assert.equal(first.status, 200); + const firstBody = await first.json(); + assert.equal(firstBody.events.length, 1); + assert.equal(firstBody.events[0].seq, 1); + assert.equal(firstBody.events[0].projectedSeq, 1); + assert.equal(firstBody.nextSinceSeq, 1); + assert.equal(firstBody.hasMore, true); + + const second = await fetch(`http://127.0.0.1:${port}/v1/agent/traces/${traceId}?sinceSeq=${firstBody.nextSinceSeq}&limit=1`); + assert.equal(second.status, 200); + const secondBody = await second.json(); + assert.equal(secondBody.events.length, 1); + assert.equal(secondBody.events[0].seq, 2); + assert.equal(secondBody.events[0].sourceSeq, 1); + assert.equal(secondBody.events[0].diagnosticOnly, true); + assert.equal(secondBody.hasMore, false); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("Code Agent steer fails closed when terminal projection authority is unavailable", async () => { + const traceId = "trc_steer_projection_store_unavailable"; + const codeAgentChatResults = new Map([[traceId, { + accepted: true, + status: "running", + traceId, + ownerUserId: TEST_AGENT_ACTOR.id, + sessionId: "ses_steer_projection_store_unavailable", + agentRun: { + adapter: "agentrun-v01", + runId: "run_steer_projection_store_unavailable", + commandId: "cmd_steer_projection_store_unavailable", + dispatchIntentId: "dispatch_steer_projection_store_unavailable", + durableDispatch: true + } + }]]); + let projectionQueryCount = 0; + const workbenchRuntime = { + async queryWorkbenchFacts() { + projectionQueryCount += 1; + const error = new Error("projection database unavailable"); + error.code = "TEST_PROJECTION_DB_UNAVAILABLE"; + throw error; + } + }; + const server = createCloudApiServer({ + codeAgentChatResults, + workbenchRuntime, + accessController: { + required: false, + async authenticate() { return { ok: true, actor: TEST_AGENT_ACTOR, session: TEST_AUTH_SESSION }; } + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat/steer`, { + method: "POST", + headers: { "content-type": "application/json", "x-trace-id": traceId }, + body: JSON.stringify({ traceId, message: "must not dispatch while terminal authority is unreadable" }) + }); + assert.equal(response.status, 503); + const body = await response.json(); + assert.equal(body.accepted, false); + assert.equal(body.status, "blocked"); + assert.equal(body.error.code, "projection_store_unavailable"); + assert.ok(projectionQueryCount > 0); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + test("cloud api turn and trace endpoints reject mismatched requested project", async () => { const traceStore = createCodeAgentTraceStore(); const traceId = "trc_issue1429_project_scope"; @@ -210,15 +380,13 @@ test("AgentRun adapter filters resource tools and credentials through access cap }; if (request.method === "POST" && url.pathname === "/api/v1/sessions") return send({ ok: true }); if (request.method === "POST" && url.pathname === "/api/v1/runs") return send({ id: "run_access_tools", status: "pending", backendProfile: "deepseek", sessionRef: body.sessionRef, resourceBundleRef: body.resourceBundleRef }); - if (request.method === "POST" && url.pathname === "/api/v1/runs/run_access_tools/commands") return send({ id: "cmd_access_tools", runId: "run_access_tools", state: "pending", type: "turn", seq: 1 }); - if (request.method === "POST" && url.pathname === "/api/v1/runs/run_access_tools/runner-jobs") return send({ - action: "create-kubernetes-job", + if (request.method === "POST" && url.pathname === "/api/v1/runs/run_access_tools/commands") return send({ + id: "cmd_access_tools", runId: "run_access_tools", - commandId: body.commandId, - attemptId: "attempt_access_tools", - runnerId: "runner_access_tools", - namespace: "agentrun-v01", - jobName: "agentrun-v01-runner-access-tools" + state: "pending", + type: "turn", + seq: 1, + dispatchIntent: { id: "dispatch_access_tools", state: "pending", runnerJobId: "rjob_access_tools", attemptCount: 0, durable: true } }); response.writeHead(404, { "content-type": "application/json" }); response.end(`${JSON.stringify({ ok: false, message: `unexpected ${request.method} ${url.pathname}` })}\n`); @@ -288,8 +456,9 @@ test("AgentRun adapter filters resource tools and credentials through access cap assert.equal(Object.hasOwn(createRun.body.resourceBundleRef, "skillRefs"), false); assert.equal(Object.hasOwn(createRun.body.resourceBundleRef, "workspaceFiles"), false); assert.deepEqual(createRun.body.executionPolicy.secretScope.toolCredentials, []); - const runnerJob = calls.find((call) => call.method === "POST" && call.path === "/api/v1/runs/run_access_tools/runner-jobs"); - assert.equal(runnerJob.body.transientEnv.some((entry) => entry.name === "HWLAB_API_KEY"), false); + const command = calls.find((call) => call.method === "POST" && call.path === "/api/v1/runs/run_access_tools/commands"); + assert.equal(command.body.dispatch.input.transientEnv.some((entry) => entry.name === "HWLAB_API_KEY"), false); + assert.equal(calls.some((call) => call.path.endsWith("/runner-jobs")), false); } finally { await new Promise((resolve, reject) => agentRunServer.close((error) => (error ? reject(error) : resolve()))); } @@ -297,6 +466,10 @@ test("AgentRun adapter filters resource tools and credentials through access cap test("cloud api /v1/agent/chat parse and params errors use structured blocker envelope", async () => { const server = createCloudApiServer({ + accessController: { + required: false, + async authenticate() { return { ok: true, actor: TEST_AGENT_ACTOR, session: TEST_AUTH_SESSION }; } + }, env: { HWLAB_CODE_AGENT_MODEL: "gpt-test" } @@ -402,7 +575,9 @@ test("cloud api /v1/agent/chat supports short submit and result polling", async assert.equal(accepted.turnId, traceId); assert.equal(accepted.userMessageId, "msg_server-test-short-submit_user"); assert.equal(accepted.assistantMessageId, "msg_server-test-short-submit_agent"); - assert.equal(accepted.resultUrl, `/v1/agent/chat/result/${traceId}`); + assert.equal(accepted.resultUrl, undefined); + assert.equal(accepted.controlSemantics, "submit-and-project-sse"); + assert.match(accepted.workbenchEventsUrl, new RegExp(`traceId=${traceId}$`, "u")); const acceptedTurn = await fetch(`http://127.0.0.1:${port}/v1/agent/turns/${encodeURIComponent(traceId)}`, { headers: { cookie: manualSession.cookie } }); assert.equal(acceptedTurn.status, 200); @@ -411,7 +586,7 @@ test("cloud api /v1/agent/chat supports short submit and result polling", async assert.equal(acceptedTurnPayload.userMessageId, accepted.userMessageId); assert.equal(acceptedTurnPayload.assistantMessageId, accepted.assistantMessageId); - const payload = await pollAgentResult(port, traceId); + const payload = await pollAgentResult(port, traceId, { headers: { cookie: manualSession.cookie } }); validateCodeAgentChatSchema(payload); assert.equal(payload.status, "completed"); assert.equal(payload.traceId, traceId); @@ -521,8 +696,36 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte assert.equal(body.payload.prompt.includes("sk-test-hwpod-secret"), false); } else { assert.equal(Object.hasOwn(body.payload, "conversationContext"), false); + assert.equal(body.dispatch?.kind, "kubernetes-job"); + const transientEnv = Object.fromEntries(body.dispatch.input.transientEnv.map((entry) => [entry.name, entry.value])); + assert.equal(transientEnv.HWLAB_RUNTIME_API_URL, "http://hwlab-cloud-api.hwlab-v03.svc.cluster.local:6667"); + assert.equal(transientEnv.HWLAB_RUNTIME_WEB_URL, "http://hwlab-cloud-web.hwlab-v03.svc.cluster.local:8080"); + assert.equal(transientEnv.HWLAB_RUNTIME_NAMESPACE, "hwlab-v03"); + assert.equal(transientEnv.HWLAB_RUNTIME_LANE, "v03"); + assert.equal(transientEnv.HWLAB_RUNTIME_ENDPOINT_LOCKED, "1"); + assert.equal(transientEnv.HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME, "1"); + assert.equal(transientEnv.UNIDESK_MAIN_SERVER_IP, "http://74.48.78.17:18081"); + assert.equal(transientEnv.HWLAB_CODE_AGENT_PROVIDER_PROFILE, "deepseek"); + assert.equal(transientEnv.HWLAB_CODE_AGENT_PARENT_TRACE_ID, "trc_server-test-agentrun-adapter"); + assert.equal(Object.hasOwn(transientEnv, "UNIDESK_SSH_CLIENT_TOKEN"), false); + assert.equal(Object.hasOwn(transientEnv, "GH_TOKEN"), false); } - return send({ id: secondTurn ? "cmd_hwlab_adapter_second" : "cmd_hwlab_adapter", runId: "run_hwlab_adapter", state: "pending", type: "turn", seq: secondTurn ? 2 : 1 }); + return send({ + id: secondTurn ? "cmd_hwlab_adapter_second" : "cmd_hwlab_adapter", + runId: "run_hwlab_adapter", + state: "pending", + type: "turn", + seq: secondTurn ? 2 : 1, + ...(secondTurn ? {} : { + dispatchIntent: { + id: "dispatch_hwlab_adapter", + state: "pending", + runnerJobId: "rjob_hwlab_adapter", + attemptCount: 0, + durable: true + } + }) + }); } if (request.method === "GET" && url.pathname === "/api/v1/runs/run_hwlab_adapter/commands") { return send({ items: [ @@ -624,8 +827,11 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte const agentRunPort = agentRunServer.address().port; const traceStore = createCodeAgentTraceStore(); + const runtimeStore = createCloudRuntimeStore(); const server = createCloudApiServer({ traceStore, + runtimeStore, + workbenchRuntime: runtimeStore, env: { HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, @@ -698,12 +904,27 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte assert.equal(accepted.turnId, traceId); assert.equal(accepted.userMessageId, "msg_server-test-agentrun-adapter_user"); assert.equal(accepted.assistantMessageId, "msg_server-test-agentrun-adapter_agent"); + assert.equal(accepted.admission.state, "promoted"); + assert.equal(accepted.admission.commandId, "cmd_hwlab_adapter"); + assert.equal(accepted.admission.dispatchIntentId, "dispatch_hwlab_adapter"); + assert.equal(accepted.admission.durableDispatch, true); - for (let index = 0; index < 50 && !traceStore.snapshot(traceId).events.some((event) => event.label === "agentrun:result:completed"); index += 1) await delay(10); - const projectedTrace = traceStore.snapshot(traceId); - assert.equal(projectedTrace.status, "completed"); - assert.ok(projectedTrace.events.some((event) => event.label === "agentrun:assistant:message")); - assert.ok(projectedTrace.events.some((event) => event.label === "agentrun:result:completed")); + await waitForTraceEvent( + traceStore, + traceId, + (event) => event.label === "agentrun:dispatch-intent:admitted", + "AgentRun command must be durably admitted before Kafka projection" + ); + await commitTestKafkaTerminal(runtimeStore, { + traceId, + sessionId: "ses_server-test-agentrun", + runId: "run_hwlab_adapter", + commandId: "cmd_hwlab_adapter", + text: "AgentRun adapter 已接管 HWLAB Code Agent。", + projectedSeq: 1, + sourceSeq: 5, + createdAt: "2026-06-01T00:00:02.000Z" + }); const acceptedTurn = await fetch(`http://127.0.0.1:${port}/v1/agent/turns/${encodeURIComponent(traceId)}`, { headers: { cookie: "hwlab_session=test-stub-session" } }); assert.equal(acceptedTurn.status, 200); @@ -713,51 +934,27 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte assert.equal(acceptedTurnPayload.assistantMessageId, accepted.assistantMessageId); const payload = await pollAgentResult(port, traceId); - validateCodeAgentChatSchema(payload); assert.equal(payload.status, "completed"); assert.equal(payload.turnId, traceId); assert.equal(payload.userMessageId, accepted.userMessageId); assert.equal(payload.assistantMessageId, accepted.assistantMessageId); - assert.equal(payload.provider, "deepseek"); - assert.equal(payload.backend, "agentrun-v01/deepseek"); - assert.equal(payload.infrastructureBackend, "agentrun-v01/deepseek"); - assert.equal(payload.capabilityLevel, "agentrun-v01-shared-code-agent-session"); - assert.equal(payload.sessionMode, "agentrun-v01-durable-session"); - assert.equal(payload.implementationType, "agentrun-v01-shared-execution-infra"); - assert.equal(payload.runner.kind, "agentrun-v01-shared-runner"); - assert.equal(payload.runner.provider, "deepseek"); - assert.equal(payload.runner.codexStdio, false); - assert.equal(payload.runner.delegatedToAgentRun, true); - assert.equal(payload.providerTrace.protocol, "agentrun-v01-jsonrpc"); - assert.equal(payload.providerTrace.command, "agentrun.v01.command.turn"); - assert.equal(payload.providerTrace.runnerKind, "agentrun-v01-shared-runner"); - assert.equal(payload.providerTrace.terminalStatus, "completed"); - assert.equal(payload.longLivedSessionGate.status, "pass"); - assert.equal(payload.longLivedSessionGate.provider, "deepseek"); - assert.equal(payload.longLivedSessionGate.codexStdio, false); - assert.equal(payload.longLivedSessionGate.delegatedToAgentRun, true); assert.equal(payload.agentRun.runId, "run_hwlab_adapter"); assert.equal(payload.agentRun.commandId, "cmd_hwlab_adapter"); - assert.equal(payload.agentRun.jobName, "agentrun-v01-runner-hwlab-adapter"); + assert.equal(payload.agentRun.dispatchIntentId, "dispatch_hwlab_adapter"); + assert.equal(payload.agentRun.runnerJobId, "rjob_hwlab_adapter"); + assert.equal(payload.agentRun.durableDispatch, true); + assert.equal(payload.projection.sourceRunId, "run_hwlab_adapter"); + assert.equal(payload.projection.sourceCommandId, "cmd_hwlab_adapter"); + assert.equal(payload.projection.projectionStatus, "caught-up"); const serializedPayload = JSON.stringify(payload); assert.equal(serializedPayload.includes("repo-owned-codex"), false); assert.equal(serializedPayload.includes("codex-app-server-stdio"), false); assert.equal(serializedPayload.includes("\"provider\":\"codex-stdio\""), false); assert.equal(serializedPayload.includes("\"codexStdio\":true"), false); - assert.match(payload.reply.content, /AgentRun adapter/u); - assert.ok(payload.runnerTrace.events.some((event) => event.label === "agentrun:backend:runner-job-created")); - const bundleEvent = payload.runnerTrace.events.find((event) => event.label === "agentrun:backend:resource-bundle-materialized"); - assert.equal(bundleEvent?.details?.kind, "gitbundle"); - assert.deepEqual(bundleEvent?.details?.bundles?.names, ["hwlab-tools", "hwlab-agent-skills"]); - assert.deepEqual(bundleEvent?.details?.tools?.names, ["hwpod", "hwpod-ctl", "hwpod-compiler", "unidesk-ssh", "hwlab-code-agent"]); - assert.deepEqual(bundleEvent?.details?.promptRefs?.names, ["hwlab-v02-runtime"]); - assert.deepEqual(bundleEvent?.details?.skillDirs?.names, ["hwpod-cli", "hwpod-ctl", "hwlab-agent-runtime", "hwlab-code-agent"]); - const promptEvent = payload.runnerTrace.events.find((event) => event.label === "agentrun:backend:initial-prompt-assembly"); - assert.equal(promptEvent?.details?.initialPromptInjected, true); - assert.equal(promptEvent?.details?.reason, "thread-start"); - assert.equal(promptEvent?.details?.initialPrompt?.promptRefCount, 1); - assert.equal(promptEvent?.details?.initialPrompt?.skillCount, 4); - assert.ok(calls.some((call) => call.path === "/api/v1/runs/run_hwlab_adapter/runner-jobs")); + assert.equal(payload.finalResponse.text, "AgentRun adapter 已接管 HWLAB Code Agent。"); + assert.equal(calls.some((call) => call.path === "/api/v1/runs/run_hwlab_adapter/runner-jobs"), false); + assert.equal(calls.some((call) => call.path === "/api/v1/runs/run_hwlab_adapter/events"), false); + assert.equal(calls.some((call) => call.path.endsWith("/result")), false); const inspectByTrace = await fetch(`http://127.0.0.1:${port}/v1/agent/chat/inspect?traceId=${traceId}`); assert.equal(inspectByTrace.status, 200); @@ -766,9 +963,9 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte assert.equal(inspectByTraceBody.latestTraceId, traceId); assert.equal(inspectByTraceBody.session.sessionId, "ses_server-test-agentrun"); assert.equal(inspectByTraceBody.session.conversationId, "cnv_server-test-agentrun"); - assert.equal(inspectByTraceBody.session.threadId, "019e8078-db67-7750-a5d9-1a99f3abd445"); + assert.equal(inspectByTraceBody.session.threadId, null); assert.equal(inspectByTraceBody.conversationFacts.conversationId, "cnv_server-test-agentrun"); - assert.equal(inspectByTraceBody.conversationFacts.threadId, "019e8078-db67-7750-a5d9-1a99f3abd445"); + assert.equal(inspectByTraceBody.conversationFacts.threadId, null); assert.equal(inspectByTraceBody.runnerTrace.traceId, traceId); assert.equal(inspectByTraceBody.valuesRedacted, true); assert.equal(JSON.stringify(inspectByTraceBody).includes("hwl_live_test_secret"), false); @@ -779,20 +976,18 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte const requestTraceEvent = traceBody.events.find((event) => event.label === "agentrun:request:accepted"); assert.equal(requestTraceEvent.eventType, "request"); assert.equal(requestTraceEvent.backend, "agentrun-v01/deepseek"); - const runnerJobTraceEvent = traceBody.events.find((event) => event.label === "agentrun:runner-job:created"); - assert.equal(runnerJobTraceEvent.eventType, "backend"); - assert.equal(runnerJobTraceEvent.backend, "agentrun-v01/deepseek"); - assert.ok(traceBody.events.some((event) => event.runId === "run_hwlab_adapter")); - assert.ok(traceBody.events.some((event) => event.label === "agentrun:assistant:message")); - assert.equal(traceBody.events.some((event) => event.details?.initialPromptInjected === true), true); - const commandTraceEvent = traceBody.events.find((event) => event.label === "item/commandExecution:completed"); - assert.equal(commandTraceEvent.eventType, "tool_call"); - assert.equal(commandTraceEvent.backend, "agentrun-v01/deepseek"); - assert.ok(Date.parse(commandTraceEvent.appendedAt) >= Date.parse(commandTraceEvent.createdAt)); - assert.equal(commandTraceEvent.toolName, "commandExecution"); - assert.equal(commandTraceEvent.createdAt, "2026-06-01T00:00:00.500Z"); - assert.match(commandTraceEvent.command, /hwpod inspect --dry-run/u); - assert.match(commandTraceEvent.stdoutSummary, /hwpod-cli\.plan/u); + assert.equal(requestTraceEvent.terminal, false); + assert.equal(requestTraceEvent.diagnosticOnly, true); + assert.equal(requestTraceEvent.authority, "trace-store-diagnostics"); + const dispatchTraceEvent = traceBody.events.find((event) => event.label === "agentrun:dispatch-intent:admitted"); + assert.equal(dispatchTraceEvent.eventType, "backend"); + assert.equal(dispatchTraceEvent.terminal, false); + assert.equal(dispatchTraceEvent.diagnosticOnly, true); + const projectionTraceEvent = traceBody.events.find((event) => event.label === "agentrun:kafka:assistant-final"); + assert.equal(projectionTraceEvent.authority, "workbench-projection"); + assert.equal(projectionTraceEvent.terminal, true); + assert.equal(traceBody.terminal, true); + assert.equal(traceBody.terminalAuthority, "workbench-projection"); conversationMessages.push( { id: "msg_693_user", @@ -860,28 +1055,31 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte }) }); assert.equal(second.status, 202); + await waitForTraceEvent( + traceStore, + secondTraceId, + (event) => event.label === "agentrun:runner-job:reused", + "second turn must reuse the admitted AgentRun runner" + ); + await commitTestKafkaTerminal(runtimeStore, { + traceId: secondTraceId, + sessionId: "ses_server-test-agentrun", + runId: "run_hwlab_adapter", + commandId: "cmd_hwlab_adapter_second", + text: "AgentRun adapter 复用已有 runner 完成第二轮。", + projectedSeq: 2, + sourceSeq: 9, + createdAt: "2026-06-01T00:00:05.000Z" + }); const secondPayload = await pollAgentResult(port, secondTraceId); - validateCodeAgentChatSchema(secondPayload); assert.equal(secondPayload.status, "completed"); - assert.equal(secondPayload.provider, "deepseek"); - assert.equal(secondPayload.backend, "agentrun-v01/deepseek"); - assert.equal(secondPayload.runner.kind, "agentrun-v01-shared-runner"); - assert.equal(secondPayload.runner.codexStdio, false); assert.equal(secondPayload.agentRun.runId, "run_hwlab_adapter"); assert.equal(secondPayload.agentRun.commandId, "cmd_hwlab_adapter_second"); - assert.equal(secondPayload.providerTrace.commandId, "cmd_hwlab_adapter_second"); - assert.equal(secondPayload.providerTrace.traceId, secondTraceId); - assert.equal(secondPayload.agentRun.providerTrace.commandId, "cmd_hwlab_adapter_second"); - assert.equal(secondPayload.agentRun.providerTrace.traceId, secondTraceId); - assert.equal(secondPayload.agentRun.jobName, "agentrun-v01-runner-hwlab-adapter-second"); - assert.equal(secondPayload.sessionReuse.reused, true); - assert.equal(secondPayload.agentRun.runnerJobCount, 1); - assert.match(secondPayload.reply.content, /复用已有 runner/u); - assert.ok(secondPayload.runnerTrace.events.some((event) => event.label === "agentrun:run:reused")); - assert.ok(secondPayload.runnerTrace.events.some((event) => event.label === "agentrun:runner-job:ensured")); - assert.equal(secondPayload.runnerTrace.events.some((event) => String(event.text ?? event.message ?? "").includes("旧 command 尾部")), false); + assert.equal(secondPayload.agentRun.runnerJobCount, 0); + assert.match(secondPayload.finalResponse.text, /复用已有 runner/u); + assert.equal(secondPayload.projection.sourceCommandId, "cmd_hwlab_adapter_second"); assert.equal(calls.filter((call) => call.method === "POST" && call.path === "/api/v1/runs").length, 1); - assert.equal(calls.filter((call) => call.method === "POST" && call.path === "/api/v1/runs/run_hwlab_adapter/runner-jobs").length, 2); + assert.equal(calls.filter((call) => call.method === "POST" && call.path === "/api/v1/runs/run_hwlab_adapter/runner-jobs").length, 0); assert.equal(calls.filter((call) => call.method === "POST" && call.path === "/api/v1/runs/run_hwlab_adapter/commands").length, 2); } finally { await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); @@ -889,7 +1087,7 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte } }); -test("cloud api keeps admitted user message when billing preflight fails before AgentRun dispatch (#1619)", async () => { +test("cloud api rejects billing failure before durable AgentRun admission without false running state (#1619)", async () => { const traceId = "trc_issue1619_billing_after_admission"; const sessionId = "ses_issue1619_billing_after_admission"; const conversationId = "cnv_issue1619_billing_after_admission"; @@ -923,7 +1121,7 @@ test("cloud api keeps admitted user message when billing preflight fails before configured: true, async billingPreflight(body) { billingCalls.push(body); - assert.ok(ownerWrites.some((write) => write.traceId === traceId && write.status === "running"), "durable admission must be recorded before billing preflight"); + assert.equal(ownerWrites.some((write) => write.traceId === traceId && write.status === "running"), false, "billing must run before UI-authoritative admission"); return { ok: false, status: 503, error: { code: "billing_preflight_unavailable", message: "billing preflight unavailable" } }; } }, @@ -962,269 +1160,31 @@ test("cloud api keeps admitted user message when billing preflight fails before message: "测试一下和 cpython 对比的性能" }) }); - assert.equal(submit.status, 202); - const accepted = await submit.json(); - assert.equal(accepted.accepted, true); - assert.equal(accepted.traceId, traceId); - - const payload = await pollAgentResult(port, traceId); - validateCodeAgentChatSchema(payload); - assert.equal(payload.status, "failed"); - assert.equal(payload.error.code, "billing_preflight_unavailable"); - assert.match(payload.finalResponse.text, /billing preflight unavailable/u); + assert.equal(submit.status, 503); + const rejected = await submit.json(); + assert.equal(rejected.accepted, false); + assert.equal(rejected.status, "failed"); + assert.equal(rejected.traceId, traceId); + assert.equal(rejected.error.code, "billing_preflight_unavailable"); assert.equal(billingCalls.length, 1); assert.equal(agentRunCalls.length, 0); - assert.ok(ownerWrites.some((write) => write.traceId === traceId && write.status === "running")); - assert.ok(ownerWrites.some((write) => write.traceId === traceId && write.status === "failed")); + assert.equal(ownerWrites.some((write) => write.traceId === traceId && ["running", "failed"].includes(write.status)), false); const stored = ownerSessions.get(sessionId); const userMessage = stored?.session?.messages?.find((message) => message.role === "user"); - assert.match(userMessage?.text ?? "", /cpython/u); - - const turn = await fetch(`http://127.0.0.1:${port}/v1/agent/turns/${traceId}`, { headers: { cookie: "hwlab_session=test-stub-session" } }); - assert.equal(turn.status, 200); - const turnBody = await turn.json(); - assert.equal(turnBody.status, "failed"); - assert.equal(turnBody.error.code, "billing_preflight_unavailable"); + assert.equal(userMessage, undefined); + const diagnosticTrace = traceStore.snapshot(traceId); + assert.equal(diagnosticTrace.events.some((event) => event.label === "turn:admitting"), true); + assert.equal(diagnosticTrace.events.some((event) => event.terminal === true), false); } finally { await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); await new Promise((resolve, reject) => agentRunServer.close((error) => (error ? reject(error) : resolve()))); } }); -test("AgentRun sync converts terminal command result even when run remains claimed (#1555)", async () => { - const calls = []; - const traceId = "trc_issue1555_terminal_command"; - const runId = "run_issue1555_claimed"; - const commandId = "cmd_issue1555_completed"; - const finalText = [ - "已新增并实际运行：", - "", - "- Go：`go test ./...` 保留两个空格", - "- Rust：`cargo test --release`", - "", - "```sh", - "go test ./...", - "cargo test --release", - "```", - "", - "| benchmark | Go | Rust |", - "|---|---:|---:|", - "| fib | 1.2ms | 1.0ms |" - ].join("\n"); - const agentRunServer = createHttpServer(async (request, response) => { - const url = new URL(request.url || "/", "http://127.0.0.1"); - calls.push({ method: request.method, path: url.pathname, search: url.search }); - const send = (data) => { - response.writeHead(200, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: true, data, traceId: "trc_fake_issue1555" })}\n`); - }; - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/events`) { - return send({ items: [ - { id: "evt_issue1555_assistant", runId, seq: 138, type: "assistant_message", payload: { commandId, text: finalText, final: true, replyAuthority: true }, createdAt: "2026-06-18T00:00:00.000Z" }, - { id: "evt_issue1555_terminal", runId, seq: 139, type: "terminal_status", payload: { commandId, terminalStatus: "completed" }, createdAt: "2026-06-18T00:00:00.000Z" } - ] }); - } - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/commands`) { - return send({ items: [ - { id: commandId, runId, state: "completed", type: "turn", seq: 1, idempotencyKey: traceId, payload: { traceId, conversationId: "cnv_issue1555", hwlabSessionId: "ses_issue1555", threadId: "thread_issue1555" } } - ] }); - } - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/commands/${commandId}/result`) { - return send({ - runId, - commandId, - attemptId: "attempt_issue1555", - runnerId: "runner_issue1555", - jobName: "agentrun-v01-runner-issue1555", - namespace: "agentrun-v02", - status: "completed", - runStatus: "claimed", - commandState: "completed", - terminalStatus: "completed", - completed: true, - reply: finalText, - lastSeq: 139, - eventCount: 139, - sessionRef: { sessionId: "ses_issue1555", conversationId: "cnv_issue1555", threadId: "thread_issue1555" } - }); - } - response.writeHead(404, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: false, message: `unexpected ${request.method} ${url.pathname}` })}\n`); - }); - await new Promise((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve)); - const agentRunPort = agentRunServer.address().port; - const traceStore = createCodeAgentTraceStore(); - try { - const currentResult = { - ok: true, - accepted: true, - shortConnection: true, - status: "running", - traceId, - conversationId: "cnv_issue1555", - sessionId: "ses_issue1555", - threadId: "thread_issue1555", - agentRun: { - adapter: "agentrun-v01", - managerUrl: `http://127.0.0.1:${agentRunPort}`, - runId, - commandId, - status: "pending", - runStatus: "claimed", - commandState: "pending", - terminalStatus: null, - lastSeq: 0, - valuesPrinted: false - }, - valuesPrinted: false - }; - const synced = await syncAgentRunChatResult({ - traceId, - currentResult, - traceStore, - options: { - env: { - AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, - HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1" - } - } - }); - assert.equal(synced.resultSynced, true); - assert.equal(synced.result.status, "completed"); - assert.equal(synced.result.agentRun.runStatus, "claimed"); - assert.equal(synced.result.agentRun.commandState, "completed"); - assert.equal(synced.result.agentRun.terminalStatus, "completed"); - assert.equal(synced.result.finalResponse.text, finalText); - assert.equal(synced.result.reply.content, finalText); - assert.ok((synced.result.finalResponse.text.match(/\n/gu) ?? []).length > 8); - assert.ok(synced.result.finalResponse.text.includes("`go test ./...` 保留两个空格")); - const assistantEvent = traceStore.snapshot(traceId).events.find((event) => event.label === "agentrun:assistant:message"); - assert.equal(assistantEvent?.message, finalText); - assert.deepEqual(calls.map((call) => call.path), [ - `/api/v1/runs/${runId}/commands`, - `/api/v1/runs/${runId}/events`, - `/api/v1/runs/${runId}/commands/${commandId}/result` - ]); - assert.ok(traceStore.snapshot(traceId).events.some((event) => event.label === "agentrun:result:completed")); - } finally { - await new Promise((resolve, reject) => agentRunServer.close((error) => (error ? reject(error) : resolve()))); - } -}); - -test("AgentRun sync seals completed final response from authoritative terminal assistant trace event (#1629)", async () => { - const calls = []; - const traceId = "trc_issue1629_terminal_assistant_final"; - const runId = "run_issue1629_trace_final"; - const commandId = "cmd_issue1629_trace_final"; - const finalText = [ - "全部六份数据到手！下面是完整的六语言终极性能对比：", - "", - "| language | runtime | status |", - "|---|---|---|", - "| Lua | LuaJIT | pass |" - ].join("\n"); - const agentRunServer = createHttpServer(async (request, response) => { - const url = new URL(request.url || "/", "http://127.0.0.1"); - calls.push({ method: request.method, path: url.pathname, search: url.search }); - const send = (data) => { - response.writeHead(200, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: true, data, traceId: "trc_fake_issue1629" })}\n`); - }; - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/commands`) { - return send({ items: [ - { id: commandId, runId, state: "completed", type: "turn", seq: 1, idempotencyKey: traceId, payload: { traceId, conversationId: "cnv_issue1629", hwlabSessionId: "ses_issue1629", threadId: "thread_issue1629" } } - ] }); - } - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/events`) { - return send({ items: [ - { id: "evt_issue1629_progress", runId, seq: 21, type: "assistant_message", payload: { commandId, text: "正在补 Lua 基准测试。" }, createdAt: "2026-06-19T15:47:13.000Z" }, - { id: "evt_issue1629_final", runId, seq: 28, type: "assistant_message", payload: { commandId, text: finalText, final: true, replyAuthority: true }, createdAt: "2026-06-19T15:47:28.000Z" }, - { id: "evt_issue1629_terminal", runId, seq: 29, type: "terminal_status", payload: { commandId, terminalStatus: "completed" }, createdAt: "2026-06-19T15:47:29.000Z" } - ] }); - } - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/commands/${commandId}/result`) { - return send({ - runId, - commandId, - attemptId: "attempt_issue1629", - runnerId: "runner_issue1629", - jobName: "agentrun-v01-runner-issue1629", - namespace: "agentrun-v01", - status: "completed", - runStatus: "completed", - commandState: "completed", - terminalStatus: "completed", - completed: true, - reply: null, - finalResponse: null, - lastSeq: 29, - eventCount: 29, - sessionRef: { sessionId: "ses_issue1629", conversationId: "cnv_issue1629", threadId: "thread_issue1629" } - }); - } - response.writeHead(404, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: false, message: `unexpected ${request.method} ${url.pathname}` })}\n`); - }); - await new Promise((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve)); - const agentRunPort = agentRunServer.address().port; - const traceStore = createCodeAgentTraceStore(); - try { - const currentResult = { - ok: true, - accepted: true, - shortConnection: true, - status: "running", - traceId, - conversationId: "cnv_issue1629", - sessionId: "ses_issue1629", - threadId: "thread_issue1629", - agentRun: { - adapter: "agentrun-v01", - managerUrl: `http://127.0.0.1:${agentRunPort}`, - runId, - commandId, - status: "running", - runStatus: "running", - commandState: "running", - terminalStatus: null, - lastSeq: 0, - valuesPrinted: false - }, - valuesPrinted: false - }; - const synced = await syncAgentRunChatResult({ - traceId, - currentResult, - traceStore, - options: { - env: { - AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, - HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1" - } - } - }); - assert.equal(synced.resultSynced, true); - assert.equal(synced.result.status, "completed"); - assert.equal(synced.result.finalResponse.text, finalText); - assert.equal(synced.result.assistantText ?? synced.result.finalResponse.text, finalText); - assert.equal(synced.result.reply.content, finalText); - assert.equal(synced.result.traceSummary.finalAssistantRow.textChars, finalText.length); - const assistantEvent = synced.result.runnerTrace.events.find((event) => event.label === "agentrun:assistant:message" && event.message === finalText); - assert.equal(assistantEvent?.message, finalText); - assert.deepEqual(calls.map((call) => call.path), [ - `/api/v1/runs/${runId}/commands`, - `/api/v1/runs/${runId}/events`, - `/api/v1/runs/${runId}/commands/${commandId}/result` - ]); - } finally { - await new Promise((resolve, reject) => agentRunServer.close((error) => (error ? reject(error) : resolve()))); - } -}); - test("cloud api AgentRun adapter reports persistent thread resume when a completed run needs a new runner", async () => { const calls = []; const hwlabSessionId = "ses_server-test-thread-resume"; - const agentRunSessionId = "ses_agentrun_deepseek_server_test_thread_resume"; + const agentRunSessionId = "ses_agentrun_server_test_thread_resume"; const conversationId = "cnv_server-test-thread-resume"; const threadId = "019e90e0-9535-7130-a894-47ef4e127206"; const ownerSessions = new Map([[hwlabSessionId, testAgentSessionRecord({ @@ -1275,7 +1235,7 @@ test("cloud api AgentRun adapter reports persistent thread resume when a complet if (request.method === "POST" && url.pathname === "/api/v1/sessions") { assert.equal(body.sessionId, agentRunSessionId); assert.equal(body.backendProfile, "deepseek"); - return send({ sessionId: body.sessionId, backendProfile: body.backendProfile }); + return send({ sessionId: body.sessionId, backendProfile: body.backendProfile, threadId }); } if (request.method === "POST" && url.pathname === "/api/v1/runs") { assert.equal(body.sessionRef.sessionId, agentRunSessionId); @@ -1288,9 +1248,22 @@ test("cloud api AgentRun adapter reports persistent thread resume when a complet assert.equal(body.type, "turn"); assert.equal(body.payload.sessionId, agentRunSessionId); assert.equal(body.payload.hwlabSessionId, hwlabSessionId); - assert.equal(body.payload.threadId, null); + assert.equal(body.payload.threadId, threadId); assert.match(body.payload.prompt, /ISSUE812_RESUME/u); - return send({ id: "cmd_issue812_resume", runId: "run_issue812_resume", state: "pending", type: "turn", seq: 1 }); + return send({ + id: "cmd_issue812_resume", + runId: "run_issue812_resume", + state: "pending", + type: "turn", + seq: 1, + dispatchIntent: { + id: "dispatch_issue812_resume", + state: "pending", + runnerJobId: "rjob_issue812_resume", + attemptCount: 0, + durable: true + } + }); } if (request.method === "GET" && url.pathname === "/api/v1/runs/run_issue812_resume/commands") { return send({ items: [ @@ -1348,8 +1321,11 @@ test("cloud api AgentRun adapter reports persistent thread resume when a complet record.session.agentRun.managerUrl = `http://127.0.0.1:${agentRunPort}`; } const traceStore = createCodeAgentTraceStore(); + const runtimeStore = createCloudRuntimeStore(); const server = createCloudApiServer({ traceStore, + runtimeStore, + workbenchRuntime: runtimeStore, env: { HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, @@ -1399,38 +1375,47 @@ test("cloud api AgentRun adapter reports persistent thread resume when a complet }); assert.equal(submit.status, 202); + await waitForTraceEvent(traceStore, traceId, (event) => event.label === "agentrun:dispatch-intent:admitted"); + await commitTestKafkaTerminal(runtimeStore, { + traceId, + sessionId: hwlabSessionId, + runId: "run_issue812_resume", + commandId: "cmd_issue812_resume", + text: "ISSUE812_RESUME_OK", + projectedSeq: 1, + sourceSeq: 4, + createdAt: "2026-06-04T00:00:02.000Z" + }); + const payload = await pollAgentResult(port, traceId); - validateCodeAgentChatSchema(payload); assert.equal(payload.status, "completed"); assert.equal(payload.sessionId, hwlabSessionId); assert.equal(payload.threadId, threadId); assert.equal(payload.agentRun.runId, "run_issue812_resume"); assert.equal(payload.agentRun.reused, false); assert.equal(payload.agentRun.runnerReused, false); - assert.equal(payload.agentRun.threadReused, false); - assert.equal(payload.agentRun.persistentResume, false); - assert.equal(payload.sessionReuse.threadId, threadId); - assert.equal(payload.sessionReuse.reused, true); - assert.equal(payload.sessionReuse.status, "thread-resumed"); - assert.equal(payload.sessionReuse.runnerReused, false); - assert.equal(payload.sessionReuse.threadReused, true); - assert.equal(payload.sessionReuse.persistentResume, true); - assert.equal(payload.reply.content, "ISSUE812_RESUME_OK"); + assert.equal(payload.agentRun.threadReused, true); + assert.equal(payload.agentRun.persistentResume, true); + assert.equal(payload.finalResponse.text, "ISSUE812_RESUME_OK"); + assert.equal(payload.agentRun.dispatchIntentId, "dispatch_issue812_resume"); assert.equal(calls.some((call) => call.method === "GET" && call.path === "/api/v1/runs/run_issue812_previous"), true); assert.equal(calls.filter((call) => call.method === "POST" && call.path === "/api/v1/runs").length, 1); - assert.equal(calls.filter((call) => call.method === "POST" && call.path === "/api/v1/runs/run_issue812_resume/runner-jobs").length, 1); + assert.equal(calls.filter((call) => call.method === "POST" && call.path === "/api/v1/runs/run_issue812_resume/runner-jobs").length, 0); + assert.equal(calls.some((call) => call.path.endsWith("/result")), false); } finally { await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); await new Promise((resolve, reject) => agentRunServer.close((error) => (error ? reject(error) : resolve()))); } }); -test("cloud api AgentRun adapter exposes invalid tool-call attribution in result payload", async () => { +test("cloud api AgentRun adapter exposes invalid tool-call attribution through Kafka projection trace", async () => { + const calls = []; const agentRunServer = createHttpServer(async (request, response) => { const url = new URL(request.url || "/", "http://127.0.0.1"); const chunks = []; for await (const chunk of request) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); const body = chunks.length ? JSON.parse(Buffer.concat(chunks).toString("utf8")) : null; + calls.push({ method: request.method, path: url.pathname, body }); const send = (data) => { response.writeHead(200, { "content-type": "application/json" }); response.end(`${JSON.stringify({ ok: true, data, traceId: "trc_fake_agentrun_invalid_tool" })}\n`); @@ -1439,7 +1424,20 @@ test("cloud api AgentRun adapter exposes invalid tool-call attribution in result return send({ id: "run_invalid_tool", status: "pending", backendProfile: "deepseek", sessionRef: body.sessionRef, resourceBundleRef: body.resourceBundleRef }); } if (request.method === "POST" && url.pathname === "/api/v1/runs/run_invalid_tool/commands") { - return send({ id: "cmd_invalid_tool", runId: "run_invalid_tool", state: "pending", type: "turn", seq: 1 }); + return send({ + id: "cmd_invalid_tool", + runId: "run_invalid_tool", + state: "pending", + type: "turn", + seq: 1, + dispatchIntent: { + id: "dispatch_invalid_tool", + state: "pending", + runnerJobId: "rjob_invalid_tool", + attemptCount: 0, + durable: true + } + }); } if (request.method === "GET" && url.pathname === "/api/v1/runs/run_invalid_tool/commands") { return send({ items: [ @@ -1494,7 +1492,12 @@ test("cloud api AgentRun adapter exposes invalid tool-call attribution in result conversationId: "cnv_invalid_tool", status: "idle" })]]); + const traceStore = createCodeAgentTraceStore(); + const runtimeStore = createCloudRuntimeStore(); const server = createCloudApiServer({ + traceStore, + runtimeStore, + workbenchRuntime: runtimeStore, env: { HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, @@ -1531,27 +1534,35 @@ test("cloud api AgentRun adapter exposes invalid tool-call attribution in result body: JSON.stringify({ conversationId: "cnv_invalid_tool", sessionId: "ses_server-test-invalid-tool", message: "invalid tool-call attribution" }) }); assert.equal(submit.status, 202); + await waitForTraceEvent(traceStore, traceId, (event) => event.label === "agentrun:dispatch-intent:admitted"); + await commitTestKafkaTerminal(runtimeStore, { + traceId, + sessionId: "ses_server-test-invalid-tool", + runId: "run_invalid_tool", + commandId: "cmd_invalid_tool", + status: "failed", + text: "invalid params, invalid function arguments json string, tool_call_id: call_function_selftest_2 (2013)", + failureKind: "provider-invalid-tool-call", + projectedSeq: 1, + sourceSeq: 2, + createdAt: "2026-06-02T00:00:01.000Z" + }); const payload = await pollAgentResult(port, traceId); - validateCodeAgentChatSchema(payload); assert.equal(payload.status, "failed"); - assert.equal(payload.providerTrace.failureKind, "provider-invalid-tool-call"); - assert.equal(payload.error.code, "provider-invalid-tool-call"); - assert.equal(payload.error.category, "provider_invalid_tool_call"); - assert.match(payload.error.userMessage, /无效 tool-call arguments JSON/u); - assert.match(payload.error.userMessage, /AgentRun\/provider/u); - assert.match(payload.error.userMessage, /不是 HWPOD/u); assert.equal(payload.finalResponse.status, "failed"); assert.match(payload.finalResponse.text, /invalid function arguments json string/u); - assert.equal(payload.traceSummary.terminalStatus, "failed"); - assert.match(payload.traceSummary.finalAssistantRow.textPreview, /invalid function arguments json string/u); - assert.equal(payload.blocker?.category ?? payload.error.category, "provider_invalid_tool_call"); - assert.match(payload.blocker?.summary ?? payload.error.userMessage, /invalid function arguments json string/u); - assert.equal(payload.session.status, "failed"); - assert.equal(payload.session.lifecycle.requiresNewSession, false); - assert.equal(payload.sessionReuse.threadId, "thread_invalid_tool"); - assert.equal(payload.sessionReuse.status, "failed-resumable"); assert.equal(payload.agentRun.reuseEligible, true); - assert.equal(payload.reuseEligible, true); + assert.equal(payload.projection.sourceRunId, "run_invalid_tool"); + assert.equal(payload.projection.sourceCommandId, "cmd_invalid_tool"); + const trace = await fetch(`http://127.0.0.1:${port}/v1/agent/traces/${traceId}`); + assert.equal(trace.status, 200); + const traceBody = await trace.json(); + const projectedFailure = traceBody.events.find((event) => event.label === "agentrun:kafka:assistant-final"); + assert.equal(projectedFailure.authority, "workbench-projection"); + assert.equal(projectedFailure.failureKind, "provider-invalid-tool-call"); + assert.equal(projectedFailure.terminal, true); + assert.equal(calls.some((call) => call.path.endsWith("/result")), false); + assert.equal(calls.some((call) => call.path.endsWith("/events")), false); const serializedPayload = JSON.stringify(payload); assert.equal(serializedPayload.includes("repo-owned-codex"), false); assert.equal(serializedPayload.includes("codex-app-server-stdio"), false); @@ -1562,7 +1573,7 @@ test("cloud api AgentRun adapter exposes invalid tool-call attribution in result } }); -test("cloud api AgentRun adapter retries evicted session storage with a fresh session", async () => { +test("cloud api surfaces manager-side evicted-session recovery through Kafka projection", async () => { const calls = []; const agentRunServer = createHttpServer(async (request, response) => { const url = new URL(request.url || "/", "http://127.0.0.1"); @@ -1686,7 +1697,12 @@ test("cloud api AgentRun adapter retries evicted session storage with a fresh se threadId: "thread_evicted", status: "idle" })]]); + const traceStore = createCodeAgentTraceStore(); + const runtimeStore = createCloudRuntimeStore(); const server = createCloudApiServer({ + traceStore, + runtimeStore, + workbenchRuntime: runtimeStore, env: { HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, @@ -1723,20 +1739,27 @@ test("cloud api AgentRun adapter retries evicted session storage with a fresh se body: JSON.stringify({ conversationId: "cnv_evicted", sessionId: "ses_server-test-evicted", threadId: "thread_evicted", message: "resume evicted session" }) }); assert.equal(submit.status, 202); + await waitForTraceEvent(traceStore, traceId, (event) => event.label === "agentrun:dispatch-intent:admitted"); + await commitTestKafkaTerminal(runtimeStore, { + traceId, + sessionId: "ses_server-test-evicted", + runId: "run_evicted_retry", + commandId: "cmd_evicted_retry", + text: "EVICTED_RETRY_OK", + projectedSeq: 1, + sourceSeq: 3, + createdAt: "2026-06-02T00:00:04.000Z" + }); const payload = await pollAgentResult(port, traceId); - validateCodeAgentChatSchema(payload); assert.equal(payload.status, "completed"); - assert.equal(payload.reply.content, "EVICTED_RETRY_OK"); - assert.equal(payload.agentRun.runId, "run_evicted_retry"); - assert.equal(payload.agentRun.commandId, "cmd_evicted_retry"); - assert.equal(payload.agentRun.freshSessionRetryOf.runId, "run_evicted"); - assert.equal(payload.agentRun.freshSessionRetryOf.commandId, "cmd_evicted"); - assert.match(payload.agentRun.sessionId, /-reset-/u); - assert.equal(payload.session.threadId, "thread_evicted_retry"); - assert.equal(payload.sessionReuse.status, "thread-resumed"); - assert.equal(payload.error, undefined); - assert.equal(ownerSessions.get("ses_server-test-evicted")?.status, "completed"); - assert.equal(calls.some((call) => call.method === "POST" && call.path === "/api/v1/runs/run_evicted_retry/commands"), true); + assert.equal(payload.finalResponse.text, "EVICTED_RETRY_OK"); + assert.equal(payload.agentRun.runId, "run_evicted"); + assert.equal(payload.agentRun.commandId, "cmd_evicted"); + assert.equal(payload.projection.sourceRunId, "run_evicted_retry"); + assert.equal(payload.projection.sourceCommandId, "cmd_evicted_retry"); + assert.equal(calls.some((call) => call.method === "POST" && call.path === "/api/v1/runs/run_evicted_retry/commands"), false); + assert.equal(calls.some((call) => call.path.endsWith("/result")), false); + assert.equal(calls.some((call) => call.path.endsWith("/events")), false); assert.equal(calls.some((call) => call.path.endsWith("/runner-jobs")), false); } finally { await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); @@ -1831,7 +1854,12 @@ test("cloud api AgentRun adapter maps minimax-m3 provider profile to AgentRun ba conversationId: "cnv_server-test-minimax-m3", status: "idle" })]]); + const traceStore = createCodeAgentTraceStore(); + const runtimeStore = createCloudRuntimeStore(); const server = createCloudApiServer({ + traceStore, + runtimeStore, + workbenchRuntime: runtimeStore, env: { HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, @@ -1876,23 +1904,31 @@ test("cloud api AgentRun adapter maps minimax-m3 provider profile to AgentRun ba const accepted = await submit.json(); assert.equal(accepted.shortConnection, true); + await waitForTraceEvent(traceStore, traceId, (event) => event.label === "agentrun:dispatch-intent:admitted"); + await commitTestKafkaTerminal(runtimeStore, { + traceId, + sessionId: "ses_server-test-minimax-m3", + runId: "run_hwlab_minimax_m3", + commandId: "cmd_hwlab_minimax_m3", + text: "AGENTRUN_MINIMAX_M3_OK", + projectedSeq: 1, + sourceSeq: 3, + createdAt: "2026-06-02T00:00:02.000Z" + }); + const payload = await pollAgentResult(port, traceId); - validateCodeAgentChatSchema(payload); assert.equal(payload.status, "completed"); - assert.equal(payload.provider, "minimax-m3"); - assert.equal(payload.backend, "agentrun-v01/minimax-m3"); - assert.equal(payload.infrastructureBackend, "agentrun-v01/minimax-m3"); - assert.equal(payload.runner.kind, "agentrun-v01-shared-runner"); - assert.equal(payload.runner.provider, "minimax-m3"); - assert.equal(payload.runner.codexStdio, false); assert.equal(payload.agentRun.backendProfile, "minimax-m3"); assert.equal(payload.sessionId, "ses_server-test-minimax-m3"); assert.equal(payload.agentRun.sessionId, "ses_agentrun_server_test_minimax_m3"); - assert.equal(payload.providerTrace.backendProfile, "minimax-m3"); - assert.equal(payload.agentRun.jobName, "agentrun-v01-runner-hwlab-minimax-m3"); - assert.equal(payload.reply.content, "AGENTRUN_MINIMAX_M3_OK"); + assert.equal(payload.agentRun.dispatchIntentId, "dispatch_hwlab_minimax_m3"); + assert.equal(payload.finalResponse.text, "AGENTRUN_MINIMAX_M3_OK"); + assert.equal(payload.projection.sourceRunId, "run_hwlab_minimax_m3"); + assert.equal(payload.projection.sourceCommandId, "cmd_hwlab_minimax_m3"); assert.equal(calls.filter((call) => call.method === "POST" && call.path === "/api/v1/runs").length, 1); assert.equal(calls.some((call) => call.path.endsWith("/runner-jobs")), false); + assert.equal(calls.some((call) => call.path.endsWith("/events")), false); + assert.equal(calls.some((call) => call.path.endsWith("/result")), false); } finally { await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); await new Promise((resolve, reject) => agentRunServer.close((error) => (error ? reject(error) : resolve()))); diff --git a/internal/cloud/server-code-agent-admission-http.ts b/internal/cloud/server-code-agent-admission-http.ts new file mode 100644 index 00000000..edb557f9 --- /dev/null +++ b/internal/cloud/server-code-agent-admission-http.ts @@ -0,0 +1,1700 @@ +// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-0102 Agent编排 draft-2026-06-17-r0; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. +// Responsibility: Code Agent HTTP session and turn admission. + +import { createHash, randomUUID } from "node:crypto"; + +import { M3_IO_CONTROL_ROUTE, M3_STATUS_ROUTE, describeM3StatusLive, handleM3IoControl } from "./m3-io-control.ts"; +import { + agentRunSessionEvidence, + cancelAgentRunChatTurn, + codeAgentAgentRunAdapterEnabled, + initialAgentRunChatResult, + loadPersistedAgentRunResult, + steerAgentRunChatTurn, + submitAgentRunChatTurn +} from "./code-agent-agentrun-adapter.ts"; +import { createCodeAgentErrorPayload, handleCodeAgentChat } from "./code-agent-chat.ts"; +import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; +import { codeAgentSessionLifecycleSummary } from "./code-agent-session-lifecycle.ts"; +import { messageAuthorityTextValue } from "./code-agent-agentrun-prompt.ts"; +import { promoteWorkbenchTurnAdmission as persistWorkbenchTurnAdmissionPromotion, writeWorkbenchSessionAdmissionFact } from "./workbench-projection-writer.ts"; +import { createWorkbenchReadModel } from "./workbench-read-model.ts"; +import { + workbenchRealtimeCapabilities +} from "./workbench-realtime-capabilities.ts"; +import { codeAgentOtelTraceFields, emitCodeAgentOtelSpan } from "./otel-trace.ts"; +import { + firstHeaderValue, + getHeader, + parsePositiveInteger, + positiveInteger, + readBody, + safeConversationId, + safeOpaqueId, + safeSessionId, + safeTraceId, + sendJson, + truthyFlag +} from "./server-http-utils.ts"; + +import { + normalizeCodeAgentProviderProfile, + firstNonEmptyValue, + recordCodeAgentSessionInputFact, + buildCodeAgentSessionInputFact, + stableCodeAgentInputId, + codeAgentInputDelivery, + codeAgentInputStatus, + codeAgentResultTimingFields, + firstTimestampIso, + firstLatestTimestampIso, + codeAgentChatShortConnectionRequested, + preflightCodeAgentBilling, + recordCodeAgentBillingUsage, + finalizeCodeAgentBillingUsage, + releaseCodeAgentBillingReservation, + withCodeAgentBillingReservation, + codeAgentBillingEnabled, + codeAgentBillingMetadata, + sanitizeBillingReservation, + sanitizeBillingRecord, + sanitizeBillingRelease, + codeAgentUsedTokens, + normalizeTurnStatus, + recordCodeAgentConversationFact, + recordCodeAgentSessionOwner, + codeAgentOwnerStatusForResult, + textValue, + timestampIsoOrNow, + timestampIsoOrNull, + codeAgentSessionOwnerEvidence, + codeAgentContinuationEvidence, + codeAgentFreshContinuationFailure, + normalizeCodeAgentFailureKind, + codeAgentFailureRequiresFreshContinuation, + codeAgentProviderProfileEvidence, + codeAgentTraceResultEvidence, + codeAgentPromptMetadata, + codeAgentPromptMetadataFromText, + codeAgentPromptMetadataFromParts, + codeAgentPromptFields, + codeAgentPromptId, + clippedPromptText, + sha256Text, + compactCodeAgentObject, + codeAgentBillingReservationEvidence, + codeAgentConversationMessagesEvidence, + boundedConversationMessageText, + conversationText, + conversationTextRaw, + codeAgentMessageTraceSuffix, + codeAgentTurnLifecycleFields, + codeAgentLifecycleMessageId, + safeTurnId, + safeMessageId, + codeAgentConversationAgentMessageStatus, + codeAgentPayloadHasSealedFinalResponse, + codeAgentConversationRunnerTraceEvidence, + codeAgentFinalResponseEvidence, + codeAgentFinalResponseText, + codeAgentTraceSummaryEvidence, + annotateOwner, + canAccessOwnedResult, + isCodeAgentResultCanceled, + codeAgentCancelScopeMismatch, + cancelExpectedValues, + cancelScopeFieldMismatch, + cancelBlockedPayload, + isTraceCommandTerminalStatus, + traceSnapshotWithTerminalEvidence, + traceSnapshotWithAttachedTerminalEvidence, + agentRunTerminalTraceEvidence, + agentRunTerminalFinalResponse, + traceRetentionSummary, + numberOrNull, + createCodeAgentChatResultStore, + compactCodeAgentChatResultPayload, + terminalEvidencePayload, + compactRunnerTraceForResult, + traceEventLabel, + resultTraceEventLimit, + createCodeAgentM3HwlabApiRequestJson, + recordCodeAgentProjectionMetrics, + recordCodeAgentTurnReadMetric, + turnReadDegradedReason, + responseBodyBytes, + statusClassLabel, + sourceLatestSeqFromResult, + sourceLatestAtFromResult, + terminalSourceAtFromResult, + projectionReason, + nowMs, + uniqueStrings +} from "./server-code-agent-http-support.ts"; + +const DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT = 120; +const DEFAULT_CODE_AGENT_TRACE_PAGE_LIMIT = 100; +const MAX_CODE_AGENT_TRACE_PAGE_LIMIT = 100; +const DEFAULT_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS = 2500; +const DEFAULT_CODE_AGENT_PROJECT_ID = "prj_hwpod_workbench"; +const DEFAULT_CODE_AGENT_PROVIDER_PROFILE = "deepseek"; +const CODE_AGENT_TERMINAL_STATUSES = new Set(["completed", "failed", "blocked", "timeout", "cancelled", "canceled"]); +const CODE_AGENT_PROVIDER_PROFILE_ALIASES = Object.freeze({ codex: "codex-api" }); +const CODE_AGENT_PROVIDER_PROFILE_ID_PATTERN = /^[a-z0-9][a-z0-9.-]{0,63}$/u; +const CODE_AGENT_PROVIDER_PROFILE_LABELS = Object.freeze({ + "deepseek": "DeepSeek", + "codex-api": "Codex API", + "gpt.pika": "gpt.pika", + "minimax-m3": "MiniMax-M3 via AgentRun", + "runtime-default": "运行默认" +}); +const DEFAULT_CODE_AGENT_CODEX_API_MODEL = "gpt-5.5"; +const DEFAULT_CODE_AGENT_CODEX_API_BASE_URL = "http://127.0.0.1:49280/responses"; +const DEFAULT_CODE_AGENT_DEEPSEEK_MODEL = "deepseek-chat"; +const DEFAULT_CODE_AGENT_MINIMAX_M3_MODEL = "MiniMax-M3"; + +export async function handleCodeAgentChatHttp(request, response, options) { + const perf = options.backendPerformance; + const body = perf ? await perf.measure("body_read", () => readBody(request, options.bodyLimitBytes)) : await readBody(request, options.bodyLimitBytes); + let params = {}; + + try { + params = body ? JSON.parse(body) : {}; + } catch (error) { + sendJson(response, 400, { + ...createCodeAgentErrorPayload({ + code: "parse_error", + message: "Invalid JSON body", + reason: error.message, + traceId: getHeader(request, "x-trace-id") || "trc_unassigned", + model: options.env?.HWLAB_CODE_AGENT_MODEL || options.env?.OPENAI_MODEL || process.env.HWLAB_CODE_AGENT_MODEL || process.env.OPENAI_MODEL || "unknown", + layer: "api", + retryable: true + }) + }); + return; + } + + if (!params || typeof params !== "object" || Array.isArray(params)) { + sendJson(response, 400, { + ...createCodeAgentErrorPayload({ + code: "invalid_params", + message: "Code Agent chat body must be a JSON object", + traceId: getHeader(request, "x-trace-id") || "trc_unassigned", + model: options.env?.HWLAB_CODE_AGENT_MODEL || options.env?.OPENAI_MODEL || process.env.HWLAB_CODE_AGENT_MODEL || process.env.OPENAI_MODEL || "unknown", + layer: "api", + retryable: true + }) + }); + return; + } + params = withMdtodoExecutionPrompt(params); + + const traceId = safeTraceId(getHeader(request, "x-trace-id") || params.traceId) || `trc_${randomUUID()}`; + const lifecycle = codeAgentTurnLifecycleFields(traceId, params); + const chatParams = { + ...params, + traceId, + ...lifecycle, + ownerUserId: options.actor?.id ?? params.ownerUserId, + ownerRole: options.actor?.role ?? params.ownerRole + }; + const manualSession = perf ? await perf.measure("manual_session", () => prepareManualCodeAgentSession({ params: chatParams, options, traceId, response })) : await prepareManualCodeAgentSession({ params: chatParams, options, traceId, response }); + if (manualSession?.blocked) return; + const nativeSessionChatParams = stripSyntheticConversationContext(chatParams, traceId, options); + + if (codeAgentChatShortConnectionRequested(request, params, options)) { + if (perf) perf.recordPhase({ phase: "short_connection_accept", durationMs: 0, outcome: "ok" }); + let submitted = null; + try { + submitted = await submitCodeAgentChatTurn({ + params: nativeSessionChatParams, + options, + traceId + }); + } catch (error) { + if (isCodeAgentSubmissionUnavailableError(error)) { + sendJson(response, error.statusCode ?? 503, error.payload); + return; + } + throw error; + } + const traceUrl = `/v1/agent/traces/${encodeURIComponent(traceId)}`; + const turnUrl = `/v1/agent/turns/${encodeURIComponent(traceId)}`; + const workbenchEventsUrl = `/v1/workbench/events?sessionId=${encodeURIComponent(safeSessionId(nativeSessionChatParams.sessionId) || "")}&traceId=${encodeURIComponent(traceId)}`; + const statusCode = 202; + const admittedTiming = codeAgentAdmissionTimingFields(submitted?.createdAt ?? submitted?.updatedAt); + void emitCodeAgentOtelSpan("POST /v1/agent/chat", traceId, options.env ?? process.env, { + kind: 2, + attributes: { "http.method": "POST", "http.route": "/v1/agent/chat", "http.status_code": statusCode, sessionId: safeSessionId(nativeSessionChatParams.sessionId) || null, turnId: lifecycle.turnId, ...agentChatOtelAttributes(nativeSessionChatParams) } + }); + sendJson(response, 202, { + accepted: true, + status: "running", + shortConnection: true, + controlSemantics: "submit-and-project-sse", + traceId, + ...codeAgentOtelTraceFields(traceId, options.env ?? process.env), + ...lifecycle, + sessionId: safeSessionId(nativeSessionChatParams.sessionId) || null, + createdAt: submitted?.createdAt ?? admittedTiming.createdAt, + updatedAt: submitted?.updatedAt ?? admittedTiming.updatedAt, + timing: submitted?.timing ?? admittedTiming.timing, + startedAt: submitted?.startedAt ?? admittedTiming.startedAt, + lastEventAt: submitted?.lastEventAt ?? admittedTiming.lastEventAt, + finishedAt: submitted?.finishedAt ?? admittedTiming.finishedAt, + durationMs: submitted?.durationMs ?? admittedTiming.durationMs, + realtimeCapabilities: workbenchRealtimeCapabilities(options.env ?? process.env), + admission: submitted?.agentRun ? { + state: submitted.admissionState ?? "promoted", + runId: submitted.agentRun.runId ?? null, + commandId: submitted.agentRun.commandId ?? null, + dispatchIntentId: submitted.agentRun.dispatchIntentId ?? null, + runnerJobId: submitted.agentRun.runnerJobId ?? null, + durableDispatch: submitted.agentRun.durableDispatch === true, + valuesRedacted: true + } : null, + traceUrl, + turnUrl, + workbenchEventsUrl, + cancelUrl: "/v1/agent/chat/cancel", + runnerTrace: (options.traceStore ?? defaultCodeAgentTraceStore).snapshot(traceId) + }); + return; + } + + const admittedBase = codeAgentAdmittedFailureBasePayload({ params: nativeSessionChatParams, options, traceId }); + try { + await recordCodeAgentTurnAdmission({ payload: admittedBase, params: nativeSessionChatParams, options, traceId }); + } catch (error) { + if (isCodeAgentAdmissionUnavailableError(error)) { + sendJson(response, error.statusCode ?? 503, error.payload); + return; + } + throw error; + } + const billingPreflight = perf ? await perf.measure("billing_preflight", () => preflightCodeAgentBilling({ params: nativeSessionChatParams, options, traceId, response: null, sendResponse: false })) : await preflightCodeAgentBilling({ params: nativeSessionChatParams, options, traceId, response: null, sendResponse: false }); + if (billingPreflight?.blocked) { + const failed = await settleAdmittedCodeAgentFailure({ + base: admittedBase, + params: nativeSessionChatParams, + options, + traceId, + results: options.codeAgentChatResults, + failure: billingPreflight, + traceLabel: "billing:preflight:failed" + }); + sendJson(response, billingPreflight.statusCode ?? 503, failed); + return; + } + const nativeSessionChatParamsWithBilling = { ...nativeSessionChatParams, userBillingReservation: billingPreflight?.reservation ?? null }; + const payload = perf ? await perf.measure("code_agent_run", () => runCodeAgentChat(nativeSessionChatParamsWithBilling, options)) : await runCodeAgentChat(nativeSessionChatParamsWithBilling, options); + await (perf ? perf.measure("billing_finalize", () => finalizeCodeAgentBillingUsage({ payload, params: nativeSessionChatParamsWithBilling, options })) : finalizeCodeAgentBillingUsage({ payload, params: nativeSessionChatParamsWithBilling, options })); + await (perf ? perf.measure("session_owner_record", () => recordCodeAgentSessionOwner({ payload, params: nativeSessionChatParamsWithBilling, options, status: codeAgentOwnerStatusForResult(payload) })) : recordCodeAgentSessionOwner({ payload, params: nativeSessionChatParamsWithBilling, options, status: codeAgentOwnerStatusForResult(payload) })); + const responsePayload = annotateOwner(payload, nativeSessionChatParamsWithBilling); + + sendJson(response, responsePayload.status === "failed" && responsePayload.error?.code === "invalid_params" ? 400 : 200, responsePayload); +} + +export async function handleCodeAgentSessionsHttp(request, response, url, options) { + const match = url.pathname.match(/^\/v1\/agent\/sessions(?:\/([^/]+)(?:\/(select))?)?$/u); + const sessionId = safeSessionId(match?.[1] ? decodeURIComponent(match[1]) : ""); + const action = match?.[2] ?? null; + if (!match) { + sendJson(response, 404, manualSessionErrorPayload({ code: "session_route_not_found", message: "Code Agent session route is not implemented." })); + return; + } + if (request.method === "GET" && !sessionId) { + await listManualCodeAgentSessions(request, response, url, options); + return; + } + if (request.method === "POST" && !sessionId) { + await createManualCodeAgentSession(request, response, options); + return; + } + if (request.method === "GET" && sessionId && !action) { + await getManualCodeAgentSession(response, options, sessionId); + return; + } + if (request.method === "DELETE" && sessionId && !action) { + await archiveManualCodeAgentSession(response, options, sessionId); + return; + } + if (request.method === "POST" && sessionId && action === "select") { + await selectManualCodeAgentSession(request, response, options, sessionId); + return; + } + sendJson(response, 405, manualSessionErrorPayload({ code: "session_method_not_allowed", message: "Unsupported Code Agent session method." })); +} + +async function listManualCodeAgentSessions(request, response, url, options) { + const projectId = textValue(url.searchParams.get("projectId")) || DEFAULT_CODE_AGENT_PROJECT_ID; + const limit = parsePositiveInteger(url.searchParams.get("limit"), 20); + const sessions = await options.accessController?.store?.listAgentSessionsForUser?.({ + ownerUserId: options.actor?.id, + actorRole: options.actor?.role, + ownerScoped: true, + projectId, + limit + }) ?? []; + sendJson(response, 200, { + ok: true, + status: "succeeded", + contractVersion: "code-agent-manual-session-v1", + projectId, + sessions: sessions.map(publicManualAgentSession), + count: sessions.length, + valuesRedacted: true, + secretMaterialStored: false + }); +} + +async function createManualCodeAgentSession(request, response, options) { + const body = await readJsonObjectBody(request, options.bodyLimitBytes); + if (!body.ok) { + sendJson(response, 400, manualSessionErrorPayload({ code: body.code, message: body.message, reason: body.reason })); + return; + } + const params = body.value; + const projectId = textValue(params.projectId) || DEFAULT_CODE_AGENT_PROJECT_ID; + const conversationId = safeConversationId(params.conversationId) || `cnv_${randomUUID()}`; + const sessionId = safeSessionId(params.sessionId) || `ses_${randomUUID()}`; + const providerProfile = normalizeCodeAgentProviderProfile(params.providerProfile ?? params.codeAgentProviderProfile); + const now = new Date().toISOString(); + const session = await options.accessController.recordAgentSessionOwner({ + ownerUserId: options.actor?.id, + ownerRole: options.actor?.role, + sessionId, + projectId, + agentId: "hwlab-code-agent", + status: "idle", + conversationId, + threadId: null, + traceId: null, + session: { + source: "manual-session-create", + providerProfile, + sessionStatus: "idle", + createdAt: now, + valuesRedacted: true, + secretMaterialStored: false + } + }); + await writeWorkbenchSessionAdmissionFact({ + runtimeStore: options.runtimeStore, + session, + ownerUserId: options.actor?.id, + ownerRole: options.actor?.role, + projectId, + conversationId, + threadId: null, + status: "idle", + now + }); + sendJson(response, 201, { + ok: true, + status: "created", + contractVersion: "code-agent-manual-session-v1", + session: publicManualAgentSession(session), + nextCommands: manualSessionNextCommands(session), + valuesRedacted: true, + secretMaterialStored: false + }); +} + +async function getManualCodeAgentSession(response, options, sessionId) { + const session = await getVisibleManualAgentSession(options, sessionId); + if (!session) { + sendJson(response, 404, manualSessionErrorPayload({ code: "session_not_found", message: "Code Agent session is not visible to the current actor.", sessionId })); + return; + } + sendJson(response, 200, { + ok: true, + status: "found", + contractVersion: "code-agent-manual-session-v1", + session: publicManualAgentSession(session), + valuesRedacted: true, + secretMaterialStored: false + }); +} + +async function archiveManualCodeAgentSession(response, options, sessionId) { + const session = await getVisibleManualAgentSession(options, sessionId); + if (!session) { + sendJson(response, 404, manualSessionErrorPayload({ code: "session_not_found", message: "Code Agent session is not visible to the current actor.", sessionId })); + return; + } + const result = await options.accessController?.store?.archiveAgentConversation?.({ + ownerUserId: options.actor?.id, + actorRole: options.actor?.role, + ownerScoped: true, + sessionId, + conversationId: session.conversationId, + projectId: session.projectId + }) ?? { count: 0 }; + sendJson(response, 200, { + ok: true, + status: "archived", + contractVersion: "code-agent-manual-session-v1", + archivedCount: result.count ?? 0, + session: publicManualAgentSession({ ...session, status: "archived", endedAt: session.endedAt ?? new Date().toISOString(), updatedAt: new Date().toISOString() }), + valuesRedacted: true, + secretMaterialStored: false + }); +} + +async function selectManualCodeAgentSession(request, response, options, sessionId) { + const body = await readJsonObjectBody(request, options.bodyLimitBytes); + if (!body.ok) { + sendJson(response, 400, manualSessionErrorPayload({ code: body.code, message: body.message, reason: body.reason, sessionId })); + return; + } + const session = await getVisibleManualAgentSession(options, sessionId); + if (!session) { + sendJson(response, 404, manualSessionErrorPayload({ code: "session_not_found", message: "Code Agent session is not visible to the current actor.", sessionId })); + return; + } + if (!manualSessionUsable(session.status)) { + sendJson(response, 409, manualSessionErrorPayload({ code: "session_not_usable", message: `Code Agent session ${sessionId} is ${session.status}; create a new session before continuing.`, sessionId, session })); + return; + } + sendJson(response, 200, { + ok: true, + status: "selected", + contractVersion: "code-agent-manual-session-v1", + session: publicManualAgentSession(session), + nextCommands: manualSessionNextCommands(session), + valuesRedacted: true, + secretMaterialStored: false + }); +} + +async function prepareManualCodeAgentSession({ params, options, traceId, response }) { + if (!options.actor?.id) { + sendJson(response, 401, manualSessionErrorPayload({ + code: "auth_required", + message: "Authentication is required before using an explicit HWLAB Code Agent session.", + traceId, + retryable: true + })); + return { blocked: true }; + } + const sessionId = safeSessionId(params.sessionId); + if (!sessionId) { + sendJson(response, 409, manualSessionErrorPayload({ + code: "session_required", + message: "Code Agent session is explicit in HWLAB v0.2; create/select a session before sending a turn.", + traceId, + retryable: true, + nextCommands: ["hwlab-cli client agent session create --provider-profile PROFILE", "hwlab-cli client agent send --session-id --message TEXT"] + })); + return { blocked: true }; + } + const session = await getVisibleManualAgentSession(options, sessionId); + if (!session) { + sendJson(response, 404, manualSessionErrorPayload({ code: "session_not_found", message: "Code Agent session is not visible to the current actor.", traceId, sessionId })); + return { blocked: true }; + } + const freshContinuation = manualSessionFreshContinuationRequired(session); + if (!manualSessionUsable(session.status) && !freshContinuation) { + sendJson(response, 409, manualSessionErrorPayload({ code: "session_not_usable", message: `Code Agent session ${sessionId} is ${session.status}; create a new session before continuing.`, traceId, sessionId, session })); + return { blocked: true }; + } + const requestedProjectId = textValue(params.projectId); + const sessionProjectId = textValue(session.projectId); + if (requestedProjectId && sessionProjectId && requestedProjectId !== sessionProjectId) { + sendJson(response, 409, manualSessionErrorPayload({ code: "session_project_mismatch", message: `sessionId ${sessionId} belongs to ${sessionProjectId}; requested ${requestedProjectId}.`, traceId, sessionId, session })); + return { blocked: true }; + } + const storedConversationId = safeConversationId(session.conversationId); + const requestedConversationId = safeConversationId(params.conversationId); + if (requestedConversationId && storedConversationId && requestedConversationId !== storedConversationId) { + sendJson(response, 409, manualSessionErrorPayload({ code: "session_conversation_mismatch", message: `sessionId ${sessionId} is bound to ${storedConversationId}; requested ${requestedConversationId}.`, traceId, sessionId, session })); + return { blocked: true }; + } + const conversationId = requestedConversationId || storedConversationId; + if (!conversationId) { + sendJson(response, 409, manualSessionErrorPayload({ code: "session_conversation_required", message: "Code Agent session has no bound conversationId; create a new session before continuing.", traceId, sessionId, session })); + return { blocked: true }; + } + params.conversationId = conversationId; + params.sessionId = sessionId; + params.projectId = requestedProjectId || sessionProjectId || DEFAULT_CODE_AGENT_PROJECT_ID; + applyManualSessionProviderProfile(params, session); + if (freshContinuation) { + params.threadId = undefined; + params.forceFreshAgentRunSession = true; + params.forceFreshAgentRunSessionReason = manualSessionFreshContinuationReason(session); + } else { + params.threadId = safeOpaqueId(params.threadId) || manualSessionContinuationThreadId(session) || undefined; + } + return { session }; +} + +function applyManualSessionProviderProfile(params = {}, session = null) { + const requested = firstNonEmptyValue(params.providerProfile, params.codeAgentProviderProfile); + if (requested) { + const normalized = normalizeCodeAgentProviderProfile(requested); + params.providerProfile = normalized; + params.codeAgentProviderProfile = normalized; + params.providerProfileSource = "request"; + return normalized; + } + const stored = manualSessionStoredProviderProfile(session); + if (!stored) return null; + params.providerProfile = stored; + params.codeAgentProviderProfile = stored; + params.providerProfileSource = "session"; + return stored; +} + +function manualSessionStoredProviderProfile(session = null) { + const evidence = session?.session && typeof session.session === "object" ? session.session : {}; + const traceProfile = latestCompletedTraceResultProviderProfile(evidence.traceResults); + const raw = firstNonEmptyValue( + traceProfile, + evidence.agentRun?.backendProfile, + evidence.providerProfile, + evidence.codeAgentProviderProfile, + evidence.backendProfile + ); + if (!raw) return null; + const normalized = normalizeCodeAgentProviderProfile(raw); + return normalized === "runtime-default" ? null : normalized; +} + +function latestCompletedTraceResultProviderProfile(traceResults = null) { + if (!traceResults || typeof traceResults !== "object") return null; + const entries = Object.values(traceResults).filter((value) => value && typeof value === "object"); + for (const result of entries.reverse()) { + const agentRun = result.agentRun && typeof result.agentRun === "object" ? result.agentRun : null; + if (!agentRun) continue; + const status = normalizeTurnStatus(result.status ?? agentRun.terminalStatus ?? agentRun.commandState ?? agentRun.status); + if (status && status !== "completed") continue; + const profile = firstNonEmptyValue(agentRun.backendProfile, result.providerProfile, result.codeAgentProviderProfile); + if (profile) return profile; + } + return null; +} + +function manualSessionFreshContinuationRequired(session = null) { + const status = String(session?.status ?? "").trim().toLowerCase().replace(/_/gu, "-"); + if (status === "thread-resume-failed") return true; + const continuation = session?.session?.continuation && typeof session.session.continuation === "object" ? session.session.continuation : null; + return continuation?.requiresFreshSession === true; +} + +function manualSessionFreshContinuationReason(session = null) { + const continuation = session?.session?.continuation && typeof session.session.continuation === "object" ? session.session.continuation : null; + return firstNonEmptyValue(continuation?.reasonCode, continuation?.reason, session?.status, "stale-continuation") ?? "stale-continuation"; +} + +function manualSessionContinuationThreadId(session = null) { + if (manualSessionFreshContinuationRequired(session)) return null; + const evidence = session?.session && typeof session.session === "object" ? session.session : {}; + return safeOpaqueId(evidence.threadId) || safeOpaqueId(evidence.agentRun?.threadId) || safeOpaqueId(session?.threadId) || null; +} + +async function getVisibleManualAgentSession(options, sessionId) { + if (!safeSessionId(sessionId) || !options.actor?.id || !options.accessController?.getAgentSession) return null; + const session = await options.accessController.getAgentSession(sessionId); + if (!session) return null; + if (session.ownerUserId === options.actor.id) return session; + return null; +} + +function publicManualAgentSession(session) { + if (!session || typeof session !== "object") return null; + return { + sessionId: session.id, + agentId: session.agentId ?? "hwlab-code-agent", + status: session.status ?? null, + usable: manualSessionUsable(session.status), + threadId: safeOpaqueId(session.threadId) || null, + lastTraceId: safeTraceId(session.lastTraceId) || null, + providerProfile: textValue(session.session?.providerProfile) || null, + createdAt: session.startedAt ?? null, + updatedAt: session.updatedAt ?? null, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +function manualSessionUsable(status) { + const value = String(status ?? "").trim().toLowerCase().replace(/_/gu, "-"); + return !["blocked", "expired", "stale"].includes(value); +} + +function manualSessionErrorPayload({ code, message, reason = null, traceId = null, sessionId = null, session = null, retryable = true, nextCommands = undefined } = {}) { + return { + ok: false, + accepted: false, + status: "blocked", + traceId: safeTraceId(traceId) || null, + sessionId: safeSessionId(sessionId) || safeSessionId(session?.id) || null, + session: session ? publicManualAgentSession(session) : null, + error: { + code, + layer: "api", + category: "session_blocked", + retryable, + message, + reason, + userMessage: message, + route: "/v1/agent/chat" + }, + blocker: { + code, + layer: "api", + retryable, + summary: message + }, + nextCommands, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +function manualSessionNextCommands(session) { + const sessionId = session?.id ?? ""; + return [ + `hwlab-cli client agent send --session-id ${sessionId} --message TEXT`, + `hwlab-cli client agent session status ${sessionId}` + ]; +} + +async function readJsonObjectBody(request, bodyLimitBytes) { + const body = await readBody(request, bodyLimitBytes); + try { + const value = body ? JSON.parse(body) : {}; + if (!value || typeof value !== "object" || Array.isArray(value)) return { ok: false, code: "invalid_params", message: "body must be a JSON object" }; + return { ok: true, value }; + } catch (error) { + return { ok: false, code: "parse_error", message: "Invalid JSON body", reason: error.message }; + } +} + +async function runCodeAgentChat(params, options) { + return handleCodeAgentChat(params, await codeAgentChatExecutionOptions(options, params)); +} + +function stripSyntheticConversationContext(params = {}, traceId, options = {}) { + if (Array.isArray(params.conversationContext?.messages) || Array.isArray(params.messages)) { + (options.traceStore ?? defaultCodeAgentTraceStore).append(traceId, { + type: "synthetic-context", + status: "ignored", + label: "synthetic-context:ignored", + message: "Synthetic conversation context was stripped; Code Agent continuation uses only Codex stdio native thread/resume and turn/start.", + valuesPrinted: false + }); + } + const { conversationContext, conversationContextTruncated, contextSource, messages, ...nativeParams } = params; + return nativeParams; +} + +async function codeAgentChatExecutionOptions(options = {}, params = {}) { + const env = codeAgentProviderProfileEnv(options.env ?? process.env, params); + Object.assign(env, await codeAgentAuthEnv(options, env, params)); + return { + callProvider: options.callCodeAgentProvider, + timeoutMs: options.codeAgentTimeoutMs, + hardTimeoutMs: options.codeAgentHardTimeoutMs, + env, + workspace: options.workspace, + skillsDirs: options.skillsDirs, + skillsDirsExact: options.skillsDirsExact, + skillsDiscoveryDelayMs: options.skillsDiscoveryDelayMs, + sessionRegistry: options.sessionRegistry, + m3IoSkillRequestJson: options.m3IoSkillRequestJson ?? createCodeAgentM3HwlabApiRequestJson(options), + codexStdioManager: options.codexStdioManager, + traceStore: options.traceStore, + gatewayRegistry: options.gatewayRegistry + }; +} + +async function codeAgentAuthEnv(options = {}, env = process.env, params = {}) { + const ownerUserId = textValue(params.ownerUserId ?? options.actor?.id ?? ""); + const store = options.accessController?.store; + const key = ownerUserId && store?.findActiveDefaultApiKeyForUser + ? await store.findActiveDefaultApiKeyForUser(ownerUserId) + : null; + const envRuntimeLane = runtimeLaneForEnv(env); + const runtimeNamespace = runtimeNamespaceForEnv(env); + const namespaceRuntimeLane = runtimeLaneForNamespace(runtimeNamespace); + const runtimeLane = resolveRuntimeLaneAuthority({ envRuntimeLane, runtimeNamespace, namespaceRuntimeLane }); + const inClusterApiUrl = internalCodeAgentApiUrl(env, runtimeNamespace); + const inClusterWebUrl = internalCodeAgentWebUrl(env, runtimeNamespace); + const apiUrl = firstNonEmptyValue( + codeAgentAgentRunAdapterEnabled(env) ? inClusterApiUrl : null, + env.HWLAB_RUNTIME_API_URL, + sameRuntimeEndpoint(env.HWLAB_CLOUD_API_URL, runtimeNamespace, runtimeLane), + env.HWLAB_PUBLIC_ENDPOINT, + localCloudApiUrl(env) + ); + return { + HWLAB_CLOUD_API_URL: apiUrl, + HWLAB_RUNTIME_API_URL: apiUrl, + HWLAB_RUNTIME_WEB_URL: firstNonEmptyValue(codeAgentAgentRunAdapterEnabled(env) ? inClusterWebUrl : null, env.HWLAB_RUNTIME_WEB_URL, apiUrl), + HWLAB_RUNTIME_NAMESPACE: runtimeNamespace, + HWLAB_RUNTIME_LANE: runtimeLane, + HWLAB_RUNTIME_ENDPOINT_SOURCE: codeAgentAgentRunAdapterEnabled(env) ? "runtime-namespace" : "runtime-env", + HWLAB_RUNTIME_ENDPOINT_LOCKED: "1", + HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME: "1", + ...(options.actorApiKeySecret ? { HWLAB_API_KEY: options.actorApiKeySecret } : key?.displaySecret ? { HWLAB_API_KEY: key.displaySecret } : {}) + }; +} + +function internalCodeAgentApiUrl(env = process.env, namespace = runtimeNamespaceForEnv(env)) { + const serviceName = firstNonEmptyValue(env.HWLAB_CODE_AGENT_INTERNAL_API_SERVICE_NAME, env.HWLAB_CLOUD_API_SERVICE_NAME, "hwlab-cloud-api"); + const port = parsePositiveInteger(firstNonEmptyValue(env.HWLAB_CODE_AGENT_INTERNAL_API_PORT, env.HWLAB_CLOUD_API_PORT), 6667); + return `http://${serviceName}.${namespace}.svc.cluster.local:${port}`; +} + +function internalCodeAgentWebUrl(env = process.env, namespace = runtimeNamespaceForEnv(env)) { + const serviceName = firstNonEmptyValue(env.HWLAB_CODE_AGENT_INTERNAL_WEB_SERVICE_NAME, env.HWLAB_CLOUD_WEB_SERVICE_NAME, "hwlab-cloud-web"); + const port = parsePositiveInteger(firstNonEmptyValue(env.HWLAB_CODE_AGENT_INTERNAL_WEB_PORT, env.HWLAB_CLOUD_WEB_PORT), 8080); + return `http://${serviceName}.${namespace}.svc.cluster.local:${port}`; +} + +function runtimeNamespaceForEnv(env = process.env) { + return firstNonEmptyValue( + env.HWLAB_RUNTIME_NAMESPACE, + env.POD_NAMESPACE, + env.HWLAB_NAMESPACE, + defaultRuntimeNamespace(env) + ); +} + +function runtimeLaneForEnv(env = process.env) { + const raw = firstNonEmptyValue(env.HWLAB_RUNTIME_LANE, env.HWLAB_GITOPS_LANE, env.HWLAB_GITOPS_PROFILE, env.HWLAB_ENVIRONMENT, env.HWLAB_BOOT_REF); + if (!raw) return null; + const lane = normalizeRuntimeLane(raw); + if (!lane) { + throw runtimeAuthorityError("unsupported_runtime_lane", `unsupported HWLAB runtime lane: ${String(raw).trim()}`); + } + return lane; +} + +function runtimeLaneForNamespace(namespace) { + const value = String(namespace ?? "").trim().toLowerCase(); + if (value === "hwlab-prod") return "prod"; + if (value === "hwlab-dev") return "dev"; + const versionMatch = value.match(/^hwlab-v(\d+)$/u); + if (versionMatch) return versionRuntimeLane(versionMatch[1]); + return null; +} + +function sameRuntimeEndpoint(value, namespace, lane) { + const text = firstNonEmptyValue(value); + if (!text) return null; + const endpointLane = runtimeLaneForEndpoint(text); + const endpointNamespace = runtimeNamespaceForEndpoint(text); + if (namespace && endpointNamespace && endpointNamespace !== namespace) return null; + if (lane && endpointLane && endpointLane !== lane) return null; + return text; +} + +function runtimeNamespaceForEndpoint(value) { + const match = String(value ?? "").match(/\.((?:hwlab-[a-z0-9-]+))\.svc\.cluster\.local(?::|\/|$)/iu); + return match?.[1] ?? null; +} + +function runtimeLaneForEndpoint(value) { + const text = String(value ?? "").toLowerCase(); + const namespaceLane = runtimeLaneForNamespace(runtimeNamespaceForEndpoint(text)); + if (namespaceLane) return namespaceLane; + if (/hwlab-prod|:18666(?:\/|$)|:18667(?:\/|$)/u.test(text)) return "prod"; + if (/hwlab-dev|:17666(?:\/|$)|:17667(?:\/|$)|:16666(?:\/|$)|:16667(?:\/|$)/u.test(text)) return "dev"; + return null; +} + +function normalizeRuntimeLane(value) { + const profile = String(value ?? "").trim().toLowerCase(); + if (!profile) return null; + if (profile === "prod" || profile === "production") return "prod"; + if (profile === "dev" || profile === "development" || profile === "local") return "dev"; + const dotted = profile.match(/^v?0+\.(\d+)$/u); + if (dotted) return versionRuntimeLane(dotted[1]); + const compact = profile.match(/^v?0*(\d+)$/u); + if (compact) return versionRuntimeLane(compact[1]); + return null; +} + +function versionRuntimeLane(value) { + const version = Number.parseInt(value, 10); + if (!Number.isFinite(version) || version <= 0) return null; + return `v${String(version).padStart(2, "0")}`; +} + +function resolveRuntimeLaneAuthority({ envRuntimeLane, runtimeNamespace, namespaceRuntimeLane }) { + if (!namespaceRuntimeLane) { + throw runtimeAuthorityError("unsupported_runtime_namespace", `unsupported HWLAB runtime namespace: ${runtimeNamespace}`); + } + if (envRuntimeLane && envRuntimeLane !== namespaceRuntimeLane) { + throw runtimeAuthorityError( + "runtime_authority_mismatch", + `HWLAB runtime lane ${envRuntimeLane} does not match namespace ${runtimeNamespace} (${namespaceRuntimeLane})` + ); + } + return namespaceRuntimeLane; +} + +function runtimeAuthorityError(code, message) { + return Object.assign(new Error(message), { + code, + statusCode: 500, + valuesPrinted: false + }); +} + +function defaultRuntimeNamespace(env = process.env) { + const profile = runtimeLaneForEnv(env) ?? "dev"; + if (profile === "prod") return "hwlab-prod"; + if (profile === "dev") return "hwlab-dev"; + return `hwlab-${profile}`; +} + +function localCloudApiUrl(env = process.env) { + const port = parsePositiveInteger(env.HWLAB_CLOUD_API_PORT, 6667); + return `http://127.0.0.1:${port}`; +} + +function codeAgentProviderProfileEnv(env = process.env, params = {}) { + const profile = normalizeCodeAgentProviderProfile(params.providerProfile ?? params.codeAgentProviderProfile ?? params.provider ?? env.HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE); + if (profile === "runtime-default") return env; + const overlay = { ...env }; + overlay.HWLAB_CODE_AGENT_SELECTED_PROVIDER_PROFILE = profile; + overlay.HWLAB_CODE_AGENT_PROVIDER_PROFILE_LABEL = CODE_AGENT_PROVIDER_PROFILE_LABELS[profile] ?? profile; + overlay.HWLAB_CODE_AGENT_PROVIDER = "codex-stdio"; + if (profile === "codex-api") { + overlay.HWLAB_CODE_AGENT_MODEL = firstNonEmptyValue(env.HWLAB_CODE_AGENT_CODEX_API_MODEL, env.HWLAB_CODE_AGENT_LEGACY_CODEX_MODEL, DEFAULT_CODE_AGENT_CODEX_API_MODEL); + overlay.HWLAB_CODE_AGENT_OPENAI_BASE_URL = firstNonEmptyValue(env.HWLAB_CODE_AGENT_CODEX_API_BASE_URL, env.HWLAB_CODE_AGENT_LEGACY_OPENAI_BASE_URL, DEFAULT_CODE_AGENT_CODEX_API_BASE_URL); + } else if (profile === "minimax-m3") { + overlay.HWLAB_CODE_AGENT_MODEL = firstNonEmptyValue(env.HWLAB_CODE_AGENT_MINIMAX_M3_MODEL, DEFAULT_CODE_AGENT_MINIMAX_M3_MODEL); + } else { + const dynamicProfile = profile !== "deepseek"; + overlay.HWLAB_CODE_AGENT_MODEL = dynamicProfile + ? firstNonEmptyValue(env[`HWLAB_CODE_AGENT_${providerProfileEnvKey(profile)}_MODEL`], profile) + : firstNonEmptyValue(env.HWLAB_CODE_AGENT_DEEPSEEK_MODEL, DEFAULT_CODE_AGENT_DEEPSEEK_MODEL); + if (!dynamicProfile || !codeAgentAgentRunAdapterEnabled(env)) { + overlay.HWLAB_CODE_AGENT_OPENAI_BASE_URL = firstNonEmptyValue(env.HWLAB_CODE_AGENT_DEEPSEEK_BASE_URL, defaultDeepSeekBaseUrlForEnv(env)); + } + } + overlay.NO_PROXY = mergeNoProxyEntries(env.NO_PROXY, codeAgentClusterNoProxyEntries(env)); + overlay.no_proxy = mergeNoProxyEntries(env.no_proxy ?? env.NO_PROXY, codeAgentClusterNoProxyEntries(env)); + return overlay; +} + +function providerProfileEnvKey(profile) { + return String(profile ?? "") + .trim() + .toUpperCase() + .replace(/[^A-Z0-9]+/gu, "_") + .replace(/^_+|_+$/gu, "") || "PROFILE"; +} + +function defaultDeepSeekBaseUrlForEnv(env = process.env) { + const namespace = env.HWLAB_GITOPS_PROFILE === "prod" || env.HWLAB_ENVIRONMENT === "prod" ? "hwlab-prod" : "hwlab-dev"; + return `http://hwlab-deepseek-proxy.${namespace}.svc.cluster.local:4000/v1/responses`; +} + +function codeAgentClusterNoProxyEntries(env = process.env) { + const namespace = env.HWLAB_GITOPS_PROFILE === "prod" || env.HWLAB_ENVIRONMENT === "prod" ? "hwlab-prod" : "hwlab-dev"; + return [ + `${namespace}.svc.cluster.local`, + ".svc", + ".cluster.local", + "127.0.0.1", + "localhost", + "::1", + "10.0.0.0/8", + "10.42.0.0/16", + "10.43.0.0/16", + "hyueapi.com", + ".hyueapi.com", + "hyui.com", + ".hyui.com", + "api.minimaxi.com", + ".minimaxi.com" + ]; +} + +function mergeNoProxyEntries(current, additions = []) { + return uniqueStrings([...String(current ?? "").split(","), ...additions]).join(","); +} + +function safeOtelConfigText(value) { + const text = firstNonEmptyValue(value); + return text === null ? null : text.slice(0, 120); +} + +function safeOtelUrlHost(value) { + const text = firstNonEmptyValue(value); + if (text === null) return null; + try { + return new URL(text).host.slice(0, 120); + } catch { + return null; + } +} + +function withMdtodoExecutionPrompt(params = {}) { + const execution = mdtodoExecutionContextFromParams(params); + if (!execution?.hwpodId || !execution?.hwpodWorkspaceArgs) return params; + const message = firstNonEmptyValue(params.message, params.prompt, params.text); + if (!message || /hwpodWorkspaceArgs\s*:/u.test(message)) return params; + const taskRef = firstNonEmptyValue(params.taskRef, execution.taskRef) ?? "-"; + const mdtodoRootRef = execution.mdtodoRootRef ?? "docs/MDTODO"; + const prefix = [ + "# MDTODO HWPOD 执行上下文", + `TaskRef: ${taskRef}`, + `sourceId: ${execution.sourceId ?? "-"}`, + `hwpodId: ${execution.hwpodId}`, + `nodeId: ${execution.nodeId ?? "-"}`, + `mdtodoRootRef: ${mdtodoRootRef}`, + `hwpodWorkspaceArgs: ${execution.hwpodWorkspaceArgs}`, + "所有 hwpod/hwpod-ctl 命令都必须携带上述 hwpodWorkspaceArgs。读取 MDTODO、报告和工程文件必须通过 HWPOD workspace/node-ops 入口，不得猜测或访问普通容器本地路径，也不得创建本地 .hwlab/hwpod-spec.yaml fallback。" + ].join("\n"); + const nextMessage = `${prefix}\n\n${message}`; + return { + ...params, + message: nextMessage, + prompt: nextMessage, + launchContext: { + ...(params.launchContext && typeof params.launchContext === "object" ? params.launchContext : {}), + hwpodId: execution.hwpodId, + nodeId: execution.nodeId, + mdtodoRootRef: execution.mdtodoRootRef, + hwpodWorkspaceArgs: execution.hwpodWorkspaceArgs, + contextFingerprint: execution.contextFingerprint, + executionContext: execution, + valuesRedacted: true + } + }; +} + +function mdtodoExecutionContextFromParams(params = {}) { + const launchContext = params.launchContext && typeof params.launchContext === "object" && !Array.isArray(params.launchContext) ? params.launchContext : null; + const execution = launchContext?.executionContext && typeof launchContext.executionContext === "object" && !Array.isArray(launchContext.executionContext) ? launchContext.executionContext : launchContext; + if (!execution || typeof execution !== "object") return null; + return { + sourceId: safeOtelConfigText(execution.sourceId ?? launchContext?.sourceId), + sourceKind: safeOtelConfigText(execution.sourceKind ?? launchContext?.sourceKind), + projectId: safeOtelConfigText(execution.projectId ?? launchContext?.projectId ?? params.projectId), + taskRef: safeOtelText(execution.taskRef ?? launchContext?.taskRef ?? params.taskRef), + fileRef: safeOtelConfigText(execution.fileRef ?? launchContext?.fileRef), + relativePath: safeOtelText(execution.relativePath ?? launchContext?.relativePath), + taskId: safeOtelConfigText(execution.taskId ?? launchContext?.taskId), + hwpodId: safeOtelText(execution.hwpodId ?? launchContext?.hwpodId), + nodeId: safeOtelText(execution.nodeId ?? launchContext?.nodeId), + mdtodoRootRef: safeOtelText(execution.mdtodoRootRef ?? launchContext?.mdtodoRootRef), + workspaceRootHash: safeOtelText(execution.workspaceRootHash ?? launchContext?.workspaceRootHash), + hwpodWorkspaceArgs: safeOtelText(execution.hwpodWorkspaceArgs ?? launchContext?.hwpodWorkspaceArgs, 500), + contextFingerprint: safeOtelText(execution.contextFingerprint ?? launchContext?.contextFingerprint), + valuesRedacted: true + }; +} + +function agentChatOtelAttributes(params = {}) { + const execution = mdtodoExecutionContextFromParams(params); + return { + "agent.chat.session_id": safeSessionId(params.sessionId) || null, + "agent.chat.provider_profile": safeOtelConfigText(params.providerProfile ?? params.codeAgentProviderProfile), + "agent.chat.provider_profile_source": safeOtelConfigText(params.providerProfileSource), + "agent.chat.task_ref": safeOtelText(params.taskRef ?? execution?.taskRef), + "agent.chat.source_id": safeOtelConfigText(execution?.sourceId ?? params.sourceId), + "agent.chat.hwpod_id": safeOtelText(execution?.hwpodId), + "agent.chat.workspace_root_hash": safeOtelText(execution?.workspaceRootHash), + "agent.chat.context_fingerprint": safeOtelText(execution?.contextFingerprint) + }; +} + +function safeOtelText(value, max = 360) { + const text = firstNonEmptyValue(value); + return text === null ? null : text.slice(0, max); +} + +async function submitCodeAgentChatTurn({ params, options, traceId }) { + const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; + const results = options.codeAgentChatResults ?? createCodeAgentChatResultStore(); + const timestamp = new Date().toISOString(); + const runtimeEnv = options.env ?? process.env; + const adapterEnabled = codeAgentAgentRunAdapterEnabled(runtimeEnv); + const acceptedPayload = { + accepted: true, + status: "running", + traceId, + ...codeAgentOtelTraceFields(traceId, process.env), + ...codeAgentTurnLifecycleFields(traceId, params), + conversationId: safeConversationId(params.conversationId) || null, + sessionId: safeSessionId(params.sessionId) || null, + threadId: safeOpaqueId(params.threadId) || null, + projectId: params.projectId ?? null, + ...codeAgentAdmissionTimingFields(timestamp) + }; + void emitCodeAgentOtelSpan("provider_decision", traceId, runtimeEnv, { + attributes: { + ...agentChatOtelAttributes(params), + adapterEnabled, + adapter: safeOtelConfigText(runtimeEnv.HWLAB_CODE_AGENT_ADAPTER ?? runtimeEnv.HWLAB_CODE_AGENT_PROVIDER), + provider: safeOtelConfigText(runtimeEnv.HWLAB_CODE_AGENT_PROVIDER), + defaultProviderProfile: safeOtelConfigText(runtimeEnv.HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE), + agentRunManagerHost: safeOtelUrlHost(runtimeEnv.AGENTRUN_MGR_URL ?? runtimeEnv.HWLAB_CODE_AGENT_AGENTRUN_MGR_URL), + agentRunRunnerNamespace: safeOtelConfigText(runtimeEnv.HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE ?? runtimeEnv.AGENTRUN_RUNTIME_NAMESPACE), + agentRunSourceCommitPresent: Boolean(firstNonEmptyValue(runtimeEnv.HWLAB_CODE_AGENT_AGENTRUN_SOURCE_COMMIT)) + } + }); + if (adapterEnabled) { + const realtimeCapabilities = workbenchRealtimeCapabilities(runtimeEnv); + const transactionalAdmission = realtimeCapabilities.transactionalProjector; + const pendingPromotion = results.get(traceId); + if (pendingPromotion?.admissionState === "promotion-pending" && pendingPromotion?.agentRun?.durableDispatch === true) { + const retryParams = { ...params, userBillingReservation: pendingPromotion.userBillingReservation ?? null }; + try { + await promoteCodeAgentTurnAdmission({ payload: pendingPromotion, params: retryParams, options, traceId }); + } catch (error) { + throw codeAgentPromotionUnavailableError(error, { payload: pendingPromotion, params: retryParams, options, traceId }); + } + const promoted = annotateOwner({ ...pendingPromotion, accepted: true, status: "running", admissionState: "promoted" }, retryParams); + results.set(traceId, promoted); + return promoted; + } + + const lifecycle = codeAgentTurnLifecycleFields(traceId, params); + if (transactionalAdmission) { + await persistCodeAgentAdmissionInputState({ + params, + options, + traceId, + status: "admitting", + messageId: lifecycle.userMessageId + }); + } + + traceStore.append(traceId, { + type: "request", + status: "admitting", + label: "turn:admitting", + message: "Code Agent turn is validating billing and durable AgentRun dispatch before UI admission.", + terminal: false, + waitingFor: "billing-and-durable-dispatch", + valuesPrinted: false + }); + const initial = initialAgentRunChatResult({ params, options, traceId }); + const billingStartedAt = Date.now(); + const billingPreflight = await preflightCodeAgentBilling({ params, options, traceId, sendResponse: false }); + void emitCodeAgentOtelSpan("billing_preflight", traceId, options.env ?? process.env, { + startTimeMs: billingStartedAt, + attributes: { sessionId: safeSessionId(params.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId, blocked: Boolean(billingPreflight?.blocked) }, + status: billingPreflight?.blocked ? "error" : "ok", + error: billingPreflight?.blocked ? billingPreflight?.payload?.error?.message ?? billingPreflight?.blocker?.summary ?? "billing preflight blocked" : null + }); + if (billingPreflight?.blocked) { + if (transactionalAdmission) { + await persistCodeAgentAdmissionInputState({ + params, + options, + traceId, + status: "blocked", + messageId: lifecycle.userMessageId, + error: billingPreflight.payload?.error ?? billingPreflight.blocker ?? { code: "billing_preflight_blocked" } + }); + } + traceStore.append(traceId, { + type: "billing", + status: "blocked", + label: "billing:preflight:blocked-before-admission", + errorCode: billingPreflight.payload?.error?.code ?? billingPreflight.blocker?.code ?? "billing_preflight_blocked", + terminal: false, + valuesPrinted: false + }); + throw codeAgentSubmissionUnavailableError(billingPreflight, { params, options, traceId }); + } + + const dispatchParams = { ...params, userBillingReservation: billingPreflight?.reservation ?? null }; + const executionOptions = { ...options, ...(await codeAgentChatExecutionOptions(options, dispatchParams)) }; + const dispatchStartedAt = Date.now(); + let payload; + try { + payload = await submitAgentRunChatTurn({ params: dispatchParams, options: executionOptions, traceId, traceStore, results }); + } catch (error) { + const failure = codeAgentDispatchFailure(error); + const rejected = codeAgentSubmissionUnavailablePayload(failure, { params: dispatchParams, options: executionOptions, traceId }); + await finalizeCodeAgentBillingUsage({ payload: rejected, params: dispatchParams, options: executionOptions }); + if (transactionalAdmission) { + await persistCodeAgentAdmissionInputState({ + params: dispatchParams, + options: executionOptions, + traceId, + status: "failed", + messageId: lifecycle.userMessageId, + error: failure.payload?.error ?? error + }); + } + void emitCodeAgentOtelSpan("agentrun_dispatch", traceId, executionOptions.env ?? process.env, { + startTimeMs: dispatchStartedAt, + status: "error", + error, + attributes: { sessionId: safeSessionId(dispatchParams.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, dispatchParams).turnId, ...codeAgentDispatchErrorOtelAttributes(error) } + }); + throw codeAgentSubmissionUnavailableError(failure, { params: dispatchParams, options: executionOptions, traceId }); + } + void emitCodeAgentOtelSpan("agentrun_dispatch", traceId, executionOptions.env ?? process.env, { + startTimeMs: dispatchStartedAt, + attributes: { sessionId: safeSessionId(dispatchParams.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, dispatchParams).turnId, runId: payload?.agentRun?.runId ?? null, commandId: payload?.agentRun?.commandId ?? null, providerProfile: dispatchParams.providerProfile ?? null } + }); + const pending = annotateOwner({ + ...withCodeAgentBillingReservation(payload, dispatchParams), + accepted: false, + status: "admitting", + admissionState: "promotion-pending" + }, dispatchParams); + results.set(traceId, pending); + try { + if (transactionalAdmission) { + await persistCodeAgentAdmissionInputState({ + params: dispatchParams, + options: executionOptions, + traceId, + status: "admitting", + messageId: lifecycle.userMessageId, + commandId: payload.agentRun?.commandId ?? null, + agentRun: payload.agentRun ?? null + }); + } + await promoteCodeAgentTurnAdmission({ payload: pending, params: dispatchParams, options: executionOptions, traceId }); + } catch (error) { + throw codeAgentPromotionUnavailableError(error, { payload: pending, params: dispatchParams, options: executionOptions, traceId }); + } + const promoted = annotateOwner({ ...pending, accepted: true, status: "running", admissionState: "promoted" }, dispatchParams); + results.set(traceId, promoted); + return promoted; + } + await recordCodeAgentTurnAdmission({ payload: acceptedPayload, params, options, traceId }); + results.set(traceId, annotateOwner(acceptedPayload, params)); + traceStore.ensure(traceId, { + runnerKind: "codex-app-server-stdio-runner", + workspace: options.workspace ?? options.env?.HWLAB_CODE_AGENT_CODEX_WORKSPACE ?? options.env?.HWLAB_CODE_AGENT_WORKSPACE ?? null, + sandbox: options.env?.HWLAB_CODE_AGENT_CODEX_SANDBOX ?? null, + sessionMode: "codex-app-server-stdio-long-lived", + implementationType: "repo-owned-codex-app-server-stdio-session" + }); + traceStore.append(traceId, { + type: "request", + status: "accepted", + label: "request:accepted-short-connection", + message: "Code Agent request accepted; committed progress is delivered through the Workbench projection event stream.", + waitingFor: "codex-stdio" + }); + + const run = async () => { + try { + const billingStartedAt = Date.now(); + const billingPreflight = await preflightCodeAgentBilling({ params, options, traceId, sendResponse: false }); + void emitCodeAgentOtelSpan("billing_preflight", traceId, options.env ?? process.env, { + startTimeMs: billingStartedAt, + attributes: { sessionId: safeSessionId(params.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId, blocked: Boolean(billingPreflight?.blocked) }, + status: billingPreflight?.blocked ? "error" : "ok", + error: billingPreflight?.blocked ? billingPreflight?.payload?.error?.message ?? billingPreflight?.blocker?.summary ?? "billing preflight blocked" : null + }); + if (billingPreflight?.blocked) { + await settleAdmittedCodeAgentFailure({ + base: codeAgentAdmittedFailureBasePayload({ params, options, traceId }), + params, + options, + traceId, + results, + failure: billingPreflight, + traceLabel: "billing:preflight:failed" + }); + return; + } + const dispatchParams = { ...params, userBillingReservation: billingPreflight?.reservation ?? null }; + const payload = await runCodeAgentChat(dispatchParams, options); + if (isCodeAgentResultCanceled(results.get(traceId))) return; + await finalizeCodeAgentBillingUsage({ payload, params: dispatchParams, options }); + await recordCodeAgentSessionOwner({ payload, params: dispatchParams, options, status: codeAgentOwnerStatusForResult(payload) }); + results.set(traceId, annotateOwner(payload, dispatchParams)); + traceStore.append(traceId, { + type: "result", + status: payload.status === "completed" ? "completed" : "failed", + label: `result:${payload.status}`, + message: payload.status === "completed" + ? "Code Agent result is committed to the Workbench projection event stream." + : payload.error?.message ?? "Code Agent completed without a successful reply.", + sessionId: payload.sessionId ?? payload.session?.sessionId, + sessionStatus: payload.session?.status, + terminal: true + }); + } catch (error) { + if (isCodeAgentResultCanceled(results.get(traceId))) return; + const payload = { + status: "failed", + traceId, + conversationId: safeConversationId(params.conversationId) || null, + sessionId: safeSessionId(params.sessionId) || null, + error: { + code: "code_agent_background_failed", + layer: "api", + retryable: true, + message: error?.message ?? "Code Agent background turn failed", + userMessage: "Code Agent 后台任务失败；错误已提交到 Workbench 实时投影。" + }, + updatedAt: new Date().toISOString() + }; + await finalizeCodeAgentBillingUsage({ payload, params, options }); + results.set(traceId, payload); + traceStore.append(traceId, { + type: "result", + status: "failed", + label: "result:failed", + errorCode: payload.error.code, + message: payload.error.message, + terminal: true + }); + } + }; + setImmediate(() => { + run(); + }); + return annotateOwner(acceptedPayload, params); +} + +async function recordCodeAgentTurnAdmission({ payload = {}, params = {}, options = {}, traceId } = {}) { + const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; + let ownerRecord = null; + try { + ownerRecord = await recordCodeAgentSessionOwner({ payload, params, options, status: "running" }); + if (!ownerRecord) { + const error = new Error("Code Agent turn admission did not durably persist Workbench session ownership."); + error.code = "workbench_admission_owner_not_persisted"; + error.valuesRedacted = true; + throw error; + } + await writeWorkbenchSessionAdmissionFact({ + runtimeStore: options.runtimeStore, + session: ownerRecord, + ownerUserId: options.actor?.id ?? params.ownerUserId, + ownerRole: options.actor?.role ?? params.ownerRole, + projectId: params.projectId ?? ownerRecord.projectId ?? null, + conversationId: params.conversationId ?? ownerRecord.conversationId ?? null, + threadId: params.threadId ?? ownerRecord.threadId ?? null, + status: "running" + }); + await recordCodeAgentSessionInputFact({ + params, + options, + traceId, + delivery: "queue", + status: "admitted", + messageId: codeAgentTurnLifecycleFields(traceId, params).userMessageId, + commandId: payload.agentRun?.commandId ?? null + }); + } catch (error) { + throw codeAgentAdmissionUnavailableError(error, { params, options, traceId }); + } + traceStore.append(traceId, { + type: "request", + status: "admitted", + label: "turn:admitted", + message: "Code Agent turn was durably admitted before billing or dispatch preflight.", + sessionId: safeSessionId(params.sessionId) || null, + threadId: safeOpaqueId(params.threadId) || null, + ...codeAgentOtelTraceFields(traceId, options.env ?? process.env), + waitingFor: "billing-or-dispatch", + valuesPrinted: false + }); + void emitCodeAgentOtelSpan("durable_admission", traceId, options.env ?? process.env, { + attributes: { sessionId: safeSessionId(params.sessionId) || null, threadId: safeOpaqueId(params.threadId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId } + }); + return ownerRecord; +} + +async function promoteCodeAgentTurnAdmission({ payload = {}, params = {}, options = {}, traceId } = {}) { + const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; + const ownerRecord = await recordCodeAgentSessionOwner({ payload, params, options, status: "admitting" }); + if (!ownerRecord) { + const error = new Error("Code Agent turn promotion did not durably persist the admitting owner record."); + error.code = "workbench_admission_owner_not_persisted"; + throw error; + } + const lifecycle = codeAgentTurnLifecycleFields(traceId, params); + const realtimeCapabilities = workbenchRealtimeCapabilities(options.env ?? process.env); + if (!realtimeCapabilities.transactionalProjector) { + const runningOwner = await recordCodeAgentSessionOwner({ payload: { ...payload, status: "running" }, params, options, status: "running" }); + if (!runningOwner) { + const error = new Error("Code Agent live turn did not durably promote the session owner to running."); + error.code = "workbench_admission_owner_not_promoted"; + error.valuesRedacted = true; + throw error; + } + traceStore.append(traceId, { + type: "request", + status: "admitted", + label: "turn:admitted-live-kafka", + message: "Code Agent turn is live after AgentRun durable dispatch; lifecycle visibility is owned only by hwlab.event.v1.", + sessionId: safeSessionId(params.sessionId) || null, + runId: payload.agentRun?.runId ?? null, + commandId: payload.agentRun?.commandId ?? null, + dispatchIntentId: payload.agentRun?.dispatchIntentId ?? null, + waitingFor: "hwlab-live-kafka-sse", + terminal: false, + valuesPrinted: false + }); + return { written: false, promoted: true, outboxCommitted: false, realtimeCapabilities, liveOnly: realtimeCapabilities.liveKafkaSse, valuesPrinted: false }; + } + const inputFact = buildCodeAgentSessionInputFact({ + params, + options, + traceId, + delivery: "queue", + status: "promoted", + messageId: lifecycle.userMessageId, + commandId: payload.agentRun?.commandId ?? null, + agentRun: payload.agentRun ?? null + }); + const promotion = await persistWorkbenchTurnAdmissionPromotion({ + runtimeStore: options.runtimeStore, + session: ownerRecord, + inputFact, + payload, + params, + ownerUserId: options.actor?.id ?? params.ownerUserId, + ownerRole: options.actor?.role ?? params.ownerRole, + projectId: params.projectId ?? ownerRecord.projectId ?? null, + conversationId: params.conversationId ?? ownerRecord.conversationId ?? null, + threadId: payload.agentRun?.threadId ?? params.threadId ?? ownerRecord.threadId ?? null + }); + const runningOwner = await recordCodeAgentSessionOwner({ payload: { ...payload, status: "running" }, params, options, status: "running" }); + if (!runningOwner) { + const error = new Error("Code Agent turn promotion did not durably promote the session owner to running."); + error.code = "workbench_admission_owner_not_promoted"; + error.valuesRedacted = true; + throw error; + } + traceStore.append(traceId, { + type: "request", + status: "admitted", + label: "turn:admitted", + message: "Code Agent turn was promoted only after billing and durable AgentRun dispatch admission committed.", + sessionId: safeSessionId(params.sessionId) || null, + threadId: safeOpaqueId(payload.agentRun?.threadId ?? params.threadId) || null, + runId: payload.agentRun?.runId ?? null, + commandId: payload.agentRun?.commandId ?? null, + dispatchIntentId: payload.agentRun?.dispatchIntentId ?? null, + runnerJobId: payload.agentRun?.runnerJobId ?? null, + waitingFor: "agentrun-kafka-projection", + terminal: false, + valuesPrinted: false + }); + void emitCodeAgentOtelSpan("durable_admission", traceId, options.env ?? process.env, { + attributes: { + sessionId: safeSessionId(params.sessionId) || null, + threadId: safeOpaqueId(payload.agentRun?.threadId ?? params.threadId) || null, + turnId: lifecycle.turnId, + runId: payload.agentRun?.runId ?? null, + commandId: payload.agentRun?.commandId ?? null, + dispatchIntentId: payload.agentRun?.dispatchIntentId ?? null, + outboxCommitted: promotion?.outboxCommitted === true + } + }); + return promotion; +} + +async function persistCodeAgentAdmissionInputState({ params = {}, options = {}, traceId, status, messageId = null, commandId = null, agentRun = null, error = null } = {}) { + try { + const written = await recordCodeAgentSessionInputFact({ + params, + options, + traceId, + delivery: "queue", + status, + messageId, + commandId, + agentRun, + error + }); + if (written?.written !== true) { + const unavailable = new Error("Code Agent admission input state was not durably persisted."); + unavailable.code = "workbench_admission_input_not_persisted"; + unavailable.valuesRedacted = true; + throw unavailable; + } + return written; + } catch (cause) { + throw codeAgentAdmissionUnavailableError(cause, { params, options, traceId }); + } +} + +function codeAgentAdmissionUnavailableError(error, { params = {}, options = {}, traceId } = {}) { + const payload = codeAgentAdmissionUnavailablePayload({ error, params, options, traceId }); + return Object.assign(new Error(payload.error.message), { + code: "admission_unavailable", + statusCode: 503, + payload, + alreadyClassified: true + }); +} + +function isCodeAgentAdmissionUnavailableError(error) { + return error?.code === "admission_unavailable" && error?.payload; +} + +function isCodeAgentSubmissionUnavailableError(error) { + return Boolean(error?.payload && Number.isInteger(error?.statusCode) && ( + isCodeAgentAdmissionUnavailableError(error) + || error.code === "submission_unavailable" + || error.code === "admission_promotion_unavailable" + )); +} + +function codeAgentSubmissionUnavailableError(failure = {}, context = {}) { + const payload = codeAgentSubmissionUnavailablePayload(failure, context); + return Object.assign(new Error(payload.error.message), { + code: "submission_unavailable", + statusCode: failure.statusCode ?? 503, + payload, + alreadyClassified: true + }); +} + +function codeAgentSubmissionUnavailablePayload(failure = {}, { params = {}, options = {}, traceId } = {}) { + const source = failure.payload ?? failure; + const error = source.error ?? {}; + const blocker = source.blocker ?? null; + const message = error.message ?? blocker?.summary ?? "Code Agent submission failed before durable admission."; + return { + ...source, + ok: false, + accepted: false, + status: "failed", + traceId: safeTraceId(traceId) || null, + ...codeAgentOtelTraceFields(traceId, options.env ?? process.env), + ...codeAgentTurnLifecycleFields(traceId, params), + conversationId: safeConversationId(params.conversationId) || null, + sessionId: safeSessionId(params.sessionId) || null, + threadId: safeOpaqueId(params.threadId) || null, + error: { + ...error, + code: error.code ?? blocker?.code ?? "submission_unavailable", + layer: error.layer ?? blocker?.layer ?? "admission", + retryable: error.retryable !== false, + message, + route: error.route ?? "/v1/agent/chat" + }, + blocker: blocker ? { ...blocker, summary: blocker.summary ?? message } : null, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +function codeAgentPromotionUnavailableError(error, { payload = {}, params = {}, options = {}, traceId } = {}) { + const message = error?.message ?? "Local Workbench admission promotion failed after AgentRun durable dispatch."; + const responsePayload = codeAgentSubmissionUnavailablePayload({ + error: { + code: "admission_promotion_unavailable", + layer: "admission", + retryable: true, + message, + userMessage: "AgentRun 已持久接纳本次请求，但本地可见性提交尚未完成；请使用同一 traceId 重试补提交。", + route: "/v1/agent/chat" + }, + blocker: { + code: "admission_promotion_unavailable", + layer: "admission", + retryable: true, + summary: message + } + }, { params, options, traceId }); + responsePayload.admission = { + state: "promotion-pending", + runId: payload.agentRun?.runId ?? null, + commandId: payload.agentRun?.commandId ?? null, + dispatchIntentId: payload.agentRun?.dispatchIntentId ?? null, + runnerJobId: payload.agentRun?.runnerJobId ?? null, + durableDispatch: payload.agentRun?.durableDispatch === true, + valuesRedacted: true + }; + return Object.assign(new Error(message), { + code: "admission_promotion_unavailable", + statusCode: 503, + payload: responsePayload, + alreadyClassified: true + }); +} + +function codeAgentAdmissionUnavailablePayload({ error, params = {}, options = {}, traceId } = {}) { + const message = error?.message ?? "Code Agent turn admission is unavailable; the user message was not persisted."; + return { + ok: false, + accepted: false, + status: "failed", + traceId: safeTraceId(traceId) || null, + ...codeAgentOtelTraceFields(traceId, options.env ?? process.env), + ...codeAgentTurnLifecycleFields(traceId, params), + conversationId: safeConversationId(params.conversationId) || null, + sessionId: safeSessionId(params.sessionId) || null, + threadId: safeOpaqueId(params.threadId) || null, + error: { + code: "admission_unavailable", + layer: "admission", + retryable: true, + message, + userMessage: "本次输入尚未持久化，请保留草稿后重试。", + route: "/v1/agent/chat" + }, + blocker: { + code: "admission_unavailable", + layer: "admission", + retryable: true, + summary: message + }, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +function codeAgentDispatchFailure(error) { + const message = error?.message ?? "AgentRun dispatch failed"; + return { + statusCode: 503, + payload: { + error: { + code: error?.code ?? "agentrun_dispatch_failed", + layer: "agentrun", + retryable: true, + message, + userMessage: "AgentRun 调度失败；错误已提交到 Workbench 实时投影。", + route: "/v1/agent/chat" + }, + blocker: { + code: error?.code ?? "agentrun_dispatch_failed", + layer: "agentrun", + retryable: true, + summary: message + } + } + }; +} + +function codeAgentDispatchErrorOtelAttributes(error) { + const agentRunError = error?.agentRunError && typeof error.agentRunError === "object" ? error.agentRunError : null; + const details = agentRunError?.error?.details && typeof agentRunError.error.details === "object" ? agentRunError.error.details : null; + return { + failureKind: safeOtelText(error?.code ?? agentRunError?.failureKind, 120), + upstreamStatusCode: Number.isInteger(error?.statusCode) ? error.statusCode : null, + upstreamTraceId: safeTraceId(agentRunError?.traceId) || null, + upstreamMessage: safeOtelText(agentRunError?.message ?? error?.message, 300), + upstreamStderrTail: safeOtelText(details?.stderr, 600) + }; +} + +function safeOtelText(value, limit = 300) { + const text = String(value ?? "").replace(/[\u0000-\u001f\u007f]+/gu, " ").trim(); + return text ? text.slice(0, limit) : null; +} + +function codeAgentAdmittedFailureBasePayload({ params = {}, options = {}, traceId } = {}) { + const now = new Date().toISOString(); + return { + accepted: true, + status: "running", + shortConnection: true, + traceId, + ...codeAgentTurnLifecycleFields(traceId, params), + messageId: codeAgentTurnLifecycleFields(traceId, params).assistantMessageId, + projectId: params.projectId ?? null, + conversationId: safeConversationId(params.conversationId) || null, + sessionId: safeSessionId(params.sessionId) || null, + threadId: safeOpaqueId(params.threadId) || null, + ...codeAgentAdmissionTimingFields(now), + provider: codeAgentAgentRunAdapterEnabled(options.env ?? process.env) ? "agentrun" : "codex-stdio", + model: options.env?.HWLAB_CODE_AGENT_MODEL ?? "unknown", + backend: codeAgentAgentRunAdapterEnabled(options.env ?? process.env) ? "agentrun-v01" : "hwlab-cloud-api/code-agent-chat", + valuesPrinted: false + }; +} + +function codeAgentAdmissionTimingFields(value = null) { + const admittedAt = timestampIsoOrNow(value); + const timing = { + startedAt: admittedAt, + lastEventAt: admittedAt, + finishedAt: null, + durationMs: null, + observedAt: admittedAt, + lastEventAgeMs: 0, + valuesRedacted: true + }; + return { + createdAt: admittedAt, + updatedAt: admittedAt, + timing, + startedAt: admittedAt, + lastEventAt: admittedAt, + finishedAt: null, + durationMs: null + }; +} + +async function settleAdmittedCodeAgentFailure({ base = {}, params = {}, options = {}, traceId, results, failure = {}, traceLabel = "code-agent:failed" } = {}) { + const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; + const error = failure.payload?.error ?? failure.error ?? {}; + const blocker = failure.payload?.blocker ?? failure.blocker ?? null; + const message = error.message ?? blocker?.summary ?? "Code Agent request failed after admission."; + traceStore.append(traceId, { + type: "result", + status: "failed", + label: traceLabel, + errorCode: error.code ?? "code_agent_failed_after_admission", + message, + terminal: true, + valuesPrinted: false + }); + const now = new Date().toISOString(); + const payload = annotateOwner({ + ...base, + accepted: true, + status: "failed", + traceId, + ...codeAgentTurnLifecycleFields(traceId, base), + messageId: base.messageId ?? codeAgentTurnLifecycleFields(traceId, base).assistantMessageId, + conversationId: safeConversationId(base.conversationId ?? params.conversationId) || null, + sessionId: safeSessionId(base.sessionId ?? params.sessionId) || null, + threadId: safeOpaqueId(base.threadId ?? params.threadId) || null, + error: { + code: error.code ?? "code_agent_failed_after_admission", + layer: error.layer ?? blocker?.layer ?? "api", + retryable: error.retryable !== false, + message, + userMessage: error.userMessage ?? message, + route: error.route ?? "/v1/agent/chat" + }, + blocker: blocker ? { + code: blocker.code ?? error.code ?? "code_agent_failed_after_admission", + layer: blocker.layer ?? error.layer ?? "api", + retryable: blocker.retryable !== false, + summary: blocker.summary ?? message + } : null, + finalResponse: { + text: error.userMessage ?? message, + textChars: String(error.userMessage ?? message).length, + role: "assistant", + status: "failed", + traceId, + updatedAt: now, + source: "code-agent-admitted-failure", + valuesPrinted: false + }, + runnerTrace: traceStore.snapshot(traceId), + updatedAt: now, + valuesPrinted: false, + secretMaterialStored: false + }, params); + await finalizeCodeAgentBillingUsage({ payload, params, options }); + payload.runnerTrace = traceStore.snapshot(traceId); + recordCodeAgentConversationFact(payload, options); + results?.set?.(traceId, payload); + await recordCodeAgentSessionOwner({ payload, params, options, status: codeAgentOwnerStatusForResult(payload) }); + return payload; +} diff --git a/internal/cloud/server-code-agent-http-support.ts b/internal/cloud/server-code-agent-http-support.ts new file mode 100644 index 00000000..a466597d --- /dev/null +++ b/internal/cloud/server-code-agent-http-support.ts @@ -0,0 +1,1614 @@ +// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-0102 Agent编排 draft-2026-06-17-r0; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. +// Responsibility: Shared Code Agent HTTP lifecycle, billing, ownership, projection-read, and response helpers. + +import { createHash, randomUUID } from "node:crypto"; + +import { M3_IO_CONTROL_ROUTE, M3_STATUS_ROUTE, describeM3StatusLive, handleM3IoControl } from "./m3-io-control.ts"; +import { + agentRunSessionEvidence, + cancelAgentRunChatTurn, + codeAgentAgentRunAdapterEnabled, + initialAgentRunChatResult, + loadPersistedAgentRunResult, + steerAgentRunChatTurn, + submitAgentRunChatTurn +} from "./code-agent-agentrun-adapter.ts"; +import { createCodeAgentErrorPayload, handleCodeAgentChat } from "./code-agent-chat.ts"; +import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; +import { codeAgentSessionLifecycleSummary } from "./code-agent-session-lifecycle.ts"; +import { messageAuthorityTextValue } from "./code-agent-agentrun-prompt.ts"; +import { recordWorkbenchSessionOwner } from "./workbench-projection-writer.ts"; +import { createWorkbenchReadModel } from "./workbench-read-model.ts"; +import { codeAgentOtelTraceFields, emitCodeAgentOtelSpan } from "./otel-trace.ts"; +import { + firstHeaderValue, + getHeader, + parsePositiveInteger, + positiveInteger, + readBody, + safeConversationId, + safeOpaqueId, + safeSessionId, + safeTraceId, + sendJson, + truthyFlag +} from "./server-http-utils.ts"; + +const DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT = 120; +const DEFAULT_CODE_AGENT_TRACE_PAGE_LIMIT = 100; +const MAX_CODE_AGENT_TRACE_PAGE_LIMIT = 100; +const DEFAULT_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS = 2500; +const DEFAULT_CODE_AGENT_PROJECT_ID = "prj_hwpod_workbench"; +const DEFAULT_CODE_AGENT_PROVIDER_PROFILE = "deepseek"; +const CODE_AGENT_TERMINAL_STATUSES = new Set(["completed", "failed", "blocked", "timeout", "cancelled", "canceled"]); +const CODE_AGENT_PROVIDER_PROFILE_ALIASES = Object.freeze({ codex: "codex-api" }); +const CODE_AGENT_PROVIDER_PROFILE_ID_PATTERN = /^[a-z0-9][a-z0-9.-]{0,63}$/u; +const CODE_AGENT_PROVIDER_PROFILE_LABELS = Object.freeze({ + "deepseek": "DeepSeek", + "codex-api": "Codex API", + "gpt.pika": "gpt.pika", + "minimax-m3": "MiniMax-M3 via AgentRun", + "runtime-default": "运行默认" +}); +const DEFAULT_CODE_AGENT_CODEX_API_MODEL = "gpt-5.5"; +const DEFAULT_CODE_AGENT_CODEX_API_BASE_URL = "http://127.0.0.1:49280/responses"; +const DEFAULT_CODE_AGENT_DEEPSEEK_MODEL = "deepseek-chat"; +const DEFAULT_CODE_AGENT_MINIMAX_M3_MODEL = "MiniMax-M3"; + + +function normalizeCodeAgentProviderProfile(value) { + const text = String(value ?? DEFAULT_CODE_AGENT_PROVIDER_PROFILE).trim().toLowerCase(); + if (!text) return DEFAULT_CODE_AGENT_PROVIDER_PROFILE; + if (text === "runtime-default") return text; + const normalized = CODE_AGENT_PROVIDER_PROFILE_ALIASES[text] ?? text; + return CODE_AGENT_PROVIDER_PROFILE_ID_PATTERN.test(normalized) ? normalized : DEFAULT_CODE_AGENT_PROVIDER_PROFILE; +} + +function firstNonEmptyValue(...values) { + for (const value of values) { + const text = String(value ?? "").trim(); + if (text) return text; + } + return null; +} + +async function recordCodeAgentSessionInputFact({ params = {}, options = {}, traceId, delivery = "queue", status = "admitted", messageId = null, commandId = null, agentRun = null, error = null } = {}) { + const runtimeStore = options.runtimeStore ?? null; + if (typeof runtimeStore?.writeWorkbenchFacts !== "function") return null; + const fact = buildCodeAgentSessionInputFact({ params, options, traceId, delivery, status, messageId, commandId, agentRun, error }); + if (!fact) return null; + return runtimeStore.writeWorkbenchFacts({ facts: { inputs: [fact] } }, codeAgentSessionInputRequestMeta(fact, params, options)); +} + +function buildCodeAgentSessionInputFact({ params = {}, options = {}, traceId, delivery = "queue", status = "admitted", messageId = null, commandId = null, agentRun = null, error = null } = {}) { + const resolvedTraceId = safeTraceId(traceId ?? params.traceId) || null; + const lifecycle = codeAgentTurnLifecycleFields(resolvedTraceId, params); + const sessionId = safeSessionId(params.sessionId ?? params.session?.sessionId) || null; + if (!sessionId) return null; + const resolvedMessageId = safeMessageId(messageId ?? params.messageId ?? lifecycle.userMessageId) || lifecycle.userMessageId; + const resolvedCommandId = textValue(commandId ?? agentRun?.commandId ?? params.commandId ?? params.agentRun?.commandId) || null; + const normalizedDelivery = codeAgentInputDelivery(delivery); + const normalizedStatus = codeAgentInputStatus(status); + const now = new Date().toISOString(); + const sourceEventId = `${sessionId}:${lifecycle.turnId}:${normalizedDelivery}:${resolvedTraceId ?? resolvedCommandId ?? resolvedMessageId}`; + // Command allocation happens after the initial admitting write. Keep one stable + // row across admitting -> blocked/failed/promoted transitions for the trace. + const inputId = stableCodeAgentInputId({ sessionId, turnId: lifecycle.turnId, traceId: resolvedTraceId, messageId: resolvedMessageId, delivery: normalizedDelivery, sourceEventId }); + return { + inputId, + sessionId, + turnId: lifecycle.turnId, + traceId: resolvedTraceId, + messageId: resolvedMessageId, + commandId: resolvedCommandId, + runId: textValue(agentRun?.runId ?? params.agentRun?.runId) || null, + dispatchIntentId: textValue(agentRun?.dispatchIntentId ?? params.agentRun?.dispatchIntentId) || null, + runnerJobId: textValue(agentRun?.runnerJobId ?? params.agentRun?.runnerJobId) || null, + durableDispatch: agentRun?.durableDispatch === true || params.agentRun?.durableDispatch === true, + delivery: normalizedDelivery, + status: normalizedStatus, + errorCode: textValue(error?.code ?? error?.error?.code) || null, + sourceEventId, + promptHash: params.message || params.prompt || params.text ? sha256Text(params.message ?? params.prompt ?? params.text).slice(0, 16) : null, + textBytes: params.message || params.prompt || params.text ? Buffer.byteLength(String(params.message ?? params.prompt ?? params.text), "utf8") : 0, + valuesRedacted: true, + secretMaterialStored: false, + createdAt: now, + updatedAt: now + }; +} + +function codeAgentSessionInputRequestMeta(fact, params = {}, options = {}) { + return { + sessionId: fact.sessionId, + traceId: fact.traceId, + turnId: fact.turnId, + messageId: fact.messageId, + ownerUserId: options.actor?.id ?? params.ownerUserId, + projectId: params.projectId ?? DEFAULT_CODE_AGENT_PROJECT_ID, + valuesPrinted: false + }; +} + +function stableCodeAgentInputId(value) { + const payload = Object.fromEntries(Object.entries(value ?? {}).filter(([, item]) => item !== undefined && item !== null && item !== "").sort(([left], [right]) => left.localeCompare(right))); + return `wsi_${sha256Text(JSON.stringify(payload)).slice(0, 32)}`; +} + +function codeAgentInputDelivery(value) { + const text = textValue(value).toLowerCase(); + return ["queue", "steer", "cancel"].includes(text) ? text : "queue"; +} + +function codeAgentInputStatus(value) { + const text = textValue(value).toLowerCase().replace(/-/gu, "_"); + return ["admitting", "admitted", "promoted", "blocked", "failed", "canceled", "completed"].includes(text) ? text : "admitted"; +} + +function codeAgentResultTimingFields(...sources) { + const records = sources.filter((source) => source && typeof source === "object"); + const timingSource = records.map((source) => source.timing).find((timing) => timing && typeof timing === "object") ?? {}; + const startedAt = firstTimestampIso(timingSource.startedAt, ...records.map((source) => source.startedAt), ...records.map((source) => source.createdAt)); + const lastEventAt = firstLatestTimestampIso(timingSource.lastEventAt, ...records.map((source) => source.lastEventAt), ...records.map((source) => source.updatedAt), ...records.map((source) => source.createdAt)); + const finishedAt = firstLatestTimestampIso(timingSource.finishedAt, ...records.map((source) => source.finishedAt), ...records.map((source) => source.completedAt)); + const durationMs = numberOrNull(timingSource.durationMs ?? records.map((source) => source.durationMs).find((value) => numberOrNull(value) !== null)); + const observedAt = timestampIsoOrNull(timingSource.observedAt); + const lastEventAgeMs = numberOrNull(timingSource.lastEventAgeMs); + if (!startedAt && !lastEventAt && !finishedAt && durationMs === null && lastEventAgeMs === null) return {}; + const timing = { + ...(timingSource && typeof timingSource === "object" ? timingSource : {}), + startedAt: startedAt ?? null, + lastEventAt: lastEventAt ?? null, + finishedAt: finishedAt ?? null, + durationMs, + observedAt: observedAt ?? null, + lastEventAgeMs, + valuesRedacted: timingSource.valuesRedacted !== false + }; + return { + timing, + startedAt: timing.startedAt, + lastEventAt: timing.lastEventAt, + finishedAt: timing.finishedAt, + durationMs: timing.durationMs + }; +} + +function firstTimestampIso(...values) { + for (const value of values) { + const timestamp = timestampIsoOrNull(value); + if (timestamp) return timestamp; + } + return null; +} + +function firstLatestTimestampIso(...values) { + let latest = null; + let latestMs = Number.NEGATIVE_INFINITY; + for (const value of values) { + const timestamp = timestampIsoOrNull(value); + if (!timestamp) continue; + const ms = Date.parse(timestamp); + if (!Number.isFinite(ms) || ms < latestMs) continue; + latest = timestamp; + latestMs = ms; + } + return latest; +} + +function codeAgentChatShortConnectionRequested(request, params, options = {}) { + const header = firstHeaderValue(request, "prefer"); + const explicit = firstHeaderValue(request, "x-hwlab-short-connection"); + const envValue = options.env?.HWLAB_CODE_AGENT_CHAT_SHORT_CONNECTION; + return codeAgentAgentRunAdapterEnabled(options.env ?? process.env) || + /(?:^|,)\s*respond-async\s*(?:,|$)/iu.test(String(header ?? "")) || + truthyFlag(explicit) || + params?.shortConnection === true || + truthyFlag(envValue); +} + +async function preflightCodeAgentBilling({ params = {}, options = {}, traceId, response, sendResponse = true } = {}) { + const client = options.userBillingClient; + if (!codeAgentBillingEnabled(options) || !client?.configured || !options.userBillingAuth?.active) return null; + const estimatedCredits = parsePositiveInteger(options.env?.HWLAB_USER_BILLING_CODE_AGENT_ESTIMATED_CREDITS, 1); + const body = { + ...(options.actorApiKeySecret ? { apiKey: options.actorApiKeySecret } : { userId: options.actor?.id }), + serviceId: "hwlab-code-agent", + estimatedCredits, + idempotencyKey: `code-agent:${traceId}:preflight`, + metadata: codeAgentBillingMetadata(params, traceId, { stage: "preflight" }) + }; + const result = await client.billingPreflight(body); + if (result.ok && result.body?.allowed !== false) return { reservation: sanitizeBillingReservation(result.body) }; + const status = [402, 403, 429].includes(Number(result.status)) ? Number(result.status) : 503; + const payload = { + ok: false, + accepted: false, + status: status === 402 ? "payment_required" : status === 403 ? "not_entitled" : status === 429 ? "rate_limited" : "billing_unavailable", + traceId, + error: { + code: result.error?.code ?? (status === 402 ? "insufficient_credits" : "user_billing_preflight_failed"), + layer: "billing", + retryable: status === 429 || status === 503, + message: result.error?.message ?? "Code Agent billing preflight failed", + route: "/v1/agent/chat" + }, + blocker: { + code: result.error?.code ?? (status === 402 ? "insufficient_credits" : "user_billing_preflight_failed"), + layer: "billing", + retryable: status === 429 || status === 503, + summary: result.error?.message ?? "Code Agent billing preflight failed" + }, + valuesRedacted: true + }; + if (sendResponse && response) sendJson(response, status, payload); + return { blocked: true, statusCode: status, payload, error: payload.error, blocker: payload.blocker }; +} + +async function recordCodeAgentBillingUsage({ payload = {}, params = {}, options = {} } = {}) { + const reservation = params.userBillingReservation ?? payload.userBillingReservation; + const reservationId = typeof reservation?.reservationId === "string" ? reservation.reservationId : ""; + const client = options.userBillingClient; + if (payload.billing?.recorded === true || payload.billing?.released === true || payload.status === "running") return payload.billing ?? null; + if (!reservationId || !client?.configured) return null; + const traceId = safeTraceId(payload.traceId ?? params.traceId); + const usedTokens = codeAgentUsedTokens(payload); + const usedCredits = parsePositiveInteger(options.env?.HWLAB_USER_BILLING_CODE_AGENT_USED_CREDITS, 1); + const result = await client.billingRecord({ + reservationId, + serviceId: "hwlab-code-agent", + usedCredits, + usedTokens, + idempotencyKey: `code-agent:${traceId}:record`, + metadata: codeAgentBillingMetadata(params, traceId, { stage: "record", status: payload.status ?? "unknown" }) + }); + const billing = result.ok ? sanitizeBillingRecord(result.body, reservation) : { + recorded: false, + reservationId, + errorCode: result.error?.code ?? "user_billing_record_failed", + valuesRedacted: true + }; + payload.billing = billing; + const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; + if (traceId) { + traceStore.append(traceId, { + type: "billing", + status: result.ok ? "recorded" : "degraded", + label: result.ok ? "billing:recorded" : "billing:record_failed", + reservationId, + recordId: result.ok ? result.body?.recordId ?? null : null, + errorCode: result.ok ? null : billing.errorCode, + valuesPrinted: false + }); + } + return billing; +} + +async function finalizeCodeAgentBillingUsage({ payload = {}, params = {}, options = {} } = {}) { + if (payload.billing?.recorded === true || payload.billing?.released === true) return payload.billing; + if (payload.status === "running") return null; + if (payload.status === "completed") { + return recordCodeAgentBillingUsage({ payload, params, options }); + } + return releaseCodeAgentBillingReservation({ payload, params, options }); +} + +async function releaseCodeAgentBillingReservation({ payload = {}, params = {}, options = {} } = {}) { + const reservation = params.userBillingReservation ?? payload.userBillingReservation; + const reservationId = typeof reservation?.reservationId === "string" ? reservation.reservationId : ""; + const client = options.userBillingClient; + if (payload.billing?.recorded === true || payload.billing?.released === true || !reservationId || !client?.configured) return payload.billing ?? null; + const traceId = safeTraceId(payload.traceId ?? params.traceId); + const releaseReason = String(payload.status ?? "not_completed").trim() || "not_completed"; + const releaseBody = { + reservationId, + serviceId: "hwlab-code-agent", + idempotencyKey: `code-agent:${traceId}:release`, + reason: releaseReason, + metadata: codeAgentBillingMetadata(params, traceId, { + stage: "release", + status: payload.status ?? "unknown", + errorCode: payload.error?.code ?? payload.blocker?.code ?? null + }) + }; + const result = typeof client.billingRelease === "function" + ? await client.billingRelease(releaseBody) + : { ok: false, error: { code: "user_billing_release_not_supported" } }; + const billing = result.ok ? sanitizeBillingRelease(result.body, reservation) : { + released: false, + reservationId, + errorCode: result.error?.code ?? "user_billing_release_failed", + valuesRedacted: true + }; + payload.billing = billing; + const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; + if (traceId) { + traceStore.append(traceId, { + type: "billing", + status: result.ok ? "released" : "degraded", + label: result.ok ? "billing:released" : "billing:release_failed", + reservationId, + reason: releaseReason, + errorCode: result.ok ? null : billing.errorCode, + valuesPrinted: false + }); + } + return billing; +} + +function withCodeAgentBillingReservation(payload = {}, params = {}) { + if (!payload || typeof payload !== "object" || payload.userBillingReservation) return payload; + const reservation = params.userBillingReservation; + return reservation ? { ...payload, userBillingReservation: reservation } : payload; +} + +function codeAgentBillingEnabled(options = {}) { + const env = options.env ?? process.env; + if (String(env.HWLAB_USER_BILLING_CODE_AGENT_ENABLED ?? "").trim() === "0") return false; + return Boolean(options.userBillingClient?.configured); +} + +function codeAgentBillingMetadata(params = {}, traceId, extra = {}) { + return { + traceId, + conversationId: safeConversationId(params.conversationId) || null, + sessionId: safeSessionId(params.sessionId) || null, + projectId: textValue(params.projectId) || null, + route: "/v1/agent/chat", + resource: "code-agent-turn", + ...extra, + valuesRedacted: true + }; +} + +function sanitizeBillingReservation(value = {}) { + return { + allowed: value.allowed === true, + reservationId: textValue(value.reservationId) || null, + resourceType: textValue(value.resourceType) || null, + planId: textValue(value.planId) || null, + estimatedCredits: Number.isFinite(Number(value.estimatedCredits)) ? Number(value.estimatedCredits) : null, + estimatedTokens: Number.isFinite(Number(value.estimatedTokens)) ? Number(value.estimatedTokens) : null, + expiresAt: textValue(value.expiresAt) || null, + valuesRedacted: true + }; +} + +function sanitizeBillingRecord(value = {}, reservation = {}) { + return { + recorded: true, + reservationId: textValue(reservation.reservationId) || null, + recordId: textValue(value.recordId) || null, + credits: Number.isFinite(Number(value.credits)) ? Number(value.credits) : null, + balance: Number.isFinite(Number(value.balance)) ? Number(value.balance) : null, + valuesRedacted: true + }; +} + +function sanitizeBillingRelease(value = {}, reservation = {}) { + return { + released: true, + reservationId: textValue(value.reservationId) || textValue(reservation.reservationId) || null, + releasedCredits: Number.isFinite(Number(value.releasedCredits)) ? Number(value.releasedCredits) : null, + status: textValue(value.status) || "cancelled", + valuesRedacted: true + }; +} + +function codeAgentUsedTokens(payload = {}) { + const usage = payload.usage && typeof payload.usage === "object" ? payload.usage : null; + const traceSummary = payload.traceSummary && typeof payload.traceSummary === "object" ? payload.traceSummary : null; + const candidates = [ + usage?.totalTokens, + usage?.total_tokens, + usage?.tokens, + Number(usage?.inputTokens ?? usage?.input_tokens ?? 0) + Number(usage?.outputTokens ?? usage?.output_tokens ?? 0), + traceSummary?.tokenCount, + traceSummary?.tokens + ]; + for (const value of candidates) { + const parsed = Number(value); + if (Number.isFinite(parsed) && parsed > 0) return Math.ceil(parsed); + } + return 0; +} + +function normalizeTurnStatus(...values) { + for (const value of values) { + const text = textValue(value).toLowerCase().replace(/_/gu, "-"); + if (!text) continue; + if (["accepted", "pending", "processing", "running", "busy", "creating", "queued", "in-flight"].includes(text)) return "running"; + if (["completed", "done", "succeeded", "success"].includes(text)) return "completed"; + if (["failed", "failure", "error", "stale", "thread-resume-failed", "aborted", "interrupted", "expired"].includes(text)) return "failed"; + if (["cancelled", "canceled"].includes(text)) return "canceled"; + if (["blocked", "timeout"].includes(text)) return text; + } + return null; +} + +function recordCodeAgentConversationFact(payload = {}, options = {}) { + const registry = options.sessionRegistry; + const conversationId = safeConversationId(payload.conversationId); + if (!conversationId || typeof registry?.recordFact !== "function") return null; + try { + return registry.recordFact(conversationId, { + kind: payload.status === "completed" ? "runner_turn" : "runner_turn_terminal", + conversationId, + sessionId: payload.sessionId, + traceId: payload.traceId, + provider: payload.provider, + backend: payload.backend, + workspace: payload.workspace, + sandbox: payload.sandbox, + session: payload.session, + sessionMode: payload.sessionMode, + sessionReuse: payload.sessionReuse, + implementationType: payload.implementationType, + runner: payload.runner, + runnerTrace: payload.runnerTrace, + partialContext: payload.partialContext ?? payload.runnerTrace?.partialContext, + capabilityLevel: payload.capabilityLevel, + toolCalls: payload.toolCalls, + skills: payload.skills + }, { now: options.now }); + } catch { + return null; + } +} + +async function recordCodeAgentSessionOwner({ payload = {}, params = {}, options = {}, status = "active", preserveLastTraceId = false } = {}) { + const ownerUserId = options.actor?.id ?? params.ownerUserId; + if (!ownerUserId || !options.accessController?.recordAgentSessionOwner) return null; + const traceId = safeTraceId(payload.traceId ?? params.traceId); + const session = payload.session && typeof payload.session === "object" ? payload.session : null; + const sessionReuse = payload.sessionReuse && typeof payload.sessionReuse === "object" ? payload.sessionReuse : null; + const sessionId = safeSessionId(payload.sessionId ?? session?.sessionId ?? sessionReuse?.sessionId ?? params.sessionId) || null; + const conversationId = safeConversationId(payload.conversationId ?? session?.conversationId ?? sessionReuse?.conversationId ?? params.conversationId) || null; + const threadId = safeOpaqueId(session?.threadId ?? sessionReuse?.threadId ?? payload.threadId ?? params.threadId) || null; + return recordWorkbenchSessionOwner({ + accessController: options.accessController, + traceId, + ownerUserId, + ownerRole: options.actor?.role ?? params.ownerRole ?? null, + sessionId, + projectId: params.projectId ?? payload.projectId ?? null, + conversationId, + threadId, + status, + preserveLastTraceId, + session: codeAgentSessionOwnerEvidence(payload, params) + }); +} + +function codeAgentOwnerStatusForResult(result = {}) { + const sessionStatus = textValue(result?.session?.status ?? result?.sessionSummary?.status ?? result?.sessionLifecycleStatus); + if (sessionStatus === "thread-resume-failed") return "thread-resume-failed"; + if (codeAgentFreshContinuationFailure(result)) return "thread-resume-failed"; + if (codeAgentPayloadHasSealedFinalResponse(result)) return "completed"; + if (result?.status === "completed") return "completed"; + if (result?.status === "canceled" || result?.status === "cancelled") return "canceled"; + return result?.status ?? "active"; +} + +function textValue(value) { + return String(value ?? "").trim(); +} + +function timestampIsoOrNow(value = null) { + const parsed = Date.parse(String(value ?? "")); + return Number.isFinite(parsed) ? new Date(parsed).toISOString() : new Date().toISOString(); +} + +function timestampIsoOrNull(value = null) { + const parsed = Date.parse(String(value ?? "")); + return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null; +} + +function codeAgentSessionOwnerEvidence(payload = {}, params = {}) { + const traceId = payload.traceId ?? params.traceId ?? null; + const finalResponse = codeAgentFinalResponseEvidence(payload, traceId); + const traceSummary = codeAgentTraceSummaryEvidence(payload, traceId, finalResponse); + const messages = codeAgentConversationMessagesEvidence(payload, params, traceId, finalResponse); + const firstUserMessagePreview = messages.find((message) => message.role === "user")?.text?.slice(0, 240) ?? null; + const promptEvidence = codeAgentPromptMetadata(payload, params, traceId); + const traceResult = codeAgentTraceResultEvidence(payload, params, traceId, finalResponse, traceSummary); + const providerProfile = codeAgentProviderProfileEvidence(payload, params); + const continuation = codeAgentContinuationEvidence(payload, params); + const sessionLifecycleStatus = continuation.requiresFreshSession ? "thread-resume-failed" : payload.sessionLifecycleStatus ?? null; + return { + ...(providerProfile ? { providerProfile, codeAgentProviderProfile: providerProfile } : {}), + provider: payload.provider ?? null, + model: payload.model ?? null, + backend: payload.backend ?? null, + status: payload.status ?? null, + sessionMode: payload.sessionMode ?? payload.session?.sessionMode ?? null, + sessionLifecycleStatus, + continuation, + runnerKind: payload.runner?.kind ?? payload.runnerTrace?.runnerKind ?? null, + conversationId: payload.conversationId ?? params.conversationId ?? null, + sessionId: payload.sessionId ?? payload.session?.sessionId ?? payload.sessionReuse?.sessionId ?? params.sessionId ?? null, + threadId: payload.session?.threadId ?? payload.sessionReuse?.threadId ?? payload.threadId ?? params.threadId ?? null, + traceId, + projectId: payload.projectId ?? params.projectId ?? null, + ...codeAgentPromptFields(promptEvidence), + finalResponse, + traceSummary, + ...(traceResult ? { traceResults: { [traceResult.traceId]: traceResult } } : {}), + ...(messages.length ? { messages, messageCount: messages.length } : {}), + ...(firstUserMessagePreview ? { firstUserMessagePreview } : {}), + ...agentRunSessionEvidence(payload), + secretMaterialStored: false, + valuesRedacted: true + }; +} + +function codeAgentContinuationEvidence(payload = {}, params = {}) { + const failure = codeAgentFreshContinuationFailure(payload); + const providerProfile = codeAgentProviderProfileEvidence(payload, params); + const traceId = safeTraceId(payload.traceId ?? params.traceId) || null; + if (!failure) { + return { + requiresFreshSession: false, + providerProfile, + traceId, + valuesRedacted: true + }; + } + return { + requiresFreshSession: true, + reasonCode: failure.code, + reason: failure.message, + staleThreadId: safeOpaqueId(payload.session?.threadId ?? payload.sessionReuse?.threadId ?? payload.threadId ?? params.threadId ?? payload.agentRun?.threadId) || null, + providerProfile, + traceId, + valuesRedacted: true + }; +} + +function codeAgentFreshContinuationFailure(payload = {}) { + if (!payload || typeof payload !== "object") return null; + const sessionStatus = String(payload.session?.status ?? payload.sessionLifecycleStatus ?? payload.sessionReuse?.status ?? "").trim().toLowerCase().replace(/_/gu, "-"); + if (sessionStatus === "thread-resume-failed") return { code: "thread-resume-failed", message: "previous AgentRun thread/session resume failed", valuesRedacted: true }; + const status = normalizeTurnStatus(payload.status, payload.agentRun?.terminalStatus, payload.agentRun?.commandState, payload.agentRun?.status, payload.session?.status, payload.sessionReuse?.status); + if (!status || !["failed", "blocked", "timeout"].includes(status)) return null; + const code = normalizeCodeAgentFailureKind(firstNonEmptyValue( + payload.error?.code, + payload.blocker?.code, + payload.providerTrace?.failureKind, + payload.providerTrace?.errorCode, + payload.agentRun?.failureKind, + payload.agentRun?.errorCode, + payload.transient?.failureKind, + payload.transient?.errorCode + )); + const message = firstNonEmptyValue( + payload.error?.message, + payload.error?.userMessage, + payload.blocker?.summary, + payload.blocker?.message, + payload.providerTrace?.failureMessage, + payload.providerTrace?.summary, + payload.agentRun?.failureMessage, + payload.agentRun?.message + ) ?? ""; + if (!codeAgentFailureRequiresFreshContinuation(code, message)) return null; + return { code: code ?? "stale-continuation", message: message || code || "stale continuation", valuesRedacted: true }; +} + +function normalizeCodeAgentFailureKind(value) { + const normalized = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); + return normalized || null; +} + +function codeAgentFailureRequiresFreshContinuation(code, message = "") { + const failureKind = normalizeCodeAgentFailureKind(code); + if (["thread-resume-failed", "session-store-evicted", "provider-stream-disconnected"].includes(failureKind)) return true; + return /session stor(?:e|age).*evicted|pvc-backed session|no rollout found for thread id|no rollout found|thread\/resume failed|provider stream disconnected/iu.test(String(message ?? "")); +} + +function codeAgentProviderProfileEvidence(payload = {}, params = {}) { + const raw = firstNonEmptyValue( + params.providerProfile, + params.codeAgentProviderProfile, + payload.providerProfile, + payload.codeAgentProviderProfile, + payload.agentRun?.backendProfile, + payload.providerTrace?.backendProfile, + payload.backendProfile + ); + if (!raw) return null; + const normalized = normalizeCodeAgentProviderProfile(raw); + return normalized === "runtime-default" ? null : normalized; +} + +function codeAgentTraceResultEvidence(payload = {}, params = {}, traceId = null, finalResponse = null, traceSummary = null) { + const resolvedTraceId = safeTraceId(traceId); + if (!resolvedTraceId) return null; + const agentRun = agentRunSessionEvidence(payload).agentRun ?? null; + const userBillingReservation = codeAgentBillingReservationEvidence(payload.userBillingReservation ?? params.userBillingReservation); + const resultSession = payload.session && typeof payload.session === "object" ? payload.session : null; + const sessionReuse = payload.sessionReuse && typeof payload.sessionReuse === "object" ? payload.sessionReuse : null; + const conversationId = safeConversationId(payload.conversationId ?? agentRun?.conversationId ?? resultSession?.conversationId ?? sessionReuse?.conversationId ?? params.conversationId) || null; + const sessionId = safeSessionId(payload.sessionId ?? resultSession?.sessionId ?? sessionReuse?.sessionId ?? params.sessionId) || null; + const threadId = safeOpaqueId(payload.threadId ?? resultSession?.threadId ?? sessionReuse?.threadId ?? params.threadId ?? agentRun?.threadId) || null; + const promptEvidence = codeAgentPromptMetadata(payload, params, resolvedTraceId); + if (!agentRun && !finalResponse && !traceSummary) return null; + return { + traceId: resolvedTraceId, + status: codeAgentPayloadHasSealedFinalResponse(payload) ? "completed" : payload.status ?? agentRun?.terminalStatus ?? agentRun?.commandState ?? null, + conversationId, + sessionId, + threadId, + ...codeAgentPromptFields(promptEvidence), + messageId: finalResponse?.messageId ?? payload.messageId ?? null, + createdAt: payload.createdAt ?? finalResponse?.createdAt ?? null, + updatedAt: payload.updatedAt ?? finalResponse?.updatedAt ?? null, + finalResponse, + traceSummary, + agentRun, + ...(userBillingReservation ? { userBillingReservation } : {}), + valuesRedacted: true, + secretMaterialStored: false + }; +} + +function codeAgentPromptMetadata(payload = {}, params = {}, traceId = null) { + const prompt = payload?.prompt && typeof payload.prompt === "object" ? payload.prompt : null; + const rawPrompt = firstNonEmptyValue( + params.message, + params.prompt, + params.text, + payload.userMessage, + typeof payload.prompt === "string" ? payload.prompt : null + ); + const fromText = codeAgentPromptMetadataFromText(rawPrompt, traceId, "code-agent-request", { fullText: Boolean(rawPrompt) }); + return codeAgentPromptMetadataFromParts({ + traceId, + source: prompt?.source ?? payload.promptSource ?? fromText?.prompt?.source ?? "code-agent-request", + promptId: firstNonEmptyValue(payload.promptId, prompt?.id, fromText?.promptId), + promptSha256: firstNonEmptyValue(payload.promptSha256, prompt?.sha256, fromText?.promptSha256), + promptSummary: firstNonEmptyValue(payload.promptSummary, prompt?.summary, fromText?.promptSummary), + promptPreview: firstNonEmptyValue(payload.promptPreview, prompt?.preview, fromText?.promptPreview), + textChars: Number.isInteger(prompt?.textChars) ? prompt.textChars : fromText?.prompt?.textChars + }); +} + +function codeAgentPromptMetadataFromText(value, traceId = null, source = "code-agent-request", options = {}) { + const promptText = String(value ?? "").trim(); + if (!promptText) return null; + return codeAgentPromptMetadataFromParts({ + traceId, + source, + promptSha256: options.fullText === true ? sha256Text(promptText) : undefined, + promptSummary: promptText, + promptPreview: promptText, + textChars: options.fullText === true ? promptText.length : undefined + }); +} + +function codeAgentPromptMetadataFromParts({ traceId = null, source = "code-agent-request", promptId = null, promptSha256 = null, promptSummary = null, promptPreview = null, textChars = undefined } = {}) { + const digest = firstNonEmptyValue(promptSha256); + const summary = clippedPromptText(firstNonEmptyValue(promptSummary, promptPreview), 240); + const preview = clippedPromptText(firstNonEmptyValue(promptPreview, promptSummary), 500); + const explicitId = firstNonEmptyValue(promptId); + if (!explicitId && !digest && !summary && !preview) return null; + const id = explicitId || codeAgentPromptId(traceId, digest); + const prompt = compactCodeAgentObject({ + id, + sha256: digest || undefined, + summary, + preview, + textChars: Number.isInteger(textChars) ? textChars : undefined, + source, + valuesPrinted: false + }); + return compactCodeAgentObject({ + promptId: id, + promptSha256: digest || undefined, + promptSummary: summary, + promptPreview: preview, + prompt, + valuesPrinted: false + }) ?? null; +} + +function codeAgentPromptFields(metadata = null) { + if (!metadata) return {}; + return compactCodeAgentObject({ + promptId: metadata.promptId, + promptSha256: metadata.promptSha256, + promptSummary: metadata.promptSummary, + promptPreview: metadata.promptPreview, + prompt: metadata.prompt + }) ?? {}; +} + +function codeAgentPromptId(traceId = null, digest = null) { + const traceSuffix = safeTraceId(traceId)?.slice(4).replace(/[^A-Za-z0-9_.:-]/gu, "_"); + const digestSuffix = firstNonEmptyValue(digest)?.slice(0, 16); + const suffix = firstNonEmptyValue(traceSuffix, digestSuffix); + return suffix ? `prm_${suffix}` : undefined; +} + +function clippedPromptText(value, limit) { + const promptText = String(value ?? "").trim(); + if (!promptText) return undefined; + return promptText.length > limit ? `${promptText.slice(0, limit)}...` : promptText; +} + +function sha256Text(value) { + return createHash("sha256").update(String(value)).digest("hex"); +} + +function compactCodeAgentObject(value) { + const result = {}; + for (const [key, item] of Object.entries(value ?? {})) { + if (item === undefined || item === null) continue; + if (Array.isArray(item) && item.length === 0) continue; + if (typeof item === "object" && !Array.isArray(item) && Object.keys(item).length === 0) continue; + result[key] = item; + } + return Object.keys(result).length > 0 ? result : undefined; +} + +function codeAgentBillingReservationEvidence(value = null) { + if (!value || typeof value !== "object") return null; + const reservationId = textValue(value.reservationId); + if (!reservationId) return null; + return { + reservationId, + resourceType: textValue(value.resourceType) || null, + planId: textValue(value.planId) || null, + estimatedCredits: numberOrNull(value.estimatedCredits), + estimatedTokens: numberOrNull(value.estimatedTokens), + expiresAt: textValue(value.expiresAt) || null, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +function codeAgentConversationMessagesEvidence(payload = {}, params = {}, traceId = null, finalResponse = null) { + const userText = boundedConversationMessageText(params.message ?? params.prompt ?? payload.userMessage ?? payload.prompt); + if (!userText) return []; + const resolvedTraceId = safeTraceId(traceId) || null; + const lifecycle = codeAgentTurnLifecycleFields(resolvedTraceId, payload); + const session = payload.session && typeof payload.session === "object" ? payload.session : null; + const sessionReuse = payload.sessionReuse && typeof payload.sessionReuse === "object" ? payload.sessionReuse : null; + const conversationId = safeConversationId(payload.conversationId ?? session?.conversationId ?? sessionReuse?.conversationId ?? params.conversationId) || null; + const sessionId = safeSessionId(payload.sessionId ?? session?.sessionId ?? sessionReuse?.sessionId ?? params.sessionId) || null; + const threadId = safeOpaqueId(session?.threadId ?? sessionReuse?.threadId ?? payload.threadId ?? params.threadId) || null; + const createdAt = payload.createdAt ?? payload.reply?.createdAt ?? new Date().toISOString(); + const updatedAt = payload.updatedAt ?? createdAt; + const userMessage = { + id: lifecycle.userMessageId, + messageId: lifecycle.userMessageId, + role: "user", + title: "用户", + text: userText, + status: "sent", + traceId: resolvedTraceId, + turnId: lifecycle.turnId, + conversationId, + sessionId, + threadId, + createdAt, + updatedAt, + source: "code-agent-submit", + secretMaterialStored: false, + valuesRedacted: true + }; + const agentText = boundedConversationMessageText( + finalResponse?.text + ?? payload.reply?.content + ?? payload.message?.content + ?? payload.assistantText + ?? payload.error?.userMessage + ?? payload.error?.message + ?? payload.blocker?.userMessage + ?? payload.blocker?.message + ?? payload.blocker?.summary + ); + const agentStatus = codeAgentConversationAgentMessageStatus(payload); + const agentMessage = { + id: lifecycle.assistantMessageId, + messageId: lifecycle.assistantMessageId, + role: "agent", + title: agentStatus === "failed" ? "Code Agent 返回阻塞" : agentStatus === "running" ? "Code Agent 处理中" : "Code Agent 回复", + text: agentText || (agentStatus === "running" ? "" : "Code Agent 请求已结束，请查看 Trace 详情。"), + status: agentStatus, + traceId: resolvedTraceId, + turnId: lifecycle.turnId, + conversationId, + sessionId, + threadId, + createdAt: payload.reply?.createdAt ?? updatedAt, + updatedAt, + source: "code-agent-result", + ...(payload.runnerTrace && typeof payload.runnerTrace === "object" ? { runnerTrace: codeAgentConversationRunnerTraceEvidence(payload.runnerTrace, resolvedTraceId) } : {}), + secretMaterialStored: false, + valuesRedacted: true + }; + return [userMessage, agentMessage]; +} + +function boundedConversationMessageText(value, limit = 12000) { + const text = conversationText(value); + if (!text) return ""; + return text.length > limit ? text.slice(0, limit) : text; +} + +function conversationText(value) { + const extracted = conversationTextRaw(value).replace(/\r\n?/gu, "\n"); + if (!extracted.trim() || extracted.trim() === "[object Object]") return ""; + return extracted; +} + +function conversationTextRaw(value) { + if (value === undefined || value === null) return ""; + if (typeof value === "string") return value; + if (typeof value === "number" || typeof value === "boolean") return String(value); + if (Array.isArray(value)) return value.map(conversationTextRaw).filter(Boolean).join("\n"); + if (typeof value === "object") { + for (const key of ["text", "content", "message", "prompt", "value", "summary", "preview", "title"]) { + const text = conversationTextRaw(value[key]); + if (text) return text; + } + } + return ""; +} + +function codeAgentMessageTraceSuffix(traceId) { + const text = safeTraceId(traceId) || `local_${Date.now().toString(36)}`; + return text.replace(/^trc_/u, "").replace(/[^A-Za-z0-9_.:-]/gu, "_").slice(0, 48) || "trace"; +} + +function codeAgentTurnLifecycleFields(traceId, source = {}) { + const resolvedTraceId = safeTraceId(source?.traceId ?? traceId) || safeTraceId(traceId) || null; + const turnId = safeTurnId(source?.turnId) || resolvedTraceId; + const traceSuffix = codeAgentMessageTraceSuffix(resolvedTraceId); + return { + turnId, + userMessageId: safeMessageId(source?.userMessageId ?? source?.userMessage?.messageId) || codeAgentLifecycleMessageId(traceSuffix, "user"), + assistantMessageId: safeMessageId(source?.assistantMessageId ?? source?.assistantMessage?.messageId) || codeAgentLifecycleMessageId(traceSuffix, "agent") + }; +} + +function codeAgentLifecycleMessageId(traceSuffix, role) { + return `msg_${traceSuffix}_${role}`; +} + +function safeTurnId(value) { + const text = textValue(value); + return /^turn_[A-Za-z0-9_.:-]+$/u.test(text) ? text : null; +} + +function safeMessageId(value) { + const text = textValue(value); + return /^msg_[A-Za-z0-9_.:-]+$/u.test(text) ? text : null; +} + +function codeAgentConversationAgentMessageStatus(payload = {}) { + const status = textValue(payload.status ?? payload.agentRun?.terminalStatus ?? payload.commandState ?? payload.runStatus).toLowerCase(); + if (codeAgentPayloadHasSealedFinalResponse(payload)) return "completed"; + if (["completed", "done", "success"].includes(status)) return "completed"; + if (["failed", "blocked", "error", "timeout", "canceled", "cancelled"].includes(status)) return "failed"; + if (payload.error || payload.blocker) return "failed"; + return "running"; +} + +function codeAgentPayloadHasSealedFinalResponse(payload = {}) { + if (!payload || typeof payload !== "object") return false; + if (payload.error || payload.blocker) return false; + if (!codeAgentFinalResponseText(payload)) return false; + const agentRun = payload.agentRun && typeof payload.agentRun === "object" ? payload.agentRun : {}; + return isTraceCommandTerminalStatus(payload.status) || + isTraceCommandTerminalStatus(agentRun.status) || + isTraceCommandTerminalStatus(agentRun.runStatus) || + isTraceCommandTerminalStatus(agentRun.commandState) || + isTraceCommandTerminalStatus(agentRun.terminalStatus); +} + +function codeAgentConversationRunnerTraceEvidence(runnerTrace = {}, traceId = null) { + const events = Array.isArray(runnerTrace.events) ? runnerTrace.events : []; + const lastEvent = runnerTrace.lastEvent && typeof runnerTrace.lastEvent === "object" ? runnerTrace.lastEvent : events.at(-1) ?? null; + const eventCount = Number.isFinite(Number(runnerTrace.eventCount)) ? Number(runnerTrace.eventCount) : events.length; + return { + traceId: safeTraceId(runnerTrace.traceId ?? traceId) || traceId || null, + status: runnerTrace.status ?? lastEvent?.status ?? null, + sessionId: runnerTrace.sessionId ?? null, + threadId: runnerTrace.threadId ?? null, + runId: runnerTrace.runId ?? lastEvent?.runId ?? null, + commandId: runnerTrace.commandId ?? lastEvent?.commandId ?? null, + eventCount, + lastEvent: lastEvent ? { + label: lastEvent.label ?? null, + status: lastEvent.status ?? null, + terminal: lastEvent.terminal === true, + createdAt: lastEvent.createdAt ?? null, + message: boundedConversationMessageText(lastEvent.message ?? lastEvent.text, 1000), + valuesPrinted: false + } : null, + eventsCompacted: true, + fullTraceLoaded: false, + valuesPrinted: false + }; +} + +function codeAgentFinalResponseEvidence(payload = {}, traceId = null) { + const text = codeAgentFinalResponseText(payload); + if (!text) return null; + return { + text, + textChars: text.length, + messageId: payload.finalResponse?.messageId ?? payload.reply?.messageId ?? payload.messageId ?? null, + role: payload.reply?.role ?? payload.message?.role ?? "assistant", + status: codeAgentPayloadHasSealedFinalResponse(payload) ? "completed" : payload.finalResponse?.status ?? payload.status ?? null, + traceId, + createdAt: payload.finalResponse?.createdAt ?? payload.reply?.createdAt ?? payload.createdAt ?? null, + updatedAt: payload.finalResponse?.updatedAt ?? payload.updatedAt ?? null, + source: payload.finalResponse?.source ?? "code-agent-result", + valuesPrinted: false + }; +} + +function codeAgentFinalResponseText(payload = {}) { + for (const value of [ + payload?.finalResponse?.text, + payload?.finalResponse?.content, + payload?.finalResponse, + payload?.reply?.content, + payload?.reply, + payload?.message?.content, + payload?.message, + payload?.assistantText, + payload?.finalText, + payload?.text, + payload?.content + ]) { + const text = conversationText(value); + if (text) return text; + } + return ""; +} + +function codeAgentTraceSummaryEvidence(payload = {}, traceId = null, finalResponse = null) { + const runnerTrace = payload.runnerTrace && typeof payload.runnerTrace === "object" ? payload.runnerTrace : null; + const events = Array.isArray(runnerTrace?.events) ? runnerTrace.events : []; + const terminalEvent = [...events].reverse().find((event) => event?.terminal === true) ?? runnerTrace?.lastEvent ?? null; + return { + traceId, + source: "code-agent-derived-session-evidence", + sourceEventCount: Number.isFinite(Number(runnerTrace?.eventCount)) ? Number(runnerTrace.eventCount) : events.length, + terminalStatus: payload.agentRun?.terminalStatus ?? payload.status ?? terminalEvent?.status ?? null, + lastEventLabel: terminalEvent?.label ?? null, + finalAssistantRow: finalResponse ? { + role: finalResponse.role, + status: finalResponse.status, + textChars: finalResponse.textChars, + textPreview: finalResponse.text.slice(0, 240), + messageId: finalResponse.messageId, + valuesPrinted: false + } : null, + agentRun: payload.agentRun ? { + runId: payload.agentRun.runId ?? null, + commandId: payload.agentRun.commandId ?? null, + attemptId: payload.agentRun.attemptId ?? null, + runnerId: payload.agentRun.runnerId ?? null, + jobName: payload.agentRun.jobName ?? null, + namespace: payload.agentRun.namespace ?? null, + lastSeq: payload.agentRun.lastSeq ?? null, + valuesPrinted: false + } : null, + updatedAt: payload.updatedAt ?? null, + valuesPrinted: false + }; +} + +function annotateOwner(payload, params = {}) { + if (!params.ownerUserId) return payload; + return { + ...payload, + ownerUserId: params.ownerUserId, + ownerRole: params.ownerRole ?? null + }; +} + +function canAccessOwnedResult(result, actor) { + if (!result?.ownerUserId || !actor) return true; + return actor.role === "admin" || actor.id === result.ownerUserId; +} + +function isCodeAgentResultCanceled(result) { + return result?.status === "canceled" || result?.canceled === true || result?.error?.code === "codex_stdio_canceled"; +} + +function codeAgentCancelScopeMismatch(params = {}, result = {}) { + return cancelScopeFieldMismatch("conversationId", safeConversationId(params.conversationId), cancelExpectedValues(safeConversationId, + result.conversationId, + result.agentRun?.conversationId, + result.session?.conversationId, + result.sessionReuse?.conversationId, + result.runnerTrace?.conversationId + )) ?? cancelScopeFieldMismatch("sessionId", safeSessionId(params.sessionId), cancelExpectedValues(safeSessionId, + result.sessionId, + result.session?.sessionId, + result.sessionReuse?.sessionId, + result.agentRun?.hwlabSessionId + )) ?? cancelScopeFieldMismatch("threadId", safeOpaqueId(params.threadId), cancelExpectedValues(safeOpaqueId, + result.threadId, + result.agentRun?.threadId, + result.session?.threadId, + result.sessionReuse?.threadId, + result.runnerTrace?.threadId + )); +} + +function cancelExpectedValues(normalize, ...values) { + return [...new Set(values.map((value) => normalize(value)).filter(Boolean))]; +} + +function cancelScopeFieldMismatch(field, requested, expectedValues) { + if (!requested || expectedValues.length === 0 || expectedValues.includes(requested)) return null; + return { field, requested, expected: expectedValues[0] }; +} + +function cancelBlockedPayload({ + code, + message, + traceId, + conversationId = null, + sessionId = null, + session = null, + runnerTrace = null, + status = "failed", + unsupported = false, + degraded = false +}) { + const sessionSummary = codeAgentSessionLifecycleSummary({ + session, + runnerTrace, + status, + unsupported, + degraded, + error: { code, userMessage: message, message } + }); + return { + accepted: false, + canceled: false, + status, + conversationId, + sessionId, + traceId, + session, + sessionLifecycleStatus: sessionSummary.status, + sessionLifecycle: sessionSummary, + sessionSummary, + unsupported, + degraded, + runnerTrace, + lastTraceEvent: runnerTrace?.lastEvent ?? null, + error: { + code, + layer: "session", + category: "cancel_blocked", + retryable: true, + userMessage: message, + message, + traceId, + route: "/v1/agent/chat/cancel", + toolName: "codex-stdio.cancel" + }, + blocker: { + code, + layer: "session", + category: "cancel_blocked", + retryable: true, + summary: message, + userMessage: message, + traceId, + route: "/v1/agent/chat/cancel", + toolName: "codex-stdio.cancel" + } + }; +} + +function isTraceCommandTerminalStatus(status) { + return CODE_AGENT_TERMINAL_STATUSES.has(String(status ?? "").trim().toLowerCase().replace(/_/gu, "-")); +} + +function traceSnapshotWithTerminalEvidence(snapshot, result, traceId, refreshError = null) { + const evidence = agentRunTerminalTraceEvidence(result, traceId); + if (snapshot && snapshot.status !== "missing") return traceSnapshotWithAttachedTerminalEvidence(snapshot, evidence, refreshError); + if (!evidence) return { + ...snapshot, + ok: false, + traceStatus: "missing", + retention: traceRetentionSummary("missing") + }; + return { + ...snapshot, + ok: true, + status: "expired", + traceStatus: "expired", + projectionStatus: "projecting", + projectionHealth: "degraded", + persisted: true, + retention: traceRetentionSummary("expired"), + conversationId: evidence.conversationId, + sessionId: evidence.sessionId, + threadId: evidence.threadId, + ...codeAgentPromptFields(evidence), + agentRun: evidence.agentRun, + finalResponse: evidence.finalResponse, + traceSummary: evidence.traceSummary, + terminalEvidence: { + available: true, + source: evidence.source, + refresh: refreshError ? { + attempted: true, + ok: false, + code: refreshError?.code ?? "agentrun_trace_refresh_failed", + message: refreshError?.message ?? "AgentRun trace events could not be refreshed; command result remains authoritative for final response.", + valuesPrinted: false + } : { attempted: false, ok: null }, + conversationId: evidence.conversationId, + sessionId: evidence.sessionId, + threadId: evidence.threadId, + ...codeAgentPromptFields(evidence), + agentRun: evidence.agentRun, + traceSummary: evidence.traceSummary, + finalResponse: evidence.finalResponse, + valuesPrinted: false + } + }; +} + +function traceSnapshotWithAttachedTerminalEvidence(snapshot, evidence, refreshError = null) { + if (!evidence) return snapshot; + return { + ...snapshot, + conversationId: snapshot.conversationId ?? evidence.conversationId, + sessionId: snapshot.sessionId ?? evidence.sessionId, + threadId: snapshot.threadId ?? evidence.threadId, + ...codeAgentPromptFields(evidence), + agentRun: snapshot.agentRun ?? evidence.agentRun, + finalResponse: snapshot.finalResponse ?? evidence.finalResponse, + traceSummary: snapshot.traceSummary ?? evidence.traceSummary, + terminalEvidence: snapshot.terminalEvidence ?? (refreshError ? { + available: true, + source: evidence.source, + refresh: { + attempted: true, + ok: false, + code: refreshError?.code ?? "agentrun_trace_refresh_failed", + message: refreshError?.message ?? "AgentRun trace events could not be refreshed; live events may be partial.", + valuesPrinted: false + }, + valuesPrinted: false + } : undefined) + }; +} + +function agentRunTerminalTraceEvidence(result, traceId) { + if (!result || typeof result !== "object") return null; + const storedSummary = result.traceSummary && typeof result.traceSummary === "object" && result.traceSummary.source === "agentrun-command-result" ? result.traceSummary : null; + const finalResponse = agentRunTerminalFinalResponse(result, traceId); + const promptEvidence = codeAgentPromptMetadata(result, {}, traceId); + const agentRun = result.agentRun && typeof result.agentRun === "object" ? { + adapter: result.agentRun.adapter ?? null, + runId: result.agentRun.runId ?? null, + commandId: result.agentRun.commandId ?? null, + attemptId: result.agentRun.attemptId ?? null, + runnerId: result.agentRun.runnerId ?? null, + jobName: result.agentRun.jobName ?? null, + namespace: result.agentRun.namespace ?? null, + terminalStatus: result.agentRun.terminalStatus ?? null, + lastSeq: result.agentRun.lastSeq ?? null, + valuesPrinted: false + } : null; + const terminalStatus = normalizeTurnStatus(result.agentRun?.terminalStatus, result.agentRun?.commandState, result.agentRun?.status, result.agentRun?.runStatus); + if (!isTraceCommandTerminalStatus(terminalStatus)) return null; + if (!agentRun?.runId || !agentRun?.commandId || (!storedSummary && !finalResponse)) return null; + const traceSummary = { + traceId, + source: "agentrun-command-result", + sourceEventCount: numberOrNull(storedSummary?.sourceEventCount ?? storedSummary?.eventCount ?? result.runnerTrace?.eventCount), + renderedRowSummary: storedSummary?.renderedRowSummary ?? null, + noiseEventCount: numberOrNull(storedSummary?.noiseEventCount), + omittedNoiseCount: numberOrNull(storedSummary?.omittedNoiseCount), + terminalStatus: storedSummary?.terminalStatus ?? terminalStatus, + updatedAt: storedSummary?.updatedAt ?? result.updatedAt ?? null, + valuesPrinted: false + }; + return { + source: "agentrun-command-result", + conversationId: safeConversationId(result.conversationId) || null, + sessionId: safeSessionId(result.sessionId) || null, + threadId: safeOpaqueId(result.threadId) || null, + ...codeAgentPromptFields(promptEvidence), + agentRun, + finalResponse, + traceSummary + }; +} + +function agentRunTerminalFinalResponse(result, traceId) { + const stored = result.finalResponse && typeof result.finalResponse === "object" ? result.finalResponse : null; + const text = textValue(stored?.text ?? result.reply?.content ?? result.message?.content ?? result.assistantText); + if (!text) return null; + const storedTraceId = safeTraceId(stored?.traceId ?? result.traceId) || null; + if (storedTraceId && storedTraceId !== traceId) return null; + return { + text, + textChars: text.length, + messageId: stored?.messageId ?? result.reply?.messageId ?? result.messageId ?? null, + role: stored?.role ?? result.reply?.role ?? "assistant", + status: stored?.status ?? result.status ?? null, + traceId: traceId, + createdAt: stored?.createdAt ?? result.reply?.createdAt ?? result.createdAt ?? null, + updatedAt: stored?.updatedAt ?? result.updatedAt ?? null, + source: "agentrun-command-result", + valuesPrinted: false + }; +} + +function traceRetentionSummary(status) { + return { + traceStatus: status, + liveTraceStore: status === "missing" ? "missing" : "expired-or-evicted", + policy: "live trace events are short-term storage; terminal final response and summary are resolved from AgentRun command result by traceId and commandId", + replacementEvidence: status === "missing" ? "resolve the trace through AgentRun command registry/result or report the trace as missing" : "use terminalEvidence.finalResponse, terminalEvidence.traceSummary, conversationId, sessionId, threadId, and agentRun ids", + valuesPrinted: false + }; +} + +function numberOrNull(value) { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function createCodeAgentChatResultStore({ maxResults = 256 } = {}) { + const limit = positiveInteger(maxResults, 256); + const results = new Map(); + function set(traceId, payload) { + const id = safeTraceId(traceId); + if (!id) return; + results.set(id, { + ...payload, + traceId: id, + cachedAt: new Date().toISOString() + }); + while (results.size > limit) { + const oldest = results.keys().next().value; + if (!oldest) break; + results.delete(oldest); + } + } + return { + set, + get: (traceId) => results.get(safeTraceId(traceId)) ?? null, + clear: () => results.clear() + }; +} + +function compactCodeAgentChatResultPayload(payload, options = {}) { + if (!payload || typeof payload !== "object") return payload; + const limit = resultTraceEventLimit(options); + const terminalEvidence = agentRunTerminalTraceEvidence(payload, payload.traceId); + const { userBillingReservation, ...publicPayload } = payload; + return { + ...publicPayload, + ...codeAgentTurnLifecycleFields(publicPayload.traceId, publicPayload), + ...(terminalEvidence ? { terminalEvidence: terminalEvidencePayload(terminalEvidence) } : {}), + ...(publicPayload.runnerTrace && typeof publicPayload.runnerTrace === "object" + ? { runnerTrace: compactRunnerTraceForResult(publicPayload.runnerTrace, limit) } + : {}) + }; +} + +function terminalEvidencePayload(evidence) { + return { + available: true, + source: evidence.source, + conversationId: evidence.conversationId, + sessionId: evidence.sessionId, + threadId: evidence.threadId, + ...codeAgentPromptFields(evidence), + agentRun: evidence.agentRun, + traceSummary: evidence.traceSummary, + finalResponse: evidence.finalResponse, + valuesPrinted: false + }; +} + +function compactRunnerTraceForResult(runnerTrace, limit = DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT) { + if (!runnerTrace || typeof runnerTrace !== "object") return runnerTrace; + const events = Array.isArray(runnerTrace.events) ? runnerTrace.events : []; + const total = Number.isInteger(runnerTrace.eventCount) ? runnerTrace.eventCount : events.length; + const effectiveLimit = Math.max(8, positiveInteger(limit, DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT)); + if (events.length <= effectiveLimit) { + return { + ...runnerTrace, + eventCount: total, + eventsCompacted: false + }; + } + const headCount = Math.min(24, Math.max(1, Math.floor(effectiveLimit / 4))); + const tailCount = Math.max(1, effectiveLimit - headCount - 1); + const omitted = Math.max(0, events.length - headCount - tailCount); + const head = events.slice(0, headCount); + const tail = events.slice(events.length - tailCount); + const marker = { + traceId: runnerTrace.traceId ?? head[0]?.traceId ?? tail[0]?.traceId ?? null, + type: "trace", + stage: "trace", + status: "truncated", + label: "trace:compacted", + createdAt: tail[0]?.createdAt ?? runnerTrace.updatedAt ?? new Date().toISOString(), + message: `Compatibility result response compacted ${omitted} trace events; fetch /v1/agent/traces/${encodeURIComponent(runnerTrace.traceId ?? "")} for the full trace.`, + omittedEventCount: omitted, + retainedEventCount: head.length + tail.length, + totalEventCount: total, + valuesPrinted: false + }; + const compactEvents = [...head, marker, ...tail]; + return { + ...runnerTrace, + eventCount: total, + events: compactEvents, + eventLabels: compactEvents.map(traceEventLabel).filter(Boolean), + eventsCompacted: true, + eventWindow: { + mode: "head-tail", + retained: compactEvents.length, + omitted, + total + } + }; +} + +function traceEventLabel(event) { + return typeof event === "string" ? event : event?.label; +} + +function resultTraceEventLimit(options = {}) { + return positiveInteger( + options.env?.HWLAB_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT, + DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT + ); +} + +function createCodeAgentM3HwlabApiRequestJson(options = {}) { + return async (targetUrl, request = {}) => { + const url = new URL(targetUrl); + const route = url.pathname; + if (route === M3_IO_CONTROL_ROUTE && request.method === "POST") { + return { + ok: true, + status: 200, + body: await handleM3IoControl(request.body ?? {}, { + runtimeStore: options.runtimeStore, + now: options.now, + env: options.env, + requestJson: options.m3IoRequestJson + }) + }; + } + if (route === M3_STATUS_ROUTE && request.method === "GET") { + return { + ok: true, + status: 200, + body: await describeM3StatusLive(options) + }; + } + return { + ok: false, + status: 404, + body: { + error: { + code: "not_found", + message: `Unsupported Code Agent M3 HWLAB API route ${route}` + } + }, + error: `Unsupported Code Agent M3 HWLAB API route ${route}` + }; + }; +} +function recordCodeAgentProjectionMetrics(options = {}, { result = null, trace = null, projection = null, refreshError = null } = {}) { + const performanceStore = options.backendPerformanceStore ?? options.backendPerformance; + if (!performanceStore?.recordWorkbenchProjection || !projection) return; + const sourceLatestSeq = sourceLatestSeqFromResult(result); + const lastProjectedSeq = Number(projection.lastProjectedSeq ?? 0); + if (!Number.isFinite(sourceLatestSeq) && !Number.isFinite(lastProjectedSeq)) return; + performanceStore.recordWorkbenchProjection({ + sourceLatestSeq: Number.isFinite(sourceLatestSeq) ? sourceLatestSeq : lastProjectedSeq, + lastProjectedSeq: Number.isFinite(lastProjectedSeq) ? lastProjectedSeq : 0, + sourceLatestEventAt: sourceLatestAtFromResult(result, trace), + projectedLatestEventAt: projection.updatedAt ?? trace?.updatedAt ?? result?.updatedAt ?? null, + terminalSourceAt: terminalSourceAtFromResult(result), + terminalProjectedAt: projection.projectionStatus === "caught-up" ? projection.updatedAt ?? trace?.updatedAt ?? result?.updatedAt ?? null : null, + status: projection?.status ?? "unknown", + reason: projectionReason(projection, refreshError), + projectionStatus: projection.projectionStatus ?? "unknown", + source: "workbench_read_model" + }); +} + +function recordCodeAgentTurnReadMetric(options = {}, url, statusCode, body = {}, startedAt = nowMs()) { + const performanceStore = options.backendPerformanceStore ?? options.backendPerformance; + if (!performanceStore?.recordWorkbenchTurnRead) return; + performanceStore.recordWorkbenchTurnRead({ + route: url?.pathname ?? "/v1/agent/turns/:id", + status: body?.status ?? statusClassLabel(statusCode), + degradedReason: turnReadDegradedReason(body), + durationMs: nowMs() - startedAt, + responseBytes: responseBodyBytes(body), + timedOut: statusCode === 504 || body?.error?.code === "timeout" || body?.refreshError?.code === "agentrun_timeout" + }); +} + +function turnReadDegradedReason(body = {}) { + if (body?.refreshError?.code) return body.refreshError.code; + if (body?.projection?.blocker?.code) return body.projection.blocker.code; + if (body?.blocker?.code) return body.blocker.code; + if (body?.error?.code) return body.error.code; + if (body?.projectionStatus === "projecting") return "projecting"; + if (body?.projectionStatus === "blocked") return "projection_blocked"; + return "none"; +} + +function responseBodyBytes(body = {}) { + try { return Buffer.byteLength(JSON.stringify(body)); } catch { return 0; } +} + +function statusClassLabel(value) { + const status = Number(value); + if (!Number.isFinite(status) || status <= 0) return "unknown"; + return `${Math.floor(status / 100)}xx`; +} + +function sourceLatestSeqFromResult(result = null) { + const candidates = [ + result?.agentRun?.lastSeq, + result?.traceSummary?.agentRun?.lastSeq, + result?.traceSummary?.lastSeq, + result?.providerTrace?.lastSeq, + result?.runnerTrace?.lastEvent?.sourceSeq, + result?.runnerTrace?.lastEvent?.seq, + result?.runnerTrace?.eventCount + ]; + for (const value of candidates) { + const seq = Number(value); + if (Number.isFinite(seq) && seq >= 0) return Math.floor(seq); + } + return NaN; +} + +function sourceLatestAtFromResult(result = null, trace = null) { + return result?.traceSummary?.updatedAt + ?? result?.agentRun?.updatedAt + ?? result?.providerTrace?.updatedAt + ?? result?.runnerTrace?.updatedAt + ?? trace?.updatedAt + ?? result?.updatedAt + ?? null; +} + +function terminalSourceAtFromResult(result = null) { + if (!result) return null; + const terminalStatus = result?.agentRun?.terminalStatus ?? result?.traceSummary?.terminalStatus ?? result?.status; + if (!isTraceCommandTerminalStatus(terminalStatus)) return null; + return result?.traceSummary?.updatedAt ?? result?.agentRun?.updatedAt ?? result?.updatedAt ?? null; +} + +function projectionReason(projection = null, refreshError = null) { + if (refreshError) return refreshError.code ?? "refresh_error"; + if (projection?.blocker) return projection.blocker.code ?? "projection_blocked"; + if (projection?.projectionStatus === "projecting") return "projecting"; + if (projection?.projectionStatus === "caught-up") return "none"; + return projection?.projectionStatus ?? "unknown"; +} + +function nowMs() { + if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now(); + return Date.now(); +} + +function uniqueStrings(values) { + return [...new Set(values.map((value) => String(value ?? "").trim()).filter(Boolean))]; +} + +export { + normalizeCodeAgentProviderProfile, + firstNonEmptyValue, + recordCodeAgentSessionInputFact, + buildCodeAgentSessionInputFact, + stableCodeAgentInputId, + codeAgentInputDelivery, + codeAgentInputStatus, + codeAgentResultTimingFields, + firstTimestampIso, + firstLatestTimestampIso, + codeAgentChatShortConnectionRequested, + preflightCodeAgentBilling, + recordCodeAgentBillingUsage, + finalizeCodeAgentBillingUsage, + releaseCodeAgentBillingReservation, + withCodeAgentBillingReservation, + codeAgentBillingEnabled, + codeAgentBillingMetadata, + sanitizeBillingReservation, + sanitizeBillingRecord, + sanitizeBillingRelease, + codeAgentUsedTokens, + normalizeTurnStatus, + recordCodeAgentConversationFact, + recordCodeAgentSessionOwner, + codeAgentOwnerStatusForResult, + textValue, + timestampIsoOrNow, + timestampIsoOrNull, + codeAgentSessionOwnerEvidence, + codeAgentContinuationEvidence, + codeAgentFreshContinuationFailure, + normalizeCodeAgentFailureKind, + codeAgentFailureRequiresFreshContinuation, + codeAgentProviderProfileEvidence, + codeAgentTraceResultEvidence, + codeAgentPromptMetadata, + codeAgentPromptMetadataFromText, + codeAgentPromptMetadataFromParts, + codeAgentPromptFields, + codeAgentPromptId, + clippedPromptText, + sha256Text, + compactCodeAgentObject, + codeAgentBillingReservationEvidence, + codeAgentConversationMessagesEvidence, + boundedConversationMessageText, + conversationText, + conversationTextRaw, + codeAgentMessageTraceSuffix, + codeAgentTurnLifecycleFields, + codeAgentLifecycleMessageId, + safeTurnId, + safeMessageId, + codeAgentConversationAgentMessageStatus, + codeAgentPayloadHasSealedFinalResponse, + codeAgentConversationRunnerTraceEvidence, + codeAgentFinalResponseEvidence, + codeAgentFinalResponseText, + codeAgentTraceSummaryEvidence, + annotateOwner, + canAccessOwnedResult, + isCodeAgentResultCanceled, + codeAgentCancelScopeMismatch, + cancelExpectedValues, + cancelScopeFieldMismatch, + cancelBlockedPayload, + isTraceCommandTerminalStatus, + traceSnapshotWithTerminalEvidence, + traceSnapshotWithAttachedTerminalEvidence, + agentRunTerminalTraceEvidence, + agentRunTerminalFinalResponse, + traceRetentionSummary, + numberOrNull, + createCodeAgentChatResultStore, + compactCodeAgentChatResultPayload, + terminalEvidencePayload, + compactRunnerTraceForResult, + traceEventLabel, + resultTraceEventLimit, + createCodeAgentM3HwlabApiRequestJson, + recordCodeAgentProjectionMetrics, + recordCodeAgentTurnReadMetric, + turnReadDegradedReason, + responseBodyBytes, + statusClassLabel, + sourceLatestSeqFromResult, + sourceLatestAtFromResult, + terminalSourceAtFromResult, + projectionReason, + nowMs, + uniqueStrings +}; diff --git a/internal/cloud/server-code-agent-http.ts b/internal/cloud/server-code-agent-http.ts index 56b80939..a41b4444 100644 --- a/internal/cloud/server-code-agent-http.ts +++ b/internal/cloud/server-code-agent-http.ts @@ -1,4756 +1,7 @@ -// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-0102 Agent编排 draft-2026-06-17-r0; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. -// Responsibility: Code Agent HTTP turn admission, lifecycle compatibility wrappers, and Workbench projection writer/finalizer integration. +// SPEC: PJ2026-0104010803 Workbench唯一投影; PJ2026-010403 API契约; PJ2026-010205 HWLAB接入; PJ2026-0102 Agent编排. +// Responsibility: Stable Code Agent HTTP export facade. -import { createHash, randomUUID } from "node:crypto"; - -import { M3_IO_CONTROL_ROUTE, M3_STATUS_ROUTE, describeM3StatusLive, handleM3IoControl } from "./m3-io-control.ts"; -import { - agentRunSessionEvidence, - cancelAgentRunChatTurn, - codeAgentAgentRunAdapterEnabled, - initialAgentRunChatResult, - loadPersistedAgentRunResult, - steerAgentRunChatTurn, - submitAgentRunChatTurn, - syncAgentRunChatResult -} from "./code-agent-agentrun-adapter.ts"; -import { createCodeAgentErrorPayload, handleCodeAgentChat } from "./code-agent-chat.ts"; -import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; -import { codeAgentSessionLifecycleSummary } from "./code-agent-session-lifecycle.ts"; -import { messageAuthorityTextValue } from "./code-agent-agentrun-prompt.ts"; -import { scheduleWorkbenchProjectionFinalizer } from "./workbench-projection-finalizer.ts"; -import { writeWorkbenchProjectionSession, writeWorkbenchSessionAdmissionFact } from "./workbench-projection-writer.ts"; -import { buildAgentRunProjectionEventsFetchPlan } from "./workbench-projection-cursor.ts"; -import { createWorkbenchReadModel } from "./workbench-read-model.ts"; -import { publishHwlabAgentRunCommandShadow } from "./kafka-shadow-producer.ts"; -import { codeAgentOtelTraceFields, emitCodeAgentOtelSpan } from "./otel-trace.ts"; -import { - firstHeaderValue, - getHeader, - parsePositiveInteger, - positiveInteger, - readBody, - safeConversationId, - safeOpaqueId, - safeSessionId, - safeTraceId, - sendJson, - truthyFlag -} from "./server-http-utils.ts"; - -const DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT = 120; -const DEFAULT_CODE_AGENT_TRACE_PAGE_LIMIT = 100; -const MAX_CODE_AGENT_TRACE_PAGE_LIMIT = 100; -const DEFAULT_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS = 2500; -const DEFAULT_CODE_AGENT_PROJECT_ID = "prj_hwpod_workbench"; -const DEFAULT_CODE_AGENT_PROVIDER_PROFILE = "deepseek"; -const DEFAULT_AGENTRUN_PROJECTION_RESUME_INITIAL_DELAY_MS = 5000; -const DEFAULT_AGENTRUN_PROJECTION_RESUME_INTERVAL_MS = 60000; -const DEFAULT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS = 30000; -const DEFAULT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE = 100; -const DEFAULT_AGENTRUN_PROJECTION_RESUME_JITTER_MS = 15000; -const DEFAULT_AGENTRUN_PROJECTION_RESUME_FAILURE_BACKOFF_MS = 120000; -const CODE_AGENT_TERMINAL_STATUSES = new Set(["completed", "failed", "blocked", "timeout", "cancelled", "canceled"]); -const CODE_AGENT_PROVIDER_PROFILE_ALIASES = Object.freeze({ codex: "codex-api" }); -const CODE_AGENT_PROVIDER_PROFILE_ID_PATTERN = /^[a-z0-9][a-z0-9.-]{0,63}$/u; -const CODE_AGENT_PROVIDER_PROFILE_LABELS = Object.freeze({ - "deepseek": "DeepSeek", - "codex-api": "Codex API", - "gpt.pika": "gpt.pika", - "minimax-m3": "MiniMax-M3 via AgentRun", - "runtime-default": "运行默认" -}); -const DEFAULT_CODE_AGENT_CODEX_API_MODEL = "gpt-5.5"; -const DEFAULT_CODE_AGENT_CODEX_API_BASE_URL = "http://127.0.0.1:49280/responses"; -const DEFAULT_CODE_AGENT_DEEPSEEK_MODEL = "deepseek-chat"; -const DEFAULT_CODE_AGENT_MINIMAX_M3_MODEL = "MiniMax-M3"; - -export async function handleCodeAgentChatHttp(request, response, options) { - const perf = options.backendPerformance; - const body = perf ? await perf.measure("body_read", () => readBody(request, options.bodyLimitBytes)) : await readBody(request, options.bodyLimitBytes); - let params = {}; - - try { - params = body ? JSON.parse(body) : {}; - } catch (error) { - sendJson(response, 400, { - ...createCodeAgentErrorPayload({ - code: "parse_error", - message: "Invalid JSON body", - reason: error.message, - traceId: getHeader(request, "x-trace-id") || "trc_unassigned", - model: options.env?.HWLAB_CODE_AGENT_MODEL || options.env?.OPENAI_MODEL || process.env.HWLAB_CODE_AGENT_MODEL || process.env.OPENAI_MODEL || "unknown", - layer: "api", - retryable: true - }) - }); - return; - } - - if (!params || typeof params !== "object" || Array.isArray(params)) { - sendJson(response, 400, { - ...createCodeAgentErrorPayload({ - code: "invalid_params", - message: "Code Agent chat body must be a JSON object", - traceId: getHeader(request, "x-trace-id") || "trc_unassigned", - model: options.env?.HWLAB_CODE_AGENT_MODEL || options.env?.OPENAI_MODEL || process.env.HWLAB_CODE_AGENT_MODEL || process.env.OPENAI_MODEL || "unknown", - layer: "api", - retryable: true - }) - }); - return; - } - params = withMdtodoExecutionPrompt(params); - - const traceId = safeTraceId(getHeader(request, "x-trace-id") || params.traceId) || `trc_${randomUUID()}`; - const lifecycle = codeAgentTurnLifecycleFields(traceId, params); - const chatParams = { - ...params, - traceId, - ...lifecycle, - ownerUserId: options.actor?.id ?? params.ownerUserId, - ownerRole: options.actor?.role ?? params.ownerRole - }; - const manualSession = perf ? await perf.measure("manual_session", () => prepareManualCodeAgentSession({ params: chatParams, options, traceId, response })) : await prepareManualCodeAgentSession({ params: chatParams, options, traceId, response }); - if (manualSession?.blocked) return; - const nativeSessionChatParams = stripSyntheticConversationContext(chatParams, traceId, options); - - if (codeAgentChatShortConnectionRequested(request, params, options)) { - if (perf) perf.recordPhase({ phase: "short_connection_accept", durationMs: 0, outcome: "ok" }); - let submitted = null; - try { - submitted = await submitCodeAgentChatTurn({ - params: nativeSessionChatParams, - options, - traceId - }); - } catch (error) { - if (isCodeAgentAdmissionUnavailableError(error)) { - sendJson(response, error.statusCode ?? 503, error.payload); - return; - } - throw error; - } - const traceUrl = `/v1/agent/traces/${encodeURIComponent(traceId)}`; - const turnUrl = `/v1/agent/turns/${encodeURIComponent(traceId)}`; - const statusCode = 202; - const admittedTiming = codeAgentAdmissionTimingFields(submitted?.createdAt ?? submitted?.updatedAt); - void emitCodeAgentOtelSpan("POST /v1/agent/chat", traceId, options.env ?? process.env, { - kind: 2, - attributes: { "http.method": "POST", "http.route": "/v1/agent/chat", "http.status_code": statusCode, sessionId: safeSessionId(nativeSessionChatParams.sessionId) || null, turnId: lifecycle.turnId, ...agentChatOtelAttributes(nativeSessionChatParams) } - }); - sendJson(response, 202, { - accepted: true, - status: "running", - shortConnection: true, - controlSemantics: "submit-and-poll", - traceId, - ...codeAgentOtelTraceFields(traceId, options.env ?? process.env), - ...lifecycle, - sessionId: safeSessionId(nativeSessionChatParams.sessionId) || null, - createdAt: submitted?.createdAt ?? admittedTiming.createdAt, - updatedAt: submitted?.updatedAt ?? admittedTiming.updatedAt, - timing: submitted?.timing ?? admittedTiming.timing, - startedAt: submitted?.startedAt ?? admittedTiming.startedAt, - lastEventAt: submitted?.lastEventAt ?? admittedTiming.lastEventAt, - finishedAt: submitted?.finishedAt ?? admittedTiming.finishedAt, - durationMs: submitted?.durationMs ?? admittedTiming.durationMs, - traceUrl, - resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`, - turnUrl, - streamUrl: `${traceUrl}/stream`, - cancelUrl: "/v1/agent/chat/cancel", - polling: { - resultIntervalMs: parsePositiveInteger(options.env?.HWLAB_CODE_AGENT_RESULT_POLL_INTERVAL_MS, 1000), - traceIntervalMs: parsePositiveInteger(options.env?.HWLAB_CODE_AGENT_TRACE_POLL_INTERVAL_MS, 1000) - }, - runnerTrace: (options.traceStore ?? defaultCodeAgentTraceStore).snapshot(traceId) - }); - return; - } - - const admittedBase = codeAgentAdmittedFailureBasePayload({ params: nativeSessionChatParams, options, traceId }); - try { - await recordCodeAgentTurnAdmission({ payload: admittedBase, params: nativeSessionChatParams, options, traceId }); - } catch (error) { - if (isCodeAgentAdmissionUnavailableError(error)) { - sendJson(response, error.statusCode ?? 503, error.payload); - return; - } - throw error; - } - const billingPreflight = perf ? await perf.measure("billing_preflight", () => preflightCodeAgentBilling({ params: nativeSessionChatParams, options, traceId, response: null, sendResponse: false })) : await preflightCodeAgentBilling({ params: nativeSessionChatParams, options, traceId, response: null, sendResponse: false }); - if (billingPreflight?.blocked) { - const failed = await settleAdmittedCodeAgentFailure({ - base: admittedBase, - params: nativeSessionChatParams, - options, - traceId, - results: options.codeAgentChatResults, - failure: billingPreflight, - traceLabel: "billing:preflight:failed" - }); - sendJson(response, billingPreflight.statusCode ?? 503, failed); - return; - } - const nativeSessionChatParamsWithBilling = { ...nativeSessionChatParams, userBillingReservation: billingPreflight?.reservation ?? null }; - const payload = perf ? await perf.measure("code_agent_run", () => runCodeAgentChat(nativeSessionChatParamsWithBilling, options)) : await runCodeAgentChat(nativeSessionChatParamsWithBilling, options); - await (perf ? perf.measure("billing_finalize", () => finalizeCodeAgentBillingUsage({ payload, params: nativeSessionChatParamsWithBilling, options })) : finalizeCodeAgentBillingUsage({ payload, params: nativeSessionChatParamsWithBilling, options })); - await (perf ? perf.measure("session_owner_record", () => recordCodeAgentSessionOwner({ payload, params: nativeSessionChatParamsWithBilling, options, status: codeAgentOwnerStatusForResult(payload) })) : recordCodeAgentSessionOwner({ payload, params: nativeSessionChatParamsWithBilling, options, status: codeAgentOwnerStatusForResult(payload) })); - const responsePayload = annotateOwner(payload, nativeSessionChatParamsWithBilling); - - sendJson(response, responsePayload.status === "failed" && responsePayload.error?.code === "invalid_params" ? 400 : 200, responsePayload); -} - -export async function handleCodeAgentSessionsHttp(request, response, url, options) { - const match = url.pathname.match(/^\/v1\/agent\/sessions(?:\/([^/]+)(?:\/(select))?)?$/u); - const sessionId = safeSessionId(match?.[1] ? decodeURIComponent(match[1]) : ""); - const action = match?.[2] ?? null; - if (!match) { - sendJson(response, 404, manualSessionErrorPayload({ code: "session_route_not_found", message: "Code Agent session route is not implemented." })); - return; - } - if (request.method === "GET" && !sessionId) { - await listManualCodeAgentSessions(request, response, url, options); - return; - } - if (request.method === "POST" && !sessionId) { - await createManualCodeAgentSession(request, response, options); - return; - } - if (request.method === "GET" && sessionId && !action) { - await getManualCodeAgentSession(response, options, sessionId); - return; - } - if (request.method === "DELETE" && sessionId && !action) { - await archiveManualCodeAgentSession(response, options, sessionId); - return; - } - if (request.method === "POST" && sessionId && action === "select") { - await selectManualCodeAgentSession(request, response, options, sessionId); - return; - } - sendJson(response, 405, manualSessionErrorPayload({ code: "session_method_not_allowed", message: "Unsupported Code Agent session method." })); -} - -async function listManualCodeAgentSessions(request, response, url, options) { - const projectId = textValue(url.searchParams.get("projectId")) || DEFAULT_CODE_AGENT_PROJECT_ID; - const limit = parsePositiveInteger(url.searchParams.get("limit"), 20); - const sessions = await options.accessController?.store?.listAgentSessionsForUser?.({ - ownerUserId: options.actor?.id, - actorRole: options.actor?.role, - ownerScoped: true, - projectId, - limit - }) ?? []; - sendJson(response, 200, { - ok: true, - status: "succeeded", - contractVersion: "code-agent-manual-session-v1", - projectId, - sessions: sessions.map(publicManualAgentSession), - count: sessions.length, - valuesRedacted: true, - secretMaterialStored: false - }); -} - -async function createManualCodeAgentSession(request, response, options) { - const body = await readJsonObjectBody(request, options.bodyLimitBytes); - if (!body.ok) { - sendJson(response, 400, manualSessionErrorPayload({ code: body.code, message: body.message, reason: body.reason })); - return; - } - const params = body.value; - const projectId = textValue(params.projectId) || DEFAULT_CODE_AGENT_PROJECT_ID; - const conversationId = safeConversationId(params.conversationId) || `cnv_${randomUUID()}`; - const sessionId = safeSessionId(params.sessionId) || `ses_${randomUUID()}`; - const providerProfile = normalizeCodeAgentProviderProfile(params.providerProfile ?? params.codeAgentProviderProfile); - const now = new Date().toISOString(); - const session = await options.accessController.recordAgentSessionOwner({ - ownerUserId: options.actor?.id, - ownerRole: options.actor?.role, - sessionId, - projectId, - agentId: "hwlab-code-agent", - status: "idle", - conversationId, - threadId: null, - traceId: null, - session: { - source: "manual-session-create", - providerProfile, - sessionStatus: "idle", - createdAt: now, - valuesRedacted: true, - secretMaterialStored: false - } - }); - await writeWorkbenchSessionAdmissionFact({ - runtimeStore: options.runtimeStore, - session, - ownerUserId: options.actor?.id, - ownerRole: options.actor?.role, - projectId, - conversationId, - threadId: null, - status: "idle", - now - }); - sendJson(response, 201, { - ok: true, - status: "created", - contractVersion: "code-agent-manual-session-v1", - session: publicManualAgentSession(session), - nextCommands: manualSessionNextCommands(session), - valuesRedacted: true, - secretMaterialStored: false - }); -} - -async function getManualCodeAgentSession(response, options, sessionId) { - const session = await getVisibleManualAgentSession(options, sessionId); - if (!session) { - sendJson(response, 404, manualSessionErrorPayload({ code: "session_not_found", message: "Code Agent session is not visible to the current actor.", sessionId })); - return; - } - sendJson(response, 200, { - ok: true, - status: "found", - contractVersion: "code-agent-manual-session-v1", - session: publicManualAgentSession(session), - valuesRedacted: true, - secretMaterialStored: false - }); -} - -async function archiveManualCodeAgentSession(response, options, sessionId) { - const session = await getVisibleManualAgentSession(options, sessionId); - if (!session) { - sendJson(response, 404, manualSessionErrorPayload({ code: "session_not_found", message: "Code Agent session is not visible to the current actor.", sessionId })); - return; - } - const result = await options.accessController?.store?.archiveAgentConversation?.({ - ownerUserId: options.actor?.id, - actorRole: options.actor?.role, - ownerScoped: true, - sessionId, - conversationId: session.conversationId, - projectId: session.projectId - }) ?? { count: 0 }; - sendJson(response, 200, { - ok: true, - status: "archived", - contractVersion: "code-agent-manual-session-v1", - archivedCount: result.count ?? 0, - session: publicManualAgentSession({ ...session, status: "archived", endedAt: session.endedAt ?? new Date().toISOString(), updatedAt: new Date().toISOString() }), - valuesRedacted: true, - secretMaterialStored: false - }); -} - -async function selectManualCodeAgentSession(request, response, options, sessionId) { - const body = await readJsonObjectBody(request, options.bodyLimitBytes); - if (!body.ok) { - sendJson(response, 400, manualSessionErrorPayload({ code: body.code, message: body.message, reason: body.reason, sessionId })); - return; - } - const session = await getVisibleManualAgentSession(options, sessionId); - if (!session) { - sendJson(response, 404, manualSessionErrorPayload({ code: "session_not_found", message: "Code Agent session is not visible to the current actor.", sessionId })); - return; - } - if (!manualSessionUsable(session.status)) { - sendJson(response, 409, manualSessionErrorPayload({ code: "session_not_usable", message: `Code Agent session ${sessionId} is ${session.status}; create a new session before continuing.`, sessionId, session })); - return; - } - sendJson(response, 200, { - ok: true, - status: "selected", - contractVersion: "code-agent-manual-session-v1", - session: publicManualAgentSession(session), - nextCommands: manualSessionNextCommands(session), - valuesRedacted: true, - secretMaterialStored: false - }); -} - -async function prepareManualCodeAgentSession({ params, options, traceId, response }) { - if (!options.actor?.id) { - sendJson(response, 401, manualSessionErrorPayload({ - code: "auth_required", - message: "Authentication is required before using an explicit HWLAB Code Agent session.", - traceId, - retryable: true - })); - return { blocked: true }; - } - const sessionId = safeSessionId(params.sessionId); - if (!sessionId) { - sendJson(response, 409, manualSessionErrorPayload({ - code: "session_required", - message: "Code Agent session is explicit in HWLAB v0.2; create/select a session before sending a turn.", - traceId, - retryable: true, - nextCommands: ["hwlab-cli client agent session create --provider-profile PROFILE", "hwlab-cli client agent send --session-id --message TEXT"] - })); - return { blocked: true }; - } - const session = await getVisibleManualAgentSession(options, sessionId); - if (!session) { - sendJson(response, 404, manualSessionErrorPayload({ code: "session_not_found", message: "Code Agent session is not visible to the current actor.", traceId, sessionId })); - return { blocked: true }; - } - const freshContinuation = manualSessionFreshContinuationRequired(session); - if (!manualSessionUsable(session.status) && !freshContinuation) { - sendJson(response, 409, manualSessionErrorPayload({ code: "session_not_usable", message: `Code Agent session ${sessionId} is ${session.status}; create a new session before continuing.`, traceId, sessionId, session })); - return { blocked: true }; - } - const requestedProjectId = textValue(params.projectId); - const sessionProjectId = textValue(session.projectId); - if (requestedProjectId && sessionProjectId && requestedProjectId !== sessionProjectId) { - sendJson(response, 409, manualSessionErrorPayload({ code: "session_project_mismatch", message: `sessionId ${sessionId} belongs to ${sessionProjectId}; requested ${requestedProjectId}.`, traceId, sessionId, session })); - return { blocked: true }; - } - const storedConversationId = safeConversationId(session.conversationId); - const requestedConversationId = safeConversationId(params.conversationId); - if (requestedConversationId && storedConversationId && requestedConversationId !== storedConversationId) { - sendJson(response, 409, manualSessionErrorPayload({ code: "session_conversation_mismatch", message: `sessionId ${sessionId} is bound to ${storedConversationId}; requested ${requestedConversationId}.`, traceId, sessionId, session })); - return { blocked: true }; - } - const conversationId = requestedConversationId || storedConversationId; - if (!conversationId) { - sendJson(response, 409, manualSessionErrorPayload({ code: "session_conversation_required", message: "Code Agent session has no bound conversationId; create a new session before continuing.", traceId, sessionId, session })); - return { blocked: true }; - } - params.conversationId = conversationId; - params.sessionId = sessionId; - params.projectId = requestedProjectId || sessionProjectId || DEFAULT_CODE_AGENT_PROJECT_ID; - applyManualSessionProviderProfile(params, session); - if (freshContinuation) { - params.threadId = undefined; - params.forceFreshAgentRunSession = true; - params.forceFreshAgentRunSessionReason = manualSessionFreshContinuationReason(session); - } else { - params.threadId = safeOpaqueId(params.threadId) || manualSessionContinuationThreadId(session) || undefined; - } - return { session }; -} - -function applyManualSessionProviderProfile(params = {}, session = null) { - const requested = firstNonEmptyValue(params.providerProfile, params.codeAgentProviderProfile); - if (requested) { - const normalized = normalizeCodeAgentProviderProfile(requested); - params.providerProfile = normalized; - params.codeAgentProviderProfile = normalized; - params.providerProfileSource = "request"; - return normalized; - } - const stored = manualSessionStoredProviderProfile(session); - if (!stored) return null; - params.providerProfile = stored; - params.codeAgentProviderProfile = stored; - params.providerProfileSource = "session"; - return stored; -} - -function manualSessionStoredProviderProfile(session = null) { - const evidence = session?.session && typeof session.session === "object" ? session.session : {}; - const traceProfile = latestCompletedTraceResultProviderProfile(evidence.traceResults); - const raw = firstNonEmptyValue( - traceProfile, - evidence.agentRun?.backendProfile, - evidence.providerProfile, - evidence.codeAgentProviderProfile, - evidence.backendProfile - ); - if (!raw) return null; - const normalized = normalizeCodeAgentProviderProfile(raw); - return normalized === "runtime-default" ? null : normalized; -} - -function latestCompletedTraceResultProviderProfile(traceResults = null) { - if (!traceResults || typeof traceResults !== "object") return null; - const entries = Object.values(traceResults).filter((value) => value && typeof value === "object"); - for (const result of entries.reverse()) { - const agentRun = result.agentRun && typeof result.agentRun === "object" ? result.agentRun : null; - if (!agentRun) continue; - const status = normalizeTurnStatus(result.status ?? agentRun.terminalStatus ?? agentRun.commandState ?? agentRun.status); - if (status && status !== "completed") continue; - const profile = firstNonEmptyValue(agentRun.backendProfile, result.providerProfile, result.codeAgentProviderProfile); - if (profile) return profile; - } - return null; -} - -function manualSessionFreshContinuationRequired(session = null) { - const status = String(session?.status ?? "").trim().toLowerCase().replace(/_/gu, "-"); - if (status === "thread-resume-failed") return true; - const continuation = session?.session?.continuation && typeof session.session.continuation === "object" ? session.session.continuation : null; - return continuation?.requiresFreshSession === true; -} - -function manualSessionFreshContinuationReason(session = null) { - const continuation = session?.session?.continuation && typeof session.session.continuation === "object" ? session.session.continuation : null; - return firstNonEmptyValue(continuation?.reasonCode, continuation?.reason, session?.status, "stale-continuation") ?? "stale-continuation"; -} - -function manualSessionContinuationThreadId(session = null) { - if (manualSessionFreshContinuationRequired(session)) return null; - const evidence = session?.session && typeof session.session === "object" ? session.session : {}; - return safeOpaqueId(evidence.threadId) || safeOpaqueId(evidence.agentRun?.threadId) || safeOpaqueId(session?.threadId) || null; -} - -async function getVisibleManualAgentSession(options, sessionId) { - if (!safeSessionId(sessionId) || !options.actor?.id || !options.accessController?.getAgentSession) return null; - const session = await options.accessController.getAgentSession(sessionId); - if (!session) return null; - if (session.ownerUserId === options.actor.id) return session; - return null; -} - -function publicManualAgentSession(session) { - if (!session || typeof session !== "object") return null; - return { - sessionId: session.id, - agentId: session.agentId ?? "hwlab-code-agent", - status: session.status ?? null, - usable: manualSessionUsable(session.status), - threadId: safeOpaqueId(session.threadId) || null, - lastTraceId: safeTraceId(session.lastTraceId) || null, - providerProfile: textValue(session.session?.providerProfile) || null, - createdAt: session.startedAt ?? null, - updatedAt: session.updatedAt ?? null, - valuesRedacted: true, - secretMaterialStored: false - }; -} - -function manualSessionUsable(status) { - const value = String(status ?? "").trim().toLowerCase().replace(/_/gu, "-"); - return !["blocked", "expired", "stale"].includes(value); -} - -function manualSessionErrorPayload({ code, message, reason = null, traceId = null, sessionId = null, session = null, retryable = true, nextCommands = undefined } = {}) { - return { - ok: false, - accepted: false, - status: "blocked", - traceId: safeTraceId(traceId) || null, - sessionId: safeSessionId(sessionId) || safeSessionId(session?.id) || null, - session: session ? publicManualAgentSession(session) : null, - error: { - code, - layer: "api", - category: "session_blocked", - retryable, - message, - reason, - userMessage: message, - route: "/v1/agent/chat" - }, - blocker: { - code, - layer: "api", - retryable, - summary: message - }, - nextCommands, - valuesRedacted: true, - secretMaterialStored: false - }; -} - -function manualSessionNextCommands(session) { - const sessionId = session?.id ?? ""; - return [ - `hwlab-cli client agent send --session-id ${sessionId} --message TEXT`, - `hwlab-cli client agent session status ${sessionId}` - ]; -} - -async function readJsonObjectBody(request, bodyLimitBytes) { - const body = await readBody(request, bodyLimitBytes); - try { - const value = body ? JSON.parse(body) : {}; - if (!value || typeof value !== "object" || Array.isArray(value)) return { ok: false, code: "invalid_params", message: "body must be a JSON object" }; - return { ok: true, value }; - } catch (error) { - return { ok: false, code: "parse_error", message: "Invalid JSON body", reason: error.message }; - } -} - -async function runCodeAgentChat(params, options) { - return handleCodeAgentChat(params, await codeAgentChatExecutionOptions(options, params)); -} - -function stripSyntheticConversationContext(params = {}, traceId, options = {}) { - if (Array.isArray(params.conversationContext?.messages) || Array.isArray(params.messages)) { - (options.traceStore ?? defaultCodeAgentTraceStore).append(traceId, { - type: "synthetic-context", - status: "ignored", - label: "synthetic-context:ignored", - message: "Synthetic conversation context was stripped; Code Agent continuation uses only Codex stdio native thread/resume and turn/start.", - valuesPrinted: false - }); - } - const { conversationContext, conversationContextTruncated, contextSource, messages, ...nativeParams } = params; - return nativeParams; -} - -async function codeAgentChatExecutionOptions(options = {}, params = {}) { - const env = codeAgentProviderProfileEnv(options.env ?? process.env, params); - Object.assign(env, await codeAgentAuthEnv(options, env, params)); - return { - callProvider: options.callCodeAgentProvider, - timeoutMs: options.codeAgentTimeoutMs, - hardTimeoutMs: options.codeAgentHardTimeoutMs, - env, - workspace: options.workspace, - skillsDirs: options.skillsDirs, - skillsDirsExact: options.skillsDirsExact, - skillsDiscoveryDelayMs: options.skillsDiscoveryDelayMs, - sessionRegistry: options.sessionRegistry, - m3IoSkillRequestJson: options.m3IoSkillRequestJson ?? createCodeAgentM3HwlabApiRequestJson(options), - codexStdioManager: options.codexStdioManager, - traceStore: options.traceStore, - gatewayRegistry: options.gatewayRegistry - }; -} - -async function codeAgentAuthEnv(options = {}, env = process.env, params = {}) { - const ownerUserId = textValue(params.ownerUserId ?? options.actor?.id ?? ""); - const store = options.accessController?.store; - const key = ownerUserId && store?.findActiveDefaultApiKeyForUser - ? await store.findActiveDefaultApiKeyForUser(ownerUserId) - : null; - const envRuntimeLane = runtimeLaneForEnv(env); - const runtimeNamespace = runtimeNamespaceForEnv(env); - const namespaceRuntimeLane = runtimeLaneForNamespace(runtimeNamespace); - const runtimeLane = resolveRuntimeLaneAuthority({ envRuntimeLane, runtimeNamespace, namespaceRuntimeLane }); - const inClusterApiUrl = internalCodeAgentApiUrl(env, runtimeNamespace); - const inClusterWebUrl = internalCodeAgentWebUrl(env, runtimeNamespace); - const apiUrl = firstNonEmptyValue( - codeAgentAgentRunAdapterEnabled(env) ? inClusterApiUrl : null, - env.HWLAB_RUNTIME_API_URL, - sameRuntimeEndpoint(env.HWLAB_CLOUD_API_URL, runtimeNamespace, runtimeLane), - env.HWLAB_PUBLIC_ENDPOINT, - localCloudApiUrl(env) - ); - return { - HWLAB_CLOUD_API_URL: apiUrl, - HWLAB_RUNTIME_API_URL: apiUrl, - HWLAB_RUNTIME_WEB_URL: firstNonEmptyValue(codeAgentAgentRunAdapterEnabled(env) ? inClusterWebUrl : null, env.HWLAB_RUNTIME_WEB_URL, apiUrl), - HWLAB_RUNTIME_NAMESPACE: runtimeNamespace, - HWLAB_RUNTIME_LANE: runtimeLane, - HWLAB_RUNTIME_ENDPOINT_SOURCE: codeAgentAgentRunAdapterEnabled(env) ? "runtime-namespace" : "runtime-env", - HWLAB_RUNTIME_ENDPOINT_LOCKED: "1", - HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME: "1", - ...(options.actorApiKeySecret ? { HWLAB_API_KEY: options.actorApiKeySecret } : key?.displaySecret ? { HWLAB_API_KEY: key.displaySecret } : {}) - }; -} - -function internalCodeAgentApiUrl(env = process.env, namespace = runtimeNamespaceForEnv(env)) { - const serviceName = firstNonEmptyValue(env.HWLAB_CODE_AGENT_INTERNAL_API_SERVICE_NAME, env.HWLAB_CLOUD_API_SERVICE_NAME, "hwlab-cloud-api"); - const port = parsePositiveInteger(firstNonEmptyValue(env.HWLAB_CODE_AGENT_INTERNAL_API_PORT, env.HWLAB_CLOUD_API_PORT), 6667); - return `http://${serviceName}.${namespace}.svc.cluster.local:${port}`; -} - -function internalCodeAgentWebUrl(env = process.env, namespace = runtimeNamespaceForEnv(env)) { - const serviceName = firstNonEmptyValue(env.HWLAB_CODE_AGENT_INTERNAL_WEB_SERVICE_NAME, env.HWLAB_CLOUD_WEB_SERVICE_NAME, "hwlab-cloud-web"); - const port = parsePositiveInteger(firstNonEmptyValue(env.HWLAB_CODE_AGENT_INTERNAL_WEB_PORT, env.HWLAB_CLOUD_WEB_PORT), 8080); - return `http://${serviceName}.${namespace}.svc.cluster.local:${port}`; -} - -function runtimeNamespaceForEnv(env = process.env) { - return firstNonEmptyValue( - env.HWLAB_RUNTIME_NAMESPACE, - env.POD_NAMESPACE, - env.HWLAB_NAMESPACE, - defaultRuntimeNamespace(env) - ); -} - -function runtimeLaneForEnv(env = process.env) { - const raw = firstNonEmptyValue(env.HWLAB_RUNTIME_LANE, env.HWLAB_GITOPS_LANE, env.HWLAB_GITOPS_PROFILE, env.HWLAB_ENVIRONMENT, env.HWLAB_BOOT_REF); - if (!raw) return null; - const lane = normalizeRuntimeLane(raw); - if (!lane) { - throw runtimeAuthorityError("unsupported_runtime_lane", `unsupported HWLAB runtime lane: ${String(raw).trim()}`); - } - return lane; -} - -function runtimeLaneForNamespace(namespace) { - const value = String(namespace ?? "").trim().toLowerCase(); - if (value === "hwlab-prod") return "prod"; - if (value === "hwlab-dev") return "dev"; - const versionMatch = value.match(/^hwlab-v(\d+)$/u); - if (versionMatch) return versionRuntimeLane(versionMatch[1]); - return null; -} - -function sameRuntimeEndpoint(value, namespace, lane) { - const text = firstNonEmptyValue(value); - if (!text) return null; - const endpointLane = runtimeLaneForEndpoint(text); - const endpointNamespace = runtimeNamespaceForEndpoint(text); - if (namespace && endpointNamespace && endpointNamespace !== namespace) return null; - if (lane && endpointLane && endpointLane !== lane) return null; - return text; -} - -function runtimeNamespaceForEndpoint(value) { - const match = String(value ?? "").match(/\.((?:hwlab-[a-z0-9-]+))\.svc\.cluster\.local(?::|\/|$)/iu); - return match?.[1] ?? null; -} - -function runtimeLaneForEndpoint(value) { - const text = String(value ?? "").toLowerCase(); - const namespaceLane = runtimeLaneForNamespace(runtimeNamespaceForEndpoint(text)); - if (namespaceLane) return namespaceLane; - if (/hwlab-prod|:18666(?:\/|$)|:18667(?:\/|$)/u.test(text)) return "prod"; - if (/hwlab-dev|:17666(?:\/|$)|:17667(?:\/|$)|:16666(?:\/|$)|:16667(?:\/|$)/u.test(text)) return "dev"; - return null; -} - -function normalizeRuntimeLane(value) { - const profile = String(value ?? "").trim().toLowerCase(); - if (!profile) return null; - if (profile === "prod" || profile === "production") return "prod"; - if (profile === "dev" || profile === "development" || profile === "local") return "dev"; - const dotted = profile.match(/^v?0+\.(\d+)$/u); - if (dotted) return versionRuntimeLane(dotted[1]); - const compact = profile.match(/^v?0*(\d+)$/u); - if (compact) return versionRuntimeLane(compact[1]); - return null; -} - -function versionRuntimeLane(value) { - const version = Number.parseInt(value, 10); - if (!Number.isFinite(version) || version <= 0) return null; - return `v${String(version).padStart(2, "0")}`; -} - -function resolveRuntimeLaneAuthority({ envRuntimeLane, runtimeNamespace, namespaceRuntimeLane }) { - if (!namespaceRuntimeLane) { - throw runtimeAuthorityError("unsupported_runtime_namespace", `unsupported HWLAB runtime namespace: ${runtimeNamespace}`); - } - if (envRuntimeLane && envRuntimeLane !== namespaceRuntimeLane) { - throw runtimeAuthorityError( - "runtime_authority_mismatch", - `HWLAB runtime lane ${envRuntimeLane} does not match namespace ${runtimeNamespace} (${namespaceRuntimeLane})` - ); - } - return namespaceRuntimeLane; -} - -function runtimeAuthorityError(code, message) { - return Object.assign(new Error(message), { - code, - statusCode: 500, - valuesPrinted: false - }); -} - -function defaultRuntimeNamespace(env = process.env) { - const profile = runtimeLaneForEnv(env) ?? "dev"; - if (profile === "prod") return "hwlab-prod"; - if (profile === "dev") return "hwlab-dev"; - return `hwlab-${profile}`; -} - -function localCloudApiUrl(env = process.env) { - const port = parsePositiveInteger(env.HWLAB_CLOUD_API_PORT, 6667); - return `http://127.0.0.1:${port}`; -} - -function codeAgentProviderProfileEnv(env = process.env, params = {}) { - const profile = normalizeCodeAgentProviderProfile(params.providerProfile ?? params.codeAgentProviderProfile ?? params.provider ?? env.HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE); - if (profile === "runtime-default") return env; - const overlay = { ...env }; - overlay.HWLAB_CODE_AGENT_SELECTED_PROVIDER_PROFILE = profile; - overlay.HWLAB_CODE_AGENT_PROVIDER_PROFILE_LABEL = CODE_AGENT_PROVIDER_PROFILE_LABELS[profile] ?? profile; - overlay.HWLAB_CODE_AGENT_PROVIDER = "codex-stdio"; - if (profile === "codex-api") { - overlay.HWLAB_CODE_AGENT_MODEL = firstNonEmptyValue(env.HWLAB_CODE_AGENT_CODEX_API_MODEL, env.HWLAB_CODE_AGENT_LEGACY_CODEX_MODEL, DEFAULT_CODE_AGENT_CODEX_API_MODEL); - overlay.HWLAB_CODE_AGENT_OPENAI_BASE_URL = firstNonEmptyValue(env.HWLAB_CODE_AGENT_CODEX_API_BASE_URL, env.HWLAB_CODE_AGENT_LEGACY_OPENAI_BASE_URL, DEFAULT_CODE_AGENT_CODEX_API_BASE_URL); - } else if (profile === "minimax-m3") { - overlay.HWLAB_CODE_AGENT_MODEL = firstNonEmptyValue(env.HWLAB_CODE_AGENT_MINIMAX_M3_MODEL, DEFAULT_CODE_AGENT_MINIMAX_M3_MODEL); - } else { - const dynamicProfile = profile !== "deepseek"; - overlay.HWLAB_CODE_AGENT_MODEL = dynamicProfile - ? firstNonEmptyValue(env[`HWLAB_CODE_AGENT_${providerProfileEnvKey(profile)}_MODEL`], profile) - : firstNonEmptyValue(env.HWLAB_CODE_AGENT_DEEPSEEK_MODEL, DEFAULT_CODE_AGENT_DEEPSEEK_MODEL); - if (!dynamicProfile || !codeAgentAgentRunAdapterEnabled(env)) { - overlay.HWLAB_CODE_AGENT_OPENAI_BASE_URL = firstNonEmptyValue(env.HWLAB_CODE_AGENT_DEEPSEEK_BASE_URL, defaultDeepSeekBaseUrlForEnv(env)); - } - } - overlay.NO_PROXY = mergeNoProxyEntries(env.NO_PROXY, codeAgentClusterNoProxyEntries(env)); - overlay.no_proxy = mergeNoProxyEntries(env.no_proxy ?? env.NO_PROXY, codeAgentClusterNoProxyEntries(env)); - return overlay; -} - -function normalizeCodeAgentProviderProfile(value) { - const text = String(value ?? DEFAULT_CODE_AGENT_PROVIDER_PROFILE).trim().toLowerCase(); - if (!text) return DEFAULT_CODE_AGENT_PROVIDER_PROFILE; - if (text === "runtime-default") return text; - const normalized = CODE_AGENT_PROVIDER_PROFILE_ALIASES[text] ?? text; - return CODE_AGENT_PROVIDER_PROFILE_ID_PATTERN.test(normalized) ? normalized : DEFAULT_CODE_AGENT_PROVIDER_PROFILE; -} - -function providerProfileEnvKey(profile) { - return String(profile ?? "") - .trim() - .toUpperCase() - .replace(/[^A-Z0-9]+/gu, "_") - .replace(/^_+|_+$/gu, "") || "PROFILE"; -} - -function defaultDeepSeekBaseUrlForEnv(env = process.env) { - const namespace = env.HWLAB_GITOPS_PROFILE === "prod" || env.HWLAB_ENVIRONMENT === "prod" ? "hwlab-prod" : "hwlab-dev"; - return `http://hwlab-deepseek-proxy.${namespace}.svc.cluster.local:4000/v1/responses`; -} - -function codeAgentClusterNoProxyEntries(env = process.env) { - const namespace = env.HWLAB_GITOPS_PROFILE === "prod" || env.HWLAB_ENVIRONMENT === "prod" ? "hwlab-prod" : "hwlab-dev"; - return [ - `${namespace}.svc.cluster.local`, - ".svc", - ".cluster.local", - "127.0.0.1", - "localhost", - "::1", - "10.0.0.0/8", - "10.42.0.0/16", - "10.43.0.0/16", - "hyueapi.com", - ".hyueapi.com", - "hyui.com", - ".hyui.com", - "api.minimaxi.com", - ".minimaxi.com" - ]; -} - -function mergeNoProxyEntries(current, additions = []) { - return uniqueStrings([...String(current ?? "").split(","), ...additions]).join(","); -} - -function firstNonEmptyValue(...values) { - for (const value of values) { - const text = String(value ?? "").trim(); - if (text) return text; - } - return null; -} - -function safeOtelConfigText(value) { - const text = firstNonEmptyValue(value); - return text === null ? null : text.slice(0, 120); -} - -function safeOtelUrlHost(value) { - const text = firstNonEmptyValue(value); - if (text === null) return null; - try { - return new URL(text).host.slice(0, 120); - } catch { - return null; - } -} - -function withMdtodoExecutionPrompt(params = {}) { - const execution = mdtodoExecutionContextFromParams(params); - if (!execution?.hwpodId || !execution?.hwpodWorkspaceArgs) return params; - const message = firstNonEmptyValue(params.message, params.prompt, params.text); - if (!message || /hwpodWorkspaceArgs\s*:/u.test(message)) return params; - const taskRef = firstNonEmptyValue(params.taskRef, execution.taskRef) ?? "-"; - const mdtodoRootRef = execution.mdtodoRootRef ?? "docs/MDTODO"; - const prefix = [ - "# MDTODO HWPOD 执行上下文", - `TaskRef: ${taskRef}`, - `sourceId: ${execution.sourceId ?? "-"}`, - `hwpodId: ${execution.hwpodId}`, - `nodeId: ${execution.nodeId ?? "-"}`, - `mdtodoRootRef: ${mdtodoRootRef}`, - `hwpodWorkspaceArgs: ${execution.hwpodWorkspaceArgs}`, - "所有 hwpod/hwpod-ctl 命令都必须携带上述 hwpodWorkspaceArgs。读取 MDTODO、报告和工程文件必须通过 HWPOD workspace/node-ops 入口，不得猜测或访问普通容器本地路径，也不得创建本地 .hwlab/hwpod-spec.yaml fallback。" - ].join("\n"); - const nextMessage = `${prefix}\n\n${message}`; - return { - ...params, - message: nextMessage, - prompt: nextMessage, - launchContext: { - ...(params.launchContext && typeof params.launchContext === "object" ? params.launchContext : {}), - hwpodId: execution.hwpodId, - nodeId: execution.nodeId, - mdtodoRootRef: execution.mdtodoRootRef, - hwpodWorkspaceArgs: execution.hwpodWorkspaceArgs, - contextFingerprint: execution.contextFingerprint, - executionContext: execution, - valuesRedacted: true - } - }; -} - -function mdtodoExecutionContextFromParams(params = {}) { - const launchContext = params.launchContext && typeof params.launchContext === "object" && !Array.isArray(params.launchContext) ? params.launchContext : null; - const execution = launchContext?.executionContext && typeof launchContext.executionContext === "object" && !Array.isArray(launchContext.executionContext) ? launchContext.executionContext : launchContext; - if (!execution || typeof execution !== "object") return null; - return { - sourceId: safeOtelConfigText(execution.sourceId ?? launchContext?.sourceId), - sourceKind: safeOtelConfigText(execution.sourceKind ?? launchContext?.sourceKind), - projectId: safeOtelConfigText(execution.projectId ?? launchContext?.projectId ?? params.projectId), - taskRef: safeOtelText(execution.taskRef ?? launchContext?.taskRef ?? params.taskRef), - fileRef: safeOtelConfigText(execution.fileRef ?? launchContext?.fileRef), - relativePath: safeOtelText(execution.relativePath ?? launchContext?.relativePath), - taskId: safeOtelConfigText(execution.taskId ?? launchContext?.taskId), - hwpodId: safeOtelText(execution.hwpodId ?? launchContext?.hwpodId), - nodeId: safeOtelText(execution.nodeId ?? launchContext?.nodeId), - mdtodoRootRef: safeOtelText(execution.mdtodoRootRef ?? launchContext?.mdtodoRootRef), - workspaceRootHash: safeOtelText(execution.workspaceRootHash ?? launchContext?.workspaceRootHash), - hwpodWorkspaceArgs: safeOtelText(execution.hwpodWorkspaceArgs ?? launchContext?.hwpodWorkspaceArgs, 500), - contextFingerprint: safeOtelText(execution.contextFingerprint ?? launchContext?.contextFingerprint), - valuesRedacted: true - }; -} - -function agentChatOtelAttributes(params = {}) { - const execution = mdtodoExecutionContextFromParams(params); - return { - "agent.chat.session_id": safeSessionId(params.sessionId) || null, - "agent.chat.provider_profile": safeOtelConfigText(params.providerProfile ?? params.codeAgentProviderProfile), - "agent.chat.provider_profile_source": safeOtelConfigText(params.providerProfileSource), - "agent.chat.task_ref": safeOtelText(params.taskRef ?? execution?.taskRef), - "agent.chat.source_id": safeOtelConfigText(execution?.sourceId ?? params.sourceId), - "agent.chat.hwpod_id": safeOtelText(execution?.hwpodId), - "agent.chat.workspace_root_hash": safeOtelText(execution?.workspaceRootHash), - "agent.chat.context_fingerprint": safeOtelText(execution?.contextFingerprint) - }; -} - -function safeOtelText(value, max = 360) { - const text = firstNonEmptyValue(value); - return text === null ? null : text.slice(0, max); -} - -async function submitCodeAgentChatTurn({ params, options, traceId }) { - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - const results = options.codeAgentChatResults ?? createCodeAgentChatResultStore(); - const timestamp = new Date().toISOString(); - const runtimeEnv = options.env ?? process.env; - const adapterEnabled = codeAgentAgentRunAdapterEnabled(runtimeEnv); - const acceptedPayload = { - accepted: true, - status: "running", - traceId, - ...codeAgentOtelTraceFields(traceId, process.env), - ...codeAgentTurnLifecycleFields(traceId, params), - conversationId: safeConversationId(params.conversationId) || null, - sessionId: safeSessionId(params.sessionId) || null, - threadId: safeOpaqueId(params.threadId) || null, - projectId: params.projectId ?? null, - ...codeAgentAdmissionTimingFields(timestamp) - }; - await recordCodeAgentTurnAdmission({ payload: acceptedPayload, params, options, traceId }); - results.set(traceId, annotateOwner(acceptedPayload, params)); - void emitCodeAgentOtelSpan("provider_decision", traceId, runtimeEnv, { - attributes: { - ...agentChatOtelAttributes(params), - adapterEnabled, - adapter: safeOtelConfigText(runtimeEnv.HWLAB_CODE_AGENT_ADAPTER ?? runtimeEnv.HWLAB_CODE_AGENT_PROVIDER), - provider: safeOtelConfigText(runtimeEnv.HWLAB_CODE_AGENT_PROVIDER), - defaultProviderProfile: safeOtelConfigText(runtimeEnv.HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE), - agentRunManagerHost: safeOtelUrlHost(runtimeEnv.AGENTRUN_MGR_URL ?? runtimeEnv.HWLAB_CODE_AGENT_AGENTRUN_MGR_URL), - agentRunRunnerNamespace: safeOtelConfigText(runtimeEnv.HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE ?? runtimeEnv.AGENTRUN_RUNTIME_NAMESPACE), - agentRunSourceCommitPresent: Boolean(firstNonEmptyValue(runtimeEnv.HWLAB_CODE_AGENT_AGENTRUN_SOURCE_COMMIT)) - } - }); - if (adapterEnabled) { - const run = async () => { - let initial = null; - let executionOptions = options; - try { - initial = initialAgentRunChatResult({ params, options, traceId }); - results.set(traceId, annotateOwner(initial, params)); - const billingStartedAt = Date.now(); - const billingPreflight = await preflightCodeAgentBilling({ params, options, traceId, sendResponse: false }); - void emitCodeAgentOtelSpan("billing_preflight", traceId, options.env ?? process.env, { - startTimeMs: billingStartedAt, - attributes: { sessionId: safeSessionId(params.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId, blocked: Boolean(billingPreflight?.blocked) }, - status: billingPreflight?.blocked ? "error" : "ok", - error: billingPreflight?.blocked ? billingPreflight?.payload?.error?.message ?? billingPreflight?.blocker?.summary ?? "billing preflight blocked" : null - }); - if (billingPreflight?.blocked) { - await settleAdmittedCodeAgentFailure({ - base: initial, - params, - options, - traceId, - results, - failure: billingPreflight, - traceLabel: "billing:preflight:failed" - }); - return; - } - const dispatchParams = { ...params, userBillingReservation: billingPreflight?.reservation ?? null }; - initial = withCodeAgentBillingReservation(initial, dispatchParams); - results.set(traceId, annotateOwner(initial, dispatchParams)); - executionOptions = { ...options, ...(await codeAgentChatExecutionOptions(options, dispatchParams)) }; - const dispatchStartedAt = Date.now(); - const payload = await submitAgentRunChatTurn({ params: dispatchParams, options: executionOptions, traceId, traceStore, results }); - void emitCodeAgentOtelSpan("agentrun_dispatch", traceId, executionOptions.env ?? process.env, { - startTimeMs: dispatchStartedAt, - attributes: { sessionId: safeSessionId(dispatchParams.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, dispatchParams).turnId, runId: payload?.agentRun?.runId ?? null, commandId: payload?.agentRun?.commandId ?? null, providerProfile: dispatchParams.providerProfile ?? null } - }); - if (isCodeAgentResultCanceled(results.get(traceId))) return; - const owned = annotateOwner(withCodeAgentBillingReservation(payload, dispatchParams), dispatchParams); - publishHwlabAgentRunCommandShadow({ - params: dispatchParams, - payload: owned, - traceId, - lifecycle: codeAgentTurnLifecycleFields(traceId, dispatchParams), - env: executionOptions.env ?? process.env - }); - await recordCodeAgentSessionOwner({ payload: owned, params: dispatchParams, options: executionOptions, status: "running" }); - results.set(traceId, owned); - scheduleAgentRunProjectionSync({ traceId, params: dispatchParams, options: executionOptions, traceStore, currentResult: owned }); - void emitCodeAgentOtelSpan("projection_write", traceId, executionOptions.env ?? process.env, { - attributes: { sessionId: safeSessionId(dispatchParams.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, dispatchParams).turnId, runId: owned?.agentRun?.runId ?? null, commandId: owned?.agentRun?.commandId ?? null, projection: "scheduled" } - }); - } catch (error) { - if (isCodeAgentResultCanceled(results.get(traceId))) return; - const dispatchErrorAttributes = codeAgentDispatchErrorOtelAttributes(error); - void emitCodeAgentOtelSpan("agentrun_dispatch", traceId, executionOptions.env ?? process.env, { - status: "error", - error, - attributes: { sessionId: safeSessionId(params.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId, ...dispatchErrorAttributes } - }); - await settleAdmittedCodeAgentFailure({ - base: initial ?? codeAgentAdmittedFailureBasePayload({ params, options, traceId }), - params, - options: executionOptions, - traceId, - results, - failure: codeAgentDispatchFailure(error), - traceLabel: "agentrun:dispatch:failed" - }); - } - }; - setImmediate(() => { run(); }); - return annotateOwner(acceptedPayload, params); - } - traceStore.ensure(traceId, { - runnerKind: "codex-app-server-stdio-runner", - workspace: options.workspace ?? options.env?.HWLAB_CODE_AGENT_CODEX_WORKSPACE ?? options.env?.HWLAB_CODE_AGENT_WORKSPACE ?? null, - sandbox: options.env?.HWLAB_CODE_AGENT_CODEX_SANDBOX ?? null, - sessionMode: "codex-app-server-stdio-long-lived", - implementationType: "repo-owned-codex-app-server-stdio-session" - }); - traceStore.append(traceId, { - type: "request", - status: "accepted", - label: "request:accepted-short-connection", - message: "Code Agent request accepted; client must poll trace/result with short HTTP requests.", - waitingFor: "codex-stdio" - }); - - const run = async () => { - try { - const billingStartedAt = Date.now(); - const billingPreflight = await preflightCodeAgentBilling({ params, options, traceId, sendResponse: false }); - void emitCodeAgentOtelSpan("billing_preflight", traceId, options.env ?? process.env, { - startTimeMs: billingStartedAt, - attributes: { sessionId: safeSessionId(params.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId, blocked: Boolean(billingPreflight?.blocked) }, - status: billingPreflight?.blocked ? "error" : "ok", - error: billingPreflight?.blocked ? billingPreflight?.payload?.error?.message ?? billingPreflight?.blocker?.summary ?? "billing preflight blocked" : null - }); - if (billingPreflight?.blocked) { - await settleAdmittedCodeAgentFailure({ - base: codeAgentAdmittedFailureBasePayload({ params, options, traceId }), - params, - options, - traceId, - results, - failure: billingPreflight, - traceLabel: "billing:preflight:failed" - }); - return; - } - const dispatchParams = { ...params, userBillingReservation: billingPreflight?.reservation ?? null }; - const payload = await runCodeAgentChat(dispatchParams, options); - if (isCodeAgentResultCanceled(results.get(traceId))) return; - await finalizeCodeAgentBillingUsage({ payload, params: dispatchParams, options }); - await recordCodeAgentSessionOwner({ payload, params: dispatchParams, options, status: codeAgentOwnerStatusForResult(payload) }); - results.set(traceId, annotateOwner(payload, dispatchParams)); - traceStore.append(traceId, { - type: "result", - status: payload.status === "completed" ? "completed" : "failed", - label: `result:${payload.status}`, - message: payload.status === "completed" - ? "Code Agent result is ready for short-connection polling." - : payload.error?.message ?? "Code Agent completed without a successful reply.", - sessionId: payload.sessionId ?? payload.session?.sessionId, - sessionStatus: payload.session?.status, - terminal: true - }); - } catch (error) { - if (isCodeAgentResultCanceled(results.get(traceId))) return; - const payload = { - status: "failed", - traceId, - conversationId: safeConversationId(params.conversationId) || null, - sessionId: safeSessionId(params.sessionId) || null, - error: { - code: "code_agent_background_failed", - layer: "api", - retryable: true, - message: error?.message ?? "Code Agent background turn failed", - userMessage: "Code Agent 后台任务失败；trace/result 轮询已保留错误。" - }, - updatedAt: new Date().toISOString() - }; - await finalizeCodeAgentBillingUsage({ payload, params, options }); - results.set(traceId, payload); - traceStore.append(traceId, { - type: "result", - status: "failed", - label: "result:failed", - errorCode: payload.error.code, - message: payload.error.message, - terminal: true - }); - } - }; - setImmediate(() => { - run(); - }); - return annotateOwner(acceptedPayload, params); -} - -async function recordCodeAgentTurnAdmission({ payload = {}, params = {}, options = {}, traceId } = {}) { - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - let ownerRecord = null; - try { - ownerRecord = await recordCodeAgentSessionOwner({ payload, params, options, status: "running" }); - if (!ownerRecord) { - const error = new Error("Code Agent turn admission did not durably persist Workbench session ownership."); - error.code = "workbench_admission_owner_not_persisted"; - error.valuesRedacted = true; - throw error; - } - await recordCodeAgentSessionInputFact({ - params, - options, - traceId, - delivery: "queue", - status: "admitted", - messageId: codeAgentTurnLifecycleFields(traceId, params).userMessageId, - commandId: payload.agentRun?.commandId ?? null - }); - } catch (error) { - throw codeAgentAdmissionUnavailableError(error, { params, traceId }); - } - traceStore.append(traceId, { - type: "request", - status: "admitted", - label: "turn:admitted", - message: "Code Agent turn was durably admitted before billing or dispatch preflight.", - sessionId: safeSessionId(params.sessionId) || null, - threadId: safeOpaqueId(params.threadId) || null, - ...codeAgentOtelTraceFields(traceId, options.env ?? process.env), - waitingFor: "billing-or-dispatch", - valuesPrinted: false - }); - void emitCodeAgentOtelSpan("durable_admission", traceId, options.env ?? process.env, { - attributes: { sessionId: safeSessionId(params.sessionId) || null, threadId: safeOpaqueId(params.threadId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId } - }); - return ownerRecord; -} - -async function recordCodeAgentSessionInputFact({ params = {}, options = {}, traceId, delivery = "queue", status = "admitted", messageId = null, commandId = null, error = null } = {}) { - const runtimeStore = options.runtimeStore ?? null; - if (typeof runtimeStore?.writeWorkbenchFacts !== "function") return null; - const resolvedTraceId = safeTraceId(traceId ?? params.traceId) || null; - const lifecycle = codeAgentTurnLifecycleFields(resolvedTraceId, params); - const sessionId = safeSessionId(params.sessionId ?? params.session?.sessionId) || null; - if (!sessionId) return null; - const resolvedMessageId = safeMessageId(messageId ?? params.messageId ?? lifecycle.userMessageId) || lifecycle.userMessageId; - const resolvedCommandId = textValue(commandId ?? params.commandId ?? params.agentRun?.commandId) || null; - const normalizedDelivery = codeAgentInputDelivery(delivery); - const normalizedStatus = codeAgentInputStatus(status); - const now = new Date().toISOString(); - const sourceEventId = `${sessionId}:${lifecycle.turnId}:${normalizedDelivery}:${resolvedTraceId ?? resolvedCommandId ?? resolvedMessageId}`; - const inputId = stableCodeAgentInputId({ sessionId, turnId: lifecycle.turnId, traceId: resolvedTraceId, messageId: resolvedMessageId, commandId: resolvedCommandId, delivery: normalizedDelivery, sourceEventId }); - const fact = { - inputId, - sessionId, - turnId: lifecycle.turnId, - traceId: resolvedTraceId, - messageId: resolvedMessageId, - commandId: resolvedCommandId, - delivery: normalizedDelivery, - status: normalizedStatus, - errorCode: textValue(error?.code ?? error?.error?.code) || null, - sourceEventId, - promptHash: params.message || params.prompt || params.text ? sha256Text(params.message ?? params.prompt ?? params.text).slice(0, 16) : null, - textBytes: params.message || params.prompt || params.text ? Buffer.byteLength(String(params.message ?? params.prompt ?? params.text), "utf8") : 0, - valuesRedacted: true, - secretMaterialStored: false, - createdAt: now, - updatedAt: now - }; - return runtimeStore.writeWorkbenchFacts({ facts: { inputs: [fact] } }, { - sessionId, - traceId: resolvedTraceId, - turnId: lifecycle.turnId, - messageId: resolvedMessageId, - ownerUserId: options.actor?.id ?? params.ownerUserId, - projectId: params.projectId ?? DEFAULT_CODE_AGENT_PROJECT_ID, - valuesPrinted: false - }); -} - -function stableCodeAgentInputId(value) { - const payload = Object.fromEntries(Object.entries(value ?? {}).filter(([, item]) => item !== undefined && item !== null && item !== "").sort(([left], [right]) => left.localeCompare(right))); - return `wsi_${sha256Text(JSON.stringify(payload)).slice(0, 32)}`; -} - -function codeAgentInputDelivery(value) { - const text = textValue(value).toLowerCase(); - return ["queue", "steer", "cancel"].includes(text) ? text : "queue"; -} - -function codeAgentInputStatus(value) { - const text = textValue(value).toLowerCase().replace(/-/gu, "_"); - return ["admitted", "promoted", "blocked", "failed", "canceled", "completed"].includes(text) ? text : "admitted"; -} - -function codeAgentAdmissionUnavailableError(error, { params = {}, traceId } = {}) { - const payload = codeAgentAdmissionUnavailablePayload({ error, params, traceId }); - return Object.assign(new Error(payload.error.message), { - code: "admission_unavailable", - statusCode: 503, - payload, - alreadyClassified: true - }); -} - -function isCodeAgentAdmissionUnavailableError(error) { - return error?.code === "admission_unavailable" && error?.payload; -} - -function codeAgentAdmissionUnavailablePayload({ error, params = {}, traceId } = {}) { - const message = error?.message ?? "Code Agent turn admission is unavailable; the user message was not persisted."; - return { - ok: false, - accepted: false, - status: "failed", - traceId: safeTraceId(traceId) || null, - ...codeAgentOtelTraceFields(traceId, options.env ?? process.env), - ...codeAgentTurnLifecycleFields(traceId, params), - conversationId: safeConversationId(params.conversationId) || null, - sessionId: safeSessionId(params.sessionId) || null, - threadId: safeOpaqueId(params.threadId) || null, - error: { - code: "admission_unavailable", - layer: "admission", - retryable: true, - message, - userMessage: "本次输入尚未持久化，请保留草稿后重试。", - route: "/v1/agent/chat" - }, - blocker: { - code: "admission_unavailable", - layer: "admission", - retryable: true, - summary: message - }, - valuesRedacted: true, - secretMaterialStored: false - }; -} - -function codeAgentDispatchFailure(error) { - const message = error?.message ?? "AgentRun dispatch failed"; - return { - statusCode: 503, - payload: { - error: { - code: error?.code ?? "agentrun_dispatch_failed", - layer: "agentrun", - retryable: true, - message, - userMessage: "AgentRun 调度失败；trace/result 轮询已保留错误。", - route: "/v1/agent/chat" - }, - blocker: { - code: error?.code ?? "agentrun_dispatch_failed", - layer: "agentrun", - retryable: true, - summary: message - } - } - }; -} - -function codeAgentDispatchErrorOtelAttributes(error) { - const agentRunError = error?.agentRunError && typeof error.agentRunError === "object" ? error.agentRunError : null; - const details = agentRunError?.error?.details && typeof agentRunError.error.details === "object" ? agentRunError.error.details : null; - return { - failureKind: safeOtelText(error?.code ?? agentRunError?.failureKind, 120), - upstreamStatusCode: Number.isInteger(error?.statusCode) ? error.statusCode : null, - upstreamTraceId: safeTraceId(agentRunError?.traceId) || null, - upstreamMessage: safeOtelText(agentRunError?.message ?? error?.message, 300), - upstreamStderrTail: safeOtelText(details?.stderr, 600) - }; -} - -function safeOtelText(value, limit = 300) { - const text = String(value ?? "").replace(/[\u0000-\u001f\u007f]+/gu, " ").trim(); - return text ? text.slice(0, limit) : null; -} - -function codeAgentAdmittedFailureBasePayload({ params = {}, options = {}, traceId } = {}) { - const now = new Date().toISOString(); - return { - accepted: true, - status: "running", - shortConnection: true, - traceId, - ...codeAgentTurnLifecycleFields(traceId, params), - messageId: codeAgentTurnLifecycleFields(traceId, params).assistantMessageId, - projectId: params.projectId ?? null, - conversationId: safeConversationId(params.conversationId) || null, - sessionId: safeSessionId(params.sessionId) || null, - threadId: safeOpaqueId(params.threadId) || null, - ...codeAgentAdmissionTimingFields(now), - provider: codeAgentAgentRunAdapterEnabled(options.env ?? process.env) ? "agentrun" : "codex-stdio", - model: options.env?.HWLAB_CODE_AGENT_MODEL ?? "unknown", - backend: codeAgentAgentRunAdapterEnabled(options.env ?? process.env) ? "agentrun-v01" : "hwlab-cloud-api/code-agent-chat", - valuesPrinted: false - }; -} - -function codeAgentAdmissionTimingFields(value = null) { - const admittedAt = timestampIsoOrNow(value); - const timing = { - startedAt: admittedAt, - lastEventAt: admittedAt, - finishedAt: null, - durationMs: null, - observedAt: admittedAt, - lastEventAgeMs: 0, - valuesRedacted: true - }; - return { - createdAt: admittedAt, - updatedAt: admittedAt, - timing, - startedAt: admittedAt, - lastEventAt: admittedAt, - finishedAt: null, - durationMs: null - }; -} - -function codeAgentResultTimingFields(...sources) { - const records = sources.filter((source) => source && typeof source === "object"); - const timingSource = records.map((source) => source.timing).find((timing) => timing && typeof timing === "object") ?? {}; - const startedAt = firstTimestampIso(timingSource.startedAt, ...records.map((source) => source.startedAt), ...records.map((source) => source.createdAt)); - const lastEventAt = firstLatestTimestampIso(timingSource.lastEventAt, ...records.map((source) => source.lastEventAt), ...records.map((source) => source.updatedAt), ...records.map((source) => source.createdAt)); - const finishedAt = firstLatestTimestampIso(timingSource.finishedAt, ...records.map((source) => source.finishedAt), ...records.map((source) => source.completedAt)); - const durationMs = numberOrNull(timingSource.durationMs ?? records.map((source) => source.durationMs).find((value) => numberOrNull(value) !== null)); - const observedAt = timestampIsoOrNull(timingSource.observedAt); - const lastEventAgeMs = numberOrNull(timingSource.lastEventAgeMs); - if (!startedAt && !lastEventAt && !finishedAt && durationMs === null && lastEventAgeMs === null) return {}; - const timing = { - ...(timingSource && typeof timingSource === "object" ? timingSource : {}), - startedAt: startedAt ?? null, - lastEventAt: lastEventAt ?? null, - finishedAt: finishedAt ?? null, - durationMs, - observedAt: observedAt ?? null, - lastEventAgeMs, - valuesRedacted: timingSource.valuesRedacted !== false - }; - return { - timing, - startedAt: timing.startedAt, - lastEventAt: timing.lastEventAt, - finishedAt: timing.finishedAt, - durationMs: timing.durationMs - }; -} - -function firstTimestampIso(...values) { - for (const value of values) { - const timestamp = timestampIsoOrNull(value); - if (timestamp) return timestamp; - } - return null; -} - -function firstLatestTimestampIso(...values) { - let latest = null; - let latestMs = Number.NEGATIVE_INFINITY; - for (const value of values) { - const timestamp = timestampIsoOrNull(value); - if (!timestamp) continue; - const ms = Date.parse(timestamp); - if (!Number.isFinite(ms) || ms < latestMs) continue; - latest = timestamp; - latestMs = ms; - } - return latest; -} - -async function settleAdmittedCodeAgentFailure({ base = {}, params = {}, options = {}, traceId, results, failure = {}, traceLabel = "code-agent:failed" } = {}) { - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - const error = failure.payload?.error ?? failure.error ?? {}; - const blocker = failure.payload?.blocker ?? failure.blocker ?? null; - const message = error.message ?? blocker?.summary ?? "Code Agent request failed after admission."; - traceStore.append(traceId, { - type: "result", - status: "failed", - label: traceLabel, - errorCode: error.code ?? "code_agent_failed_after_admission", - message, - terminal: true, - valuesPrinted: false - }); - const now = new Date().toISOString(); - const payload = annotateOwner({ - ...base, - accepted: true, - status: "failed", - traceId, - ...codeAgentTurnLifecycleFields(traceId, base), - messageId: base.messageId ?? codeAgentTurnLifecycleFields(traceId, base).assistantMessageId, - conversationId: safeConversationId(base.conversationId ?? params.conversationId) || null, - sessionId: safeSessionId(base.sessionId ?? params.sessionId) || null, - threadId: safeOpaqueId(base.threadId ?? params.threadId) || null, - error: { - code: error.code ?? "code_agent_failed_after_admission", - layer: error.layer ?? blocker?.layer ?? "api", - retryable: error.retryable !== false, - message, - userMessage: error.userMessage ?? message, - route: error.route ?? "/v1/agent/chat" - }, - blocker: blocker ? { - code: blocker.code ?? error.code ?? "code_agent_failed_after_admission", - layer: blocker.layer ?? error.layer ?? "api", - retryable: blocker.retryable !== false, - summary: blocker.summary ?? message - } : null, - finalResponse: { - text: error.userMessage ?? message, - textChars: String(error.userMessage ?? message).length, - role: "assistant", - status: "failed", - traceId, - updatedAt: now, - source: "code-agent-admitted-failure", - valuesPrinted: false - }, - runnerTrace: traceStore.snapshot(traceId), - updatedAt: now, - valuesPrinted: false, - secretMaterialStored: false - }, params); - await finalizeCodeAgentBillingUsage({ payload, params, options }); - payload.runnerTrace = traceStore.snapshot(traceId); - recordCodeAgentConversationFact(payload, options); - results?.set?.(traceId, payload); - await recordCodeAgentSessionOwner({ payload, params, options, status: codeAgentOwnerStatusForResult(payload) }); - return payload; -} - -function scheduleAgentRunProjectionSync({ traceId, params = {}, options = {}, traceStore = defaultCodeAgentTraceStore, currentResult = null } = {}) { - if (!safeTraceId(traceId) || !currentResult?.agentRun?.runId || !currentResult?.agentRun?.commandId) return null; - const env = options.env ?? process.env; - const noResponseTimeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_NO_RESPONSE_TIMEOUT_MS, null); - const pollIntervalMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_POLL_INTERVAL_MS, 1_000); - const refreshOptions = codeAgentTurnStatusRefreshOptions(options); - return scheduleWorkbenchProjectionFinalizer({ - traceId, - currentResult, - traceStore, - noResponseTimeoutMs, - pollIntervalMs, - getCachedResult: () => options.codeAgentChatResults?.get?.(traceId), - isCanceled: isCodeAgentResultCanceled, - isTerminal: agentRunProjectionObservedTerminal, - activitySignature: agentRunProjectionActivitySignature, - sleep: sleepAgentRunProjectionSync, - performanceStore: options.backendPerformanceStore, - syncResult: ({ result }) => syncAgentRunChatResult({ - traceId, - currentResult: result, - options: refreshOptions, - traceStore, - forceResultSync: agentRunProjectionObservedTerminal(result, traceStore?.snapshot?.(traceId)) || traceSnapshotIsTerminal(traceStore?.snapshot?.(traceId)), - retryParams: params - }), - onTerminal: (payload, finalizerOptions = {}) => scheduleCodeAgentTerminalTurnStatusEffects({ payload: codeAgentPayloadWithObservedTerminalStatus(payload, traceStore?.snapshot?.(traceId)) ?? payload, params, options, preserveLastTraceId: finalizerOptions.preserveLastTraceId === true }) - }); -} - -export function startAgentRunProjectionResume({ options = {}, traceStore = defaultCodeAgentTraceStore, logger = console } = {}) { - const env = options.env ?? process.env; - if (!truthyFlag(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_ENABLED)) { - return { started: false, reason: "disabled", stop() {}, valuesPrinted: false }; - } - const initialDelayMs = parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INITIAL_DELAY_MS, DEFAULT_AGENTRUN_PROJECTION_RESUME_INITIAL_DELAY_MS); - const intervalMs = parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INTERVAL_MS, DEFAULT_AGENTRUN_PROJECTION_RESUME_INTERVAL_MS); - const minAgeMs = parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS, DEFAULT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS); - const batchSize = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE, DEFAULT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE); - const jitterMs = intervalMs > 0 ? Math.min(parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_JITTER_MS, DEFAULT_AGENTRUN_PROJECTION_RESUME_JITTER_MS), intervalMs) : 0; - const failureBackoffMs = Math.max(intervalMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_FAILURE_BACKOFF_MS, DEFAULT_AGENTRUN_PROJECTION_RESUME_FAILURE_BACKOFF_MS)); - const retryInitialDelayFloorMs = Math.max(1000, intervalMs * 2); - const candidateRetryMaxAttempts = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_CANDIDATE_RETRY_MAX_ATTEMPTS, 5); - const candidateRetryInitialDelayMs = Math.max(retryInitialDelayFloorMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_CANDIDATE_RETRY_INITIAL_DELAY_MS, retryInitialDelayFloorMs)); - const candidateRetryMaxDelayMs = Math.max(candidateRetryInitialDelayMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_CANDIDATE_RETRY_MAX_DELAY_MS, 600000)); - const passRetryMaxAttempts = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_PASS_RETRY_MAX_ATTEMPTS, 5); - const passRetryInitialDelayMs = Math.max(failureBackoffMs, retryInitialDelayFloorMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_PASS_RETRY_INITIAL_DELAY_MS, retryInitialDelayFloorMs)); - const passRetryMaxDelayMs = Math.max(passRetryInitialDelayMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_PASS_RETRY_MAX_DELAY_MS, 600000)); - const candidateRetryState = new Map(); - let passFailureAttempt = 0; - let stopped = false; - let active = false; - let timer = null; - let nextOffset = 0; - - const schedule = (delayMs, reason) => { - if (stopped) return; - const scheduledDelayMs = Math.max(0, Number(delayMs) || 0) + randomJitterMs(jitterMs); - timer = setTimeout(() => { - timer = null; - const offset = nextOffset; - active = true; - let nextDelayMs = intervalMs; - let nextReason = "interval"; - void runAgentRunProjectionResumePass({ options, traceStore, logger, batchSize, minAgeMs, offset, reason, candidateRetryState, candidateRetryMaxAttempts, candidateRetryInitialDelayMs, candidateRetryMaxDelayMs }) - .then((result) => { - passFailureAttempt = 0; - const candidateFailed = result?.failed > 0; - nextOffset = candidateFailed ? offset : (result?.scannedSessions >= batchSize ? offset + batchSize : 0); - nextDelayMs = candidateFailed ? failureBackoffMs : intervalMs; - nextReason = candidateFailed ? "failure_backoff" : "interval"; - }) - .catch((error) => { - passFailureAttempt += 1; - const retryDelayMs = projectionResumeCandidateRetryDelayMs(passFailureAttempt, passRetryInitialDelayMs, passRetryMaxDelayMs); - const retryExhausted = passFailureAttempt >= passRetryMaxAttempts; - nextDelayMs = retryDelayMs; - nextReason = retryExhausted ? "pass_retry_exhausted" : "pass_failure_backoff"; - logAgentRunProjectionResume(logger, { - event: "workbench_projection_resume", - ok: false, - status: retryExhausted ? "stopped" : "failed", - reason, - offset, - errorCode: error?.code ?? "projection_resume_pass_failed", - message: error?.message ?? "AgentRun projection resume pass failed.", - passRetryAttempt: passFailureAttempt, - passRetryMaxAttempts, - passRetryLabel: passFailureAttempt + "/" + passRetryMaxAttempts, - passRetryDelayMs: retryDelayMs, - passRetryExhausted: retryExhausted, - valuesRedacted: true - }); - if (retryExhausted) stopped = true; - }) - .finally(() => { - active = false; - if (!stopped && intervalMs > 0) schedule(nextDelayMs, nextReason); - }); - }, scheduledDelayMs); - timer.unref?.(); - }; - - schedule(initialDelayMs, "startup"); - return { - started: true, - initialDelayMs, - intervalMs, - minAgeMs, - batchSize, - jitterMs, - failureBackoffMs, - candidateRetryMaxAttempts, - candidateRetryInitialDelayMs, - candidateRetryMaxDelayMs, - passRetryMaxAttempts, - passRetryInitialDelayMs, - passRetryMaxDelayMs, - valuesPrinted: false, - stop() { - stopped = true; - if (timer) clearTimeout(timer); - timer = null; - }, - active: () => active - }; -} - -async function runAgentRunProjectionResumePass({ options = {}, traceStore = defaultCodeAgentTraceStore, logger = console, batchSize, minAgeMs, offset = 0, reason = "manual", candidateRetryState = new Map(), candidateRetryMaxAttempts = 5, candidateRetryInitialDelayMs = 30000, candidateRetryMaxDelayMs = 600000 } = {}) { - const stateCandidates = await listAgentRunProjectionResumeStateCandidates(options, batchSize, offset, { minAgeMs }); - const sessions = stateCandidates.length > 0 ? [] : await listAgentRunProjectionResumeSessions(options, batchSize, offset); - const candidates = stateCandidates.length > 0 ? stateCandidates : sessions.flatMap((session) => agentRunProjectionResumeCandidates(session, { minAgeMs })); - let resumed = 0; - let terminal = 0; - let nonTerminal = 0; - let failed = 0; - let skippedCooldown = 0; - let skippedExhausted = 0; - const nowMs = Date.now(); - for (const candidate of candidates) { - const retryKey = projectionResumeCandidateRetryKey(candidate); - const retryState = candidateRetryState.get(retryKey); - if (retryState?.exhausted === true) { - skippedExhausted += 1; - continue; - } - if (Number.isFinite(retryState?.nextRetryAtMs) && retryState.nextRetryAtMs > nowMs) { - skippedCooldown += 1; - continue; - } - try { - const result = await resumeAgentRunProjectionCandidate({ candidate, options, traceStore }); - resumed += 1; - const currentTraceSnapshot = safeTraceStoreSnapshot(traceStore, candidate.traceId); - if (agentRunProjectionObservedTerminal(result, result?.runnerTrace ?? currentTraceSnapshot) || traceSnapshotIsTerminal(currentTraceSnapshot)) { - terminal += 1; - candidateRetryState.delete(retryKey); - } else { - nonTerminal += 1; - const previousAttempts = Number.isFinite(retryState?.attempts) ? retryState.attempts : 0; - const retryAttempt = previousAttempts + 1; - const retryExhausted = retryAttempt >= candidateRetryMaxAttempts; - const retryDelayMs = retryExhausted ? 0 : projectionResumeCandidateRetryDelayMs(retryAttempt, candidateRetryInitialDelayMs, candidateRetryMaxDelayMs); - const nextRetryAtMs = retryExhausted ? null : Date.now() + retryDelayMs; - const nextRetryAt = Number.isFinite(nextRetryAtMs) ? new Date(nextRetryAtMs).toISOString() : null; - candidateRetryState.set(retryKey, { - attempts: retryAttempt, - nextRetryAtMs, - exhausted: retryExhausted, - lastErrorCode: "projection_resume_non_terminal" - }); - appendProjectionResumeRetry(traceStore, candidate, { retryAttempt, retryMaxAttempts: candidateRetryMaxAttempts, retryLabel: retryAttempt + "/" + candidateRetryMaxAttempts, retryDelayMs, nextRetryAt, retryExhausted, status: result?.status ?? null }); - } - } catch (error) { - failed += 1; - const previousAttempts = Number.isFinite(retryState?.attempts) ? retryState.attempts : 0; - const retryAttempt = previousAttempts + 1; - const retryExhausted = retryAttempt >= candidateRetryMaxAttempts; - const retryDelayMs = retryExhausted ? 0 : projectionResumeCandidateRetryDelayMs(retryAttempt, candidateRetryInitialDelayMs, candidateRetryMaxDelayMs); - const nextRetryAtMs = retryExhausted ? null : Date.now() + retryDelayMs; - const nextRetryAt = Number.isFinite(nextRetryAtMs) ? new Date(nextRetryAtMs).toISOString() : null; - candidateRetryState.set(retryKey, { - attempts: retryAttempt, - nextRetryAtMs, - exhausted: retryExhausted, - lastErrorCode: error?.code ?? "projection_resume_sync_failed", - lastErrorMessage: safeOtelText(error?.message, 300) - }); - appendProjectionResumeError(traceStore, candidate, error, { retryAttempt, retryMaxAttempts: candidateRetryMaxAttempts, retryLabel: retryAttempt + "/" + candidateRetryMaxAttempts, retryDelayMs, nextRetryAt, retryExhausted }); - } - } - logAgentRunProjectionResume(logger, { - event: "workbench_projection_resume", - ok: failed === 0, - status: failed === 0 ? "completed" : "degraded", - reason, - offset, - scannedProjectionStates: stateCandidates.length, - scannedSessions: sessions.length, - candidates: candidates.length, - resumed, - terminal, - nonTerminal, - failed, - skippedCooldown, - skippedExhausted, - candidateRetryTracked: candidateRetryState.size, - candidateRetryLastErrorCode: latestProjectionResumeRetryField(candidateRetryState, "lastErrorCode"), - candidateRetryLastErrorMessage: latestProjectionResumeRetryField(candidateRetryState, "lastErrorMessage"), - candidateRetryMaxAttempts, - valuesRedacted: true - }); - return { offset, scannedSessions: sessions.length, candidates: candidates.length, resumed, terminal, nonTerminal, failed, skippedCooldown, skippedExhausted }; -} - -function latestProjectionResumeRetryField(candidateRetryState, field) { - const values = [...candidateRetryState.values()]; - for (let index = values.length - 1; index >= 0; index -= 1) { - const text = textValue(values[index]?.[field]); - if (text) return text; - } - return null; -} - -async function listAgentRunProjectionResumeStateCandidates(options = {}, batchSize = DEFAULT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE, offset = 0, { minAgeMs = DEFAULT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS } = {}) { - const runtimeStore = options.runtimeStore; - if (typeof runtimeStore?.queryWorkbenchProjectionStates !== "function") return []; - let result; - try { - result = await runtimeStore.queryWorkbenchProjectionStates({ - projectionStatuses: ["projecting", "degraded"], - dueAt: new Date().toISOString(), - limit: batchSize, - offset - }); - } catch { - return []; - } - return (Array.isArray(result?.states) ? result.states : []) - .map((state) => agentRunProjectionResumeCandidateFromState(state, { minAgeMs })) - .filter(Boolean); -} - -async function listAgentRunProjectionResumeSessions(options = {}, batchSize = DEFAULT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE, offset = 0) { - const accessStore = options.accessController?.store ?? options.accessController ?? null; - if (typeof accessStore?.listAgentSessionsForUser !== "function") return []; - return await accessStore.listAgentSessionsForUser({ - actorRole: "admin", - ownerScoped: false, - includeArchived: false, - limit: batchSize, - offset - }) ?? []; -} - -function agentRunProjectionResumeCandidates(session = {}, { minAgeMs = DEFAULT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS, nowMs = Date.now() } = {}) { - const snapshot = session?.session && typeof session.session === "object" ? session.session : {}; - const records = new Map(); - const traceResults = snapshot.traceResults && typeof snapshot.traceResults === "object" ? snapshot.traceResults : null; - if (traceResults) { - for (const [traceId, record] of Object.entries(traceResults)) { - const safeId = safeTraceId(traceId); - if (safeId && record && typeof record === "object") records.set(safeId, { ...record, traceId: safeId }); - } - } - const directTraceId = safeTraceId(snapshot.traceId ?? session.lastTraceId); - if (directTraceId && snapshot.agentRun && typeof snapshot.agentRun === "object") { - records.set(directTraceId, { ...snapshot, traceId: directTraceId }); - } - return [...records.values()] - .map((record) => agentRunProjectionResumeCandidate(session, record, { minAgeMs, nowMs })) - .filter(Boolean); -} - -function agentRunProjectionResumeCandidate(session = {}, record = {}, { minAgeMs = DEFAULT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS, nowMs = Date.now() } = {}) { - const traceId = safeTraceId(record.traceId ?? session.lastTraceId); - const agentRun = record.agentRun && typeof record.agentRun === "object" ? record.agentRun : null; - if (!traceId || !agentRun?.runId || !agentRun?.commandId) return null; - if (isTraceCommandTerminalStatus(record.status ?? agentRun.terminalStatus ?? agentRun.commandState ?? agentRun.status)) return null; - const updatedAtMs = Date.parse(record.updatedAt ?? session.updatedAt ?? ""); - if (minAgeMs > 0 && Number.isFinite(updatedAtMs) && nowMs - updatedAtMs < minAgeMs) return null; - const sessionId = safeSessionId(record.sessionId ?? session.id) || null; - const conversationId = safeConversationId(record.conversationId ?? session.conversationId) || null; - const threadId = safeOpaqueId(record.threadId ?? session.threadId) || null; - const ownerUserId = textValue(record.ownerUserId ?? session.ownerUserId); - const ownerRole = textValue(record.ownerRole ?? session.ownerRole) || "user"; - const retryParams = agentRunProjectionRetryParams(record.retryParams ?? record.projectionRetryParams ?? session.retryParams); - return { - traceId, - params: { - ...retryParams, - traceId, - sessionId, - conversationId, - threadId, - projectId: record.projectId ?? session.projectId ?? null, - ownerUserId, - ownerRole - }, - result: { - ...record, - traceId, - sessionId, - conversationId, - threadId, - ownerUserId, - ownerRole, - status: record.status || session.status || "running", - finalResponse: null, - traceSummary: null, - agentRun: { - ...agentRun, - traceId, - providerTrace: agentRun.providerTrace ?? { traceId, runId: agentRun.runId, commandId: agentRun.commandId, valuesPrinted: false }, - lastSeq: Number.isFinite(Number(agentRun.lastSeq)) ? Number(agentRun.lastSeq) : 0, - valuesPrinted: false - }, - valuesRedacted: true - } - }; -} - -function agentRunProjectionResumeCandidateFromState(state = {}, { minAgeMs = DEFAULT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS, nowMs = Date.now() } = {}) { - const traceId = safeTraceId(state.traceId); - const runId = textValue(state.runId ?? state.sourceRunId); - const commandId = textValue(state.commandId ?? state.sourceCommandId); - if (!traceId || !runId || !commandId) return null; - if (isTraceCommandTerminalStatus(state.projectionStatus)) return null; - const updatedAtMs = Date.parse(state.updatedAt ?? ""); - if (minAgeMs > 0 && Number.isFinite(updatedAtMs) && nowMs - updatedAtMs < minAgeMs) return null; - const sessionId = safeSessionId(state.sessionId) || null; - const conversationId = safeConversationId(state.conversationId) || null; - const threadId = safeOpaqueId(state.threadId) || null; - const ownerUserId = textValue(state.ownerUserId); - const ownerRole = textValue(state.ownerRole) || "user"; - const lastSeq = Number.isFinite(Number(state.lastSourceSeq ?? state.lastAgentRunSeq)) ? Number(state.lastSourceSeq ?? state.lastAgentRunSeq) : 0; - const retryParams = agentRunProjectionRetryParams(state.retryParams ?? state.projectionRetryParams); - return { - traceId, - params: { - ...retryParams, - traceId, - sessionId, - conversationId, - threadId, - ownerUserId, - ownerRole - }, - result: { - accepted: true, - status: "running", - shortConnection: true, - traceId, - sessionId, - conversationId, - threadId, - ownerUserId, - ownerRole, - finalResponse: null, - traceSummary: null, - agentRun: { - traceId, - runId, - commandId, - lastSeq, - managerUrl: state.managerUrl ?? null, - backendProfile: state.backendProfile ?? null, - providerId: state.providerId ?? null, - status: "running", - commandState: "running", - resultSyncState: state.resultSyncState ?? null, - providerTrace: { traceId, runId, commandId, valuesPrinted: false }, - valuesPrinted: false - }, - valuesRedacted: true - } - }; -} - -function agentRunProjectionRetryParams(value = null) { - if (!value || typeof value !== "object" || Array.isArray(value)) return {}; - const message = textValue(value.message ?? value.prompt ?? value.text); - if (!message) return {}; - const result = { - message, - prompt: message, - text: message, - projectId: textValue(value.projectId) || undefined, - ownerUserId: textValue(value.ownerUserId) || undefined, - ownerRole: textValue(value.ownerRole) || undefined, - providerProfile: textValue(value.providerProfile ?? value.codeAgentProviderProfile ?? value.backendProfile) || undefined, - codeAgentProviderProfile: textValue(value.codeAgentProviderProfile ?? value.providerProfile ?? value.backendProfile) || undefined, - backendProfile: textValue(value.backendProfile ?? value.providerProfile ?? value.codeAgentProviderProfile) || undefined, - providerId: textValue(value.providerId) || undefined, - valuesPrinted: false - }; - for (const [key, item] of Object.entries(result)) { - if (item === undefined || item === "") delete result[key]; - } - return result; -} - -async function resumeAgentRunProjectionCandidate({ candidate, options = {}, traceStore = defaultCodeAgentTraceStore } = {}) { - const { currentResult } = await agentRunProjectionResumeResultWithCursor(candidate, options); - const resumeOptions = { ...options, deferAgentRunResultSync: false }; - const synced = await syncAgentRunChatResult({ - traceId: candidate.traceId, - currentResult, - options: resumeOptions, - traceStore, - forceResultSync: true, - refreshEvents: true, - retryParams: candidate.params - }); - const payload = codeAgentPayloadWithObservedTerminalStatus(synced?.result ?? currentResult, synced?.runnerTrace ?? traceStore?.snapshot?.(candidate.traceId)) ?? (synced?.result ?? currentResult); - if (isTraceCommandTerminalStatus(payload?.status)) { - await recordCodeAgentTerminalTurnStatusEffects({ payload, params: candidate.params, options, preserveLastTraceId: true }); - } - return payload; -} - -async function agentRunProjectionResumeResultWithCursor(candidate, options = {}) { - const projectionState = await loadAgentRunProjectionState(options.runtimeStore, candidate); - const plan = buildAgentRunProjectionEventsFetchPlan({ - projectionState, - agentRun: candidate.result.agentRun, - pageLimit: options.env?.HWLAB_CODE_AGENT_AGENTRUN_EVENTS_PAGE_LIMIT - }); - return { - projectionState, - currentResult: { - ...candidate.result, - traceSummary: null, - agentRun: { - ...candidate.result.agentRun, - lastSeq: plan.afterSeq, - valuesPrinted: false - } - } - }; -} - -async function loadAgentRunProjectionState(runtimeStore, candidate) { - if (!runtimeStore || typeof runtimeStore.getWorkbenchProjectionState !== "function") return null; - try { - const result = await runtimeStore.getWorkbenchProjectionState({ traceId: candidate.traceId }); - const state = result?.projectionState ?? null; - if (!state) return null; - if (String(state.runId ?? state.sourceRunId ?? "") !== String(candidate.result.agentRun.runId ?? "")) return null; - if (String(state.commandId ?? state.sourceCommandId ?? "") !== String(candidate.result.agentRun.commandId ?? "")) return null; - return state; - } catch { - return null; - } -} - -function projectionResumeCandidateRetryKey(candidate) { - const traceId = safeTraceId(candidate?.traceId) || "trace-unknown"; - const runId = String(candidate?.result?.agentRun?.runId ?? "run-unknown"); - const commandId = String(candidate?.result?.agentRun?.commandId ?? "command-unknown"); - return [traceId, runId, commandId].join("|"); -} - -function projectionResumeCandidateRetryDelayMs(retryAttempt, initialDelayMs, maxDelayMs) { - const attempt = Math.max(1, Number(retryAttempt) || 1); - return Math.min(maxDelayMs, initialDelayMs * (2 ** Math.max(0, attempt - 1))); -} - -function appendProjectionResumeError(traceStore, candidate, error, retry = {}) { - const traceId = safeTraceId(candidate?.traceId); - if (!traceId) return; - if (traceSnapshotIsTerminal(safeTraceStoreSnapshot(traceStore, traceId))) return; - traceStore.append(traceId, { - type: "turn-status", - status: "degraded", - label: "projection-resume:sync-failed", - errorCode: error?.code ?? "projection_resume_sync_failed", - message: error?.message ?? "AgentRun projection resume failed.", - runId: candidate?.result?.agentRun?.runId ?? null, - commandId: candidate?.result?.agentRun?.commandId ?? null, - waitingFor: "agentrun-result-resume", - retryAttempt: retry.retryAttempt ?? null, - retryMaxAttempts: retry.retryMaxAttempts ?? null, - retryLabel: retry.retryLabel ?? null, - retryDelayMs: retry.retryDelayMs ?? null, - nextRetryAt: retry.nextRetryAt ?? null, - retryExhausted: retry.retryExhausted === true, - terminal: false, - valuesPrinted: false - }); -} - -function appendProjectionResumeRetry(traceStore, candidate, retry = {}) { - const traceId = safeTraceId(candidate?.traceId); - if (!traceId) return; - if (traceSnapshotIsTerminal(safeTraceStoreSnapshot(traceStore, traceId))) return; - traceStore.append(traceId, { - type: "turn-status", - status: "degraded", - label: "projection-resume:non-terminal-backoff", - errorCode: "projection_resume_non_terminal", - message: "AgentRun projection resume did not reach a terminal status; next resume is delayed.", - runId: candidate?.result?.agentRun?.runId ?? null, - commandId: candidate?.result?.agentRun?.commandId ?? null, - waitingFor: "agentrun-result-resume", - observedStatus: retry.status ?? null, - retryAttempt: retry.retryAttempt ?? null, - retryMaxAttempts: retry.retryMaxAttempts ?? null, - retryLabel: retry.retryLabel ?? null, - retryDelayMs: retry.retryDelayMs ?? null, - nextRetryAt: retry.nextRetryAt ?? null, - retryExhausted: retry.retryExhausted === true, - terminal: false, - valuesPrinted: false - }); -} - -function safeTraceStoreSnapshot(traceStore, traceId) { - try { - return typeof traceStore?.snapshot === "function" ? traceStore.snapshot(traceId) : null; - } catch { - return null; - } -} - -function traceSnapshotIsTerminal(snapshot) { - if (!snapshot || typeof snapshot !== "object") return false; - if (snapshot.finishedAt) return true; - if (isTraceCommandTerminalStatus(snapshot.status)) return true; - if (isTraceCommandTerminalStatus(snapshot.lastEvent?.status) || snapshot.lastEvent?.terminal === true) return true; - const events = Array.isArray(snapshot.events) ? snapshot.events : []; - return events.some((event) => event?.terminal === true || isTraceCommandTerminalStatus(event?.status)); -} - -function logAgentRunProjectionResume(logger, payload) { - try { - const line = JSON.stringify(payload); - if (payload.ok === false && typeof logger?.warn === "function") logger.warn(line); - else if (typeof logger?.log === "function") logger.log(line); - } catch { - // Projection resume logging must not affect the projector. - } -} - -function parseNonNegativeInteger(value, fallback) { - const parsed = Number.parseInt(value ?? "", 10); - return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback; -} - -function randomJitterMs(maxJitterMs) { - return maxJitterMs > 0 ? Math.floor(Math.random() * (maxJitterMs + 1)) : 0; -} - -function agentRunProjectionActivitySignature(result, runnerTrace) { - const agentRun = result?.agentRun && typeof result.agentRun === "object" ? result.agentRun : {}; - const trace = runnerTrace && typeof runnerTrace === "object" ? runnerTrace : result?.runnerTrace; - const events = Array.isArray(trace?.events) ? trace.events : []; - const lastEvent = trace?.lastEvent ?? events.at(-1) ?? null; - return [ - result?.status, - result?.updatedAt, - agentRun.status, - agentRun.runStatus, - agentRun.commandState, - agentRun.terminalStatus, - agentRun.lastSeq, - agentRun.updatedAt, - trace?.status, - trace?.eventCount ?? events.length, - trace?.updatedAt, - trace?.lastEventLabel, - lastEvent?.seq, - lastEvent?.sourceSeq, - lastEvent?.label, - lastEvent?.type, - lastEvent?.status, - lastEvent?.createdAt - ].map((value) => value ?? "").join("|"); -} - -function agentRunProjectionObservedTerminal(result, runnerTrace) { - return Boolean(codeAgentObservedTerminalStatus(result, runnerTrace)); -} - -function codeAgentPayloadWithObservedTerminalStatus(payload, runnerTrace) { - const terminalStatus = codeAgentObservedTerminalStatus(payload, runnerTrace); - if (!terminalStatus || !payload || typeof payload !== "object") return null; - const agentRun = payload.agentRun && typeof payload.agentRun === "object" ? payload.agentRun : null; - const agentRunTerminalStatus = terminalStatus === "canceled" ? "cancelled" : terminalStatus; - return { - ...payload, - status: terminalStatus, - runnerTrace: runnerTrace && typeof runnerTrace === "object" ? runnerTrace : payload.runnerTrace, - ...(agentRun ? { agentRun: { ...agentRun, terminalStatus: agentRun.terminalStatus ?? agentRunTerminalStatus, valuesPrinted: false } } : {}) - }; -} - -function codeAgentObservedTerminalStatus(result, runnerTrace) { - const agentRun = result?.agentRun && typeof result.agentRun === "object" ? result.agentRun : {}; - if (codeAgentPayloadHasSealedFinalResponse(result)) return "completed"; - for (const value of [agentRun.terminalStatus, agentRun.commandState]) { - const terminalStatus = normalizeCodeAgentTerminalStatus(value); - if (terminalStatus) return terminalStatus; - } - const commandId = textValue(agentRun.commandId); - const trace = runnerTrace && typeof runnerTrace === "object" ? runnerTrace : result?.runnerTrace; - const events = Array.isArray(trace?.events) ? trace.events : []; - for (const event of [...events].reverse()) { - const payload = event?.payload && typeof event.payload === "object" ? event.payload : {}; - const eventCommandId = textValue(event?.commandId ?? payload.commandId); - if (commandId && eventCommandId && eventCommandId !== commandId) continue; - for (const value of [ - event?.type === "terminal_status" ? payload.terminalStatus : null, - payload.phase === "command-terminal" ? payload.terminalStatus : null, - payload.phase === "turn:completed" || payload.phase === "turn/steer:completed" ? "completed" : null, - event?.terminal === true ? payload.terminalStatus ?? payload.status ?? event.status ?? "completed" : null, - payload.terminal === true ? payload.terminalStatus ?? payload.status ?? "completed" : null - ]) { - const terminalStatus = normalizeCodeAgentTerminalStatus(value); - if (terminalStatus) return terminalStatus; - } - } - return null; -} - -function normalizeCodeAgentTerminalStatus(value) { - const status = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); - if (!CODE_AGENT_TERMINAL_STATUSES.has(status)) return null; - return status === "cancelled" ? "canceled" : status; -} - -function sleepAgentRunProjectionSync(ms) { - return new Promise((resolve) => setTimeout(resolve, Math.max(0, Number(ms) || 0))); -} - -function codeAgentChatShortConnectionRequested(request, params, options = {}) { - const header = firstHeaderValue(request, "prefer"); - const explicit = firstHeaderValue(request, "x-hwlab-short-connection"); - const envValue = options.env?.HWLAB_CODE_AGENT_CHAT_SHORT_CONNECTION; - return codeAgentAgentRunAdapterEnabled(options.env ?? process.env) || - /(?:^|,)\s*respond-async\s*(?:,|$)/iu.test(String(header ?? "")) || - truthyFlag(explicit) || - params?.shortConnection === true || - truthyFlag(envValue); -} - -async function preflightCodeAgentBilling({ params = {}, options = {}, traceId, response, sendResponse = true } = {}) { - const client = options.userBillingClient; - if (!codeAgentBillingEnabled(options) || !client?.configured || !options.userBillingAuth?.active) return null; - const estimatedCredits = parsePositiveInteger(options.env?.HWLAB_USER_BILLING_CODE_AGENT_ESTIMATED_CREDITS, 1); - const body = { - ...(options.actorApiKeySecret ? { apiKey: options.actorApiKeySecret } : { userId: options.actor?.id }), - serviceId: "hwlab-code-agent", - estimatedCredits, - idempotencyKey: `code-agent:${traceId}:preflight`, - metadata: codeAgentBillingMetadata(params, traceId, { stage: "preflight" }) - }; - const result = await client.billingPreflight(body); - if (result.ok && result.body?.allowed !== false) return { reservation: sanitizeBillingReservation(result.body) }; - const status = [402, 403, 429].includes(Number(result.status)) ? Number(result.status) : 503; - const payload = { - ok: false, - accepted: false, - status: status === 402 ? "payment_required" : status === 403 ? "not_entitled" : status === 429 ? "rate_limited" : "billing_unavailable", - traceId, - error: { - code: result.error?.code ?? (status === 402 ? "insufficient_credits" : "user_billing_preflight_failed"), - layer: "billing", - retryable: status === 429 || status === 503, - message: result.error?.message ?? "Code Agent billing preflight failed", - route: "/v1/agent/chat" - }, - blocker: { - code: result.error?.code ?? (status === 402 ? "insufficient_credits" : "user_billing_preflight_failed"), - layer: "billing", - retryable: status === 429 || status === 503, - summary: result.error?.message ?? "Code Agent billing preflight failed" - }, - valuesRedacted: true - }; - if (sendResponse && response) sendJson(response, status, payload); - return { blocked: true, statusCode: status, payload, error: payload.error, blocker: payload.blocker }; -} - -async function recordCodeAgentBillingUsage({ payload = {}, params = {}, options = {} } = {}) { - const reservation = params.userBillingReservation ?? payload.userBillingReservation; - const reservationId = typeof reservation?.reservationId === "string" ? reservation.reservationId : ""; - const client = options.userBillingClient; - if (payload.billing?.recorded === true || payload.billing?.released === true || payload.status === "running") return payload.billing ?? null; - if (!reservationId || !client?.configured) return null; - const traceId = safeTraceId(payload.traceId ?? params.traceId); - const usedTokens = codeAgentUsedTokens(payload); - const usedCredits = parsePositiveInteger(options.env?.HWLAB_USER_BILLING_CODE_AGENT_USED_CREDITS, 1); - const result = await client.billingRecord({ - reservationId, - serviceId: "hwlab-code-agent", - usedCredits, - usedTokens, - idempotencyKey: `code-agent:${traceId}:record`, - metadata: codeAgentBillingMetadata(params, traceId, { stage: "record", status: payload.status ?? "unknown" }) - }); - const billing = result.ok ? sanitizeBillingRecord(result.body, reservation) : { - recorded: false, - reservationId, - errorCode: result.error?.code ?? "user_billing_record_failed", - valuesRedacted: true - }; - payload.billing = billing; - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - if (traceId) { - traceStore.append(traceId, { - type: "billing", - status: result.ok ? "recorded" : "degraded", - label: result.ok ? "billing:recorded" : "billing:record_failed", - reservationId, - recordId: result.ok ? result.body?.recordId ?? null : null, - errorCode: result.ok ? null : billing.errorCode, - valuesPrinted: false - }); - } - return billing; -} - -async function finalizeCodeAgentBillingUsage({ payload = {}, params = {}, options = {} } = {}) { - if (payload.billing?.recorded === true || payload.billing?.released === true) return payload.billing; - if (payload.status === "running") return null; - if (payload.status === "completed") { - return recordCodeAgentBillingUsage({ payload, params, options }); - } - return releaseCodeAgentBillingReservation({ payload, params, options }); -} - -async function releaseCodeAgentBillingReservation({ payload = {}, params = {}, options = {} } = {}) { - const reservation = params.userBillingReservation ?? payload.userBillingReservation; - const reservationId = typeof reservation?.reservationId === "string" ? reservation.reservationId : ""; - const client = options.userBillingClient; - if (payload.billing?.recorded === true || payload.billing?.released === true || !reservationId || !client?.configured) return payload.billing ?? null; - const traceId = safeTraceId(payload.traceId ?? params.traceId); - const releaseReason = String(payload.status ?? "not_completed").trim() || "not_completed"; - const releaseBody = { - reservationId, - serviceId: "hwlab-code-agent", - idempotencyKey: `code-agent:${traceId}:release`, - reason: releaseReason, - metadata: codeAgentBillingMetadata(params, traceId, { - stage: "release", - status: payload.status ?? "unknown", - errorCode: payload.error?.code ?? payload.blocker?.code ?? null - }) - }; - const result = typeof client.billingRelease === "function" - ? await client.billingRelease(releaseBody) - : { ok: false, error: { code: "user_billing_release_not_supported" } }; - const billing = result.ok ? sanitizeBillingRelease(result.body, reservation) : { - released: false, - reservationId, - errorCode: result.error?.code ?? "user_billing_release_failed", - valuesRedacted: true - }; - payload.billing = billing; - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - if (traceId) { - traceStore.append(traceId, { - type: "billing", - status: result.ok ? "released" : "degraded", - label: result.ok ? "billing:released" : "billing:release_failed", - reservationId, - reason: releaseReason, - errorCode: result.ok ? null : billing.errorCode, - valuesPrinted: false - }); - } - return billing; -} - -function withCodeAgentBillingReservation(payload = {}, params = {}) { - if (!payload || typeof payload !== "object" || payload.userBillingReservation) return payload; - const reservation = params.userBillingReservation; - return reservation ? { ...payload, userBillingReservation: reservation } : payload; -} - -function codeAgentBillingEnabled(options = {}) { - const env = options.env ?? process.env; - if (String(env.HWLAB_USER_BILLING_CODE_AGENT_ENABLED ?? "").trim() === "0") return false; - return Boolean(options.userBillingClient?.configured); -} - -function codeAgentBillingMetadata(params = {}, traceId, extra = {}) { - return { - traceId, - conversationId: safeConversationId(params.conversationId) || null, - sessionId: safeSessionId(params.sessionId) || null, - projectId: textValue(params.projectId) || null, - route: "/v1/agent/chat", - resource: "code-agent-turn", - ...extra, - valuesRedacted: true - }; -} - -function sanitizeBillingReservation(value = {}) { - return { - allowed: value.allowed === true, - reservationId: textValue(value.reservationId) || null, - resourceType: textValue(value.resourceType) || null, - planId: textValue(value.planId) || null, - estimatedCredits: Number.isFinite(Number(value.estimatedCredits)) ? Number(value.estimatedCredits) : null, - estimatedTokens: Number.isFinite(Number(value.estimatedTokens)) ? Number(value.estimatedTokens) : null, - expiresAt: textValue(value.expiresAt) || null, - valuesRedacted: true - }; -} - -function sanitizeBillingRecord(value = {}, reservation = {}) { - return { - recorded: true, - reservationId: textValue(reservation.reservationId) || null, - recordId: textValue(value.recordId) || null, - credits: Number.isFinite(Number(value.credits)) ? Number(value.credits) : null, - balance: Number.isFinite(Number(value.balance)) ? Number(value.balance) : null, - valuesRedacted: true - }; -} - -function sanitizeBillingRelease(value = {}, reservation = {}) { - return { - released: true, - reservationId: textValue(value.reservationId) || textValue(reservation.reservationId) || null, - releasedCredits: Number.isFinite(Number(value.releasedCredits)) ? Number(value.releasedCredits) : null, - status: textValue(value.status) || "cancelled", - valuesRedacted: true - }; -} - -function codeAgentUsedTokens(payload = {}) { - const usage = payload.usage && typeof payload.usage === "object" ? payload.usage : null; - const traceSummary = payload.traceSummary && typeof payload.traceSummary === "object" ? payload.traceSummary : null; - const candidates = [ - usage?.totalTokens, - usage?.total_tokens, - usage?.tokens, - Number(usage?.inputTokens ?? usage?.input_tokens ?? 0) + Number(usage?.outputTokens ?? usage?.output_tokens ?? 0), - traceSummary?.tokenCount, - traceSummary?.tokens - ]; - for (const value of candidates) { - const parsed = Number(value); - if (Number.isFinite(parsed) && parsed > 0) return Math.ceil(parsed); - } - return 0; -} - -export async function handleCodeAgentChatResultHttp(request, response, url, options) { - const parts = url.pathname.split("/").filter(Boolean); - const traceId = decodeURIComponent(parts[4] ?? ""); - if (!safeTraceId(traceId)) { - sendJson(response, 400, { - error: { - code: "invalid_trace_id", - message: "traceId must start with trc_ and contain only safe identifier characters" - } - }); - return; - } - const context = await measureCodeAgentHttpPhase(options, "result_projection_read", () => readCodeAgentCompatProjection(traceId, options)); - if (context.statusCode) { - sendJson(response, context.statusCode, context.body); - return; - } - const result = context.result; - if (result && result.status !== "running") { - sendJson(response, 200, codeAgentCompatProjectionPayload(compactCodeAgentChatResultPayload(result, options), context)); - return; - } - if (context.found) { - const body = codeAgentTurnStatusPayload({ traceId, result, snapshot: context.trace, resultPollError: null, refreshError: context.refreshError ?? null, options }); - const resultPollTerminalReady = codeAgentResultPollTerminalReady(body); - sendJson(response, resultPollTerminalReady ? 200 : 202, codeAgentCompatProjectionPayload({ - accepted: body.ok, - shortConnection: true, - ...body, - waitingFor: resultPollTerminalReady ? body.waitingFor ?? context.trace?.waitingFor ?? "workbench-projection" : "agentrun-result", - ...(body.terminal === true && !resultPollTerminalReady ? { status: "running", running: true, terminal: false } : {}) - }, context)); - return; - } - sendJson(response, 404, { - ...codeAgentCompatProjectionPayload({ traceId, status: "unknown" }, context), - error: { - code: "code_agent_result_not_found", - message: `No Code Agent result is registered for ${traceId}` - } - }); -} - -export async function handleCodeAgentTurnHttp(request, response, url, options) { - const startedAt = nowMs(); - const parts = url.pathname.split("/").filter(Boolean); - const traceId = decodeURIComponent(parts[3] ?? ""); - if (!safeTraceId(traceId)) { - const body = { - ok: false, - status: "unknown", - running: false, - terminal: false, - error: { - code: "invalid_trace_id", - message: "traceId must start with trc_ and contain only safe identifier characters" - } - }; - recordCodeAgentTurnReadMetric(options, url, 400, body, startedAt); - sendJson(response, 400, body); - return; - } - const requestOptions = codeAgentRequestScopedOptions(options); - let completed = false; - const emitTurnStatusReadSpan = (name, attributes = {}, error = null) => { - void emitCodeAgentOtelSpan(name, traceId, requestOptions.env ?? process.env, { - startTimeMs: startedAt, - status: error ? "error" : undefined, - error, - attributes: { - "http.method": "GET", - "http.route": "/v1/agent/turns/:traceId", - durationMs: nowMs() - startedAt, - ...attributes - } - }); - }; - response.once?.("close", () => { - if (completed) return; - emitTurnStatusReadSpan("turn_status_read.client_closed", { - status: "client_closed", - phase: "response_close", - terminal: false - }, new Error("Code Agent turn status client connection closed before response completed.")); - }); - try { - const projectMismatch = await measureCodeAgentHttpPhase(requestOptions, "turn_project_check", () => traceProjectMismatchSnapshot(traceId, url, requestOptions, "turn")); - if (projectMismatch) { - completed = true; - recordCodeAgentTurnReadMetric(options, url, projectMismatch.statusCode, projectMismatch.body, startedAt); - emitTurnStatusReadSpan("turn_status_read", { "http.status_code": projectMismatch.statusCode, status: projectMismatch.body?.status ?? null, terminal: projectMismatch.body?.terminal ?? null, phase: "project_check" }); - sendJson(response, projectMismatch.statusCode, projectMismatch.body); - return; - } - const resolved = await measureCodeAgentHttpPhase(requestOptions, "turn_resolve_status", () => resolveCodeAgentTurnStatusSnapshot(traceId, requestOptions)); - completed = true; - recordCodeAgentTurnReadMetric(options, url, resolved.statusCode, resolved.body, startedAt); - emitTurnStatusReadSpan("turn_status_read", { "http.status_code": resolved.statusCode, status: resolved.body?.status ?? null, terminal: resolved.body?.terminal ?? null, phase: "resolved" }); - sendJson(response, resolved.statusCode, resolved.body); - } catch (error) { - completed = true; - const body = { - ok: false, - status: "unknown", - running: false, - terminal: false, - traceId, - error: { - code: error?.code ?? "turn_status_read_failed", - message: error?.message ?? "Code Agent turn status read failed." - }, - valuesRedacted: true - }; - recordCodeAgentTurnReadMetric(options, url, 500, body, startedAt); - emitTurnStatusReadSpan("turn_status_read", { "http.status_code": 500, status: body.status, terminal: false, phase: "failed", errorCode: body.error.code }, error); - sendJson(response, 500, body); - } -} - -async function resolveCodeAgentTurnStatusSnapshot(traceId, options) { - const context = await readCodeAgentCompatProjection(traceId, options); - if (context.statusCode) return context; - const body = await measureCodeAgentHttpPhase(options, "turn_payload", () => codeAgentTurnStatusPayload({ - traceId, - result: context.result, - snapshot: context.trace, - resultPollError: null, - refreshError: context.refreshError ?? null, - options - })); - return { statusCode: body.ok ? 200 : 404, body: codeAgentCompatProjectionPayload(body, context) }; -} - -function codeAgentTurnStatusRefreshOptions(options = {}) { - const env = options.env ?? process.env; - const configuredBudgetMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS, DEFAULT_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS); - const upstreamTimeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, configuredBudgetMs); - const timeoutMs = Math.max(1, Math.min(configuredBudgetMs, upstreamTimeoutMs)); - return { - ...options, - skipAgentRunTerminalRefreshIfComplete: true, - env: { - ...env, - HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS: String(timeoutMs), - HWLAB_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS: String(timeoutMs) - } - }; -} - -function codeAgentRequestScopedOptions(options = {}) { - return { - ...options, - codeAgentTraceSessionCache: options.codeAgentTraceSessionCache instanceof Map ? options.codeAgentTraceSessionCache : new Map() - }; -} - -async function measureCodeAgentHttpPhase(options, phase, callback) { - const perf = options?.backendPerformance; - return perf && typeof perf.measure === "function" ? await perf.measure(phase, callback) : await callback(); -} - -function forbiddenTurnSnapshot(traceId) { - return { - statusCode: 403, - body: { - ok: false, - status: "unknown", - running: false, - terminal: false, - traceId, - error: { - code: "agent_session_owner_required", - message: "Only the session owner or admin can read this Code Agent turn status" - } - } - }; -} - -async function readCodeAgentCompatProjection(traceId, options = {}) { - const readModel = createWorkbenchReadModel(options, options.actor ?? null); - const session = await readModel.getSessionByTraceId(traceId); - let result = readModel.resultForTrace(traceId) ?? projectedSessionTraceResult(session, traceId); - if (result && !canAccessOwnedResult(result, options.actor)) return forbiddenTurnSnapshot(traceId); - if (result?.ownerUserId && !readModel.canReadOwner(result.ownerUserId)) return forbiddenTurnSnapshot(traceId); - const projectionTrace = await readModel.traceSnapshot(traceId); - let trace = traceSnapshotWithTerminalEvidence(projectionTrace, result, traceId, null); - const projection = readModel.projectionDiagnostics({ traceId, result, trace, refreshError: null }); - recordCodeAgentProjectionMetrics(options, { result, trace, projection, refreshError: null }); - const found = Boolean(result || session || (trace && trace.status !== "missing") || trace?.persisted === true); - return { result, session, trace, projection, found, refreshError: null }; -} - -function projectedSessionTraceResult(session, traceId) { - const safeId = safeTraceId(traceId); - if (!session || !safeId) return null; - const snapshot = session.session && typeof session.session === "object" ? session.session : null; - const traceResults = snapshot?.traceResults && typeof snapshot.traceResults === "object" ? snapshot.traceResults : null; - const stored = traceResults?.[safeId] && typeof traceResults[safeId] === "object" ? traceResults[safeId] : null; - if (!stored) return null; - return { - ...stored, - traceId: safeId, - conversationId: safeConversationId(stored.conversationId ?? session.conversationId) || null, - sessionId: safeSessionId(stored.sessionId ?? session.id) || null, - threadId: safeOpaqueId(stored.threadId ?? session.threadId) || null, - ownerUserId: session.ownerUserId ?? stored.ownerUserId ?? null, - ownerRole: session.ownerRole ?? stored.ownerRole ?? null, - status: stored.status ?? session.status ?? null, - valuesRedacted: true - }; -} - -function codeAgentCompatProjectionPayload(payload = {}, context = {}) { - const traceId = safeTraceId(payload.traceId ?? context.trace?.traceId) ?? null; - const projection = context.projection && typeof context.projection === "object" ? context.projection : {}; - return { - ...payload, - projection, - projectionStatus: projection.projectionStatus ?? null, - projectionHealth: projection.projectionHealth ?? null, - lastProjectedSeq: projection.lastProjectedSeq ?? null, - sourceRunId: projection.sourceRunId ?? null, - sourceCommandId: projection.sourceCommandId ?? null, - staleMs: projection.staleMs ?? null, - blocker: projection.blocker ?? null, - workbench: traceId ? { - detailOnly: true, - turnDetailUrl: `/v1/workbench/turns/${encodeURIComponent(traceId)}`, - traceEventsDetailUrl: `/v1/workbench/traces/${encodeURIComponent(traceId)}/events` - } : null, - valuesRedacted: true, - secretMaterialStored: false - }; -} - -function codeAgentResultPollTerminalReady(body = {}) { - if (body?.terminal !== true) return false; - return Boolean(messageAuthorityTextValue(body.finalResponse ?? body.reply ?? body.terminalEvidence?.finalResponse)); -} - -async function traceProjectMismatchSnapshot(traceId, url, options, resourceKind) { - const requestedProjectId = textValue(url.searchParams.get("projectId")); - if (!requestedProjectId || !options.accessController?.getAgentSessionByTraceId) return null; - const session = await getCodeAgentSessionByTraceId(traceId, options); - if (!session) return null; - if (session.ownerUserId && options.actor?.role !== "admin" && session.ownerUserId !== options.actor?.id) return forbiddenTurnSnapshot(traceId); - const actualProjectId = textValue(session.projectId); - if (!actualProjectId || actualProjectId === requestedProjectId) return null; - return { - statusCode: 404, - body: { - ok: false, - status: "not_found", - running: false, - terminal: false, - traceId, - error: { - code: "trace_project_mismatch", - message: `${resourceKind === "turn" ? "Agent turn" : "Agent trace"} is not visible in the requested project` - } - } - }; -} - -async function getCodeAgentSessionByTraceId(traceId, options = {}) { - const safeId = safeTraceId(traceId); - if (!safeId || typeof options.accessController?.getAgentSessionByTraceId !== "function") return null; - const cache = options.codeAgentTraceSessionCache; - if (cache instanceof Map) { - if (cache.has(safeId)) return cache.get(safeId); - const session = await options.accessController.getAgentSessionByTraceId(safeId); - cache.set(safeId, session ?? null); - return session ?? null; - } - return await options.accessController.getAgentSessionByTraceId(safeId); -} - -export function codeAgentTurnStatusPayload({ traceId, result, snapshot, resultPollError, refreshError, options }) { - const resultObject = result && typeof result === "object" ? result : null; - const snapshotObject = snapshot && typeof snapshot === "object" ? snapshot : null; - const lifecycle = codeAgentTurnLifecycleFields(traceId, resultObject ?? snapshotObject ?? {}); - const events = Array.isArray(snapshotObject?.events) ? snapshotObject.events : Array.isArray(resultObject?.runnerTrace?.events) ? resultObject.runnerTrace.events : []; - const lastEvent = events.at(-1) ?? null; - const finalResponse = resultObject?.finalResponse ?? snapshotObject?.finalResponse ?? snapshotObject?.terminalEvidence?.finalResponse ?? codeAgentFinalResponseEvidence(resultObject ?? snapshotObject ?? {}, traceId); - const terminalStatus = codeAgentAuthoritativeTerminalStatus(resultObject, snapshotObject, traceId); - const terminalSealBlocked = Boolean(terminalStatus && isTurnTerminalStatus(terminalStatus) && !codeAgentTerminalFinalText(finalResponse, resultObject, snapshotObject)); - const rawStatus = normalizeTurnStatus( - terminalStatus, - resultObject?.agentRun?.commandState, - resultObject?.agentRun?.status, - resultObject?.agentRun?.runStatus, - codeAgentRunningStatus(snapshotObject?.status), - codeAgentRunningStatus(snapshotObject?.traceStatus), - codeAgentRunningStatus(snapshotObject?.runnerTrace?.status), - codeAgentRunningStatus(resultObject?.status), - resultObject?.agentRun?.terminalStatus, - snapshotObject?.terminalEvidence?.traceSummary?.terminalStatus - ); - const status = terminalSealBlocked ? "running" : rawStatus; - const found = Boolean(resultObject || (snapshotObject && snapshotObject.status !== "missing") || snapshotObject?.persisted === true); - const running = isTurnRunningStatus(status); - const terminal = isTurnTerminalStatus(status); - const runnerTrace = snapshotObject && snapshotObject.status !== "missing" ? snapshotObject : resultObject?.runnerTrace ?? null; - const timingFields = codeAgentResultTimingFields(resultObject, snapshotObject, runnerTrace); - return { - ok: found, - action: "code-agent.turn.status", - contractVersion: "code-agent-turn-status-v1", - status: found ? status ?? "unknown" : "unknown", - running: found ? running : false, - terminal: found ? terminal : false, - traceId, - ...lifecycle, - conversationId: safeConversationId(resultObject?.conversationId ?? snapshotObject?.conversationId) || null, - sessionId: safeSessionId(resultObject?.sessionId ?? resultObject?.session?.sessionId ?? snapshotObject?.sessionId) || null, - threadId: safeOpaqueId(resultObject?.threadId ?? resultObject?.session?.threadId ?? snapshotObject?.threadId) || null, - updatedAt: textValue(resultObject?.updatedAt ?? resultObject?.agentRun?.updatedAt ?? snapshotObject?.updatedAt ?? timingFields.lastEventAt ?? timingFields.startedAt) || null, - ...timingFields, - lastEventLabel: textValue(snapshotObject?.lastEventLabel ?? runnerTrace?.lastEventLabel ?? lastEvent?.label ?? lastEvent?.type) || null, - waitingFor: terminalSealBlocked ? "final_response" : textValue(snapshotObject?.waitingFor ?? runnerTrace?.waitingFor) || null, - terminalObserved: Boolean(terminalStatus), - terminalObservedStatus: terminalStatus ?? null, - terminalSealBlocked, - resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`, - traceUrl: `/v1/agent/traces/${encodeURIComponent(traceId)}`, - turnUrl: `/v1/agent/turns/${encodeURIComponent(traceId)}`, - runnerTrace: runnerTrace ? compactRunnerTraceForResult(runnerTrace, resultTraceEventLimit(options)) : null, - agentRun: resultObject?.agentRun ?? snapshotObject?.agentRun ?? null, - terminalEvidence: snapshotObject?.terminalEvidence ?? null, - finalResponse, - traceSummary: resultObject?.traceSummary ?? snapshotObject?.traceSummary ?? snapshotObject?.terminalEvidence?.traceSummary ?? null, - retention: snapshotObject?.retention ?? null, - eventCount: numberOrNull(snapshotObject?.eventCount ?? runnerTrace?.eventCount ?? events.length), - error: resultPollError || refreshError - ? codeAgentRefreshErrorPayload(resultPollError ?? refreshError, traceId, resultObject?.agentRun ?? snapshotObject?.agentRun, "turn_status_degraded") - : resultObject?.error ?? snapshotObject?.error ?? null, - valuesRedacted: true, - secretMaterialStored: false - }; -} - -function codeAgentTerminalFinalText(finalResponse = null, resultObject = null, snapshotObject = null) { - const candidates = [ - finalResponse, - resultObject?.finalResponse, - snapshotObject?.finalResponse, - snapshotObject?.terminalEvidence?.finalResponse, - resultObject?.terminalEvidence?.finalResponse - ]; - for (const value of candidates) { - const text = conversationText(value); - if (text) return text; - } - return codeAgentFinalResponseText(resultObject ?? {}) || codeAgentFinalResponseText(snapshotObject ?? {}); -} - -function codeAgentAuthoritativeTerminalStatus(resultObject, snapshotObject, traceId) { - const sealedPayload = codeAgentPayloadHasSealedFinalResponse(resultObject ?? {}) ? resultObject : codeAgentPayloadHasSealedFinalResponse(snapshotObject ?? {}) ? snapshotObject : null; - if (sealedPayload) { - const sealedStatus = normalizeTurnStatus(sealedPayload?.finalResponse?.status, sealedPayload?.terminalEvidence?.finalResponse?.status, sealedPayload?.agentRun?.terminalStatus, sealedPayload?.agentRun?.status, sealedPayload?.status); - return sealedStatus && isTurnTerminalStatus(sealedStatus) ? sealedStatus : "completed"; - } - const snapshotStatus = normalizeTurnStatus(snapshotObject?.terminalEvidence?.traceSummary?.terminalStatus, snapshotObject?.terminalEvidence?.agentRun?.terminalStatus, snapshotObject?.terminalEvidence?.status, snapshotObject?.status); - if (snapshotStatus && isTurnTerminalStatus(snapshotStatus) && codeAgentSnapshotHasTerminalAuthority(snapshotObject)) return snapshotStatus; - const agentRunStatus = normalizeTurnStatus(resultObject?.agentRun?.terminalStatus, resultObject?.agentRun?.commandState, resultObject?.agentRun?.status, resultObject?.agentRun?.runStatus); - if (agentRunStatus && isTurnTerminalStatus(agentRunStatus)) return agentRunStatus; - const resultStatus = normalizeTurnStatus(resultObject?.status); - if (resultStatus && isTurnTerminalStatus(resultStatus) && codeAgentResultHasTerminalAuthority(resultObject, traceId)) return resultStatus; - return null; -} - -function codeAgentSnapshotHasTerminalAuthority(snapshot = null) { - if (!snapshot || typeof snapshot !== "object") return false; - if (snapshot.terminal === true || snapshot.sealed === true) return true; - if (snapshot.terminalEvidence?.available === true || snapshot.terminalEvidence?.source) return true; - const events = Array.isArray(snapshot.events) ? snapshot.events : []; - return events.some((event) => event?.terminal === true); -} - -function codeAgentResultHasTerminalAuthority(result = null, traceId = null) { - if (!result || typeof result !== "object") return false; - if (result.terminal === true || result.sealed === true) return true; - if (result.error || result.blocker) return true; - if (codeAgentPayloadHasSealedFinalResponse(result)) return true; - if (agentRunTerminalTraceEvidence(result, traceId)) return true; - return Boolean(textValue(result.finishedAt ?? result.completedAt ?? result.endedAt)); -} - -function codeAgentRefreshErrorPayload(error, traceId, agentRun, fallbackCode) { - return { - code: error?.code ?? fallbackCode, - layer: error?.layer ?? "agentrun", - category: error?.category ?? (error?.code === "agentrun_timeout" ? "upstream-timeout" : "upstream-refresh-failed"), - retryable: error?.retryable !== false, - message: error?.message ?? "Code Agent turn status refresh degraded", - traceId, - runId: agentRun?.runId ?? error?.runId ?? null, - commandId: agentRun?.commandId ?? error?.commandId ?? null, - method: error?.method ?? null, - route: error?.route ?? error?.path ?? null, - timeoutMs: numberOrNull(error?.timeoutMs), - timeoutStage: error?.timeoutStage ?? null, - upstreamStatus: numberOrNull(error?.statusCode), - managerHost: error?.managerHost ?? null, - valuesPrinted: false - }; -} - -function normalizeTurnStatus(...values) { - for (const value of values) { - const text = textValue(value).toLowerCase().replace(/_/gu, "-"); - if (!text) continue; - if (["accepted", "pending", "processing", "running", "busy", "creating", "queued", "in-flight"].includes(text)) return "running"; - if (["completed", "done", "succeeded", "success"].includes(text)) return "completed"; - if (["failed", "failure", "error", "stale", "thread-resume-failed", "aborted", "interrupted", "expired"].includes(text)) return "failed"; - if (["cancelled", "canceled"].includes(text)) return "canceled"; - if (["blocked", "timeout"].includes(text)) return text; - } - return null; -} - -function isTurnRunningStatus(status) { - return status === "running"; -} - -function codeAgentRunningStatus(value) { - const status = normalizeTurnStatus(value); - return isTurnRunningStatus(status) ? status : null; -} - -function isTurnTerminalStatus(status) { - return CODE_AGENT_TERMINAL_STATUSES.has(String(status ?? "").trim().toLowerCase().replace(/_/gu, "-")); -} - -export async function handleCodeAgentInspectHttp(request, response, url, options) { - const query = { - conversationId: safeConversationId(url.searchParams.get("conversationId")), - sessionId: safeSessionId(url.searchParams.get("sessionId")), - threadId: safeOpaqueId(url.searchParams.get("threadId")), - traceId: safeTraceId(url.searchParams.get("traceId")) - }; - const sessionRegistry = options.sessionRegistry; - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - const manager = options.codexStdioManager; - const managerDescription = manager && typeof manager.describe === "function" ? manager.describe() : null; - const managerRecentSessions = Array.isArray(managerDescription?.recentSessions) ? managerDescription.recentSessions : []; - const directManagerSession = query.sessionId && manager && typeof manager.get === "function" - ? manager.get(query.sessionId, { conversationId: query.conversationId }) - : null; - const matchedManagerSession = directManagerSession ?? managerRecentSessions.find((session) => sessionMatchesInspectQuery(session, query)) ?? null; - const registryInspect = sessionRegistry && typeof sessionRegistry.inspect === "function" - ? sessionRegistry.inspect(query) - : { ok: false, status: "unavailable", traceIds: [], session: null, conversationFacts: null }; - const resultEvidence = await codeAgentInspectResultEvidence(query.traceId, options); - const session = matchedManagerSession ?? registryInspect.session ?? resultEvidence.session ?? null; - const conversationFacts = registryInspect.conversationFacts ?? resultEvidence.conversationFacts ?? null; - const requestedRunnerTrace = query.traceId ? traceStore.snapshot(query.traceId) : null; - const requestedTraceFound = Boolean(requestedRunnerTrace && requestedRunnerTrace.status !== "missing"); - const traceIds = uniqueStrings([ - requestedTraceFound ? query.traceId : null, - session?.currentTraceId, - session?.lastTraceId, - conversationFacts?.latestTraceId, - resultEvidence.latestTraceId, - ...(Array.isArray(conversationFacts?.traceIds) ? conversationFacts.traceIds : []), - ...(Array.isArray(registryInspect.traceIds) ? registryInspect.traceIds : []) - ]); - const latestTraceId = traceIds[0] ?? null; - const runnerTrace = latestTraceId - ? latestTraceId === query.traceId && requestedRunnerTrace - ? requestedRunnerTrace - : traceStore.snapshot(latestTraceId) - : null; - const found = Boolean(registryInspect.ok || session || latestTraceId); - sendJson(response, found ? 200 : 404, { - ok: found, - action: "code-agent.chat.inspect", - status: found ? "found" : "not_found", - query, - session, - conversationFacts, - traceIds, - latestTraceId, - traceUrl: latestTraceId ? `/v1/agent/traces/${encodeURIComponent(latestTraceId)}` : null, - resultUrl: latestTraceId ? `/v1/agent/chat/result/${encodeURIComponent(latestTraceId)}` : null, - runnerTrace, - valuesRedacted: true, - secretMaterialStored: false - }); -} - -async function codeAgentInspectResultEvidence(traceId, options = {}) { - if (!safeTraceId(traceId)) return { session: null, conversationFacts: null, latestTraceId: null }; - const cached = options.codeAgentChatResults?.get?.(traceId) ?? null; - const persisted = cached ? null : await loadPersistedAgentRunResult(traceId, options); - const result = cached ?? persisted ?? null; - if (!result || typeof result !== "object") return { session: null, conversationFacts: null, latestTraceId: null }; - const agentRun = result.agentRun && typeof result.agentRun === "object" ? result.agentRun : null; - const resultSession = result.session && typeof result.session === "object" ? result.session : null; - const sessionReuse = result.sessionReuse && typeof result.sessionReuse === "object" ? result.sessionReuse : null; - const providerTrace = result.providerTrace && typeof result.providerTrace === "object" ? result.providerTrace : null; - const sessionId = safeSessionId(result.sessionId ?? resultSession?.sessionId ?? sessionReuse?.sessionId ?? agentRun?.sessionId) || null; - const conversationId = safeConversationId(result.conversationId ?? resultSession?.conversationId ?? sessionReuse?.conversationId ?? agentRun?.conversationId) || null; - const threadId = safeOpaqueId(result.threadId ?? resultSession?.threadId ?? sessionReuse?.threadId ?? providerTrace?.threadId ?? agentRun?.threadId) || null; - const status = textValue(resultSession?.status ?? result.status ?? agentRun?.status) || null; - const updatedAt = textValue(result.updatedAt ?? agentRun?.updatedAt ?? resultSession?.updatedAt) || null; - const session = sessionId || conversationId || threadId ? { - sessionId, - conversationId, - threadId, - status, - lastTraceId: traceId, - source: "code-agent-result", - agentRun: agentRun ? agentRunSessionEvidence(result).agentRun : null, - valuesRedacted: true, - secretMaterialStored: false - } : null; - const conversationFacts = conversationId ? { - conversationId, - sessionId, - threadId, - latestTraceId: traceId, - traceIds: [traceId], - turnCount: 1, - source: "code-agent-result", - latestStatus: status, - updatedAt, - valuesRedacted: true, - secretMaterialStored: false - } : null; - return { session, conversationFacts, latestTraceId: traceId }; -} - -function sessionMatchesInspectQuery(session, query) { - if (!session || typeof session !== "object") return false; - if (query.sessionId && session.sessionId === query.sessionId) return true; - if (query.threadId && session.threadId === query.threadId) return true; - if (query.conversationId && session.conversationId === query.conversationId) return true; - if (query.traceId && (session.currentTraceId === query.traceId || session.lastTraceId === query.traceId)) return true; - return false; -} - -export async function handleCodeAgentCancelHttp(request, response, options) { - const body = await readBody(request, options.bodyLimitBytes); - let params = {}; - - try { - params = body ? JSON.parse(body) : {}; - } catch (error) { - sendJson(response, 400, { - ...createCodeAgentErrorPayload({ - code: "parse_error", - message: "Invalid JSON body", - reason: error.message, - traceId: getHeader(request, "x-trace-id") || "trc_unassigned", - layer: "api", - retryable: true - }) - }); - return; - } - - if (!params || typeof params !== "object" || Array.isArray(params)) { - sendJson(response, 400, { - ...createCodeAgentErrorPayload({ - code: "invalid_params", - message: "Code Agent cancel body must be a JSON object", - traceId: getHeader(request, "x-trace-id") || "trc_unassigned", - layer: "api", - retryable: true - }) - }); - return; - } - - const traceId = safeTraceId(getHeader(request, "x-trace-id") || params.traceId); - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - const snapshot = traceId ? traceStore.snapshot(traceId) : null; - const currentResult = traceId ? options.codeAgentChatResults?.get(traceId) ?? await loadPersistedAgentRunResult(traceId, options) : null; - if (currentResult?.agentRun?.commandId) { - const mismatch = codeAgentCancelScopeMismatch(params, currentResult); - if (mismatch) { - traceStore.append(traceId, { - type: "cancel", - status: "blocked", - label: "agentrun:cancel:scope_mismatch", - errorCode: "cancel_scope_mismatch", - message: `Cancel request ${mismatch.field} does not match the trace owner; AgentRun cancel was not forwarded.`, - requested: mismatch.requested, - expected: mismatch.expected, - waitingFor: "cancel-scope", - valuesPrinted: false - }); - sendJson(response, 409, cancelBlockedPayload({ - code: "cancel_scope_mismatch", - message: `取消请求的 ${mismatch.field} 与 trace 归属不一致，已拒绝转发 AgentRun cancel，避免误取消其他 session。`, - traceId, - conversationId: safeConversationId(params.conversationId) || safeConversationId(currentResult.conversationId) || null, - sessionId: safeSessionId(params.sessionId) || safeSessionId(currentResult.sessionId) || null, - runnerTrace: traceStore.snapshot(traceId), - status: "blocked" - })); - return; - } - await recordCodeAgentSessionInputFact({ - params: { - ...params, - sessionId: currentResult.sessionId ?? currentResult.session?.sessionId ?? params.sessionId, - messageId: codeAgentTurnLifecycleFields(traceId, currentResult).userMessageId, - ownerUserId: options.actor?.id, - ownerRole: options.actor?.role - }, - options, - traceId, - delivery: "cancel", - status: "promoted", - commandId: currentResult.agentRun.commandId - }); - const payload = await cancelAgentRunChatTurn({ traceId, currentResult, options, traceStore }); - if (payload) { - if (payload.alreadyTerminal === true || isTraceCommandTerminalStatus(payload.status)) { - await recordCodeAgentTerminalTurnStatusEffects({ payload, params: { ...params, traceId, ownerUserId: options.actor?.id, ownerRole: options.actor?.role }, options, preserveLastTraceId: true }); - } else { - await recordCodeAgentSessionOwner({ payload, params: { ...params, traceId, ownerUserId: options.actor?.id, ownerRole: options.actor?.role }, options, status: "canceled" }); - } - sendJson(response, 200, payload); - return; - } - } - const sessionId = safeSessionId(params.sessionId) || safeSessionId(snapshot?.sessionId); - const conversationId = safeConversationId(params.conversationId); - const manager = options.codexStdioManager; - - if (!traceId) { - sendJson(response, 400, cancelBlockedPayload({ - code: "cancel_trace_missing", - message: "traceId is required to cancel the current Code Agent request.", - traceId: "trc_unassigned", - conversationId, - sessionId - })); - return; - } - - if (!manager || typeof manager.get !== "function" || typeof manager.cancel !== "function") { - traceStore.append(traceId, { - type: "cancel", - status: "unsupported", - label: "cancel:unsupported", - errorCode: "cancel_unsupported", - message: "Codex stdio cancel/interrupt backend is not available on this runtime.", - waitingFor: "session-control" - }); - const runnerTrace = traceStore.snapshot(traceId); - sendJson(response, 501, cancelBlockedPayload({ - code: "cancel_unsupported", - message: "当前后端不支持 Code Agent interrupt/cancel；本次按 unsupported/degraded 返回，不会静默假装已中断。", - traceId, - conversationId, - sessionId, - runnerTrace, - status: "degraded", - unsupported: true, - degraded: true - })); - return; - } - - if (!sessionId) { - traceStore.append(traceId, { - type: "cancel", - status: "blocked", - label: "cancel:not_cancelable", - errorCode: "cancel_session_missing", - message: "Cancel request did not include a bound Codex stdio sessionId.", - waitingFor: "session-binding" - }); - sendJson(response, 409, cancelBlockedPayload({ - code: "cancel_session_missing", - message: "当前请求尚未暴露可取消的 Codex stdio sessionId；页面不能只隐藏 UI，已保留输入和 trace。", - traceId, - conversationId, - sessionId: null, - runnerTrace: traceStore.snapshot(traceId) - })); - return; - } - - const currentSession = manager.get(sessionId, { conversationId }) ?? null; - if (!currentSession || !["busy", "creating"].includes(currentSession.status)) { - traceStore.append(traceId, { - type: "cancel", - status: "blocked", - label: "cancel:not_in_flight", - errorCode: "cancel_not_in_flight", - message: `Session ${sessionId} is not an in-flight Codex stdio request.`, - sessionId, - sessionStatus: currentSession?.status ?? "missing" - }); - sendJson(response, 409, cancelBlockedPayload({ - code: currentSession ? "cancel_not_in_flight" : "cancel_session_not_found", - message: currentSession - ? `当前 session 状态为 ${currentSession.status}，没有可取消的 in-flight Codex stdio 请求。` - : `没有找到 sessionId=${sessionId} 的 Codex stdio session。`, - traceId, - conversationId, - sessionId, - session: currentSession, - runnerTrace: traceStore.snapshot(traceId) - })); - return; - } - - const canceledSession = manager.cancel(sessionId, { - traceId, - conversationId, - reason: "user_cancel" - }); - traceStore.append(traceId, { - type: "cancel", - status: "canceled", - label: "cancel:canceled", - message: "User canceled the current Codex stdio request.", - sessionId, - sessionStatus: canceledSession?.status ?? "canceled", - sessionLifecycleStatus: codeAgentSessionLifecycleSummary({ session: canceledSession, status: "canceled" }).status, - waitingFor: "user-retry", - terminal: true - }); - const runnerTraceSnapshot = traceStore.snapshot(traceId); - const sessionSummary = codeAgentSessionLifecycleSummary({ - session: canceledSession, - runnerTrace: runnerTraceSnapshot, - status: "canceled" - }); - const runnerTrace = { - ...runnerTraceSnapshot, - sessionLifecycleStatus: sessionSummary.status - }; - const payload = { - accepted: true, - canceled: true, - status: "canceled", - conversationId: conversationId ?? canceledSession?.conversationId ?? null, - sessionId, - traceId, - session: canceledSession, - sessionLifecycleStatus: sessionSummary.status, - sessionLifecycle: sessionSummary, - sessionSummary, - runnerTrace, - lastTraceEvent: runnerTrace.lastEvent, - retryable: true, - error: { - code: "codex_stdio_canceled", - layer: "session", - category: "canceled", - retryable: true, - message: "user canceled current Code Agent request", - userMessage: "当前 Codex stdio 请求已取消；输入、sessionId、traceId 和最后 trace event 已保留，可重试上一条消息。", - traceId, - route: "/v1/agent/chat/cancel", - toolName: "codex-stdio.cancel" - }, - userMessage: "当前 Codex stdio 请求已取消；输入、sessionId、traceId 和最后 trace event 已保留，可重试上一条消息。", - updatedAt: new Date().toISOString() - }; - await recordCodeAgentSessionOwner({ payload, params: { ...params, traceId, ownerUserId: options.actor?.id, ownerRole: options.actor?.role }, options, status: "canceled" }); - recordCodeAgentConversationFact(payload, options); - options.codeAgentChatResults?.set(traceId, annotateOwner(payload, { ownerUserId: options.actor?.id, ownerRole: options.actor?.role })); - sendJson(response, 200, payload); -} - -function recordCodeAgentConversationFact(payload = {}, options = {}) { - const registry = options.sessionRegistry; - const conversationId = safeConversationId(payload.conversationId); - if (!conversationId || typeof registry?.recordFact !== "function") return null; - try { - return registry.recordFact(conversationId, { - kind: payload.status === "completed" ? "runner_turn" : "runner_turn_terminal", - conversationId, - sessionId: payload.sessionId, - traceId: payload.traceId, - provider: payload.provider, - backend: payload.backend, - workspace: payload.workspace, - sandbox: payload.sandbox, - session: payload.session, - sessionMode: payload.sessionMode, - sessionReuse: payload.sessionReuse, - implementationType: payload.implementationType, - runner: payload.runner, - runnerTrace: payload.runnerTrace, - partialContext: payload.partialContext ?? payload.runnerTrace?.partialContext, - capabilityLevel: payload.capabilityLevel, - toolCalls: payload.toolCalls, - skills: payload.skills - }, { now: options.now }); - } catch { - return null; - } -} - -async function recordCodeAgentTerminalTurnStatusEffects({ payload = {}, params = {}, options = {}, preserveLastTraceId = false } = {}) { - payload = codeAgentPayloadWithObservedTerminalStatus(payload, payload?.runnerTrace) ?? payload; - if (!payload || typeof payload !== "object" || !isTraceCommandTerminalStatus(payload.status)) return null; - if (payload.turnStatusTerminalEffects?.recorded === true) return payload.turnStatusTerminalEffects; - const billing = await finalizeCodeAgentBillingUsage({ payload, params, options }); - recordCodeAgentConversationFact(payload, options); - const owner = await recordCodeAgentSessionOwner({ payload, params, options, status: codeAgentOwnerStatusForResult(payload), preserveLastTraceId }); - const billingSettled = codeAgentTerminalBillingSettled({ payload, params, options, billing }); - const ownerRequired = Boolean(options.actor?.id && options.accessController?.recordAgentSessionOwner); - const ownerSettled = !ownerRequired || Boolean(owner); - const effects = { - recorded: billingSettled && ownerSettled, - pending: false, - billingSettled, - ownerSettled, - preserveLastTraceId: Boolean(preserveLastTraceId), - recordedAt: new Date().toISOString(), - valuesPrinted: false - }; - payload.turnStatusTerminalEffects = effects; - const traceId = safeTraceId(payload.traceId ?? params.traceId); - if (traceId) void emitCodeAgentOtelSpan("projection_write", traceId, options.env ?? process.env, { - attributes: { - projection: "terminal-effects", - status: payload.status, - terminal: true, - billingSettled, - ownerSettled, - sessionId: safeSessionId(payload.sessionId ?? params.sessionId) || null, - turnId: codeAgentTurnLifecycleFields(traceId, payload).turnId, - runId: payload.agentRun?.runId ?? null, - commandId: payload.agentRun?.commandId ?? null - } - }); - if (traceId) options.codeAgentChatResults?.set?.(traceId, payload); - return effects; -} - -function scheduleCodeAgentTerminalTurnStatusEffects({ payload = {}, params = {}, options = {}, preserveLastTraceId = false } = {}) { - payload = codeAgentPayloadWithObservedTerminalStatus(payload, payload?.runnerTrace) ?? payload; - if (!payload || typeof payload !== "object" || !isTraceCommandTerminalStatus(payload.status)) return null; - const existing = payload.turnStatusTerminalEffects; - if (existing?.recorded === true || existing?.pending === true) return existing; - const traceId = safeTraceId(payload.traceId ?? params.traceId); - const pending = { - recorded: false, - pending: true, - billingSettled: false, - ownerSettled: false, - preserveLastTraceId: Boolean(preserveLastTraceId), - scheduledAt: new Date().toISOString(), - valuesPrinted: false - }; - payload.turnStatusTerminalEffects = pending; - if (traceId) options.codeAgentChatResults?.set?.(traceId, payload); - setImmediate(() => { - void recordCodeAgentTerminalTurnStatusEffects({ payload, params, options, preserveLastTraceId }).catch((error) => { - const failed = { - recorded: false, - pending: false, - billingSettled: false, - ownerSettled: false, - preserveLastTraceId: Boolean(preserveLastTraceId), - errorCode: error?.code ?? "terminal_turn_status_effects_failed", - recordedAt: new Date().toISOString(), - valuesPrinted: false - }; - payload.turnStatusTerminalEffects = failed; - if (traceId) { - options.codeAgentChatResults?.set?.(traceId, payload); - (options.traceStore ?? defaultCodeAgentTraceStore).append(traceId, { - type: "turn-status", - status: "degraded", - label: "turn-status:terminal-effects-failed", - errorCode: failed.errorCode, - message: error?.message ?? "Terminal turn status side effects failed and will retry on the next poll.", - valuesPrinted: false - }); - } - }); - }); - return pending; -} - -function codeAgentTerminalBillingSettled({ payload = {}, params = {}, options = {}, billing = null } = {}) { - const reservation = params.userBillingReservation ?? payload.userBillingReservation; - const reservationId = typeof reservation?.reservationId === "string" ? reservation.reservationId : ""; - if (!reservationId || !options.userBillingClient?.configured) return true; - return billing?.recorded === true || billing?.released === true || payload.billing?.recorded === true || payload.billing?.released === true; -} - -export async function handleCodeAgentSteerHttp(request, response, options) { - const body = await readBody(request, options.bodyLimitBytes); - let params = {}; - - try { - params = body ? JSON.parse(body) : {}; - } catch (error) { - sendJson(response, 400, { - ...createCodeAgentErrorPayload({ - code: "parse_error", - message: "Invalid JSON body", - reason: error.message, - traceId: getHeader(request, "x-trace-id") || "trc_unassigned", - layer: "api", - retryable: true, - route: "/v1/agent/chat/steer" - }) - }); - return; - } - - if (!params || typeof params !== "object" || Array.isArray(params)) { - sendJson(response, 400, { - ...createCodeAgentErrorPayload({ - code: "invalid_params", - message: "Code Agent steer body must be a JSON object", - traceId: getHeader(request, "x-trace-id") || "trc_unassigned", - layer: "api", - retryable: true, - route: "/v1/agent/chat/steer" - }) - }); - return; - } - - const traceId = safeTraceId(getHeader(request, "x-trace-id") || params.traceId || params.targetTraceId); - const message = firstNonEmptyValue(params.message, params.prompt, params.text); - if (!traceId || !message) { - sendJson(response, 400, { - ...createCodeAgentErrorPayload({ - code: !traceId ? "steer_trace_missing" : "steer_message_missing", - message: !traceId ? "traceId is required to steer the current Code Agent turn." : "message is required to steer the current Code Agent turn.", - traceId: traceId || "trc_unassigned", - layer: "api", - retryable: true, - route: "/v1/agent/chat/steer" - }) - }); - return; - } - - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - const currentResult = options.codeAgentChatResults?.get?.(traceId) ?? await loadPersistedAgentRunResult(traceId, options); - if (!canAccessOwnedResult(currentResult, options.actor)) { - sendJson(response, 403, { - ok: false, - status: "forbidden", - traceId, - error: { code: "code_agent_trace_forbidden", message: "Code Agent trace is not visible to the current actor" } - }); - return; - } - if (!currentResult?.agentRun?.runId || !currentResult?.agentRun?.commandId) { - sendJson(response, 404, { - ok: false, - status: "not_found", - traceId, - error: { code: "steer_trace_not_found", message: "No AgentRun-backed Code Agent turn was found for the requested traceId." }, - route: "/v1/agent/chat/steer", - runnerTrace: traceStore.snapshot(traceId) - }); - return; - } - const unsealedTerminalError = traceSnapshotTerminalFailureDiagnostic(traceStore.snapshot(traceId)); - if (unsealedTerminalError) { - const terminalSync = await syncAgentRunTerminalBeforeSteer({ traceId, currentResult, params, options, traceStore }); - const syncedResult = terminalSync?.payload ?? currentResult; - if (terminalSync?.terminal === true) { - sendJson(response, 409, { - ok: false, - accepted: false, - status: "blocked", - traceId, - route: "/v1/agent/chat/steer", - sessionId: safeSessionId(syncedResult.sessionId ?? params.sessionId) || null, - threadId: safeOpaqueId(syncedResult.threadId ?? params.threadId) || null, - agentRun: syncedResult.agentRun ?? currentResult.agentRun, - runnerTrace: terminalSync.runnerTrace ?? traceStore.snapshot(traceId), - terminalDiagnostic: unsealedTerminalError, - error: { - code: "steer_trace_terminal", - layer: "api", - category: "not_in_flight", - retryable: false, - message: `Trace ${traceId} is already terminal and cannot accept a steer command.`, - traceId, - route: "/v1/agent/chat/steer", - toolName: "agentrun.command.steer" - }, - valuesPrinted: false - }); - return; - } - traceStore.append(traceId, { - type: "backend", - status: "blocked", - label: "agentrun:steer:blocked-unsealed-terminal-error", - errorCode: terminalSync?.error?.code ?? unsealedTerminalError.failureKind ?? "steer_trace_unsealed_terminal_error", - message: terminalSync?.error?.message ?? "Trace has an unsealed terminal error diagnostic; HWLAB refused to steer a new prompt into the old trace.", - waitingFor: "terminal-projection-reconciliation", - terminalDiagnostic: unsealedTerminalError, - terminal: false, - valuesPrinted: false - }); - sendJson(response, terminalSync?.error ? 503 : 409, { - ok: false, - accepted: false, - status: "blocked", - traceId, - route: "/v1/agent/chat/steer", - sessionId: safeSessionId(currentResult.sessionId ?? params.sessionId) || null, - threadId: safeOpaqueId(currentResult.threadId ?? params.threadId) || null, - agentRun: currentResult.agentRun, - runnerTrace: traceStore.snapshot(traceId), - terminalDiagnostic: unsealedTerminalError, - error: { - code: terminalSync?.error?.code ?? "steer_trace_unsealed_terminal_error", - layer: terminalSync?.error?.layer ?? "api", - category: "not_in_flight", - retryable: false, - message: terminalSync?.error?.message ?? "Trace has an unsealed terminal error diagnostic; start a new turn instead of steering into the old trace.", - traceId, - route: "/v1/agent/chat/steer", - toolName: "agentrun.command.steer" - }, - valuesPrinted: false - }); - return; - } - if (isTraceCommandTerminalStatus(currentResult.status)) { - sendJson(response, 409, { - ok: false, - accepted: false, - status: "blocked", - traceId, - route: "/v1/agent/chat/steer", - sessionId: safeSessionId(currentResult.sessionId ?? params.sessionId) || null, - threadId: safeOpaqueId(currentResult.threadId ?? params.threadId) || null, - agentRun: currentResult.agentRun, - runnerTrace: traceStore.snapshot(traceId), - error: { - code: "steer_trace_terminal", - layer: "api", - category: "not_in_flight", - retryable: false, - message: `Trace ${traceId} is already terminal and cannot accept a steer command.`, - traceId, - route: "/v1/agent/chat/steer", - toolName: "agentrun.command.steer" - }, - valuesPrinted: false - }); - return; - } - - try { - await recordCodeAgentSessionInputFact({ - params: { - ...params, - sessionId: currentResult.sessionId ?? currentResult.session?.sessionId ?? params.sessionId, - ownerUserId: options.actor?.id, - ownerRole: options.actor?.role - }, - options, - traceId, - delivery: "steer", - status: "promoted", - commandId: currentResult.agentRun.commandId - }); - const payload = await steerAgentRunChatTurn({ - traceId, - currentResult, - params: { - ...params, - message, - ownerUserId: options.actor?.id ?? params.ownerUserId, - ownerRole: options.actor?.role ?? params.ownerRole - }, - options, - traceStore - }); - sendJson(response, 202, payload); - setImmediate(() => { - void recordCodeAgentSessionOwner({ payload: currentResult, params: { ...params, traceId, ownerUserId: options.actor?.id, ownerRole: options.actor?.role }, options, status: "running" }); - }); - } catch (error) { - traceStore.append(traceId, { - type: "backend", - status: "failed", - label: "agentrun:steer:failed", - errorCode: error?.code ?? "agentrun_steer_failed", - message: error?.message ?? "AgentRun steer command failed.", - terminal: false, - valuesPrinted: false - }); - sendJson(response, error?.statusCode === 404 ? 404 : 409, { - ok: false, - accepted: false, - status: "failed", - traceId, - route: "/v1/agent/chat/steer", - sessionId: safeSessionId(currentResult.sessionId ?? params.sessionId) || null, - threadId: safeOpaqueId(currentResult.threadId ?? params.threadId) || null, - agentRun: currentResult.agentRun, - runnerTrace: traceStore.snapshot(traceId), - error: { - code: error?.code ?? "agentrun_steer_failed", - layer: "agentrun", - category: "steer_failed", - retryable: true, - message: error?.message ?? "AgentRun steer command failed.", - traceId, - route: "/v1/agent/chat/steer", - toolName: "agentrun.command.steer" - } - }); - } -} - -async function syncAgentRunTerminalBeforeSteer({ traceId, currentResult = null, params = {}, options = {}, traceStore = defaultCodeAgentTraceStore } = {}) { - try { - const synced = await syncAgentRunChatResult({ - traceId, - currentResult, - options: { ...options, deferAgentRunResultSync: false }, - traceStore, - forceResultSync: true, - refreshEvents: true - }); - const payload = codeAgentPayloadWithObservedTerminalStatus(synced?.result ?? null, synced?.runnerTrace ?? traceStore.snapshot(traceId)) ?? synced?.result ?? currentResult; - if (isTraceCommandTerminalStatus(payload?.status)) { - await recordCodeAgentTerminalTurnStatusEffects({ - payload, - params: { ...params, traceId, ownerUserId: options.actor?.id ?? params.ownerUserId, ownerRole: options.actor?.role ?? params.ownerRole }, - options, - preserveLastTraceId: true - }); - return { terminal: true, payload, runnerTrace: synced?.runnerTrace ?? payload?.runnerTrace ?? traceStore.snapshot(traceId), valuesPrinted: false }; - } - return { terminal: false, payload, runnerTrace: synced?.runnerTrace ?? payload?.runnerTrace ?? traceStore.snapshot(traceId), valuesPrinted: false }; - } catch (error) { - return { terminal: false, error, runnerTrace: traceStore.snapshot(traceId), valuesPrinted: false }; - } -} - -function traceSnapshotTerminalFailureDiagnostic(snapshot = null) { - const events = Array.isArray(snapshot?.events) ? snapshot.events : []; - for (const event of [...events].reverse()) { - const payload = event?.payload && typeof event.payload === "object" ? event.payload : {}; - const label = textValue(event?.label ?? event?.type); - const status = normalizeTurnStatus(event?.status ?? payload.status); - const failureKind = textValue(event?.failureKind ?? event?.errorCode ?? payload.failureKind ?? payload.errorCode); - const retrying = event?.willRetry === true || payload.willRetry === true; - const retryExhausted = event?.retryExhausted === true || payload.retryExhausted === true; - const explicitlyNonRetryable = event?.retryable === false || payload.retryable === false; - const hardFailure = retryExhausted || explicitlyNonRetryable || failureKind === "backend-timeout" || /^agentrun:error:/u.test(label); - if (retrying || !hardFailure || (status !== "failed" && !failureKind)) continue; - return { - label: label || null, - status: status || null, - failureKind: failureKind || null, - retryExhausted, - message: textValue(event?.message ?? payload.message) || null, - valuesPrinted: false - }; - } - return null; -} - -async function recordCodeAgentSessionOwner({ payload = {}, params = {}, options = {}, status = "active", preserveLastTraceId = false } = {}) { - const ownerUserId = options.actor?.id ?? params.ownerUserId; - if (!ownerUserId || !options.accessController?.recordAgentSessionOwner) return null; - const traceId = safeTraceId(payload.traceId ?? params.traceId); - const session = payload.session && typeof payload.session === "object" ? payload.session : null; - const sessionReuse = payload.sessionReuse && typeof payload.sessionReuse === "object" ? payload.sessionReuse : null; - const sessionId = safeSessionId(payload.sessionId ?? session?.sessionId ?? sessionReuse?.sessionId ?? params.sessionId) || null; - const conversationId = safeConversationId(payload.conversationId ?? session?.conversationId ?? sessionReuse?.conversationId ?? params.conversationId) || null; - const threadId = safeOpaqueId(session?.threadId ?? sessionReuse?.threadId ?? payload.threadId ?? params.threadId) || null; - return writeWorkbenchProjectionSession({ - accessController: options.accessController, - runtimeStore: options.runtimeStore, - traceStore: options.traceStore ?? defaultCodeAgentTraceStore, - traceId, - ownerUserId, - ownerRole: options.actor?.role ?? params.ownerRole ?? null, - sessionId, - projectId: params.projectId ?? payload.projectId ?? null, - conversationId, - threadId, - status, - preserveLastTraceId, - session: codeAgentSessionOwnerEvidence(payload, params), - payload, - params - }); -} - -function codeAgentOwnerStatusForResult(result = {}) { - const sessionStatus = textValue(result?.session?.status ?? result?.sessionSummary?.status ?? result?.sessionLifecycleStatus); - if (sessionStatus === "thread-resume-failed") return "thread-resume-failed"; - if (codeAgentFreshContinuationFailure(result)) return "thread-resume-failed"; - if (codeAgentPayloadHasSealedFinalResponse(result)) return "completed"; - if (result?.status === "completed") return "completed"; - if (result?.status === "canceled" || result?.status === "cancelled") return "canceled"; - return result?.status ?? "active"; -} - -function textValue(value) { - return String(value ?? "").trim(); -} - -function timestampIsoOrNow(value = null) { - const parsed = Date.parse(String(value ?? "")); - return Number.isFinite(parsed) ? new Date(parsed).toISOString() : new Date().toISOString(); -} - -function timestampIsoOrNull(value = null) { - const parsed = Date.parse(String(value ?? "")); - return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null; -} - -function codeAgentSessionOwnerEvidence(payload = {}, params = {}) { - const traceId = payload.traceId ?? params.traceId ?? null; - const finalResponse = codeAgentFinalResponseEvidence(payload, traceId); - const traceSummary = codeAgentTraceSummaryEvidence(payload, traceId, finalResponse); - const messages = codeAgentConversationMessagesEvidence(payload, params, traceId, finalResponse); - const firstUserMessagePreview = messages.find((message) => message.role === "user")?.text?.slice(0, 240) ?? null; - const promptEvidence = codeAgentPromptMetadata(payload, params, traceId); - const traceResult = codeAgentTraceResultEvidence(payload, params, traceId, finalResponse, traceSummary); - const providerProfile = codeAgentProviderProfileEvidence(payload, params); - const continuation = codeAgentContinuationEvidence(payload, params); - const sessionLifecycleStatus = continuation.requiresFreshSession ? "thread-resume-failed" : payload.sessionLifecycleStatus ?? null; - return { - ...(providerProfile ? { providerProfile, codeAgentProviderProfile: providerProfile } : {}), - provider: payload.provider ?? null, - model: payload.model ?? null, - backend: payload.backend ?? null, - status: payload.status ?? null, - sessionMode: payload.sessionMode ?? payload.session?.sessionMode ?? null, - sessionLifecycleStatus, - continuation, - runnerKind: payload.runner?.kind ?? payload.runnerTrace?.runnerKind ?? null, - conversationId: payload.conversationId ?? params.conversationId ?? null, - sessionId: payload.sessionId ?? payload.session?.sessionId ?? payload.sessionReuse?.sessionId ?? params.sessionId ?? null, - threadId: payload.session?.threadId ?? payload.sessionReuse?.threadId ?? payload.threadId ?? params.threadId ?? null, - traceId, - projectId: payload.projectId ?? params.projectId ?? null, - ...codeAgentPromptFields(promptEvidence), - finalResponse, - traceSummary, - ...(traceResult ? { traceResults: { [traceResult.traceId]: traceResult } } : {}), - ...(messages.length ? { messages, messageCount: messages.length } : {}), - ...(firstUserMessagePreview ? { firstUserMessagePreview } : {}), - ...agentRunSessionEvidence(payload), - secretMaterialStored: false, - valuesRedacted: true - }; -} - -function codeAgentContinuationEvidence(payload = {}, params = {}) { - const failure = codeAgentFreshContinuationFailure(payload); - const providerProfile = codeAgentProviderProfileEvidence(payload, params); - const traceId = safeTraceId(payload.traceId ?? params.traceId) || null; - if (!failure) { - return { - requiresFreshSession: false, - providerProfile, - traceId, - valuesRedacted: true - }; - } - return { - requiresFreshSession: true, - reasonCode: failure.code, - reason: failure.message, - staleThreadId: safeOpaqueId(payload.session?.threadId ?? payload.sessionReuse?.threadId ?? payload.threadId ?? params.threadId ?? payload.agentRun?.threadId) || null, - providerProfile, - traceId, - valuesRedacted: true - }; -} - -function codeAgentFreshContinuationFailure(payload = {}) { - if (!payload || typeof payload !== "object") return null; - const sessionStatus = String(payload.session?.status ?? payload.sessionLifecycleStatus ?? payload.sessionReuse?.status ?? "").trim().toLowerCase().replace(/_/gu, "-"); - if (sessionStatus === "thread-resume-failed") return { code: "thread-resume-failed", message: "previous AgentRun thread/session resume failed", valuesRedacted: true }; - const status = normalizeTurnStatus(payload.status, payload.agentRun?.terminalStatus, payload.agentRun?.commandState, payload.agentRun?.status, payload.session?.status, payload.sessionReuse?.status); - if (!status || !["failed", "blocked", "timeout"].includes(status)) return null; - const code = normalizeCodeAgentFailureKind(firstNonEmptyValue( - payload.error?.code, - payload.blocker?.code, - payload.providerTrace?.failureKind, - payload.providerTrace?.errorCode, - payload.agentRun?.failureKind, - payload.agentRun?.errorCode, - payload.transient?.failureKind, - payload.transient?.errorCode - )); - const message = firstNonEmptyValue( - payload.error?.message, - payload.error?.userMessage, - payload.blocker?.summary, - payload.blocker?.message, - payload.providerTrace?.failureMessage, - payload.providerTrace?.summary, - payload.agentRun?.failureMessage, - payload.agentRun?.message - ) ?? ""; - if (!codeAgentFailureRequiresFreshContinuation(code, message)) return null; - return { code: code ?? "stale-continuation", message: message || code || "stale continuation", valuesRedacted: true }; -} - -function normalizeCodeAgentFailureKind(value) { - const normalized = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); - return normalized || null; -} - -function codeAgentFailureRequiresFreshContinuation(code, message = "") { - const failureKind = normalizeCodeAgentFailureKind(code); - if (["thread-resume-failed", "session-store-evicted", "provider-stream-disconnected"].includes(failureKind)) return true; - return /session stor(?:e|age).*evicted|pvc-backed session|no rollout found for thread id|no rollout found|thread\/resume failed|provider stream disconnected/iu.test(String(message ?? "")); -} - -function codeAgentProviderProfileEvidence(payload = {}, params = {}) { - const raw = firstNonEmptyValue( - params.providerProfile, - params.codeAgentProviderProfile, - payload.providerProfile, - payload.codeAgentProviderProfile, - payload.agentRun?.backendProfile, - payload.providerTrace?.backendProfile, - payload.backendProfile - ); - if (!raw) return null; - const normalized = normalizeCodeAgentProviderProfile(raw); - return normalized === "runtime-default" ? null : normalized; -} - -function codeAgentTraceResultEvidence(payload = {}, params = {}, traceId = null, finalResponse = null, traceSummary = null) { - const resolvedTraceId = safeTraceId(traceId); - if (!resolvedTraceId) return null; - const agentRun = agentRunSessionEvidence(payload).agentRun ?? null; - const userBillingReservation = codeAgentBillingReservationEvidence(payload.userBillingReservation ?? params.userBillingReservation); - const resultSession = payload.session && typeof payload.session === "object" ? payload.session : null; - const sessionReuse = payload.sessionReuse && typeof payload.sessionReuse === "object" ? payload.sessionReuse : null; - const conversationId = safeConversationId(payload.conversationId ?? agentRun?.conversationId ?? resultSession?.conversationId ?? sessionReuse?.conversationId ?? params.conversationId) || null; - const sessionId = safeSessionId(payload.sessionId ?? resultSession?.sessionId ?? sessionReuse?.sessionId ?? params.sessionId) || null; - const threadId = safeOpaqueId(payload.threadId ?? resultSession?.threadId ?? sessionReuse?.threadId ?? params.threadId ?? agentRun?.threadId) || null; - const promptEvidence = codeAgentPromptMetadata(payload, params, resolvedTraceId); - if (!agentRun && !finalResponse && !traceSummary) return null; - return { - traceId: resolvedTraceId, - status: codeAgentPayloadHasSealedFinalResponse(payload) ? "completed" : payload.status ?? agentRun?.terminalStatus ?? agentRun?.commandState ?? null, - conversationId, - sessionId, - threadId, - ...codeAgentPromptFields(promptEvidence), - messageId: finalResponse?.messageId ?? payload.messageId ?? null, - createdAt: payload.createdAt ?? finalResponse?.createdAt ?? null, - updatedAt: payload.updatedAt ?? finalResponse?.updatedAt ?? null, - finalResponse, - traceSummary, - agentRun, - ...(userBillingReservation ? { userBillingReservation } : {}), - valuesRedacted: true, - secretMaterialStored: false - }; -} - -function codeAgentPromptMetadata(payload = {}, params = {}, traceId = null) { - const prompt = payload?.prompt && typeof payload.prompt === "object" ? payload.prompt : null; - const rawPrompt = firstNonEmptyValue( - params.message, - params.prompt, - params.text, - payload.userMessage, - typeof payload.prompt === "string" ? payload.prompt : null - ); - const fromText = codeAgentPromptMetadataFromText(rawPrompt, traceId, "code-agent-request", { fullText: Boolean(rawPrompt) }); - return codeAgentPromptMetadataFromParts({ - traceId, - source: prompt?.source ?? payload.promptSource ?? fromText?.prompt?.source ?? "code-agent-request", - promptId: firstNonEmptyValue(payload.promptId, prompt?.id, fromText?.promptId), - promptSha256: firstNonEmptyValue(payload.promptSha256, prompt?.sha256, fromText?.promptSha256), - promptSummary: firstNonEmptyValue(payload.promptSummary, prompt?.summary, fromText?.promptSummary), - promptPreview: firstNonEmptyValue(payload.promptPreview, prompt?.preview, fromText?.promptPreview), - textChars: Number.isInteger(prompt?.textChars) ? prompt.textChars : fromText?.prompt?.textChars - }); -} - -function codeAgentPromptMetadataFromText(value, traceId = null, source = "code-agent-request", options = {}) { - const promptText = String(value ?? "").trim(); - if (!promptText) return null; - return codeAgentPromptMetadataFromParts({ - traceId, - source, - promptSha256: options.fullText === true ? sha256Text(promptText) : undefined, - promptSummary: promptText, - promptPreview: promptText, - textChars: options.fullText === true ? promptText.length : undefined - }); -} - -function codeAgentPromptMetadataFromParts({ traceId = null, source = "code-agent-request", promptId = null, promptSha256 = null, promptSummary = null, promptPreview = null, textChars = undefined } = {}) { - const digest = firstNonEmptyValue(promptSha256); - const summary = clippedPromptText(firstNonEmptyValue(promptSummary, promptPreview), 240); - const preview = clippedPromptText(firstNonEmptyValue(promptPreview, promptSummary), 500); - const explicitId = firstNonEmptyValue(promptId); - if (!explicitId && !digest && !summary && !preview) return null; - const id = explicitId || codeAgentPromptId(traceId, digest); - const prompt = compactCodeAgentObject({ - id, - sha256: digest || undefined, - summary, - preview, - textChars: Number.isInteger(textChars) ? textChars : undefined, - source, - valuesPrinted: false - }); - return compactCodeAgentObject({ - promptId: id, - promptSha256: digest || undefined, - promptSummary: summary, - promptPreview: preview, - prompt, - valuesPrinted: false - }) ?? null; -} - -function codeAgentPromptFields(metadata = null) { - if (!metadata) return {}; - return compactCodeAgentObject({ - promptId: metadata.promptId, - promptSha256: metadata.promptSha256, - promptSummary: metadata.promptSummary, - promptPreview: metadata.promptPreview, - prompt: metadata.prompt - }) ?? {}; -} - -function codeAgentPromptId(traceId = null, digest = null) { - const traceSuffix = safeTraceId(traceId)?.slice(4).replace(/[^A-Za-z0-9_.:-]/gu, "_"); - const digestSuffix = firstNonEmptyValue(digest)?.slice(0, 16); - const suffix = firstNonEmptyValue(traceSuffix, digestSuffix); - return suffix ? `prm_${suffix}` : undefined; -} - -function clippedPromptText(value, limit) { - const promptText = String(value ?? "").trim(); - if (!promptText) return undefined; - return promptText.length > limit ? `${promptText.slice(0, limit)}...` : promptText; -} - -function sha256Text(value) { - return createHash("sha256").update(String(value)).digest("hex"); -} - -function compactCodeAgentObject(value) { - const result = {}; - for (const [key, item] of Object.entries(value ?? {})) { - if (item === undefined || item === null) continue; - if (Array.isArray(item) && item.length === 0) continue; - if (typeof item === "object" && !Array.isArray(item) && Object.keys(item).length === 0) continue; - result[key] = item; - } - return Object.keys(result).length > 0 ? result : undefined; -} - -function codeAgentBillingReservationEvidence(value = null) { - if (!value || typeof value !== "object") return null; - const reservationId = textValue(value.reservationId); - if (!reservationId) return null; - return { - reservationId, - resourceType: textValue(value.resourceType) || null, - planId: textValue(value.planId) || null, - estimatedCredits: numberOrNull(value.estimatedCredits), - estimatedTokens: numberOrNull(value.estimatedTokens), - expiresAt: textValue(value.expiresAt) || null, - valuesRedacted: true, - secretMaterialStored: false - }; -} - -function codeAgentConversationMessagesEvidence(payload = {}, params = {}, traceId = null, finalResponse = null) { - const userText = boundedConversationMessageText(params.message ?? params.prompt ?? payload.userMessage ?? payload.prompt); - if (!userText) return []; - const resolvedTraceId = safeTraceId(traceId) || null; - const lifecycle = codeAgentTurnLifecycleFields(resolvedTraceId, payload); - const session = payload.session && typeof payload.session === "object" ? payload.session : null; - const sessionReuse = payload.sessionReuse && typeof payload.sessionReuse === "object" ? payload.sessionReuse : null; - const conversationId = safeConversationId(payload.conversationId ?? session?.conversationId ?? sessionReuse?.conversationId ?? params.conversationId) || null; - const sessionId = safeSessionId(payload.sessionId ?? session?.sessionId ?? sessionReuse?.sessionId ?? params.sessionId) || null; - const threadId = safeOpaqueId(session?.threadId ?? sessionReuse?.threadId ?? payload.threadId ?? params.threadId) || null; - const createdAt = payload.createdAt ?? payload.reply?.createdAt ?? new Date().toISOString(); - const updatedAt = payload.updatedAt ?? createdAt; - const userMessage = { - id: lifecycle.userMessageId, - messageId: lifecycle.userMessageId, - role: "user", - title: "用户", - text: userText, - status: "sent", - traceId: resolvedTraceId, - turnId: lifecycle.turnId, - conversationId, - sessionId, - threadId, - createdAt, - updatedAt, - source: "code-agent-submit", - secretMaterialStored: false, - valuesRedacted: true - }; - const agentText = boundedConversationMessageText( - finalResponse?.text - ?? payload.reply?.content - ?? payload.message?.content - ?? payload.assistantText - ?? payload.error?.userMessage - ?? payload.error?.message - ?? payload.blocker?.userMessage - ?? payload.blocker?.message - ?? payload.blocker?.summary - ); - const agentStatus = codeAgentConversationAgentMessageStatus(payload); - const agentMessage = { - id: lifecycle.assistantMessageId, - messageId: lifecycle.assistantMessageId, - role: "agent", - title: agentStatus === "failed" ? "Code Agent 返回阻塞" : agentStatus === "running" ? "Code Agent 处理中" : "Code Agent 回复", - text: agentText || (agentStatus === "running" ? "" : "Code Agent 请求已结束，请查看 Trace 详情。"), - status: agentStatus, - traceId: resolvedTraceId, - turnId: lifecycle.turnId, - conversationId, - sessionId, - threadId, - createdAt: payload.reply?.createdAt ?? updatedAt, - updatedAt, - source: "code-agent-result", - ...(payload.runnerTrace && typeof payload.runnerTrace === "object" ? { runnerTrace: codeAgentConversationRunnerTraceEvidence(payload.runnerTrace, resolvedTraceId) } : {}), - secretMaterialStored: false, - valuesRedacted: true - }; - return [userMessage, agentMessage]; -} - -function boundedConversationMessageText(value, limit = 12000) { - const text = conversationText(value); - if (!text) return ""; - return text.length > limit ? text.slice(0, limit) : text; -} - -function conversationText(value) { - const extracted = conversationTextRaw(value).replace(/\r\n?/gu, "\n"); - if (!extracted.trim() || extracted.trim() === "[object Object]") return ""; - return extracted; -} - -function conversationTextRaw(value) { - if (value === undefined || value === null) return ""; - if (typeof value === "string") return value; - if (typeof value === "number" || typeof value === "boolean") return String(value); - if (Array.isArray(value)) return value.map(conversationTextRaw).filter(Boolean).join("\n"); - if (typeof value === "object") { - for (const key of ["text", "content", "message", "prompt", "value", "summary", "preview", "title"]) { - const text = conversationTextRaw(value[key]); - if (text) return text; - } - } - return ""; -} - -function codeAgentMessageTraceSuffix(traceId) { - const text = safeTraceId(traceId) || `local_${Date.now().toString(36)}`; - return text.replace(/^trc_/u, "").replace(/[^A-Za-z0-9_.:-]/gu, "_").slice(0, 48) || "trace"; -} - -function codeAgentTurnLifecycleFields(traceId, source = {}) { - const resolvedTraceId = safeTraceId(source?.traceId ?? traceId) || safeTraceId(traceId) || null; - const turnId = safeTurnId(source?.turnId) || resolvedTraceId; - const traceSuffix = codeAgentMessageTraceSuffix(resolvedTraceId); - return { - turnId, - userMessageId: safeMessageId(source?.userMessageId ?? source?.userMessage?.messageId) || codeAgentLifecycleMessageId(traceSuffix, "user"), - assistantMessageId: safeMessageId(source?.assistantMessageId ?? source?.assistantMessage?.messageId) || codeAgentLifecycleMessageId(traceSuffix, "agent") - }; -} - -function codeAgentLifecycleMessageId(traceSuffix, role) { - return `msg_${traceSuffix}_${role}`; -} - -function safeTurnId(value) { - const text = textValue(value); - return /^turn_[A-Za-z0-9_.:-]+$/u.test(text) ? text : null; -} - -function safeMessageId(value) { - const text = textValue(value); - return /^msg_[A-Za-z0-9_.:-]+$/u.test(text) ? text : null; -} - -function codeAgentConversationAgentMessageStatus(payload = {}) { - const status = textValue(payload.status ?? payload.agentRun?.terminalStatus ?? payload.commandState ?? payload.runStatus).toLowerCase(); - if (codeAgentPayloadHasSealedFinalResponse(payload)) return "completed"; - if (["completed", "done", "success"].includes(status)) return "completed"; - if (["failed", "blocked", "error", "timeout", "canceled", "cancelled"].includes(status)) return "failed"; - if (payload.error || payload.blocker) return "failed"; - return "running"; -} - -function codeAgentPayloadHasSealedFinalResponse(payload = {}) { - if (!payload || typeof payload !== "object") return false; - if (payload.error || payload.blocker) return false; - if (!codeAgentFinalResponseText(payload)) return false; - const agentRun = payload.agentRun && typeof payload.agentRun === "object" ? payload.agentRun : {}; - return isTraceCommandTerminalStatus(payload.status) || - isTraceCommandTerminalStatus(agentRun.status) || - isTraceCommandTerminalStatus(agentRun.runStatus) || - isTraceCommandTerminalStatus(agentRun.commandState) || - isTraceCommandTerminalStatus(agentRun.terminalStatus); -} - -function codeAgentConversationRunnerTraceEvidence(runnerTrace = {}, traceId = null) { - const events = Array.isArray(runnerTrace.events) ? runnerTrace.events : []; - const lastEvent = runnerTrace.lastEvent && typeof runnerTrace.lastEvent === "object" ? runnerTrace.lastEvent : events.at(-1) ?? null; - const eventCount = Number.isFinite(Number(runnerTrace.eventCount)) ? Number(runnerTrace.eventCount) : events.length; - return { - traceId: safeTraceId(runnerTrace.traceId ?? traceId) || traceId || null, - status: runnerTrace.status ?? lastEvent?.status ?? null, - sessionId: runnerTrace.sessionId ?? null, - threadId: runnerTrace.threadId ?? null, - runId: runnerTrace.runId ?? lastEvent?.runId ?? null, - commandId: runnerTrace.commandId ?? lastEvent?.commandId ?? null, - eventCount, - lastEvent: lastEvent ? { - label: lastEvent.label ?? null, - status: lastEvent.status ?? null, - terminal: lastEvent.terminal === true, - createdAt: lastEvent.createdAt ?? null, - message: boundedConversationMessageText(lastEvent.message ?? lastEvent.text, 1000), - valuesPrinted: false - } : null, - eventsCompacted: true, - fullTraceLoaded: false, - valuesPrinted: false - }; -} - -function codeAgentFinalResponseEvidence(payload = {}, traceId = null) { - const text = codeAgentFinalResponseText(payload); - if (!text) return null; - return { - text, - textChars: text.length, - messageId: payload.finalResponse?.messageId ?? payload.reply?.messageId ?? payload.messageId ?? null, - role: payload.reply?.role ?? payload.message?.role ?? "assistant", - status: codeAgentPayloadHasSealedFinalResponse(payload) ? "completed" : payload.finalResponse?.status ?? payload.status ?? null, - traceId, - createdAt: payload.finalResponse?.createdAt ?? payload.reply?.createdAt ?? payload.createdAt ?? null, - updatedAt: payload.finalResponse?.updatedAt ?? payload.updatedAt ?? null, - source: payload.finalResponse?.source ?? "code-agent-result", - valuesPrinted: false - }; -} - -function codeAgentFinalResponseText(payload = {}) { - for (const value of [ - payload?.finalResponse?.text, - payload?.finalResponse?.content, - payload?.finalResponse, - payload?.reply?.content, - payload?.reply, - payload?.message?.content, - payload?.message, - payload?.assistantText, - payload?.finalText, - payload?.text, - payload?.content - ]) { - const text = conversationText(value); - if (text) return text; - } - return ""; -} - -function codeAgentTraceSummaryEvidence(payload = {}, traceId = null, finalResponse = null) { - const runnerTrace = payload.runnerTrace && typeof payload.runnerTrace === "object" ? payload.runnerTrace : null; - const events = Array.isArray(runnerTrace?.events) ? runnerTrace.events : []; - const terminalEvent = [...events].reverse().find((event) => event?.terminal === true) ?? runnerTrace?.lastEvent ?? null; - return { - traceId, - source: "code-agent-derived-session-evidence", - sourceEventCount: Number.isFinite(Number(runnerTrace?.eventCount)) ? Number(runnerTrace.eventCount) : events.length, - terminalStatus: payload.agentRun?.terminalStatus ?? payload.status ?? terminalEvent?.status ?? null, - lastEventLabel: terminalEvent?.label ?? null, - finalAssistantRow: finalResponse ? { - role: finalResponse.role, - status: finalResponse.status, - textChars: finalResponse.textChars, - textPreview: finalResponse.text.slice(0, 240), - messageId: finalResponse.messageId, - valuesPrinted: false - } : null, - agentRun: payload.agentRun ? { - runId: payload.agentRun.runId ?? null, - commandId: payload.agentRun.commandId ?? null, - attemptId: payload.agentRun.attemptId ?? null, - runnerId: payload.agentRun.runnerId ?? null, - jobName: payload.agentRun.jobName ?? null, - namespace: payload.agentRun.namespace ?? null, - lastSeq: payload.agentRun.lastSeq ?? null, - valuesPrinted: false - } : null, - updatedAt: payload.updatedAt ?? null, - valuesPrinted: false - }; -} - -function annotateOwner(payload, params = {}) { - if (!params.ownerUserId) return payload; - return { - ...payload, - ownerUserId: params.ownerUserId, - ownerRole: params.ownerRole ?? null - }; -} - -function canAccessOwnedResult(result, actor) { - if (!result?.ownerUserId || !actor) return true; - return actor.role === "admin" || actor.id === result.ownerUserId; -} - -function isCodeAgentResultCanceled(result) { - return result?.status === "canceled" || result?.canceled === true || result?.error?.code === "codex_stdio_canceled"; -} - -function codeAgentCancelScopeMismatch(params = {}, result = {}) { - return cancelScopeFieldMismatch("conversationId", safeConversationId(params.conversationId), cancelExpectedValues(safeConversationId, - result.conversationId, - result.agentRun?.conversationId, - result.session?.conversationId, - result.sessionReuse?.conversationId, - result.runnerTrace?.conversationId - )) ?? cancelScopeFieldMismatch("sessionId", safeSessionId(params.sessionId), cancelExpectedValues(safeSessionId, - result.sessionId, - result.session?.sessionId, - result.sessionReuse?.sessionId, - result.agentRun?.hwlabSessionId - )) ?? cancelScopeFieldMismatch("threadId", safeOpaqueId(params.threadId), cancelExpectedValues(safeOpaqueId, - result.threadId, - result.agentRun?.threadId, - result.session?.threadId, - result.sessionReuse?.threadId, - result.runnerTrace?.threadId - )); -} - -function cancelExpectedValues(normalize, ...values) { - return [...new Set(values.map((value) => normalize(value)).filter(Boolean))]; -} - -function cancelScopeFieldMismatch(field, requested, expectedValues) { - if (!requested || expectedValues.length === 0 || expectedValues.includes(requested)) return null; - return { field, requested, expected: expectedValues[0] }; -} - -async function syncAlreadyTerminalAgentRunBeforeCancel({ traceId, currentResult = null, params = {}, options = {}, traceStore = defaultCodeAgentTraceStore } = {}) { - if (!safeTraceId(traceId) || !currentResult?.agentRun?.runId || !currentResult?.agentRun?.commandId) return null; - const currentTerminalStatus = currentResult.agentRun?.terminalStatus ?? currentResult.agentRun?.commandState ?? currentResult.agentRun?.commandStatus; - if (isTraceCommandTerminalStatus(currentTerminalStatus)) { - return alreadyTerminalCancelPayload({ traceId, payload: currentResult, params, options, traceStore }); - } - try { - const synced = await syncAgentRunChatResult({ - traceId, - currentResult, - options: { ...options, deferAgentRunResultSync: false }, - traceStore, - forceResultSync: true, - refreshEvents: true - }); - const payload = synced?.result ?? null; - const payloadTerminalStatus = payload?.agentRun?.terminalStatus ?? payload?.agentRun?.commandState ?? payload?.agentRun?.commandStatus; - if (!isTraceCommandTerminalStatus(payloadTerminalStatus)) return null; - return alreadyTerminalCancelPayload({ traceId, payload, params, options, traceStore }); - } catch (error) { - traceStore.append(traceId, { - type: "cancel", - status: "degraded", - label: "agentrun:cancel:terminal-check-degraded", - errorCode: error?.code ?? "terminal_check_failed", - message: error?.message ?? "Failed to check AgentRun terminal state before forwarding cancel.", - waitingFor: "cancel-terminal-check", - valuesPrinted: false - }); - return null; - } -} - -async function alreadyTerminalCancelPayload({ traceId, payload = {}, params = {}, options = {}, traceStore = defaultCodeAgentTraceStore } = {}) { - traceStore.append(traceId, { - type: "cancel", - status: "blocked", - label: "agentrun:cancel:already-terminal", - message: "AgentRun command was already terminal; HWLAB did not forward cancel and kept the sealed terminal projection.", - runId: payload.agentRun?.runId ?? null, - commandId: payload.agentRun?.commandId ?? null, - terminalStatus: payload.agentRun?.terminalStatus ?? payload.status, - waitingFor: "already-terminal-projection", - valuesPrinted: false - }); - await recordCodeAgentTerminalTurnStatusEffects({ - payload, - params: { ...params, traceId, ownerUserId: options.actor?.id, ownerRole: options.actor?.role }, - options, - preserveLastTraceId: true - }); - return { - ...payload, - cancelDisposition: { - status: "already_terminal", - forwarded: false, - route: "/v1/agent/chat/cancel", - message: "AgentRun command was already terminal; cancel was not forwarded.", - valuesPrinted: false - }, - runnerTrace: traceStore.snapshot(traceId) - }; -} - -function cancelBlockedPayload({ - code, - message, - traceId, - conversationId = null, - sessionId = null, - session = null, - runnerTrace = null, - status = "failed", - unsupported = false, - degraded = false -}) { - const sessionSummary = codeAgentSessionLifecycleSummary({ - session, - runnerTrace, - status, - unsupported, - degraded, - error: { code, userMessage: message, message } - }); - return { - accepted: false, - canceled: false, - status, - conversationId, - sessionId, - traceId, - session, - sessionLifecycleStatus: sessionSummary.status, - sessionLifecycle: sessionSummary, - sessionSummary, - unsupported, - degraded, - runnerTrace, - lastTraceEvent: runnerTrace?.lastEvent ?? null, - error: { - code, - layer: "session", - category: "cancel_blocked", - retryable: true, - userMessage: message, - message, - traceId, - route: "/v1/agent/chat/cancel", - toolName: "codex-stdio.cancel" - }, - blocker: { - code, - layer: "session", - category: "cancel_blocked", - retryable: true, - summary: message, - userMessage: message, - traceId, - route: "/v1/agent/chat/cancel", - toolName: "codex-stdio.cancel" - } - }; -} - -export async function handleCodeAgentTraceHttp(request, response, url, options) { - const startedAt = nowMs(); - const parts = url.pathname.split("/").filter(Boolean); - const traceId = decodeURIComponent(parts[3] ?? ""); - const streamSegment = parts[4]; - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - const pageOptions = codeAgentTracePageOptions(url); - if (!/^trc_[A-Za-z0-9_.:-]+$/u.test(traceId)) { - sendJson(response, 400, { - error: { - code: "invalid_trace_id", - message: "traceId must start with trc_ and contain only safe identifier characters" - } - }); - return; - } - const requestOptions = codeAgentRequestScopedOptions(options); - const projectMismatch = await measureCodeAgentHttpPhase(requestOptions, "trace_project_check", () => traceProjectMismatchSnapshot(traceId, url, requestOptions, "trace")); - if (projectMismatch) { - sendJson(response, projectMismatch.statusCode, projectMismatch.body); - return; - } - const context = await measureCodeAgentHttpPhase(requestOptions, "trace_projection_read", () => readCodeAgentCompatProjection(traceId, requestOptions)); - if (context.statusCode) { - sendJson(response, context.statusCode, context.body); - return; - } - - if (streamSegment === "stream") { - sendTraceSse(response, traceStore, traceId, requestOptions); - return; - } - - const body = await measureCodeAgentHttpPhase(requestOptions, "trace_paginate", () => { - return paginateTraceSnapshot(context.trace, pageOptions); - }); - const statusCode = context.found ? 200 : 404; - void emitCodeAgentOtelSpan("trace_events_read", traceId, requestOptions.env ?? process.env, { - startTimeMs: startedAt, - attributes: { - "http.method": "GET", - "http.route": "/v1/agent/traces/:traceId", - "http.status_code": statusCode, - found: Boolean(context.found), - returnedEvents: Array.isArray(body?.events) ? body.events.length : 0, - sinceSeq: pageOptions.sinceSeq, - limit: pageOptions.limit, - fromSeq: body?.range?.fromSeq ?? null, - toSeq: body?.range?.toSeq ?? null, - totalEvents: body?.range?.total ?? body?.eventCount ?? null, - hasMore: Boolean(body?.hasMore), - fullTraceLoaded: Boolean(body?.fullTraceLoaded) - } - }); - sendJson(response, statusCode, codeAgentCompatProjectionPayload(body, context)); -} - -function codeAgentTracePageOptions(url) { - const sinceSeq = nonNegativeInteger(url.searchParams.get("sinceSeq") ?? url.searchParams.get("since"), 0); - const requestedLimit = parsePositiveInteger(url.searchParams.get("limit"), DEFAULT_CODE_AGENT_TRACE_PAGE_LIMIT); - return { - sinceSeq, - limit: Math.min(requestedLimit, MAX_CODE_AGENT_TRACE_PAGE_LIMIT) - }; -} - -function paginateTraceSnapshot(snapshot, { sinceSeq, limit }) { - const sourceEvents = Array.isArray(snapshot?.events) ? snapshot.events : []; - const indexedEvents = sourceEvents.map((event, index) => ({ event, seq: traceEventSequence(event, index) })); - const firstIndex = indexedEvents.findIndex((item) => item.seq > sinceSeq); - const startIndex = firstIndex >= 0 ? firstIndex : indexedEvents.length; - const pageItems = indexedEvents.slice(startIndex, startIndex + limit); - const events = pageItems.map((item) => item.event); - const fromSeq = pageItems.length > 0 ? pageItems[0].seq : null; - const toSeq = pageItems.length > 0 ? pageItems[pageItems.length - 1].seq : null; - const hasMore = startIndex + pageItems.length < indexedEvents.length; - const totalEvents = snapshot?.eventCount ?? indexedEvents.length; - return { - ...snapshot, - events, - eventCount: totalEvents, - range: { - sinceSeq, - fromSeq, - toSeq, - limit, - returned: events.length, - total: totalEvents - }, - truncated: hasMore, - hasMore, - nextSinceSeq: pageItems.length > 0 ? toSeq : sinceSeq, - fullTraceLoaded: !hasMore - }; -} - -function traceEventSequence(event, index) { - const seq = Number(event?.seq); - return Number.isFinite(seq) && seq > 0 ? Math.trunc(seq) : index + 1; -} - -function nonNegativeInteger(value, fallback) { - const parsed = Number.parseInt(value ?? "", 10); - return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback; -} - -function isTraceCommandTerminalStatus(status) { - return CODE_AGENT_TERMINAL_STATUSES.has(String(status ?? "").trim().toLowerCase().replace(/_/gu, "-")); -} - -function traceSnapshotWithTerminalEvidence(snapshot, result, traceId, refreshError = null) { - const evidence = agentRunTerminalTraceEvidence(result, traceId); - if (snapshot?.status !== "missing") return traceSnapshotWithAttachedTerminalEvidence(snapshot, evidence, refreshError); - if (!evidence) return { - ...snapshot, - ok: false, - traceStatus: "missing", - retention: traceRetentionSummary("missing") - }; - return { - ...snapshot, - ok: true, - status: "expired", - traceStatus: "expired", - projectionStatus: "projecting", - projectionHealth: "degraded", - persisted: true, - retention: traceRetentionSummary("expired"), - conversationId: evidence.conversationId, - sessionId: evidence.sessionId, - threadId: evidence.threadId, - ...codeAgentPromptFields(evidence), - agentRun: evidence.agentRun, - finalResponse: evidence.finalResponse, - traceSummary: evidence.traceSummary, - terminalEvidence: { - available: true, - source: evidence.source, - refresh: refreshError ? { - attempted: true, - ok: false, - code: refreshError?.code ?? "agentrun_trace_refresh_failed", - message: refreshError?.message ?? "AgentRun trace events could not be refreshed; command result remains authoritative for final response.", - valuesPrinted: false - } : { attempted: false, ok: null }, - conversationId: evidence.conversationId, - sessionId: evidence.sessionId, - threadId: evidence.threadId, - ...codeAgentPromptFields(evidence), - agentRun: evidence.agentRun, - traceSummary: evidence.traceSummary, - finalResponse: evidence.finalResponse, - valuesPrinted: false - } - }; -} - -function traceSnapshotWithAttachedTerminalEvidence(snapshot, evidence, refreshError = null) { - if (!evidence) return snapshot; - return { - ...snapshot, - conversationId: snapshot.conversationId ?? evidence.conversationId, - sessionId: snapshot.sessionId ?? evidence.sessionId, - threadId: snapshot.threadId ?? evidence.threadId, - ...codeAgentPromptFields(evidence), - agentRun: snapshot.agentRun ?? evidence.agentRun, - finalResponse: snapshot.finalResponse ?? evidence.finalResponse, - traceSummary: snapshot.traceSummary ?? evidence.traceSummary, - terminalEvidence: snapshot.terminalEvidence ?? (refreshError ? { - available: true, - source: evidence.source, - refresh: { - attempted: true, - ok: false, - code: refreshError?.code ?? "agentrun_trace_refresh_failed", - message: refreshError?.message ?? "AgentRun trace events could not be refreshed; live events may be partial.", - valuesPrinted: false - }, - valuesPrinted: false - } : undefined) - }; -} - -function agentRunTerminalTraceEvidence(result, traceId) { - if (!result || typeof result !== "object") return null; - const storedSummary = result.traceSummary && typeof result.traceSummary === "object" && result.traceSummary.source === "agentrun-command-result" ? result.traceSummary : null; - const finalResponse = agentRunTerminalFinalResponse(result, traceId); - const promptEvidence = codeAgentPromptMetadata(result, {}, traceId); - const agentRun = result.agentRun && typeof result.agentRun === "object" ? { - adapter: result.agentRun.adapter ?? null, - runId: result.agentRun.runId ?? null, - commandId: result.agentRun.commandId ?? null, - attemptId: result.agentRun.attemptId ?? null, - runnerId: result.agentRun.runnerId ?? null, - jobName: result.agentRun.jobName ?? null, - namespace: result.agentRun.namespace ?? null, - terminalStatus: result.agentRun.terminalStatus ?? null, - lastSeq: result.agentRun.lastSeq ?? null, - valuesPrinted: false - } : null; - const terminalStatus = normalizeTurnStatus(result.agentRun?.terminalStatus, result.agentRun?.commandState, result.agentRun?.status, result.agentRun?.runStatus); - if (!isTraceCommandTerminalStatus(terminalStatus)) return null; - if (!agentRun?.runId || !agentRun?.commandId || (!storedSummary && !finalResponse)) return null; - const traceSummary = { - traceId, - source: "agentrun-command-result", - sourceEventCount: numberOrNull(storedSummary?.sourceEventCount ?? storedSummary?.eventCount ?? result.runnerTrace?.eventCount), - renderedRowSummary: storedSummary?.renderedRowSummary ?? null, - noiseEventCount: numberOrNull(storedSummary?.noiseEventCount), - omittedNoiseCount: numberOrNull(storedSummary?.omittedNoiseCount), - terminalStatus: storedSummary?.terminalStatus ?? terminalStatus, - updatedAt: storedSummary?.updatedAt ?? result.updatedAt ?? null, - valuesPrinted: false - }; - return { - source: "agentrun-command-result", - conversationId: safeConversationId(result.conversationId) || null, - sessionId: safeSessionId(result.sessionId) || null, - threadId: safeOpaqueId(result.threadId) || null, - ...codeAgentPromptFields(promptEvidence), - agentRun, - finalResponse, - traceSummary - }; -} - -function agentRunTerminalFinalResponse(result, traceId) { - const stored = result.finalResponse && typeof result.finalResponse === "object" ? result.finalResponse : null; - const text = textValue(stored?.text ?? result.reply?.content ?? result.message?.content ?? result.assistantText); - if (!text) return null; - const storedTraceId = safeTraceId(stored?.traceId ?? result.traceId) || null; - if (storedTraceId && storedTraceId !== traceId) return null; - return { - text, - textChars: text.length, - messageId: stored?.messageId ?? result.reply?.messageId ?? result.messageId ?? null, - role: stored?.role ?? result.reply?.role ?? "assistant", - status: stored?.status ?? result.status ?? null, - traceId: traceId, - createdAt: stored?.createdAt ?? result.reply?.createdAt ?? result.createdAt ?? null, - updatedAt: stored?.updatedAt ?? result.updatedAt ?? null, - source: "agentrun-command-result", - valuesPrinted: false - }; -} - -function traceRetentionSummary(status) { - return { - traceStatus: status, - liveTraceStore: status === "missing" ? "missing" : "expired-or-evicted", - policy: "live trace events are short-term storage; terminal final response and summary are resolved from AgentRun command result by traceId and commandId", - replacementEvidence: status === "missing" ? "resolve the trace through AgentRun command registry/result or report the trace as missing" : "use terminalEvidence.finalResponse, terminalEvidence.traceSummary, conversationId, sessionId, threadId, and agentRun ids", - valuesPrinted: false - }; -} - -function numberOrNull(value) { - const parsed = Number(value); - return Number.isFinite(parsed) ? parsed : null; -} - -function sendTraceSse(response, traceStore, traceId, options = {}) { - response.writeHead(200, { - "content-type": "text/event-stream; charset=utf-8", - "cache-control": "no-store", - connection: "keep-alive", - "x-accel-buffering": "no" - }); - const writeEvent = (eventName, payload) => { - response.write(`event: ${eventName}\n`); - response.write(`data: ${JSON.stringify(payload)}\n\n`); - }; - writeEvent("snapshot", traceStore.snapshot(traceId)); - const unsubscribe = traceStore.subscribe(traceId, (event, snapshot) => { - writeEvent("runnerTrace", { event, snapshot }); - }); - const heartbeatMs = parsePositiveInteger(options.env?.HWLAB_CODE_AGENT_TRACE_HEARTBEAT_MS, 15000); - const heartbeat = setInterval(() => { - writeEvent("heartbeat", traceStore.snapshot(traceId)); - }, heartbeatMs); - response.on("close", () => { - clearInterval(heartbeat); - unsubscribe(); - }); -} - -export function createCodeAgentChatResultStore({ maxResults = 256 } = {}) { - const limit = positiveInteger(maxResults, 256); - const results = new Map(); - function set(traceId, payload) { - const id = safeTraceId(traceId); - if (!id) return; - results.set(id, { - ...payload, - traceId: id, - cachedAt: new Date().toISOString() - }); - while (results.size > limit) { - const oldest = results.keys().next().value; - if (!oldest) break; - results.delete(oldest); - } - } - return { - set, - get: (traceId) => results.get(safeTraceId(traceId)) ?? null, - clear: () => results.clear() - }; -} - -function compactCodeAgentChatResultPayload(payload, options = {}) { - if (!payload || typeof payload !== "object") return payload; - const limit = resultTraceEventLimit(options); - const terminalEvidence = agentRunTerminalTraceEvidence(payload, payload.traceId); - const { userBillingReservation, ...publicPayload } = payload; - return { - ...publicPayload, - ...codeAgentTurnLifecycleFields(publicPayload.traceId, publicPayload), - ...(terminalEvidence ? { terminalEvidence: terminalEvidencePayload(terminalEvidence) } : {}), - ...(publicPayload.runnerTrace && typeof publicPayload.runnerTrace === "object" - ? { runnerTrace: compactRunnerTraceForResult(publicPayload.runnerTrace, limit) } - : {}) - }; -} - -function terminalEvidencePayload(evidence) { - return { - available: true, - source: evidence.source, - conversationId: evidence.conversationId, - sessionId: evidence.sessionId, - threadId: evidence.threadId, - ...codeAgentPromptFields(evidence), - agentRun: evidence.agentRun, - traceSummary: evidence.traceSummary, - finalResponse: evidence.finalResponse, - valuesPrinted: false - }; -} - -function compactRunnerTraceForResult(runnerTrace, limit = DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT) { - if (!runnerTrace || typeof runnerTrace !== "object") return runnerTrace; - const events = Array.isArray(runnerTrace.events) ? runnerTrace.events : []; - const total = Number.isInteger(runnerTrace.eventCount) ? runnerTrace.eventCount : events.length; - const effectiveLimit = Math.max(8, positiveInteger(limit, DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT)); - if (events.length <= effectiveLimit) { - return { - ...runnerTrace, - eventCount: total, - eventsCompacted: false - }; - } - const headCount = Math.min(24, Math.max(1, Math.floor(effectiveLimit / 4))); - const tailCount = Math.max(1, effectiveLimit - headCount - 1); - const omitted = Math.max(0, events.length - headCount - tailCount); - const head = events.slice(0, headCount); - const tail = events.slice(events.length - tailCount); - const marker = { - traceId: runnerTrace.traceId ?? head[0]?.traceId ?? tail[0]?.traceId ?? null, - type: "trace", - stage: "trace", - status: "truncated", - label: "trace:compacted", - createdAt: tail[0]?.createdAt ?? runnerTrace.updatedAt ?? new Date().toISOString(), - message: `Result polling response compacted ${omitted} trace events; fetch /v1/agent/traces/${encodeURIComponent(runnerTrace.traceId ?? "")} for the full trace.`, - omittedEventCount: omitted, - retainedEventCount: head.length + tail.length, - totalEventCount: total, - valuesPrinted: false - }; - const compactEvents = [...head, marker, ...tail]; - return { - ...runnerTrace, - eventCount: total, - events: compactEvents, - eventLabels: compactEvents.map(traceEventLabel).filter(Boolean), - eventsCompacted: true, - eventWindow: { - mode: "head-tail", - retained: compactEvents.length, - omitted, - total - } - }; -} - -function traceEventLabel(event) { - return typeof event === "string" ? event : event?.label; -} - -function resultTraceEventLimit(options = {}) { - return positiveInteger( - options.env?.HWLAB_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT, - DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT - ); -} - -function createCodeAgentM3HwlabApiRequestJson(options = {}) { - return async (targetUrl, request = {}) => { - const url = new URL(targetUrl); - const route = url.pathname; - if (route === M3_IO_CONTROL_ROUTE && request.method === "POST") { - return { - ok: true, - status: 200, - body: await handleM3IoControl(request.body ?? {}, { - runtimeStore: options.runtimeStore, - now: options.now, - env: options.env, - requestJson: options.m3IoRequestJson - }) - }; - } - if (route === M3_STATUS_ROUTE && request.method === "GET") { - return { - ok: true, - status: 200, - body: await describeM3StatusLive(options) - }; - } - return { - ok: false, - status: 404, - body: { - error: { - code: "not_found", - message: `Unsupported Code Agent M3 HWLAB API route ${route}` - } - }, - error: `Unsupported Code Agent M3 HWLAB API route ${route}` - }; - }; -} -function recordCodeAgentProjectionMetrics(options = {}, { result = null, trace = null, projection = null, refreshError = null } = {}) { - const performanceStore = options.backendPerformanceStore ?? options.backendPerformance; - if (!performanceStore?.recordWorkbenchProjection || !projection) return; - const sourceLatestSeq = sourceLatestSeqFromResult(result); - const lastProjectedSeq = Number(projection.lastProjectedSeq ?? 0); - if (!Number.isFinite(sourceLatestSeq) && !Number.isFinite(lastProjectedSeq)) return; - performanceStore.recordWorkbenchProjection({ - sourceLatestSeq: Number.isFinite(sourceLatestSeq) ? sourceLatestSeq : lastProjectedSeq, - lastProjectedSeq: Number.isFinite(lastProjectedSeq) ? lastProjectedSeq : 0, - sourceLatestEventAt: sourceLatestAtFromResult(result, trace), - projectedLatestEventAt: projection.updatedAt ?? trace?.updatedAt ?? result?.updatedAt ?? null, - terminalSourceAt: terminalSourceAtFromResult(result), - terminalProjectedAt: projection.projectionStatus === "caught-up" ? projection.updatedAt ?? trace?.updatedAt ?? result?.updatedAt ?? null : null, - status: projection?.status ?? "unknown", - reason: projectionReason(projection, refreshError), - projectionStatus: projection.projectionStatus ?? "unknown", - source: "workbench_read_model" - }); -} - -function recordCodeAgentTurnReadMetric(options = {}, url, statusCode, body = {}, startedAt = nowMs()) { - const performanceStore = options.backendPerformanceStore ?? options.backendPerformance; - if (!performanceStore?.recordWorkbenchTurnRead) return; - performanceStore.recordWorkbenchTurnRead({ - route: url?.pathname ?? "/v1/agent/turns/:id", - status: body?.status ?? statusClassLabel(statusCode), - degradedReason: turnReadDegradedReason(body), - durationMs: nowMs() - startedAt, - responseBytes: responseBodyBytes(body), - timedOut: statusCode === 504 || body?.error?.code === "timeout" || body?.refreshError?.code === "agentrun_timeout" - }); -} - -function turnReadDegradedReason(body = {}) { - if (body?.refreshError?.code) return body.refreshError.code; - if (body?.projection?.blocker?.code) return body.projection.blocker.code; - if (body?.blocker?.code) return body.blocker.code; - if (body?.error?.code) return body.error.code; - if (body?.projectionStatus === "projecting") return "projecting"; - if (body?.projectionStatus === "blocked") return "projection_blocked"; - return "none"; -} - -function responseBodyBytes(body = {}) { - try { return Buffer.byteLength(JSON.stringify(body)); } catch { return 0; } -} - -function statusClassLabel(value) { - const status = Number(value); - if (!Number.isFinite(status) || status <= 0) return "unknown"; - return `${Math.floor(status / 100)}xx`; -} - -function sourceLatestSeqFromResult(result = null) { - const candidates = [ - result?.agentRun?.lastSeq, - result?.traceSummary?.agentRun?.lastSeq, - result?.traceSummary?.lastSeq, - result?.providerTrace?.lastSeq, - result?.runnerTrace?.lastEvent?.sourceSeq, - result?.runnerTrace?.lastEvent?.seq, - result?.runnerTrace?.eventCount - ]; - for (const value of candidates) { - const seq = Number(value); - if (Number.isFinite(seq) && seq >= 0) return Math.floor(seq); - } - return NaN; -} - -function sourceLatestAtFromResult(result = null, trace = null) { - return result?.traceSummary?.updatedAt - ?? result?.agentRun?.updatedAt - ?? result?.providerTrace?.updatedAt - ?? result?.runnerTrace?.updatedAt - ?? trace?.updatedAt - ?? result?.updatedAt - ?? null; -} - -function terminalSourceAtFromResult(result = null) { - if (!result) return null; - const terminalStatus = result?.agentRun?.terminalStatus ?? result?.traceSummary?.terminalStatus ?? result?.status; - if (!isTraceCommandTerminalStatus(terminalStatus)) return null; - return result?.traceSummary?.updatedAt ?? result?.agentRun?.updatedAt ?? result?.updatedAt ?? null; -} - -function projectionReason(projection = null, refreshError = null) { - if (refreshError) return refreshError.code ?? "refresh_error"; - if (projection?.blocker) return projection.blocker.code ?? "projection_blocked"; - if (projection?.projectionStatus === "projecting") return "projecting"; - if (projection?.projectionStatus === "caught-up") return "none"; - return projection?.projectionStatus ?? "unknown"; -} - -function nowMs() { - if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now(); - return Date.now(); -} - -function uniqueStrings(values) { - return [...new Set(values.map((value) => String(value ?? "").trim()).filter(Boolean))]; -} +export { handleCodeAgentChatHttp, handleCodeAgentSessionsHttp } from "./server-code-agent-admission-http.ts"; +export { codeAgentTurnStatusPayload, handleCodeAgentCancelHttp, handleCodeAgentChatResultHttp, handleCodeAgentInspectHttp, handleCodeAgentTurnHttp } from "./server-code-agent-turn-http.ts"; +export { handleCodeAgentSteerHttp, handleCodeAgentTraceHttp } from "./server-code-agent-trace-http.ts"; +export { createCodeAgentChatResultStore } from "./server-code-agent-http-support.ts"; diff --git a/internal/cloud/server-code-agent-trace-http.ts b/internal/cloud/server-code-agent-trace-http.ts new file mode 100644 index 00000000..5ab319f9 --- /dev/null +++ b/internal/cloud/server-code-agent-trace-http.ts @@ -0,0 +1,520 @@ +// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-0102 Agent编排 draft-2026-06-17-r0; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. +// Responsibility: Code Agent HTTP steer and trace resources. + +import { createHash, randomUUID } from "node:crypto"; + +import { M3_IO_CONTROL_ROUTE, M3_STATUS_ROUTE, describeM3StatusLive, handleM3IoControl } from "./m3-io-control.ts"; +import { + agentRunSessionEvidence, + cancelAgentRunChatTurn, + codeAgentAgentRunAdapterEnabled, + initialAgentRunChatResult, + loadPersistedAgentRunResult, + steerAgentRunChatTurn, + submitAgentRunChatTurn +} from "./code-agent-agentrun-adapter.ts"; +import { createCodeAgentErrorPayload, handleCodeAgentChat } from "./code-agent-chat.ts"; +import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; +import { codeAgentSessionLifecycleSummary } from "./code-agent-session-lifecycle.ts"; +import { messageAuthorityTextValue } from "./code-agent-agentrun-prompt.ts"; +import { createWorkbenchReadModel } from "./workbench-read-model.ts"; +import { codeAgentOtelTraceFields, emitCodeAgentOtelSpan } from "./otel-trace.ts"; +import { + firstHeaderValue, + getHeader, + parsePositiveInteger, + positiveInteger, + readBody, + safeConversationId, + safeOpaqueId, + safeSessionId, + safeTraceId, + sendJson, + truthyFlag +} from "./server-http-utils.ts"; + +import { + normalizeCodeAgentProviderProfile, + firstNonEmptyValue, + recordCodeAgentSessionInputFact, + stableCodeAgentInputId, + codeAgentInputDelivery, + codeAgentInputStatus, + codeAgentResultTimingFields, + firstTimestampIso, + firstLatestTimestampIso, + codeAgentChatShortConnectionRequested, + preflightCodeAgentBilling, + recordCodeAgentBillingUsage, + finalizeCodeAgentBillingUsage, + releaseCodeAgentBillingReservation, + withCodeAgentBillingReservation, + codeAgentBillingEnabled, + codeAgentBillingMetadata, + sanitizeBillingReservation, + sanitizeBillingRecord, + sanitizeBillingRelease, + codeAgentUsedTokens, + normalizeTurnStatus, + recordCodeAgentConversationFact, + recordCodeAgentSessionOwner, + codeAgentOwnerStatusForResult, + textValue, + timestampIsoOrNow, + timestampIsoOrNull, + codeAgentSessionOwnerEvidence, + codeAgentContinuationEvidence, + codeAgentFreshContinuationFailure, + normalizeCodeAgentFailureKind, + codeAgentFailureRequiresFreshContinuation, + codeAgentProviderProfileEvidence, + codeAgentTraceResultEvidence, + codeAgentPromptMetadata, + codeAgentPromptMetadataFromText, + codeAgentPromptMetadataFromParts, + codeAgentPromptFields, + codeAgentPromptId, + clippedPromptText, + sha256Text, + compactCodeAgentObject, + codeAgentBillingReservationEvidence, + codeAgentConversationMessagesEvidence, + boundedConversationMessageText, + conversationText, + conversationTextRaw, + codeAgentMessageTraceSuffix, + codeAgentTurnLifecycleFields, + codeAgentLifecycleMessageId, + safeTurnId, + safeMessageId, + codeAgentConversationAgentMessageStatus, + codeAgentPayloadHasSealedFinalResponse, + codeAgentConversationRunnerTraceEvidence, + codeAgentFinalResponseEvidence, + codeAgentFinalResponseText, + codeAgentTraceSummaryEvidence, + annotateOwner, + canAccessOwnedResult, + isCodeAgentResultCanceled, + codeAgentCancelScopeMismatch, + cancelExpectedValues, + cancelScopeFieldMismatch, + cancelBlockedPayload, + isTraceCommandTerminalStatus, + traceSnapshotWithTerminalEvidence, + traceSnapshotWithAttachedTerminalEvidence, + agentRunTerminalTraceEvidence, + agentRunTerminalFinalResponse, + traceRetentionSummary, + numberOrNull, + createCodeAgentChatResultStore, + compactCodeAgentChatResultPayload, + terminalEvidencePayload, + compactRunnerTraceForResult, + traceEventLabel, + resultTraceEventLimit, + createCodeAgentM3HwlabApiRequestJson, + recordCodeAgentProjectionMetrics, + recordCodeAgentTurnReadMetric, + turnReadDegradedReason, + responseBodyBytes, + statusClassLabel, + sourceLatestSeqFromResult, + sourceLatestAtFromResult, + terminalSourceAtFromResult, + projectionReason, + nowMs, + uniqueStrings +} from "./server-code-agent-http-support.ts"; + +import { + codeAgentCompatProjectionPayload, + codeAgentRequestScopedOptions, + measureCodeAgentHttpPhase, + readCodeAgentCompatProjection, + traceProjectMismatchSnapshot +} from "./server-code-agent-turn-http.ts"; + +const DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT = 120; +const DEFAULT_CODE_AGENT_TRACE_PAGE_LIMIT = 100; +const MAX_CODE_AGENT_TRACE_PAGE_LIMIT = 100; +const DEFAULT_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS = 2500; +const DEFAULT_CODE_AGENT_PROJECT_ID = "prj_hwpod_workbench"; +const DEFAULT_CODE_AGENT_PROVIDER_PROFILE = "deepseek"; +const CODE_AGENT_TERMINAL_STATUSES = new Set(["completed", "failed", "blocked", "timeout", "cancelled", "canceled"]); +const CODE_AGENT_PROVIDER_PROFILE_ALIASES = Object.freeze({ codex: "codex-api" }); +const CODE_AGENT_PROVIDER_PROFILE_ID_PATTERN = /^[a-z0-9][a-z0-9.-]{0,63}$/u; +const CODE_AGENT_PROVIDER_PROFILE_LABELS = Object.freeze({ + "deepseek": "DeepSeek", + "codex-api": "Codex API", + "gpt.pika": "gpt.pika", + "minimax-m3": "MiniMax-M3 via AgentRun", + "runtime-default": "运行默认" +}); +const DEFAULT_CODE_AGENT_CODEX_API_MODEL = "gpt-5.5"; +const DEFAULT_CODE_AGENT_CODEX_API_BASE_URL = "http://127.0.0.1:49280/responses"; +const DEFAULT_CODE_AGENT_DEEPSEEK_MODEL = "deepseek-chat"; +const DEFAULT_CODE_AGENT_MINIMAX_M3_MODEL = "MiniMax-M3"; + +export async function handleCodeAgentSteerHttp(request, response, options) { + const body = await readBody(request, options.bodyLimitBytes); + let params = {}; + + try { + params = body ? JSON.parse(body) : {}; + } catch (error) { + sendJson(response, 400, { + ...createCodeAgentErrorPayload({ + code: "parse_error", + message: "Invalid JSON body", + reason: error.message, + traceId: getHeader(request, "x-trace-id") || "trc_unassigned", + layer: "api", + retryable: true, + route: "/v1/agent/chat/steer" + }) + }); + return; + } + + if (!params || typeof params !== "object" || Array.isArray(params)) { + sendJson(response, 400, { + ...createCodeAgentErrorPayload({ + code: "invalid_params", + message: "Code Agent steer body must be a JSON object", + traceId: getHeader(request, "x-trace-id") || "trc_unassigned", + layer: "api", + retryable: true, + route: "/v1/agent/chat/steer" + }) + }); + return; + } + + const traceId = safeTraceId(getHeader(request, "x-trace-id") || params.traceId || params.targetTraceId); + const message = firstNonEmptyValue(params.message, params.prompt, params.text); + if (!traceId || !message) { + sendJson(response, 400, { + ...createCodeAgentErrorPayload({ + code: !traceId ? "steer_trace_missing" : "steer_message_missing", + message: !traceId ? "traceId is required to steer the current Code Agent turn." : "message is required to steer the current Code Agent turn.", + traceId: traceId || "trc_unassigned", + layer: "api", + retryable: true, + route: "/v1/agent/chat/steer" + }) + }); + return; + } + + const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; + const durableResult = await loadPersistedAgentRunResult(traceId, options); + const currentResult = durableResult ?? options.codeAgentChatResults?.get?.(traceId); + const projectionContext = await readCodeAgentCompatProjection(traceId, options); + if (!canAccessOwnedResult(currentResult, options.actor)) { + sendJson(response, 403, { + ok: false, + status: "forbidden", + traceId, + error: { code: "code_agent_trace_forbidden", message: "Code Agent trace is not visible to the current actor" } + }); + return; + } + const projectionStoreBlocker = codeAgentProjectionStoreBlocker(projectionContext); + if (projectionStoreBlocker) { + sendJson(response, 503, { + ok: false, + accepted: false, + status: "blocked", + traceId, + route: "/v1/agent/chat/steer", + error: { + ...projectionStoreBlocker, + code: "projection_store_unavailable", + layer: projectionStoreBlocker.layer ?? "runtime-store", + category: projectionStoreBlocker.category ?? "projection-store-unavailable", + retryable: true, + message: projectionStoreBlocker.message ?? "Workbench projection store is unavailable; steer is blocked until terminal authority can be checked.", + traceId, + route: "/v1/agent/chat/steer" + }, + valuesPrinted: false + }); + return; + } + if (!currentResult?.agentRun?.runId || !currentResult?.agentRun?.commandId) { + sendJson(response, 404, { + ok: false, + status: "not_found", + traceId, + error: { code: "steer_trace_not_found", message: "No AgentRun-backed Code Agent turn was found for the requested traceId." }, + route: "/v1/agent/chat/steer", + runnerTrace: traceStore.snapshot(traceId) + }); + return; + } + const projectedTerminalStatus = normalizeTurnStatus( + projectionContext.trace?.terminalEvidence?.status, + projectionContext.trace?.status + ); + if (isTraceCommandTerminalStatus(projectedTerminalStatus)) { + sendJson(response, 409, { + ok: false, + accepted: false, + status: "blocked", + traceId, + route: "/v1/agent/chat/steer", + sessionId: safeSessionId(currentResult.sessionId ?? params.sessionId) || null, + threadId: safeOpaqueId(currentResult.threadId ?? params.threadId) || null, + agentRun: currentResult.agentRun, + runnerTrace: traceStore.snapshot(traceId), + error: { + code: "steer_trace_terminal", + layer: "api", + category: "not_in_flight", + retryable: false, + message: `Trace ${traceId} is already terminal and cannot accept a steer command.`, + traceId, + route: "/v1/agent/chat/steer", + toolName: "agentrun.command.steer" + }, + valuesPrinted: false + }); + return; + } + + try { + await recordCodeAgentSessionInputFact({ + params: { + ...params, + sessionId: currentResult.sessionId ?? currentResult.session?.sessionId ?? params.sessionId, + ownerUserId: options.actor?.id, + ownerRole: options.actor?.role + }, + options, + traceId, + delivery: "steer", + status: "promoted", + commandId: currentResult.agentRun.commandId + }); + const payload = await steerAgentRunChatTurn({ + traceId, + currentResult, + params: { + ...params, + message, + ownerUserId: options.actor?.id ?? params.ownerUserId, + ownerRole: options.actor?.role ?? params.ownerRole + }, + options, + traceStore + }); + sendJson(response, 202, payload); + } catch (error) { + traceStore.append(traceId, { + type: "backend", + status: "failed", + label: "agentrun:steer:failed", + errorCode: error?.code ?? "agentrun_steer_failed", + message: error?.message ?? "AgentRun steer command failed.", + terminal: false, + valuesPrinted: false + }); + sendJson(response, error?.statusCode === 404 ? 404 : 409, { + ok: false, + accepted: false, + status: "failed", + traceId, + route: "/v1/agent/chat/steer", + sessionId: safeSessionId(currentResult.sessionId ?? params.sessionId) || null, + threadId: safeOpaqueId(currentResult.threadId ?? params.threadId) || null, + agentRun: currentResult.agentRun, + runnerTrace: traceStore.snapshot(traceId), + error: { + code: error?.code ?? "agentrun_steer_failed", + layer: "agentrun", + category: "steer_failed", + retryable: true, + message: error?.message ?? "AgentRun steer command failed.", + traceId, + route: "/v1/agent/chat/steer", + toolName: "agentrun.command.steer" + } + }); + } +} + +export async function handleCodeAgentTraceHttp(request, response, url, options) { + const startedAt = nowMs(); + const parts = url.pathname.split("/").filter(Boolean); + const traceId = decodeURIComponent(parts[3] ?? ""); + const streamSegment = parts[4]; + const pageOptions = codeAgentTracePageOptions(url); + if (!/^trc_[A-Za-z0-9_.:-]+$/u.test(traceId)) { + sendJson(response, 400, { + error: { + code: "invalid_trace_id", + message: "traceId must start with trc_ and contain only safe identifier characters" + } + }); + return; + } + const requestOptions = codeAgentRequestScopedOptions(options); + const projectMismatch = await measureCodeAgentHttpPhase(requestOptions, "trace_project_check", () => traceProjectMismatchSnapshot(traceId, url, requestOptions, "trace")); + if (projectMismatch) { + sendJson(response, projectMismatch.statusCode, projectMismatch.body); + return; + } + const context = await measureCodeAgentHttpPhase(requestOptions, "trace_projection_read", () => readCodeAgentCompatProjection(traceId, requestOptions)); + if (context.statusCode) { + sendJson(response, context.statusCode, context.body); + return; + } + + if (streamSegment === "stream") { + sendJson(response, 410, { + ok: false, + status: "removed", + traceId, + error: { + code: "trace_store_sse_removed", + message: "Trace-store SSE was removed; use the committed Workbench projection event stream.", + workbenchEventsUrl: `/v1/workbench/events?traceId=${encodeURIComponent(traceId)}` + }, + valuesRedacted: true + }); + return; + } + + const diagnosticTrace = (requestOptions.traceStore ?? defaultCodeAgentTraceStore).snapshot(traceId); + const trace = mergeCodeAgentDiagnosticTrace(context.trace, diagnosticTrace, traceId); + const diagnosticFound = trace.events.length > 0; + const body = await measureCodeAgentHttpPhase(requestOptions, "trace_paginate", () => { + return paginateTraceSnapshot(trace, pageOptions); + }); + const statusCode = context.found || diagnosticFound ? 200 : 404; + void emitCodeAgentOtelSpan("trace_events_read", traceId, requestOptions.env ?? process.env, { + startTimeMs: startedAt, + attributes: { + "http.method": "GET", + "http.route": "/v1/agent/traces/:traceId", + "http.status_code": statusCode, + found: Boolean(context.found), + returnedEvents: Array.isArray(body?.events) ? body.events.length : 0, + sinceSeq: pageOptions.sinceSeq, + limit: pageOptions.limit, + fromSeq: body?.range?.fromSeq ?? null, + toSeq: body?.range?.toSeq ?? null, + totalEvents: body?.range?.total ?? body?.eventCount ?? null, + hasMore: Boolean(body?.hasMore), + fullTraceLoaded: Boolean(body?.fullTraceLoaded) + } + }); + sendJson(response, statusCode, codeAgentCompatProjectionPayload(body, context)); +} + +function mergeCodeAgentDiagnosticTrace(projectionTrace, diagnosticTrace, traceId) { + const projected = projectionTrace && typeof projectionTrace === "object" ? projectionTrace : {}; + const diagnostics = diagnosticTrace && typeof diagnosticTrace === "object" ? diagnosticTrace : {}; + const merged = new Map(); + for (const [source, events] of [["diagnostic", diagnostics.events], ["projection", projected.events]]) { + for (const [index, event] of (Array.isArray(events) ? events : []).entries()) { + const originalSeq = traceEventSequence(event, index); + const key = textValue(event?.id ?? event?.sourceEventId) || `${originalSeq}:${textValue(event?.label ?? event?.type)}:${textValue(event?.createdAt ?? event?.occurredAt)}`; + const normalized = source === "projection" + ? { + ...event, + projectedSeq: Number.isFinite(Number(event?.projectedSeq)) ? Number(event.projectedSeq) : originalSeq, + authority: "workbench-projection" + } + : { + ...event, + sourceSeq: Number.isFinite(Number(event?.sourceSeq)) ? Number(event.sourceSeq) : originalSeq, + terminal: false, + sealed: false, + diagnosticOnly: true, + authority: "trace-store-diagnostics" + }; + merged.set(key, { event: normalized, source, originalSeq, sourceIndex: index, key }); + } + } + const events = [...merged.values()] + .sort((left, right) => left.originalSeq - right.originalSeq + || (left.source === right.source ? 0 : left.source === "projection" ? -1 : 1) + || left.sourceIndex - right.sourceIndex + || left.key.localeCompare(right.key)) + .map(({ event }, index) => ({ ...event, seq: index + 1, paginationSeq: index + 1 })); + const projectionFound = projected.status && projected.status !== "missing"; + return { + ...projected, + traceId, + status: projectionFound ? projected.status : events.length > 0 ? "running" : "missing", + terminal: projected.terminal === true, + sealed: projected.sealed === true, + events, + eventCount: events.length, + diagnosticEventCount: events.filter((event) => event.diagnosticOnly === true).length, + terminalAuthority: "workbench-projection", + valuesRedacted: true + }; +} + +function codeAgentProjectionStoreBlocker(context = {}) { + const candidates = [ + context?.projection?.blocker, + context?.trace?.projection?.blocker, + context?.trace?.blocker, + context?.result?.projection?.blocker, + context?.result?.blocker + ]; + return candidates.find((blocker) => blocker?.code === "projection_store_unavailable") ?? null; +} + +function codeAgentTracePageOptions(url) { + const sinceSeq = nonNegativeInteger(url.searchParams.get("sinceSeq") ?? url.searchParams.get("since"), 0); + const requestedLimit = parsePositiveInteger(url.searchParams.get("limit"), DEFAULT_CODE_AGENT_TRACE_PAGE_LIMIT); + return { + sinceSeq, + limit: Math.min(requestedLimit, MAX_CODE_AGENT_TRACE_PAGE_LIMIT) + }; +} + +function paginateTraceSnapshot(snapshot, { sinceSeq, limit }) { + const sourceEvents = Array.isArray(snapshot?.events) ? snapshot.events : []; + const indexedEvents = sourceEvents.map((event, index) => ({ event, seq: traceEventSequence(event, index) })); + const firstIndex = indexedEvents.findIndex((item) => item.seq > sinceSeq); + const startIndex = firstIndex >= 0 ? firstIndex : indexedEvents.length; + const pageItems = indexedEvents.slice(startIndex, startIndex + limit); + const events = pageItems.map((item) => item.event); + const fromSeq = pageItems.length > 0 ? pageItems[0].seq : null; + const toSeq = pageItems.length > 0 ? pageItems[pageItems.length - 1].seq : null; + const hasMore = startIndex + pageItems.length < indexedEvents.length; + const totalEvents = snapshot?.eventCount ?? indexedEvents.length; + return { + ...snapshot, + events, + eventCount: totalEvents, + range: { + sinceSeq, + fromSeq, + toSeq, + limit, + returned: events.length, + total: totalEvents + }, + truncated: hasMore, + hasMore, + nextSinceSeq: pageItems.length > 0 ? toSeq : sinceSeq, + fullTraceLoaded: !hasMore + }; +} + +function traceEventSequence(event, index) { + const seq = Number(event?.seq); + return Number.isFinite(seq) && seq > 0 ? Math.trunc(seq) : index + 1; +} + +function nonNegativeInteger(value, fallback) { + const parsed = Number.parseInt(value ?? "", 10); + return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback; +} diff --git a/internal/cloud/server-code-agent-turn-http.ts b/internal/cloud/server-code-agent-turn-http.ts new file mode 100644 index 00000000..399cba68 --- /dev/null +++ b/internal/cloud/server-code-agent-turn-http.ts @@ -0,0 +1,921 @@ +// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-0102 Agent编排 draft-2026-06-17-r0; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. +// Responsibility: Code Agent HTTP result, turn, inspect, and cancel resources. + +import { createHash, randomUUID } from "node:crypto"; + +import { M3_IO_CONTROL_ROUTE, M3_STATUS_ROUTE, describeM3StatusLive, handleM3IoControl } from "./m3-io-control.ts"; +import { + agentRunSessionEvidence, + cancelAgentRunChatTurn, + codeAgentAgentRunAdapterEnabled, + initialAgentRunChatResult, + loadPersistedAgentRunResult, + steerAgentRunChatTurn, + submitAgentRunChatTurn +} from "./code-agent-agentrun-adapter.ts"; +import { createCodeAgentErrorPayload, handleCodeAgentChat } from "./code-agent-chat.ts"; +import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; +import { codeAgentSessionLifecycleSummary } from "./code-agent-session-lifecycle.ts"; +import { messageAuthorityTextValue } from "./code-agent-agentrun-prompt.ts"; +import { createWorkbenchReadModel } from "./workbench-read-model.ts"; +import { codeAgentOtelTraceFields, emitCodeAgentOtelSpan } from "./otel-trace.ts"; +import { + firstHeaderValue, + getHeader, + parsePositiveInteger, + positiveInteger, + readBody, + safeConversationId, + safeOpaqueId, + safeSessionId, + safeTraceId, + sendJson, + truthyFlag +} from "./server-http-utils.ts"; + +import { + normalizeCodeAgentProviderProfile, + firstNonEmptyValue, + recordCodeAgentSessionInputFact, + stableCodeAgentInputId, + codeAgentInputDelivery, + codeAgentInputStatus, + codeAgentResultTimingFields, + firstTimestampIso, + firstLatestTimestampIso, + codeAgentChatShortConnectionRequested, + preflightCodeAgentBilling, + recordCodeAgentBillingUsage, + finalizeCodeAgentBillingUsage, + releaseCodeAgentBillingReservation, + withCodeAgentBillingReservation, + codeAgentBillingEnabled, + codeAgentBillingMetadata, + sanitizeBillingReservation, + sanitizeBillingRecord, + sanitizeBillingRelease, + codeAgentUsedTokens, + normalizeTurnStatus, + recordCodeAgentConversationFact, + recordCodeAgentSessionOwner, + codeAgentOwnerStatusForResult, + textValue, + timestampIsoOrNow, + timestampIsoOrNull, + codeAgentSessionOwnerEvidence, + codeAgentContinuationEvidence, + codeAgentFreshContinuationFailure, + normalizeCodeAgentFailureKind, + codeAgentFailureRequiresFreshContinuation, + codeAgentProviderProfileEvidence, + codeAgentTraceResultEvidence, + codeAgentPromptMetadata, + codeAgentPromptMetadataFromText, + codeAgentPromptMetadataFromParts, + codeAgentPromptFields, + codeAgentPromptId, + clippedPromptText, + sha256Text, + compactCodeAgentObject, + codeAgentBillingReservationEvidence, + codeAgentConversationMessagesEvidence, + boundedConversationMessageText, + conversationText, + conversationTextRaw, + codeAgentMessageTraceSuffix, + codeAgentTurnLifecycleFields, + codeAgentLifecycleMessageId, + safeTurnId, + safeMessageId, + codeAgentConversationAgentMessageStatus, + codeAgentPayloadHasSealedFinalResponse, + codeAgentConversationRunnerTraceEvidence, + codeAgentFinalResponseEvidence, + codeAgentFinalResponseText, + codeAgentTraceSummaryEvidence, + annotateOwner, + canAccessOwnedResult, + isCodeAgentResultCanceled, + codeAgentCancelScopeMismatch, + cancelExpectedValues, + cancelScopeFieldMismatch, + cancelBlockedPayload, + isTraceCommandTerminalStatus, + traceSnapshotWithTerminalEvidence, + traceSnapshotWithAttachedTerminalEvidence, + agentRunTerminalTraceEvidence, + agentRunTerminalFinalResponse, + traceRetentionSummary, + numberOrNull, + createCodeAgentChatResultStore, + compactCodeAgentChatResultPayload, + terminalEvidencePayload, + compactRunnerTraceForResult, + traceEventLabel, + resultTraceEventLimit, + createCodeAgentM3HwlabApiRequestJson, + recordCodeAgentProjectionMetrics, + recordCodeAgentTurnReadMetric, + turnReadDegradedReason, + responseBodyBytes, + statusClassLabel, + sourceLatestSeqFromResult, + sourceLatestAtFromResult, + terminalSourceAtFromResult, + projectionReason, + nowMs, + uniqueStrings +} from "./server-code-agent-http-support.ts"; + +const DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT = 120; +const DEFAULT_CODE_AGENT_TRACE_PAGE_LIMIT = 100; +const MAX_CODE_AGENT_TRACE_PAGE_LIMIT = 100; +const DEFAULT_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS = 2500; +const DEFAULT_CODE_AGENT_PROJECT_ID = "prj_hwpod_workbench"; +const DEFAULT_CODE_AGENT_PROVIDER_PROFILE = "deepseek"; +const CODE_AGENT_TERMINAL_STATUSES = new Set(["completed", "failed", "blocked", "timeout", "cancelled", "canceled"]); +const CODE_AGENT_PROVIDER_PROFILE_ALIASES = Object.freeze({ codex: "codex-api" }); +const CODE_AGENT_PROVIDER_PROFILE_ID_PATTERN = /^[a-z0-9][a-z0-9.-]{0,63}$/u; +const CODE_AGENT_PROVIDER_PROFILE_LABELS = Object.freeze({ + "deepseek": "DeepSeek", + "codex-api": "Codex API", + "gpt.pika": "gpt.pika", + "minimax-m3": "MiniMax-M3 via AgentRun", + "runtime-default": "运行默认" +}); +const DEFAULT_CODE_AGENT_CODEX_API_MODEL = "gpt-5.5"; +const DEFAULT_CODE_AGENT_CODEX_API_BASE_URL = "http://127.0.0.1:49280/responses"; +const DEFAULT_CODE_AGENT_DEEPSEEK_MODEL = "deepseek-chat"; +const DEFAULT_CODE_AGENT_MINIMAX_M3_MODEL = "MiniMax-M3"; + +export async function handleCodeAgentChatResultHttp(request, response, url, options) { + const parts = url.pathname.split("/").filter(Boolean); + const traceId = decodeURIComponent(parts[4] ?? ""); + if (!safeTraceId(traceId)) { + sendJson(response, 400, { + error: { + code: "invalid_trace_id", + message: "traceId must start with trc_ and contain only safe identifier characters" + } + }); + return; + } + const context = await measureCodeAgentHttpPhase(options, "result_projection_read", () => readCodeAgentCompatProjection(traceId, options)); + if (context.statusCode) { + sendJson(response, context.statusCode, context.body); + return; + } + const result = context.result; + if (result && result.status !== "running" && !codeAgentResultUsesProjectionTerminalAuthority(result)) { + sendJson(response, 200, codeAgentCompatProjectionPayload(compactCodeAgentChatResultPayload(result, options), context)); + return; + } + if (context.found) { + const body = codeAgentTurnStatusPayload({ traceId, result, snapshot: context.trace, resultPollError: null, refreshError: context.refreshError ?? null, options }); + const resultPollTerminalReady = codeAgentResultPollTerminalReady(body); + sendJson(response, resultPollTerminalReady ? 200 : 202, codeAgentCompatProjectionPayload({ + accepted: body.ok, + shortConnection: true, + ...body, + waitingFor: resultPollTerminalReady ? body.waitingFor ?? context.trace?.waitingFor ?? "workbench-projection" : "agentrun-result", + ...(body.terminal === true && !resultPollTerminalReady ? { status: "running", running: true, terminal: false } : {}) + }, context)); + return; + } + sendJson(response, 404, { + ...codeAgentCompatProjectionPayload({ traceId, status: "unknown" }, context), + error: { + code: "code_agent_result_not_found", + message: `No Code Agent result is registered for ${traceId}` + } + }); +} + +export async function handleCodeAgentTurnHttp(request, response, url, options) { + const startedAt = nowMs(); + const parts = url.pathname.split("/").filter(Boolean); + const traceId = decodeURIComponent(parts[3] ?? ""); + if (!safeTraceId(traceId)) { + const body = { + ok: false, + status: "unknown", + running: false, + terminal: false, + error: { + code: "invalid_trace_id", + message: "traceId must start with trc_ and contain only safe identifier characters" + } + }; + recordCodeAgentTurnReadMetric(options, url, 400, body, startedAt); + sendJson(response, 400, body); + return; + } + const requestOptions = codeAgentRequestScopedOptions(options); + let completed = false; + const emitTurnStatusReadSpan = (name, attributes = {}, error = null) => { + void emitCodeAgentOtelSpan(name, traceId, requestOptions.env ?? process.env, { + startTimeMs: startedAt, + status: error ? "error" : undefined, + error, + attributes: { + "http.method": "GET", + "http.route": "/v1/agent/turns/:traceId", + durationMs: nowMs() - startedAt, + ...attributes + } + }); + }; + response.once?.("close", () => { + if (completed) return; + emitTurnStatusReadSpan("turn_status_read.client_closed", { + status: "client_closed", + phase: "response_close", + terminal: false + }, new Error("Code Agent turn status client connection closed before response completed.")); + }); + try { + const projectMismatch = await measureCodeAgentHttpPhase(requestOptions, "turn_project_check", () => traceProjectMismatchSnapshot(traceId, url, requestOptions, "turn")); + if (projectMismatch) { + completed = true; + recordCodeAgentTurnReadMetric(options, url, projectMismatch.statusCode, projectMismatch.body, startedAt); + emitTurnStatusReadSpan("turn_status_read", { "http.status_code": projectMismatch.statusCode, status: projectMismatch.body?.status ?? null, terminal: projectMismatch.body?.terminal ?? null, phase: "project_check" }); + sendJson(response, projectMismatch.statusCode, projectMismatch.body); + return; + } + const resolved = await measureCodeAgentHttpPhase(requestOptions, "turn_resolve_status", () => resolveCodeAgentTurnStatusSnapshot(traceId, requestOptions)); + completed = true; + recordCodeAgentTurnReadMetric(options, url, resolved.statusCode, resolved.body, startedAt); + emitTurnStatusReadSpan("turn_status_read", { "http.status_code": resolved.statusCode, status: resolved.body?.status ?? null, terminal: resolved.body?.terminal ?? null, phase: "resolved" }); + sendJson(response, resolved.statusCode, resolved.body); + } catch (error) { + completed = true; + const body = { + ok: false, + status: "unknown", + running: false, + terminal: false, + traceId, + error: { + code: error?.code ?? "turn_status_read_failed", + message: error?.message ?? "Code Agent turn status read failed." + }, + valuesRedacted: true + }; + recordCodeAgentTurnReadMetric(options, url, 500, body, startedAt); + emitTurnStatusReadSpan("turn_status_read", { "http.status_code": 500, status: body.status, terminal: false, phase: "failed", errorCode: body.error.code }, error); + sendJson(response, 500, body); + } +} + +async function resolveCodeAgentTurnStatusSnapshot(traceId, options) { + const context = await readCodeAgentCompatProjection(traceId, options); + if (context.statusCode) return context; + const body = await measureCodeAgentHttpPhase(options, "turn_payload", () => codeAgentTurnStatusPayload({ + traceId, + result: context.result, + snapshot: context.trace, + resultPollError: null, + refreshError: context.refreshError ?? null, + options + })); + return { statusCode: body.ok ? 200 : 404, body: codeAgentCompatProjectionPayload(body, context) }; +} + +function codeAgentRequestScopedOptions(options = {}) { + return { + ...options, + codeAgentTraceSessionCache: options.codeAgentTraceSessionCache instanceof Map ? options.codeAgentTraceSessionCache : new Map() + }; +} + +async function measureCodeAgentHttpPhase(options, phase, callback) { + const perf = options?.backendPerformance; + return perf && typeof perf.measure === "function" ? await perf.measure(phase, callback) : await callback(); +} + +function forbiddenTurnSnapshot(traceId) { + return { + statusCode: 403, + body: { + ok: false, + status: "unknown", + running: false, + terminal: false, + traceId, + error: { + code: "agent_session_owner_required", + message: "Only the session owner or admin can read this Code Agent turn status" + } + } + }; +} + +async function readCodeAgentCompatProjection(traceId, options = {}) { + const readModel = createWorkbenchReadModel(options, options.actor ?? null); + const session = await readModel.getSessionByTraceId(traceId); + let result = readModel.resultForTrace(traceId) ?? projectedSessionTraceResult(session, traceId); + if (result && !canAccessOwnedResult(result, options.actor)) return forbiddenTurnSnapshot(traceId); + if (result?.ownerUserId && !readModel.canReadOwner(result.ownerUserId)) return forbiddenTurnSnapshot(traceId); + const projectionTrace = await readModel.traceSnapshot(traceId); + let trace = traceSnapshotWithTerminalEvidence(projectionTrace, result, traceId, null); + const projection = readModel.projectionDiagnostics({ traceId, result, trace, refreshError: null }); + recordCodeAgentProjectionMetrics(options, { result, trace, projection, refreshError: null }); + const found = Boolean(result || session || (trace && trace.status !== "missing") || trace?.persisted === true); + return { result, session, trace, projection, found, refreshError: null }; +} + +function projectedSessionTraceResult(session, traceId) { + const safeId = safeTraceId(traceId); + if (!session || !safeId) return null; + const snapshot = session.session && typeof session.session === "object" ? session.session : null; + const traceResults = snapshot?.traceResults && typeof snapshot.traceResults === "object" ? snapshot.traceResults : null; + const stored = traceResults?.[safeId] && typeof traceResults[safeId] === "object" ? traceResults[safeId] : null; + if (!stored) return null; + return { + ...stored, + traceId: safeId, + conversationId: safeConversationId(stored.conversationId ?? session.conversationId) || null, + sessionId: safeSessionId(stored.sessionId ?? session.id) || null, + threadId: safeOpaqueId(stored.threadId ?? session.threadId) || null, + ownerUserId: session.ownerUserId ?? stored.ownerUserId ?? null, + ownerRole: session.ownerRole ?? stored.ownerRole ?? null, + status: stored.status ?? session.status ?? null, + valuesRedacted: true + }; +} + +function codeAgentCompatProjectionPayload(payload = {}, context = {}) { + const traceId = safeTraceId(payload.traceId ?? context.trace?.traceId) ?? null; + const projection = context.projection && typeof context.projection === "object" ? context.projection : {}; + return { + ...payload, + projection, + projectionStatus: projection.projectionStatus ?? null, + projectionHealth: projection.projectionHealth ?? null, + lastProjectedSeq: projection.lastProjectedSeq ?? null, + sourceRunId: projection.sourceRunId ?? null, + sourceCommandId: projection.sourceCommandId ?? null, + staleMs: projection.staleMs ?? null, + blocker: projection.blocker ?? null, + workbench: traceId ? { + detailOnly: true, + turnDetailUrl: `/v1/workbench/turns/${encodeURIComponent(traceId)}`, + traceEventsDetailUrl: `/v1/workbench/traces/${encodeURIComponent(traceId)}/events` + } : null, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +function codeAgentResultPollTerminalReady(body = {}) { + if (body?.terminal !== true) return false; + return Boolean(messageAuthorityTextValue(body.finalResponse ?? body.reply ?? body.terminalEvidence?.finalResponse)); +} + +async function traceProjectMismatchSnapshot(traceId, url, options, resourceKind) { + const requestedProjectId = textValue(url.searchParams.get("projectId")); + if (!requestedProjectId || !options.accessController?.getAgentSessionByTraceId) return null; + const session = await getCodeAgentSessionByTraceId(traceId, options); + if (!session) return null; + if (session.ownerUserId && options.actor?.role !== "admin" && session.ownerUserId !== options.actor?.id) return forbiddenTurnSnapshot(traceId); + const actualProjectId = textValue(session.projectId); + if (!actualProjectId || actualProjectId === requestedProjectId) return null; + return { + statusCode: 404, + body: { + ok: false, + status: "not_found", + running: false, + terminal: false, + traceId, + error: { + code: "trace_project_mismatch", + message: `${resourceKind === "turn" ? "Agent turn" : "Agent trace"} is not visible in the requested project` + } + } + }; +} + +async function getCodeAgentSessionByTraceId(traceId, options = {}) { + const safeId = safeTraceId(traceId); + if (!safeId || typeof options.accessController?.getAgentSessionByTraceId !== "function") return null; + const cache = options.codeAgentTraceSessionCache; + if (cache instanceof Map) { + if (cache.has(safeId)) return cache.get(safeId); + const session = await options.accessController.getAgentSessionByTraceId(safeId); + cache.set(safeId, session ?? null); + return session ?? null; + } + return await options.accessController.getAgentSessionByTraceId(safeId); +} + +export function codeAgentTurnStatusPayload({ traceId, result, snapshot, resultPollError, refreshError, options }) { + const resultObject = result && typeof result === "object" ? result : null; + const snapshotObject = snapshot && typeof snapshot === "object" ? snapshot : null; + const projectionTerminalAuthority = codeAgentResultUsesProjectionTerminalAuthority(resultObject); + const lifecycle = codeAgentTurnLifecycleFields(traceId, resultObject ?? snapshotObject ?? {}); + const events = Array.isArray(snapshotObject?.events) ? snapshotObject.events : Array.isArray(resultObject?.runnerTrace?.events) ? resultObject.runnerTrace.events : []; + const lastEvent = events.at(-1) ?? null; + const finalResponse = projectionTerminalAuthority + ? snapshotObject?.finalResponse ?? snapshotObject?.terminalEvidence?.finalResponse ?? codeAgentFinalResponseEvidence(snapshotObject ?? {}, traceId) + : resultObject?.finalResponse ?? snapshotObject?.finalResponse ?? snapshotObject?.terminalEvidence?.finalResponse ?? codeAgentFinalResponseEvidence(resultObject ?? snapshotObject ?? {}, traceId); + const terminalStatus = projectionTerminalAuthority + ? codeAgentProjectionTerminalStatus(snapshotObject) + : codeAgentAuthoritativeTerminalStatus(resultObject, snapshotObject, traceId); + const terminalSealBlocked = Boolean(terminalStatus && isTurnTerminalStatus(terminalStatus) && !codeAgentTerminalFinalText(finalResponse, projectionTerminalAuthority ? null : resultObject, snapshotObject)); + const rawStatus = projectionTerminalAuthority + ? normalizeTurnStatus( + terminalStatus, + codeAgentRunningStatus(snapshotObject?.status), + codeAgentRunningStatus(snapshotObject?.traceStatus), + codeAgentRunningStatus(snapshotObject?.runnerTrace?.status), + codeAgentRunningStatus(resultObject?.status), + "running" + ) + : normalizeTurnStatus( + terminalStatus, + resultObject?.agentRun?.commandState, + resultObject?.agentRun?.status, + resultObject?.agentRun?.runStatus, + codeAgentRunningStatus(snapshotObject?.status), + codeAgentRunningStatus(snapshotObject?.traceStatus), + codeAgentRunningStatus(snapshotObject?.runnerTrace?.status), + codeAgentRunningStatus(resultObject?.status), + resultObject?.agentRun?.terminalStatus, + snapshotObject?.terminalEvidence?.traceSummary?.terminalStatus + ); + const status = terminalSealBlocked ? "running" : rawStatus; + const found = Boolean(resultObject || (snapshotObject && snapshotObject.status !== "missing") || snapshotObject?.persisted === true); + const running = isTurnRunningStatus(status); + const terminal = isTurnTerminalStatus(status); + const runnerTrace = snapshotObject && snapshotObject.status !== "missing" ? snapshotObject : resultObject?.runnerTrace ?? null; + const timingFields = codeAgentResultTimingFields(resultObject, snapshotObject, runnerTrace); + return { + ok: found, + action: "code-agent.turn.status", + contractVersion: "code-agent-turn-status-v1", + status: found ? status ?? "unknown" : "unknown", + running: found ? running : false, + terminal: found ? terminal : false, + traceId, + ...lifecycle, + conversationId: safeConversationId(resultObject?.conversationId ?? snapshotObject?.conversationId) || null, + sessionId: safeSessionId(resultObject?.sessionId ?? resultObject?.session?.sessionId ?? snapshotObject?.sessionId) || null, + threadId: safeOpaqueId(resultObject?.threadId ?? resultObject?.session?.threadId ?? snapshotObject?.threadId) || null, + updatedAt: textValue(resultObject?.updatedAt ?? resultObject?.agentRun?.updatedAt ?? snapshotObject?.updatedAt ?? timingFields.lastEventAt ?? timingFields.startedAt) || null, + ...timingFields, + lastEventLabel: textValue(snapshotObject?.lastEventLabel ?? runnerTrace?.lastEventLabel ?? lastEvent?.label ?? lastEvent?.type) || null, + waitingFor: terminalSealBlocked ? "final_response" : textValue(snapshotObject?.waitingFor ?? runnerTrace?.waitingFor) || null, + terminalObserved: Boolean(terminalStatus), + terminalObservedStatus: terminalStatus ?? null, + terminalSealBlocked, + resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`, + traceUrl: `/v1/agent/traces/${encodeURIComponent(traceId)}`, + turnUrl: `/v1/agent/turns/${encodeURIComponent(traceId)}`, + runnerTrace: runnerTrace ? compactRunnerTraceForResult(runnerTrace, resultTraceEventLimit(options)) : null, + agentRun: resultObject?.agentRun ?? snapshotObject?.agentRun ?? null, + terminalEvidence: snapshotObject?.terminalEvidence ?? null, + finalResponse, + traceSummary: resultObject?.traceSummary ?? snapshotObject?.traceSummary ?? snapshotObject?.terminalEvidence?.traceSummary ?? null, + retention: snapshotObject?.retention ?? null, + eventCount: numberOrNull(snapshotObject?.eventCount ?? runnerTrace?.eventCount ?? events.length), + error: resultPollError || refreshError + ? codeAgentRefreshErrorPayload(resultPollError ?? refreshError, traceId, resultObject?.agentRun ?? snapshotObject?.agentRun, "turn_status_degraded") + : resultObject?.error ?? snapshotObject?.error ?? null, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +function codeAgentResultUsesProjectionTerminalAuthority(result = null) { + const agentRun = result?.agentRun && typeof result.agentRun === "object" ? result.agentRun : null; + if (!agentRun || agentRun.adapter !== "agentrun-v01") return false; + return agentRun.durableDispatch === true + || Boolean(textValue(agentRun.dispatchIntentId ?? agentRun.commandId)); +} + +function codeAgentProjectionTerminalStatus(snapshotObject = null) { + const status = normalizeTurnStatus( + snapshotObject?.terminalEvidence?.status, + snapshotObject?.terminalEvidence?.traceSummary?.terminalStatus, + snapshotObject?.status + ); + return status && isTurnTerminalStatus(status) && codeAgentSnapshotHasTerminalAuthority(snapshotObject) ? status : null; +} + +function codeAgentTerminalFinalText(finalResponse = null, resultObject = null, snapshotObject = null) { + const candidates = [ + finalResponse, + resultObject?.finalResponse, + snapshotObject?.finalResponse, + snapshotObject?.terminalEvidence?.finalResponse, + resultObject?.terminalEvidence?.finalResponse + ]; + for (const value of candidates) { + const text = conversationText(value); + if (text) return text; + } + return codeAgentFinalResponseText(resultObject ?? {}) || codeAgentFinalResponseText(snapshotObject ?? {}); +} + +function codeAgentAuthoritativeTerminalStatus(resultObject, snapshotObject, traceId) { + const sealedPayload = codeAgentPayloadHasSealedFinalResponse(resultObject ?? {}) ? resultObject : codeAgentPayloadHasSealedFinalResponse(snapshotObject ?? {}) ? snapshotObject : null; + if (sealedPayload) { + const sealedStatus = normalizeTurnStatus(sealedPayload?.finalResponse?.status, sealedPayload?.terminalEvidence?.finalResponse?.status, sealedPayload?.agentRun?.terminalStatus, sealedPayload?.agentRun?.status, sealedPayload?.status); + return sealedStatus && isTurnTerminalStatus(sealedStatus) ? sealedStatus : "completed"; + } + const snapshotStatus = normalizeTurnStatus(snapshotObject?.terminalEvidence?.traceSummary?.terminalStatus, snapshotObject?.terminalEvidence?.agentRun?.terminalStatus, snapshotObject?.terminalEvidence?.status, snapshotObject?.status); + if (snapshotStatus && isTurnTerminalStatus(snapshotStatus) && codeAgentSnapshotHasTerminalAuthority(snapshotObject)) return snapshotStatus; + const agentRunStatus = normalizeTurnStatus(resultObject?.agentRun?.terminalStatus, resultObject?.agentRun?.commandState, resultObject?.agentRun?.status, resultObject?.agentRun?.runStatus); + if (agentRunStatus && isTurnTerminalStatus(agentRunStatus)) return agentRunStatus; + const resultStatus = normalizeTurnStatus(resultObject?.status); + if (resultStatus && isTurnTerminalStatus(resultStatus) && codeAgentResultHasTerminalAuthority(resultObject, traceId)) return resultStatus; + return null; +} + +function codeAgentSnapshotHasTerminalAuthority(snapshot = null) { + if (!snapshot || typeof snapshot !== "object") return false; + if (snapshot.terminal === true || snapshot.sealed === true) return true; + if (snapshot.terminalEvidence?.available === true || snapshot.terminalEvidence?.source) return true; + const events = Array.isArray(snapshot.events) ? snapshot.events : []; + return events.some((event) => event?.terminal === true); +} + +function codeAgentResultHasTerminalAuthority(result = null, traceId = null) { + if (!result || typeof result !== "object") return false; + if (result.terminal === true || result.sealed === true) return true; + if (result.error || result.blocker) return true; + if (codeAgentPayloadHasSealedFinalResponse(result)) return true; + if (agentRunTerminalTraceEvidence(result, traceId)) return true; + return Boolean(textValue(result.finishedAt ?? result.completedAt ?? result.endedAt)); +} + +function codeAgentRefreshErrorPayload(error, traceId, agentRun, fallbackCode) { + return { + code: error?.code ?? fallbackCode, + layer: error?.layer ?? "agentrun", + category: error?.category ?? (error?.code === "agentrun_timeout" ? "upstream-timeout" : "upstream-refresh-failed"), + retryable: error?.retryable !== false, + message: error?.message ?? "Code Agent turn status refresh degraded", + traceId, + runId: agentRun?.runId ?? error?.runId ?? null, + commandId: agentRun?.commandId ?? error?.commandId ?? null, + method: error?.method ?? null, + route: error?.route ?? error?.path ?? null, + timeoutMs: numberOrNull(error?.timeoutMs), + timeoutStage: error?.timeoutStage ?? null, + upstreamStatus: numberOrNull(error?.statusCode), + managerHost: error?.managerHost ?? null, + valuesPrinted: false + }; +} + +function isTurnRunningStatus(status) { + return status === "running"; +} + +function codeAgentRunningStatus(value) { + const status = normalizeTurnStatus(value); + return isTurnRunningStatus(status) ? status : null; +} + +function isTurnTerminalStatus(status) { + return CODE_AGENT_TERMINAL_STATUSES.has(String(status ?? "").trim().toLowerCase().replace(/_/gu, "-")); +} + +export async function handleCodeAgentInspectHttp(request, response, url, options) { + const query = { + conversationId: safeConversationId(url.searchParams.get("conversationId")), + sessionId: safeSessionId(url.searchParams.get("sessionId")), + threadId: safeOpaqueId(url.searchParams.get("threadId")), + traceId: safeTraceId(url.searchParams.get("traceId")) + }; + const sessionRegistry = options.sessionRegistry; + const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; + const manager = options.codexStdioManager; + const managerDescription = manager && typeof manager.describe === "function" ? manager.describe() : null; + const managerRecentSessions = Array.isArray(managerDescription?.recentSessions) ? managerDescription.recentSessions : []; + const directManagerSession = query.sessionId && manager && typeof manager.get === "function" + ? manager.get(query.sessionId, { conversationId: query.conversationId }) + : null; + const matchedManagerSession = directManagerSession ?? managerRecentSessions.find((session) => sessionMatchesInspectQuery(session, query)) ?? null; + const registryInspect = sessionRegistry && typeof sessionRegistry.inspect === "function" + ? sessionRegistry.inspect(query) + : { ok: false, status: "unavailable", traceIds: [], session: null, conversationFacts: null }; + const resultEvidence = await codeAgentInspectResultEvidence(query.traceId, options); + const session = matchedManagerSession ?? registryInspect.session ?? resultEvidence.session ?? null; + const conversationFacts = registryInspect.conversationFacts ?? resultEvidence.conversationFacts ?? null; + const requestedRunnerTrace = query.traceId ? traceStore.snapshot(query.traceId) : null; + const requestedTraceFound = Boolean(requestedRunnerTrace && requestedRunnerTrace.status !== "missing"); + const traceIds = uniqueStrings([ + requestedTraceFound ? query.traceId : null, + session?.currentTraceId, + session?.lastTraceId, + conversationFacts?.latestTraceId, + resultEvidence.latestTraceId, + ...(Array.isArray(conversationFacts?.traceIds) ? conversationFacts.traceIds : []), + ...(Array.isArray(registryInspect.traceIds) ? registryInspect.traceIds : []) + ]); + const latestTraceId = traceIds[0] ?? null; + const runnerTrace = latestTraceId + ? latestTraceId === query.traceId && requestedRunnerTrace + ? requestedRunnerTrace + : traceStore.snapshot(latestTraceId) + : null; + const found = Boolean(registryInspect.ok || session || latestTraceId); + sendJson(response, found ? 200 : 404, { + ok: found, + action: "code-agent.chat.inspect", + status: found ? "found" : "not_found", + query, + session, + conversationFacts, + traceIds, + latestTraceId, + traceUrl: latestTraceId ? `/v1/agent/traces/${encodeURIComponent(latestTraceId)}` : null, + resultUrl: latestTraceId ? `/v1/agent/chat/result/${encodeURIComponent(latestTraceId)}` : null, + runnerTrace, + valuesRedacted: true, + secretMaterialStored: false + }); +} + +async function codeAgentInspectResultEvidence(traceId, options = {}) { + if (!safeTraceId(traceId)) return { session: null, conversationFacts: null, latestTraceId: null }; + const cached = options.codeAgentChatResults?.get?.(traceId) ?? null; + const persisted = cached ? null : await loadPersistedAgentRunResult(traceId, options); + const result = cached ?? persisted ?? null; + if (!result || typeof result !== "object") return { session: null, conversationFacts: null, latestTraceId: null }; + const agentRun = result.agentRun && typeof result.agentRun === "object" ? result.agentRun : null; + const resultSession = result.session && typeof result.session === "object" ? result.session : null; + const sessionReuse = result.sessionReuse && typeof result.sessionReuse === "object" ? result.sessionReuse : null; + const providerTrace = result.providerTrace && typeof result.providerTrace === "object" ? result.providerTrace : null; + const sessionId = safeSessionId(result.sessionId ?? resultSession?.sessionId ?? sessionReuse?.sessionId ?? agentRun?.sessionId) || null; + const conversationId = safeConversationId(result.conversationId ?? resultSession?.conversationId ?? sessionReuse?.conversationId ?? agentRun?.conversationId) || null; + const threadId = safeOpaqueId(result.threadId ?? resultSession?.threadId ?? sessionReuse?.threadId ?? providerTrace?.threadId ?? agentRun?.threadId) || null; + const status = textValue(resultSession?.status ?? result.status ?? agentRun?.status) || null; + const updatedAt = textValue(result.updatedAt ?? agentRun?.updatedAt ?? resultSession?.updatedAt) || null; + const session = sessionId || conversationId || threadId ? { + sessionId, + conversationId, + threadId, + status, + lastTraceId: traceId, + source: "code-agent-result", + agentRun: agentRun ? agentRunSessionEvidence(result).agentRun : null, + valuesRedacted: true, + secretMaterialStored: false + } : null; + const conversationFacts = conversationId ? { + conversationId, + sessionId, + threadId, + latestTraceId: traceId, + traceIds: [traceId], + turnCount: 1, + source: "code-agent-result", + latestStatus: status, + updatedAt, + valuesRedacted: true, + secretMaterialStored: false + } : null; + return { session, conversationFacts, latestTraceId: traceId }; +} + +function sessionMatchesInspectQuery(session, query) { + if (!session || typeof session !== "object") return false; + if (query.sessionId && session.sessionId === query.sessionId) return true; + if (query.threadId && session.threadId === query.threadId) return true; + if (query.conversationId && session.conversationId === query.conversationId) return true; + if (query.traceId && (session.currentTraceId === query.traceId || session.lastTraceId === query.traceId)) return true; + return false; +} + +export async function handleCodeAgentCancelHttp(request, response, options) { + const body = await readBody(request, options.bodyLimitBytes); + let params = {}; + + try { + params = body ? JSON.parse(body) : {}; + } catch (error) { + sendJson(response, 400, { + ...createCodeAgentErrorPayload({ + code: "parse_error", + message: "Invalid JSON body", + reason: error.message, + traceId: getHeader(request, "x-trace-id") || "trc_unassigned", + layer: "api", + retryable: true + }) + }); + return; + } + + if (!params || typeof params !== "object" || Array.isArray(params)) { + sendJson(response, 400, { + ...createCodeAgentErrorPayload({ + code: "invalid_params", + message: "Code Agent cancel body must be a JSON object", + traceId: getHeader(request, "x-trace-id") || "trc_unassigned", + layer: "api", + retryable: true + }) + }); + return; + } + + const traceId = safeTraceId(getHeader(request, "x-trace-id") || params.traceId); + const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; + const snapshot = traceId ? traceStore.snapshot(traceId) : null; + const durableResult = traceId ? await loadPersistedAgentRunResult(traceId, options) : null; + const currentResult = durableResult ?? (traceId ? options.codeAgentChatResults?.get(traceId) : null); + if (currentResult?.agentRun?.commandId) { + const mismatch = codeAgentCancelScopeMismatch(params, currentResult); + if (mismatch) { + traceStore.append(traceId, { + type: "cancel", + status: "blocked", + label: "agentrun:cancel:scope_mismatch", + errorCode: "cancel_scope_mismatch", + message: `Cancel request ${mismatch.field} does not match the trace owner; AgentRun cancel was not forwarded.`, + requested: mismatch.requested, + expected: mismatch.expected, + waitingFor: "cancel-scope", + valuesPrinted: false + }); + sendJson(response, 409, cancelBlockedPayload({ + code: "cancel_scope_mismatch", + message: `取消请求的 ${mismatch.field} 与 trace 归属不一致，已拒绝转发 AgentRun cancel，避免误取消其他 session。`, + traceId, + conversationId: safeConversationId(params.conversationId) || safeConversationId(currentResult.conversationId) || null, + sessionId: safeSessionId(params.sessionId) || safeSessionId(currentResult.sessionId) || null, + runnerTrace: traceStore.snapshot(traceId), + status: "blocked" + })); + return; + } + await recordCodeAgentSessionInputFact({ + params: { + ...params, + sessionId: currentResult.sessionId ?? currentResult.session?.sessionId ?? params.sessionId, + messageId: codeAgentTurnLifecycleFields(traceId, currentResult).userMessageId, + ownerUserId: options.actor?.id, + ownerRole: options.actor?.role + }, + options, + traceId, + delivery: "cancel", + status: "promoted", + commandId: currentResult.agentRun.commandId + }); + const payload = await cancelAgentRunChatTurn({ traceId, currentResult, options, traceStore }); + if (payload) { + sendJson(response, 200, payload); + return; + } + } + const sessionId = safeSessionId(params.sessionId) || safeSessionId(snapshot?.sessionId); + const conversationId = safeConversationId(params.conversationId); + const manager = options.codexStdioManager; + + if (!traceId) { + sendJson(response, 400, cancelBlockedPayload({ + code: "cancel_trace_missing", + message: "traceId is required to cancel the current Code Agent request.", + traceId: "trc_unassigned", + conversationId, + sessionId + })); + return; + } + + if (!manager || typeof manager.get !== "function" || typeof manager.cancel !== "function") { + traceStore.append(traceId, { + type: "cancel", + status: "unsupported", + label: "cancel:unsupported", + errorCode: "cancel_unsupported", + message: "Codex stdio cancel/interrupt backend is not available on this runtime.", + waitingFor: "session-control" + }); + const runnerTrace = traceStore.snapshot(traceId); + sendJson(response, 501, cancelBlockedPayload({ + code: "cancel_unsupported", + message: "当前后端不支持 Code Agent interrupt/cancel；本次按 unsupported/degraded 返回，不会静默假装已中断。", + traceId, + conversationId, + sessionId, + runnerTrace, + status: "degraded", + unsupported: true, + degraded: true + })); + return; + } + + if (!sessionId) { + traceStore.append(traceId, { + type: "cancel", + status: "blocked", + label: "cancel:not_cancelable", + errorCode: "cancel_session_missing", + message: "Cancel request did not include a bound Codex stdio sessionId.", + waitingFor: "session-binding" + }); + sendJson(response, 409, cancelBlockedPayload({ + code: "cancel_session_missing", + message: "当前请求尚未暴露可取消的 Codex stdio sessionId；页面不能只隐藏 UI，已保留输入和 trace。", + traceId, + conversationId, + sessionId: null, + runnerTrace: traceStore.snapshot(traceId) + })); + return; + } + + const currentSession = manager.get(sessionId, { conversationId }) ?? null; + if (!currentSession || !["busy", "creating"].includes(currentSession.status)) { + traceStore.append(traceId, { + type: "cancel", + status: "blocked", + label: "cancel:not_in_flight", + errorCode: "cancel_not_in_flight", + message: `Session ${sessionId} is not an in-flight Codex stdio request.`, + sessionId, + sessionStatus: currentSession?.status ?? "missing" + }); + sendJson(response, 409, cancelBlockedPayload({ + code: currentSession ? "cancel_not_in_flight" : "cancel_session_not_found", + message: currentSession + ? `当前 session 状态为 ${currentSession.status}，没有可取消的 in-flight Codex stdio 请求。` + : `没有找到 sessionId=${sessionId} 的 Codex stdio session。`, + traceId, + conversationId, + sessionId, + session: currentSession, + runnerTrace: traceStore.snapshot(traceId) + })); + return; + } + + const canceledSession = manager.cancel(sessionId, { + traceId, + conversationId, + reason: "user_cancel" + }); + traceStore.append(traceId, { + type: "cancel", + status: "canceled", + label: "cancel:canceled", + message: "User canceled the current Codex stdio request.", + sessionId, + sessionStatus: canceledSession?.status ?? "canceled", + sessionLifecycleStatus: codeAgentSessionLifecycleSummary({ session: canceledSession, status: "canceled" }).status, + waitingFor: "user-retry", + terminal: true + }); + const runnerTraceSnapshot = traceStore.snapshot(traceId); + const sessionSummary = codeAgentSessionLifecycleSummary({ + session: canceledSession, + runnerTrace: runnerTraceSnapshot, + status: "canceled" + }); + const runnerTrace = { + ...runnerTraceSnapshot, + sessionLifecycleStatus: sessionSummary.status + }; + const payload = { + accepted: true, + canceled: true, + status: "canceled", + conversationId: conversationId ?? canceledSession?.conversationId ?? null, + sessionId, + traceId, + session: canceledSession, + sessionLifecycleStatus: sessionSummary.status, + sessionLifecycle: sessionSummary, + sessionSummary, + runnerTrace, + lastTraceEvent: runnerTrace.lastEvent, + retryable: true, + error: { + code: "codex_stdio_canceled", + layer: "session", + category: "canceled", + retryable: true, + message: "user canceled current Code Agent request", + userMessage: "当前 Codex stdio 请求已取消；输入、sessionId、traceId 和最后 trace event 已保留，可重试上一条消息。", + traceId, + route: "/v1/agent/chat/cancel", + toolName: "codex-stdio.cancel" + }, + userMessage: "当前 Codex stdio 请求已取消；输入、sessionId、traceId 和最后 trace event 已保留，可重试上一条消息。", + updatedAt: new Date().toISOString() + }; + await recordCodeAgentSessionOwner({ payload, params: { ...params, traceId, ownerUserId: options.actor?.id, ownerRole: options.actor?.role }, options, status: "canceled" }); + recordCodeAgentConversationFact(payload, options); + options.codeAgentChatResults?.set(traceId, annotateOwner(payload, { ownerUserId: options.actor?.id, ownerRole: options.actor?.role })); + sendJson(response, 200, payload); +} + +export { codeAgentCompatProjectionPayload, codeAgentRequestScopedOptions, measureCodeAgentHttpPhase, readCodeAgentCompatProjection, traceProjectMismatchSnapshot }; diff --git a/internal/cloud/server-health.test.ts b/internal/cloud/server-health.test.ts index 016567e9..4c74c5a5 100644 --- a/internal/cloud/server-health.test.ts +++ b/internal/cloud/server-health.test.ts @@ -348,7 +348,7 @@ test("cloud api health separates DB connected from durable runtime schema readin try { const { port } = server.address(); - const response = await fetch(`http://127.0.0.1:${port}/health/live`); + const response = await fetch(`http://127.0.0.1:${port}/health`); const payload = await response.json(); assert.equal(response.status, 200); assert.equal(payload.status, "degraded"); @@ -449,7 +449,7 @@ test("cloud api health keeps DB live evidence separate when durable adapter quer try { const { port } = server.address(); - const response = await fetch(`http://127.0.0.1:${port}/health/live`); + const response = await fetch(`http://127.0.0.1:${port}/health`); const payload = await response.json(); assert.equal(response.status, 200); assert.equal(payload.status, "degraded"); @@ -593,7 +593,7 @@ test("cloud api health contract distinguishes durable SSL, auth, schema, migrati try { const { port } = server.address(); - const response = await fetch(`http://127.0.0.1:${port}/health/live`); + const response = await fetch(`http://127.0.0.1:${port}/health`); const payload = await response.json(); assert.equal(response.status, 200); assert.equal(payload.status, "degraded"); @@ -653,7 +653,7 @@ test("cloud api health does not treat DB env presence-only as live readiness", a try { const { port } = server.address(); - const response = await fetch(`http://127.0.0.1:${port}/health/live`); + const response = await fetch(`http://127.0.0.1:${port}/health`); const payload = await response.json(); assert.equal(response.status, 200); assert.equal(payload.db.configReady, true); @@ -782,7 +782,7 @@ test("cloud api health reports provider, durable DB, and codex stdio ready when try { const { port } = server.address(); - const response = await fetch(`http://127.0.0.1:${port}/health/live`); + const response = await fetch(`http://127.0.0.1:${port}/health`); assert.equal(response.status, 200); const payload = await response.json(); assert.equal(payload.status, "ok"); @@ -868,7 +868,7 @@ test("cloud api health reports AgentRun adapter readiness without repo-owned cod try { const { port } = server.address(); - const response = await fetch(`http://127.0.0.1:${port}/health/live`); + const response = await fetch(`http://127.0.0.1:${port}/health`); assert.equal(response.status, 200); const payload = await response.json(); assert.equal(payload.status, "ok"); diff --git a/internal/cloud/server-hwpod-http.ts b/internal/cloud/server-hwpod-http.ts new file mode 100644 index 00000000..8d4d7617 --- /dev/null +++ b/internal/cloud/server-hwpod-http.ts @@ -0,0 +1,528 @@ +/* + * Responsibility: Cloud API HTTP handlers for HWLAB node updates, HWPOD discovery, and node-ops dispatch. + */ +import { createHash, randomUUID } from "node:crypto"; +import { readFileSync } from "node:fs"; +import path from "node:path"; + +import { ENVIRONMENT_DEV } from "../protocol/index.mjs"; +import { CLOUD_API_SERVICE_ID } from "../audit/index.mjs"; +import { + discoverHwpodSpecs, + hwpodSpecDiscoveryPayload, + hwpodSpecWorkspaceProbePlan +} from "./hwpod-spec-discovery.ts"; +import { + getHeader, + parsePositiveInteger, + readBody, + safeOpaqueId, + sendJson, + truthyFlag +} from "./server-http-utils.ts"; +import { HWPOD_NODE_OPS, HWPOD_NODE_OPS_CONTRACT_VERSION } from "../../tools/src/hwpod-node-ops-contract.ts"; + +const DEFAULT_BODY_LIMIT_BYTES = 1024 * 1024; + +export async function handleHwpodNodeOpsHttp(request, response, options) { + if (request.method === "GET") { + sendJson(response, 200, { + ok: true, + status: "ready", + contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, + route: "/v1/hwpod-node-ops", + specAuthority: "code-agent-workspace", + compiler: "hwpod-compiler-cli", + apiRole: "node-ops-forwarder", + nodeRole: "thin-hwpod-node-executor", + supportedOps: Array.from(HWPOD_NODE_OPS), + websocket: options.hwpodNodeWsRegistry.describe() + }); + return; + } + if (request.method !== "POST") { + sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "hwpod-node-ops only supports GET and POST" } }); + return; + } + + const body = await readJsonObject(request, options.bodyLimitBytes); + if (!body.ok) { + sendJson(response, 400, body.error); + return; + } + const validation = validateHwpodNodeOpsPlan(body.value); + if (!validation.ok) { + sendJson(response, 400, { + ok: false, + status: "rejected", + contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, + error: validation.error + }); + return; + } + + const plan = validation.plan; + const requestMeta = hwpodNodeOpsRequestMeta(request, options, "hwpod"); + try { + const handled = await dispatchHwpodNodeOpsPlan(plan, requestMeta, options, { request }); + sendJson(response, handled.httpStatus, handled.payload); + } catch (error) { + const summary = error?.message ?? "hwpod-node-ops handler failed"; + recordHwpodNodeOpsBlocked(request, options, plan, requestMeta, "hwpod_node_handler_failed", summary, { dispatchMode: "handler-exception" }); + sendJson(response, 200, hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, { dispatchMode: "handler-exception" })); + } +} + +export function handleHwlabNodeUpdateHttp(request, response, url, options) { + if (request.method !== "GET") { + sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "GET required." } }); + return; + } + const env = options.env ?? process.env; + const currentVersion = cleanText(url.searchParams.get("current") || url.searchParams.get("version")); + const channel = cleanText(url.searchParams.get("channel")) || "stable"; + const platform = cleanText(url.searchParams.get("platform")) || "unknown"; + const bundled = hwlabNodeBundledMetadata(env); + const latestVersion = cleanText(env.HWLAB_NODE_PY_LATEST_VERSION) || bundled.version || "0.1.0"; + const releaseNotesUrl = cleanText(env.HWLAB_NODE_PY_RELEASE_NOTES_URL) || null; + const downloadUrl = hwlabNodeDownloadUrl(env); + const sha256 = cleanText(env.HWLAB_NODE_PY_SHA256) || bundled.sha256 || null; + const newer = currentVersion ? compareVersionStrings(latestVersion, currentVersion) > 0 : Boolean(latestVersion); + const updateAvailable = Boolean(downloadUrl && newer); + sendJson(response, 200, { + ok: true, + contractVersion: "hwlab-node-update-v1", + serviceId: "hwlab-node", + route: "/v1/hwlab-node/update", + channel, + platform, + currentVersion: currentVersion || null, + latestVersion, + updateAvailable, + downloadUrl: updateAvailable ? downloadUrl : null, + sha256: updateAvailable ? sha256 : null, + releaseNotesUrl, + manualDefault: true, + autoApplyDefault: false, + checkIntervalSeconds: parsePositiveInteger(env.HWLAB_NODE_PY_UPDATE_INTERVAL_SECONDS, 300), + observedAt: new Date().toISOString() + }); +} + +export function handleHwlabNodeDownloadHttp(request, response, options) { + if (request.method !== "GET") { + sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "GET required." } }); + return; + } + const bundled = hwlabNodeBundledMetadata(options.env ?? process.env); + if (!bundled.content) { + sendJson(response, 404, { ok: false, error: { code: "hwlab_node_bundle_missing", message: "bundled hwlab-node.py is not available" } }); + return; + } + const bytes = Buffer.from(bundled.content, "utf8"); + response.writeHead(200, { + "content-type": "text/x-python; charset=utf-8", + "cache-control": "no-store", + "x-hwlab-node-version": bundled.version || "unknown", + "x-hwlab-node-sha256": bundled.sha256 || "", + "content-length": String(bytes.length) + }); + response.end(bytes); +} + +export async function handleHwpodSpecDiscoveryHttp(request, response, url, options) { + const probe = truthyFlag(url.searchParams.get("probe")); + const observedAt = new Date().toISOString(); + let specs = await discoverHwpodSpecs(options); + if (probe) { + specs = await Promise.all(specs.map((spec) => probeDiscoveredHwpodSpec(spec, { request, options, observedAt }))); + } + sendJson(response, 200, hwpodSpecDiscoveryPayload(specs, { observedAt })); +} + +function runtimeEnvironment(env = process.env) { + const value = env.HWLAB_GITOPS_PROFILE || env.HWLAB_ENVIRONMENT || ENVIRONMENT_DEV; + return typeof value === "string" && value.trim() ? value.trim() : ENVIRONMENT_DEV; +} + +function hwpodNodeOpsRequestMeta(request, options, label) { + const httpContext = request?.hwlabHttpRequestContext ?? {}; + return { + requestId: getHeader(request, "x-request-id") || httpContext.requestId || `req_${label}_${randomUUID()}`, + traceId: getHeader(request, "x-trace-id") || `trc_${label}_${randomUUID()}`, + serviceId: getHeader(request, "x-source-service-id") || CLOUD_API_SERVICE_ID, + environment: runtimeEnvironment(options.env ?? process.env), + otelTraceId: httpContext.traceId ?? null, + traceparent: httpContext.traceparent ?? null, + valuesPrinted: false + }; +} + +function hwlabNodeBundledMetadata(env) { + const bundlePath = cleanText(env.HWLAB_NODE_PY_BUNDLE_PATH) || path.resolve(process.cwd(), "tools/hwlab-node.py"); + try { + const content = readFileSync(bundlePath, "utf8"); + const version = cleanText(content.match(/APP_VERSION\s*=\s*["']([^"']+)["']/u)?.[1]); + const sha256 = createHash("sha256").update(content, "utf8").digest("hex"); + return { path: bundlePath, content, version, sha256 }; + } catch { + return { path: bundlePath, content: "", version: "", sha256: "" }; + } +} + +function hwlabNodeDownloadUrl(env) { + const explicit = cleanText(env.HWLAB_NODE_PY_DOWNLOAD_URL); + if (explicit) return explicit; + const downloadPath = cleanText(env.HWLAB_NODE_PY_DOWNLOAD_PATH) || "/v1/hwlab-node/download/hwlab-node.py"; + if (/^https?:\/\//iu.test(downloadPath)) return downloadPath; + const publicEndpoint = cleanText(env.HWLAB_PUBLIC_ENDPOINT) || "https://hwlab.pikapython.com"; + try { + return new URL(downloadPath, publicEndpoint.endsWith("/") ? publicEndpoint : `${publicEndpoint}/`).toString(); + } catch { + return null; + } +} + +function compareVersionStrings(left, right) { + const a = versionParts(left); + const b = versionParts(right); + for (let index = 0; index < Math.max(a.length, b.length, 3); index += 1) { + const delta = (a[index] ?? 0) - (b[index] ?? 0); + if (delta !== 0) return delta > 0 ? 1 : -1; + } + return 0; +} + +function versionParts(value) { + return String(value ?? "") + .trim() + .replace(/^v/iu, "") + .split(/[.+-]/u) + .slice(0, 4) + .map((part) => parseInt(part, 10)) + .map((part) => (Number.isFinite(part) && part >= 0 ? part : 0)); +} + +function cleanText(value) { + const text = String(value ?? "").trim(); + return text || ""; +} + +async function probeDiscoveredHwpodSpec(spec, { request, options, observedAt }) { + if (spec.ok === false) return spec; + const plan = hwpodSpecWorkspaceProbePlan(spec); + const validation = validateHwpodNodeOpsPlan(plan); + if (!validation.ok) { + return { + ...spec, + availability: { + ok: false, + status: "invalid_probe_plan", + checkedAt: observedAt, + blocker: validation.error + } + }; + } + const requestMeta = hwpodNodeOpsRequestMeta(request, options, "hwpod_spec"); + const handled = await dispatchHwpodNodeOpsPlan(validation.plan, requestMeta, options, { request }); + const payload = handled.payload; + return { + ...spec, + availability: { + ok: payload.ok === true, + status: payload.ok === true ? "available" : "blocked", + checkedAt: observedAt, + probePlanId: payload.planId, + nodeOpsRoute: "/v1/hwpod-node-ops", + results: payload.results ?? [], + blocker: payload.blocker ?? null, + httpStatus: handled.httpStatus + } + }; +} + +async function dispatchHwpodNodeOpsPlan(plan, requestMeta, options, context = {}) { + const hwpodNodeOpsUrl = normalizedHwpodNodeOpsUrl(options.env ?? process.env); + const hwpodNodeWsRegistry = options.hwpodNodeWsRegistry; + const hasWsNode = hwpodNodeWsRegistry?.hasNode?.(plan.nodeId); + if (typeof options.hwpodNodeOpsHandler !== "function" && !hasWsNode && !hwpodNodeOpsUrl) { + const summary = "no outbound WebSocket hwpod-node is connected and HWLAB_HWPOD_NODE_OPS_URL is not configured; cloud-api is only validating the hwpod-node-ops contract"; + const details = { + dispatchMode: "none", + websocketRegistryConfigured: Boolean(hwpodNodeWsRegistry), + websocketConnected: false, + directUrlConfigured: false + }; + recordHwpodNodeOpsBlocked(context.request, options, plan, requestMeta, "hwpod_node_unavailable", summary, details); + return { + httpStatus: 200, + payload: hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, details) + }; + } + const handled = typeof options.hwpodNodeOpsHandler === "function" + ? await options.hwpodNodeOpsHandler(plan, { request: context.request, requestMeta, env: options.env ?? process.env }) + : hasWsNode + ? await hwpodNodeWsRegistry.dispatch(plan, requestMeta, { timeoutMs: parsePositiveInteger(options.env?.HWLAB_HWPOD_NODE_OPS_TIMEOUT_MS, 30000) }) + : await forwardHwpodNodeOpsPlan(hwpodNodeOpsUrl, plan, requestMeta, options); + const results = Array.isArray(handled?.results) ? handled.results : []; + const failed = results.some((item) => item?.ok === false); + const payload = attachHwpodNodeOpsDiagnostics({ + ok: handled?.ok ?? !failed, + status: handled?.status ?? (failed ? "failed" : "completed"), + contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, + planId: plan.planId, + hwpodId: plan.hwpodId, + nodeId: plan.nodeId, + acceptedOps: plan.ops.length, + results, + blocker: handled?.blocker ?? null, + requestMeta + }, requestMeta, { dispatchMode: hasWsNode ? "websocket" : hwpodNodeOpsUrl ? "direct-url" : "handler" }); + if (payload.ok === false && payload.status === "blocked" && payload.blocker?.code) { + recordHwpodNodeOpsBlocked(context.request, options, plan, requestMeta, payload.blocker.code, payload.blocker.summary ?? "hwpod-node-ops blocked", payload.blocker.details ?? {}); + } + return { httpStatus: handled?.httpStatus ?? (failed ? 409 : 200), payload }; +} + +async function forwardHwpodNodeOpsPlan(targetUrl, plan, requestMeta, options) { + const target = await describeDirectHwpodNode(targetUrl, options); + if (!target.ok || !target.nodeId) { + return directHwpodNodeBlocked(plan, "hwpod_node_identity_unverified", "configured hwpod-node URL did not expose a verifiable nodeId", { + requestedNodeId: plan.nodeId, + targetUrl: redactNodeOpsUrl(targetUrl), + dispatchMode: "direct-url", + targetStatus: target.status ?? null, + error: target.error ?? null + }, requestMeta); + } + if (target.ok && target.nodeId && target.nodeId !== plan.nodeId) { + return directHwpodNodeBlocked(plan, "hwpod_node_id_mismatch", `HWLAB_HWPOD_NODE_OPS_URL points to ${target.nodeId}, not requested node ${plan.nodeId}`, { + requestedNodeId: plan.nodeId, + targetNodeId: target.nodeId, + targetUrl: redactNodeOpsUrl(targetUrl), + dispatchMode: "direct-url" + }, requestMeta); + } + const response = await fetch(targetUrl, { + method: "POST", + headers: { + "content-type": "application/json", + "x-request-id": requestMeta.requestId, + "x-trace-id": requestMeta.traceId, + "x-source-service-id": CLOUD_API_SERVICE_ID, + ...(requestMeta.otelTraceId ? { "x-hwlab-otel-trace-id": requestMeta.otelTraceId } : {}), + ...(requestMeta.traceparent ? { traceparent: requestMeta.traceparent } : {}) + }, + body: JSON.stringify(plan), + signal: AbortSignal.timeout(parsePositiveInteger(options.env?.HWLAB_HWPOD_NODE_OPS_TIMEOUT_MS, 30000)) + }); + const body = await response.json().catch(() => null); + if (!body || typeof body !== "object") { + return directHwpodNodeBlocked(plan, "hwpod_node_response_invalid", `hwpod-node returned HTTP ${response.status} without a JSON object payload`, { + targetUrl: redactNodeOpsUrl(targetUrl), + dispatchMode: "direct-url", + targetStatus: response.status + }, requestMeta); + } + return { + ok: body.ok, + status: body.status, + httpStatus: response.status, + results: body.results, + blocker: body.blocker ?? null + }; +} + +function directHwpodNodeBlocked(plan, code, summary, details, requestMeta = {}) { + const blocker = hwpodNodeOpsBlocker({ code, layer: "hwpod-node", retryable: true, summary, details }, requestMeta, details); + return { + ok: false, + status: "blocked", + httpStatus: 200, + results: plan.ops.map((op) => ({ opId: op.opId, op: op.op, ok: false, status: "blocked", blocker })), + blocker, + otelTraceId: blocker.otelTraceId ?? null, + diagnostic: blocker.diagnostic ?? null + }; +} + +async function describeDirectHwpodNode(targetUrl, options) { + try { + const response = await fetch(targetUrl, { + method: "GET", + signal: AbortSignal.timeout(parsePositiveInteger(options.env?.HWLAB_HWPOD_NODE_OPS_TIMEOUT_MS, 30000)) + }); + const body = await response.json().catch(() => null); + return { ok: response.ok && body && typeof body === "object", status: response.status, nodeId: safeOpaqueId(body?.nodeId), body }; + } catch (error) { + return { ok: false, error: error instanceof Error ? error.message : String(error), nodeId: "" }; + } +} + +function redactNodeOpsUrl(value) { + try { + const parsed = new URL(value); + parsed.username = ""; + parsed.password = ""; + parsed.search = parsed.search ? "?redacted=1" : ""; + return parsed.toString(); + } catch { + return ""; + } +} + +function normalizedHwpodNodeOpsUrl(env = process.env) { + const direct = String(env.HWLAB_HWPOD_NODE_OPS_URL ?? "").trim(); + if (direct) return direct; + const base = String(env.HWLAB_HWPOD_NODE_URL ?? "").trim().replace(/\/+$/u, ""); + return base ? `${base}/v1/hwpod-node-ops` : ""; +} + +function validateHwpodNodeOpsPlan(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return { ok: false, error: { code: "invalid_hwpod_node_ops_plan", message: "hwpod-node-ops body must be a JSON object" } }; + } + if (value.contractVersion !== HWPOD_NODE_OPS_CONTRACT_VERSION) { + return { ok: false, error: { code: "invalid_hwpod_node_ops_contract", message: `contractVersion must be ${HWPOD_NODE_OPS_CONTRACT_VERSION}`, actual: value.contractVersion ?? null } }; + } + const planId = safeOpaqueId(value.planId) || `hwpod_plan_${randomUUID()}`; + const nodeId = safeOpaqueId(value.nodeId); + const hwpodId = safeOpaqueId(value.hwpodId); + if (!nodeId) return { ok: false, error: { code: "invalid_hwpod_node_id", message: "nodeId is required" } }; + if (!hwpodId) return { ok: false, error: { code: "invalid_hwpod_id", message: "hwpodId is required" } }; + if (!Array.isArray(value.ops) || value.ops.length === 0) { + return { ok: false, error: { code: "invalid_hwpod_node_ops", message: "ops must be a non-empty array" } }; + } + const ops = []; + for (const [index, item] of value.ops.entries()) { + if (!item || typeof item !== "object" || Array.isArray(item)) { + return { ok: false, error: { code: "invalid_hwpod_node_op", message: `ops[${index}] must be a JSON object` } }; + } + const op = typeof item.op === "string" ? item.op.trim() : ""; + if (!HWPOD_NODE_OPS.has(op)) { + return { ok: false, error: { code: "unsupported_hwpod_node_op", message: `unsupported hwpod-node op: ${op || ""}`, supportedOps: Array.from(HWPOD_NODE_OPS) } }; + } + ops.push({ + opId: safeOpaqueId(item.opId) || `op_${index + 1}`, + op, + args: item.args && typeof item.args === "object" && !Array.isArray(item.args) ? item.args : {} + }); + } + return { + ok: true, + plan: { + ...value, + planId, + hwpodId, + nodeId, + ops + } + }; +} + +function hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, details = {}) { + const blocker = hwpodNodeOpsBlocker({ + code: "hwpod_node_unavailable", + layer: "hwpod-node", + retryable: true, + summary, + details, + userMessage: "hwpod-node 尚未接入执行面；cloud-api 已完成 hwpod-node-ops 合同校验。" + }, requestMeta, details); + return { + ok: false, + status: "blocked", + contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, + planId: plan.planId, + hwpodId: plan.hwpodId, + nodeId: plan.nodeId, + acceptedOps: plan.ops.length, + results: plan.ops.map((op) => ({ + opId: op.opId, + op: op.op, + ok: false, + status: "blocked", + blocker + })), + blocker, + otelTraceId: blocker.otelTraceId ?? null, + diagnostic: blocker.diagnostic ?? null, + requestMeta + }; +} + +function hwpodNodeOpsBlocker(blocker, requestMeta = {}, details = {}) { + const code = cleanText(blocker?.code) || "hwpod_node_ops_blocked"; + const summary = cleanText(blocker?.summary ?? blocker?.message) || "hwpod-node-ops blocked"; + const otelTraceId = cleanText(blocker?.otelTraceId ?? requestMeta?.otelTraceId) || null; + const diagnostic = { + code, + rootCauseCode: code, + summary, + otelTraceId, + traceLine: otelTraceId ? `OTel traceId: ${otelTraceId}` : null, + details: details && typeof details === "object" ? details : {}, + valuesPrinted: false + }; + const userMessage = cleanText(blocker?.userMessage); + return { + ...blocker, + code, + layer: blocker?.layer ?? "hwpod-node", + retryable: blocker?.retryable ?? true, + summary, + ...(otelTraceId ? { otelTraceId } : {}), + diagnostic, + ...(userMessage ? { userMessage: `${userMessage}\n${diagnostic.traceLine ?? "OTel traceId: unavailable"}` } : {}) + }; +} + +function attachHwpodNodeOpsDiagnostics(payload, requestMeta, details = {}) { + if (!payload || typeof payload !== "object" || payload.ok !== false) return payload; + const topBlocker = payload.blocker ? hwpodNodeOpsBlocker(payload.blocker, requestMeta, payload.blocker.details ?? details) : null; + const results = Array.isArray(payload.results) + ? payload.results.map((result) => result?.blocker ? { ...result, blocker: hwpodNodeOpsBlocker(result.blocker, requestMeta, result.blocker.details ?? details) } : result) + : payload.results; + const diagnostic = topBlocker?.diagnostic ?? results?.find?.((result) => result?.blocker?.diagnostic)?.blocker?.diagnostic ?? null; + return { + ...payload, + results, + blocker: topBlocker, + ...(diagnostic?.otelTraceId ? { otelTraceId: diagnostic.otelTraceId } : {}), + ...(diagnostic ? { diagnostic } : {}) + }; +} + +function recordHwpodNodeOpsBlocked(request, options, plan, requestMeta, code, summary, details = {}) { + const httpContext = request?.hwlabHttpRequestContext; + const attributes = { + "hwlab.hwpod.plan_id": cleanText(plan?.planId) || null, + "hwlab.hwpod.hwpod_id": cleanText(plan?.hwpodId) || null, + "hwlab.hwpod.node_id": cleanText(plan?.nodeId) || null, + "hwlab.hwpod.accepted_ops": Array.isArray(plan?.ops) ? plan.ops.length : 0, + "hwlab.hwpod.ops": Array.isArray(plan?.ops) ? plan.ops.map((op) => cleanText(op?.op)).filter(Boolean).join(",").slice(0, 240) : null, + "hwlab.hwpod.blocker.code": code, + "hwlab.hwpod.blocker.summary": String(summary ?? "").slice(0, 500), + "hwlab.hwpod.dispatch_mode": cleanText(details?.dispatchMode) || "unknown", + "hwlab.hwpod.websocket_connected": Boolean(details?.websocketConnected), + "hwlab.hwpod.direct_url_configured": Boolean(details?.directUrlConfigured), + "hwlab.hwpod.otel_trace_id": cleanText(requestMeta?.otelTraceId) || null, + traceId: cleanText(requestMeta?.traceId) || null + }; + if (httpContext) httpContext.otelAttributes = { ...(httpContext.otelAttributes ?? {}), ...attributes }; + const error = Object.assign(new Error(summary), { code }); + options.emitHttpRoutePhaseSpan?.(request, options, "hwpod-node-ops.blocked", Date.now(), Date.now(), "error", error, attributes); +} + +async function readJsonObject(request, limitBytes) { + const body = await readBody(request, limitBytes ?? DEFAULT_BODY_LIMIT_BYTES); + try { + const value = body ? JSON.parse(body) : {}; + if (!value || typeof value !== "object" || Array.isArray(value)) { + return { ok: false, error: { accepted: false, error: { code: "invalid_params", message: "body must be a JSON object" } } }; + } + return { ok: true, value }; + } catch (error) { + return { ok: false, error: { accepted: false, error: { code: "parse_error", message: "Invalid JSON body", reason: error.message } } }; + } +} diff --git a/internal/cloud/server-live-builds.test.ts b/internal/cloud/server-live-builds.test.ts index 58a40a92..0aa6b330 100644 --- a/internal/cloud/server-live-builds.test.ts +++ b/internal/cloud/server-live-builds.test.ts @@ -149,6 +149,10 @@ test("cloud api aggregates live HWLAB build times from health and controlled dep }; const server = createCloudApiServer({ + accessController: { + required: false, + async authenticate() { return { ok: true, actor: { id: "usr_live_builds", role: "admin" }, session: { id: "uss_live_builds" } }; } + }, env: { PATH: "", HWLAB_COMMIT_ID: "apiabcdef123456", @@ -285,6 +289,10 @@ test("cloud api ignores old repository reports and keeps missing buildCreatedAt const observe = async () => { const server = createCloudApiServer({ + accessController: { + required: false, + async authenticate() { return { ok: true, actor: { id: "usr_live_builds", role: "admin" }, session: { id: "uss_live_builds" } }; } + }, env: { PATH: "", HWLAB_GATEWAY_URL: "http://live.test/hwlab-gateway" diff --git a/internal/cloud/server-m3-http.test.ts b/internal/cloud/server-m3-http.test.ts index 592a664d..79fa0b86 100644 --- a/internal/cloud/server-m3-http.test.ts +++ b/internal/cloud/server-m3-http.test.ts @@ -146,6 +146,10 @@ test("cloud api /v1/diagnostics/gate exposes Chinese single-table rows from live now: () => "2026-05-23T00:00:00.000Z" }); const server = createCloudApiServer({ + accessController: { + required: false, + async authenticate() { return { ok: true, actor: { id: "usr_m3_diagnostics", role: "admin" }, session: { id: "uss_m3_diagnostics" } }; } + }, runtimeStore, env: { HWLAB_M3_GATEWAY_SIMU_1_URL: "http://gateway-1", diff --git a/internal/cloud/server-test-helpers.ts b/internal/cloud/server-test-helpers.ts index 982096dd..a9bcd85e 100644 --- a/internal/cloud/server-test-helpers.ts +++ b/internal/cloud/server-test-helpers.ts @@ -683,15 +683,16 @@ function sessionCookieFromResponse(response) { return value.split(";")[0] || ""; } -export async function pollAgentResult(port, traceId) { +export async function pollAgentResult(port, traceId, init = {}) { let last = null; for (let attempt = 0; attempt < 20; attempt += 1) { - const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat/result/${encodeURIComponent(traceId)}`); + const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat/result/${encodeURIComponent(traceId)}`, init); last = { status: response.status, body: await response.json() }; if (response.status === 200) return last.body; + if (response.status !== 202) throw new Error(`Code Agent result polling failed: ${JSON.stringify(last)}`); await delay(10); } throw new Error(`Code Agent result did not complete: ${JSON.stringify(last)}`); diff --git a/internal/cloud/server-workbench-facts.ts b/internal/cloud/server-workbench-facts.ts index 97ce0952..6b5805e4 100644 --- a/internal/cloud/server-workbench-facts.ts +++ b/internal/cloud/server-workbench-facts.ts @@ -17,7 +17,6 @@ import { createWorkbenchReadModel } from "./workbench-read-model.ts"; import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts"; import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts"; import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts"; -import { openKafkaEventStream } from "./kafka-event-bridge.ts"; import { durableTraceStatus, RUNNING_STATUSES, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts"; import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts"; diff --git a/internal/cloud/server-workbench-launch-http.test.ts b/internal/cloud/server-workbench-launch-http.test.ts index 9d4532b0..2bb9e284 100644 --- a/internal/cloud/server-workbench-launch-http.test.ts +++ b/internal/cloud/server-workbench-launch-http.test.ts @@ -52,9 +52,9 @@ test("workbench launch records OTel stage for Project Management link write 404" } }, runtimeStore: { - async writeWorkbenchFacts(params, requestMeta) { + async writeWorkbenchSessionAdmissionFact(params, requestMeta) { factWrites.push({ params, requestMeta }); - return { ok: true, facts: params.facts }; + return { ok: true, fact: params.fact, admissionOnly: true }; } } }); @@ -86,7 +86,7 @@ test("workbench launch records OTel stage for Project Management link write 404" assert.match(context.otelAttributes["workbench.launch.task_ref_hash"], /^[0-9a-f]{24}$/u); assert.equal(context.otelAttributes["workbench.launch.values_redacted"], true); assert.equal(factWrites.length, 1); - const sessionJson = factWrites[0].params.facts.sessions[0].sessionJson; + const sessionJson = factWrites[0].params.fact.sessionJson; assert.equal(sessionJson.launchContext.sourceId, "hwlab-v03-mdtodo"); assert.equal(sessionJson.launchContext.hwpodId, "constart-71freq-c"); assert.equal(sessionJson.launchContext.contextFingerprint, "ctx_launch_otel"); diff --git a/internal/cloud/server-workbench-read-http.ts b/internal/cloud/server-workbench-read-http.ts index 5a4a1f4c..78fb4c86 100644 --- a/internal/cloud/server-workbench-read-http.ts +++ b/internal/cloud/server-workbench-read-http.ts @@ -17,7 +17,7 @@ import { createWorkbenchReadModel } from "./workbench-read-model.ts"; import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts"; import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts"; import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts"; -import { openKafkaEventStream } from "./kafka-event-bridge.ts"; +import { workbenchRealtimeCapabilities } from "./workbench-realtime-capabilities.ts"; import { durableTraceStatus, RUNNING_STATUSES, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts"; import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts"; import * as workbenchFacts from "./server-workbench-facts.ts"; @@ -157,6 +157,19 @@ export async function handleWorkbenchReadModelHttp(request, response, url, optio if (!auth) return; if (url.pathname === "/v1/workbench/sync") { + const capabilities = workbenchRealtimeCapabilities(options.env ?? process.env); + if (!capabilities.projectionRealtime) { + sendJson(response, 503, { + ok: false, + error: { + code: "workbench_projection_realtime_disabled", + message: "Workbench projection sync/replay capability is disabled.", + capabilities, + valuesRedacted: true + } + }); + return; + } await (perf ? perf.measure("workbench_sync", () => handleWorkbenchSyncHttp(request, response, url, options, auth.actor)) : handleWorkbenchSyncHttp(request, response, url, options, auth.actor)); return; } diff --git a/internal/cloud/server-workbench-realtime-http.test.ts b/internal/cloud/server-workbench-realtime-http.test.ts index ad278165..17811e7d 100644 --- a/internal/cloud/server-workbench-realtime-http.test.ts +++ b/internal/cloud/server-workbench-realtime-http.test.ts @@ -2,12 +2,14 @@ // Responsibility: Workbench realtime and projection blocker regression tests. import assert from "node:assert/strict"; +import { get as httpGet } from "node:http"; import { test } from "bun:test"; import { createCloudApiServer } from "./server.ts"; import { createBackendPerformanceStore } from "./backend-performance.ts"; import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; import { classifyWorkbenchReadModelFailure } from "./server-workbench-http.ts"; +import { projectionOutboxRealtimeEvents, workbenchRealtimeAfterSeq } from "./server-workbench-realtime-http.ts"; import { codeAgentTurnStatusPayload, createCodeAgentChatResultStore } from "./server-code-agent-http.ts"; import { createWorkbenchTurnProjection, durableTraceStatus, projectionDiagnostics, traceTerminalEvidence } from "./workbench-turn-projection.ts"; @@ -32,11 +34,660 @@ import { emptyFacts, normalizeTestStatus, getSseEvents, - parseSseBlock, - createFakeKafkaFactory, - waitFor + parseSseBlock } from "./server-workbench-http-test-helpers.ts"; +const TRANSACTIONAL_REALTIME_ENV = Object.freeze({ + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true" +}); + +test("projection outbox emits immutable assistant versions in row order", () => { + const traceId = "trc_outbox_versions"; + const sessionId = "ses_outbox_versions"; + const messageId = "msg_outbox_versions_agent"; + const events = projectionOutboxRealtimeEvents({ + facts: { messages: [{ messageId, traceId, sessionId, text: "current state must not replace history", projectedSeq: 99 }] }, + events: [ + { outboxSeq: 10, outboxEventId: "outbox-progress-1", entityFamily: "messages", entityId: messageId, projectedSeq: 1, projectionRevision: 1, traceId, sessionId, commitType: "message", payload: { family: "messages", fact: { messageId, traceId, sessionId, text: "first progress", projectedSeq: 1, status: "running" } } }, + { outboxSeq: 11, outboxEventId: "outbox-progress-2", entityFamily: "messages", entityId: messageId, projectedSeq: 2, projectionRevision: 2, traceId, sessionId, commitType: "message", payload: { family: "messages", fact: { messageId, traceId, sessionId, text: "second progress", projectedSeq: 2, status: "running" } } } + ] + }); + + assert.deepEqual(events.map((item) => item.payload.message.text), ["first progress", "second progress"]); + assert.deepEqual(events.map((item) => item.payload.cursor.outboxSeq), [10, 11]); +}); + +test("live Kafka SSE transparently fans out one envelope without DB, snapshot, cursor, replay, or SSE id", async () => { + const sessionId = "ses_live_kafka_sse"; + const traceId = "trc_live_kafka_sse"; + const subscribers = new Set(); + const otelSpans = []; + let releaseBridgeReady; + const bridgeReady = new Promise((resolve) => { releaseBridgeReady = resolve; }); + const bridge = { + capabilities: { directPublish: true, liveKafkaSse: true, transactionalProjector: false, projectionOutboxRelay: false, projectionRealtime: false }, + ready: bridgeReady, + subscribeLiveHwlabEvents(listener) { + subscribers.add(listener); + return () => subscribers.delete(listener); + }, + async stop() {} + }; + let dbCalls = 0; + const workbenchRuntime = new Proxy({}, { + get() { + dbCalls += 1; + throw new Error("live SSE must not access Workbench runtime facts"); + } + }); + const server = createCloudApiServer({ + accessController: realtimeAccessController({ + sessions: [{ id: sessionId, ownerUserId: ACTOR.id, lastTraceId: traceId }] + }), + workbenchRuntime, + kafkaEventBridge: bridge, + otelSpanEmitter(name, emittedTraceId, env, spanOptions) { otelSpans.push({ name, emittedTraceId, env, spanOptions }); }, + env: { + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const envelope = { + schema: "hwlab.event.v1", + eventType: "hwlab.trace.event.projected", + eventId: "hwlab:evt_live_kafka_sse", + traceId, + hwlabSessionId: sessionId, + sessionId, + runId: "run_live_kafka_sse", + commandId: "cmd_live_kafka_sse", + context: { runId: "run_live_kafka_sse", commandId: "cmd_live_kafka_sse", sourceSeq: 4, valuesRedacted: true }, + event: { type: "assistant", eventType: "assistant", status: "running", traceId, sessionId, text: "live increment", sourceSeq: 4, terminal: false, valuesPrinted: false }, + valuesPrinted: false + }; + + try { + const { port } = server.address(); + const firstPromise = getSseEvents(port, `/v1/workbench/events?sessionId=${sessionId}&afterSeq=999`, 2); + const secondPromise = getSseEvents(port, `/v1/workbench/events?sessionId=${sessionId}&traceId=${traceId}&afterSeq=888`, 2); + await waitForCondition(() => subscribers.size === 2); + for (const listener of subscribers) listener(envelope, { topic: "hwlab.event.v1", partition: 6, offset: "101" }); + releaseBridgeReady(); + const [first, second] = await Promise.all([firstPromise, secondPromise]); + for (const events of [first, second]) { + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "hwlab.event.v1"]); + assert.deepEqual(events[0].data.capabilities, bridge.capabilities); + assert.equal(events[0].data.liveOnly, true); + assert.equal(events[0].data.replay, false); + assert.equal(events[0].data.lossPossible, true); + assert.equal(events[0].data.cursor, undefined); + assert.equal(events[1].id, null); + assert.deepEqual(events[1].data, envelope); + } + assert.equal(otelSpans.length, 2); + for (const span of otelSpans) { + assert.equal(span.name, "hwlab.workbench.live_sse.business_event_write"); + assert.equal(span.emittedTraceId, traceId); + assert.deepEqual(span.spanOptions.attributes, { + businessTraceId: traceId, + hwlabSessionId: sessionId, + runId: "run_live_kafka_sse", + commandId: "cmd_live_kafka_sse", + topic: "hwlab.event.v1", + partition: 6, + offset: "101", + sourceTopic: null, + sourcePartition: null, + sourceOffset: null, + eventType: "assistant", + terminal: false, + valuesRedacted: true + }); + assert.equal(JSON.stringify(span.spanOptions.attributes).includes("live increment"), false); + } + assert.equal(dbCalls, 0); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("live Kafka SSE heartbeat keeps the transport open without DB, cursor, snapshot, or replay", async () => { + const sessionId = "ses_live_kafka_heartbeat"; + let dbCalls = 0; + const server = createCloudApiServer({ + accessController: realtimeAccessController({ sessions: [{ id: sessionId, ownerUserId: ACTOR.id }] }), + workbenchRuntime: new Proxy({}, { + get() { + dbCalls += 1; + throw new Error("live heartbeat must not access Workbench projection state"); + } + }), + kafkaEventBridge: { + capabilities: { liveKafkaSse: true }, + ready: Promise.resolve(), + subscribeLiveHwlabEvents() { return () => {}; }, + async stop() {} + }, + env: { + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false", + HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "5" + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const events = await getSseEvents(server.address().port, `/v1/workbench/events?sessionId=${sessionId}`, 2); + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.heartbeat"]); + assert.equal(events[1].id, null); + assert.equal(events[1].data.liveOnly, true); + assert.equal(events[1].data.replay, false); + assert.equal(events[1].data.cursor, undefined); + assert.equal(events[1].data.snapshot, undefined); + assert.equal(dbCalls, 0); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("live Kafka SSE rejects foreign or inconsistent ownership scopes before fanout subscription", async () => { + const ownedSession = { id: "ses_live_kafka_owned", ownerUserId: ACTOR.id, lastTraceId: "trc_live_kafka_owned" }; + const foreignSession = { id: "ses_live_kafka_foreign", ownerUserId: "usr_live_kafka_foreign", lastTraceId: "trc_live_kafka_foreign" }; + let subscriptions = 0; + let projectionReads = 0; + const server = createCloudApiServer({ + accessController: realtimeAccessController({ sessions: [ownedSession, foreignSession] }), + workbenchRuntime: new Proxy({}, { + get() { + projectionReads += 1; + throw new Error("live SSE authorization must not access Workbench projection state"); + } + }), + kafkaEventBridge: { + capabilities: { liveKafkaSse: true }, + ready: Promise.resolve(), + subscribeLiveHwlabEvents() { + subscriptions += 1; + return () => {}; + }, + async stop() {} + }, + env: { + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const foreignSessionResponse = await getJson(port, `/v1/workbench/events?sessionId=${foreignSession.id}`); + const foreignTraceResponse = await getJson(port, `/v1/workbench/events?traceId=${foreignSession.lastTraceId}`); + const inconsistentResponse = await getJson(port, `/v1/workbench/events?sessionId=${ownedSession.id}&traceId=${foreignSession.lastTraceId}`); + const missingResponse = await getJson(port, "/v1/workbench/events?sessionId=ses_live_kafka_missing"); + + for (const response of [foreignSessionResponse, foreignTraceResponse, inconsistentResponse, missingResponse]) { + assert.equal(response.status, 404); + assert.equal(response.body.error.code, "workbench_realtime_scope_not_found"); + } + assert.equal(subscriptions, 0); + assert.equal(projectionReads, 0); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("live Kafka SSE fails closed when ownership lookup is not configured", async () => { + let subscriptions = 0; + const server = createCloudApiServer({ + accessController: { + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_realtime_unconfigured" } }; } + }, + kafkaEventBridge: { + capabilities: { liveKafkaSse: true }, + ready: Promise.resolve(), + subscribeLiveHwlabEvents() { + subscriptions += 1; + return () => {}; + }, + async stop() {} + }, + env: { + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const response = await getJson(server.address().port, "/v1/workbench/events?sessionId=ses_live_kafka_unconfigured"); + assert.equal(response.status, 503); + assert.equal(response.body.error.code, "workbench_realtime_authorization_unconfigured"); + assert.equal(subscriptions, 0); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("live Kafka SSE lets admin subscribe to another owner's consistent session and trace", async () => { + const session = { id: "ses_live_kafka_admin", ownerUserId: "usr_live_kafka_owner", lastTraceId: "trc_live_kafka_admin" }; + const admin = { ...ACTOR, id: "usr_live_kafka_admin", role: "admin" }; + let subscriptions = 0; + const server = createCloudApiServer({ + accessController: realtimeAccessController({ actor: admin, sessions: [session] }), + kafkaEventBridge: { + capabilities: { liveKafkaSse: true }, + ready: Promise.resolve(), + subscribeLiveHwlabEvents() { + subscriptions += 1; + return () => {}; + }, + async stop() {} + }, + env: { + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const events = await getSseEvents(server.address().port, `/v1/workbench/events?sessionId=${session.id}&traceId=${session.lastTraceId}`, 1); + assert.equal(events[0].event, "workbench.connected"); + assert.deepEqual(events[0].data.filters, { sessionId: session.id, traceId: session.lastTraceId }); + assert.equal(subscriptions, 1); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("workbench realtime initial connection emits current snapshot without replaying historical outbox", async () => { + const sessionId = "ses_realtime_snapshot_only"; + const traceId = "trc_realtime_snapshot_only"; + const calls = []; + const runtime = { + async readAtomicWorkbenchProjectionSync(params = {}) { + calls.push({ ...params }); + return { + facts: { + sessions: [{ sessionId, lastTraceId: traceId }], + messages: [{ messageId: "msg_snapshot_only", sessionId, traceId, role: "agent", text: "current", projectedSeq: 9 }], + turns: [{ turnId: traceId, sessionId, traceId, status: "running", projectedSeq: 9 }] + }, + events: [{ outboxSeq: 1, entityFamily: "messages", entityId: "msg_old", payload: { family: "messages", fact: { messageId: "msg_old", sessionId, traceId, text: "historical" } } }], + cutoffOutboxSeq: 90, + cursorOutboxSeq: 90, + hasMore: true + }; + } + }; + const server = createCloudApiServer({ + accessController: realtimeAccessController(), + workbenchRuntime: runtime, + env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const events = await getSseEvents(server.address().port, `/v1/workbench/events?sessionId=${sessionId}`, 3); + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.message.snapshot", "workbench.turn.snapshot"]); + assert.equal(JSON.stringify(events).includes("historical"), false); + assert.equal(events[1].id, "90"); + assert.equal(calls.length, 1); + assert.equal(calls[0].snapshotOnly, true); + assert.equal(calls[0].deltaOnly, false); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("live and projection realtime remain independently reachable when both capabilities are enabled", async () => { + const sessionId = "ses_composable_realtime"; + const traceId = "trc_composable_realtime"; + let projectionReads = 0; + const runtime = { + async readAtomicWorkbenchProjectionSync() { + projectionReads += 1; + return { + facts: { + sessions: [{ sessionId, lastTraceId: traceId }], + messages: [{ messageId: "msg_composable_realtime", sessionId, traceId, role: "agent", text: "projection snapshot", projectedSeq: 1 }], + turns: [] + }, + events: [], + cutoffOutboxSeq: 1, + cursorOutboxSeq: 1, + hasMore: false + }; + } + }; + const server = createCloudApiServer({ + accessController: realtimeAccessController(), + workbenchRuntime: runtime, + env: { + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true", + HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", + HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const events = await getSseEvents(server.address().port, `/v1/workbench/projection-events?sessionId=${sessionId}`, 2); + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.message.snapshot"]); + assert.equal(events[0].data.realtimeSource, "projection-outbox"); + assert.equal(projectionReads, 1); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("synchronous projection notification is buffered until the unique initial snapshot completes", async () => { + const sessionId = "ses_realtime_initial_barrier"; + const traceId = "trc_realtime_initial_barrier"; + const calls = []; + let releaseInitial; + let unsubscribeCount = 0; + const initialGate = new Promise((resolve) => { releaseInitial = resolve; }); + const kafkaEventBridge = { + subscribeProjectionCommits(listener) { + listener({ sessionId, traceId }); + return () => { unsubscribeCount += 1; }; + }, + async stop() {} + }; + const runtime = { + async readAtomicWorkbenchProjectionSync(params = {}) { + calls.push({ ...params }); + if (calls.length === 1) { + await initialGate; + return { + facts: { + sessions: [{ sessionId, lastTraceId: traceId }], + messages: [{ messageId: "msg_initial_barrier", sessionId, traceId, role: "agent", text: "snapshot before delta", projectedSeq: 10 }], + turns: [] + }, + events: [], + cutoffOutboxSeq: 10, + cursorOutboxSeq: 10, + hasMore: false + }; + } + const event = { id: "wte_initial_barrier_delta", sessionId, traceId, projectedSeq: 11, message: "delta after snapshot" }; + return { + facts: {}, + events: [{ outboxSeq: 11, entityFamily: "traceEvents", entityId: event.id, projectedSeq: 11, projectionRevision: 11, sessionId, traceId, commitType: "event", payload: { family: "traceEvents", fact: event } }], + cutoffOutboxSeq: 11, + cursorOutboxSeq: 11, + hasMore: false + }; + } + }; + const server = createCloudApiServer({ + accessController: realtimeAccessController(), + workbenchRuntime: runtime, + kafkaEventBridge, + env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const eventsPromise = getSseEvents(server.address().port, `/v1/workbench/events?sessionId=${sessionId}`, 3); + await waitForCondition(() => calls.length === 1); + assert.equal(calls[0].snapshotOnly, true); + assert.equal(calls[0].deltaOnly, false); + assert.equal(calls.length, 1); + releaseInitial(); + const events = await eventsPromise; + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.message.snapshot", "workbench.trace.event"]); + assert.equal(events[1].data.message.text, "snapshot before delta"); + assert.equal(events[2].data.event.message, "delta after snapshot"); + assert.equal(calls.length, 2); + assert.equal(calls[1].snapshotOnly, false); + assert.equal(calls[1].deltaOnly, true); + assert.equal(calls[1].afterOutboxSeq, 10); + } finally { + releaseInitial?.(); + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } + assert.equal(unsubscribeCount, 1); +}); + +test("initial atomic snapshot failure emits an error and closes SSE for reconnect", async () => { + const sessionId = "ses_realtime_initial_failure"; + let unsubscribeCount = 0; + const server = createCloudApiServer({ + accessController: realtimeAccessController(), + workbenchRuntime: { + async readAtomicWorkbenchProjectionSync() { + const error = new Error("initial snapshot failed"); + error.code = "workbench_initial_snapshot_failed"; + throw error; + } + }, + kafkaEventBridge: { + subscribeProjectionCommits() { return () => { unsubscribeCount += 1; }; }, + async stop() {} + }, + env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const controller = new AbortController(); + let timeout = null; + try { + const response = await fetch(`http://127.0.0.1:${server.address().port}/v1/workbench/events?sessionId=${sessionId}`, { signal: controller.signal }); + assert.equal(response.status, 200); + const body = await Promise.race([ + response.text(), + new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error("initial snapshot failure left SSE open")), 500); }) + ]); + const events = body.trim().split("\n\n").filter(Boolean).map(parseSseBlock); + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.error"]); + assert.equal(events[1].data.error.code, "workbench_initial_snapshot_failed"); + assert.equal(body.includes("workbench.heartbeat"), false); + await waitForCondition(() => unsubscribeCount === 1); + } finally { + if (timeout) clearTimeout(timeout); + controller.abort(); + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("delta scan failure after a successful snapshot closes SSE for cursor reconnect", async () => { + const sessionId = "ses_realtime_delta_failure"; + const traceId = "trc_realtime_delta_failure"; + let unsubscribeCount = 0; + let calls = 0; + const server = createCloudApiServer({ + accessController: realtimeAccessController(), + workbenchRuntime: { + async readAtomicWorkbenchProjectionSync() { + calls += 1; + if (calls === 1) { + return { + facts: { + sessions: [{ sessionId, lastTraceId: traceId }], + messages: [{ messageId: "msg_delta_failure", sessionId, traceId, role: "agent", text: "durable snapshot", projectedSeq: 5 }], + turns: [] + }, + events: [], + cutoffOutboxSeq: 5, + cursorOutboxSeq: 5, + hasMore: false + }; + } + const error = new Error("delta scan failed"); + error.code = "workbench_delta_scan_failed"; + throw error; + } + }, + kafkaEventBridge: { + subscribeProjectionCommits(listener) { + listener({ sessionId, traceId }); + return () => { unsubscribeCount += 1; }; + }, + async stop() {} + }, + env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const controller = new AbortController(); + let timeout = null; + try { + const response = await fetch(`http://127.0.0.1:${server.address().port}/v1/workbench/events?sessionId=${sessionId}`, { signal: controller.signal }); + assert.equal(response.status, 200); + const body = await Promise.race([ + response.text(), + new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error("delta failure left SSE open")), 500); }) + ]); + const events = body.trim().split("\n\n").filter(Boolean).map(parseSseBlock); + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.message.snapshot", "workbench.error"]); + assert.equal(events[2].data.error.code, "workbench_delta_scan_failed"); + assert.equal(body.includes("workbench.heartbeat"), false); + assert.equal(calls, 2); + await waitForCondition(() => unsubscribeCount === 1); + } finally { + if (timeout) clearTimeout(timeout); + controller.abort(); + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("projector commit during initial snapshot wakes one coalesced delta scan without idle polling", async () => { + const sessionId = "ses_realtime_commit_wakeup"; + const traceId = "trc_realtime_commit_wakeup"; + const calls = []; + let releaseInitial; + let notify = null; + let unsubscribeCount = 0; + const initialGate = new Promise((resolve) => { releaseInitial = resolve; }); + const kafkaEventBridge = { + subscribeProjectionCommits(listener) { notify = listener; return () => { unsubscribeCount += 1; }; }, + async stop() {} + }; + const runtime = { + async readAtomicWorkbenchProjectionSync(params = {}) { + calls.push({ ...params }); + if (calls.length === 1) { + await initialGate; + return { facts: { sessions: [{ sessionId, lastTraceId: traceId }], messages: [], turns: [] }, events: [], cutoffOutboxSeq: 10, cursorOutboxSeq: 10, hasMore: false }; + } + const event = { id: "wte_commit_wakeup", sessionId, traceId, projectedSeq: 11, message: "commit wakeup" }; + return { facts: {}, events: [{ outboxSeq: 11, entityFamily: "traceEvents", entityId: event.id, projectedSeq: 11, projectionRevision: 11, sessionId, traceId, commitType: "event", payload: { family: "traceEvents", fact: event } }], cutoffOutboxSeq: 11, cursorOutboxSeq: 11, hasMore: false }; + } + }; + const server = createCloudApiServer({ + accessController: realtimeAccessController(), + workbenchRuntime: runtime, + kafkaEventBridge, + env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const eventsPromise = getSseEvents(server.address().port, `/v1/workbench/events?sessionId=${sessionId}`, 2); + await waitForCondition(() => calls.length === 1 && typeof notify === "function"); + notify({ sessionId, traceId }); + notify({ sessionId, traceId }); + releaseInitial(); + const events = await eventsPromise; + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.trace.event"]); + assert.equal(events[1].data.event.message, "commit wakeup"); + assert.equal(calls.length, 2); + assert.equal(calls[0].snapshotOnly, true); + assert.equal(calls[1].deltaOnly, true); + await new Promise((resolve) => setTimeout(resolve, 30)); + assert.equal(calls.length, 2); + } finally { + releaseInitial?.(); + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } + assert.equal(unsubscribeCount, 1); +}); + +test("workbench realtime early disconnect releases commit subscription during initial scan", async () => { + const sessionId = "ses_realtime_early_disconnect"; + let releaseInitial; + let subscribed = 0; + let unsubscribed = 0; + const gate = new Promise((resolve) => { releaseInitial = resolve; }); + const server = createCloudApiServer({ + accessController: realtimeAccessController(), + workbenchRuntime: { async readAtomicWorkbenchProjectionSync() { await gate; return { facts: {}, events: [], cutoffOutboxSeq: 0, cursorOutboxSeq: 0, hasMore: false }; } }, + kafkaEventBridge: { subscribeProjectionCommits() { subscribed += 1; return () => { unsubscribed += 1; }; }, async stop() {} }, + env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + let clientRequest = null; + try { + const response = await new Promise((resolve, reject) => { + clientRequest = httpGet(`http://127.0.0.1:${server.address().port}/v1/workbench/events?sessionId=${sessionId}`, resolve); + clientRequest.once("error", reject); + }); + assert.equal(response.statusCode, 200); + await waitForCondition(() => subscribed === 1); + response.destroy(); + clientRequest.destroy(); + await waitForCondition(() => unsubscribed === 1); + releaseInitial(); + } finally { + clientRequest?.destroy(); + releaseInitial?.(); + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("workbench realtime prefers automatic Last-Event-ID over stale URL cursor", () => { + const url = new URL("http://localhost/v1/workbench/events?afterSeq=10&afterOutboxSeq=11"); + assert.equal(workbenchRealtimeAfterSeq({ headers: { "last-event-id": "42" } }, url), 42); +}); + +test("workbench realtime resets a cursor ahead of scoped cutoff with one fresh snapshot", async () => { + const sessionId = "ses_realtime_cursor_reset"; + const traceId = "trc_realtime_cursor_reset"; + const calls = []; + const runtime = { + async readAtomicWorkbenchProjectionSync(params = {}) { + calls.push({ ...params }); + if (params.snapshotOnly === true) { + return { facts: { sessions: [{ sessionId, lastTraceId: traceId }], messages: [{ messageId: "msg_cursor_reset", sessionId, traceId, text: "fresh", projectedSeq: 5 }], turns: [] }, events: [], cutoffOutboxSeq: 5, cursorOutboxSeq: 5, hasMore: false }; + } + return { facts: {}, events: [], cutoffOutboxSeq: 5, cursorOutboxSeq: 5, hasMore: false }; + } + }; + const server = createCloudApiServer({ accessController: realtimeAccessController(), workbenchRuntime: runtime, env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const events = await getSseEvents(server.address().port, `/v1/workbench/events?sessionId=${sessionId}&afterSeq=42`, 2); + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.message.snapshot"]); + assert.equal(events[1].id, "5"); + assert.deepEqual(calls.map((call) => [call.deltaOnly, call.snapshotOnly]), [[true, false], [false, true]]); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + test("workbench realtime stream surfaces facts blocker instead of legacy trace fallback", async () => { const traceStore = createCodeAgentTraceStore(); const results = createCodeAgentChatResultStore(); @@ -63,36 +714,44 @@ test("workbench realtime stream surfaces facts blocker instead of legacy trace f async ensureBootstrap() {}, async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } }; - const server = createCloudApiServer({ accessController, traceStore, codeAgentChatResults: results, env: { HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000" } }); + const workbenchRuntime = { + async readAtomicWorkbenchProjectionSync() { + const error = new Error("atomic projection unavailable"); + error.code = "TEST_ATOMIC_PROJECTION_UNAVAILABLE"; + throw error; + } + }; + const server = createCloudApiServer({ + accessController, + traceStore, + codeAgentChatResults: results, + workbenchRuntime, + env: { + ...TRANSACTIONAL_REALTIME_ENV, + HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", + HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" + } + }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); try { const { port } = server.address(); - const eventsPromise = getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}&traceId=${encodeURIComponent(traceId)}`, 4); + const eventsPromise = getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}&traceId=${encodeURIComponent(traceId)}`, 2); const events = await eventsPromise; assert.deepEqual(events.map((event) => event.event), [ "workbench.connected", - "workbench.error", - "workbench.trace.snapshot", - "workbench.turn.snapshot" + "workbench.error" ]); assert.equal(events[0].data.filters.sessionId, session.id); - assert.equal(events[1].data.error.code, "workbench_facts_trace_missing"); - assert.equal(events[2].data.traceId, traceId); - assert.equal(events[2].data.snapshot.status, "unknown"); - assert.equal(events[2].data.snapshot.eventCount, 0); - assert.equal(events[3].data.turn.traceId, traceId); - assert.equal(events[3].data.turn.status, "unknown"); - assert.equal(events[3].data.turn.terminal, false); - assert.equal(events[3].data.turn.finalResponse, null); + assert.equal(events[1].data.error.code, "TEST_ATOMIC_PROJECTION_UNAVAILABLE"); assert.equal(JSON.stringify(events).includes("legacy result answer"), false); assert.equal(JSON.stringify(events).includes("legacy trace answer"), false); } finally { await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); } }); -test("workbench realtime stream forwards HWLAB Kafka events after initial connection", async () => { - const fakeKafka = createFakeKafkaFactory(); +test("workbench session realtime follows durable outbox beyond stale lastTraceId", async () => { + const staleTraceId = "trc_workbench_realtime_stale"; const traceId = "trc_workbench_realtime_after_seq"; const session = { id: "ses_workbench_realtime_after_seq", @@ -102,17 +761,25 @@ test("workbench realtime stream forwards HWLAB Kafka events after initial connec ownerUserId: ACTOR.id, conversationId: "cnv_workbench_realtime_after_seq", threadId: "thread-workbench-realtime-after-seq", - lastTraceId: traceId, + lastTraceId: staleTraceId, updatedAt: "2026-06-24T14:00:00.000Z", - session: { sessionStatus: "running", lastTraceId: traceId } + session: { sessionStatus: "running", lastTraceId: staleTraceId } }; const outboxQueries = []; const workbenchRuntime = { - async readWorkbenchProjectionOutbox(params = {}) { + async readAtomicWorkbenchProjectionSync(params = {}) { outboxQueries.push({ ...params }); - return [ - { outboxSeq: 11, projectedSeq: 7, traceId, sessionId: session.id, turnId: traceId, commitType: "event", terminal: false, sealed: false, createdAt: "2026-06-24T14:00:01.000Z" } - ]; + const after = Number(params.afterOutboxSeq ?? 0); + const outboxSeq = after + 1; + const projectedSeq = outboxSeq === 11 ? 7 : 8; + const event = { id: `wte_workbench_realtime_after_seq_${outboxSeq}`, sourceEventId: `src_workbench_realtime_after_seq_${outboxSeq}`, projectedSeq, sourceSeq: projectedSeq, traceId, sessionId: session.id, turnId: traceId, eventType: "backend", status: "running", message: outboxSeq === 11 ? "durable outbox realtime event" : "second outbox page", terminal: false, sealed: false, updatedAt: "2026-06-24T14:00:01.000Z" }; + return { + facts: { sessions: [{ sessionId: session.id, ownerUserId: ACTOR.id, lastTraceId: staleTraceId, status: "running" }], messages: [], parts: [], turns: [], traceEvents: [event], checkpoints: [] }, + events: [{ outboxSeq, outboxEventId: `outbox-workbench-realtime-after-seq-${outboxSeq}`, entityFamily: "traceEvents", entityId: event.id, projectionRevision: projectedSeq, projectedSeq, traceId, sessionId: session.id, turnId: traceId, commitType: "event", terminal: false, sealed: false, payload: { family: "traceEvents", fact: event }, createdAt: event.updatedAt }], + cutoffOutboxSeq: 12, + cursorOutboxSeq: outboxSeq, + hasMore: outboxSeq < 12 + }; } }; const accessController = { @@ -126,110 +793,34 @@ test("workbench realtime stream forwards HWLAB Kafka events after initial connec const serverWithKafka = createCloudApiServer({ accessController, workbenchRuntime, - kafkaFactory: fakeKafka.factory, env: { + ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", - HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", - HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", - HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1" + HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } }); await new Promise((resolve) => serverWithKafka.listen(0, "127.0.0.1", resolve)); try { const { port } = serverWithKafka.address(); - setTimeout(() => { void fakeKafka.emit({ - eventType: "hwlab.trace.event.projected", - sessionId: "ses_agentrun_workbench_realtime_after_seq", - traceId, - context: { sourceSeq: 7, runId: "run_workbench_realtime_after_seq", commandId: "cmd_workbench_realtime_after_seq" }, - event: { type: "backend", eventType: "backend", status: "running", label: "agentrun:event:test", message: "Kafka realtime event", sourceSeq: 7 } - }); }, 25); - const events = await getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}&traceId=${encodeURIComponent(traceId)}&afterSeq=10`, 2); - assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.trace.event"]); + const events = await getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}&afterSeq=10`, 3); + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.trace.event", "workbench.trace.event"]); assert.equal(events[0].id, "10"); - assert.equal(events[1].id, "hwlab.event.v1:0:0"); - assert.equal(events[1].data.realtimeSource, "kafka"); + assert.equal(events[1].id, "11"); + assert.equal(events[1].data.realtimeSource, "projection-outbox"); assert.equal(events[1].data.realtimeAuthority, "workbench-realtime-authority-v2"); assert.equal(events[1].data.entity.family, "traceEvents"); assert.equal(events[1].data.entity.version, 7); - assert.equal(events[1].data.entity.projectionRevision, "kafka:hwlab.event.v1:0:0"); - assert.equal(events[1].data.event.message, "Kafka realtime event"); - assert.equal(events[1].data.kafka.topic, "hwlab.event.v1"); + assert.equal(events[1].data.entity.projectionRevision, "7"); + assert.equal(events[1].data.event.message, "durable outbox realtime event"); + assert.equal(events[1].data.traceId, traceId); assert.equal(events[1].data.cursor.traceSeq, 7); - assert.deepEqual(outboxQueries, []); - } finally { - await new Promise((resolve, reject) => serverWithKafka.close((error) => error ? reject(error) : resolve())); - } -}); - -test("workbench session realtime Kafka stream does not pin subscription to stale lastTraceId", async () => { - const fakeKafka = createFakeKafkaFactory(); - const staleTraceId = "trc_workbench_realtime_stale_trace"; - const liveTraceId = "trc_workbench_realtime_live_trace"; - const session = { - id: "ses_workbench_realtime_session_wide", - projectId: "prj_hwpod_workbench", - agentId: "hwlab-code-agent", - status: "running", - ownerUserId: ACTOR.id, - conversationId: "cnv_workbench_realtime_session_wide", - threadId: "thread-workbench-realtime-session-wide", - lastTraceId: staleTraceId, - updatedAt: "2026-06-24T14:00:00.000Z", - session: { sessionStatus: "running", lastTraceId: staleTraceId } - }; - const accessController = { - store: { - async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, - async getAgentSessionByTraceId() { return null; } - }, - async ensureBootstrap() {}, - async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } - }; - const serverWithKafka = createCloudApiServer({ - accessController, - workbenchRuntime: { - async queryWorkbenchFacts(params = {}) { - return { - facts: { - sessions: [{ sessionId: session.id, ownerUserId: ACTOR.id, threadId: session.threadId, lastTraceId: staleTraceId, status: "running", valuesRedacted: true }], - messages: [], - parts: [], - turns: [], - checkpoints: [] - }, - count: 1, - persistence: { adapter: "test-session-wide-kafka", durable: true }, - params - }; - } - }, - kafkaFactory: fakeKafka.factory, - env: { - HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", - HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", - HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", - HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1" - } - }); - await new Promise((resolve) => serverWithKafka.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = serverWithKafka.address(); - setTimeout(() => { void fakeKafka.emit({ - eventType: "hwlab.trace.event.projected", - sessionId: "ses_agentrun_workbench_realtime_session_wide", - traceId: liveTraceId, - context: { runId: "run_workbench_realtime_session_wide", commandId: "cmd_workbench_realtime_session_wide" }, - event: { type: "backend", eventType: "backend", status: "running", label: "agentrun:event:live", message: "Kafka live trace event" } - }); }, 100); - const events = await getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}`, 4); - assert.equal(events[0].event, "workbench.connected"); - const liveEvent = events.find((event) => event.event === "workbench.trace.event" && event.data?.traceId === liveTraceId); - assert.ok(liveEvent, JSON.stringify(events.map((event) => ({ event: event.event, traceId: event.data?.traceId, message: event.data?.event?.message })))); - assert.equal(liveEvent.data.event.message, "Kafka live trace event"); - assert.equal(liveEvent.data.realtimeAuthority, "workbench-realtime-authority-v2"); + assert.equal(events[2].id, "12"); + assert.equal(events[2].data.event.message, "second outbox page"); + assert.equal(outboxQueries.length, 2); + assert.deepEqual(outboxQueries.map((query) => query.afterOutboxSeq), [10, 11]); + assert.equal(outboxQueries[0].sessionId, session.id); + assert.deepEqual(outboxQueries[0].actor, { id: ACTOR.id, role: ACTOR.role }); } finally { await new Promise((resolve, reject) => serverWithKafka.close((error) => error ? reject(error) : resolve())); } @@ -277,7 +868,7 @@ test("workbench read model exposes runtime trace projection query failures as pr throw error; } }; - const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: results }); + const server = createCloudApiServer({ accessController, traceStore, workbenchRuntime: runtimeStore, codeAgentChatResults: results }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); try { @@ -355,7 +946,7 @@ test("workbench trace events remain visible after session lastTraceId moves", as async ensureBootstrap() {}, async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } }; - const server = createCloudApiServer({ accessController, runtimeStore }); + const server = createCloudApiServer({ accessController, workbenchRuntime: runtimeStore }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); try { @@ -446,3 +1037,14 @@ test("workbench trace event page keeps per-trace terminal status after later ses await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); } }); + +function realtimeAccessController({ actor = ACTOR, sessions = [] } = {}) { + const bySessionId = new Map(sessions.map((session) => [session.id, session])); + const byTraceId = new Map(sessions.filter((session) => session.lastTraceId).map((session) => [session.lastTraceId, session])); + return { + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor, session: { id: "uss_realtime_test" } }; }, + async getAgentSession(sessionId) { return bySessionId.get(sessionId) ?? null; }, + async getAgentSessionByTraceId(traceId) { return byTraceId.get(traceId) ?? null; } + }; +} diff --git a/internal/cloud/server-workbench-realtime-http.ts b/internal/cloud/server-workbench-realtime-http.ts index 548b8ec8..b83fdb14 100644 --- a/internal/cloud/server-workbench-realtime-http.ts +++ b/internal/cloud/server-workbench-realtime-http.ts @@ -4,7 +4,6 @@ */ import { createHash } from "node:crypto"; -import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; import { parsePositiveInteger, safeConversationId, @@ -15,11 +14,15 @@ import { } from "./server-http-utils.ts"; import { createWorkbenchReadModel } from "./workbench-read-model.ts"; import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts"; +import { emitLiveKafkaOtelSpan } from "./kafka-event-bridge.ts"; import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts"; import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts"; -import { openKafkaEventStream } from "./kafka-event-bridge.ts"; -import { durableTraceStatus, RUNNING_STATUSES, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts"; +import { projectionOutboxRealtimeEvents } from "./workbench-projection-outbox-events.ts"; +import { durableTraceStatus, RUNNING_STATUSES, TERMINAL_STATUSES } from "./workbench-turn-projection.ts"; import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts"; +import { + workbenchRealtimeCapabilities +} from "./workbench-realtime-capabilities.ts"; import * as workbenchFacts from "./server-workbench-facts.ts"; const DEFAULT_PAGE_LIMIT = 50; @@ -27,7 +30,6 @@ const DEFAULT_SESSION_LIST_LIMIT = 20; const MAX_PAGE_LIMIT = 100; const DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS = 15000; const WORKBENCH_REALTIME_DRAIN_TIMEOUT_MS = 2500; -const WORKBENCH_REALTIME_AUTHORITY_VERSION = "workbench-realtime-authority-v2"; const WORKBENCH_SESSION_LIST_PAGE_FAMILIES = Object.freeze(["sessions"]); const WORKBENCH_SESSION_DETAIL_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]); const WORKBENCH_SESSION_MESSAGE_PAGE_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]); @@ -180,14 +182,25 @@ export async function drainWorkbenchRealtimeConnections(options = {}) { export async function handleWorkbenchRealtimeHttp(request, response, url, options = {}) { try { if (request.method !== "GET") return methodNotAllowed(response, "GET"); + const realtimeCapabilities = workbenchRealtimeCapabilities(options.env ?? process.env); + const projectionOnly = url.pathname === "/v1/workbench/projection-events"; + if (!projectionOnly && realtimeCapabilities.liveKafkaSse) { + await handleLiveKafkaWorkbenchRealtimeHttp(request, response, url, options); + return; + } + if (!realtimeCapabilities.projectionRealtime) { + sendJson(response, 503, workbenchError("workbench_realtime_disabled", "No Workbench realtime SSE capability is enabled.")); + return; + } const requestedSessionId = safeSessionId(url.searchParams.get("sessionId") ?? url.searchParams.get("includeSessionId")); const requestedTraceId = safeTraceId(url.searchParams.get("traceId")); const heartbeatMs = parsePositiveInteger(options.env?.HWLAB_WORKBENCH_SSE_HEARTBEAT_MS, DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS); attachWorkbenchRealtimeOtelContext(request, { + route: projectionOnly ? "/v1/workbench/projection-events" : "/v1/workbench/events", sessionId: requestedSessionId, traceId: requestedTraceId, heartbeatMs, - realtimeSource: "kafka" + realtimeSource: "projection-outbox" }); const perf = options.backendPerformance; const auth = perf ? await perf.measure("workbench_auth", () => authenticateWorkbenchRead(request, response, options)) : await authenticateWorkbenchRead(request, response, options); @@ -198,9 +211,17 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option return; } - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - const readModel = createWorkbenchReadModel(options, auth.actor); + if (!requestedSessionId && !requestedTraceId) { + sendJson(response, 400, workbenchError("workbench_realtime_scope_required", "Workbench realtime requires sessionId or traceId.")); + return; + } + const runtime = options.workbenchRuntime ?? createWorkbenchRuntimeClient({ env: options.env ?? process.env, fetch: options.fetch, traceparent: request?.hwlabHttpRequestContext?.traceparent }); + if (typeof runtime?.readAtomicWorkbenchProjectionSync !== "function") { + sendJson(response, 503, workbenchError("workbench_realtime_runtime_unconfigured", "Workbench realtime requires an atomic projection snapshot reader.")); + return; + } const requestedAfterSeq = workbenchRealtimeAfterSeq(request, url); + const outboxTailBatchSize = requiredPositiveRuntimeSetting(options.env ?? process.env, "HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE"); let closed = false; let realtimeCloseReason = "client_close"; let realtimeCloseSignal = null; @@ -210,6 +231,315 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option const realtimeStartedAtMs = Date.now(); const cleanup = []; + let streamSessionId = requestedSessionId ?? null; + let streamThreadId = null; + let activeTraceId = requestedTraceId ?? null; + let outboxCursor = requestedAfterSeq; + let scanRunning = false; + let scanRequested = false; + let initialScanState = "pending"; + let initialScanWakeRequested = false; + let realtimeConnection = null; + + const isActive = () => !closed && !response.destroyed && !response.writableEnded; + const closeConnection = () => { + if (closed) return; + closed = true; + emitWorkbenchRealtimeClosedOtelSpan(request, options.env, { + reason: realtimeCloseReason, + signal: realtimeCloseSignal, + startedAtMs: realtimeStartedAtMs, + sessionId: realtimeSessionId, + threadId: realtimeThreadId, + traceId: realtimeTraceId, + activeConnectionCount: activeWorkbenchRealtimeConnections.size + }); + for (const item of cleanup.splice(0)) item(); + }; + + response.writeHead(200, { + "content-type": "text/event-stream; charset=utf-8", + "cache-control": "no-store, no-transform", + connection: "keep-alive", + "x-accel-buffering": "no", + "x-content-type-options": "nosniff" + }); + response.once("close", closeConnection); + request.once?.("aborted", closeConnection); + request.socket?.once?.("close", closeConnection); + cleanup.push(() => request.off?.("aborted", closeConnection)); + cleanup.push(() => request.socket?.off?.("close", closeConnection)); + if (typeof response.flushHeaders === "function") response.flushHeaders(); + + const writeEvent = async (name, payload = {}) => { + if (!isActive()) return false; + const startedAt = nowMs(); + const eventCreatedAt = realtimeEventCreatedAt(payload); + const traceSeq = realtimeTraceSeq(payload); + const eventId = realtimeEventId(payload); + const block = `event: ${name}\n${eventId ? `id: ${eventId}\n` : ""}data: ${JSON.stringify({ contractVersion: "workbench-events-v1", serverSentAt: new Date().toISOString(), eventCreatedAt, traceSeq, ...payload })}\n\n`; + const writable = response.write(block); + if (!writable) await waitForResponseDrain(response, isActive); + perf?.recordPhase({ phase: "sse_write", durationMs: nowMs() - startedAt, outcome: isActive() ? "ok" : "closed" }); + return isActive(); + }; + + realtimeConnection = { + close(fields = {}) { + if (!isActive()) return false; + const reason = textValue(fields.reason) || "server_shutdown"; + realtimeCloseReason = reason; + realtimeCloseSignal = textValue(fields.signal) || null; + void writeEvent("workbench.server_draining", { + type: "server.draining", + status: "closing", + reason, + signal: realtimeCloseSignal, + sessionId: realtimeSessionId, + threadId: realtimeThreadId, + traceId: realtimeTraceId, + observedAt: new Date().toISOString() + }).finally(() => { + if (!response.writableEnded) response.end(); + }); + return true; + }, + isActive + }; + activeWorkbenchRealtimeConnections.add(realtimeConnection); + cleanup.push(() => activeWorkbenchRealtimeConnections.delete(realtimeConnection)); + + await writeEvent("workbench.connected", { + type: "connected", + status: "connected", + realtimeSource: "projection-outbox", + snapshotSource: "atomic-projection-sync", + heartbeatMs, + cursor: { outboxSeq: requestedAfterSeq }, + filters: { sessionId: requestedSessionId, traceId: requestedTraceId } + }); + + realtimeSessionId = streamSessionId ?? requestedSessionId ?? null; + realtimeTraceId = activeTraceId ?? requestedTraceId ?? null; + realtimeThreadId = streamThreadId ?? null; + attachWorkbenchRealtimeOtelContext(request, { + sessionId: streamSessionId ?? requestedSessionId, + traceId: activeTraceId, + threadId: streamThreadId, + heartbeatMs, + realtimeSource: "projection-outbox" + }); + emitWorkbenchRealtimeAcceptedOtelSpan(request, options.env); + + const syncParams = (extra = {}) => ({ + sessionId: requestedSessionId, + traceId: requestedTraceId, + afterOutboxSeq: outboxCursor, + afterSeq: outboxCursor, + limit: outboxTailBatchSize, + actor: { id: auth.actor?.id, role: auth.actor?.role ?? "user" }, + ...extra + }); + const updateStreamScope = (snapshot) => { + const session = factArray(snapshot?.facts?.sessions)[0] ?? null; + streamSessionId = textValue(session?.sessionId) || requestedSessionId || streamSessionId; + streamThreadId = safeOpaqueId(session?.threadId) || textValue(session?.threadId) || streamThreadId; + activeTraceId = requestedTraceId || textValue(session?.lastTraceId) || activeTraceId; + realtimeSessionId = streamSessionId; + realtimeThreadId = streamThreadId; + realtimeTraceId = activeTraceId; + }; + const emitProjectionSnapshot = async (snapshot) => { + updateStreamScope(snapshot); + for (const item of projectionOutboxRealtimeEvents(snapshot, { includeSnapshot: true, includeEvents: false })) { + if (!await writeEvent(item.name, item.payload)) break; + } + outboxCursor = nonNegativeInteger(snapshot?.cutoffOutboxSeq ?? snapshot?.cursorOutboxSeq); + }; + const recoverCursor = async () => { + const snapshot = await runtime.readAtomicWorkbenchProjectionSync(syncParams({ afterOutboxSeq: 0, afterSeq: 0, snapshotOnly: true, deltaOnly: false })); + await emitProjectionSnapshot(snapshot); + }; + const runProjectionScan = async ({ initial = false } = {}) => { + const snapshotOnly = initial && requestedAfterSeq <= 0; + if (snapshotOnly) { + await recoverCursor(); + return; + } + let hasMore = true; + while (hasMore && isActive()) { + const snapshot = await runtime.readAtomicWorkbenchProjectionSync(syncParams({ snapshotOnly: false, deltaOnly: true })); + const nextCursor = nonNegativeInteger(snapshot?.cursorOutboxSeq); + if (nextCursor < outboxCursor) { + await recoverCursor(); + return; + } + for (const item of projectionOutboxRealtimeEvents(snapshot)) { + if (!await writeEvent(item.name, item.payload)) return; + } + outboxCursor = nextCursor; + hasMore = snapshot?.hasMore === true; + } + }; + const emitProjectionScanError = async (error) => { + await writeEvent("workbench.error", { type: "error", realtimeSource: "projection-outbox", sessionId: streamSessionId, traceId: activeTraceId, error: { code: error?.code ?? "workbench_outbox_scan_failed", message: error?.message ?? "Workbench projection outbox scan failed.", valuesRedacted: true } }); + }; + const scanProjectionOutbox = async () => { + if (!isActive()) return; + if (initialScanState !== "complete") { + if (initialScanState === "pending") initialScanWakeRequested = true; + return; + } + if (scanRunning) { + scanRequested = true; + return; + } + scanRunning = true; + try { + do { + scanRequested = false; + await runProjectionScan({ initial: false }); + } while (scanRequested && isActive()); + } catch (error) { + await emitProjectionScanError(error); + if (!response.writableEnded) response.end(); + } finally { + scanRunning = false; + } + }; + const requestProjectionScan = () => { + if (!isActive()) return; + if (initialScanState !== "complete") { + if (initialScanState === "pending") initialScanWakeRequested = true; + return; + } + scanRequested = true; + void scanProjectionOutbox(); + }; + const unsubscribeProjectionCommits = options.kafkaEventBridge?.subscribeProjectionCommits?.((signal = {}) => { + if (workbenchProjectionSignalMatches(signal, requestedSessionId, requestedTraceId)) requestProjectionScan(); + }); + if (typeof unsubscribeProjectionCommits === "function") cleanup.push(unsubscribeProjectionCommits); + + try { + await runProjectionScan({ initial: true }); + initialScanState = "complete"; + } catch (error) { + initialScanState = "failed"; + await emitProjectionScanError(error); + if (!response.writableEnded) response.end(); + return; + } + if (initialScanState === "complete" && initialScanWakeRequested && isActive()) { + initialScanWakeRequested = false; + scanRequested = true; + await scanProjectionOutbox(); + } + if (!isActive()) return; + + const heartbeat = setInterval(() => { + void writeEvent("workbench.heartbeat", { + type: "heartbeat", + sessionId: requestedSessionId ?? null, + traceId: activeTraceId ?? null, + observedAt: new Date().toISOString() + }); + }, Math.max(1000, heartbeatMs)); + heartbeat.unref?.(); + cleanup.push(() => clearInterval(heartbeat)); + } catch (error) { + handleWorkbenchRealtimeFailure(request, response, url, options, error); + } +} + +async function handleLiveKafkaWorkbenchRealtimeHttp(request, response, url, options = {}) { + const requestedSessionId = safeSessionId(url.searchParams.get("sessionId") ?? url.searchParams.get("includeSessionId")); + const requestedTraceId = safeTraceId(url.searchParams.get("traceId")); + const realtimeCapabilities = workbenchRealtimeCapabilities(options.env ?? process.env); + const heartbeatMs = parsePositiveInteger(options.env?.HWLAB_WORKBENCH_SSE_HEARTBEAT_MS, DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS); + attachWorkbenchRealtimeOtelContext(request, { + sessionId: requestedSessionId, + traceId: requestedTraceId, + heartbeatMs, + realtimeSource: "live-kafka-sse", + realtimeCapabilities + }); + const perf = options.backendPerformance; + const auth = perf ? await perf.measure("workbench_auth", () => authenticateWorkbenchRead(request, response, options)) : await authenticateWorkbenchRead(request, response, options); + if (!auth) return; + if (url.searchParams.has("projectId") || url.searchParams.has("workspaceId")) { + sendJson(response, 400, workbenchError("workbench_authority_removed", "Workbench realtime is keyed by sessionId/traceId only.")); + return; + } + if (!requestedSessionId && !requestedTraceId) { + sendJson(response, 400, workbenchError("workbench_realtime_scope_required", "Workbench realtime requires sessionId or traceId.")); + return; + } + const authorization = await authorizeLiveKafkaWorkbenchRealtimeScope(options, auth.actor, { + sessionId: requestedSessionId, + traceId: requestedTraceId + }); + if (!authorization.ok) { + sendJson(response, authorization.status, authorization.body); + return; + } + const bridge = options.kafkaEventBridge; + if (bridge?.capabilities?.liveKafkaSse !== true || typeof bridge?.subscribeLiveHwlabEvents !== "function") { + sendJson(response, 503, workbenchError("workbench_live_kafka_unconfigured", "Workbench live Kafka fanout is not configured.")); + return; + } + + let closed = false; + let realtimeCloseReason = "client_close"; + let realtimeCloseSignal = null; + const realtimeStartedAtMs = Date.now(); + const cleanup = []; + const isActive = () => !closed && !response.destroyed && !response.writableEnded; + const closeConnection = () => { + if (closed) return; + closed = true; + emitWorkbenchRealtimeClosedOtelSpan(request, options.env, { + reason: realtimeCloseReason, + signal: realtimeCloseSignal, + startedAtMs: realtimeStartedAtMs, + sessionId: requestedSessionId, + threadId: null, + traceId: requestedTraceId, + activeConnectionCount: activeWorkbenchRealtimeConnections.size + }); + for (const item of cleanup.splice(0)) item(); + }; + + response.once("close", closeConnection); + request.once?.("aborted", closeConnection); + request.socket?.once?.("close", closeConnection); + cleanup.push(() => request.off?.("aborted", closeConnection)); + cleanup.push(() => request.socket?.off?.("close", closeConnection)); + + const bufferedEnvelopes = []; + let liveDeliveryStarted = false; + let enqueueEvent = null; + const unsubscribe = bridge.subscribeLiveHwlabEvents((envelope, transport) => { + if (!liveKafkaEnvelopeMatches(envelope, requestedSessionId, requestedTraceId)) return; + if (!liveDeliveryStarted || typeof enqueueEvent !== "function") { + bufferedEnvelopes.push({ envelope, transport }); + return; + } + void enqueueEvent("hwlab.event.v1", envelope, transport); + }); + if (typeof unsubscribe === "function") cleanup.push(unsubscribe); + + try { + await bridge.ready; + } catch (error) { + closeConnection(); + throw error; + } + if (!isActive()) { + closeConnection(); + return; + } + response.writeHead(200, { "content-type": "text/event-stream; charset=utf-8", "cache-control": "no-store, no-transform", @@ -219,174 +549,129 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option }); if (typeof response.flushHeaders === "function") response.flushHeaders(); - const writeEvent = (name, payload = {}) => { - if (closed || response.destroyed) return; - const startedAt = nowMs(); - const eventCreatedAt = realtimeEventCreatedAt(payload); - const traceSeq = realtimeTraceSeq(payload); - const eventId = realtimeEventId(payload); - response.write(`event: ${name}\n`); - if (eventId) response.write(`id: ${eventId}\n`); - response.write(`data: ${JSON.stringify({ contractVersion: "workbench-events-v1", serverSentAt: new Date().toISOString(), eventCreatedAt, traceSeq, ...payload })}\n\n`); - perf?.recordPhase({ phase: "sse_write", durationMs: nowMs() - startedAt, outcome: "ok" }); + const writeEvent = async (name, payload, transport = null) => { + if (!isActive()) return false; + const block = `event: ${name}\ndata: ${JSON.stringify(payload)}\n\n`; + const writable = response.write(block); + if (!writable) await waitForResponseDrain(response, isActive); + const active = isActive(); + if (active && name === "hwlab.event.v1") { + emitLiveKafkaOtelSpan("hwlab.workbench.live_sse.business_event_write", payload, transport, { + env: options.env, + otelSpanEmitter: options.otelSpanEmitter ?? emitCodeAgentOtelSpan + }); + } + return active; + }; + let writeChain = Promise.resolve(true); + enqueueEvent = (name, payload, transport = null) => { + writeChain = writeChain.then(() => writeEvent(name, payload, transport)); + return writeChain; }; const realtimeConnection = { close(fields = {}) { - if (closed || response.destroyed || response.writableEnded) return false; - const reason = textValue(fields.reason) || "server_shutdown"; - realtimeCloseReason = reason; + if (!isActive()) return false; + realtimeCloseReason = textValue(fields.reason) || "server_shutdown"; realtimeCloseSignal = textValue(fields.signal) || null; - try { - writeEvent("workbench.server_draining", { - type: "server.draining", - status: "closing", - reason, - signal: realtimeCloseSignal, - sessionId: realtimeSessionId, - threadId: realtimeThreadId, - traceId: realtimeTraceId, - observedAt: new Date().toISOString() - }); - response.end(); - return true; - } catch { - try { response.destroy(); } catch {} - return false; - } + void enqueueEvent("workbench.server_draining", { + type: "server.draining", + status: "closing", + reason: realtimeCloseReason, + signal: realtimeCloseSignal, + capabilities: realtimeCapabilities, + lossPossible: true + }).finally(() => { + if (!response.writableEnded) response.end(); + }); + return true; }, - isActive() { - return !closed && !response.destroyed && !response.writableEnded; - } + isActive }; activeWorkbenchRealtimeConnections.add(realtimeConnection); cleanup.push(() => activeWorkbenchRealtimeConnections.delete(realtimeConnection)); - writeEvent("workbench.connected", { + await enqueueEvent("workbench.connected", { type: "connected", status: "connected", - realtimeSource: "kafka", - snapshotSource: "read-model", - heartbeatMs, - cursor: { outboxSeq: requestedAfterSeq }, - filters: { - sessionId: requestedSessionId, - traceId: requestedTraceId - } + capabilities: realtimeCapabilities, + realtimeSource: "hwlab.event.v1", + deliverySemantics: "live-only", + liveOnly: true, + replay: false, + replaySupported: false, + lossPossible: true, + filters: { sessionId: requestedSessionId, traceId: requestedTraceId } }); - - const resolveInitialSession = requestedSessionId && requestedAfterSeq <= 0; - const initialContext = resolveInitialSession - ? await realtimeInitialSessionContext(readModel, auth.actor, requestedSessionId) - : { facts: emptyFactSet(), session: null, blocker: null }; - if (initialContext.blocker) { - writeEvent("workbench.error", { - type: "error", - reason: "initial-session", - sessionId: requestedSessionId, - traceId: requestedTraceId, - error: initialContext.blocker - }); + liveDeliveryStarted = true; + const readyBufferedEnvelopes = bufferedEnvelopes.splice(0); + for (const { envelope, transport } of readyBufferedEnvelopes) void enqueueEvent("hwlab.event.v1", envelope, transport); + await writeChain; + if (!isActive()) { + closeConnection(); + return; } - const initialFactSession = initialContext.session; - const streamSessionId = factSessionId(initialFactSession) ?? requestedSessionId ?? null; - const streamThreadId = safeOpaqueId(initialFactSession?.threadId) ?? (textValue(initialFactSession?.threadId) || null); - const activeTraceId = requestedTraceId - ?? factLastTraceId(initialFactSession) - ?? factLatestTraceIdForSession(initialContext.facts, streamSessionId) - ?? null; - realtimeSessionId = streamSessionId ?? requestedSessionId ?? null; - realtimeTraceId = activeTraceId ?? requestedTraceId ?? null; - realtimeThreadId = streamThreadId ?? null; - attachWorkbenchRealtimeOtelContext(request, { - sessionId: streamSessionId ?? requestedSessionId, - traceId: activeTraceId, - threadId: streamThreadId, - heartbeatMs, - realtimeSource: "kafka" - }); + const heartbeatTimer = setInterval(() => { + void enqueueEvent("workbench.heartbeat", { + type: "heartbeat", + status: "connected", + realtimeSource: "hwlab.event.v1", + liveOnly: true, + replay: false, + lossPossible: true, + serverSentAt: new Date().toISOString(), + valuesPrinted: false + }); + }, heartbeatMs); + heartbeatTimer.unref?.(); + cleanup.push(() => clearInterval(heartbeatTimer)); emitWorkbenchRealtimeAcceptedOtelSpan(request, options.env); - if (activeTraceId && requestedAfterSeq <= 0) { - await (perf ? perf.measure("workbench_initial_trace", () => writeTraceRealtimeSnapshotSafe({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" })) : writeTraceRealtimeSnapshotSafe({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" })); - } - const kafkaFilters = resolveWorkbenchRealtimeKafkaFilters({ traceId: requestedTraceId, sessionId: streamSessionId, traceStore }); - try { - const kafkaStream = await openKafkaEventStream({ - env: options.env ?? process.env, - stream: "hwlab", - fromBeginning: false, - ...kafkaFilters, - kafkaFactory: options.kafkaFactory, - onEvent: async (record) => { - if (closed || response.destroyed) return; - const realtime = workbenchRealtimeEventFromKafka(record, { sessionId: streamSessionId, threadId: streamThreadId, traceId: activeTraceId }); - if (!realtime) return; - writeEvent("workbench.trace.event", realtime.traceEvent); - if (realtime.turnSnapshot) writeEvent("workbench.turn.snapshot", realtime.turnSnapshot); - }, - onError: (error) => { - writeEvent("workbench.error", { - type: "error", - realtimeSource: "kafka", - sessionId: streamSessionId, - threadId: streamThreadId, - traceId: activeTraceId, - error: { code: "workbench_kafka_stream_failed", message: error?.message ?? "Workbench Kafka realtime stream failed." } - }); - } - }); - cleanup.push(() => { void kafkaStream.stop?.(); }); - } catch (kafkaError) { - writeEvent("workbench.error", { - type: "error", - realtimeSource: "kafka", - sessionId: streamSessionId, - threadId: streamThreadId, - traceId: activeTraceId, - error: { code: "workbench_kafka_stream_unavailable", message: kafkaError?.message ?? "Workbench Kafka realtime stream is unavailable." } - }); +} + +async function authorizeLiveKafkaWorkbenchRealtimeScope(options, actor, { sessionId, traceId }) { + const access = options.accessController; + if ((sessionId && typeof access?.getAgentSession !== "function") || (traceId && typeof access?.getAgentSessionByTraceId !== "function")) { + return { + ok: false, + status: 503, + body: workbenchError("workbench_realtime_authorization_unconfigured", "Workbench live realtime ownership authorization is not configured.") + }; } - const heartbeat = setInterval(async () => { - try { - writeEvent("workbench.heartbeat", { - type: "heartbeat", - sessionId: requestedSessionId ?? null, - traceId: activeTraceId ?? null, - observedAt: new Date().toISOString() - }); - } catch (error) { - writeEvent("workbench.error", { - type: "error", - error: { code: "workbench_realtime_heartbeat_failed", message: error?.message ?? "Workbench realtime heartbeat failed." } - }); - } - }, Math.max(1000, heartbeatMs)); - cleanup.push(() => clearInterval(heartbeat)); - - response.on("close", () => { - if (closed) return; - closed = true; - emitWorkbenchRealtimeClosedOtelSpan(request, options.env, { - reason: realtimeCloseReason, - signal: realtimeCloseSignal, - startedAtMs: realtimeStartedAtMs, - sessionId: realtimeSessionId, - threadId: realtimeThreadId, - traceId: realtimeTraceId, - activeConnectionCount: activeWorkbenchRealtimeConnections.size - }); - for (const item of cleanup.splice(0)) item(); - }); - } catch (error) { - handleWorkbenchRealtimeFailure(request, response, url, options, error); + const [sessionById, sessionByTrace] = await Promise.all([ + sessionId ? access.getAgentSession(sessionId) : null, + traceId ? access.getAgentSessionByTraceId(traceId) : null + ]); + const sessionByIdId = sessionById ? safeSessionId(sessionById.id ?? sessionById.sessionId) : null; + const sessionByTraceId = sessionByTrace ? safeSessionId(sessionByTrace.id ?? sessionByTrace.sessionId) : null; + const scopeMissing = (sessionId && sessionByIdId !== sessionId) + || (traceId && !sessionByTraceId) + || (sessionId && traceId && sessionByIdId !== sessionByTraceId); + const resolvedSessions = [sessionById, sessionByTrace].filter(Boolean); + const ownedByActor = actor?.role === "admin" || (Boolean(actor?.id) + && resolvedSessions.length > 0 + && resolvedSessions.every((session) => textValue(session.ownerUserId) === actor.id)); + if (scopeMissing || !ownedByActor) { + return { + ok: false, + status: 404, + body: workbenchError("workbench_realtime_scope_not_found", "Workbench realtime scope is not visible to the current actor.") + }; } + return { ok: true, sessionId: sessionByIdId ?? sessionByTraceId }; +} + +function liveKafkaEnvelopeMatches(envelope, sessionId, traceId) { + if (!envelope || envelope.schema !== "hwlab.event.v1") return false; + if (sessionId && safeSessionId(envelope.hwlabSessionId ?? envelope.sessionId) !== sessionId) return false; + if (traceId && safeTraceId(envelope.traceId) !== traceId) return false; + return true; } export function attachWorkbenchRealtimeOtelContext(request, fields = {}) { const context = request?.hwlabHttpRequestContext; if (!context) return; - context.route = "/v1/workbench/events"; + context.route = textValue(fields.route) || "/v1/workbench/events"; context.otelAttributes = { ...(context.otelAttributes ?? {}), "workbench.route": "events", @@ -396,151 +681,13 @@ export function attachWorkbenchRealtimeOtelContext(request, fields = {}) { "workbench.thread_id": fields.threadId ?? null, "workbench.sse.heartbeat_ms": fields.heartbeatMs ?? null, "workbench.sse.realtime_source": fields.realtimeSource ?? null, - "workbench.sse.outbox_mode": false + "workbench.sse.live_kafka_enabled": fields.realtimeCapabilities?.liveKafkaSse ?? null, + "workbench.sse.projection_realtime_enabled": fields.realtimeCapabilities?.projectionRealtime ?? null, + "workbench.sse.outbox_mode": fields.realtimeCapabilities?.liveKafkaSse === true ? false : true }; } -export function resolveWorkbenchRealtimeKafkaFilters({ traceId = null, sessionId = null, traceStore = null } = {}) { - const resolved = traceId && typeof traceStore?.snapshot === "function" ? collectTraceLinkedIds(traceStore.snapshot(traceId)) : {}; - const hasAgentRunKey = Boolean(resolved.runId || resolved.commandId); - const fallbackSessionId = resolved.sessionId || agentRunScopedSessionIdFromWorkbenchSession(sessionId) || sessionId; - return compactObject({ - traceId: hasAgentRunKey ? null : traceId, - sessionId: hasAgentRunKey ? null : fallbackSessionId, - runId: resolved.runId, - commandId: resolved.commandId - }); -} - -export function agentRunScopedSessionIdFromWorkbenchSession(sessionId) { - const safeId = safeSessionId(sessionId); - if (!safeId || isAgentRunAliasSessionId(safeId)) return safeId; - const base = safeId.replace(/^ses_/u, "").replace(/[^A-Za-z0-9_]+/gu, "_").replace(/^_+|_+$/gu, "") || "session"; - return safeSessionId(`ses_agentrun_${base}`); -} - -export function collectTraceLinkedIds(snapshot) { - const out = { sessionId: null, runId: null, commandId: null }; - const events = Array.isArray(snapshot?.events) ? snapshot.events : []; - for (const event of events) collectTraceLinkedIdsFromRecord(event, out); - collectTraceLinkedIdsFromRecord(snapshot?.lastEvent, out); - return out; -} - -export function collectTraceLinkedIdsFromRecord(record, out) { - const value = record && typeof record === "object" && !Array.isArray(record) ? record : {}; - const agentRun = value.agentRun && typeof value.agentRun === "object" && !Array.isArray(value.agentRun) ? value.agentRun : {}; - const payload = value.payload && typeof value.payload === "object" && !Array.isArray(value.payload) ? value.payload : {}; - out.sessionId ||= safeSessionId(value.sessionId ?? value.sourceSessionId ?? agentRun.sessionId ?? payload.sessionId); - out.runId ||= textValue(value.runId ?? value.sourceRunId ?? agentRun.runId ?? payload.runId); - out.commandId ||= textValue(value.commandId ?? value.sourceCommandId ?? agentRun.commandId ?? payload.commandId); -} - -export function workbenchRealtimeEventFromKafka(record, context = {}) { - const value = record?.value && typeof record.value === "object" && !Array.isArray(record.value) ? record.value : null; - if (!value) return null; - const hwlabEvent = value.event && typeof value.event === "object" && !Array.isArray(value.event) ? value.event : {}; - const sourceContext = value.context && typeof value.context === "object" && !Array.isArray(value.context) ? value.context : {}; - const traceId = safeTraceId(value.traceId ?? hwlabEvent.traceId ?? context.traceId) ?? textValue(value.traceId ?? hwlabEvent.traceId ?? context.traceId); - const sessionId = safeSessionId(context.sessionId ?? value.sessionId ?? hwlabEvent.sessionId) ?? textValue(context.sessionId ?? value.sessionId ?? hwlabEvent.sessionId); - const threadId = safeOpaqueId(context.threadId ?? sourceContext.threadId) ?? textValue(context.threadId ?? sourceContext.threadId); - if (!traceId && !sessionId) return null; - const sourceSeq = integerValue(hwlabEvent.sourceSeq ?? sourceContext.sourceSeq); - const kafka = { - topic: textValue(record.topic), - partition: integerValue(record.partition), - offset: textValue(record.offset), - key: textValue(record.key), - timestamp: textValue(record.timestamp), - valuesRedacted: true - }; - const event = { - ...hwlabEvent, - traceId: traceId ?? hwlabEvent.traceId ?? null, - sessionId: sessionId ?? hwlabEvent.sessionId ?? null, - threadId: threadId ?? sourceContext.threadId ?? null, - runId: textValue(hwlabEvent.runId ?? sourceContext.runId), - commandId: textValue(hwlabEvent.commandId ?? sourceContext.commandId), - source: textValue(hwlabEvent.source) || "hwlab.kafka", - sourceSeq, - kafka, - valuesPrinted: false - }; - const status = normalizeStatus(event.status); - const terminal = event.terminal === true || TERMINAL_STATUSES.has(status); - const cursor = { - traceSeq: sourceSeq ?? integerValue(record.offset), - kafkaOffset: textValue(record.offset), - kafkaPartition: integerValue(record.partition) - }; - const entityVersion = sourceSeq ?? integerValue(record.offset) ?? 0; - const projectionRevision = ["kafka", kafka.topic, kafka.partition, kafka.offset].filter((part) => textValue(part)).join(":"); - const traceEntity = { - family: "traceEvents", - id: [traceId ?? "trace", event.runId, event.commandId, kafka.partition, kafka.offset].filter((part) => textValue(part)).join(":"), - version: entityVersion, - traceSeq: cursor.traceSeq, - projectionRevision, - authority: WORKBENCH_REALTIME_AUTHORITY_VERSION - }; - const traceEvent = { - id: kafka.topic && kafka.partition !== null && kafka.offset ? `${kafka.topic}:${kafka.partition}:${kafka.offset}` : null, - type: "trace.event", - contractVersion: "workbench-sync-v1", - realtimeAuthority: WORKBENCH_REALTIME_AUTHORITY_VERSION, - realtimeSource: "kafka", - sessionId, - threadId, - traceId, - event, - snapshot: { traceId, sessionId, threadId, status: status || event.status || null, events: [event], eventCount: 1 }, - cursor, - entity: traceEntity, - projectionRevision, - kafka, - valuesPrinted: false - }; - const turnSnapshot = terminal ? { - type: "turn.snapshot", - contractVersion: "workbench-sync-v1", - realtimeAuthority: WORKBENCH_REALTIME_AUTHORITY_VERSION, - realtimeSource: "kafka", - sessionId, - threadId, - traceId, - reason: "kafka-terminal", - cursor, - turn: { - traceId, - sessionId, - threadId, - status: status || event.status || "completed", - running: false, - terminal: true, - finalResponse: terminalFinalResponse(status || event.status || "completed", { - traceId, - status: status || event.status || "completed", - finalResponse: event.text ? { text: event.text, status: status || event.status || "completed", traceId, source: "kafka-terminal-event", valuesPrinted: false } : null, - finalText: event.text ?? event.message ?? null, - error: event.errorCode ? { message: event.message ?? event.errorCode } : null - }), - agentRun: compactObject({ runId: event.runId, commandId: event.commandId }), - valuesPrinted: false - }, - entity: { - family: "turns", - id: traceId ?? event.runId ?? event.commandId ?? traceEntity.id, - version: entityVersion, - traceSeq: cursor.traceSeq, - projectionRevision, - authority: WORKBENCH_REALTIME_AUTHORITY_VERSION - }, - projectionRevision, - kafka, - valuesPrinted: false - } : null; - return { traceEvent, turnSnapshot }; -} +export { projectionOutboxRealtimeEvents } from "./workbench-projection-outbox-events.ts"; export function compactObject(value) { return Object.fromEntries(Object.entries(value || {}).filter(([, entry]) => textValue(entry))); @@ -587,7 +734,7 @@ export function emitWorkbenchRealtimeAcceptedOtelSpan(request, env = process.env endTimeMs: endedAtMs, statusCode: 200, method: "GET", - route: "/v1/workbench/events", + route: context.route ?? "/v1/workbench/events", attributes: { ...(context.otelAttributes ?? {}), "http.closed_by": "sse-accepted", @@ -608,7 +755,7 @@ export function emitWorkbenchRealtimeClosedOtelSpan(request, env = process.env, endTimeMs: endedAtMs, statusCode: 200, method: "GET", - route: "/v1/workbench/events", + route: context.route ?? "/v1/workbench/events", attributes: { ...(context.otelAttributes ?? {}), "http.closed_by": reason, @@ -713,13 +860,49 @@ export function realtimeEventId(payload) { } export function workbenchRealtimeAfterSeq(request, url) { - for (const value of [url.searchParams.get("afterSeq"), url.searchParams.get("afterOutboxSeq"), request?.headers?.["last-event-id"]]) { + for (const value of [request?.headers?.["last-event-id"], url.searchParams.get("afterSeq"), url.searchParams.get("afterOutboxSeq")]) { const parsed = nonNegativeInteger(value, 0); if (parsed > 0) return parsed; } return 0; } +function requiredPositiveRuntimeSetting(env, name) { + const value = Number(env?.[name]); + if (!Number.isInteger(value) || value <= 0) { + const error = new Error(`${name} must be explicitly configured as a positive integer.`); + error.code = "workbench_outbox_tail_config_invalid"; + throw error; + } + return value; +} + +export function workbenchProjectionSignalMatches(signal = {}, sessionId = null, traceId = null) { + if (signal.recovery === true) return true; + const signalSessionId = textValue(signal.sessionId); + const signalTraceId = textValue(signal.traceId); + if (traceId && signalTraceId !== traceId) return false; + if (sessionId && signalSessionId !== sessionId) return false; + return Boolean(signalSessionId || signalTraceId); +} + +export function waitForResponseDrain(response, isActive = () => true) { + if (!isActive()) return Promise.resolve(false); + return new Promise((resolve) => { + const settle = (result) => { + response.off?.("drain", onDrain); + response.off?.("close", onClose); + response.off?.("error", onClose); + resolve(result); + }; + const onDrain = () => settle(isActive()); + const onClose = () => settle(false); + response.once("drain", onDrain); + response.once("close", onClose); + response.once("error", onClose); + }); +} + export function nowMs() { if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now(); return Date.now(); diff --git a/internal/cloud/server.ts b/internal/cloud/server.ts index a2ffde74..56f3bc29 100644 --- a/internal/cloud/server.ts +++ b/internal/cloud/server.ts @@ -3,8 +3,8 @@ * 职责: Cloud API REST/SSE route dispatcher。Workbench 新资源路由应委托 read model / compat wrapper，不在 dispatcher 内写业务事实。 */ import { createServer } from "node:http"; -import { createHash, randomUUID } from "node:crypto"; -import { copyFileSync, existsSync, lstatSync, mkdirSync, readFileSync, symlinkSync } from "node:fs"; +import { randomUUID } from "node:crypto"; +import { copyFileSync, existsSync, lstatSync, mkdirSync, symlinkSync } from "node:fs"; import path from "node:path"; import { @@ -28,7 +28,6 @@ import { handleCodeAgentChat } from "./code-agent-chat.ts"; import { createDurableCodeAgentTraceStore, defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; -import { writeWorkbenchProjectionEvent } from "./workbench-projection-writer.ts"; import { createCodeAgentSessionRegistry } from "./code-agent-session-registry.ts"; import { codeAgentSessionLifecycleSummary } from "./code-agent-session-lifecycle.ts"; import { createCodexStdioSessionManager } from "./codex-stdio-session.ts"; @@ -66,8 +65,7 @@ import { handleCodeAgentSessionsHttp, handleCodeAgentSteerHttp, handleCodeAgentTurnHttp, - handleCodeAgentTraceHttp, - startAgentRunProjectionResume + handleCodeAgentTraceHttp } from "./server-code-agent-http.ts"; import { drainWorkbenchRealtimeConnections, handleWorkbenchReadModelHttp, handleWorkbenchRealtimeHttp } from "./server-workbench-http.ts"; import { handleWorkbenchDebugFakeSseHttp } from "./workbench-debug-fake-sse.ts"; @@ -87,10 +85,11 @@ import { } from "./skills-store.ts"; import { createHwpodNodeWsRegistry } from "./hwpod-node-ws-registry.ts"; import { - discoverHwpodSpecs, - hwpodSpecDiscoveryPayload, - hwpodSpecWorkspaceProbePlan -} from "./hwpod-spec-discovery.ts"; + handleHwlabNodeDownloadHttp, + handleHwlabNodeUpdateHttp, + handleHwpodNodeOpsHttp, + handleHwpodSpecDiscoveryHttp +} from "./server-hwpod-http.ts"; import { handleCaseRunHttp } from "./server-caserun-http.ts"; import { handleProjectManagementProxyHttp } from "./project-management-proxy.ts"; import { HWPOD_NODE_OPS, HWPOD_NODE_OPS_CONTRACT_VERSION } from "../../tools/src/hwpod-node-ops-contract.ts"; @@ -155,8 +154,7 @@ export function createCloudApiServer(options = {}) { const runtimeStore = options.runtimeStore || createConfiguredCloudRuntimeStore({ ...options, env }); const traceStore = createDurableCodeAgentTraceStore({ traceStore: options.traceStore || defaultCodeAgentTraceStore, - traceEventStore: runtimeStore, - workbenchEventWriter: (event, meta) => writeWorkbenchProjectionEvent({ runtimeStore, event, requestMeta: meta }) + traceEventStore: runtimeStore }); const codexStdioManager = options.codexStdioManager || createCodexStdioSessionManager({ traceStore }); const codeAgentChatResults = options.codeAgentChatResults || createCodeAgentChatResultStore({ @@ -173,13 +171,8 @@ export function createCloudApiServer(options = {}) { if (typeof accessController.configureCodeAgentWorkspaceContext === "function") { accessController.configureCodeAgentWorkspaceContext({ env, fetchImpl: options.fetchImpl, traceStore, codeAgentChatResults }); } - const workbenchEmptySessionGc = startWorkbenchEmptySessionGc({ env, accessController, logger: console }); - const agentRunProjectionResume = startAgentRunProjectionResume({ - options: { ...options, env, runtimeStore, accessController, traceStore, codeAgentChatResults }, - traceStore, - logger: console - }); - const kafkaEventBridge = startHwlabKafkaEventBridge({ env, logger: console }); + const workbenchEmptySessionGc = startWorkbenchEmptySessionGc({ env, accessController, logger: options.logger ?? console }); + const kafkaEventBridge = options.kafkaEventBridge ?? startHwlabKafkaEventBridge({ env, logger: options.logger ?? console, runtimeStore, kafkaFactory: options.kafkaFactory }); const server = createServer(async (request, response) => { const requestContext = buildHttpRequestContext(request); attachHttpRequestContext(request, response, requestContext); @@ -214,7 +207,7 @@ export function createCloudApiServer(options = {}) { response.once("close", () => emitHttpSpan("close")); await withBackendPerformanceContext(backendPerformance, async () => { try { - await routeRequest(request, response, { ...options, env, runtimeStore, gatewayRegistry, hwpodNodeWsRegistry, accessController, sessionRegistry, traceStore, codexStdioManager, codeAgentChatResults, webPerformanceStore, backendPerformanceStore, backendPerformance }); + await routeRequest(request, response, { ...options, env, runtimeStore, kafkaEventBridge, gatewayRegistry, hwpodNodeWsRegistry, accessController, sessionRegistry, traceStore, codexStdioManager, codeAgentChatResults, webPerformanceStore, backendPerformanceStore, backendPerformance, emitHttpRoutePhaseSpan }); } catch (error) { if (error?.alreadySent || response.headersSent || response.writableEnded) return; backendPerformance.recordPhase({ phase: "route_exception", durationMs: 0, outcome: "error" }); @@ -230,9 +223,10 @@ export function createCloudApiServer(options = {}) { }); server.on("close", () => { workbenchEmptySessionGc.stop(); - agentRunProjectionResume.stop(); void kafkaEventBridge.stop(); }); + server.hwlabStartupReady = kafkaEventBridge?.ready ?? Promise.resolve(); + server.hwlabAbortStartup = async () => { workbenchEmptySessionGc.stop(); await kafkaEventBridge.stop(); }; return server; } @@ -339,8 +333,9 @@ export async function buildHealthPayload(options = {}) { const dbProbe = await buildDbRuntimeReadiness(env, liveProbe ? { ...(options.dbProbe ?? {}), probe: false } : options.dbProbe); const codeAgent = await describeCodeAgentAvailability(env, options); const runtime = liveProbe ? runtimeReadinessSnapshot(runtimeStore) : await runtimeReadiness(runtimeStore); + const kafkaProjector = await kafkaProjectorHealth(options.kafkaEventBridge, { liveProbe }); const db = applyRuntimeDbReadinessLayers(dbProbe, runtime); - const readiness = buildCloudApiReadiness({ db, codeAgent, runtime }); + const readiness = buildCloudApiReadiness({ db, codeAgent, runtime, kafkaProjector }); return { serviceId, @@ -366,10 +361,29 @@ export async function buildHealthPayload(options = {}) { observedAt: new Date().toISOString(), db, codeAgent, - runtime + runtime, + kafkaProjector }; } +async function kafkaProjectorHealth(projector, { liveProbe = false } = {}) { + if (!projector?.started) return { started: false, reason: projector?.reason ?? "disabled", valuesRedacted: true }; + const identity = { + capabilities: projector.capabilities ?? null, + directPublishGroupId: projector.directPublishGroupId ?? null, + projectorGroupId: projector.projectorGroupId ?? null, + hwlabEventGroupId: projector.hwlabEventGroupId ?? null, + agentRunTopic: projector.agentRunTopic, + hwlabTopic: projector.hwlabTopic + }; + if (liveProbe) return { started: true, ...identity, ...(projector.startupStatus?.() ?? { status: "initializing" }), statusSource: "startup-state", valuesRedacted: true }; + try { + return { started: true, ...identity, ...(await projector.status()), valuesRedacted: true }; + } catch (error) { + return { started: true, status: "blocked", errorCode: error?.code ?? "hwlab_kafka_projector_status_failed", message: error?.message ?? "Kafka projector status query failed.", valuesRedacted: true }; + } +} + function runtimeReadinessSnapshot(runtimeStore) { if (runtimeStore && typeof runtimeStore.summary === "function") { try { @@ -610,7 +624,8 @@ async function handleRestAdapter(request, response, url, options) { observeHttpRoutePhase(request, options, "v1.runtime_readiness", () => runtimeReadinessSnapshot(options.runtimeStore)) ]); const db = applyRuntimeDbReadinessLayers(dbProbe, runtime); - const readiness = buildCloudApiReadiness({ db, codeAgent, runtime }); + const kafkaProjector = await kafkaProjectorHealth(options.kafkaEventBridge, { liveProbe: false }); + const readiness = buildCloudApiReadiness({ db, codeAgent, runtime, kafkaProjector }); sendJson(response, 200, { serviceId: CLOUD_API_SERVICE_ID, adapter: "rest", @@ -728,7 +743,7 @@ async function handleRestAdapter(request, response, url, options) { return; } - if (url.pathname === "/v1/workbench/events") { + if (url.pathname === "/v1/workbench/events" || url.pathname === "/v1/workbench/projection-events") { await handleWorkbenchRealtimeHttp(request, response, url, options); return; } @@ -980,7 +995,7 @@ function navIdForRestPath(pathname, method = "GET") { if (pathname === "/v1/api-keys" || pathname === "/v1/api-keys/default" || pathname.startsWith("/v1/api-keys/")) return "user.apiKeys"; if (pathname === "/v1/users/me/profile" || pathname === "/v1/users/me/password") return "system.settings"; if (pathname === "/v1/workbench/debug/fake-sse" || pathname.startsWith("/v1/workbench/debug/fake-sse/") || pathname === "/v1/workbench/debug/kafka-sse" || pathname.startsWith("/v1/workbench/debug/kafka-sse/")) return "workbench.debug"; - if (pathname === "/v1/workbench/events" || pathname === "/v1/workbench/sync" || pathname === "/v1/workbench/launches" || pathname === "/v1/workbench/sessions" || pathname.startsWith("/v1/workbench/sessions/") || pathname.startsWith("/v1/workbench/turns/") || pathname.startsWith("/v1/workbench/traces/")) return "workbench.code"; + if (pathname === "/v1/workbench/events" || pathname === "/v1/workbench/projection-events" || pathname === "/v1/workbench/sync" || pathname === "/v1/workbench/launches" || pathname === "/v1/workbench/sessions" || pathname.startsWith("/v1/workbench/sessions/") || pathname.startsWith("/v1/workbench/turns/") || pathname.startsWith("/v1/workbench/traces/")) return "workbench.code"; if (pathname === "/v1/agent/chat" || pathname === "/v1/agent/sessions" || pathname.startsWith("/v1/agent/sessions/") || pathname === "/v1/agent/chat/inspect" || pathname.startsWith("/v1/agent/chat/result/") || pathname.startsWith("/v1/agent/turns/") || pathname.startsWith("/v1/agent/traces/") || pathname === "/v1/agent/chat/cancel" || pathname === "/v1/agent/chat/steer") return "workbench.code"; if (pathname === "/v1/admin/provider-profiles" || pathname.startsWith("/v1/admin/provider-profiles/")) return "admin.providerProfiles"; if (pathname === "/v1/admin/secrets" || pathname.startsWith("/v1/admin/secrets/")) return "admin.secrets"; @@ -994,492 +1009,6 @@ function navIdForRestPath(pathname, method = "GET") { return ""; } -async function handleHwpodNodeOpsHttp(request, response, options) { - if (request.method === "GET") { - sendJson(response, 200, { - ok: true, - status: "ready", - contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, - route: "/v1/hwpod-node-ops", - specAuthority: "code-agent-workspace", - compiler: "hwpod-compiler-cli", - apiRole: "node-ops-forwarder", - nodeRole: "thin-hwpod-node-executor", - supportedOps: Array.from(HWPOD_NODE_OPS), - websocket: options.hwpodNodeWsRegistry.describe() - }); - return; - } - if (request.method !== "POST") { - sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "hwpod-node-ops only supports GET and POST" } }); - return; - } - - const body = await readJsonObject(request, options.bodyLimitBytes); - if (!body.ok) { - sendJson(response, 400, body.error); - return; - } - const validation = validateHwpodNodeOpsPlan(body.value); - if (!validation.ok) { - sendJson(response, 400, { - ok: false, - status: "rejected", - contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, - error: validation.error - }); - return; - } - - const plan = validation.plan; - const requestMeta = hwpodNodeOpsRequestMeta(request, options, "hwpod"); - try { - const handled = await dispatchHwpodNodeOpsPlan(plan, requestMeta, options, { request }); - sendJson(response, handled.httpStatus, handled.payload); - } catch (error) { - const summary = error?.message ?? "hwpod-node-ops handler failed"; - recordHwpodNodeOpsBlocked(request, options, plan, requestMeta, "hwpod_node_handler_failed", summary, { dispatchMode: "handler-exception" }); - sendJson(response, 200, hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, { dispatchMode: "handler-exception" })); - } -} - -function hwpodNodeOpsRequestMeta(request, options, label) { - const httpContext = request?.hwlabHttpRequestContext ?? {}; - return { - requestId: getHeader(request, "x-request-id") || httpContext.requestId || `req_${label}_${randomUUID()}`, - traceId: getHeader(request, "x-trace-id") || `trc_${label}_${randomUUID()}`, - serviceId: getHeader(request, "x-source-service-id") || CLOUD_API_SERVICE_ID, - environment: runtimeEnvironment(options.env ?? process.env), - otelTraceId: httpContext.traceId ?? null, - traceparent: httpContext.traceparent ?? null, - valuesPrinted: false - }; -} - -function handleHwlabNodeUpdateHttp(request, response, url, options) { - if (request.method !== "GET") { - sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "GET required." } }); - return; - } - const env = options.env ?? process.env; - const currentVersion = cleanText(url.searchParams.get("current") || url.searchParams.get("version")); - const channel = cleanText(url.searchParams.get("channel")) || "stable"; - const platform = cleanText(url.searchParams.get("platform")) || "unknown"; - const bundled = hwlabNodeBundledMetadata(env); - const latestVersion = cleanText(env.HWLAB_NODE_PY_LATEST_VERSION) || bundled.version || "0.1.0"; - const releaseNotesUrl = cleanText(env.HWLAB_NODE_PY_RELEASE_NOTES_URL) || null; - const downloadUrl = hwlabNodeDownloadUrl(env); - const sha256 = cleanText(env.HWLAB_NODE_PY_SHA256) || bundled.sha256 || null; - const newer = currentVersion ? compareVersionStrings(latestVersion, currentVersion) > 0 : Boolean(latestVersion); - const updateAvailable = Boolean(downloadUrl && newer); - sendJson(response, 200, { - ok: true, - contractVersion: "hwlab-node-update-v1", - serviceId: "hwlab-node", - route: "/v1/hwlab-node/update", - channel, - platform, - currentVersion: currentVersion || null, - latestVersion, - updateAvailable, - downloadUrl: updateAvailable ? downloadUrl : null, - sha256: updateAvailable ? sha256 : null, - releaseNotesUrl, - manualDefault: true, - autoApplyDefault: false, - checkIntervalSeconds: parsePositiveInteger(env.HWLAB_NODE_PY_UPDATE_INTERVAL_SECONDS, 300), - observedAt: new Date().toISOString() - }); -} - -function handleHwlabNodeDownloadHttp(request, response, options) { - if (request.method !== "GET") { - sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "GET required." } }); - return; - } - const bundled = hwlabNodeBundledMetadata(options.env ?? process.env); - if (!bundled.content) { - sendJson(response, 404, { ok: false, error: { code: "hwlab_node_bundle_missing", message: "bundled hwlab-node.py is not available" } }); - return; - } - const bytes = Buffer.from(bundled.content, "utf8"); - response.writeHead(200, { - "content-type": "text/x-python; charset=utf-8", - "cache-control": "no-store", - "x-hwlab-node-version": bundled.version || "unknown", - "x-hwlab-node-sha256": bundled.sha256 || "", - "content-length": String(bytes.length) - }); - response.end(bytes); -} - -function hwlabNodeBundledMetadata(env) { - const bundlePath = cleanText(env.HWLAB_NODE_PY_BUNDLE_PATH) || path.resolve(process.cwd(), "tools/hwlab-node.py"); - try { - const content = readFileSync(bundlePath, "utf8"); - const version = cleanText(content.match(/APP_VERSION\s*=\s*["']([^"']+)["']/u)?.[1]); - const sha256 = createHash("sha256").update(content, "utf8").digest("hex"); - return { path: bundlePath, content, version, sha256 }; - } catch { - return { path: bundlePath, content: "", version: "", sha256: "" }; - } -} - -function hwlabNodeDownloadUrl(env) { - const explicit = cleanText(env.HWLAB_NODE_PY_DOWNLOAD_URL); - if (explicit) return explicit; - const downloadPath = cleanText(env.HWLAB_NODE_PY_DOWNLOAD_PATH) || "/v1/hwlab-node/download/hwlab-node.py"; - if (/^https?:\/\//iu.test(downloadPath)) return downloadPath; - const publicEndpoint = cleanText(env.HWLAB_PUBLIC_ENDPOINT) || "https://hwlab.pikapython.com"; - try { - return new URL(downloadPath, publicEndpoint.endsWith("/") ? publicEndpoint : `${publicEndpoint}/`).toString(); - } catch { - return null; - } -} - -function compareVersionStrings(left, right) { - const a = versionParts(left); - const b = versionParts(right); - for (let index = 0; index < Math.max(a.length, b.length, 3); index += 1) { - const delta = (a[index] ?? 0) - (b[index] ?? 0); - if (delta !== 0) return delta > 0 ? 1 : -1; - } - return 0; -} - -function versionParts(value) { - return String(value ?? "") - .trim() - .replace(/^v/iu, "") - .split(/[.+-]/u) - .slice(0, 4) - .map((part) => parseInt(part, 10)) - .map((part) => (Number.isFinite(part) && part >= 0 ? part : 0)); -} - -function cleanText(value) { - const text = String(value ?? "").trim(); - return text || ""; -} - -async function handleHwpodSpecDiscoveryHttp(request, response, url, options) { - const probe = truthyFlag(url.searchParams.get("probe")); - const observedAt = new Date().toISOString(); - let specs = await discoverHwpodSpecs(options); - if (probe) { - specs = await Promise.all(specs.map((spec) => probeDiscoveredHwpodSpec(spec, { request, options, observedAt }))); - } - sendJson(response, 200, hwpodSpecDiscoveryPayload(specs, { observedAt })); -} - -async function probeDiscoveredHwpodSpec(spec, { request, options, observedAt }) { - if (spec.ok === false) return spec; - const plan = hwpodSpecWorkspaceProbePlan(spec); - const validation = validateHwpodNodeOpsPlan(plan); - if (!validation.ok) { - return { - ...spec, - availability: { - ok: false, - status: "invalid_probe_plan", - checkedAt: observedAt, - blocker: validation.error - } - }; - } - const requestMeta = hwpodNodeOpsRequestMeta(request, options, "hwpod_spec"); - const handled = await dispatchHwpodNodeOpsPlan(validation.plan, requestMeta, options, { request }); - const payload = handled.payload; - return { - ...spec, - availability: { - ok: payload.ok === true, - status: payload.ok === true ? "available" : "blocked", - checkedAt: observedAt, - probePlanId: payload.planId, - nodeOpsRoute: "/v1/hwpod-node-ops", - results: payload.results ?? [], - blocker: payload.blocker ?? null, - httpStatus: handled.httpStatus - } - }; -} - -async function dispatchHwpodNodeOpsPlan(plan, requestMeta, options, context = {}) { - const hwpodNodeOpsUrl = normalizedHwpodNodeOpsUrl(options.env ?? process.env); - const hwpodNodeWsRegistry = options.hwpodNodeWsRegistry; - const hasWsNode = hwpodNodeWsRegistry?.hasNode?.(plan.nodeId); - if (typeof options.hwpodNodeOpsHandler !== "function" && !hasWsNode && !hwpodNodeOpsUrl) { - const summary = "no outbound WebSocket hwpod-node is connected and HWLAB_HWPOD_NODE_OPS_URL is not configured; cloud-api is only validating the hwpod-node-ops contract"; - const details = { - dispatchMode: "none", - websocketRegistryConfigured: Boolean(hwpodNodeWsRegistry), - websocketConnected: false, - directUrlConfigured: false - }; - recordHwpodNodeOpsBlocked(context.request, options, plan, requestMeta, "hwpod_node_unavailable", summary, details); - return { - httpStatus: 200, - payload: hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, details) - }; - } - const handled = typeof options.hwpodNodeOpsHandler === "function" - ? await options.hwpodNodeOpsHandler(plan, { request: context.request, requestMeta, env: options.env ?? process.env }) - : hasWsNode - ? await hwpodNodeWsRegistry.dispatch(plan, requestMeta, { timeoutMs: parsePositiveInteger(options.env?.HWLAB_HWPOD_NODE_OPS_TIMEOUT_MS, 30000) }) - : await forwardHwpodNodeOpsPlan(hwpodNodeOpsUrl, plan, requestMeta, options); - const results = Array.isArray(handled?.results) ? handled.results : []; - const failed = results.some((item) => item?.ok === false); - const payload = attachHwpodNodeOpsDiagnostics({ - ok: handled?.ok ?? !failed, - status: handled?.status ?? (failed ? "failed" : "completed"), - contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, - planId: plan.planId, - hwpodId: plan.hwpodId, - nodeId: plan.nodeId, - acceptedOps: plan.ops.length, - results, - blocker: handled?.blocker ?? null, - requestMeta - }, requestMeta, { dispatchMode: hasWsNode ? "websocket" : hwpodNodeOpsUrl ? "direct-url" : "handler" }); - if (payload.ok === false && payload.status === "blocked" && payload.blocker?.code) { - recordHwpodNodeOpsBlocked(context.request, options, plan, requestMeta, payload.blocker.code, payload.blocker.summary ?? "hwpod-node-ops blocked", payload.blocker.details ?? {}); - } - return { httpStatus: handled?.httpStatus ?? (failed ? 409 : 200), payload }; -} - -async function forwardHwpodNodeOpsPlan(targetUrl, plan, requestMeta, options) { - const target = await describeDirectHwpodNode(targetUrl, options); - if (!target.ok || !target.nodeId) { - return directHwpodNodeBlocked(plan, "hwpod_node_identity_unverified", "configured hwpod-node URL did not expose a verifiable nodeId", { - requestedNodeId: plan.nodeId, - targetUrl: redactNodeOpsUrl(targetUrl), - dispatchMode: "direct-url", - targetStatus: target.status ?? null, - error: target.error ?? null - }, requestMeta); - } - if (target.ok && target.nodeId && target.nodeId !== plan.nodeId) { - return directHwpodNodeBlocked(plan, "hwpod_node_id_mismatch", `HWLAB_HWPOD_NODE_OPS_URL points to ${target.nodeId}, not requested node ${plan.nodeId}`, { - requestedNodeId: plan.nodeId, - targetNodeId: target.nodeId, - targetUrl: redactNodeOpsUrl(targetUrl), - dispatchMode: "direct-url" - }, requestMeta); - } - const response = await fetch(targetUrl, { - method: "POST", - headers: { - "content-type": "application/json", - "x-request-id": requestMeta.requestId, - "x-trace-id": requestMeta.traceId, - "x-source-service-id": CLOUD_API_SERVICE_ID, - ...(requestMeta.otelTraceId ? { "x-hwlab-otel-trace-id": requestMeta.otelTraceId } : {}), - ...(requestMeta.traceparent ? { traceparent: requestMeta.traceparent } : {}) - }, - body: JSON.stringify(plan), - signal: AbortSignal.timeout(parsePositiveInteger(options.env?.HWLAB_HWPOD_NODE_OPS_TIMEOUT_MS, 30000)) - }); - const body = await response.json().catch(() => null); - if (!body || typeof body !== "object") { - return directHwpodNodeBlocked(plan, "hwpod_node_response_invalid", `hwpod-node returned HTTP ${response.status} without a JSON object payload`, { - targetUrl: redactNodeOpsUrl(targetUrl), - dispatchMode: "direct-url", - targetStatus: response.status - }, requestMeta); - } - return { - ok: body.ok, - status: body.status, - httpStatus: response.status, - results: body.results, - blocker: body.blocker ?? null - }; -} - -function directHwpodNodeBlocked(plan, code, summary, details, requestMeta = {}) { - const blocker = hwpodNodeOpsBlocker({ code, layer: "hwpod-node", retryable: true, summary, details }, requestMeta, details); - return { - ok: false, - status: "blocked", - httpStatus: 200, - results: plan.ops.map((op) => ({ opId: op.opId, op: op.op, ok: false, status: "blocked", blocker })), - blocker, - otelTraceId: blocker.otelTraceId ?? null, - diagnostic: blocker.diagnostic ?? null - }; -} - -async function describeDirectHwpodNode(targetUrl, options) { - try { - const response = await fetch(targetUrl, { - method: "GET", - signal: AbortSignal.timeout(parsePositiveInteger(options.env?.HWLAB_HWPOD_NODE_OPS_TIMEOUT_MS, 30000)) - }); - const body = await response.json().catch(() => null); - return { ok: response.ok && body && typeof body === "object", status: response.status, nodeId: safeOpaqueId(body?.nodeId), body }; - } catch (error) { - return { ok: false, error: error instanceof Error ? error.message : String(error), nodeId: "" }; - } -} - -function redactNodeOpsUrl(value) { - try { - const parsed = new URL(value); - parsed.username = ""; - parsed.password = ""; - parsed.search = parsed.search ? "?redacted=1" : ""; - return parsed.toString(); - } catch { - return ""; - } -} - -function normalizedHwpodNodeOpsUrl(env = process.env) { - const direct = String(env.HWLAB_HWPOD_NODE_OPS_URL ?? "").trim(); - if (direct) return direct; - const base = String(env.HWLAB_HWPOD_NODE_URL ?? "").trim().replace(/\/+$/u, ""); - return base ? `${base}/v1/hwpod-node-ops` : ""; -} - - -function validateHwpodNodeOpsPlan(value) { - if (!value || typeof value !== "object" || Array.isArray(value)) { - return { ok: false, error: { code: "invalid_hwpod_node_ops_plan", message: "hwpod-node-ops body must be a JSON object" } }; - } - if (value.contractVersion !== HWPOD_NODE_OPS_CONTRACT_VERSION) { - return { ok: false, error: { code: "invalid_hwpod_node_ops_contract", message: `contractVersion must be ${HWPOD_NODE_OPS_CONTRACT_VERSION}`, actual: value.contractVersion ?? null } }; - } - const planId = safeOpaqueId(value.planId) || `hwpod_plan_${randomUUID()}`; - const nodeId = safeOpaqueId(value.nodeId); - const hwpodId = safeOpaqueId(value.hwpodId); - if (!nodeId) return { ok: false, error: { code: "invalid_hwpod_node_id", message: "nodeId is required" } }; - if (!hwpodId) return { ok: false, error: { code: "invalid_hwpod_id", message: "hwpodId is required" } }; - if (!Array.isArray(value.ops) || value.ops.length === 0) { - return { ok: false, error: { code: "invalid_hwpod_node_ops", message: "ops must be a non-empty array" } }; - } - const ops = []; - for (const [index, item] of value.ops.entries()) { - if (!item || typeof item !== "object" || Array.isArray(item)) { - return { ok: false, error: { code: "invalid_hwpod_node_op", message: `ops[${index}] must be a JSON object` } }; - } - const op = typeof item.op === "string" ? item.op.trim() : ""; - if (!HWPOD_NODE_OPS.has(op)) { - return { ok: false, error: { code: "unsupported_hwpod_node_op", message: `unsupported hwpod-node op: ${op || ""}`, supportedOps: Array.from(HWPOD_NODE_OPS) } }; - } - ops.push({ - opId: safeOpaqueId(item.opId) || `op_${index + 1}`, - op, - args: item.args && typeof item.args === "object" && !Array.isArray(item.args) ? item.args : {} - }); - } - return { - ok: true, - plan: { - ...value, - planId, - hwpodId, - nodeId, - ops - } - }; -} - -function hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, details = {}) { - const blocker = hwpodNodeOpsBlocker({ - code: "hwpod_node_unavailable", - layer: "hwpod-node", - retryable: true, - summary, - details, - userMessage: "hwpod-node 尚未接入执行面；cloud-api 已完成 hwpod-node-ops 合同校验。" - }, requestMeta, details); - return { - ok: false, - status: "blocked", - contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, - planId: plan.planId, - hwpodId: plan.hwpodId, - nodeId: plan.nodeId, - acceptedOps: plan.ops.length, - results: plan.ops.map((op) => ({ - opId: op.opId, - op: op.op, - ok: false, - status: "blocked", - blocker - })), - blocker, - otelTraceId: blocker.otelTraceId ?? null, - diagnostic: blocker.diagnostic ?? null, - requestMeta - }; -} - -function hwpodNodeOpsBlocker(blocker, requestMeta = {}, details = {}) { - const code = cleanText(blocker?.code) || "hwpod_node_ops_blocked"; - const summary = cleanText(blocker?.summary ?? blocker?.message) || "hwpod-node-ops blocked"; - const otelTraceId = cleanText(blocker?.otelTraceId ?? requestMeta?.otelTraceId) || null; - const diagnostic = { - code, - rootCauseCode: code, - summary, - otelTraceId, - traceLine: otelTraceId ? `OTel traceId: ${otelTraceId}` : null, - details: details && typeof details === "object" ? details : {}, - valuesPrinted: false - }; - const userMessage = cleanText(blocker?.userMessage); - return { - ...blocker, - code, - layer: blocker?.layer ?? "hwpod-node", - retryable: blocker?.retryable ?? true, - summary, - ...(otelTraceId ? { otelTraceId } : {}), - diagnostic, - ...(userMessage ? { userMessage: `${userMessage}\n${diagnostic.traceLine ?? "OTel traceId: unavailable"}` } : {}) - }; -} - -function attachHwpodNodeOpsDiagnostics(payload, requestMeta, details = {}) { - if (!payload || typeof payload !== "object" || payload.ok !== false) return payload; - const topBlocker = payload.blocker ? hwpodNodeOpsBlocker(payload.blocker, requestMeta, payload.blocker.details ?? details) : null; - const results = Array.isArray(payload.results) - ? payload.results.map((result) => result?.blocker ? { ...result, blocker: hwpodNodeOpsBlocker(result.blocker, requestMeta, result.blocker.details ?? details) } : result) - : payload.results; - const diagnostic = topBlocker?.diagnostic ?? results?.find?.((result) => result?.blocker?.diagnostic)?.blocker?.diagnostic ?? null; - return { - ...payload, - results, - blocker: topBlocker, - ...(diagnostic?.otelTraceId ? { otelTraceId: diagnostic.otelTraceId } : {}), - ...(diagnostic ? { diagnostic } : {}) - }; -} - -function recordHwpodNodeOpsBlocked(request, options, plan, requestMeta, code, summary, details = {}) { - const httpContext = request?.hwlabHttpRequestContext; - const attributes = { - "hwlab.hwpod.plan_id": cleanText(plan?.planId) || null, - "hwlab.hwpod.hwpod_id": cleanText(plan?.hwpodId) || null, - "hwlab.hwpod.node_id": cleanText(plan?.nodeId) || null, - "hwlab.hwpod.accepted_ops": Array.isArray(plan?.ops) ? plan.ops.length : 0, - "hwlab.hwpod.ops": Array.isArray(plan?.ops) ? plan.ops.map((op) => cleanText(op?.op)).filter(Boolean).join(",").slice(0, 240) : null, - "hwlab.hwpod.blocker.code": code, - "hwlab.hwpod.blocker.summary": String(summary ?? "").slice(0, 500), - "hwlab.hwpod.dispatch_mode": cleanText(details?.dispatchMode) || "unknown", - "hwlab.hwpod.websocket_connected": Boolean(details?.websocketConnected), - "hwlab.hwpod.direct_url_configured": Boolean(details?.directUrlConfigured), - "hwlab.hwpod.otel_trace_id": cleanText(requestMeta?.otelTraceId) || null, - traceId: cleanText(requestMeta?.traceId) || null - }; - if (httpContext) httpContext.otelAttributes = { ...(httpContext.otelAttributes ?? {}), ...attributes }; - const error = Object.assign(new Error(summary), { code }); - emitHttpRoutePhaseSpan(request, options, "hwpod-node-ops.blocked", Date.now(), Date.now(), "error", error, attributes); -} - async function codeAgentOptions(request, response, options, authOptions = {}) { const auth = await options.accessController.authenticate(request, { required: authOptions.required ?? options.accessController.required }); if (!auth.ok) { diff --git a/internal/cloud/user-billing-integration.test.ts b/internal/cloud/user-billing-integration.test.ts index 41dbc4c1..bf2124d1 100644 --- a/internal/cloud/user-billing-integration.test.ts +++ b/internal/cloud/user-billing-integration.test.ts @@ -204,158 +204,6 @@ test("cloud api accepts user-billing API keys and records Code Agent billing usa } }); -test("cloud api defers AgentRun Code Agent billing record until terminal result", async () => { - const calls = []; - const agentRunCalls = []; - let eventPolls = 0; - let resultPolls = 0; - const agentRunServer = createHttpServer(async (request, response) => { - const url = new URL(request.url || "/", "http://127.0.0.1"); - const chunks = []; - for await (const chunk of request) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); - const body = chunks.length ? JSON.parse(Buffer.concat(chunks).toString("utf8")) : null; - agentRunCalls.push({ method: request.method, path: url.pathname, search: url.search, body }); - const send = (data) => { - response.writeHead(200, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: true, data, traceId: "trc_fake_agentrun_billing" })}\n`); - }; - if (request.method === "POST" && url.pathname === "/api/v1/runs") { - assert.equal(body.backendProfile, "deepseek"); - return send({ id: "run_billing_deferred", status: "pending", backendProfile: "deepseek", sessionRef: body.sessionRef, resourceBundleRef: body.resourceBundleRef }); - } - if (request.method === "POST" && url.pathname === "/api/v1/runs/run_billing_deferred/commands") { - assert.equal(body.type, "turn"); - assert.equal(body.idempotencyKey, "trc_user_billing_agentrun_terminal"); - return send({ id: "cmd_billing_deferred", runId: "run_billing_deferred", state: "pending", type: "turn", seq: 1 }); - } - if (request.method === "GET" && url.pathname === "/api/v1/runs/run_billing_deferred/commands") { - return send({ items: [{ id: "cmd_billing_deferred", runId: "run_billing_deferred", state: "running", type: "turn", seq: 1, idempotencyKey: "trc_user_billing_agentrun_terminal", payload: { traceId: "trc_user_billing_agentrun_terminal", conversationId: "cnv_user_billing_agentrun", hwlabSessionId: "ses_user_billing_agentrun", providerProfile: "deepseek" } }] }); - } - if (request.method === "POST" && url.pathname === "/api/v1/runs/run_billing_deferred/runner-jobs") { - assert.equal(body.commandId, "cmd_billing_deferred"); - return send({ - action: "create-kubernetes-job", - runId: "run_billing_deferred", - commandId: "cmd_billing_deferred", - attemptId: "attempt_billing_deferred", - runnerId: "runner_billing_deferred", - namespace: "agentrun-v02", - jobName: "agentrun-v01-runner-billing-deferred" - }); - } - if (request.method === "GET" && url.pathname === "/api/v1/runs/run_billing_deferred/events") { - eventPolls += 1; - return send({ items: eventPolls > 1 ? [{ id: "evt_done", runId: "run_billing_deferred", seq: 1, type: "terminal_status", payload: { commandId: "cmd_billing_deferred", terminalStatus: "completed" }, createdAt: "2026-06-14T04:20:00.000Z" }] : [] }); - } - if (request.method === "GET" && url.pathname === "/api/v1/runs/run_billing_deferred/commands/cmd_billing_deferred/result") { - resultPolls += 1; - if (resultPolls === 1) { - return send({ runId: "run_billing_deferred", commandId: "cmd_billing_deferred", status: "running", runStatus: "claimed", commandState: "running", terminalStatus: null }); - } - return send({ - runId: "run_billing_deferred", - commandId: "cmd_billing_deferred", - attemptId: "attempt_billing_deferred", - runnerId: "runner_billing_deferred", - jobName: "agentrun-v01-runner-billing-deferred", - namespace: "agentrun-v02", - status: "completed", - runStatus: "completed", - commandState: "completed", - terminalStatus: "completed", - completed: true, - reply: "AgentRun billing completed.", - lastSeq: 1, - eventCount: 1, - sessionRef: { sessionId: "ses_agentrun_deepseek_billing", conversationId: "cnv_user_billing_agentrun", threadId: "thr_billing" } - }); - } - response.writeHead(404, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: false, failureKind: "unexpected", message: `${request.method} ${url.pathname}` })}\n`); - }); - await new Promise((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve)); - - const userBillingClient = { - configured: true, - async introspect(token) { - calls.push({ op: "introspect", tokenPrefix: token.slice(0, 8) }); - assert.equal(token, "hwl_user_billing_agentrun_secret"); - return { ok: true, status: 200, body: { active: true, principal: { userId: "usr_user_billing_agentrun", email: "agentrun@hwlab.local", username: "billing-agentrun", role: "user", scopes: ["api"], authType: "api-key", keyId: "key_user_billing_agentrun" } } }; - }, - async billingPreflight(body) { - calls.push({ op: "preflight", body }); - assert.equal(body.apiKey, "hwl_user_billing_agentrun_secret"); - assert.equal(body.idempotencyKey, "code-agent:trc_user_billing_agentrun_terminal:preflight"); - return { ok: true, status: 200, body: { allowed: true, reservationId: "res_user_billing_agentrun", estimatedCredits: 1, expiresAt: "2026-06-14T05:00:00.000Z" } }; - }, - async billingRecord(body) { - calls.push({ op: "record", body }); - assert.equal(body.reservationId, "res_user_billing_agentrun"); - assert.equal(body.idempotencyKey, "code-agent:trc_user_billing_agentrun_terminal:record"); - assert.equal(body.metadata.status, "completed"); - return { ok: true, status: 200, body: { recordId: "use_user_billing_agentrun", credits: 1, balance: 9 } }; - }, - async billingRelease() { - throw new Error("completed AgentRun billing should record usage instead of releasing reservation"); - } - }; - const { port: agentRunPort } = agentRunServer.address(); - const server = createCloudApiServer({ - traceStore: createCodeAgentTraceStore(), - env: { - HWLAB_ACCESS_CONTROL_REQUIRED: "1", - HWLAB_USER_BILLING_CODE_AGENT_ENABLED: "1", - HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", - AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, - AGENTRUN_API_KEY: "test-agentrun-key", - HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1", - HWLAB_CODE_AGENT_AGENTRUN_SOURCE_COMMIT: "0123456789abcdef0123456789abcdef01234567", - HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_POLL_INTERVAL_MS: "10", - HWLAB_CODE_AGENT_AGENTRUN_SESSION_STORAGE: "metadata-only", - HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE: "deepseek" - }, - userBillingClient - }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = server.address(); - const authHeader = { authorization: "Bearer hwl_user_billing_agentrun_secret" }; - const sessionResponse = await fetch(`http://127.0.0.1:${port}/v1/agent/sessions`, { - method: "POST", - headers: { "content-type": "application/json", ...authHeader }, - body: JSON.stringify({ conversationId: "cnv_user_billing_agentrun", sessionId: "ses_user_billing_agentrun", providerProfile: "deepseek" }) - }); - assert.equal(sessionResponse.status, 201); - - const submit = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, { - method: "POST", - headers: { "content-type": "application/json", authorization: "Bearer hwl_user_billing_agentrun_secret", "x-trace-id": "trc_user_billing_agentrun_terminal", prefer: "respond-async" }, - body: JSON.stringify({ conversationId: "cnv_user_billing_agentrun", sessionId: "ses_user_billing_agentrun", shortConnection: true, message: "billing AgentRun terminal smoke" }) - }); - assert.equal(submit.status, 202); - await waitForUserBillingCondition(() => agentRunCalls.some((call) => call.path === "/api/v1/runs/run_billing_deferred/runner-jobs")); - assert.deepEqual(calls.map((call) => call.op), ["introspect", "introspect", "preflight"]); - - const running = await fetch(`http://127.0.0.1:${port}/v1/agent/chat/result/trc_user_billing_agentrun_terminal`, { headers: authHeader }); - assert.equal(running.status === 202 || running.status === 200, true); - assert.equal(calls.some((call) => call.op === "record"), false); - - const payload = running.status === 200 ? await running.json() : await pollUserBillingAgentResult(port, "trc_user_billing_agentrun_terminal", authHeader); - assert.equal(payload.status, "completed"); - assert.equal(payload.billing.recorded, true); - assert.equal(payload.billing.reservationId, "res_user_billing_agentrun"); - assert.equal(JSON.stringify(payload).includes("userBillingReservation"), false); - assert.equal(calls.filter((call) => call.op === "preflight").length, 1); - assert.equal(calls.filter((call) => call.op === "record").length, 1); - assert.equal(calls.some((call) => call.op === "release"), false); - } finally { - await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); - await new Promise((resolve, reject) => agentRunServer.close((error) => (error ? reject(error) : resolve()))); - } -}); - test("AgentRun persisted trace evidence restores billing reservation without exposing it on agentRun", async () => { const restored = await loadPersistedAgentRunResult("trc_user_billing_agentrun_restored", { env: { HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01" }, diff --git a/internal/cloud/web-performance.test.ts b/internal/cloud/web-performance.test.ts index 809980d1..227ece53 100644 --- a/internal/cloud/web-performance.test.ts +++ b/internal/cloud/web-performance.test.ts @@ -491,7 +491,13 @@ test("web performance route templates keep metrics low-cardinality", () => { }); test("cloud api ingests WebUI performance samples and exposes metrics only on loopback host", async () => { - const server = createCloudApiServer({ env: { HWLAB_METRICS_NAMESPACE: "hwlab-v02", HWLAB_GITOPS_TARGET: "v02" } }); + const server = createCloudApiServer({ + accessController: { + required: false, + async authenticate() { return { ok: true, actor: { id: "usr_web_performance", role: "admin" }, session: { id: "uss_web_performance" } }; } + }, + env: { HWLAB_METRICS_NAMESPACE: "hwlab-v02", HWLAB_GITOPS_TARGET: "v02" } + }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); try { const address = server.address(); diff --git a/internal/cloud/workbench-facts-store.ts b/internal/cloud/workbench-facts-store.ts index e3cd815b..f636d149 100644 --- a/internal/cloud/workbench-facts-store.ts +++ b/internal/cloud/workbench-facts-store.ts @@ -2,11 +2,10 @@ * SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-18-p0-unique-projection; draft-2026-06-20-p1-zero-split-durable-realtime; PJ2026-010403 API契约 draft-2026-06-18-r1. * 职责: Workbench durable facts store adapter。统一读取 session/message/turn/trace projection facts，不在 GET 中推进 AgentRun facts。 */ -import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; import { emitCodeAgentOtelSpan } from "./otel-trace.ts"; import { safeConversationId, safeSessionId, safeTraceId } from "./server-http-utils.ts"; import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts"; -import { durableTraceStatus, normalizeWorkbenchStatus, TERMINAL_STATUSES, traceTerminalEvidence } from "./workbench-turn-projection.ts"; +import { durableTraceStatus, TERMINAL_STATUSES, traceTerminalEvidence } from "./workbench-turn-projection.ts"; const TERMINAL_DURABLE_TRACE_CACHE_MAX = 256; const DEFAULT_WORKBENCH_FACTS_QUERY_MAX_ATTEMPTS = 5; @@ -17,9 +16,7 @@ const terminalDurableTraceCache = new Map(); export function createWorkbenchFactsStore(options = {}, actor = null) { const accessStore = options.accessController?.store ?? options.accessController ?? null; const traceSessionCache = options.codeAgentTraceSessionCache instanceof Map ? options.codeAgentTraceSessionCache : null; - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - const runtimeStore = options.runtimeStore ?? null; - const runtimeReader = options.workbenchRuntime ?? createWorkbenchRuntimeClient({ env: options.env ?? process.env, fetch: options.fetch, logger: options.logger }) ?? runtimeStore; + const runtimeReader = options.workbenchRuntime ?? createWorkbenchRuntimeClient({ env: options.env ?? process.env, fetch: options.fetch, logger: options.logger }); const results = options.codeAgentChatResults ?? null; async function queryFacts(params = {}) { @@ -135,18 +132,8 @@ export function createWorkbenchFactsStore(options = {}, actor = null) { return safeId ? results?.get?.(safeId) ?? null : null; } - function traceSnapshotSync(traceId) { - return traceStore.snapshot(traceId); - } - async function traceSnapshot(traceId) { - const memory = traceSnapshotSync(traceId); - if (hasTraceProjection(memory) && TERMINAL_STATUSES.has(normalizeStatus(memory.status))) return memory; - const durable = await durableTraceSnapshot(runtimeReader, traceId); - if (isProjectionDiagnosticTrace(durable) && hasTraceProjection(memory)) return mergeTraceProjectionDiagnostic(memory, durable); - if (durable && shouldPreferDurableTrace(memory, durable)) return durable; - if (hasTraceProjection(memory)) return memory; - return durable ?? memory; + return durableTraceSnapshot(runtimeReader, traceId); } return { @@ -157,7 +144,6 @@ export function createWorkbenchFactsStore(options = {}, actor = null) { listSessions, resultForTrace, traceSnapshot, - traceSnapshotSync, canReadOwner: (ownerUserId) => canActorReadOwner(ownerUserId, actor) }; } @@ -220,7 +206,7 @@ function objectValue(value) { } async function durableTraceSnapshot(runtimeStore, traceId) { - if (!runtimeStore || typeof runtimeStore.queryAgentTraceEvents !== "function") return null; + if (!runtimeStore || typeof runtimeStore.queryWorkbenchFacts !== "function") return null; const safeId = safeTraceId(traceId); if (!safeId) return null; const cached = terminalDurableTraceCache.get(safeId); @@ -231,11 +217,11 @@ async function durableTraceSnapshot(runtimeStore, traceId) { } let result; try { - result = await runtimeStore.queryAgentTraceEvents({ traceId: safeId }); + result = await runtimeStore.queryWorkbenchFacts({ traceId: safeId, families: ["traceEvents"] }); } catch (error) { return projectionStoreUnavailableTrace(safeId, error); } - const events = Array.isArray(result?.events) ? result.events : []; + const events = Array.isArray(result?.facts?.traceEvents) ? result.facts.traceEvents : []; if (events.length === 0) return null; const normalizedEvents = events.map((event, index) => ({ ...event, traceId: safeId, seq: eventSeq(event, index) })); const firstEvent = normalizedEvents[0] ?? null; @@ -391,41 +377,6 @@ function finiteNonNegativeNumber(value) { return Number.isFinite(parsed) && parsed >= 0 ? Math.trunc(parsed) : null; } -function isProjectionDiagnosticTrace(snapshot) { - return Boolean(snapshot?.projection?.blocker || snapshot?.projectionHealth || snapshot?.blocker); -} - -function mergeTraceProjectionDiagnostic(memory, diagnostic) { - return { - ...memory, - projection: diagnostic.projection ?? memory?.projection ?? null, - projectionStatus: diagnostic.projectionStatus ?? diagnostic.projection?.projectionStatus ?? memory?.projectionStatus ?? null, - projectionHealth: diagnostic.projectionHealth ?? diagnostic.projection?.projectionHealth ?? memory?.projectionHealth ?? null, - blocker: diagnostic.blocker ?? diagnostic.projection?.blocker ?? memory?.blocker ?? null, - valuesPrinted: false - }; -} - -function hasTraceProjection(snapshot) { - return Boolean(snapshot?.status !== "missing" && (Number(snapshot?.eventCount ?? 0) > 0 || snapshot?.lastEvent)); -} - -function shouldPreferDurableTrace(memory, durable) { - if (!durable) return false; - if (!hasTraceProjection(memory)) return true; - const memoryStatus = normalizeWorkbenchStatus(memory?.status); - const durableStatus = normalizeWorkbenchStatus(durable?.status); - if (TERMINAL_STATUSES.has(durableStatus) && !TERMINAL_STATUSES.has(memoryStatus)) return true; - return traceLastSeq(durable) > traceLastSeq(memory); -} - -function traceLastSeq(snapshot) { - const events = Array.isArray(snapshot?.events) ? snapshot.events : []; - const lastEvent = snapshot?.lastEvent ?? events.at(-1) ?? null; - const indexedMax = events.reduce((max, event, index) => Math.max(max, eventSeq(event, index)), 0); - return Math.max(indexedMax, lastEvent ? eventSeq(lastEvent, Math.max(0, events.length - 1)) : 0); -} - function canActorReadSession(session, actor) { if (!session || !actor) return false; if (normalizeStatus(session.status) === "archived") return false; diff --git a/internal/cloud/workbench-kafka-sse-debug.test.ts b/internal/cloud/workbench-kafka-sse-debug.test.ts index 62f54c46..f9e200a5 100644 --- a/internal/cloud/workbench-kafka-sse-debug.test.ts +++ b/internal/cloud/workbench-kafka-sse-debug.test.ts @@ -13,7 +13,7 @@ test("workbench Kafka SSE debug endpoint streams filtered raw HWLAB Kafka events }; const server = createServer((request, response) => { const url = new URL(request.url ?? "/", "http://127.0.0.1"); - void handleWorkbenchKafkaSseDebugHttp(request, response, url, { env, kafkaFactory: fakeKafka.factory, logger: null }); + void handleWorkbenchKafkaSseDebugHttp(request, response, url, { env, kafkaFactory: fakeKafka.factory, accessController: fakeAccessController(), logger: null }); }); await listen(server); const abort = new AbortController(); @@ -25,6 +25,8 @@ test("workbench Kafka SSE debug endpoint streams filtered raw HWLAB Kafka events assert.ok(reader, "SSE response must expose a readable stream"); const connected = await readUntil(reader, "hwlab.kafka.connected"); assert.match(connected, /hwlab\.event\.v1/u); + assert.match(connected, /"consumerReady":true/u); + assert.match(connected, new RegExp(String(fakeKafka.groupId).replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"), "u")); await fakeKafka.emit({ eventType: "hwlab.trace.event.projected", sessionId: "ses_other", event: { label: "ignored" } }); await fakeKafka.emit({ eventType: "hwlab.trace.event.projected", sessionId: "ses_kafka_sse_debug", traceId: "trc_kafka_sse_debug", context: { runId: "run_kafka_sse_debug", commandId: "cmd_kafka_sse_debug" }, event: { label: "agentrun:event:debug", status: "running" } }); @@ -47,7 +49,7 @@ test("workbench Kafka SSE debug endpoint streams filtered codex stdio Kafka even }; const server = createServer((request, response) => { const url = new URL(request.url ?? "/", "http://127.0.0.1"); - void handleWorkbenchKafkaSseDebugHttp(request, response, url, { env, kafkaFactory: fakeKafka.factory, logger: null }); + void handleWorkbenchKafkaSseDebugHttp(request, response, url, { env, kafkaFactory: fakeKafka.factory, accessController: fakeAccessController(), logger: null }); }); await listen(server); const abort = new AbortController(); @@ -90,7 +92,7 @@ test("workbench Kafka SSE debug endpoint resolves traceId to AgentRun Kafka ids" }; const server = createServer((request, response) => { const url = new URL(request.url ?? "/", "http://127.0.0.1"); - void handleWorkbenchKafkaSseDebugHttp(request, response, url, { env, kafkaFactory: fakeKafka.factory, traceStore, logger: null }); + void handleWorkbenchKafkaSseDebugHttp(request, response, url, { env, kafkaFactory: fakeKafka.factory, traceStore, accessController: fakeAccessController(), logger: null }); }); await listen(server); const abort = new AbortController(); @@ -115,18 +117,131 @@ test("workbench Kafka SSE debug endpoint resolves traceId to AgentRun Kafka ids" } }); -function createFakeKafkaFactory() { +test("workbench Kafka SSE debug endpoint is admin-only", async () => { + const fakeKafka = createFakeKafkaFactory(); + const server = createServer((request, response) => { + const url = new URL(request.url ?? "/", "http://127.0.0.1"); + void handleWorkbenchKafkaSseDebugHttp(request, response, url, { + env: { HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092" }, + kafkaFactory: fakeKafka.factory, + accessController: fakeAccessController("user"), + logger: null + }); + }); + await listen(server); + try { + const response = await fetch(`${serverUrl(server)}/v1/workbench/debug/kafka-sse/events?stream=hwlab`); + assert.equal(response.status, 403); + assert.match(await response.text(), /admin_required/u); + assert.equal(fakeKafka.consumerCreated, false); + } finally { + await close(server); + } +}); + +test("workbench Kafka SSE debug endpoint reports connected only after consumer run is ready", async () => { + const fakeKafka = createFakeKafkaFactory({ deferRun: true }); + const server = createServer((request, response) => { + const url = new URL(request.url ?? "/", "http://127.0.0.1"); + void handleWorkbenchKafkaSseDebugHttp(request, response, url, { + env: { + HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", + HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", + HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1" + }, + kafkaFactory: fakeKafka.factory, + accessController: fakeAccessController(), + logger: null + }); + }); + await listen(server); + const abort = new AbortController(); + try { + const response = await fetch(`${serverUrl(server)}/v1/workbench/debug/kafka-sse/events?stream=hwlab`, { signal: abort.signal }); + const reader = response.body?.getReader(); + assert.ok(reader); + const firstRead = reader.read(); + const early = await Promise.race([firstRead.then(() => "event"), delay(25).then(() => "pending")]); + assert.equal(early, "pending"); + fakeKafka.releaseRun(); + const first = await firstRead; + assert.equal(first.done, false); + const connected = new TextDecoder().decode(first.value); + assert.match(connected, /hwlab\.kafka\.connected/u); + assert.match(connected, /"consumerReady":true/u); + assert.match(connected, /"groupId":"test-hwlab-cloud-api-debug-sse-/u); + } finally { + abort.abort(); + fakeKafka.releaseRun(); + await close(server); + } +}); + +test("workbench Kafka SSE debug endpoint stops a consumer that becomes ready after client close", async () => { + const fakeKafka = createFakeKafkaFactory({ deferRun: true }); + const server = createServer((request, response) => { + const url = new URL(request.url ?? "/", "http://127.0.0.1"); + void handleWorkbenchKafkaSseDebugHttp(request, response, url, { + env: { HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1" }, + kafkaFactory: fakeKafka.factory, + accessController: fakeAccessController(), + logger: null + }); + }); + await listen(server); + const abort = new AbortController(); + try { + const response = await fetch(`${serverUrl(server)}/v1/workbench/debug/kafka-sse/events?stream=hwlab`, { signal: abort.signal }); + const reader = response.body?.getReader(); + assert.ok(reader); + const firstRead = reader.read().catch(() => null); + await delay(20); + abort.abort(); + fakeKafka.releaseRun(); + await waitUntil(() => fakeKafka.stopCalls > 0); + const first = await firstRead; + const text = first && !first.done && first.value ? new TextDecoder().decode(first.value) : ""; + assert.doesNotMatch(text, /hwlab\.kafka\.connected/u); + assert.equal(fakeKafka.disconnectCalls > 0, true); + } finally { + abort.abort(); + fakeKafka.releaseRun(); + await close(server); + } +}); + +function createFakeKafkaFactory(options: { deferRun?: boolean } = {}) { let eachMessage: ((input: any) => Promise) | null = null; let subscribedTopic = "hwlab.event.v1"; + let releaseRun: (() => void) | null = null; + const runGate = options.deferRun ? new Promise((resolve) => { releaseRun = resolve; }) : null; + let groupId: string | null = null; + let stopCalls = 0; + let disconnectCalls = 0; + let consumerCreated = false; const consumer = { connect: async () => undefined, subscribe: async (input: { topic?: string }) => { subscribedTopic = String(input.topic ?? subscribedTopic); }, - run: async (input: any) => { eachMessage = input.eachMessage; }, - stop: async () => undefined, - disconnect: async () => undefined + run: async (input: any) => { + if (runGate) await runGate; + eachMessage = input.eachMessage; + }, + stop: async () => { stopCalls += 1; }, + disconnect: async () => { disconnectCalls += 1; } }; return { - factory: () => ({ consumer: () => consumer }), + factory: () => ({ + consumer: (input: { groupId?: string }) => { + consumerCreated = true; + groupId = String(input.groupId ?? ""); + return consumer; + } + }), + get groupId() { return groupId; }, + get stopCalls() { return stopCalls; }, + get disconnectCalls() { return disconnectCalls; }, + get consumerCreated() { return consumerCreated; }, + releaseRun: () => releaseRun?.(), emit: async (value: Record) => { assert.ok(eachMessage, "consumer.run must be called before emitting fake Kafka events"); await eachMessage({ @@ -143,6 +258,27 @@ function createFakeKafkaFactory() { }; } +function fakeAccessController(role: "admin" | "user" = "admin") { + return { + async ensureBootstrap() {}, + async authenticate() { + return { ok: true, status: 200, actor: { id: role === "admin" ? "usr_debug_admin" : "usr_debug_user", role } }; + } + }; +} + +async function delay(ms: number) { + await new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function waitUntil(predicate: () => boolean, timeoutMs = 1000) { + const deadline = Date.now() + timeoutMs; + while (!predicate()) { + if (Date.now() >= deadline) throw new Error("condition did not become true before timeout"); + await delay(10); + } +} + async function readUntil(reader: ReadableStreamDefaultReader, pattern: string): Promise { const decoder = new TextDecoder(); let text = ""; diff --git a/internal/cloud/workbench-kafka-sse-debug.ts b/internal/cloud/workbench-kafka-sse-debug.ts index ca7df2b7..10d4b13a 100644 --- a/internal/cloud/workbench-kafka-sse-debug.ts +++ b/internal/cloud/workbench-kafka-sse-debug.ts @@ -5,6 +5,7 @@ import { sendJson } from "./server-http-utils.ts"; import { openKafkaEventStream } from "./kafka-event-bridge.ts"; import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; +import { authenticateWorkbenchRead } from "./server-workbench-read-http.ts"; const CONTRACT_VERSION = "workbench-debug-kafka-sse-v1"; const DEFAULT_STREAM = "hwlab"; @@ -13,6 +14,11 @@ const ALLOWED_STREAMS = new Set(["stdio", "agentrun", "hwlab"]); export async function handleWorkbenchKafkaSseDebugHttp(request, response, url, options = {}) { const route = routeSuffix(url.pathname); try { + const auth = await authenticateWorkbenchRead(request, response, options); + if (!auth) return; + if (auth.actor?.role !== "admin") { + return sendJson(response, 403, debugError("admin_required", "Only admin users can access raw Workbench Kafka debug streams.")); + } if (route === "") { if (request.method !== "GET") return methodNotAllowed(response, "GET"); return sendJson(response, 200, describeKafkaSseDebug(url, options.env ?? process.env)); @@ -64,16 +70,6 @@ async function openKafkaDebugSse(request, response, url, options) { "x-content-type-options": "nosniff" }); if (typeof response.flushHeaders === "function") response.flushHeaders(); - writeSse(response, "hwlab.kafka.connected", { - ok: true, - contractVersion: CONTRACT_VERSION, - stream, - topic: topicForStream(stream, env), - filters, - resolvedFilters, - serverSentAt: new Date().toISOString(), - valuesPrinted: false - }); let kafkaStream = null; let closed = false; const close = () => { @@ -81,7 +77,9 @@ async function openKafkaDebugSse(request, response, url, options) { closed = true; void kafkaStream?.stop?.(); }; - request.on("close", close); + response.once("close", close); + request.once?.("aborted", close); + request.socket?.once?.("close", close); try { kafkaStream = await openKafkaEventStream({ env, @@ -103,8 +101,23 @@ async function openKafkaDebugSse(request, response, url, options) { serverSentAt: new Date().toISOString(), valuesPrinted: false }); - }, - onError: (error) => writeSse(response, "hwlab.kafka.error", { ok: false, error: errorMessagePayload(error), valuesPrinted: false }) + } + }); + if (closed) { + await kafkaStream.stop?.(); + return; + } + writeSse(response, "hwlab.kafka.connected", { + ok: true, + contractVersion: CONTRACT_VERSION, + consumerReady: true, + groupId: kafkaStream.groupId, + stream, + topic: kafkaStream.topic ?? topicForStream(stream, env), + filters, + resolvedFilters, + serverSentAt: new Date().toISOString(), + valuesPrinted: false }); } catch (error) { writeSse(response, "hwlab.kafka.error", { ok: false, error: errorMessagePayload(error), valuesPrinted: false }); diff --git a/internal/cloud/workbench-projection-cursor.test.ts b/internal/cloud/workbench-projection-cursor.test.ts deleted file mode 100644 index 03d20970..00000000 --- a/internal/cloud/workbench-projection-cursor.test.ts +++ /dev/null @@ -1,153 +0,0 @@ -// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor. -// Responsibility: Independent backend unit tests for AgentRun projection cursor planning. - -import assert from "node:assert/strict"; -import { test } from "bun:test"; - -import { buildAgentRunProjectionEventsFetchPlan, buildWorkbenchProjectionStateUpdate } from "./workbench-projection-cursor.ts"; - -test("AgentRun projection fetch plan uses durable cursor without scanning long trace", () => { - const poisonTrace = {}; - Object.defineProperty(poisonTrace, "events", { - get() { - throw new Error("trace events must not be read to compute AgentRun afterSeq"); - } - }); - const poisonResult = {}; - Object.defineProperty(poisonResult, "events", { - get() { - throw new Error("result events must not be read to compute AgentRun afterSeq"); - } - }); - - const plan = buildAgentRunProjectionEventsFetchPlan({ - projectionState: { - traceId: "trc_cursor_o1", - sourceRunId: "run_cursor_o1", - sourceCommandId: "cmd_cursor_o1", - lastSourceSeq: 3700, - lastProjectedSeq: 129 - }, - agentRun: { - runId: "run_cursor_o1", - commandId: "cmd_cursor_o1", - lastSeq: 12 - }, - trace: poisonTrace, - result: poisonResult, - pageLimit: 500 - }); - - assert.equal(plan.runId, "run_cursor_o1"); - assert.equal(plan.commandId, "cmd_cursor_o1"); - assert.equal(plan.afterSeq, 3700); - assert.equal(plan.limit, 500); - assert.equal(plan.cursorSource, "projection-state"); -}); - -test("AgentRun projection state advances from incremental events response", () => { - const state = buildWorkbenchProjectionStateUpdate({ - currentState: { - traceId: "trc_cursor_update", - sourceRunId: "run_cursor_update", - sourceCommandId: "cmd_cursor_update", - lastSourceSeq: 3700, - lastProjectedSeq: 3700, - sourceLatestSeq: 3700, - projectionStatus: "projecting", - resultSyncState: "not_started", - createdAt: "2026-06-19T12:00:00.000Z" - }, - agentRun: { - runId: "run_cursor_update", - commandId: "cmd_cursor_update", - lastSeq: 3700 - }, - eventsResponse: { - traceLastSeq: 3720, - maxSeq: 3721, - rawEventCount: 20 - }, - now: () => "2026-06-19T12:01:00.000Z" - }); - - assert.equal(state.lastSourceSeq, 3720); - assert.equal(state.lastAgentRunSeq, 3720); - assert.equal(state.lastProjectedSeq, 3720); - assert.equal(state.sourceLatestSeq, 3721); - assert.equal(state.projectionStatus, "projecting"); - assert.equal(state.resultSyncState, "not_started"); - assert.equal(state.createdAt, "2026-06-19T12:00:00.000Z"); - assert.equal(state.updatedAt, "2026-06-19T12:01:00.000Z"); -}); - -test("AgentRun projection state preserves retry params for background resume", () => { - const state = buildWorkbenchProjectionStateUpdate({ - currentState: { - traceId: "trc_cursor_retry_params", - sourceRunId: "run_cursor_retry_params", - sourceCommandId: "cmd_cursor_retry_params", - retryParams: { - message: "preserve this prompt for fresh-session retry", - sessionId: "ses_cursor_retry", - providerProfile: "dsflash-go", - valuesPrinted: false - }, - createdAt: "2026-06-19T12:00:00.000Z" - }, - agentRun: { - runId: "run_cursor_retry_params", - commandId: "cmd_cursor_retry_params", - lastSeq: 3 - }, - now: () => "2026-06-19T12:01:00.000Z" - }); - - assert.equal(state.retryParams.message, "preserve this prompt for fresh-session retry"); - assert.equal(state.retryParams.prompt, "preserve this prompt for fresh-session retry"); - assert.equal(state.retryParams.sessionId, "ses_cursor_retry"); - assert.equal(state.retryParams.providerProfile, "dsflash-go"); - assert.equal(state.retryParams.valuesPrinted, false); -}); - -test("AgentRun projection state resets source cursor when fresh retry switches run", () => { - const state = buildWorkbenchProjectionStateUpdate({ - currentState: { - traceId: "trc_cursor_fresh_retry", - sourceRunId: "run_cursor_failed", - sourceCommandId: "cmd_cursor_failed", - lastSourceSeq: 3700, - lastAgentRunSeq: 3700, - lastProjectedSeq: 3700, - sourceLatestSeq: 3700, - projectionStatus: "terminal", - resultSyncState: "pending", - retryParams: { - message: "retry on a fresh AgentRun session", - sessionId: "ses_cursor_fresh_retry", - providerProfile: "dsflash-go", - valuesPrinted: false - }, - createdAt: "2026-06-19T12:00:00.000Z" - }, - agentRun: { - runId: "run_cursor_fresh_retry", - commandId: "cmd_cursor_fresh_retry", - lastSeq: 0 - }, - projectionStatus: "projecting", - resultSyncState: null, - now: () => "2026-06-19T12:02:00.000Z" - }); - - assert.equal(state.sourceRunId, "run_cursor_fresh_retry"); - assert.equal(state.sourceCommandId, "cmd_cursor_fresh_retry"); - assert.equal(state.lastSourceSeq, 0); - assert.equal(state.lastAgentRunSeq, 0); - assert.equal(state.lastProjectedSeq, 0); - assert.equal(state.sourceLatestSeq, 0); - assert.equal(state.projectionStatus, "projecting"); - assert.equal(state.retryParams.message, "retry on a fresh AgentRun session"); - const plan = buildAgentRunProjectionEventsFetchPlan({ projectionState: state, agentRun: { runId: "run_cursor_fresh_retry", commandId: "cmd_cursor_fresh_retry", lastSeq: 0 } }); - assert.equal(plan.afterSeq, 0); -}); diff --git a/internal/cloud/workbench-projection-cursor.ts b/internal/cloud/workbench-projection-cursor.ts deleted file mode 100644 index 3f286390..00000000 --- a/internal/cloud/workbench-projection-cursor.ts +++ /dev/null @@ -1,205 +0,0 @@ -/* - * SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; PJ2026-010205 HWLAB接入 draft-2026-06-17-r0. - * Responsibility: Pure cursor planning for AgentRun events projection; never derives cursor by scanning trace events or result payloads. - */ - -const DEFAULT_AGENTRUN_EVENTS_PAGE_LIMIT = 500; -const MAX_AGENTRUN_EVENTS_PAGE_LIMIT = 1000; - -export function buildAgentRunProjectionEventsFetchPlan({ projectionState = null, agentRun = null, pageLimit = DEFAULT_AGENTRUN_EVENTS_PAGE_LIMIT } = {}) { - const state = normalizeProjectionState(projectionState); - const mapping = normalizeAgentRunMapping(agentRun); - const runId = textValue(state?.sourceRunId ?? state?.runId ?? mapping.runId); - const commandId = textValue(state?.sourceCommandId ?? state?.commandId ?? mapping.commandId); - const stateSeq = maxNonNegativeInteger(state?.lastSourceSeq, state?.lastAgentRunSeq); - const mappingSeq = maxNonNegativeInteger(mapping.lastSeq); - return { - runId, - commandId, - afterSeq: Math.max(stateSeq, mappingSeq), - limit: boundedPageLimit(pageLimit), - cursorSource: state ? "projection-state" : "agent-run-result", - valuesPrinted: false - }; -} - -export function buildWorkbenchProjectionStateUpdate({ - currentState = null, - result = null, - agentRun = null, - eventsResponse = null, - resultSyncState = null, - projectionStatus = null, - projectionHealth = null, - polling = null, - retryParams = null, - error = null, - now = () => new Date().toISOString() -} = {}) { - const previous = normalizeProjectionState(currentState) ?? {}; - const mapping = normalizeAgentRunMapping(agentRun ?? result?.agentRun); - const timestamp = timestampValue(now); - const traceId = textValue(previous.traceId ?? result?.traceId ?? mapping.traceId); - const runId = textValue(mapping.runId) || textValue(previous.sourceRunId ?? previous.runId); - const commandId = textValue(mapping.commandId) || textValue(previous.sourceCommandId ?? previous.commandId); - const previousRunId = textValue(previous.sourceRunId ?? previous.runId); - const previousCommandId = textValue(previous.sourceCommandId ?? previous.commandId); - const sourceChanged = Boolean((previousRunId && runId && previousRunId !== runId) || (previousCommandId && commandId && previousCommandId !== commandId)); - const previousCursorSeq = sourceChanged ? 0 : maxNonNegativeInteger(previous.lastSourceSeq, previous.lastAgentRunSeq); - const lastSourceSeq = Math.max( - previousCursorSeq, - maxNonNegativeInteger(mapping.lastSeq), - maxNonNegativeInteger(eventsResponse?.traceLastSeq) - ); - const sourceLatestSeq = Math.max( - sourceChanged ? 0 : maxNonNegativeInteger(previous.sourceLatestSeq, previous.upstreamLatestSeq), - maxNonNegativeInteger(eventsResponse?.maxSeq), - lastSourceSeq - ); - const nextRetryParams = normalizeProjectionRetryParams(retryParams) ?? normalizeProjectionRetryParams(previous.retryParams); - const nextPolling = normalizeProjectionPolling(polling); - const nextResultSyncState = textValue(resultSyncState ?? previous.resultSyncState) || "not_started"; - const nextProjectionStatus = textValue(projectionStatus) || inferProjectionStatus({ lastSourceSeq, sourceLatestSeq, resultSyncState: nextResultSyncState, previous }); - const nextProjectionHealth = textValue(projectionHealth) || (error ? "degraded" : "healthy"); - const nextRetryAt = nextPolling?.nextRetryAt ?? (error ? previous.nextRetryAt ?? null : null); - return { - ...previous, - traceId, - sessionId: textValue(previous.sessionId ?? result?.sessionId ?? mapping.sessionId) || null, - conversationId: textValue(previous.conversationId ?? result?.conversationId ?? mapping.conversationId) || null, - threadId: textValue(previous.threadId ?? result?.threadId ?? mapping.threadId) || null, - ownerUserId: textValue(previous.ownerUserId ?? result?.ownerUserId) || null, - ownerRole: textValue(previous.ownerRole ?? result?.ownerRole) || null, - managerUrl: textValue(previous.managerUrl ?? mapping.managerUrl) || null, - backendProfile: textValue(previous.backendProfile ?? mapping.backendProfile) || null, - providerId: textValue(previous.providerId ?? mapping.providerId) || null, - sourceRunId: runId, - sourceCommandId: commandId, - runId, - commandId, - lastSourceSeq, - lastAgentRunSeq: lastSourceSeq, - lastProjectedSeq: sourceChanged ? lastSourceSeq : Math.max(maxNonNegativeInteger(previous.lastProjectedSeq), lastSourceSeq), - sourceLatestSeq, - upstreamLatestSeq: sourceLatestSeq, - projectionStatus: nextProjectionStatus, - projectionHealth: nextProjectionHealth, - resultSyncState: nextResultSyncState, - lastProjectedAt: sourceChanged || lastSourceSeq > maxNonNegativeInteger(previous.lastSourceSeq, previous.lastAgentRunSeq) ? timestamp : previous.lastProjectedAt ?? timestamp, - lastResultSyncAt: nextResultSyncState === "synced" ? timestamp : previous.lastResultSyncAt ?? null, - lastErrorCode: error ? textValue(error.code ?? error.errorCode ?? "projection_sync_failed") : null, - lastErrorMessage: error ? textValue(error.message ?? "Projection sync failed.") : null, - failureCount: error ? maxNonNegativeInteger(previous.failureCount) + 1 : 0, - nextRetryAt, - pollCount: nextPolling?.pollCount ?? maxNonNegativeInteger(previous.pollCount), - noProgressPollCount: nextPolling?.noProgressPollCount ?? (error ? maxNonNegativeInteger(previous.noProgressPollCount) : 0), - backoffMs: nextPolling?.backoffMs ?? 0, - backoffReason: nextPolling?.backoffReason ?? null, - rootCause: nextPolling?.rootCause ?? null, - terminalConsistencyGap: nextPolling?.terminalConsistencyGap ?? null, - retryParams: nextRetryParams, - createdAt: previous.createdAt ?? timestamp, - updatedAt: timestamp, - valuesPrinted: false - }; -} - -export function agentRunResultSyncDeferred({ forceResultSync = false, options = {}, env = process.env } = {}) { - if (forceResultSync) return false; - if (options.deferAgentRunResultSync === true) return true; - const configured = String(env?.HWLAB_CODE_AGENT_AGENTRUN_RESULT_SYNC_IN_RUNNING ?? "").trim().toLowerCase(); - return ["0", "false", "off", "no"].includes(configured); -} - -function normalizeProjectionState(value) { - if (!value || typeof value !== "object" || Array.isArray(value)) return null; - return value; -} - -function normalizeAgentRunMapping(value) { - return value && typeof value === "object" && !Array.isArray(value) ? value : {}; -} - -function normalizeProjectionRetryParams(value) { - if (!value || typeof value !== "object" || Array.isArray(value)) return null; - const message = textValue(value.message ?? value.prompt ?? value.text); - if (!message) return null; - const result = { - message, - prompt: message, - text: message, - sessionId: textValue(value.sessionId) || null, - conversationId: textValue(value.conversationId) || null, - threadId: textValue(value.threadId) || null, - projectId: textValue(value.projectId) || null, - ownerUserId: textValue(value.ownerUserId) || null, - ownerRole: textValue(value.ownerRole) || null, - providerProfile: textValue(value.providerProfile ?? value.codeAgentProviderProfile ?? value.backendProfile) || null, - codeAgentProviderProfile: textValue(value.codeAgentProviderProfile ?? value.providerProfile ?? value.backendProfile) || null, - backendProfile: textValue(value.backendProfile ?? value.providerProfile ?? value.codeAgentProviderProfile) || null, - providerId: textValue(value.providerId) || null, - valuesPrinted: false - }; - for (const [key, item] of Object.entries(result)) { - if (item === null || item === "") delete result[key]; - } - return result; -} - -function normalizeProjectionPolling(value) { - if (!value || typeof value !== "object" || Array.isArray(value)) return null; - const result = { - pollCount: maxNonNegativeInteger(value.pollCount), - noProgressPollCount: maxNonNegativeInteger(value.noProgressPollCount), - backoffMs: maxNonNegativeInteger(value.backoffMs), - backoffReason: clippedText(value.backoffReason, 80), - rootCause: clippedText(value.rootCause, 120), - terminalConsistencyGap: maxNonNegativeInteger(value.terminalConsistencyGap), - nextRetryAt: timestampText(value.nextRetryAt) - }; - return result; -} - -function inferProjectionStatus({ lastSourceSeq, sourceLatestSeq, resultSyncState, previous }) { - if (["pending", "timed_out", "failed"].includes(resultSyncState)) return "terminal"; - if (sourceLatestSeq > lastSourceSeq) return "projecting"; - return textValue(previous?.projectionStatus) || "projecting"; -} - -function boundedPageLimit(value) { - const parsed = Number.parseInt(String(value ?? ""), 10); - if (!Number.isInteger(parsed) || parsed <= 0) return DEFAULT_AGENTRUN_EVENTS_PAGE_LIMIT; - return Math.min(parsed, MAX_AGENTRUN_EVENTS_PAGE_LIMIT); -} - -function maxNonNegativeInteger(...values) { - let max = 0; - for (const value of values) { - const parsed = Number.parseInt(String(value ?? ""), 10); - if (Number.isInteger(parsed) && parsed >= 0 && parsed > max) max = parsed; - } - return max; -} - -function timestampValue(now) { - const value = typeof now === "function" ? now() : now; - const parsed = Date.parse(String(value ?? "")); - return Number.isFinite(parsed) ? new Date(parsed).toISOString() : new Date().toISOString(); -} - -function textValue(value) { - return String(value ?? "").trim(); -} - -function clippedText(value, limit) { - const text = textValue(value); - if (!text) return null; - return text.length > limit ? text.slice(0, limit) : text; -} - -function timestampText(value) { - const text = textValue(value); - if (!text) return null; - const parsed = Date.parse(text); - return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null; -} diff --git a/internal/cloud/workbench-projection-finalizer.ts b/internal/cloud/workbench-projection-finalizer.ts deleted file mode 100644 index c5b72a10..00000000 --- a/internal/cloud/workbench-projection-finalizer.ts +++ /dev/null @@ -1,106 +0,0 @@ -/* - * SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-18-p0-unique-projection; PJ2026-010205 HWLAB接入 draft-2026-06-17-r0. - * 职责: WorkbenchProjectionFinalizer 组件入口。以 checkpoint/轮询驱动 AgentRun facts 追平，不让 GET path 承担 finalize/repair。 - */ -import { appendProjectionDiagnostic } from "./workbench-projection-writer.ts"; - -export function scheduleWorkbenchProjectionFinalizer({ traceId, currentResult = null, traceStore, getCachedResult, isCanceled, isTerminal, syncResult, onTerminal, activitySignature, noResponseTimeoutMs = null, pollIntervalMs = 1000, sleep = defaultSleep, performanceStore = null } = {}) { - if (!traceId || !currentResult?.agentRun?.runId || !currentResult?.agentRun?.commandId || typeof syncResult !== "function") return null; - let lastActivityAt = Date.now(); - let lastActivitySignature = activitySignature?.(currentResult, traceStore?.snapshot?.(traceId)) ?? null; - let idleDegraded = false; - performanceStore?.recordWorkbenchProjectorCandidate?.({ status: "scheduled", reason: "finalizer" }); - - setImmediate(() => { - void (async () => { - let result = currentResult; - let lastError = null; - let nextDelayMs = pollIntervalMs; - while (true) { - const cached = getCachedResult?.(traceId); - if (isCanceled?.(cached)) return; - if (cached && isTerminal?.(cached, traceStore?.snapshot?.(traceId))) { - performanceStore?.recordWorkbenchProjectorCandidate?.({ status: "completed", reason: "cached_terminal" }); - onTerminal?.(cached, { preserveLastTraceId: true }); - return; - } - const syncStartedAt = Date.now(); - try { - const synced = await syncResult({ result }); - performanceStore?.recordWorkbenchProjectorBatch?.({ phase: "candidate_scan", status: "ok", durationMs: Date.now() - syncStartedAt }); - result = synced?.result ?? result; - nextDelayMs = projectionFinalizerDelayMs(synced?.polling, pollIntervalMs); - const runnerTrace = synced?.runnerTrace ?? traceStore?.snapshot?.(traceId); - if (result && isTerminal?.(result, runnerTrace)) { - performanceStore?.recordWorkbenchProjectorCandidate?.({ status: "completed", reason: "synced_terminal" }); - onTerminal?.(result, { preserveLastTraceId: true }); - return; - } - const signature = activitySignature?.(result, runnerTrace); - if (signature && signature !== lastActivitySignature) { - lastActivitySignature = signature; - lastActivityAt = Date.now(); - idleDegraded = false; - } - lastError = null; - } catch (error) { - performanceStore?.recordWorkbenchProjectorBatch?.({ phase: "candidate_scan", status: error?.code === "agentrun_timeout" ? "timeout" : "error", durationMs: Date.now() - syncStartedAt }); - performanceStore?.recordWorkbenchProjectorError?.({ reason: error?.code ?? "finalizer_sync_failed" }); - nextDelayMs = projectionFinalizerDelayMs(error?.polling, pollIntervalMs); - lastError = error; - } - const idleMs = Date.now() - lastActivityAt; - if (noResponseTimeoutMs && idleMs >= noResponseTimeoutMs && !idleDegraded) { - performanceStore?.recordWorkbenchProjectorCandidate?.({ status: "degraded", reason: "no_response_idle_timeout" }); - performanceStore?.recordWorkbenchProjectorError?.({ reason: lastError?.code ?? "no_response_idle_timeout" }); - appendProjectionDiagnostic(traceStore, traceId, { - type: "turn-status", - status: "degraded", - label: "projection-finalizer:no-response-idle-timeout", - errorCode: lastError?.code ?? "no_response_idle_timeout", - degradedReason: lastError ? "projection_activity_stale" : "no_response_idle_timeout", - message: lastError?.message ?? `AgentRun projection has no new activity for ${idleMs}ms; idle threshold ${noResponseTimeoutMs}ms.`, - runId: result?.agentRun?.runId ?? currentResult.agentRun.runId, - commandId: result?.agentRun?.commandId ?? currentResult.agentRun.commandId, - timeoutMs: noResponseTimeoutMs, - idleMs, - lastActivityAt: new Date(lastActivityAt).toISOString(), - waitingFor: "agentrun-result-activity", - terminal: false - }); - idleDegraded = true; - } - await sleep(nextDelayMs); - } - })().catch((error) => { - performanceStore?.recordWorkbenchProjectorError?.({ reason: error?.code ?? "workbench_projection_finalizer_crashed" }); - appendProjectionDiagnostic(traceStore, traceId, { - type: "turn-status", - status: "degraded", - label: "projection-finalizer:sync-crashed", - errorCode: error?.code ?? "workbench_projection_finalizer_crashed", - message: error?.message ?? "Workbench projection finalizer crashed.", - runId: currentResult.agentRun.runId, - commandId: currentResult.agentRun.commandId, - waitingFor: "agentrun-result", - terminal: false - }); - }); - }); - return { scheduled: true, traceId, noResponseTimeoutMs, pollIntervalMs, valuesPrinted: false }; -} - -function defaultSleep(ms) { - return new Promise((resolve) => setTimeout(resolve, Math.max(0, Number(ms) || 0))); -} - -function projectionFinalizerDelayMs(polling = null, fallbackMs = 1000) { - const fallback = positiveInteger(fallbackMs, 1000); - const backoffMs = positiveInteger(polling?.backoffMs, 0); - return backoffMs > 0 ? Math.max(fallback, backoffMs) : fallback; -} - -function positiveInteger(value, fallback) { - const parsed = Number.parseInt(String(value ?? ""), 10); - return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; -} diff --git a/internal/cloud/workbench-projection-outbox-events.ts b/internal/cloud/workbench-projection-outbox-events.ts new file mode 100644 index 00000000..7e5fed25 --- /dev/null +++ b/internal/cloud/workbench-projection-outbox-events.ts @@ -0,0 +1,99 @@ +/* + * Immutable Workbench projection outbox rows are the common SSE and sync-replay authority. + */ + +export const WORKBENCH_REALTIME_AUTHORITY_VERSION = "workbench-realtime-authority-v2"; + +export function projectionOutboxRealtimeEvents(snapshot = {}, { includeSnapshot = false, includeEvents = true } = {}) { + const facts = objectValue(snapshot.facts); + const rows = includeEvents ? factArray(snapshot.events) : []; + const emitted = new Set(); + const output = []; + for (const row of rows) { + const family = textValue(row.entityFamily) || textValue(row.payload?.family) || projectionOutboxFamily(row); + const fact = projectionOutboxFact(row); + if (!fact || !SUPPORTED_FAMILIES.has(family)) continue; + const item = projectionRealtimeItem(row, fact, family); + if (!item) continue; + emitted.add(`${family}:${item.payload.entity.id}`); + output.push(item); + } + if (includeSnapshot) { + const cutoff = nonNegativeInteger(snapshot.cutoffOutboxSeq); + for (const [family, name] of [["messages", "workbench.message.snapshot"], ["turns", "workbench.turn.snapshot"]]) { + for (const fact of factArray(facts[family])) { + const id = projectionFactId(fact, family); + if (!id || emitted.has(`${family}:${id}`)) continue; + const item = projectionRealtimeItem({ outboxSeq: cutoff, entityFamily: family, entityId: id, projectedSeq: fact.projectedSeq, projectionRevision: fact.projectedSeq, createdAt: fact.updatedAt }, fact, family, name); + if (item) output.push(item); + } + } + } + return output.sort((left, right) => nonNegativeInteger(left.payload.cursor?.outboxSeq) - nonNegativeInteger(right.payload.cursor?.outboxSeq)); +} + +function projectionRealtimeItem(row, fact, family, forcedName = null) { + const id = textValue(row.entityId) || projectionFactId(fact, family); + if (!id) return null; + const outboxSeq = nonNegativeInteger(row.outboxSeq); + const version = nonNegativeInteger(row.projectionRevision ?? row.projectedSeq ?? fact.projectedSeq ?? outboxSeq); + const projectionRevision = String(version); + const base = { + contractVersion: "workbench-sync-v1", + realtimeAuthority: WORKBENCH_REALTIME_AUTHORITY_VERSION, + realtimeSource: "projection-outbox", + sessionId: textValue(fact.sessionId ?? row.sessionId), + traceId: textValue(fact.traceId ?? row.traceId), + cursor: { outboxSeq, traceSeq: nonNegativeInteger(row.projectedSeq ?? fact.projectedSeq) }, + entity: { family, id, version, entityVersion: version, outboxSeq, traceSeq: nonNegativeInteger(row.projectedSeq ?? fact.projectedSeq), projectionRevision, authority: WORKBENCH_REALTIME_AUTHORITY_VERSION }, + family, + id, + commitType: textValue(row.commitType), + terminal: row.terminal === true || fact.terminal === true, + sealed: row.sealed === true || fact.sealed === true, + projectionRevision, + outboxEventId: textValue(row.outboxEventId), + eventCreatedAt: textValue(row.createdAt ?? fact.updatedAt ?? fact.createdAt), + valuesRedacted: true + }; + if (family === "messages") return { name: forcedName ?? "workbench.message.snapshot", payload: { ...base, type: "message.snapshot", reason: "projection-outbox", message: fact } }; + if (family === "turns") return { name: forcedName ?? "workbench.turn.snapshot", payload: { ...base, type: "turn.snapshot", reason: "projection-outbox", turn: fact } }; + return { name: forcedName ?? "workbench.trace.event", payload: { ...base, type: "trace.event", event: fact, snapshot: { traceId: base.traceId, sessionId: base.sessionId, status: fact.status ?? null, events: [fact], eventCount: 1 } } }; +} + +function projectionOutboxFact(row) { + const fact = row?.payload?.fact; + return fact && typeof fact === "object" && !Array.isArray(fact) ? fact : null; +} + +function projectionOutboxFamily(row) { + if (row.commitType === "message") return "messages"; + if (row.commitType === "terminal" && row.entityFamily !== "traceEvents") return "turns"; + return "traceEvents"; +} + +function projectionFactId(fact, family) { + if (family === "messages") return textValue(fact.messageId ?? fact.id); + if (family === "turns") return textValue(fact.turnId ?? fact.traceId); + return textValue(fact.id ?? fact.sourceEventId); +} + +function factArray(value) { + return Array.isArray(value) ? value : []; +} + +function objectValue(value) { + return value && typeof value === "object" && !Array.isArray(value) ? value : {}; +} + +function textValue(value) { + const text = typeof value === "string" ? value.trim() : value === null || value === undefined ? "" : String(value).trim(); + return text || ""; +} + +function nonNegativeInteger(value) { + const parsed = Number.parseInt(String(value ?? ""), 10); + return Number.isInteger(parsed) && parsed >= 0 ? parsed : 0; +} + +const SUPPORTED_FAMILIES = new Set(["messages", "turns", "traceEvents"]); diff --git a/internal/cloud/workbench-projection-writer.test.ts b/internal/cloud/workbench-projection-writer.test.ts index 5d4368d7..3336f1c8 100644 --- a/internal/cloud/workbench-projection-writer.test.ts +++ b/internal/cloud/workbench-projection-writer.test.ts @@ -1,1377 +1,288 @@ -// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor. -// Responsibility: Workbench projection writer/finalizer durable facts regression tests. - import assert from "node:assert/strict"; import { test } from "bun:test"; -import { createCloudRuntimeStore } from "../db/runtime-store.ts"; -import { writeWorkbenchProjectionEvent, writeWorkbenchProjectionSession } from "./workbench-projection-writer.ts"; +import { + buildWorkbenchProjectionEventFacts, + recordWorkbenchSessionOwner, + writeWorkbenchSessionAdmissionFact +} from "./workbench-projection-writer.ts"; +import { projectionOutboxRealtimeEvents } from "./server-workbench-realtime-http.ts"; -test("workbench projection writer preserves Workbench launch context on Code Agent session facts", async () => { - const factWrites = []; - const accessWrites = []; - const launchContext = { - source: "project-management", - projectId: "project_constart_71freq", - taskRef: "mdtodo:constart-71freq-mdtodo:file_c5abae4e69c60370:R1", - sourceId: "constart-71freq-mdtodo", - sourceKind: "hwpod-workspace", - fileRef: "file_c5abae4e69c60370", - taskId: "R1", - hwpodId: "constart-71freq-c", - nodeId: "node-d601-f103-v2", - mdtodoRootRef: "docs/MDTODO", - workspaceRootHash: "workspace-hash", - hwpodWorkspaceArgs: "--hwpod-id constart-71freq-c --workspace-path 'F:\\Work\\ConStart'", - contextFingerprint: "ctx_projection_launch", - executionContext: { - sourceId: "constart-71freq-mdtodo", - sourceKind: "hwpod-workspace", - projectId: "project_constart_71freq", - taskRef: "mdtodo:constart-71freq-mdtodo:file_c5abae4e69c60370:R1", - fileRef: "file_c5abae4e69c60370", - taskId: "R1", - hwpodId: "constart-71freq-c", - nodeId: "node-d601-f103-v2", - mdtodoRootRef: "docs/MDTODO", - workspaceRootHash: "workspace-hash", - hwpodWorkspaceArgs: "--hwpod-id constart-71freq-c --workspace-path 'F:\\Work\\ConStart'", - contextFingerprint: "ctx_projection_launch", - valuesRedacted: true - }, - valuesRedacted: true - }; - const runtimeStore = { - async writeWorkbenchFacts(params, requestMeta) { - factWrites.push({ params, requestMeta }); - return { written: true, facts: params.facts }; - } - }; +test("HTTP lifecycle records ownership without writing projection facts", async () => { + const calls = []; const accessController = { async recordAgentSessionOwner(input) { - accessWrites.push(input); - return { - id: input.sessionId, - projectId: input.projectId, - ownerUserId: input.ownerUserId, - conversationId: input.conversationId, - threadId: input.threadId, - lastTraceId: input.traceId, - status: input.status, - session: input.session, - updatedAt: "2026-06-28T11:15:00.000Z" - }; + calls.push(input); + return { id: input.sessionId, ownerUserId: input.ownerUserId, status: input.status }; } }; - - await writeWorkbenchProjectionSession({ + let projectionWrites = 0; + const owner = await recordWorkbenchSessionOwner({ accessController, + runtimeStore: { writeWorkbenchFacts() { projectionWrites += 1; } }, + traceId: "trc_owner_only", + ownerUserId: "usr_owner", + ownerRole: "user", + sessionId: "ses_owner_only", + projectId: "prj_owner", + status: "completed", + session: { source: "http-result" } + }); + + assert.equal(owner.id, "ses_owner_only"); + assert.equal(calls.length, 1); + assert.equal(calls[0].traceId, "trc_owner_only"); + assert.equal(projectionWrites, 0); +}); + +test("admission persists only the dedicated session scope fact", async () => { + const calls = []; + const runtimeStore = { + async writeWorkbenchSessionAdmissionFact(params, meta) { + calls.push({ params, meta }); + return { written: true, admissionOnly: true }; + }, + writeWorkbenchFacts() { + throw new Error("generic projection writer must not be used by admission"); + } + }; + const result = await writeWorkbenchSessionAdmissionFact({ runtimeStore, - traceId: "trc_projection_launch_context", - ownerUserId: "usr_projection_launch", - ownerRole: "admin", - sessionId: "ses_projection_launch_context", - projectId: "project_constart_71freq", - conversationId: "cnv_projection_launch_context", - threadId: "thread-projection-launch-context", - status: "running", - payload: { - traceId: "trc_projection_launch_context", + session: { + id: "ses_admission_only", + ownerUserId: "usr_owner", + ownerRole: "user", + projectId: "prj_owner", + conversationId: "cnv_owner", + lastTraceId: "trc_admission_only", status: "running", - updatedAt: "2026-06-28T11:15:00.000Z" - }, - params: { - projectId: "project_constart_71freq", - taskRef: "mdtodo:constart-71freq-mdtodo:file_c5abae4e69c60370:R1", - message: "run this MDTODO task", - launchContext - }, - session: { - sessionStatus: "running", - messages: [ - { messageId: "msg_projection_launch_user", role: "user", text: "run this MDTODO task", status: "sent", turnId: "trc_projection_launch_context", traceId: "trc_projection_launch_context" } - ], - valuesRedacted: true, - secretMaterialStored: false + session: { source: "manual-session-create", launchContext: { source: "workbench" } }, + createdAt: "2026-07-10T10:00:00.000Z", + updatedAt: "2026-07-10T10:00:01.000Z" } }); - assert.equal(accessWrites.length, 1); - assert.equal(factWrites.length, 1); - const sessionJson = factWrites[0].params.facts.sessions[0].sessionJson; - assert.equal(sessionJson.launchContext.sourceId, "constart-71freq-mdtodo"); - assert.equal(sessionJson.launchContext.hwpodId, "constart-71freq-c"); - assert.equal(sessionJson.launchContext.contextFingerprint, "ctx_projection_launch"); - assert.equal(sessionJson.launchContext.executionContext.hwpodWorkspaceArgs, "--hwpod-id constart-71freq-c --workspace-path 'F:\\Work\\ConStart'"); - assert.equal(sessionJson.launchContext.valuesRedacted, true); - assert.equal(sessionJson.secretMaterialStored, false); + assert.equal(result.admissionOnly, true); + assert.equal(calls.length, 1); + const fact = calls[0].params.fact; + assert.equal(fact.sessionId, "ses_admission_only"); + assert.equal(fact.ownerUserId, "usr_owner"); + assert.equal(fact.lastTraceId, "trc_admission_only"); + assert.equal(fact.projectedSeq, 0); + assert.equal(fact.sourceEventId, "ses_admission_only:session-admission"); + assert.deepEqual(fact.sessionJson.launchContext, { source: "workbench", valuesRedacted: true }); }); -test("workbench projection writer commits terminal owner evidence as sealed durable facts", async () => { - const factWrites = []; - const accessWrites = []; - const runtimeStore = { - async writeWorkbenchFacts(params, requestMeta) { - factWrites.push({ params, requestMeta }); - return { written: true, facts: params.facts }; - } - }; - const accessController = { - async recordAgentSessionOwner(input) { - accessWrites.push(input); - return { - id: input.sessionId, - projectId: input.projectId, - ownerUserId: input.ownerUserId, - conversationId: input.conversationId, - threadId: input.threadId, - lastTraceId: input.traceId, - status: input.status, - session: input.session, - updatedAt: "2026-06-20T11:00:00.000Z" - }; - } - }; - - await writeWorkbenchProjectionSession({ - accessController, - runtimeStore, - traceId: "trc_writer_terminal", - ownerUserId: "usr_writer", - ownerRole: "user", - sessionId: "ses_writer_terminal", - projectId: "prj_writer", - conversationId: "cnv_writer_terminal", - threadId: "thread-writer-terminal", - status: "completed", - payload: { - traceId: "trc_writer_terminal", - status: "completed", - assistantText: "final answer", - runnerTrace: { - traceId: "trc_writer_terminal", - status: "completed", - startedAt: "2026-06-20T10:59:30.000Z", - lastEventAt: "2026-06-20T11:00:00.000Z", - finishedAt: "2026-06-20T11:00:00.000Z", - elapsedMs: 30000 - }, - agentRun: { runId: "run_writer_terminal", commandId: "cmd_writer_terminal", status: "completed", terminalStatus: "completed", lastSeq: 42 }, - updatedAt: "2026-06-20T11:00:00.000Z" - }, - session: { - sessionStatus: "completed", - messages: [ - { messageId: "msg_writer_user", role: "user", text: "question", status: "sent", turnId: "trc_writer_terminal", traceId: "trc_writer_terminal" }, - { messageId: "msg_writer_agent", role: "agent", text: "final answer", status: "completed", turnId: "trc_writer_terminal", traceId: "trc_writer_terminal" } - ], - finalResponse: { text: "final answer", status: "completed", traceId: "trc_writer_terminal" } - } - }); - - assert.equal(accessWrites.length, 1); - assert.equal(factWrites.length, 1); - const facts = factWrites[0].params.facts; - assert.equal(facts.sessions[0].status, "completed"); - assert.equal(facts.sessions[0].sealed, true); - assert.equal(facts.messages.length, 2); - assert.equal(facts.messages.find((message) => message.messageId === "msg_writer_agent").sealed, true); - assert.equal(facts.parts.some((part) => part.messageId === "msg_writer_agent" && part.text === "final answer" && part.sealed === true), true); - assert.equal(facts.parts.some((part) => part.messageId === "msg_writer_agent" && part.partType === "final_response" && part.text === "final answer" && part.sealed === true), true); - assert.equal(facts.turns[0].terminal, true); - assert.equal(facts.turns[0].sealed, true); - assert.equal(facts.turns[0].finalResponse.text, "final answer"); - assert.equal(facts.messages.find((message) => message.messageId === "msg_writer_agent").lastEventAt, "2026-06-20T11:00:00.000Z"); - assert.equal(facts.messages.find((message) => message.messageId === "msg_writer_agent").durationMs, 30000); - assert.equal(facts.turns[0].startedAt, "2026-06-20T10:59:30.000Z"); - assert.equal(facts.turns[0].durationMs, 30000); - assert.equal(facts.checkpoints[0].projectionStatus, "caught_up"); - assert.equal(facts.checkpoints[0].sealed, true); - assert.equal(facts.checkpoints[0].timing.finishedAt, "2026-06-20T11:00:00.000Z"); +test("admission fails closed when the dedicated durable entry is absent", async () => { + await assert.rejects( + writeWorkbenchSessionAdmissionFact({ runtimeStore: { writeWorkbenchFacts() {} }, session: { id: "ses_missing_entry" } }), + (error) => error?.code === "workbench_admission_store_unconfigured" + ); }); -test("workbench projection writer keeps completed AgentRun without final response unsealed", async () => { - const factWrites = []; - const runtimeStore = { - async writeWorkbenchFacts(params, requestMeta) { - factWrites.push({ params, requestMeta }); - return { written: true, facts: params.facts }; - } - }; - const accessController = { - async recordAgentSessionOwner(input) { - return { - id: input.sessionId, - projectId: input.projectId, - ownerUserId: input.ownerUserId, - conversationId: input.conversationId, - threadId: input.threadId, - lastTraceId: input.traceId, - status: input.status, - session: input.session, - updatedAt: "2026-06-20T11:02:00.000Z" - }; - } - }; - - await writeWorkbenchProjectionSession({ - accessController, - runtimeStore, - traceId: "trc_writer_terminal_missing_final", - ownerUserId: "usr_writer", - ownerRole: "user", - sessionId: "ses_writer_terminal_missing_final", - projectId: "prj_writer", - conversationId: "cnv_writer_terminal_missing_final", - threadId: "thread-writer-terminal-missing-final", - status: "completed", - payload: { - traceId: "trc_writer_terminal_missing_final", - status: "completed", - runnerTrace: { - traceId: "trc_writer_terminal_missing_final", - status: "completed", - startedAt: "2026-06-20T11:01:30.000Z", - lastEventAt: "2026-06-20T11:02:00.000Z", - finishedAt: "2026-06-20T11:02:00.000Z", - events: [ - { seq: 1, type: "assistant_message", status: "running", message: "progress only" }, - { seq: 2, type: "result", status: "completed", terminal: true, label: "agentrun:terminal:completed" } - ], - eventCount: 2 - }, - agentRun: { runId: "run_writer_terminal_missing_final", commandId: "cmd_writer_terminal_missing_final", status: "completed", terminalStatus: "completed", lastSeq: 2 }, - updatedAt: "2026-06-20T11:02:00.000Z" - }, - session: { - sessionStatus: "completed", - messages: [ - { messageId: "msg_writer_missing_final_user", role: "user", text: "question", status: "sent", turnId: "trc_writer_terminal_missing_final", traceId: "trc_writer_terminal_missing_final" }, - { messageId: "msg_writer_missing_final_agent", role: "agent", text: "", status: "completed", turnId: "trc_writer_terminal_missing_final", traceId: "trc_writer_terminal_missing_final" } - ] - } - }); - - assert.equal(factWrites.length, 1); - const facts = factWrites[0].params.facts; - const agentMessage = facts.messages.find((message) => message.messageId === "msg_writer_missing_final_agent"); - assert.equal(facts.sessions[0].status, "running"); - assert.equal(facts.sessions[0].terminal, false); - assert.equal(facts.sessions[0].sealed, false); - assert.equal(agentMessage.status, "running"); - assert.equal(agentMessage.terminal, false); - assert.equal(agentMessage.sealed, false); - assert.equal(facts.parts.some((part) => part.partType === "final_response"), false); - assert.equal(facts.turns[0].status, "running"); - assert.equal(facts.turns[0].terminal, false); - assert.equal(facts.turns[0].sealed, false); - assert.equal(facts.turns[0].finalResponse, null); - assert.equal(facts.turns[0].diagnostic.projectionStatus, "projecting"); - assert.equal(facts.turns[0].diagnostic.waitingFor, "final_response"); - assert.equal(facts.checkpoints[0].projectionStatus, "projecting"); - assert.equal(facts.checkpoints[0].terminal, false); - assert.equal(facts.checkpoints[0].sealed, false); -}); - -test("workbench projection writer seals failed AgentRun turns with failure final response", async () => { - const factWrites = []; - const runtimeStore = { - async writeWorkbenchFacts(params, requestMeta) { - factWrites.push({ params, requestMeta }); - return { written: true, facts: params.facts }; - } - }; - const accessController = { - async recordAgentSessionOwner(input) { - return { - id: input.sessionId, - projectId: input.projectId, - ownerUserId: input.ownerUserId, - conversationId: input.conversationId, - threadId: input.threadId, - lastTraceId: input.traceId, - status: input.status, - session: input.session, - updatedAt: "2026-06-20T11:03:00.000Z" - }; - } - }; - - await writeWorkbenchProjectionSession({ - accessController, - runtimeStore, - traceId: "trc_writer_terminal_failed", - ownerUserId: "usr_writer", - ownerRole: "user", - sessionId: "ses_writer_terminal_failed", - projectId: "prj_writer", - conversationId: "cnv_writer_terminal_failed", - threadId: "thread-writer-terminal-failed", - status: "failed", - payload: { - traceId: "trc_writer_terminal_failed", - status: "failed", - error: { code: "provider-stream-disconnected", message: "provider stream disconnected" }, - runnerTrace: { - traceId: "trc_writer_terminal_failed", - status: "failed", - startedAt: "2026-06-20T11:02:30.000Z", - lastEventAt: "2026-06-20T11:03:00.000Z", - finishedAt: "2026-06-20T11:03:00.000Z", - events: [ - { seq: 1, type: "assistant_message", status: "running", message: "progress only" }, - { seq: 2, type: "result", status: "failed", terminal: true, label: "agentrun:terminal:failed", errorCode: "provider-stream-disconnected" } - ], - eventCount: 2 - }, - agentRun: { runId: "run_writer_terminal_failed", commandId: "cmd_writer_terminal_failed", status: "failed", terminalStatus: "failed", failureKind: "provider-stream-disconnected", lastSeq: 2 }, - updatedAt: "2026-06-20T11:03:00.000Z" - }, - session: { - sessionStatus: "failed", - messages: [ - { messageId: "msg_writer_failed_user", role: "user", text: "question", status: "sent", turnId: "trc_writer_terminal_failed", traceId: "trc_writer_terminal_failed" }, - { messageId: "msg_writer_failed_agent", role: "agent", text: "", status: "failed", turnId: "trc_writer_terminal_failed", traceId: "trc_writer_terminal_failed" } - ] - } - }); - - assert.equal(factWrites.length, 1); - const facts = factWrites[0].params.facts; - const agentMessage = facts.messages.find((message) => message.messageId === "msg_writer_failed_agent"); - const finalPart = facts.parts.find((part) => part.messageId === "msg_writer_failed_agent" && part.partType === "final_response"); - assert.equal(facts.sessions[0].status, "failed"); - assert.equal(facts.sessions[0].terminal, true); - assert.equal(facts.sessions[0].sealed, true); - assert.equal(agentMessage.status, "failed"); - assert.equal(agentMessage.terminal, true); - assert.equal(agentMessage.sealed, true); - assert.match(agentMessage.text, /provider-stream-disconnected/u); - assert.equal(finalPart?.status, "failed"); - assert.match(finalPart?.text ?? "", /provider-stream-disconnected/u); - assert.equal(facts.turns[0].status, "failed"); - assert.equal(facts.turns[0].terminal, true); - assert.equal(facts.turns[0].sealed, true); - assert.match(facts.turns[0].finalResponse.text, /provider-stream-disconnected/u); - assert.equal(facts.turns[0].diagnostic.projectionStatus, "caught-up"); - assert.equal(facts.turns[0].diagnostic.projectionHealth, "healthy"); - assert.equal(facts.turns[0].diagnostic.blocker, null); - assert.equal(facts.checkpoints[0].projectionStatus, "caught_up"); - assert.equal(facts.checkpoints[0].terminal, true); - assert.equal(facts.checkpoints[0].sealed, true); - assert.match(facts.checkpoints[0].finalResponse.text, /provider-stream-disconnected/u); -}); - -test("workbench projection writer does not synthesize terminal duration from updatedAt", async () => { - const factWrites = []; - const runtimeStore = { - async writeWorkbenchFacts(params, requestMeta) { - factWrites.push({ params, requestMeta }); - return { written: true, facts: params.facts }; - } - }; - const accessController = { - async recordAgentSessionOwner(input) { - return { - id: input.sessionId, - projectId: input.projectId, - ownerUserId: input.ownerUserId, - conversationId: input.conversationId, - threadId: input.threadId, - lastTraceId: input.traceId, - status: input.status, - session: input.session, - updatedAt: "2026-06-20T11:05:00.000Z" - }; - } - }; - - await writeWorkbenchProjectionSession({ - accessController, - runtimeStore, - traceId: "trc_writer_terminal_no_finished_at", - ownerUserId: "usr_writer", - ownerRole: "user", - sessionId: "ses_writer_terminal_no_finished_at", - projectId: "prj_writer", - conversationId: "cnv_writer_terminal_no_finished_at", - threadId: "thread-writer-terminal-no-finished-at", - status: "canceled", - payload: { - traceId: "trc_writer_terminal_no_finished_at", - status: "canceled", - startedAt: "2026-06-20T11:00:00.000Z", - updatedAt: "2026-06-20T11:05:00.000Z", - agentRun: { runId: "run_writer_terminal_no_finished_at", commandId: "cmd_writer_terminal_no_finished_at", terminalStatus: "canceled" } - }, - session: { - sessionStatus: "canceled", - messages: [ - { messageId: "msg_writer_terminal_no_finished_user", role: "user", text: "cancel me", status: "sent", turnId: "trc_writer_terminal_no_finished_at", traceId: "trc_writer_terminal_no_finished_at" }, - { messageId: "msg_writer_terminal_no_finished_agent", role: "agent", text: "hwlab-user-cancel", status: "canceled", turnId: "trc_writer_terminal_no_finished_at", traceId: "trc_writer_terminal_no_finished_at" } - ] - } - }); - - assert.equal(factWrites.length, 1); - const facts = factWrites[0].params.facts; - const agentMessage = facts.messages.find((message) => message.messageId === "msg_writer_terminal_no_finished_agent"); - assert.equal(facts.turns[0].status, "canceled"); - assert.equal(facts.turns[0].finishedAt, null); - assert.equal(facts.turns[0].durationMs, null); - assert.equal(agentMessage.finishedAt, null); - assert.equal(agentMessage.durationMs, null); - assert.equal(facts.checkpoints[0].timing.finishedAt, null); - assert.equal(facts.checkpoints[0].timing.durationMs, null); - assert.equal(facts.checkpoints[0].diagnostic.blocker.code, "workbench_terminal_timing_authority_missing"); -}); - -test("workbench projection writer keeps event writes durable when checkpoint reads are blocked", async () => { - const factWrites = []; - const baseStore = createCloudRuntimeStore({ now: () => "2026-06-20T11:30:00.000Z" }); - const blocked = new Error("Postgres durable runtime adapter is blocked for runtime evidence queries"); - blocked.code = "runtime_evidence_queries_blocked"; - const runtimeStore = { - allocateWorkbenchProjectedSeq(params, requestMeta) { - return baseStore.allocateWorkbenchProjectedSeq(params, requestMeta); - }, - async queryWorkbenchFacts() { - throw blocked; - }, - async writeWorkbenchFacts(params, requestMeta) { - const result = baseStore.writeWorkbenchFacts(params, requestMeta); - factWrites.push({ params, requestMeta }); - return result; - } - }; - - await writeWorkbenchProjectionEvent({ - runtimeStore, +test("Kafka projector builds running facts from one canonical source event", () => { + const built = buildWorkbenchProjectionEventFacts({ + projectedSeq: 4, + projectedAt: "2026-07-10T10:00:02.000Z", event: { - traceId: "trc_writer_blocked_checkpoint_event", - sessionId: "ses_writer_blocked_checkpoint_event", - sourceSeq: 1, - type: "backend", - eventType: "backend", + traceId: "trc_running", + sessionId: "ses_running", + runId: "run_running", + commandId: "cmd_running", + sourceEventId: "evt_running", + sourceSeq: 7, + type: "assistant", + eventType: "assistant", status: "running", - label: "agentrun:backend:running", - runId: "run_writer_blocked_checkpoint_event", - commandId: "cmd_writer_blocked_checkpoint_event", - startedAt: "2026-06-20T11:29:50.000Z", - createdAt: "2026-06-20T11:29:55.000Z" + text: "committed delta", + createdAt: "2026-07-10T10:00:01.000Z" } }); - assert.equal(factWrites.length, 1); - assert.equal(factWrites[0].params.facts.traceEvents[0].traceId, "trc_writer_blocked_checkpoint_event"); - assert.equal(factWrites[0].params.facts.traceEvents[0].projectedSeq, 1); - assert.equal(factWrites[0].params.facts.checkpoints[0].projectionStatus, "projecting"); + assert.equal(built.written, true); + assert.equal(built.facts.sessions[0].lastTraceId, "trc_running"); + assert.equal(built.facts.messages[0].text, "committed delta"); + assert.equal(built.facts.turns[0].terminal, false); + assert.equal(built.facts.checkpoints[0].projectedSeq, 4); + assert.equal(built.facts.traceEvents[0].sourceEventId, "evt_running"); }); -test("workbench projection writer keeps terminal session facts durable without trace backfill queries", async () => { - const factWrites = []; - let allocationSeq = 0; - let queryCount = 0; - const blocked = new Error("Postgres durable runtime adapter is blocked for runtime evidence queries"); - blocked.code = "runtime_evidence_queries_blocked"; - const runtimeStore = { - async writeWorkbenchFacts(params, requestMeta) { - factWrites.push({ params, requestMeta }); - return { written: true, facts: params.facts }; +test("authoritative assistant final seals terminal with a final response", () => { + const built = buildWorkbenchProjectionEventFacts({ + projectedSeq: 5, + projectedAt: "2026-07-10T10:00:04.000Z", + previousCheckpoint: { + traceId: "trc_terminal", + sessionId: "ses_terminal", + projectedSeq: 4, + sourceSeq: 4, + startedAt: "2026-07-10T10:00:00.000Z", + lastEventAt: "2026-07-10T10:00:03.000Z", + timing: { startedAt: "2026-07-10T10:00:00.000Z", lastEventAt: "2026-07-10T10:00:03.000Z", finishedAt: null, durationMs: null } }, - async queryWorkbenchFacts() { - queryCount += 1; - throw blocked; - }, - async allocateWorkbenchProjectedSeq() { - allocationSeq += 1; - return { projectedSeq: allocationSeq }; - } - }; - const accessController = { - async recordAgentSessionOwner(input) { - return { - id: input.sessionId, - projectId: input.projectId, - ownerUserId: input.ownerUserId, - conversationId: input.conversationId, - threadId: input.threadId, - lastTraceId: input.traceId, - status: input.status, - session: input.session, - updatedAt: "2026-06-20T11:40:00.000Z" - }; - } - }; - - const owner = await writeWorkbenchProjectionSession({ - accessController, - runtimeStore, - traceId: "trc_writer_blocked_backfill", - ownerUserId: "usr_writer", - ownerRole: "user", - sessionId: "ses_writer_blocked_backfill", - projectId: "prj_writer", - conversationId: "cnv_writer_blocked_backfill", - threadId: "thread-writer-blocked-backfill", - status: "completed", - payload: { - traceId: "trc_writer_blocked_backfill", - status: "completed", - finalResponse: { text: "BLOCKED_BACKFILL_DONE", status: "completed", traceId: "trc_writer_blocked_backfill" }, - agentRun: { runId: "run_writer_blocked_backfill", commandId: "cmd_writer_blocked_backfill", status: "completed", terminalStatus: "completed", lastSeq: 3 }, - updatedAt: "2026-06-20T11:40:00.000Z" - }, - session: { - sessionStatus: "completed", - messages: [ - { messageId: "msg_writer_blocked_backfill_user", role: "user", text: "question", status: "sent", turnId: "trc_writer_blocked_backfill", traceId: "trc_writer_blocked_backfill" }, - { messageId: "msg_writer_blocked_backfill_agent", role: "agent", text: "BLOCKED_BACKFILL_DONE", status: "completed", turnId: "trc_writer_blocked_backfill", traceId: "trc_writer_blocked_backfill" } - ], - finalResponse: { text: "BLOCKED_BACKFILL_DONE", status: "completed", traceId: "trc_writer_blocked_backfill" } - } - }); - - assert.equal(owner.id, "ses_writer_blocked_backfill"); - assert.equal(queryCount, 1); - assert.equal(factWrites.length, 1); - assert.equal(factWrites[0].params.facts.checkpoints[0].traceId, "trc_writer_blocked_backfill"); - assert.equal(factWrites[0].params.facts.turns[0].finalResponse.text, "BLOCKED_BACKFILL_DONE"); - assert.equal(factWrites[0].params.facts.traceEvents?.length ?? 0, 0); - assert.equal(allocationSeq, 0); -}); - -test("workbench projection writer does not synthesize terminal trace events from sealed facts", async () => { - const runtimeStore = createCloudRuntimeStore({ now: () => "2026-06-20T11:20:00.000Z" }); - const accessController = { - async recordAgentSessionOwner(input) { - return { - id: input.sessionId, - projectId: input.projectId, - ownerUserId: input.ownerUserId, - conversationId: input.conversationId, - threadId: input.threadId, - lastTraceId: input.traceId, - status: input.status, - session: input.session, - updatedAt: "2026-06-20T11:20:00.000Z" - }; - } - }; - - for (const [sourceSeq, label] of [ - [1, "agentrun:backend:admitted"], - [2, "agentrun:run:created"], - [3, "agentrun:runner-job:queued"] - ]) { - await writeWorkbenchProjectionEvent({ - runtimeStore, - event: { - traceId: "trc_writer_terminal_backfill", - sessionId: "ses_writer_terminal_backfill", - source: "agentrun", - sourceSeq, - type: "backend", - eventType: "backend", - status: "running", - label, - runId: "run_writer_terminal_backfill", - commandId: "cmd_writer_terminal_backfill", - createdAt: `2026-06-20T11:19:0${sourceSeq}.000Z` - } - }); - } - - await writeWorkbenchProjectionSession({ - accessController, - runtimeStore, - traceId: "trc_writer_terminal_backfill", - ownerUserId: "usr_writer", - ownerRole: "user", - sessionId: "ses_writer_terminal_backfill", - projectId: "prj_writer", - conversationId: "cnv_writer_terminal_backfill", - threadId: "thread-writer-terminal-backfill", - status: "completed", - payload: { - traceId: "trc_writer_terminal_backfill", - status: "completed", - finalResponse: { text: "BACKFILL_DONE", status: "completed", traceId: "trc_writer_terminal_backfill" }, - runnerTrace: { - traceId: "trc_writer_terminal_backfill", - status: "completed", - startedAt: "2026-06-20T11:19:01.000Z", - lastEventAt: "2026-06-20T11:19:13.000Z", - finishedAt: "2026-06-20T11:19:13.000Z", - eventCount: 3, - events: [] - }, - agentRun: { runId: "run_writer_terminal_backfill", commandId: "cmd_writer_terminal_backfill", status: "completed", terminalStatus: "completed", lastSeq: 3 }, - updatedAt: "2026-06-20T11:19:13.000Z" - }, - session: { - sessionStatus: "completed", - messages: [ - { messageId: "msg_writer_backfill_user", role: "user", text: "question", status: "sent", turnId: "trc_writer_terminal_backfill", traceId: "trc_writer_terminal_backfill" }, - { messageId: "msg_writer_backfill_agent", role: "agent", text: "BACKFILL_DONE", status: "completed", turnId: "trc_writer_terminal_backfill", traceId: "trc_writer_terminal_backfill" } - ], - finalResponse: { text: "BACKFILL_DONE", status: "completed", traceId: "trc_writer_terminal_backfill" } - } - }); - - const loaded = runtimeStore.queryWorkbenchFacts({ traceId: "trc_writer_terminal_backfill", families: ["traceEvents", "turns", "messages"], limit: 20 }); - const events = loaded.facts.traceEvents; - assert.deepEqual(events.map((event) => event.projectedSeq), [1, 2, 3]); - assert.deepEqual(events.map((event) => event.label), ["agentrun:backend:admitted", "agentrun:run:created", "agentrun:runner-job:queued"]); - assert.equal(events.some((event) => event.source === "workbench-terminal-projection"), false); - const terminalTurn = loaded.facts.turns.find((turn) => turn.terminal === true); - assert.equal(terminalTurn.finalResponse.text, "BACKFILL_DONE"); - assert.equal(loaded.facts.messages.some((message) => message.role !== "user" && message.text === "BACKFILL_DONE"), true); -}); - -test("workbench projection terminal facts do not advance checkpoint past durable trace events", async () => { - const runtimeStore = createCloudRuntimeStore({ now: () => "2026-06-20T11:45:00.000Z" }); - const traceId = "trc_writer_terminal_checkpoint_gap"; - const sessionId = "ses_writer_terminal_checkpoint_gap"; - const finalText = "CHECKPOINT_GAP_DONE"; - const accessController = { - async recordAgentSessionOwner(input) { - return { - id: input.sessionId, - projectId: input.projectId, - ownerUserId: input.ownerUserId, - conversationId: input.conversationId, - threadId: input.threadId, - lastTraceId: input.traceId, - status: input.status, - session: input.session, - updatedAt: "2026-06-20T11:45:00.000Z" - }; - } - }; - - for (let sourceSeq = 1; sourceSeq <= 33; sourceSeq += 1) { - await writeWorkbenchProjectionEvent({ - runtimeStore, - event: { - traceId, - sessionId, - source: "agentrun", - sourceSeq, - type: sourceSeq === 33 ? "terminal" : "backend", - eventType: sourceSeq === 33 ? "terminal" : "backend", - status: sourceSeq === 33 ? "completed" : "running", - label: sourceSeq === 33 ? "agentrun:result:completed" : `agentrun:backend:${sourceSeq}`, - terminal: sourceSeq === 33, - runId: "run_writer_terminal_checkpoint_gap", - commandId: "cmd_writer_terminal_checkpoint_gap", - createdAt: `2026-06-20T11:44:${String(sourceSeq).padStart(2, "0")}.000Z` - } - }); - } - - await writeWorkbenchProjectionSession({ - accessController, - runtimeStore, - traceId, - ownerUserId: "usr_writer", - ownerRole: "user", - sessionId, - projectId: "prj_writer", - conversationId: "cnv_writer_terminal_checkpoint_gap", - threadId: "thread-writer-terminal-checkpoint-gap", - status: "completed", - payload: { - traceId, - status: "completed", - finalResponse: { text: finalText, status: "completed", traceId }, - runnerTrace: { - traceId, - status: "completed", - eventCount: 34, - events: Array.from({ length: 34 }, (_, index) => ({ seq: index + 1, type: "backend", status: index === 33 ? "completed" : "running", terminal: index === 33 })) - }, - agentRun: { runId: "run_writer_terminal_checkpoint_gap", commandId: "cmd_writer_terminal_checkpoint_gap", status: "completed", terminalStatus: "completed", lastSeq: 30 }, - updatedAt: "2026-06-20T11:45:00.000Z" - }, - session: { - sessionStatus: "completed", - messages: [ - { messageId: "msg_writer_checkpoint_gap_user", role: "user", text: "question", status: "sent", turnId: traceId, traceId }, - { messageId: "msg_writer_checkpoint_gap_agent", role: "agent", text: finalText, status: "completed", turnId: traceId, traceId } - ], - finalResponse: { text: finalText, status: "completed", traceId } - } - }); - - const loaded = runtimeStore.queryWorkbenchFacts({ traceId, families: ["traceEvents", "checkpoints", "turns"], limit: 80 }); - assert.equal(loaded.facts.traceEvents.length, 33); - assert.equal(Math.max(...loaded.facts.traceEvents.map((event) => event.projectedSeq)), 33); - assert.equal(loaded.facts.checkpoints[0].projectedSeq, 33); - assert.equal(loaded.facts.checkpoints[0].sourceSeq, 30); - assert.equal(loaded.facts.turns[0].terminal, true); -}); - -test("workbench projection writer does not duplicate terminal trace events that already exist", async () => { - const runtimeStore = createCloudRuntimeStore({ now: () => "2026-06-20T11:25:00.000Z" }); - const traceId = "trc_writer_terminal_backfill_existing"; - const accessController = { - async recordAgentSessionOwner(input) { - return { - id: input.sessionId, - projectId: input.projectId, - ownerUserId: input.ownerUserId, - conversationId: input.conversationId, - threadId: input.threadId, - lastTraceId: input.traceId, - status: input.status, - session: input.session, - updatedAt: "2026-06-20T11:25:00.000Z" - }; - } - }; - - await writeWorkbenchProjectionEvent({ - runtimeStore, event: { - traceId, - sessionId: "ses_writer_terminal_backfill_existing", - sourceSeq: 1, - type: "backend", - status: "running", - label: "agentrun:backend:admitted", - createdAt: "2026-06-20T11:24:01.000Z" - } - }); - await writeWorkbenchProjectionEvent({ - runtimeStore, - event: { - traceId, - sessionId: "ses_writer_terminal_backfill_existing", - sourceSeq: 2, + traceId: "trc_terminal", + sessionId: "ses_terminal", + sourceEventId: "evt_terminal", + sourceSeq: 5, type: "assistant", eventType: "assistant", status: "completed", - label: "agentrun:assistant:message", - text: "ALREADY_DONE", terminal: true, - createdAt: "2026-06-20T11:24:12.000Z" + replyAuthority: true, + text: "durable final answer", + createdAt: "2026-07-10T10:00:04.000Z" } }); - await writeWorkbenchProjectionEvent({ - runtimeStore, + + const turn = built.facts.turns[0]; + assert.equal(turn.terminal, true); + assert.equal(turn.sealed, true); + assert.equal(turn.finalResponse.text, "durable final answer"); + assert.equal(built.facts.messages[0].terminal, true); + assert.equal(built.facts.parts[0].partType, "final_response"); + assert.equal(built.facts.checkpoints[0].projectionStatus, "caught_up"); +}); + +test("completed terminal without final response remains visibly unsealed", () => { + const built = buildWorkbenchProjectionEventFacts({ + projectedSeq: 1, event: { - traceId, - sessionId: "ses_writer_terminal_backfill_existing", - sourceSeq: 3, + traceId: "trc_missing_final", + sessionId: "ses_missing_final", + sourceEventId: "evt_missing_final", + sourceSeq: 1, type: "result", eventType: "terminal", status: "completed", - label: "agentrun:result:completed", terminal: true, - createdAt: "2026-06-20T11:24:13.000Z" + createdAt: "2026-07-10T10:00:00.000Z" } }); - await writeWorkbenchProjectionSession({ - accessController, - runtimeStore, - traceId, - ownerUserId: "usr_writer", - ownerRole: "user", - sessionId: "ses_writer_terminal_backfill_existing", - projectId: "prj_writer", - conversationId: "cnv_writer_terminal_backfill_existing", - threadId: "thread-writer-terminal-backfill-existing", - status: "completed", - payload: { - traceId, - status: "completed", - finalResponse: { text: "ALREADY_DONE", status: "completed", traceId }, - runnerTrace: { - traceId, - status: "completed", - startedAt: "2026-06-20T11:24:01.000Z", - lastEventAt: "2026-06-20T11:24:13.000Z", - finishedAt: "2026-06-20T11:24:13.000Z", - eventCount: 3, - events: [] - }, - agentRun: { runId: "run_writer_terminal_backfill_existing", commandId: "cmd_writer_terminal_backfill_existing", status: "completed", terminalStatus: "completed", lastSeq: 3 }, - updatedAt: "2026-06-20T11:24:13.000Z" - }, - session: { - sessionStatus: "completed", - messages: [ - { messageId: "msg_writer_backfill_existing_user", role: "user", text: "question", status: "sent", turnId: traceId, traceId }, - { messageId: "msg_writer_backfill_existing_agent", role: "agent", text: "ALREADY_DONE", status: "completed", turnId: traceId, traceId } - ], - finalResponse: { text: "ALREADY_DONE", status: "completed", traceId } - } - }); + const session = built.facts.sessions[0]; + const message = built.facts.messages[0]; + const turn = built.facts.turns[0]; + const traceEvent = built.facts.traceEvents[0]; + const checkpoint = built.facts.checkpoints[0]; + for (const fact of [session, message, turn, traceEvent, checkpoint]) { + assert.equal(fact.status ?? "running", "running"); + assert.equal(fact.terminal, false); + assert.equal(fact.sealed, false); + } + assert.equal(built.terminal, false); + assert.equal(built.sourceTerminal, true); + assert.equal(turn.finalResponse, null); + assert.equal(turn.diagnostic.terminalSealBlocked, true); + assert.equal(traceEvent.diagnostic.sourceStatus, "completed"); + assert.equal(traceEvent.finishedAt, null); + assert.equal(traceEvent.durationMs, null); + assert.equal(checkpoint.projectionStatus, "projecting"); + assert.equal(checkpoint.projectionHealth, "degraded"); - const loaded = runtimeStore.queryWorkbenchFacts({ traceId, families: ["traceEvents", "checkpoints"], limit: 20 }); - assert.deepEqual(loaded.facts.traceEvents.map((event) => event.projectedSeq), [1, 2, 3]); - assert.deepEqual(loaded.facts.traceEvents.map((event) => event.label), ["agentrun:backend:admitted", "agentrun:assistant:message", "agentrun:result:completed"]); - assert.equal(loaded.facts.checkpoints[0].projectedSeq, 3); + const [sse] = projectionOutboxRealtimeEvents({ + events: [{ + outboxSeq: 1, + entityFamily: "traceEvents", + entityId: traceEvent.id, + projectedSeq: 1, + projectionRevision: 1, + traceId: traceEvent.traceId, + sessionId: traceEvent.sessionId, + commitType: "event", + terminal: traceEvent.terminal, + sealed: traceEvent.sealed, + payload: { family: "traceEvents", fact: traceEvent } + }] + }); + assert.equal(sse.name, "workbench.trace.event"); + assert.equal(sse.payload.terminal, false); + assert.equal(sse.payload.sealed, false); + assert.equal(sse.payload.snapshot.status, "running"); + assert.equal(sse.payload.event.terminal, false); }); -test("workbench projection event writer aggregates assistant updates and skips late non-terminal rows after seal", async () => { - const runtimeStore = createCloudRuntimeStore({ now: () => "2026-06-20T11:32:00.000Z" }); - const traceId = "trc_writer_assistant_aggregate"; - const sessionId = "ses_writer_assistant_aggregate"; - - await writeWorkbenchProjectionEvent({ - runtimeStore, - event: { - traceId, - sessionId, - source: "agentrun", - sourceSeq: 1, - type: "backend", - eventType: "backend", - status: "running", - label: "agentrun:backend:running", - createdAt: "2026-06-20T11:31:00.000Z" - } - }); - await writeWorkbenchProjectionEvent({ - runtimeStore, - event: { - traceId, - sessionId, - source: "agentrun", - sourceSeq: 25, - type: "assistant", - eventType: "assistant", - status: "running", - label: "agentrun:assistant:message", - text: "AGGREGATED_FINAL", - createdAt: "2026-06-20T11:31:10.000Z" - } - }); - await writeWorkbenchProjectionEvent({ - runtimeStore, - event: { - traceId, - sessionId, - source: "agentrun", - sourceSeq: 26, - type: "assistant", - eventType: "assistant", - status: "completed", - label: "agentrun:assistant:message", - text: "AGGREGATED_FINAL", - finalResponse: { text: "AGGREGATED_FINAL", status: "completed", traceId }, +test("late running event cannot overwrite a sealed terminal", () => { + const built = buildWorkbenchProjectionEventFacts({ + projectedSeq: 10, + previousCheckpoint: { + traceId: "trc_sealed", + sessionId: "ses_sealed", + projectedSeq: 9, + sourceSeq: 9, terminal: true, - createdAt: "2026-06-20T11:31:12.000Z" - } - }); - await writeWorkbenchProjectionEvent({ - runtimeStore, + sealed: true, + finalResponse: { text: "sealed answer", status: "completed" } + }, event: { - traceId, - sessionId, - source: "agentrun", - sourceSeq: 24, - type: "assistant", - eventType: "assistant", - status: "running", - label: "agentrun:assistant:message", - text: "AGGREGATED_FINAL", - createdAt: "2026-06-20T11:31:09.000Z" - } - }); - - const loaded = runtimeStore.queryWorkbenchFacts({ traceId, families: ["traceEvents", "messages", "checkpoints"], limit: 20 }); - const events = loaded.facts.traceEvents; - assert.equal(events.length, 2); - assert.deepEqual(events.map((event) => event.projectedSeq), [1, 2]); - assert.equal(events[1].sourceEventId, `${traceId}:assistant-message:msg_writer_assistant_aggregate_agent`); - assert.equal(events[1].status, "completed"); - assert.equal(events[1].text, "AGGREGATED_FINAL"); - assert.equal(events.some((event) => event.sourceSeq === 24), false); - assert.equal(loaded.facts.messages.filter((message) => message.role !== "user").length, 1); - assert.equal(loaded.facts.checkpoints[0].terminal, true); -}); - -test("workbench projection writer does not rewrite prior-turn messages from merged owner session", async () => { - const factWrites = []; - const runtimeStore = { - async writeWorkbenchFacts(params, requestMeta) { - factWrites.push({ params, requestMeta }); - return { written: true, facts: params.facts }; - } - }; - const accessController = { - async recordAgentSessionOwner(input) { - return { - id: input.sessionId, - projectId: input.projectId, - ownerUserId: input.ownerUserId, - conversationId: input.conversationId, - threadId: input.threadId, - lastTraceId: input.traceId, - status: input.status, - updatedAt: "2026-06-20T11:10:00.000Z", - session: { - sessionStatus: "completed", - lastTraceId: input.traceId, - messages: [ - { messageId: "msg_writer_r1_user", role: "user", text: "round1 prompt", status: "sent", turnId: "trc_writer_r1", traceId: "trc_writer_r1" }, - { messageId: "msg_writer_r1_agent", role: "agent", text: "WRONG_ROUND2_TEXT", status: "completed", turnId: "trc_writer_r1", traceId: "trc_writer_r1" }, - ...input.session.messages - ] - } - }; - } - }; - - await writeWorkbenchProjectionSession({ - accessController, - runtimeStore, - traceId: "trc_writer_r2", - ownerUserId: "usr_writer", - ownerRole: "user", - sessionId: "ses_writer_multi_turn", - projectId: "prj_writer", - conversationId: "cnv_writer_multi_turn", - threadId: "thread-writer-multi-turn", - status: "completed", - payload: { - traceId: "trc_writer_r2", - status: "completed", - finalResponse: { text: "ROUND2_DONE", status: "completed", traceId: "trc_writer_r2" }, - agentRun: { runId: "run_writer_r2", commandId: "cmd_writer_r2", status: "completed", terminalStatus: "completed", lastSeq: 9 }, - updatedAt: "2026-06-20T11:10:00.000Z" - }, - session: { - sessionStatus: "completed", - messages: [ - { messageId: "msg_writer_r2_user", role: "user", text: "round2 prompt", status: "sent", turnId: "trc_writer_r2", traceId: "trc_writer_r2" }, - { messageId: "msg_writer_r2_agent", role: "agent", text: "ROUND2_DONE", status: "completed", turnId: "trc_writer_r2", traceId: "trc_writer_r2" } - ] - } - }); - - assert.equal(factWrites.length, 1); - const facts = factWrites[0].params.facts; - assert.deepEqual(facts.messages.map((message) => message.messageId), ["msg_writer_r2_user", "msg_writer_r2_agent"]); - assert.equal(facts.messages.some((message) => message.traceId === "trc_writer_r1"), false); - assert.equal(facts.messages.find((message) => message.messageId === "msg_writer_r2_agent").text, "ROUND2_DONE"); - assert.equal(facts.turns[0].traceId, "trc_writer_r2"); - assert.equal(facts.turns[0].finalResponse.text, "ROUND2_DONE"); -}); - -test("workbench projection writer seals canceled AgentRun turns with canonical cancel final response", async () => { - const factWrites = []; - const runtimeStore = { - async writeWorkbenchFacts(params, requestMeta) { - factWrites.push({ params, requestMeta }); - return { written: true, facts: params.facts }; - } - }; - const accessController = { - async recordAgentSessionOwner(input) { - return { - id: input.sessionId, - projectId: input.projectId, - ownerUserId: input.ownerUserId, - conversationId: input.conversationId, - threadId: input.threadId, - lastTraceId: input.traceId, - status: input.status, - session: input.session, - updatedAt: "2026-06-20T11:02:00.000Z" - }; - } - }; - - await writeWorkbenchProjectionSession({ - accessController, - runtimeStore, - traceId: "trc_writer_canceled", - ownerUserId: "usr_writer", - ownerRole: "user", - sessionId: "ses_writer_canceled", - projectId: "prj_writer", - conversationId: "cnv_writer_canceled", - threadId: "thread-writer-canceled", - status: "canceled", - payload: { - traceId: "trc_writer_canceled", - status: "canceled", - finalResponse: { text: "command cancelled before backend start", status: "canceled", traceId: "trc_writer_canceled" }, - userMessage: "当前 AgentRun 请求已取消；traceId/runId/commandId 已保留，可重试上一条消息。", - runnerTrace: { - traceId: "trc_writer_canceled", - status: "canceled", - events: [{ type: "cancel", status: "canceled", terminal: true, createdAt: "2026-06-20T11:02:00.000Z" }], - updatedAt: "2026-06-20T11:02:00.000Z" - }, - agentRun: { runId: "run_writer_canceled", commandId: "cmd_writer_canceled", status: "cancelled", commandState: "cancelled", terminalStatus: "cancelled", lastSeq: 12 }, - updatedAt: "2026-06-20T11:02:00.000Z" - }, - session: { - sessionStatus: "canceled", - messages: [ - { messageId: "msg_writer_canceled_user", role: "user", text: "cancel me", status: "sent", turnId: "trc_writer_canceled", traceId: "trc_writer_canceled" }, - { messageId: "msg_writer_canceled_agent", role: "agent", text: "", status: "canceled", turnId: "trc_writer_canceled", traceId: "trc_writer_canceled" } - ] - } - }); - - assert.equal(factWrites.length, 1); - const facts = factWrites[0].params.facts; - const agentMessage = facts.messages.find((message) => message.messageId === "msg_writer_canceled_agent"); - assert.equal(facts.sessions[0].status, "canceled"); - assert.equal(facts.turns[0].terminal, true); - assert.equal(facts.turns[0].finalResponse.text, "hwlab-user-cancel"); - assert.equal(facts.turns[0].assistantText, "hwlab-user-cancel"); - assert.equal(agentMessage.text, "hwlab-user-cancel"); - assert.equal(facts.parts.some((part) => part.messageId === "msg_writer_canceled_agent" && part.partType === "final_response" && part.text === "hwlab-user-cancel" && part.sealed === true), true); -}); - -test("workbench projection writer does not replace prior user prompt with cancel notice", async () => { - const runtimeStore = createCloudRuntimeStore({ now: () => "2026-06-20T11:04:00.000Z" }); - const traceId = "trc_writer_canceled_prompt_preserved"; - const accessController = { - async recordAgentSessionOwner(input) { - return { - id: input.sessionId, - projectId: input.projectId, - ownerUserId: input.ownerUserId, - conversationId: input.conversationId, - threadId: input.threadId, - lastTraceId: input.traceId, - status: input.status, - session: input.session, - updatedAt: input.status === "canceled" ? "2026-06-20T11:04:30.000Z" : "2026-06-20T11:04:00.000Z" - }; - } - }; - - await writeWorkbenchProjectionSession({ - accessController, - runtimeStore, - traceId, - ownerUserId: "usr_writer", - ownerRole: "user", - sessionId: "ses_writer_canceled_prompt_preserved", - projectId: "prj_writer", - conversationId: "cnv_writer_canceled_prompt_preserved", - threadId: "thread-writer-canceled-prompt-preserved", - status: "running", - payload: { - traceId, - status: "running", - prompt: "ORIGINAL CANCEL PROMPT", - updatedAt: "2026-06-20T11:04:00.000Z" - }, - session: { sessionStatus: "running", lastTraceId: traceId } - }); - await writeWorkbenchProjectionEvent({ - runtimeStore, - event: { - traceId, - sessionId: "ses_writer_canceled_prompt_preserved", - source: "agentrun", - sourceSeq: 1, + traceId: "trc_sealed", + sessionId: "ses_sealed", + sourceEventId: "evt_late", + sourceSeq: 8, type: "backend", - eventType: "backend", - status: "running", - label: "agentrun:request:accepted", - createdAt: "2026-06-20T11:04:10.000Z" - } - }); - await writeWorkbenchProjectionSession({ - accessController, - runtimeStore, - traceId, - ownerUserId: "usr_writer", - ownerRole: "user", - sessionId: "ses_writer_canceled_prompt_preserved", - projectId: "prj_writer", - conversationId: "cnv_writer_canceled_prompt_preserved", - threadId: "thread-writer-canceled-prompt-preserved", - status: "canceled", - payload: { - traceId, - status: "canceled", - message: "当前 AgentRun 请求已取消；traceId/runId/commandId 已保留，可重试上一条消息。", - finalResponse: { text: "command cancelled before backend start", status: "canceled", traceId }, - runnerTrace: { - traceId, - status: "canceled", - events: [{ type: "cancel", status: "canceled", terminal: true, createdAt: "2026-06-20T11:04:30.000Z" }], - updatedAt: "2026-06-20T11:04:30.000Z" - }, - agentRun: { runId: "run_writer_canceled_prompt_preserved", commandId: "cmd_writer_canceled_prompt_preserved", status: "cancelled", commandState: "cancelled", terminalStatus: "cancelled", lastSeq: 12 }, - updatedAt: "2026-06-20T11:04:30.000Z" - }, - session: { - sessionStatus: "canceled", - lastTraceId: traceId, - messages: [ - { messageId: "msg_writer_canceled_prompt_preserved_user", role: "user", text: "当前 AgentRun 请求已取消；traceId/runId/commandId 已保留，可重试上一条消息。", status: "sent", turnId: traceId, traceId }, - { messageId: "msg_writer_canceled_prompt_preserved_agent", role: "agent", text: "", status: "canceled", turnId: traceId, traceId } - ] + status: "running" } }); - const loaded = runtimeStore.queryWorkbenchFacts({ traceId, families: ["messages", "parts", "turns"], limit: 20 }); - const userMessages = loaded.facts.messages.filter((message) => message.role === "user"); - const agentMessage = loaded.facts.messages.find((message) => message.role !== "user"); - assert.deepEqual(userMessages.map((message) => message.text), ["ORIGINAL CANCEL PROMPT"]); - assert.equal(agentMessage.text, "hwlab-user-cancel"); - assert.equal(loaded.facts.parts.some((part) => part.messageId === "msg_writer_canceled_prompt_preserved_user" && part.text.includes("请求已取消")), false); + assert.equal(built.written, false); + assert.equal(built.suppressedAfterSeal, true); + assert.equal(built.projectedSeq, 9); + assert.equal(built.sourceEventId, "evt_late"); + assert.equal("facts" in built, false); }); -test("workbench projection writer does not seal running assistant text as final response", async () => { - const factWrites = []; - const runtimeStore = { - async writeWorkbenchFacts(params, requestMeta) { - factWrites.push({ params, requestMeta }); - return { written: true, facts: params.facts }; - } - }; - const accessController = { - async recordAgentSessionOwner(input) { - return { - id: input.sessionId, - projectId: input.projectId, - ownerUserId: input.ownerUserId, - conversationId: input.conversationId, - threadId: input.threadId, - lastTraceId: input.traceId, - status: input.status, - session: input.session, - updatedAt: "2026-06-20T11:05:00.000Z" - }; - } - }; - - await writeWorkbenchProjectionSession({ - accessController, - runtimeStore, - traceId: "trc_writer_running", - ownerUserId: "usr_writer", - ownerRole: "user", - sessionId: "ses_writer_running", - projectId: "prj_writer", - conversationId: "cnv_writer_running", - threadId: "thread-writer-running", - status: "running", - payload: { - traceId: "trc_writer_running", - status: "running", - assistantText: "backend diagnostic text should not become final", - updatedAt: "2026-06-20T11:05:00.000Z" - }, - session: { - sessionStatus: "running", - messages: [ - { messageId: "msg_writer_running_user", role: "user", text: "question", status: "sent", turnId: "trc_writer_running", traceId: "trc_writer_running" }, - { messageId: "msg_writer_running_agent", role: "agent", text: "", status: "running", turnId: "trc_writer_running", traceId: "trc_writer_running" } - ] - } - }); - - assert.equal(factWrites.length, 1); - const facts = factWrites[0].params.facts; - const agentMessage = facts.messages.find((message) => message.messageId === "msg_writer_running_agent"); - assert.equal(agentMessage.text, ""); - assert.equal(facts.parts.some((part) => part.messageId === "msg_writer_running_agent"), false); - assert.equal(facts.turns[0].terminal, false); - assert.equal(facts.turns[0].finalResponse, null); - assert.equal(facts.turns[0].assistantText, null); -}); - -test("workbench projection event commit writes trace event and checkpoint facts", async () => { - const factWrites = []; - let now = "2026-06-20T11:01:05.000Z"; - const baseStore = createCloudRuntimeStore({ now: () => now }); - const runtimeStore = { - allocateWorkbenchProjectedSeq(params, requestMeta) { - return baseStore.allocateWorkbenchProjectedSeq(params, requestMeta); - }, - queryWorkbenchFacts(params) { - return baseStore.queryWorkbenchFacts(params); - }, - async writeWorkbenchFacts(params, requestMeta) { - const result = baseStore.writeWorkbenchFacts(params, requestMeta); - factWrites.push({ params, requestMeta }); - return result; - } - }; - - now = "2026-06-20T11:03:30.000Z"; - await writeWorkbenchProjectionEvent({ - runtimeStore, - event: { - traceId: "trc_writer_event", - sessionId: "ses_writer_event", - seq: 7, - sourceSeq: 70, - type: "backend", - status: "running", - label: "agentrun:backend:running", - runId: "run_writer_event", - commandId: "cmd_writer_event", - startedAt: "2026-06-20T11:00:30.000Z", - durationMs: 3000, - createdAt: "2026-06-20T11:01:00.000Z" - } - }); - await writeWorkbenchProjectionEvent({ - runtimeStore, - event: { - traceId: "trc_writer_event", - sessionId: "ses_writer_event", - seq: 8, - sourceSeq: 71, - type: "result", - status: "completed", - label: "agentrun:result:completed", - terminal: true, - runId: "run_writer_event", - commandId: "cmd_writer_event", - createdAt: "2026-06-20T11:02:00.000Z" - } - }); - - assert.equal(factWrites.length, 2); - assert.equal(factWrites[0].params.facts.traceEvents[0].projectedSeq, 1); - assert.equal(factWrites[0].params.facts.checkpoints[0].projectionStatus, "projecting"); - assert.equal(factWrites[0].params.facts.checkpoints[0].startedAt, "2026-06-20T11:00:30.000Z"); - assert.equal(factWrites[0].params.facts.checkpoints[0].lastEventAt, "2026-06-20T11:01:00.000Z"); - assert.equal(factWrites[0].params.facts.checkpoints[0].durationMs, null); - assert.equal(factWrites[0].params.facts.checkpoints[0].timing.durationMs, null); - assert.equal(factWrites[1].params.facts.sessions[0].status, "completed"); - assert.equal(factWrites[1].params.facts.traceEvents[0].sealed, true); - assert.equal(factWrites[1].params.facts.traceEvents[0].projectedSeq, 2); - assert.equal(factWrites[1].params.facts.checkpoints[0].projectionStatus, "caught_up"); - assert.equal(factWrites[1].params.facts.checkpoints[0].durationMs, 90000); - assert.equal(factWrites[1].params.facts.checkpoints[0].finishedAt, "2026-06-20T11:02:00.000Z"); - assert.equal(factWrites[1].params.facts.checkpoints[0].lastEventAt, "2026-06-20T11:02:00.000Z"); -}); - -test("workbench projection writer reuses lifecycle assistant message id for terminal session projection", async () => { - const runtimeStore = createCloudRuntimeStore({ now: () => "2026-06-20T11:10:00.000Z" }); - const traceId = "trc_writer_terminal_same_message_id"; - const expectedAssistantMessageId = "msg_writer_terminal_same_message_id_agent"; - const accessController = { - async recordAgentSessionOwner(input) { - return { - id: input.sessionId, - projectId: input.projectId, - ownerUserId: input.ownerUserId, - ownerRole: input.ownerRole, - conversationId: input.conversationId, - threadId: input.threadId, - lastTraceId: input.traceId, - status: input.status, - session: input.session, - updatedAt: "2026-06-20T11:10:05.000Z" - }; - } - }; - - await writeWorkbenchProjectionEvent({ - runtimeStore, - event: { - traceId, - sessionId: "ses_writer_terminal_same_message_id", +test("late terminal event cannot overwrite a different sealed terminal", () => { + const built = buildWorkbenchProjectionEventFacts({ + projectedSeq: 11, + previousCheckpoint: { + traceId: "trc_sealed_terminal", + sessionId: "ses_sealed_terminal", + projectedSeq: 10, sourceSeq: 10, - type: "backend", - status: "running", - label: "agentrun:backend:running", - startedAt: "2026-06-20T11:09:55.000Z", - createdAt: "2026-06-20T11:09:55.000Z" - } - }); - await writeWorkbenchProjectionSession({ - accessController, - runtimeStore, - traceId, - ownerUserId: "usr_writer", - ownerRole: "user", - sessionId: "ses_writer_terminal_same_message_id", - projectId: "prj_writer", - conversationId: "cnv_writer_terminal_same_message_id", - threadId: "thread-writer-terminal-same-message-id", - status: "completed", - payload: { - traceId, status: "completed", - assistantText: "same lifecycle final", - finalResponse: { text: "same lifecycle final", status: "completed", traceId }, - runnerTrace: { - traceId, - status: "completed", - startedAt: "2026-06-20T11:09:55.000Z", - lastEventAt: "2026-06-20T11:10:05.000Z", - finishedAt: "2026-06-20T11:10:05.000Z", - elapsedMs: 10000 - }, - agentRun: { runId: "run_writer_terminal_same_message_id", commandId: "cmd_writer_terminal_same_message_id", status: "completed", terminalStatus: "completed", lastSeq: 11 }, - updatedAt: "2026-06-20T11:10:05.000Z" + terminal: true, + sealed: true, + finalResponse: { text: "immutable completed answer", status: "completed" } }, - session: { - sessionStatus: "completed", - lastTraceId: traceId, - messages: [ - { messageId: "msg_writer_terminal_same_message_id_user", role: "user", text: "same lifecycle", traceId, status: "sent" } - ], - finalResponse: { text: "same lifecycle final", status: "completed", traceId } - } - }); - - const loaded = runtimeStore.queryWorkbenchFacts({ traceId, families: ["messages", "parts", "turns"], limit: 20 }); - const assistantMessages = loaded.facts.messages.filter((message) => message.role !== "user"); - const terminalTurn = loaded.facts.turns.find((turn) => turn.terminal === true); - assert.deepEqual(assistantMessages.map((message) => message.messageId), [expectedAssistantMessageId]); - assert.equal(terminalTurn.messageId, expectedAssistantMessageId); - assert.equal(loaded.facts.parts.some((part) => part.messageId === expectedAssistantMessageId && part.partType === "final_response" && part.text === "same lifecycle final"), true); -}); - -test("workbench projection event writer allocates projectedSeq idempotently by source event identity", async () => { - const runtimeStore = createCloudRuntimeStore({ now: () => "2026-06-20T12:00:00.000Z" }); - - const repeatedSourceEvent = { - traceId: "trc_writer_idempotent", - sessionId: "ses_writer_idempotent", - seq: 1, - source: "agentrun", - sourceSeq: 70, - type: "backend", - status: "running", - label: "agentrun:backend:running", - runId: "run_writer_idempotent", - commandId: "cmd_writer_idempotent", - startedAt: "2026-06-20T12:00:01.000Z", - createdAt: "2026-06-20T12:00:01.000Z" - }; - await writeWorkbenchProjectionEvent({ runtimeStore, event: repeatedSourceEvent }); - await writeWorkbenchProjectionEvent({ - runtimeStore, event: { - ...repeatedSourceEvent, - seq: 1, - sourceSeq: 71, - label: "agentrun:backend:runner-ready", - startedAt: null, - durationMs: 999, - createdAt: "2026-06-20T12:00:02.000Z" + traceId: "trc_sealed_terminal", + sessionId: "ses_sealed_terminal", + sourceEventId: "evt_late_failed_terminal", + sourceSeq: 11, + type: "terminal_status", + status: "failed", + terminal: true, + errorCode: "late-failure" } }); - await writeWorkbenchProjectionEvent({ runtimeStore, event: repeatedSourceEvent }); - const loaded = runtimeStore.queryWorkbenchFacts({ traceId: "trc_writer_idempotent", families: ["traceEvents", "checkpoints"], limit: 10 }); - assert.deepEqual(loaded.facts.traceEvents.map((event) => event.projectedSeq), [1, 2]); - assert.deepEqual(loaded.facts.traceEvents.map((event) => event.sourceSeq), [70, 71]); - assert.equal(loaded.facts.checkpoints[0].projectedSeq, 2); - assert.equal(loaded.facts.checkpoints[0].startedAt, "2026-06-20T12:00:01.000Z"); - assert.equal(loaded.facts.checkpoints[0].lastEventAt, "2026-06-20T12:00:02.000Z"); - assert.equal(loaded.facts.checkpoints[0].durationMs, null); + assert.equal(built.written, false); + assert.equal(built.suppressedAfterSeal, true); + assert.equal(built.projectedSeq, 10); + assert.equal(built.sourceEventId, "evt_late_failed_terminal"); + assert.equal("facts" in built, false); +}); + +test("same canonical source event derives deterministic fact identities", () => { + const input = { + projectedSeq: 3, + projectedAt: "2026-07-10T10:00:03.000Z", + event: { + traceId: "trc_deterministic", + sessionId: "ses_deterministic", + sourceEventId: "evt_deterministic", + sourceSeq: 3, + type: "assistant", + eventType: "assistant", + status: "running", + text: "same", + createdAt: "2026-07-10T10:00:02.000Z" + } + }; + const first = buildWorkbenchProjectionEventFacts(input); + const second = buildWorkbenchProjectionEventFacts(input); + + assert.equal(first.facts.traceEvents[0].id, second.facts.traceEvents[0].id); + assert.equal(first.facts.messages[0].messageId, second.facts.messages[0].messageId); + assert.equal(first.facts.parts[0].partId, second.facts.parts[0].partId); }); diff --git a/internal/cloud/workbench-projection-writer.ts b/internal/cloud/workbench-projection-writer.ts index d5d34076..ab05e8d0 100644 --- a/internal/cloud/workbench-projection-writer.ts +++ b/internal/cloud/workbench-projection-writer.ts @@ -1,75 +1,156 @@ /* - * SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-18-p0-unique-projection; draft-2026-06-20-p1-zero-split-durable-realtime; PJ2026-010205 HWLAB接入 draft-2026-06-17-r0. - * 职责: WorkbenchProjectionWriter 组件入口。唯一封装 Code Agent/AgentRun facts 到 Workbench session projection facts 的持久化写入。 + * SPEC: PJ2026-0104010803 Workbench唯一投影; HWLAB#2464 transactional Kafka projector. + * Responsibility: atomically promote durable admission into initial nonterminal Workbench facts and build pure Kafka projection facts. + * Local admission may create the first running session/message/turn snapshot; all later lifecycle and terminal authority comes only from commitAgentRunKafkaProjection. */ import { createHash } from "node:crypto"; -import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; -import { emitCodeAgentOtelSpan } from "./otel-trace.ts"; import { safeTraceId } from "./server-http-utils.ts"; -import { createWorkbenchTurnProjection, normalizeWorkbenchStatus, projectionDiagnostics, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts"; +import { normalizeWorkbenchStatus, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts"; -export async function writeWorkbenchProjectionSession({ accessController, runtimeStore = null, traceStore = defaultCodeAgentTraceStore, traceId, ownerUserId, ownerRole = null, sessionId = null, projectId = null, conversationId = null, threadId = null, status = "active", session = {}, payload = null, params = {}, preserveLastTraceId = false } = {}) { +export async function recordWorkbenchSessionOwner({ accessController, traceId, ownerUserId, ownerRole = null, sessionId = null, projectId = null, conversationId = null, threadId = null, status = "active", session = {}, preserveLastTraceId = false } = {}) { if (!ownerUserId || typeof accessController?.recordAgentSessionOwner !== "function") return null; - const safeId = safeTraceId(traceId); - try { - const ownerRecord = await accessController.recordAgentSessionOwner({ - ownerUserId, - ownerRole, - sessionId, - projectId, - conversationId, - threadId, - traceId: preserveLastTraceId ? undefined : safeId, - status, - session - }); - const factsWrite = await writeWorkbenchProjectionFacts({ - runtimeStore, - traceStore, - traceId: safeId, - ownerUserId, - ownerRole, - sessionId: ownerRecord?.id ?? sessionId, - projectId: ownerRecord?.projectId ?? projectId, - conversationId: ownerRecord?.conversationId ?? conversationId, - threadId: ownerRecord?.threadId ?? threadId, - status: ownerRecord?.status ?? status, - session, - payload, - params - }); - if (!factsWrite) { - throw workbenchProjectionError("workbench_projection_facts_not_written", "Workbench projection facts were not durably written."); - } - return ownerRecord; - } catch (error) { - emitWorkbenchProjectionOtel("workbench.projection.session.persist", safeId, "error", { - "workbench.projection.phase": "session-owner", - "workbench.projection.has_owner_user_id": Boolean(ownerUserId), - "workbench.projection.has_session_id": Boolean(sessionId) - }, error); - if (safeId) appendProjectionDiagnostic(traceStore, safeId, { - type: "session-owner", - status: "degraded", - label: "projection-writer:session-owner:persist-failed", - errorCode: "workbench_projection_session_persist_failed", - message: error?.message ?? "Workbench projection session persistence failed.", - valuesPrinted: false - }); - throw error; - } + return accessController.recordAgentSessionOwner({ + ownerUserId, + ownerRole, + sessionId, + projectId, + conversationId, + threadId, + traceId: preserveLastTraceId ? undefined : safeTraceId(traceId), + status, + session + }); } export async function writeWorkbenchSessionAdmissionFact({ runtimeStore = null, session = null, ownerUserId = null, ownerRole = null, projectId = null, conversationId = null, threadId = null, status = "idle", now = new Date().toISOString() } = {}) { - if (typeof runtimeStore?.writeWorkbenchFacts !== "function" || !session || typeof session !== "object") return null; + if (typeof runtimeStore?.writeWorkbenchSessionAdmissionFact !== "function") { + throw workbenchProjectionError("workbench_admission_store_unconfigured", "Workbench admission requires the durable session admission store."); + } + const fact = buildWorkbenchSessionAdmissionFact({ session, ownerUserId, ownerRole, projectId, conversationId, threadId, status, now }); + return runtimeStore.writeWorkbenchSessionAdmissionFact({ fact }, { + sessionId: fact.sessionId, + ownerUserId: fact.ownerUserId, + projectId: fact.projectId, + valuesPrinted: false + }); +} + +export async function promoteWorkbenchTurnAdmission({ runtimeStore = null, session = null, inputFact = null, payload = {}, params = {}, ownerUserId = null, ownerRole = null, projectId = null, conversationId = null, threadId = null, now = new Date().toISOString() } = {}) { + if (typeof runtimeStore?.promoteWorkbenchTurnAdmission !== "function") { + throw workbenchProjectionError("workbench_admission_promotion_store_unconfigured", "Workbench admission promotion requires the durable atomic promotion store."); + } + if (!inputFact || typeof inputFact !== "object") { + throw workbenchProjectionError("workbench_admission_input_required", "Workbench admission promotion requires an input fact."); + } + const facts = buildWorkbenchAdmissionPromotionFacts({ session, inputFact, payload, params, ownerUserId, ownerRole, projectId, conversationId, threadId, now }); + const sessionFact = facts.sessions[0]; + return runtimeStore.promoteWorkbenchTurnAdmission({ facts }, { + sessionId: sessionFact.sessionId, + traceId: inputFact.traceId ?? null, + turnId: inputFact.turnId ?? null, + messageId: inputFact.messageId ?? null, + ownerUserId: sessionFact.ownerUserId, + projectId: sessionFact.projectId, + valuesPrinted: false + }); +} + +export function buildWorkbenchAdmissionPromotionFacts({ session = null, inputFact = null, payload = {}, params = {}, ownerUserId = null, ownerRole = null, projectId = null, conversationId = null, threadId = null, now = new Date().toISOString() } = {}) { + const sessionFact = buildWorkbenchSessionAdmissionFact({ session, ownerUserId, ownerRole, projectId, conversationId, threadId, status: "running", now }); + const traceId = safeTraceId(payload.traceId ?? params.traceId ?? inputFact?.traceId); + if (!traceId) throw workbenchProjectionError("workbench_admission_trace_required", "Workbench admission promotion requires traceId."); + const turnId = textValue(payload.turnId ?? params.turnId ?? inputFact?.turnId) || traceId; + const userMessageId = textValue(payload.userMessageId ?? params.userMessageId ?? inputFact?.messageId) || eventProjectionUserMessageId(traceId, {}); + const assistantMessageId = textValue(payload.assistantMessageId ?? params.assistantMessageId) || eventProjectionAssistantMessageId(traceId, {}); + const sourceEventId = `${traceId}:admission-promotion`; + const occurredAt = timestampValue(payload.createdAt ?? now); + const timing = eventTimingProjection({ startedAt: occurredAt, lastEventAt: occurredAt, terminal: false }); + const userText = textValue(params.message ?? params.prompt ?? params.text); + const common = { + sessionId: sessionFact.sessionId, + turnId, + traceId, + projectedSeq: 0, + sourceSeq: 0, + sourceEventId, + terminal: false, + sealed: false, + timing, + startedAt: timing.startedAt, + lastEventAt: timing.lastEventAt, + finishedAt: null, + durationMs: null, + createdAt: occurredAt, + updatedAt: occurredAt, + valuesPrinted: false + }; + const userMessage = { ...common, messageId: userMessageId, role: "user", status: "sent", text: userText }; + const assistantMessage = { ...common, messageId: assistantMessageId, role: "agent", status: "running", text: "" }; + const turn = { + ...common, + messageId: assistantMessageId, + status: "running", + finalResponse: null, + assistantText: "", + failureKind: null, + diagnostic: { + authority: "local-admission-promotion", + admissionState: "promoted", + sourceRunId: textValue(payload.agentRun?.runId) || null, + sourceCommandId: textValue(payload.agentRun?.commandId) || null, + dispatchIntentId: textValue(payload.agentRun?.dispatchIntentId) || null, + runnerJobId: textValue(payload.agentRun?.runnerJobId) || null, + valuesRedacted: true + } + }; + return { + inputs: [{ + ...inputFact, + status: "promoted", + commandId: textValue(payload.agentRun?.commandId ?? inputFact.commandId) || null, + runId: textValue(payload.agentRun?.runId) || null, + dispatchIntentId: textValue(payload.agentRun?.dispatchIntentId) || null, + runnerJobId: textValue(payload.agentRun?.runnerJobId) || null, + durableDispatch: payload.agentRun?.durableDispatch === true, + updatedAt: occurredAt, + valuesRedacted: true, + secretMaterialStored: false + }], + sessions: [{ + ...sessionFact, + status: "running", + lastTraceId: traceId, + sessionJson: { + ...sessionFact.sessionJson, + admissionState: "promoted", + agentRun: payload.agentRun ?? sessionFact.sessionJson?.agentRun ?? null, + valuesRedacted: true, + secretMaterialStored: false + }, + updatedAt: occurredAt + }], + messages: [userMessage, assistantMessage], + parts: userText ? messagePartFacts(userMessage) : [], + turns: [turn], + traceEvents: [], + checkpoints: [] + }; +} + +export function buildWorkbenchSessionAdmissionFact({ session = null, ownerUserId = null, ownerRole = null, projectId = null, conversationId = null, threadId = null, status = "idle", now = new Date().toISOString() } = {}) { + if (!session || typeof session !== "object") { + throw workbenchProjectionError("workbench_admission_session_required", "Workbench admission requires a session record."); + } const sessionId = textValue(session.id ?? session.sessionId); - if (!sessionId) return null; + if (!sessionId) { + throw workbenchProjectionError("workbench_admission_session_required", "Workbench admission requires sessionId."); + } const createdAt = timestampValue(session.startedAt ?? session.createdAt ?? session.session?.createdAt ?? now); const updatedAt = timestampValue(session.updatedAt ?? createdAt); const normalizedStatus = normalizeWorkbenchStatus(session.status ?? session.session?.sessionStatus ?? status) || "idle"; const sessionJson = workbenchSessionJsonWithLaunchContext(plainObjectValue(session.session) ?? {}, { payload: session }); - const fact = { + return { sessionId, ownerUserId: textValue(session.ownerUserId ?? ownerUserId) || null, ownerRole: textValue(session.ownerRole ?? ownerRole) || null, @@ -87,7 +168,7 @@ export async function writeWorkbenchSessionAdmissionFact({ runtimeStore = null, sessionJson: { ...sessionJson, sessionStatus: normalizedStatus, - source: textValue(sessionJson.source ?? session.session?.source) || "manual-session-create", + source: textValue(sessionJson.source ?? session.session?.source) || "session-admission", providerProfile: textValue(sessionJson.providerProfile ?? session.session?.providerProfile) || null, valuesRedacted: true, secretMaterialStored: false @@ -96,103 +177,71 @@ export async function writeWorkbenchSessionAdmissionFact({ runtimeStore = null, updatedAt, valuesRedacted: true }; - return runtimeStore.writeWorkbenchFacts({ facts: { sessions: [fact] } }, { - sessionId, - ownerUserId: fact.ownerUserId, - projectId: fact.projectId, - valuesPrinted: false - }); } -export async function writeWorkbenchProjectionEvent({ runtimeStore = null, event = {}, requestMeta = {}, previousCheckpoint = null } = {}) { - if (typeof runtimeStore?.writeWorkbenchFacts !== "function") return null; - if (typeof runtimeStore?.allocateWorkbenchProjectedSeq !== "function") throw new Error("Workbench projection event writer requires a durable projectedSeq allocator."); +export function buildWorkbenchProjectionEventFacts({ event = {}, requestMeta = {}, previousCheckpoint = null, projectedSeq, projectedAt = new Date().toISOString() } = {}) { const traceId = safeTraceId(event.traceId ?? requestMeta.traceId); - if (!traceId) return null; - const projectedAt = new Date().toISOString(); + if (!traceId) throw workbenchProjectionError("workbench_projection_trace_required", "Kafka projection event requires traceId."); + const durableProjectedSeq = nonNegativeInteger(projectedSeq); + if (durableProjectedSeq <= 0) throw workbenchProjectionError("workbench_projection_seq_required", "Kafka projection event requires an allocated projectedSeq."); const sourceSeq = nonNegativeInteger(event.sourceSeq ?? event.seq); const sessionId = textValue(event.sessionId ?? requestMeta.sessionId) || null; const turnId = textValue(event.turnId) || traceId; const eventType = textValue(event.eventType ?? event.type ?? event.label) || "event"; - const terminal = event.terminal === true || TERMINAL_STATUSES.has(normalizeWorkbenchStatus(event.status)); - const status = terminal ? normalizeWorkbenchStatus(event.status) : "running"; - const checkpointHint = previousCheckpoint && typeof previousCheckpoint === "object" ? previousCheckpoint : null; - const resolvedPreviousCheckpoint = checkpointHint ?? await latestWorkbenchCheckpoint(runtimeStore, traceId, { traceStore: defaultCodeAgentTraceStore, phase: "event-checkpoint" }); - const previousTiming = normalizeTimingProjection(resolvedPreviousCheckpoint?.timing) ?? normalizeTimingProjection(resolvedPreviousCheckpoint); + const sourceTerminal = event.terminal === true || TERMINAL_STATUSES.has(normalizeWorkbenchStatus(event.status)); + const sourceStatus = sourceTerminal ? normalizeWorkbenchStatus(event.status) : "running"; + const previousTiming = normalizeTimingProjection(previousCheckpoint?.timing) ?? normalizeTimingProjection(previousCheckpoint); const sourceOccurredAt = timestampValue(event.createdAt ?? event.occurredAt ?? event.updatedAt ?? projectedAt); const occurredAt = latestTimestamp(previousTiming?.lastEventAt, sourceOccurredAt) ?? sourceOccurredAt; - const previousTerminal = checkpointIsTerminal(resolvedPreviousCheckpoint); - const suppressedAfterSeal = previousTerminal && !terminal; - if (suppressedAfterSeal) { - emitWorkbenchProjectionOtel("workbench.projection.event.write", traceId, "ok", { - "workbench.projection.phase": "event-suppressed-after-seal", - "workbench.projection.terminal": false, - "workbench.projection.suppressed_after_seal": true, - "workbench.projection.session_id": sessionId, - "workbench.projection.turn_id": turnId, - "workbench.projection.source_seq": sourceSeq, - "workbench.projection.event_type": eventType - }); - return { written: false, suppressedAfterSeal: true, valuesPrinted: false }; + if (checkpointIsTerminal(previousCheckpoint)) { + return { + written: false, + suppressedAfterSeal: true, + traceId, + sessionId, + sourceSeq, + sourceEventId: workbenchSourceEventId(event, { traceId, sourceSeq, occurredAt }), + projectedSeq: nonNegativeInteger(previousCheckpoint?.projectedSeq) || durableProjectedSeq, + valuesPrinted: false + }; } const explicitEventTiming = normalizeTimingProjection(event.timing) ?? normalizeTimingProjection(event); const explicitDurationMs = durationValue(event.durationMs ?? explicitEventTiming?.durationMs); const explicitStartedAt = optionalTimestampValue(event.startedAt ?? event.traceStartedAt ?? event.runnerStartedAt ?? explicitEventTiming?.startedAt); - // Once a trace has a durable checkpoint, startedAt is sealed as the card - // elapsed-time source. Later trace/result hydration can carry older or newer - // startedAt values, but it must not make the visible Code Agent card jump by - // switching to a different clock origin. const provisionalLastEventAt = latestTimestamp(previousTiming?.lastEventAt, explicitEventTiming?.lastEventAt, occurredAt); - const provisionalFinishedAt = terminal ? latestTimestamp(explicitEventTiming?.finishedAt, optionalTimestampValue(event.finishedAt), provisionalLastEventAt, occurredAt) : null; + const provisionalFinishedAt = sourceTerminal ? latestTimestamp(explicitEventTiming?.finishedAt, optionalTimestampValue(event.finishedAt), provisionalLastEventAt, occurredAt) : null; const startedAt = previousTiming?.startedAt ?? explicitStartedAt ?? startedAtFromDuration(provisionalFinishedAt, explicitDurationMs) ?? (sourceSeq <= 1 ? occurredAt : null); const lastEventAt = provisionalLastEventAt; - const finishedAt = terminal ? latestTimestamp(provisionalFinishedAt, lastEventAt) : null; - const timing = eventTimingProjection({ startedAt, lastEventAt, finishedAt, terminal, durationMs: explicitDurationMs ?? previousTiming?.durationMs }); - const timingAuthorityIssue = terminalTimingAuthorityIssue(timing, { terminal, traceId, source: "event", status, label: event.label, sourceSeq }); - if (timingAuthorityIssue) emitTerminalTimingAuthorityDiagnostic(defaultCodeAgentTraceStore, traceId, timingAuthorityIssue, { - "workbench.projection.phase": "event-timing", - "workbench.projection.session_id": sessionId, - "workbench.projection.turn_id": turnId, - "workbench.projection.source_seq": sourceSeq - }); - const eventTiming = suppressedAfterSeal ? previousTiming ?? timing : timing; + const finishedAt = sourceTerminal ? latestTimestamp(provisionalFinishedAt, lastEventAt) : null; const sourceEventId = workbenchSourceEventId(event, { traceId, sourceSeq, occurredAt }); - const allocation = await runtimeStore.allocateWorkbenchProjectedSeq({ - traceId, - sessionId, - turnId, - runId: textValue(event.runId) || null, - commandId: textValue(event.commandId) || null, - sourceSeq, - sourceEventId, - eventType, - occurredAt, - updatedAt: projectedAt - }, { - traceId, - sessionId, - agentSessionId: requestMeta.agentSessionId ?? sessionId, - valuesPrinted: false - }); - const projectedSeq = nonNegativeInteger(allocation?.projectedSeq); - if (projectedSeq <= 0) throw new Error("Workbench projection allocator returned an invalid projectedSeq."); - const eventId = isAssistantMessageTraceEvent(event) ? stableFactId("wte", { traceId, sourceEventId }) : textValue(event.id) || stableFactId("wte", { traceId, sourceEventId }); - const previousFinalText = finalResponseTextValue(resolvedPreviousCheckpoint?.finalResponse, resolvedPreviousCheckpoint?.assistantText, resolvedPreviousCheckpoint?.finalText); - const eventTerminalFinalResponse = terminal ? terminalFinalResponse(status, event, { evidence: event, finalResponse: event.finalResponse }) : null; - const messageFinalText = terminal ? finalResponseTextValue(event.finalResponse, event.assistantText, event.reply, eventTerminalFinalResponse, previousFinalText) : null; - const userMessageFact = sessionId && !suppressedAfterSeal && resolvedPreviousCheckpoint?.userMessage ? normalizeMessageFact(resolvedPreviousCheckpoint.userMessage, 0, { traceId, sessionId, turnId, terminal: false, terminalStatus: "sent", finalText: null, timestamp: projectedAt, timing }) : null; - const messageFact = sessionId && !suppressedAfterSeal ? normalizeMessageFact({ + const eventId = textValue(event.id) || stableFactId("wte", { traceId, sourceEventId }); + const previousFinalText = finalResponseTextValue(previousCheckpoint?.finalResponse, previousCheckpoint?.assistantText, previousCheckpoint?.finalText); + const liveAssistantText = isAssistantMessageTraceEvent(event) ? finalResponseTextValue(event.assistantText, event.text, event.reply, event.message) : finalResponseTextValue(event.assistantText, event.reply); + const eventTerminalFinalResponse = sourceTerminal ? terminalFinalResponse(sourceStatus, event, { evidence: event, finalResponse: event.finalResponse }) : null; + const messageFinalText = sourceTerminal ? finalResponseTextValue(event.finalResponse, liveAssistantText, previousFinalText, eventTerminalFinalResponse) : null; + const checkpointAssistantText = messageFinalText ?? liveAssistantText ?? previousFinalText ?? null; + const checkpointFinalResponse = sourceTerminal && messageFinalText + ? { ...(eventTerminalFinalResponse ?? {}), text: messageFinalText, status: sourceStatus, traceId, source: "agentrun-kafka-projector", valuesPrinted: false } + : previousCheckpoint?.finalResponse ?? null; + const terminalSealBlocked = sourceTerminal && sourceStatus === "completed" && !messageFinalText; + const effectiveTerminal = sourceTerminal && !terminalSealBlocked; + const effectiveStatus = effectiveTerminal ? sourceStatus : "running"; + const timing = eventTimingProjection({ startedAt, lastEventAt, finishedAt, terminal: effectiveTerminal, durationMs: explicitDurationMs ?? previousTiming?.durationMs }); + const timingAuthorityIssue = terminalTimingAuthorityIssue(timing, { terminal: effectiveTerminal, traceId, source: "kafka", status: effectiveStatus, label: event.label, sourceSeq }); + const projectionDiagnostic = { timingAuthorityIssue, terminalSealBlocked, sourceTerminal, sourceStatus, authority: "agentrun-kafka-projector", valuesRedacted: true }; + const userMessageFact = sessionId && previousCheckpoint?.userMessage ? normalizeMessageFact(previousCheckpoint.userMessage, 0, { traceId, sessionId, turnId, terminal: false, terminalStatus: "sent", finalText: null, timestamp: projectedAt, timing }) : null; + const messageFact = sessionId ? normalizeMessageFact({ messageId: eventProjectionAssistantMessageId(traceId, event), role: "agent", - status, + status: effectiveStatus, traceId, turnId, - text: terminal && messageFinalText ? messageFinalText : "", - projectedSeq, + text: messageFinalText ?? liveAssistantText ?? previousFinalText ?? "", + projectedSeq: durableProjectedSeq, sourceSeq, sourceEventId, - terminal, - sealed: terminal, + terminal: effectiveTerminal, + sealed: effectiveTerminal, timing, startedAt: timing.startedAt, lastEventAt: timing.lastEventAt, @@ -200,126 +249,61 @@ export async function writeWorkbenchProjectionEvent({ runtimeStore = null, event durationMs: timing.durationMs, createdAt: occurredAt, updatedAt: projectedAt, - source: "workbench-event-projection" - }, userMessageFact ? 1 : 0, { traceId, sessionId, turnId, terminal, terminalStatus: status, finalText: messageFinalText, timestamp: projectedAt, timing }) : null; + source: "agentrun-kafka-projector" + }, userMessageFact ? 1 : 0, { traceId, sessionId, turnId, terminal: effectiveTerminal, terminalStatus: effectiveStatus, finalText: messageFinalText, terminalSealBlocked, timestamp: projectedAt, timing }) : null; const facts = { - sessions: sessionId && !suppressedAfterSeal ? [{ + sessions: sessionId ? [{ sessionId, status: effectiveStatus, lastTraceId: traceId, projectedSeq: durableProjectedSeq, sourceSeq, sourceEventId, terminal: effectiveTerminal, sealed: effectiveTerminal, timing, startedAt: timing.startedAt, lastEventAt: timing.lastEventAt, finishedAt: timing.finishedAt, durationMs: timing.durationMs, updatedAt: projectedAt }] : [], + traceEvents: [{ ...event, id: eventId, traceId, sessionId, turnId, messageId: textValue(event.messageId) || null, status: effectiveStatus, sourceSeq, sourceEventId, projectedSeq: durableProjectedSeq, eventType, terminal: effectiveTerminal, sealed: effectiveTerminal, diagnostic: projectionDiagnostic, timing, startedAt: timing.startedAt, lastEventAt: timing.lastEventAt, finishedAt: timing.finishedAt, durationMs: timing.durationMs, createdAt: occurredAt, occurredAt, updatedAt: projectedAt }], + messages: [userMessageFact, messageFact].filter(Boolean), + parts: [userMessageFact, messageFact].filter(Boolean).flatMap((message) => messagePartFacts(message, { finalText: message === messageFact ? messageFinalText : null })), + turns: sessionId ? [{ + turnId, sessionId, - status, - lastTraceId: traceId, - projectedSeq, + traceId, + messageId: messageFact?.messageId ?? null, + status: effectiveStatus, + projectedSeq: durableProjectedSeq, sourceSeq, sourceEventId, - terminal, - sealed: terminal, + terminal: effectiveTerminal, + sealed: effectiveTerminal, + finalResponse: checkpointFinalResponse, + assistantText: checkpointAssistantText, + failureKind: textValue(event.failureKind ?? event.errorCode) || null, + diagnostic: projectionDiagnostic, timing, startedAt: timing.startedAt, lastEventAt: timing.lastEventAt, finishedAt: timing.finishedAt, durationMs: timing.durationMs, + createdAt: occurredAt, updatedAt: projectedAt }] : [], - traceEvents: [{ - ...event, - id: eventId, - traceId, - sessionId, - turnId, - messageId: textValue(event.messageId) || null, - sourceSeq, - sourceEventId, - projectedSeq, - eventType, - terminal, - sealed: terminal, - timing: eventTiming, - startedAt: eventTiming?.startedAt ?? null, - lastEventAt: eventTiming?.lastEventAt ?? null, - finishedAt: eventTiming?.finishedAt ?? null, - durationMs: eventTiming?.durationMs ?? null, - suppressedAfterSeal, - sourceOccurredAt: sourceOccurredAt !== occurredAt ? sourceOccurredAt : null, - createdAt: occurredAt, - occurredAt, - updatedAt: projectedAt - }], - messages: [userMessageFact, messageFact].filter(Boolean), - parts: [userMessageFact, messageFact].filter(Boolean).flatMap((message) => messagePartFacts(message, { finalText: message === messageFact ? messageFinalText : null })), - checkpoints: !suppressedAfterSeal ? [{ + checkpoints: [{ traceId, sessionId, turnId, runId: textValue(event.runId) || null, commandId: textValue(event.commandId) || null, - projectedSeq, + projectedSeq: durableProjectedSeq, sourceSeq, sourceEventId, - projectionStatus: terminal ? "caught_up" : "projecting", - projectionHealth: "healthy", - terminal, - sealed: terminal, + projectionStatus: effectiveTerminal ? "caught_up" : "projecting", + projectionHealth: terminalSealBlocked ? "degraded" : "healthy", + terminal: effectiveTerminal, + sealed: effectiveTerminal, + assistantText: checkpointAssistantText, + finalResponse: checkpointFinalResponse, timing, startedAt: timing.startedAt, lastEventAt: timing.lastEventAt, finishedAt: timing.finishedAt, durationMs: timing.durationMs, - diagnostic: { - lastEventLabel: textValue(event.label) || null, - lastEventStatus: textValue(event.status) || null, - timingAuthorityIssue, - valuesRedacted: true - }, + diagnostic: { ...projectionDiagnostic, lastEventLabel: textValue(event.label) || null, lastEventStatus: textValue(event.status) || null }, updatedAt: projectedAt - }] : [] + }] }; - try { - const result = await runtimeStore.writeWorkbenchFacts({ facts }, { - traceId, - sessionId, - agentSessionId: requestMeta.agentSessionId ?? sessionId, - valuesPrinted: false - }); - emitWorkbenchProjectionOtel("workbench.projection.event.write", traceId, "ok", { - ...workbenchProjectionFactCountAttributes(facts), - "workbench.projection.phase": terminal ? "event-terminal" : "event-running", - "workbench.projection.terminal": terminal, - "workbench.projection.suppressed_after_seal": suppressedAfterSeal, - "workbench.projection.session_id": sessionId, - "workbench.projection.turn_id": turnId, - "workbench.projection.projected_seq": projectedSeq, - "workbench.projection.source_seq": sourceSeq, - "workbench.projection.source_event_id": sourceEventId, - "workbench.projection.event_type": eventType, - "workbench.projection.event_id": eventId, - "workbench.projection.message_id": messageFact?.messageId ?? null - }); - return result; - } catch (error) { - emitWorkbenchProjectionOtel("workbench.projection.event.write", traceId, "error", { - ...workbenchProjectionFactCountAttributes(facts), - "workbench.projection.phase": terminal ? "event-terminal" : "event-running", - "workbench.projection.terminal": terminal, - "workbench.projection.suppressed_after_seal": suppressedAfterSeal, - "workbench.projection.session_id": sessionId, - "workbench.projection.turn_id": turnId, - "workbench.projection.projected_seq": projectedSeq, - "workbench.projection.source_seq": sourceSeq, - "workbench.projection.source_event_id": sourceEventId, - "workbench.projection.event_type": eventType, - "workbench.projection.event_id": eventId, - "workbench.projection.message_id": messageFact?.messageId ?? null - }, error); - appendProjectionDiagnostic(defaultCodeAgentTraceStore, traceId, { - type: "facts", - status: "degraded", - label: "projection-writer:event-facts:persist-failed", - errorCode: error?.code ?? "workbench_event_facts_persist_failed", - message: error?.message ?? "Workbench event facts persistence failed.", - terminal: false, - valuesPrinted: false - }); - throw error; - } + return { written: true, suppressedAfterSeal: false, traceId, sessionId, sourceSeq, sourceEventId, projectedSeq: durableProjectedSeq, facts, terminal: effectiveTerminal, sourceTerminal, terminalSealBlocked, valuesPrinted: false }; } function eventProjectionAssistantMessageId(traceId, event = {}) { @@ -342,293 +326,6 @@ function eventProjectionMessageId(traceId, role, event = {}) { return `msg_${traceSuffix}_${role === "user" ? "user" : "agent"}`; } -async function writeWorkbenchProjectionFacts({ runtimeStore = null, traceStore = defaultCodeAgentTraceStore, traceId = null, ownerUserId = null, ownerRole = null, sessionId = null, projectId = null, conversationId = null, threadId = null, status = "active", session = {}, payload = null, params = {} } = {}) { - if (typeof runtimeStore?.writeWorkbenchFacts !== "function") return null; - const safeId = safeTraceId(traceId ?? session?.traceId ?? payload?.traceId ?? params?.traceId); - const previousCheckpoint = safeId ? await latestWorkbenchCheckpoint(runtimeStore, safeId, { traceStore, phase: "facts-checkpoint" }) : null; - const facts = buildWorkbenchProjectionFacts({ traceId, ownerUserId, ownerRole, sessionId, projectId, conversationId, threadId, status, session, payload, params, previousCheckpoint }); - if (isEmptyFacts(facts)) return null; - const timingAuthorityIssue = facts.checkpoints?.[0]?.diagnostic?.timingAuthorityIssue ?? null; - if (timingAuthorityIssue) emitTerminalTimingAuthorityDiagnostic(traceStore, safeId, timingAuthorityIssue, { - "workbench.projection.phase": "facts-timing", - "workbench.projection.session_id": facts.sessions[0]?.sessionId ?? sessionId ?? null, - "workbench.projection.turn_id": facts.turns[0]?.turnId ?? null - }); - try { - const result = await runtimeStore.writeWorkbenchFacts({ facts }, { - traceId: facts.checkpoints[0]?.traceId ?? traceId, - sessionId: facts.sessions[0]?.sessionId ?? sessionId, - ownerUserId, - projectId, - valuesPrinted: false - }); - emitWorkbenchProjectionOtel("workbench.projection.facts.write", safeId, "ok", { - ...workbenchProjectionFactCountAttributes(facts), - "workbench.projection.terminal_trace_backfill_written": 0, - "workbench.projection.phase": "session-facts", - "workbench.projection.terminal": facts.turns.some((turn) => turn.terminal === true), - "workbench.projection.session_id": facts.sessions[0]?.sessionId ?? sessionId ?? null, - "workbench.projection.turn_id": facts.turns[0]?.turnId ?? null, - "workbench.projection.projected_seq": facts.checkpoints[0]?.projectedSeq ?? facts.turns[0]?.projectedSeq ?? facts.messages[0]?.projectedSeq ?? null, - "workbench.projection.source_seq": facts.checkpoints[0]?.sourceSeq ?? facts.turns[0]?.sourceSeq ?? facts.messages[0]?.sourceSeq ?? null, - "workbench.projection.source_event_id": facts.checkpoints[0]?.sourceEventId ?? facts.turns[0]?.sourceEventId ?? facts.messages[0]?.sourceEventId ?? null, - "workbench.projection.message_id": facts.messages[0]?.messageId ?? null - }); - return result; - } catch (error) { - const safeId = safeTraceId(traceId); - emitWorkbenchProjectionOtel("workbench.projection.facts.write", safeId, "error", { - ...workbenchProjectionFactCountAttributes(facts), - "workbench.projection.phase": "session-facts", - "workbench.projection.terminal": facts.turns.some((turn) => turn.terminal === true), - "workbench.projection.session_id": facts.sessions[0]?.sessionId ?? sessionId ?? null, - "workbench.projection.turn_id": facts.turns[0]?.turnId ?? null, - "workbench.projection.projected_seq": facts.checkpoints[0]?.projectedSeq ?? facts.turns[0]?.projectedSeq ?? facts.messages[0]?.projectedSeq ?? null, - "workbench.projection.source_seq": facts.checkpoints[0]?.sourceSeq ?? facts.turns[0]?.sourceSeq ?? facts.messages[0]?.sourceSeq ?? null, - "workbench.projection.source_event_id": facts.checkpoints[0]?.sourceEventId ?? facts.turns[0]?.sourceEventId ?? facts.messages[0]?.sourceEventId ?? null, - "workbench.projection.message_id": facts.messages[0]?.messageId ?? null - }, error); - if (safeId) appendProjectionDiagnostic(traceStore, safeId, { - type: "facts", - status: "degraded", - label: "projection-writer:facts:persist-failed", - errorCode: error?.code ?? "workbench_facts_persist_failed", - message: error?.message ?? "Workbench facts persistence failed.", - terminal: false - }); - throw error; - } -} - -async function backfillTerminalTraceEvents({ runtimeStore = null, traceStore = defaultCodeAgentTraceStore, traceId = null, facts = {}, payload = null, status = null } = {}) { - if (typeof runtimeStore?.writeWorkbenchFacts !== "function" || typeof runtimeStore?.allocateWorkbenchProjectedSeq !== "function") return null; - const safeId = safeTraceId(traceId ?? facts?.checkpoints?.[0]?.traceId ?? facts?.turns?.[0]?.traceId); - const checkpoint = facts?.checkpoints?.[0] ?? null; - const turn = facts?.turns?.[0] ?? null; - if (!safeId || checkpoint?.terminal !== true || turn?.terminal !== true) return null; - const terminalStatus = normalizeWorkbenchStatus(turn?.status ?? checkpoint?.status ?? status); - if (terminalStatus !== "completed") return null; - const finalText = finalResponseTextValue(turn?.finalResponse, checkpoint?.finalResponse, turn?.assistantText, checkpoint?.assistantText, payload?.finalResponse, payload?.assistantText); - if (!finalText) return null; - const sessionId = textValue(checkpoint?.sessionId ?? facts?.sessions?.[0]?.sessionId) || null; - const turnId = textValue(turn?.turnId ?? checkpoint?.turnId) || safeId; - const runId = textValue(checkpoint?.runId ?? payload?.agentRun?.runId) || null; - const commandId = textValue(checkpoint?.commandId ?? payload?.agentRun?.commandId) || null; - const assistantMessageId = textValue(turn?.messageId ?? facts?.messages?.find((message) => message.role !== "user" && message.traceId === safeId)?.messageId) || null; - try { - const existingEvents = await existingWorkbenchTraceEvents(runtimeStore, safeId, { traceStore }); - if (existingEvents === null) return { written: 0, skipped: true, errorCode: "workbench_terminal_trace_backfill_probe_failed", valuesPrinted: false }; - const writeAssistantFinal = !hasTerminalAssistantFinalTraceEvent(existingEvents, finalText); - const writeCompletion = !hasTerminalResultTraceEvent(existingEvents); - if (!writeAssistantFinal && !writeCompletion) return { written: 0, idempotent: true, skippedExisting: true, valuesPrinted: false }; - const timestamp = latestTimestamp(turn?.finishedAt, checkpoint?.finishedAt, turn?.lastEventAt, checkpoint?.lastEventAt, turn?.updatedAt, checkpoint?.updatedAt, payload?.updatedAt) ?? new Date().toISOString(); - const baseSourceSeq = Math.max(nonNegativeInteger(checkpoint?.sourceSeq), nonNegativeInteger(turn?.sourceSeq)); - let offset = 1; - let written = 0; - if (writeAssistantFinal) { - await writeWorkbenchProjectionEvent({ - runtimeStore, - previousCheckpoint: checkpoint, - event: { - traceId: safeId, - sessionId, - turnId, - messageId: assistantMessageId, - source: "workbench-terminal-projection", - sourceSeq: baseSourceSeq + offset, - sourceEventId: `${safeId}:workbench-terminal-projection:assistant-final`, - type: "assistant", - eventType: "assistant", - status: "completed", - label: "agentrun:assistant:message", - message: finalText, - text: finalText, - finalResponse: { text: finalText, status: "completed", traceId: safeId, valuesPrinted: false }, - final: true, - replyAuthority: true, - terminal: true, - runId, - commandId, - timing: checkpoint?.timing, - startedAt: checkpoint?.startedAt, - lastEventAt: checkpoint?.lastEventAt, - finishedAt: checkpoint?.finishedAt, - durationMs: checkpoint?.durationMs, - createdAt: timestamp, - occurredAt: timestamp, - updatedAt: timestamp, - valuesPrinted: false - } - }); - written += 1; - offset += 1; - } - if (writeCompletion) { - await writeWorkbenchProjectionEvent({ - runtimeStore, - previousCheckpoint: checkpoint, - event: { - traceId: safeId, - sessionId, - turnId, - messageId: assistantMessageId, - source: "workbench-terminal-projection", - sourceSeq: baseSourceSeq + offset, - sourceEventId: `${safeId}:workbench-terminal-projection:completion`, - type: "result", - eventType: "terminal", - status: "completed", - label: "agentrun:result:completed", - message: "AgentRun result is ready for HWLAB short-connection polling.", - finalResponse: { text: finalText, status: "completed", traceId: safeId, valuesPrinted: false }, - terminal: true, - sealed: true, - runId, - commandId, - timing: checkpoint?.timing, - startedAt: checkpoint?.startedAt, - lastEventAt: checkpoint?.lastEventAt, - finishedAt: checkpoint?.finishedAt, - durationMs: checkpoint?.durationMs, - createdAt: timestamp, - occurredAt: timestamp, - updatedAt: timestamp, - valuesPrinted: false - } - }); - written += 1; - } - return { written, idempotent: true, valuesPrinted: false }; - } catch (error) { - emitWorkbenchProjectionOtel("workbench.projection.terminal_trace_backfill", safeId, "error", { - "workbench.projection.phase": "terminal-trace-backfill", - "workbench.projection.session_id": sessionId, - "workbench.projection.turn_id": turnId - }, error); - appendProjectionDiagnostic(traceStore, safeId, { - type: "facts", - status: "degraded", - label: "projection-writer:terminal-trace-backfill-failed", - errorCode: error?.code ?? "workbench_terminal_trace_backfill_failed", - message: error?.message ?? "Workbench terminal trace event backfill failed.", - terminal: false, - valuesPrinted: false - }); - return { written: 0, errorCode: error?.code ?? "workbench_terminal_trace_backfill_failed", valuesPrinted: false }; - } -} - -function buildWorkbenchProjectionFacts({ traceId = null, ownerUserId = null, ownerRole = null, sessionId = null, projectId = null, conversationId = null, threadId = null, status = "active", session = {}, payload = null, params = {}, previousCheckpoint = null } = {}) { - const safeId = safeTraceId(traceId ?? session?.traceId ?? payload?.traceId ?? params?.traceId); - const resolvedSessionId = textValue(sessionId ?? session?.sessionId ?? payload?.sessionId ?? params?.sessionId) || null; - const resolvedConversationId = textValue(conversationId ?? session?.conversationId ?? payload?.conversationId ?? params?.conversationId) || null; - const resolvedThreadId = textValue(threadId ?? session?.threadId ?? payload?.threadId ?? params?.threadId) || null; - const agentRun = payload?.agentRun ?? session?.agentRun ?? null; - const timestamp = timestampValue(payload?.updatedAt ?? session?.updatedAt ?? payload?.createdAt ?? session?.createdAt); - const normalizedStatus = normalizeWorkbenchStatus(payload?.status ?? session?.sessionStatus ?? status); - const projection = createWorkbenchTurnProjection({ traceId: safeId, result: payload, session: { id: resolvedSessionId, status: normalizedStatus, session }, trace: payload?.runnerTrace ?? null }); - const projectedStatus = normalizeWorkbenchStatus(projection.status ?? normalizedStatus); - const terminal = projection.terminal === true; - const terminalSealBlocked = projection.terminalSealBlocked === true; - const terminalStatus = terminal ? (TERMINAL_STATUSES.has(projectedStatus) ? projectedStatus : normalizedStatus) : projectedStatus; - const timing = terminalTimingAtLeastProjectedAt(projectionTimingForStatus(projection.timing, terminal), terminal); - const timingAuthorityIssue = terminalTimingAuthorityIssue(timing, { terminal, traceId: safeId, source: "facts", status: terminalStatus, label: payload?.lastEventLabel, sourceSeq: projection.lastProjectedSeq }); - const baseDiagnostic = projectionDiagnostics({ traceId: safeId, result: payload, trace: payload?.runnerTrace ?? null, projection }); - const diagnostic = timingAuthorityIssue - ? { ...baseDiagnostic, projectionHealth: baseDiagnostic.projectionHealth === "unavailable" ? "unavailable" : "degraded", blocker: baseDiagnostic.blocker ?? timingAuthorityIssue, timingAuthorityIssue, valuesRedacted: true } - : baseDiagnostic; - const factSeq = workbenchProjectionFactSequence({ projection, previousCheckpoint, agentRun }); - const previousFinalText = finalResponseTextValue(previousCheckpoint?.finalResponse, previousCheckpoint?.assistantText, previousCheckpoint?.finalText); - const finalText = terminal ? finalResponseTextValue(projection.finalResponse, payload?.finalResponse, session?.finalResponse, payload?.assistantText, previousFinalText) : null; - const projectedFinalResponse = terminal && finalText ? { text: finalText, status: terminalStatus, traceId: safeId, valuesPrinted: false } : null; - const messages = normalizeMessages(workbenchProjectionInputMessages({ session, payload, params, traceId: safeId, timestamp, terminal, terminalStatus, finalText, previousCheckpoint }), { - traceId: safeId, - sessionId: resolvedSessionId, - conversationId: resolvedConversationId, - threadId: resolvedThreadId, - terminal, - terminalStatus, - finalText, - terminalSealBlocked, - timestamp, - timing - }); - const sessionJson = workbenchSessionJsonWithLaunchContext(session, { payload, params }); - return { - sessions: resolvedSessionId ? [{ - sessionId: resolvedSessionId, - ownerUserId, - ownerRole, - projectId: projectId ?? session?.projectId ?? payload?.projectId ?? params?.projectId ?? null, - conversationId: resolvedConversationId, - threadId: resolvedThreadId, - status: terminal ? terminalStatus : "running", - lastTraceId: safeId, - projectedSeq: factSeq.projectedSeq, - sourceSeq: factSeq.sourceSeq, - sourceEventId: safeId, - terminal, - sealed: terminal, - sessionJson, - timing, - startedAt: timing.startedAt, - lastEventAt: timing.lastEventAt, - finishedAt: timing.finishedAt, - durationMs: timing.durationMs, - createdAt: timestampValue(session?.createdAt ?? payload?.createdAt ?? timestamp), - updatedAt: timestamp - }] : [], - messages, - parts: messages.flatMap((message) => messagePartFacts(message, { finalText: message.role !== "user" ? finalText : null })), - turns: safeId && resolvedSessionId ? [{ - turnId: projection.turnId ?? safeId, - sessionId: resolvedSessionId, - traceId: safeId, - messageId: messages.find((message) => message.role !== "user")?.messageId ?? null, - status: projection.status ?? normalizedStatus, - projectedSeq: factSeq.projectedSeq, - sourceSeq: factSeq.sourceSeq, - sourceEventId: safeId, - terminal: projection.terminal, - sealed: projection.terminal, - finalResponse: projectedFinalResponse, - diagnostic, - assistantText: finalText, - userMessage: previousCheckpoint?.userMessage ?? null, - timing, - startedAt: timing.startedAt, - lastEventAt: timing.lastEventAt, - finishedAt: timing.finishedAt, - durationMs: timing.durationMs, - updatedAt: timestamp - }] : [], - checkpoints: safeId ? [{ - traceId: safeId, - sessionId: resolvedSessionId, - turnId: projection.turnId ?? safeId, - runId: textValue(agentRun?.runId ?? projection.sourceRunId) || null, - commandId: textValue(agentRun?.commandId ?? projection.sourceCommandId) || null, - projectedSeq: factSeq.projectedSeq, - sourceSeq: factSeq.sourceSeq, - sourceEventId: safeId, - projectionStatus: projection.terminal ? "caught_up" : "projecting", - projectionHealth: diagnostic.projectionHealth === "unavailable" ? "unavailable" : diagnostic.projectionHealth === "stalled" ? "stalled" : "healthy", - terminal: projection.terminal, - sealed: projection.terminal, - diagnostic, - finalResponse: projectedFinalResponse, - assistantText: finalText, - userMessage: messages.find((message) => message.role === "user") ?? null, - timing, - startedAt: timing.startedAt, - lastEventAt: timing.lastEventAt, - finishedAt: timing.finishedAt, - durationMs: timing.durationMs, - updatedAt: timestamp - }] : [] - }; -} - function workbenchSessionJsonWithLaunchContext(session = {}, { payload = null, params = null } = {}) { const base = plainObjectValue(session) ?? {}; const launchContext = firstPlainObjectValue( @@ -643,14 +340,7 @@ function workbenchSessionJsonWithLaunchContext(session = {}, { payload = null, p params?.session?.launchContext ); if (!launchContext) return base; - const next = { - ...base, - launchContext: { - ...launchContext, - valuesRedacted: true - }, - valuesRedacted: true - }; + const next = { ...base, launchContext: { ...launchContext, valuesRedacted: true }, valuesRedacted: true }; if (!Object.prototype.hasOwnProperty.call(next, "secretMaterialStored")) next.secretMaterialStored = false; return next; } @@ -667,16 +357,6 @@ function plainObjectValue(value) { return value && typeof value === "object" && !Array.isArray(value) ? value : null; } -function workbenchProjectionFactSequence({ projection = {}, previousCheckpoint = null, agentRun = null } = {}) { - const projectionSeq = nonNegativeInteger(projection?.lastProjectedSeq); - const previousSeq = nonNegativeInteger(previousCheckpoint?.projectedSeq ?? previousCheckpoint?.lastProjectedSeq); - const sourceSeq = nonNegativeInteger(agentRun?.lastSeq ?? projection?.sourceSeq ?? projectionSeq); - return { - projectedSeq: previousSeq > 0 ? Math.min(projectionSeq || previousSeq, previousSeq) : projectionSeq, - sourceSeq - }; -} - function workbenchProjectionError(code, message) { const error = new Error(message); error.code = code; @@ -684,121 +364,6 @@ function workbenchProjectionError(code, message) { return error; } -function emitWorkbenchProjectionOtel(name, traceId, status = "ok", attributes = {}, error = null) { - void emitCodeAgentOtelSpan(name, safeTraceId(traceId) ?? "trc_unassigned", process.env, { - status, - error, - attributes: { - ...attributes, - "workbench.projection.zero_implicit_fallback": true, - "workbench.projection.error_visible": status === "error" - } - }); -} - -function workbenchProjectionFactCountAttributes(facts = {}) { - return { - "workbench.facts.sessions": Array.isArray(facts.sessions) ? facts.sessions.length : 0, - "workbench.facts.messages": Array.isArray(facts.messages) ? facts.messages.length : 0, - "workbench.facts.parts": Array.isArray(facts.parts) ? facts.parts.length : 0, - "workbench.facts.turns": Array.isArray(facts.turns) ? facts.turns.length : 0, - "workbench.facts.trace_events": Array.isArray(facts.traceEvents) ? facts.traceEvents.length : 0, - "workbench.facts.checkpoints": Array.isArray(facts.checkpoints) ? facts.checkpoints.length : 0 - }; -} - -function workbenchProjectionInputMessages({ session = {}, payload = {}, params = {}, traceId = null, timestamp = null, terminal = false, terminalStatus = "completed", finalText = null, previousCheckpoint = null } = {}) { - if (Array.isArray(session?.messages) && session.messages.length > 0) return preserveTerminalUserMessage(session.messages, { terminal, traceId, previousCheckpoint }); - if (Array.isArray(payload?.messages) && payload.messages.length > 0) return preserveTerminalUserMessage(payload.messages, { terminal, traceId, previousCheckpoint }); - if (!traceId) return []; - const turnId = textValue(payload?.turnId) || traceId; - const assistantMessageId = eventProjectionAssistantMessageId(traceId, payload); - const createdAt = timestampValue(timestamp ?? payload?.createdAt ?? payload?.updatedAt); - const messages = []; - const userMessage = workbenchProjectionUserMessage({ payload, params, traceId, turnId, timestamp: createdAt, terminal }); - if (userMessage) messages.push(userMessage); - messages.push({ - messageId: assistantMessageId, - role: "agent", - status: terminal ? terminalStatus : "running", - text: terminal && finalText ? finalText : "", - traceId, - turnId, - timing: payload?.timing, - startedAt: payload?.startedAt, - lastEventAt: payload?.lastEventAt, - finishedAt: payload?.finishedAt, - durationMs: payload?.durationMs, - createdAt, - updatedAt: createdAt, - valuesPrinted: false - }); - return messages; -} - -function workbenchProjectionUserMessage({ payload = {}, params = {}, traceId = null, turnId = null, timestamp = null, terminal = false } = {}) { - const userText = textValue(params?.message ?? params?.prompt ?? payload?.prompt ?? payload?.userText ?? payload?.userMessage?.text ?? payload?.userMessage?.content ?? (!terminal ? payload?.message : null)); - if (!traceId || !userText) return null; - const createdAt = timestampValue(timestamp ?? payload?.createdAt ?? payload?.updatedAt); - return { - messageId: eventProjectionUserMessageId(traceId, payload), - role: "user", - status: "sent", - text: userText, - traceId, - turnId: textValue(turnId) || traceId, - createdAt, - updatedAt: createdAt, - valuesPrinted: false - }; -} - -function preserveTerminalUserMessage(messages = [], { terminal = false, traceId = null, previousCheckpoint = null } = {}) { - if (!terminal || !traceId) return messages; - const previousUserMessage = previousCheckpoint?.userMessage ?? null; - const previousText = textValue(previousUserMessage?.text ?? previousUserMessage?.content ?? previousUserMessage?.message); - let handledUser = false; - const normalizedTraceId = safeTraceId(traceId) ?? traceId; - const preserved = messages.map((message) => { - const role = textValue(message?.role) || "agent"; - const messageTraceId = safeTraceId(message?.traceId ?? message?.turnId ?? normalizedTraceId) ?? normalizedTraceId; - if (role !== "user" || messageTraceId !== normalizedTraceId) return message; - handledUser = true; - const messageText = textValue(message?.text ?? message?.content ?? message?.message); - if (!previousText && isSyntheticTerminalUserNotice(messageText)) return null; - if (!previousText) return message; - return { - ...message, - ...previousUserMessage, - messageId: textValue(previousUserMessage.messageId ?? previousUserMessage.id ?? message?.messageId ?? message?.id) || eventProjectionUserMessageId(normalizedTraceId, message), - role: "user", - status: "sent", - text: previousText, - traceId: normalizedTraceId, - turnId: textValue(previousUserMessage.turnId ?? message?.turnId) || normalizedTraceId, - terminal: false, - sealed: false - }; - }).filter(Boolean); - if (handledUser) return preserved; - if (!previousText) return preserved; - return [previousUserMessage, ...messages]; -} - -function isSyntheticTerminalUserNotice(text) { - const value = textValue(text); - return /当前\s*AgentRun\s*请求已取消|请求已取消|hwlab-user-cancel/u.test(value); -} - -function normalizeMessages(messages, context) { - const normalized = Array.isArray(messages) ? messages.map((message, index) => normalizeMessageFact(message, index, context)).filter(Boolean) : []; - if (context?.terminal && context?.finalText && context?.traceId && !normalized.some((message) => message.role !== "user" && message.traceId === context.traceId)) { - const projected = normalizeMessageFact({ messageId: eventProjectionAssistantMessageId(context.traceId), role: "agent", status: context.terminalStatus ?? "completed", text: context.finalText, traceId: context.traceId, source: "workbench-terminal-projection" }, normalized.length, context); - if (projected) normalized.push(projected); - } - return normalized; -} - function messagePartFacts(message = {}, { finalText = null } = {}) { const messageId = textValue(message.messageId); if (!messageId || !message.sessionId) return []; @@ -829,50 +394,6 @@ function messagePartFacts(message = {}, { finalText = null } = {}) { }]; } -async function existingWorkbenchTraceEvents(runtimeStore, traceId, { traceStore = defaultCodeAgentTraceStore } = {}) { - if (typeof runtimeStore?.queryWorkbenchFacts !== "function") return []; - const safeId = safeTraceId(traceId); - try { - const result = await runtimeStore.queryWorkbenchFacts({ traceId: safeId ?? traceId, families: ["traceEvents"], limit: 500 }); - return Array.isArray(result?.facts?.traceEvents) ? result.facts.traceEvents : []; - } catch (error) { - emitWorkbenchProjectionOtel("workbench.projection.terminal_trace_backfill.probe", safeId, "error", { - "workbench.projection.phase": "terminal-trace-backfill-probe" - }, error); - if (safeId) appendProjectionDiagnostic(traceStore, safeId, { - type: "facts", - status: "degraded", - label: "projection-writer:terminal-trace-backfill-probe-failed", - errorCode: error?.code ?? "workbench_terminal_trace_backfill_probe_failed", - message: error?.message ?? "Workbench terminal trace backfill probe failed.", - terminal: false, - valuesPrinted: false - }); - return null; - } -} - -function hasTerminalAssistantFinalTraceEvent(events = [], finalText = null) { - const expectedText = finalResponseTextValue(finalText); - return events.some((event) => { - const type = textValue(event?.eventType ?? event?.type).toLowerCase(); - const label = textValue(event?.label).toLowerCase(); - if (type !== "assistant" && !/assistant:message|assistant_message/u.test(label)) return false; - if (event?.terminal !== true && normalizeWorkbenchStatus(event?.status) !== "completed") return false; - const text = finalResponseTextValue(event?.finalResponse, event?.text, event?.message, event?.content); - return !expectedText || text === expectedText; - }); -} - -function hasTerminalResultTraceEvent(events = []) { - return events.some((event) => { - const type = textValue(event?.eventType ?? event?.type).toLowerCase(); - const label = textValue(event?.label).toLowerCase(); - const status = normalizeWorkbenchStatus(event?.status); - return (type === "terminal" || type === "result" || /result:completed|result_completed/u.test(label)) && event?.terminal === true && status === "completed"; - }); -} - function normalizeMessageFact(message = {}, index, { traceId, sessionId, conversationId, threadId, terminal, terminalStatus = "completed", finalText = null, terminalSealBlocked = false, timestamp, timing: contextTiming }) { const role = textValue(message.role) || "agent"; const messageId = textValue(message.messageId ?? message.id) || (role === "user" ? eventProjectionUserMessageId(traceId, message) : eventProjectionAssistantMessageId(traceId, message)); @@ -928,48 +449,6 @@ function finalResponseTextValue(...values) { return null; } -async function latestWorkbenchCheckpoint(runtimeStore, traceId, { traceStore = defaultCodeAgentTraceStore, phase = "checkpoint-read" } = {}) { - if (typeof runtimeStore?.queryWorkbenchFacts !== "function") return null; - const safeId = safeTraceId(traceId); - try { - const result = await runtimeStore.queryWorkbenchFacts({ traceId: safeId ?? traceId, families: ["checkpoints"], limit: 1 }); - return Array.isArray(result?.facts?.checkpoints) ? result.facts.checkpoints[0] ?? null : null; - } catch (error) { - emitWorkbenchProjectionOtel("workbench.projection.checkpoint.read", safeId, "error", { - "workbench.projection.phase": phase - }, error); - if (safeId) appendProjectionDiagnostic(traceStore, safeId, { - type: "facts", - status: "degraded", - label: "projection-writer:checkpoint-read-failed", - errorCode: error?.code ?? "workbench_checkpoint_read_failed", - message: error?.message ?? "Workbench checkpoint read failed.", - terminal: false, - valuesPrinted: false - }); - return null; - } -} - -function projectionTimingForStatus(value, terminal) { - const timing = normalizeTimingProjection(value) ?? emptyTimingProjection(); - if (terminal) return timing; - return { ...timing, finishedAt: null, durationMs: null, valuesRedacted: timing.valuesRedacted !== false }; -} - -function terminalTimingAtLeastProjectedAt(value, terminal) { - const timing = normalizeTimingProjection(value) ?? emptyTimingProjection(); - if (!terminal) return timing; - const finishedAt = timing.finishedAt ?? null; - const durationMs = durationValue(timing.durationMs) ?? elapsedBetween(timing.startedAt, finishedAt); - return { - ...timing, - finishedAt, - durationMs, - valuesRedacted: timing.valuesRedacted !== false - }; -} - function checkpointIsTerminal(value) { const source = value && typeof value === "object" ? value : null; if (!source) return false; @@ -1051,27 +530,6 @@ function terminalTimingAuthorityIssue(timing, { terminal = false, traceId = null }; } -function emitTerminalTimingAuthorityDiagnostic(traceStore, traceId, issue, attributes = {}) { - if (!issue) return null; - emitWorkbenchProjectionOtel("workbench.projection.terminal_timing_authority", traceId, "error", { - ...attributes, - "workbench.projection.timing_authority_missing": true, - "workbench.projection.missing_started_at": issue.missingStartedAt === true, - "workbench.projection.missing_finished_at": issue.missingFinishedAt === true, - "workbench.projection.missing_duration_ms": issue.missingDurationMs === true - }, workbenchProjectionError(issue.code, issue.message)); - return appendProjectionDiagnostic(traceStore, traceId, { - type: "projection-timing", - status: "degraded", - label: "projection-writer:terminal-timing:missing-authority", - errorCode: issue.code, - message: issue.message, - terminal: false, - timingAuthorityIssue: issue, - valuesPrinted: false - }); -} - function latestTimestamp(...values) { let latest = null; let latestMs = Number.NEGATIVE_INFINITY; @@ -1085,21 +543,17 @@ function latestTimestamp(...values) { return latest; } -function isEmptyFacts(facts) { - return Object.values(facts).every((items) => !Array.isArray(items) || items.length === 0); -} - function stableFactId(prefix, value) { return `${prefix}_${createHash("sha256").update(JSON.stringify(sortJson(value))).digest("hex").slice(0, 32)}`; } function workbenchSourceEventId(event = {}, { traceId, sourceSeq, occurredAt } = {}) { + const explicit = textValue(event.sourceEventId ?? event.id); + if (explicit) return explicit; if (isAssistantMessageTraceEvent(event)) { const messageId = eventProjectionAssistantMessageId(traceId, event); return `${safeTraceId(traceId) ?? traceId}:assistant-message:${messageId}`; } - const explicit = textValue(event.sourceEventId ?? event.id); - if (explicit) return explicit; const source = textValue(event.source); const label = textValue(event.label ?? event.type ?? event.eventType) || "event"; if (source && sourceSeq > 0) return `${source}:${sourceSeq}:${label}`; @@ -1147,15 +601,3 @@ function optionalTimestampValue(value) { function textValue(value) { return String(value ?? "").trim(); } - -export function appendProjectionDiagnostic(traceStore = defaultCodeAgentTraceStore, traceId, event = {}) { - const safeId = safeTraceId(traceId); - if (!safeId || typeof traceStore?.append !== "function") return null; - return traceStore.append(safeId, { - type: "projection-diagnostic", - status: "degraded", - label: "projection:diagnostic", - ...event, - valuesPrinted: false - }); -} diff --git a/internal/cloud/workbench-read-model.ts b/internal/cloud/workbench-read-model.ts index fcde5182..580daeb9 100644 --- a/internal/cloud/workbench-read-model.ts +++ b/internal/cloud/workbench-read-model.ts @@ -16,7 +16,6 @@ export function createWorkbenchReadModel(options = {}, actor = null) { getSessionByTraceId: (traceId) => facts.getSessionByTraceId(traceId), resultForTrace: (traceId) => facts.resultForTrace(traceId), traceSnapshot: (traceId) => facts.traceSnapshot(traceId), - traceSnapshotSync: (traceId) => facts.traceSnapshotSync(traceId), canReadOwner: (ownerUserId) => facts.canReadOwner(ownerUserId), projectionDiagnostics: ({ traceId, result = null, trace = null, projection = null, refreshError = null } = {}) => workbenchProjectionDiagnostics({ traceId, result, trace, projection, refreshError }) }; diff --git a/internal/cloud/workbench-realtime-authority-contract.test.ts b/internal/cloud/workbench-realtime-authority-contract.test.ts index 2033c8b9..f1a6e689 100644 --- a/internal/cloud/workbench-realtime-authority-contract.test.ts +++ b/internal/cloud/workbench-realtime-authority-contract.test.ts @@ -7,8 +7,61 @@ import { test } from "bun:test"; import { createCloudApiServer } from "./server.ts"; const ACTOR = { id: "usr_workbench_realtime_authority", username: "reader", displayName: "Reader", role: "user", status: "active" }; +const PROJECTION_REALTIME_ENV = Object.freeze({ + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true" +}); +const LIVE_REALTIME_ENV = Object.freeze({ + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" +}); -test("workbench sync delta stays durable while SSE realtime forwards Kafka events", async () => { +test("workbench sync fails closed before projection storage in live Kafka capability", async () => { + let syncReads = 0; + const server = createCloudApiServer({ + accessController: createAccessController(), + workbenchRuntime: { + async readAtomicWorkbenchProjectionSync() { + syncReads += 1; + throw new Error("live Kafka sync must not reach projection storage"); + } + }, + kafkaEventBridge: { + started: true, + capabilities: { + directPublish: true, + liveKafkaSse: true, + transactionalProjector: false, + projectionOutboxRelay: false, + projectionRealtime: false + }, + ready: Promise.resolve(), + subscribeLiveHwlabEvents() { return () => {}; }, + async stop() {} + }, + env: { ...LIVE_REALTIME_ENV } + }); + await listen(server); + + try { + const { port } = server.address(); + const response = await getJson(port, "/v1/workbench/sync?sessionId=ses_live_only&since=0"); + assert.equal(response.status, 503); + assert.equal(response.body.error.code, "workbench_projection_realtime_disabled"); + assert.equal(response.body.error.capabilities.liveKafkaSse, true); + assert.equal(syncReads, 0); + } finally { + await close(server); + } +}); + +test("workbench sync and SSE read the same atomic projection outbox", async () => { const sessionId = "ses_realtime_authority_p1"; const traceId = "trc_realtime_authority_p1"; const outboxRows = [{ @@ -28,20 +81,33 @@ test("workbench sync delta stays durable while SSE realtime forwards Kafka event terminal: true, sealed: true, createdAt: "2026-07-08T12:20:00.000Z", - payload: { type: "terminal", status: "completed", text: "redacted final" } + entityFamily: "turns", + entityId: traceId, + payload: { + family: "turns", + fact: { + turnId: traceId, + sessionId, + traceId, + messageId: "msg_realtime_authority_agent", + status: "completed", + projectedSeq: 8, + terminal: true, + sealed: true, + finalResponse: { text: "redacted final", status: "completed", traceId } + }, + valuesRedacted: true + } }]; const outboxQueries = []; const runtime = createRuntime({ facts: durableFacts({ sessionId, traceId }), outboxRows, outboxQueries }); - const fakeKafka = createFakeKafkaFactory(); const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: runtime, - kafkaFactory: fakeKafka.factory, env: { + ...PROJECTION_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", - HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", - HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", - HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1" + HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } }); await listen(server); @@ -59,23 +125,16 @@ test("workbench sync delta stays durable while SSE realtime forwards Kafka event assert.equal(sync.body.events[0].terminal, true); assert.equal(sync.body.events[0].sealed, true); - setTimeout(() => { void fakeKafka.emit({ - eventType: "hwlab.trace.event.projected", - sessionId: "ses_agentrun_realtime_authority_p1", - traceId, - context: { sourceSeq: 8, runId: "run_realtime_authority_p1", commandId: "cmd_realtime_authority_p1" }, - event: { type: "result", eventType: "terminal", status: "completed", label: "agentrun:terminal:completed", message: "redacted final", terminal: true, sourceSeq: 8 } - }); }, 0); const events = await getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(sessionId)}&traceId=${encodeURIComponent(traceId)}&afterSeq=10`, 2); - assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.trace.event"]); - assert.equal(events[1].data.realtimeSource, "kafka"); + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.turn.snapshot"]); + assert.equal(events[1].data.realtimeSource, "projection-outbox"); assert.equal(events[1].data.realtimeAuthority, "workbench-realtime-authority-v2"); - assert.equal(events[1].data.entity.family, "traceEvents"); - assert.equal(events[1].data.entity.version, 8); - assert.equal(events[1].data.entity.projectionRevision, "kafka:hwlab.event.v1:0:0"); - assert.equal(events[1].data.event.message, "redacted final"); - assert.deepEqual(outboxQueries.map((query) => query.afterSeq), [10]); + assert.equal(events[1].data.entity.family, "turns"); + assert.equal(events[1].data.entity.version, 42); + assert.equal(events[1].data.turn.finalResponse.text, "redacted final"); + assert.deepEqual(outboxQueries.map((query) => query.afterOutboxSeq), [10, 10]); assert.equal(outboxQueries.every((query) => query.sessionId === sessionId), true); + assert.deepEqual(outboxQueries.at(-1).actor, { id: ACTOR.id, role: ACTOR.role }); } finally { await close(server); } @@ -93,7 +152,7 @@ test("workbench sync delta includes all authority families and marks trace rows }), outboxRows: [] }); - const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: runtime }); + const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: runtime, env: { ...PROJECTION_REALTIME_ENV } }); await listen(server); try { @@ -118,6 +177,27 @@ test("workbench sync delta includes all authority families and marks trace rows } }); +test("workbench sync preserves session and trace scope for narrow replay", async () => { + const sessionId = "ses_realtime_authority_narrow"; + const traceId = "trc_realtime_authority_narrow"; + const outboxQueries = []; + const runtime = createRuntime({ facts: durableFacts({ sessionId, traceId }), outboxQueries }); + const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: runtime, env: { ...PROJECTION_REALTIME_ENV } }); + await listen(server); + + try { + const { port } = server.address(); + const sync = await getJson(port, `/v1/workbench/sync?sessionId=${encodeURIComponent(sessionId)}&traceId=${encodeURIComponent(traceId)}&since=7`); + assert.equal(sync.status, 200); + assert.deepEqual(sync.body.scope, { kind: "trace", sessionId, traceId, since: 7 }); + assert.equal(outboxQueries.length, 1); + assert.equal(outboxQueries[0].sessionId, sessionId); + assert.equal(outboxQueries[0].traceId, traceId); + } finally { + await close(server); + } +}); + test("workbench trace event detail route declares detail-only authority", async () => { const sessionId = "ses_realtime_authority_detail"; const traceId = "trc_realtime_authority_detail"; @@ -125,7 +205,7 @@ test("workbench trace event detail route declares detail-only authority", async facts: durableFacts({ sessionId, traceId, finalText: "sealed final", traceEventText: "detail row" }), outboxRows: [] }); - const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: runtime }); + const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: runtime, env: { ...PROJECTION_REALTIME_ENV } }); await listen(server); try { @@ -144,7 +224,7 @@ test("workbench trace event detail route declares detail-only authority", async }); test("workbench sync rejects unscoped automatic repair requests", async () => { - const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: createRuntime({ facts: durableFacts({}) }) }); + const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: createRuntime({ facts: durableFacts({}) }), env: { ...PROJECTION_REALTIME_ENV } }); await listen(server); try { @@ -168,9 +248,19 @@ function createAccessController() { function createRuntime({ facts, outboxRows = [], outboxQueries = [] }) { return { - async readWorkbenchProjectionOutbox(params = {}) { + async readAtomicWorkbenchProjectionSync(params = {}) { outboxQueries.push({ ...params }); - return outboxRows.filter((row) => Number(row.outboxSeq) > Number(params.afterSeq ?? 0)); + const after = Number(params.afterOutboxSeq ?? params.afterSeq ?? 0); + const events = outboxRows.filter((row) => Number(row.outboxSeq) > after); + const cutoffOutboxSeq = outboxRows.reduce((max, row) => Math.max(max, Number(row.outboxSeq) || 0), 0); + return { + facts: filterFacts(facts, params), + events, + cutoffOutboxSeq, + cursorOutboxSeq: cutoffOutboxSeq, + hasMore: false, + valuesRedacted: true + }; }, async queryWorkbenchFacts(params = {}) { return { @@ -310,44 +400,6 @@ function parseSseBlock(block) { return { event, id, data: JSON.parse(dataLine) }; } -function createFakeKafkaFactory() { - let eachMessage = null; - let offset = 0; - let subscribedTopic = "hwlab.event.v1"; - const consumer = { - connect: async () => undefined, - subscribe: async (input = {}) => { subscribedTopic = String(input.topic ?? subscribedTopic); }, - run: async (input = {}) => { eachMessage = input.eachMessage; }, - stop: async () => undefined, - disconnect: async () => undefined - }; - return { - factory: () => ({ consumer: () => consumer }), - emit: async (value = {}) => { - await waitFor(() => typeof eachMessage === "function"); - await eachMessage({ - topic: subscribedTopic, - partition: 0, - message: { - offset: String(offset++), - key: Buffer.from(value.traceId ?? value.sessionId ?? "fake"), - timestamp: String(Date.now()), - value: Buffer.from(JSON.stringify(value)) - } - }); - } - }; -} - -async function waitFor(predicate, timeoutMs = 1000) { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - if (predicate()) return; - await new Promise((resolve) => setTimeout(resolve, 10)); - } - throw new Error("waitFor timed out"); -} - function listen(server) { return new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); } diff --git a/internal/cloud/workbench-realtime-authority.ts b/internal/cloud/workbench-realtime-authority.ts index 98a1735d..1a23d3da 100644 --- a/internal/cloud/workbench-realtime-authority.ts +++ b/internal/cloud/workbench-realtime-authority.ts @@ -4,18 +4,31 @@ */ import { safeSessionId, safeTraceId, sendJson } from "./server-http-utils.ts"; import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts"; +import { projectionOutboxRealtimeEvents } from "./workbench-projection-outbox-events.ts"; +import { workbenchRealtimeCapabilities } from "./workbench-realtime-capabilities.ts"; const SYNC_CONTRACT_VERSION = "workbench-sync-v1"; const REALTIME_AUTHORITY_VERSION = "workbench-realtime-authority-v2"; const DEFAULT_SYNC_LIMIT = 100; const MAX_SYNC_LIMIT = 500; -const SYNC_FACT_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "traceEvents", "checkpoints"]); export async function handleWorkbenchSyncHttp(request, response, url, options = {}, actor = null) { if (request.method !== "GET") { sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "GET required." } }); return; } + const capabilities = workbenchRealtimeCapabilities(options.env ?? process.env); + if (!capabilities.projectionRealtime) { + sendJson(response, 503, { + ok: false, + error: { + code: "workbench_projection_realtime_disabled", + message: "Workbench projection sync/replay capability is disabled.", + valuesRedacted: true + } + }); + return; + } const sessionId = safeSessionId(url.searchParams.get("sessionId") ?? url.searchParams.get("includeSessionId")); const traceId = safeTraceId(url.searchParams.get("traceId")); if (!sessionId && !traceId) { @@ -30,12 +43,12 @@ export async function handleWorkbenchSyncHttp(request, response, url, options = return; } const runtime = workbenchRealtimeRuntime(request, options); - if (typeof runtime?.queryWorkbenchFacts !== "function" || typeof runtime?.readWorkbenchProjectionOutbox !== "function") { + if (typeof runtime?.readAtomicWorkbenchProjectionSync !== "function") { sendJson(response, 503, { ok: false, error: { code: "workbench_sync_runtime_unconfigured", - message: "Workbench sync requires durable facts and outbox readers.", + message: "Workbench sync requires one atomic durable projection snapshot reader.", valuesRedacted: true } }); @@ -45,27 +58,25 @@ export async function handleWorkbenchSyncHttp(request, response, url, options = try { const since = realtimeSinceCursor(request, url); const limit = boundedSyncLimit(url.searchParams.get("limit")); - const outboxQuery = { afterSeq: since, limit }; - if (sessionId) outboxQuery.sessionId = sessionId; - else outboxQuery.traceId = traceId; - const rows = await runtime.readWorkbenchProjectionOutbox(outboxQuery); - const events = rows.map(workbenchRealtimeEntityDelta); - const latestOutboxSeq = events.reduce((max, event) => Math.max(max, Number(event.cursor?.outboxSeq ?? 0)), since); - const facts = await runtime.queryWorkbenchFacts({ - families: SYNC_FACT_FAMILIES, - ...(sessionId ? { sessionId } : { traceId }), + const snapshot = await runtime.readAtomicWorkbenchProjectionSync({ + afterOutboxSeq: since, + afterSeq: since, limit, - actor: actor ? { id: actor.id, role: actor.role ?? "user", valuesRedacted: true } : undefined + ...(sessionId ? { sessionId } : {}), + ...(traceId ? { traceId } : {}), + actor: actor ? { id: actor.id, role: actor.role ?? "user" } : undefined }); - if (facts?.error) throw facts.error; - const delta = normalizeSyncFacts(facts?.facts); + const rows = Array.isArray(snapshot?.events) ? snapshot.events : []; + const events = projectionOutboxRealtimeEvents({ events: rows, facts: {} }).map((item) => item.payload); + const latestOutboxSeq = nonNegativeInteger(snapshot?.cursorOutboxSeq); + const delta = normalizeSyncFacts(snapshot?.facts); sendJson(response, 200, { ok: true, status: "succeeded", contractVersion: SYNC_CONTRACT_VERSION, realtimeAuthority: REALTIME_AUTHORITY_VERSION, scope: { - kind: sessionId ? "session" : "trace", + kind: traceId ? "trace" : "session", sessionId, traceId, since @@ -73,7 +84,8 @@ export async function handleWorkbenchSyncHttp(request, response, url, options = cursor: { outboxSeq: latestOutboxSeq, since, - hasMore: rows.length >= limit + cutoffOutboxSeq: nonNegativeInteger(snapshot?.cutoffOutboxSeq), + hasMore: snapshot?.hasMore === true }, events, delta, @@ -101,48 +113,8 @@ export async function handleWorkbenchSyncHttp(request, response, url, options = } } -export function workbenchRealtimeEntityDelta(row = {}) { - const commitType = textValue(row.commitType) || "event"; - const projectionRevision = nonNegativeInteger(row.projectionRevision ?? row.projectedSeq ?? row.outboxSeq); - const family = workbenchRealtimeEntityFamily(row, commitType); - const id = workbenchRealtimeEntityId(row, family); - const outboxSeq = nonNegativeInteger(row.outboxSeq); - const projectedSeq = nonNegativeInteger(row.projectedSeq); - return { - type: "workbench.entity.delta", - family, - id, - entity: { - family, - id, - version: projectionRevision, - entityVersion: projectionRevision - }, - cursor: { - outboxSeq, - traceSeq: projectedSeq - }, - sessionId: safeSessionId(row.sessionId) ?? null, - turnId: textValue(row.turnId) || null, - traceId: safeTraceId(row.traceId) ?? null, - messageId: textValue(row.messageId) || null, - commitType, - eventSeq: row.eventSeq === null || row.eventSeq === undefined ? null : nonNegativeInteger(row.eventSeq), - aggregateId: textValue(row.aggregateId) || null, - aggregateSeq: nonNegativeInteger(row.aggregateSeq), - projectionRevision, - entityVersion: projectionRevision, - terminal: row.terminal === true, - sealed: row.sealed === true, - serverCommittedAt: textValue(row.createdAt) || null, - payload: redactedPayload(row.payload), - valuesRedacted: true - }; -} - function workbenchRealtimeRuntime(request, options = {}) { return options.workbenchRuntime - ?? options.runtimeStore ?? createWorkbenchRuntimeClient({ env: options.env ?? process.env, fetch: options.fetch, logger: options.logger, traceparent: request?.hwlabHttpRequestContext?.traceparent }); } @@ -175,31 +147,6 @@ function normalizeSyncFacts(facts = {}) { }; } -function workbenchRealtimeEntityFamily(row = {}, commitType = "event") { - if (commitType === "terminal" || row.terminal === true || row.sealed === true) return "turns"; - if (commitType === "message" || row.messageId) return "messages"; - if (row.traceId) return "traceEvents"; - if (row.sessionId) return "sessions"; - return "diagnostics"; -} - -function workbenchRealtimeEntityId(row = {}, family = "diagnostics") { - if (family === "messages") return textValue(row.messageId) || textValue(row.turnId) || textValue(row.traceId) || `outbox:${nonNegativeInteger(row.outboxSeq)}`; - if (family === "turns") return textValue(row.turnId) || textValue(row.traceId) || `outbox:${nonNegativeInteger(row.outboxSeq)}`; - if (family === "traceEvents") return textValue(row.sourceEventId) || [textValue(row.traceId), nonNegativeInteger(row.projectedSeq)].filter(Boolean).join(":") || `outbox:${nonNegativeInteger(row.outboxSeq)}`; - if (family === "sessions") return textValue(row.sessionId) || `outbox:${nonNegativeInteger(row.outboxSeq)}`; - return textValue(row.aggregateId) || `outbox:${nonNegativeInteger(row.outboxSeq)}`; -} - -function redactedPayload(payload) { - if (!payload || typeof payload !== "object" || Array.isArray(payload)) return null; - return { - kind: textValue(payload.kind ?? payload.type ?? payload.eventType) || null, - status: textValue(payload.status) || null, - valuesRedacted: true - }; -} - function factArray(value) { return Array.isArray(value) ? value : []; } diff --git a/internal/cloud/workbench-realtime-capabilities.ts b/internal/cloud/workbench-realtime-capabilities.ts new file mode 100644 index 00000000..c62e3624 --- /dev/null +++ b/internal/cloud/workbench-realtime-capabilities.ts @@ -0,0 +1,34 @@ +// SPEC: PJ2026-0104010803 Workbench composable realtime capabilities. +// Responsibility: validate YAML-owned capability switches without code defaults or implied mode selection. + +export const WORKBENCH_REALTIME_CAPABILITY_ENVS = Object.freeze({ + directPublish: "HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED", + liveKafkaSse: "HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED", + transactionalProjector: "HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED", + projectionOutboxRelay: "HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED", + projectionRealtime: "HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED" +}); + +export function workbenchRealtimeCapabilities(env = process.env) { + return Object.fromEntries(Object.entries(WORKBENCH_REALTIME_CAPABILITY_ENVS).map(([key, name]) => [key, requiredBooleanEnv(env, name)])); +} + +export function requiredBooleanEnv(env, name) { + const value = String(env?.[name] ?? "").trim().toLowerCase(); + if (value === "true" || value === "1") return true; + if (value === "false" || value === "0") return false; + throw capabilityError( + "hwlab_workbench_realtime_capability_invalid", + `${name} is required and must be explicitly true or false in the owning YAML.`, + name + ); +} + +function capabilityError(code, message, envName) { + return Object.assign(new Error(message), { + code, + envName, + statusCode: 503, + valuesRedacted: true + }); +} diff --git a/internal/cloud/workbench-runtime-client.ts b/internal/cloud/workbench-runtime-client.ts index f54b13f9..8e95003b 100644 --- a/internal/cloud/workbench-runtime-client.ts +++ b/internal/cloud/workbench-runtime-client.ts @@ -17,6 +17,7 @@ export function createWorkbenchRuntimeClient(options = {}) { queryWorkbenchFacts: (params = {}) => postRuntime({ fetchFn, baseUrl, timeoutMs, traceparent, path: "/v1/workbench-runtime/facts", body: params }), queryAgentTraceEvents: (params = {}) => postRuntime({ fetchFn, baseUrl, timeoutMs, traceparent, path: "/v1/workbench-runtime/trace-events", body: params }), listWorkbenchSessions: (params = {}) => postRuntime({ fetchFn, baseUrl, timeoutMs, traceparent, path: "/v1/workbench-runtime/sessions", body: params }), + readAtomicWorkbenchProjectionSync: (params = {}) => postRuntime({ fetchFn, baseUrl, timeoutMs, traceparent, path: "/v1/workbench-runtime/sync", body: params }), readWorkbenchProjectionOutbox: (params = {}) => postRuntime({ fetchFn, baseUrl, timeoutMs, traceparent, path: "/v1/workbench-runtime/projection-outbox", body: params }).then((result) => Array.isArray(result?.rows) ? result.rows : []) }; } diff --git a/internal/cloud/workbench-turn-projection.ts b/internal/cloud/workbench-turn-projection.ts index fc6221ad..d2f6148b 100644 --- a/internal/cloud/workbench-turn-projection.ts +++ b/internal/cloud/workbench-turn-projection.ts @@ -39,8 +39,8 @@ export function createWorkbenchTurnProjection({ turnId = null, traceId = null, r finalResponse: finalText ? { text: finalText, status, traceId: projectionTraceId, valuesPrinted: false } : null, assistantText: finalText, lastProjectedSeq: traceLastSeq(trace), - sourceRunId: agentRun?.runId ?? lastEvent?.runId ?? lastEvent?.payload?.runId ?? null, - sourceCommandId: agentRun?.commandId ?? lastEvent?.commandId ?? lastEvent?.payload?.commandId ?? null, + sourceRunId: lastEvent?.runId ?? lastEvent?.payload?.runId ?? agentRun?.runId ?? null, + sourceCommandId: lastEvent?.commandId ?? lastEvent?.payload?.commandId ?? agentRun?.commandId ?? null, eventCount: normalizedEventCount(trace), updatedAt: trace?.updatedAt ?? result?.updatedAt ?? session?.updatedAt ?? null, timing, @@ -164,7 +164,13 @@ export function traceTerminalEvidence(trace = null) { if (direct) { const status = terminalStatusFromValue(direct.status ?? direct.terminalStatus ?? trace?.status) ?? "completed"; if (status !== "completed" && retryableProviderInterruptionEvidence(direct, trace)) return null; - return { source: "trace-terminal-evidence", status, evidence: direct, valuesRedacted: true }; + return { + source: "trace-terminal-evidence", + status, + finalResponse: direct.finalResponse ?? null, + evidence: direct, + valuesRedacted: true + }; } const events = Array.isArray(trace?.events) ? trace.events : []; return terminalTraceEventEvidence(events); diff --git a/internal/db/migrations/0008_workbench_kafka_realtime.sql b/internal/db/migrations/0008_workbench_kafka_realtime.sql new file mode 100644 index 00000000..9c225562 --- /dev/null +++ b/internal/db/migrations/0008_workbench_kafka_realtime.sql @@ -0,0 +1,106 @@ +-- Versioned v8 upgrade for immutable Workbench replay and the AgentRun Kafka projector. + +ALTER TABLE workbench_projection_outbox ADD COLUMN IF NOT EXISTS outbox_event_id TEXT; +ALTER TABLE workbench_projection_outbox ADD COLUMN IF NOT EXISTS entity_family TEXT; +ALTER TABLE workbench_projection_outbox ADD COLUMN IF NOT EXISTS entity_id TEXT; + +-- Pre-v8 rows did not store immutable fact payloads. Mark them as cutover rows so +-- recovery advances its cursor without fabricating message/turn/trace entities. +UPDATE workbench_projection_outbox +SET outbox_event_id = COALESCE(outbox_event_id, 'legacy:' || outbox_seq::text), + entity_family = COALESCE(entity_family, 'legacy'), + entity_id = COALESCE(entity_id, 'legacy:' || outbox_seq::text) +WHERE outbox_event_id IS NULL OR entity_family IS NULL OR entity_id IS NULL; + +ALTER TABLE workbench_projection_outbox ALTER COLUMN outbox_event_id SET NOT NULL; +ALTER TABLE workbench_projection_outbox ALTER COLUMN entity_family SET NOT NULL; +ALTER TABLE workbench_projection_outbox ALTER COLUMN entity_id SET NOT NULL; +DROP INDEX IF EXISTS idx_workbench_projection_outbox_event_id; +CREATE UNIQUE INDEX idx_workbench_projection_outbox_event_id ON workbench_projection_outbox(outbox_event_id); + +CREATE TABLE IF NOT EXISTS workbench_kafka_inbox ( + source_topic TEXT NOT NULL, + source_partition INTEGER NOT NULL, + source_offset BIGINT NOT NULL, + source_key TEXT, + source_event_id TEXT, + input_sha256 TEXT NOT NULL, + trace_id TEXT, + session_id TEXT, + run_id TEXT, + command_id TEXT, + source_seq INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'processing', + projected_seq INTEGER, + envelope_json TEXT NOT NULL DEFAULT '{}', + error_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + projected_at TEXT, + PRIMARY KEY (source_topic, source_partition, source_offset) +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_workbench_kafka_inbox_source_event ON workbench_kafka_inbox(source_topic, source_event_id) WHERE source_event_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_workbench_kafka_inbox_status_updated ON workbench_kafka_inbox(status, updated_at, source_topic, source_partition, source_offset); +CREATE INDEX IF NOT EXISTS idx_workbench_kafka_inbox_trace ON workbench_kafka_inbox(trace_id, source_seq); + +CREATE TABLE IF NOT EXISTS workbench_kafka_dlq ( + dlq_seq BIGSERIAL PRIMARY KEY, + source_topic TEXT NOT NULL, + source_partition INTEGER NOT NULL, + source_offset BIGINT NOT NULL, + source_event_id TEXT, + input_sha256 TEXT NOT NULL, + error_code TEXT NOT NULL, + error_message TEXT NOT NULL, + envelope_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_workbench_kafka_dlq_transport ON workbench_kafka_dlq(source_topic, source_partition, source_offset); + +CREATE TABLE IF NOT EXISTS hwlab_kafka_outbox ( + outbox_seq BIGSERIAL PRIMARY KEY, + event_id TEXT NOT NULL UNIQUE, + topic TEXT NOT NULL, + partition_key TEXT NOT NULL, + payload_json TEXT NOT NULL, + headers_json TEXT NOT NULL DEFAULT '{}', + attempt_count INTEGER NOT NULL DEFAULT 0, + next_attempt_at TEXT NOT NULL, + lease_owner TEXT, + lease_expires_at TEXT, + published_at TEXT, + last_error TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_hwlab_kafka_outbox_due ON hwlab_kafka_outbox(published_at, next_attempt_at, lease_expires_at, outbox_seq); + +CREATE OR REPLACE FUNCTION notify_hwlab_workbench_projection() RETURNS trigger AS $$ +BEGIN + PERFORM pg_notify( + 'hwlab_workbench_projection', + json_build_object( + 'sessionId', NEW.session_id, + 'traceId', NEW.trace_id, + 'outboxSeq', NEW.outbox_seq + )::text + ); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_notify_hwlab_workbench_projection ON workbench_projection_outbox; +CREATE TRIGGER trg_notify_hwlab_workbench_projection +AFTER INSERT ON workbench_projection_outbox +FOR EACH ROW EXECUTE FUNCTION notify_hwlab_workbench_projection(); + +INSERT INTO hwlab_schema_migrations (id, schema_version, applied_at, migration_json) +VALUES ( + '0008_workbench_kafka_realtime', + 'runtime-durable-postgres-v8', + CURRENT_TIMESTAMP, + '{"path":"internal/db/migrations/0008_workbench_kafka_realtime.sql","runtime":"cloud-api"}' +) +ON CONFLICT (id) DO UPDATE SET + schema_version = EXCLUDED.schema_version, + migration_json = EXCLUDED.migration_json; diff --git a/internal/db/runtime-store-core.ts b/internal/db/runtime-store-core.ts index 07678476..aa39889c 100644 --- a/internal/db/runtime-store-core.ts +++ b/internal/db/runtime-store-core.ts @@ -421,7 +421,7 @@ export function workbenchInputDelivery(value) { export function workbenchInputStatus(value) { const text = textOr(value, "admitted").toLowerCase().replace(/-/gu, "_"); - return ["admitted", "promoted", "blocked", "failed", "canceled", "completed"].includes(text) ? text : "admitted"; + return ["admitting", "admitted", "promoted", "blocked", "failed", "canceled", "completed"].includes(text) ? text : "admitted"; } export function indexWorkbenchAggregateEvents(events = []) { @@ -1301,7 +1301,7 @@ export function isRuntimeDbSslError(error) { ].some((pattern) => message.includes(pattern)); } -export function summarizeRuntimeSchema(rows) { +export function summarizeRuntimeSchema(rows, requiredSchema = requiredPostgresSchema) { const columnsByTable = new Map(); for (const row of rows) { const table = row.table_name; @@ -1315,7 +1315,7 @@ export function summarizeRuntimeSchema(rows) { const missingTables = []; const missingColumns = []; - for (const [table, columns] of Object.entries(requiredPostgresSchema)) { + for (const [table, columns] of Object.entries(requiredSchema)) { const observed = columnsByTable.get(table); if (!observed) { missingTables.push(table); @@ -1332,7 +1332,7 @@ export function summarizeRuntimeSchema(rows) { return { ready: missingTables.length === 0 && missingColumns.length === 0, checked: true, - requiredTables: Object.keys(requiredPostgresSchema), + requiredTables: Object.keys(requiredSchema), missingTables, missingColumns }; diff --git a/internal/db/runtime-store-memory.ts b/internal/db/runtime-store-memory.ts index 61fe2aff..41cf409e 100644 --- a/internal/db/runtime-store-memory.ts +++ b/internal/db/runtime-store-memory.ts @@ -16,7 +16,6 @@ import { deriveProtocolActorFromMeta } from "../audit/index.mjs"; import { - CLOUD_CORE_MIGRATION_ID, CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS @@ -214,6 +213,20 @@ export class CloudRuntimeStore { }); } + async workbenchTransactionalRealtimeReadiness() { + return { + ready: true, + adapter: RUNTIME_STORE_KIND, + durable: false, + testRuntime: true, + valuesRedacted: true + }; + } + + async assertWorkbenchTransactionalRealtimeReady() { + return this.workbenchTransactionalRealtimeReadiness(); + } + registerGatewaySession(params = {}, requestMeta = {}) { const now = this.now(); const input = asObject(params.gatewaySession ?? params, "gatewaySession"); @@ -665,9 +678,15 @@ export class CloudRuntimeStore { const inputFacts = facts.inputs.map((fact) => memoryWorkbenchSessionInputWithSeq(fact, this.workbenchSessionInputs)); for (const fact of inputFacts) this.workbenchSessionInputs.set(fact.inputId, fact); for (const fact of facts.sessions) this.workbenchSessions.set(fact.sessionId, fact); - for (const fact of facts.messages) this.workbenchMessages.set(fact.messageId, fact); + for (const fact of facts.messages) { + const previous = this.workbenchMessages.get(fact.messageId) ?? null; + this.workbenchMessages.set(fact.messageId, previous && (nonNegativeInteger(previous.projectedSeq) > 0 || previous.sealed === true) ? previous : fact); + } for (const fact of facts.parts) this.workbenchParts.set(fact.partId, fact); - for (const fact of facts.turns) this.workbenchTurns.set(fact.turnId, fact); + for (const fact of facts.turns) { + const previous = this.workbenchTurns.get(fact.turnId) ?? null; + this.workbenchTurns.set(fact.turnId, previous && (nonNegativeInteger(previous.projectedSeq) > 0 || previous.sealed === true) ? previous : fact); + } for (const fact of facts.traceEvents) this.workbenchTraceEvents.set(fact.id, fact); for (const fact of facts.checkpoints) { const previous = this.workbenchProjectionCheckpoints.get(fact.traceId) ?? null; @@ -682,6 +701,57 @@ export class CloudRuntimeStore { }; } + writeWorkbenchSessionAdmissionFact(params = {}, requestMeta = {}) { + const facts = normalizeWorkbenchFacts({ sessions: [params.fact ?? params.session ?? params] }, requestMeta, this.now()); + const fact = facts.sessions[0]; + if (!fact?.sessionId) throw new Error("Workbench session admission fact requires sessionId."); + const previous = this.workbenchSessions.get(fact.sessionId) ?? null; + const projected = previous && (nonNegativeInteger(previous.projectedSeq) > 0 || previous.sealed === true); + const ownership = { + ownerUserId: previous?.ownerUserId ?? fact.ownerUserId ?? null, + ownerRole: previous?.ownerRole ?? fact.ownerRole ?? null, + projectId: previous?.projectId ?? fact.projectId ?? null, + conversationId: previous?.conversationId ?? fact.conversationId ?? null, + threadId: previous?.threadId ?? fact.threadId ?? null + }; + const stored = projected + ? { ...previous, ...ownership } + : { ...previous, ...fact, ...ownership, createdAt: previous?.createdAt ?? fact.createdAt }; + this.workbenchSessions.set(fact.sessionId, stored); + return { written: true, admissionOnly: true, fact: stored, persistence: this.summary(), valuesPrinted: false }; + } + + promoteWorkbenchTurnAdmission(params = {}, requestMeta = {}) { + const facts = normalizeWorkbenchFacts(params.facts ?? params, requestMeta, this.now()); + const sessionFact = facts.sessions[0]; + const inputFact = facts.inputs[0]; + const turnFact = facts.turns[0]; + if (!sessionFact?.sessionId || !inputFact?.inputId || !turnFact?.turnId) throw new Error("Workbench turn admission promotion requires session, input, and turn facts."); + const storedSession = this.writeWorkbenchSessionAdmissionFact({ fact: sessionFact }, requestMeta).fact; + const storedInput = memoryWorkbenchSessionInputWithSeq(inputFact, this.workbenchSessionInputs); + this.workbenchSessionInputs.set(storedInput.inputId, storedInput); + for (const fact of facts.messages) { + const previous = this.workbenchMessages.get(fact.messageId) ?? null; + if (!previous || (nonNegativeInteger(previous.projectedSeq) <= 0 && previous.sealed !== true)) this.workbenchMessages.set(fact.messageId, fact); + } + for (const fact of facts.parts) { + const previous = this.workbenchParts.get(fact.partId) ?? null; + if (!previous || (nonNegativeInteger(previous.projectedSeq) <= 0 && previous.sealed !== true)) this.workbenchParts.set(fact.partId, fact); + } + for (const fact of facts.turns) { + const previous = this.workbenchTurns.get(fact.turnId) ?? null; + if (!previous || (nonNegativeInteger(previous.projectedSeq) <= 0 && previous.sealed !== true)) this.workbenchTurns.set(fact.turnId, fact); + } + return { + written: true, + promoted: true, + outboxCommitted: true, + facts: { ...facts, sessions: [storedSession], inputs: [storedInput] }, + persistence: this.summary(), + valuesPrinted: false + }; + } + queryWorkbenchFacts(params = {}) { const families = workbenchFactFamilySet(params); const sessions = [...this.workbenchSessions.values()].filter((record) => matchesWorkbenchSessionFactQuery(record, params)); diff --git a/internal/db/runtime-store-postgres-kafka.integration.test.ts b/internal/db/runtime-store-postgres-kafka.integration.test.ts new file mode 100644 index 00000000..a3bc7748 --- /dev/null +++ b/internal/db/runtime-store-postgres-kafka.integration.test.ts @@ -0,0 +1,307 @@ +import assert from "node:assert/strict"; +import { randomUUID } from "node:crypto"; +import { test } from "bun:test"; +import pg from "pg"; + +import { buildWorkbenchProjectionEventFacts } from "../cloud/workbench-projection-writer.ts"; +import { PostgresCloudRuntimeStore } from "./runtime-store-postgres.ts"; +import { commitAgentRunKafkaProjection as commitProjection } from "./runtime-store-postgres-kafka.ts"; + +const dbUrl = String(process.env.HWLAB_KAFKA_PROJECTOR_INTEGRATION_DB_URL ?? "").trim(); +const enabled = Boolean(dbUrl) && process.env.HWLAB_KAFKA_PROJECTOR_INTEGRATION_CONFIRM_NON_PRODUCTION === "1"; + +if (!enabled) { + test.skip("Kafka projector real PostgreSQL transaction and fault-injection contract", () => {}); +} else { + test("Kafka projector real PostgreSQL transaction and fault-injection contract", async () => { + const { Pool } = pg; + const pool = new Pool({ connectionString: dbUrl, ssl: false, max: 4 }); + const suffix = randomUUID().replaceAll("-", ""); + const traceId = `trc_it_${suffix}`; + const sessionId = `ses_it_${suffix}`; + const ownerUserId = `usr_it_${suffix}`; + const now = "2026-07-10T12:00:00.000Z"; + const store = new PostgresCloudRuntimeStore({ + dbUrl, + sslMode: "disable", + env: { ...process.env, HWLAB_CLOUD_DB_SSL_MODE: "disable" }, + queryClient: pool, + now: () => now, + logger: { error() {}, warn() {}, info() {} } + }); + + try { + const readiness = await store.readiness(); + assert.equal(readiness.ready, true); + assert.equal(readiness.durable, true); + assert.equal(readiness.migration?.ready, true); + + await pool.query( + "INSERT INTO users (id, username, display_name, role, status, created_at, updated_at) VALUES ($1,$2,$2,'user','active',$3,$3)", + [ownerUserId, `integration-${suffix}`, now] + ); + + await store.writeWorkbenchSessionAdmissionFact({ + fact: { + sessionId, + ownerUserId, + projectId: `prj_it_${suffix}`, + conversationId: `cnv_it_${suffix}`, + threadId: `thread_it_${suffix}`, + status: "queued", + lastTraceId: traceId, + projectedSeq: 0, + sourceSeq: 0, + terminal: false, + sealed: false, + sessionJson: { sessionId, launchContext: { source: "integration-test" }, valuesRedacted: true }, + createdAt: now, + updatedAt: now + } + }); + + const running = await project(store, { + traceId, + sessionId, + sourceEventId: `evt_running_${suffix}`, + sourceSeq: 1, + sourceOffset: "100", + inputSha256: "1".repeat(64), + event: { eventType: "assistant_message", status: "running", assistantText: "partial", createdAt: now } + }); + assert.equal(running.duplicate, false); + assert.equal(running.projectedSeq, 1); + assert.equal(await count(pool, "workbench_kafka_inbox", "trace_id = $1", [traceId]), 1); + assert.equal(await count(pool, "workbench_trace_events", "trace_id = $1", [traceId]), 1); + assert.equal(await count(pool, "workbench_projection_outbox", "trace_id = $1", [traceId]), 2); + assert.equal(await count(pool, "hwlab_kafka_outbox", "partition_key = $1", [traceId]), 1); + + const sameTransport = await project(store, { + traceId, + sessionId, + sourceEventId: `evt_running_${suffix}`, + sourceSeq: 1, + sourceOffset: "100", + inputSha256: "1".repeat(64), + event: { eventType: "assistant_message", status: "running", assistantText: "partial", createdAt: now } + }); + const newOffsetSameEvent = await project(store, { + traceId, + sessionId, + sourceEventId: `evt_running_${suffix}`, + sourceSeq: 1, + sourceOffset: "101", + inputSha256: "1".repeat(64), + event: { eventType: "assistant_message", status: "running", assistantText: "partial", createdAt: now } + }); + assert.equal(sameTransport.duplicate, true); + assert.equal(newOffsetSameEvent.duplicate, true); + assert.equal(await count(pool, "workbench_kafka_inbox", "trace_id = $1", [traceId]), 1); + assert.equal(await count(pool, "workbench_projection_outbox", "trace_id = $1", [traceId]), 2); + + const conflict = await project(store, { + traceId, + sessionId, + sourceEventId: `evt_running_${suffix}`, + sourceSeq: 1, + sourceOffset: "102", + inputSha256: "2".repeat(64), + event: { eventType: "assistant_message", status: "running", assistantText: "mutated", createdAt: now } + }); + assert.equal(conflict.conflict, true); + assert.equal(await count(pool, "workbench_kafka_dlq", "source_event_id = $1", [`evt_running_${suffix}`]), 1); + const projectedInbox = await pool.query( + "SELECT status, projected_seq FROM workbench_kafka_inbox WHERE source_topic = $1 AND source_event_id = $2", + ["agentrun.event.v1", `evt_running_${suffix}`] + ); + assert.deepEqual(projectedInbox.rows[0], { status: "projected", projected_seq: 1 }); + + await pool.query(` + CREATE OR REPLACE FUNCTION issue_2464_delay_inbox_insert() RETURNS trigger AS $$ + BEGIN + IF NEW.source_topic = 'agentrun.event.v1' AND NEW.source_partition = 0 AND NEW.source_offset = 300 THEN + PERFORM pg_sleep(0.25); + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + DROP TRIGGER IF EXISTS issue_2464_delay_inbox_insert ON workbench_kafka_inbox; + CREATE TRIGGER issue_2464_delay_inbox_insert BEFORE INSERT ON workbench_kafka_inbox + FOR EACH ROW EXECUTE FUNCTION issue_2464_delay_inbox_insert(); + `); + try { + const concurrent = await Promise.allSettled([ + project(store, { + traceId: `trc_transport_a_${suffix}`, + sessionId: `ses_transport_a_${suffix}`, + sourceEventId: `evt_transport_a_${suffix}`, + sourceSeq: 1, + sourceOffset: "300", + inputSha256: "6".repeat(64), + event: { eventType: "backend_status", status: "running", createdAt: now } + }), + project(store, { + traceId: `trc_transport_b_${suffix}`, + sessionId: `ses_transport_b_${suffix}`, + sourceEventId: `evt_transport_b_${suffix}`, + sourceSeq: 1, + sourceOffset: "300", + inputSha256: "7".repeat(64), + event: { eventType: "backend_status", status: "running", createdAt: now } + }) + ]); + assert.deepEqual(concurrent.map((result) => result.status), ["fulfilled", "fulfilled"]); + const values = concurrent.map((result) => { + if (result.status !== "fulfilled") throw result.reason; + return result.value; + }); + assert.equal(values.filter((result) => result.conflict === true).length, 1); + assert.equal(values.filter((result) => result.conflict !== true && result.duplicate !== true).length, 1); + assert.equal(await count(pool, "workbench_kafka_inbox", "source_topic = $1 AND source_partition = 0 AND source_offset = 300", ["agentrun.event.v1"]), 1); + assert.equal(await count(pool, "workbench_kafka_dlq", "source_topic = $1 AND source_partition = 0 AND source_offset = 300", ["agentrun.event.v1"]), 1); + } finally { + await pool.query("DROP TRIGGER IF EXISTS issue_2464_delay_inbox_insert ON workbench_kafka_inbox; DROP FUNCTION IF EXISTS issue_2464_delay_inbox_insert()"); + } + + const faultTraceId = `trc_fault_${suffix}`; + const faultSessionId = `ses_fault_${suffix}`; + const faultStore = new PostgresCloudRuntimeStore({ + dbUrl, + sslMode: "disable", + env: { ...process.env, HWLAB_CLOUD_DB_SSL_MODE: "disable" }, + queryClient: faultInjectingPool(pool, /^INSERT INTO workbench_projection_outbox/u), + now: () => now, + logger: { error() {}, warn() {}, info() {} } + }); + await assert.rejects(project(faultStore, { + traceId: faultTraceId, + sessionId: faultSessionId, + sourceEventId: `evt_fault_${suffix}`, + sourceSeq: 1, + sourceOffset: "200", + inputSha256: "3".repeat(64), + event: { eventType: "assistant_message", status: "running", assistantText: "must roll back", createdAt: now } + }), /injected projection outbox failure/u); + assert.equal(await count(pool, "workbench_kafka_inbox", "trace_id = $1", [faultTraceId]), 0); + assert.equal(await count(pool, "workbench_trace_events", "trace_id = $1", [faultTraceId]), 0); + assert.equal(await count(pool, "workbench_projection_outbox", "trace_id = $1", [faultTraceId]), 0); + assert.equal(await count(pool, "hwlab_kafka_outbox", "partition_key = $1", [faultTraceId]), 0); + + const terminal = await project(store, { + traceId, + sessionId, + sourceEventId: `evt_terminal_${suffix}`, + sourceSeq: 2, + sourceOffset: "103", + inputSha256: "4".repeat(64), + event: { + eventType: "terminal_status", + status: "completed", + terminal: true, + assistantText: "authoritative final", + finalResponse: { text: "authoritative final", status: "completed" }, + startedAt: now, + finishedAt: "2026-07-10T12:00:01.000Z", + durationMs: 1000, + createdAt: "2026-07-10T12:00:01.000Z" + } + }); + assert.equal(terminal.projectedSeq, 2); + const sealed = await pool.query( + "SELECT projected_seq, terminal, sealed, checkpoint_json FROM workbench_projection_checkpoints WHERE trace_id = $1", + [traceId] + ); + assert.equal(sealed.rows[0].projected_seq, 2); + assert.equal(sealed.rows[0].terminal, true); + assert.equal(sealed.rows[0].sealed, true); + assert.equal(JSON.parse(sealed.rows[0].checkpoint_json).finalResponse.text, "authoritative final"); + assert.equal(await count(pool, "workbench_projection_outbox", "trace_id = $1", [traceId]), 5); + + const late = await project(store, { + traceId, + sessionId, + sourceEventId: `evt_late_${suffix}`, + sourceSeq: 3, + sourceOffset: "104", + inputSha256: "5".repeat(64), + event: { eventType: "backend_status", status: "running", terminal: false, createdAt: "2026-07-10T12:00:02.000Z" } + }); + assert.equal(late.suppressedAfterSeal, true); + assert.equal(late.projectedSeq, 2); + assert.equal(await count(pool, "workbench_trace_events", "trace_id = $1", [traceId]), 2); + assert.equal(await count(pool, "workbench_projection_outbox", "trace_id = $1", [traceId]), 5); + const lateInbox = await pool.query( + "SELECT status, projected_seq FROM workbench_kafka_inbox WHERE source_topic = $1 AND source_event_id = $2", + ["agentrun.event.v1", `evt_late_${suffix}`] + ); + assert.deepEqual(lateInbox.rows[0], { status: "projected", projected_seq: 2 }); + + const firstSync = await store.readAtomicWorkbenchProjectionSync({ traceId, afterOutboxSeq: 0, limit: 100 }); + assert.equal(firstSync.events.length, 5); + assert.equal(firstSync.hasMore, false); + assert.equal(firstSync.cursorOutboxSeq, firstSync.cutoffOutboxSeq); + assert.equal(new Set(firstSync.events.map((event) => event.outboxSeq)).size, firstSync.events.length); + assert.equal(firstSync.facts.checkpoints[0].projectedSeq, 2); + assert.equal(firstSync.facts.turns[0].finalResponse.text, "authoritative final"); + const caughtUp = await store.readAtomicWorkbenchProjectionSync({ traceId, afterOutboxSeq: firstSync.cursorOutboxSeq, limit: 100, deltaOnly: true }); + assert.equal(caughtUp.events.length, 0); + assert.equal(caughtUp.cursorOutboxSeq, firstSync.cursorOutboxSeq); + } finally { + await pool.end(); + } + }, 30_000); +} + +async function project(store, { traceId, sessionId, sourceEventId, sourceSeq, sourceOffset, inputSha256, event }) { + const canonicalEvent = { + schema: "agentrun.event.v1", + eventId: sourceEventId, + sourceSeq, + traceId, + hwlabSessionId: sessionId, + runId: `run_${traceId}`, + commandId: `cmd_${traceId}`, + event + }; + const projectedEvent = { ...event, traceId, sessionId, sourceEventId, sourceSeq, runId: canonicalEvent.runId, commandId: canonicalEvent.commandId }; + return commitProjection(store, { + transport: { sourceTopic: "agentrun.event.v1", sourcePartition: 0, sourceOffset, sourceKey: canonicalEvent.runId, inputSha256 }, + canonicalEvent, + projectedEvent, + requestMeta: { traceId, sessionId, valuesPrinted: false }, + factsFactory: ({ projectedSeq, previousCheckpoint, projectedAt }) => buildWorkbenchProjectionEventFacts({ projectedSeq, previousCheckpoint, projectedAt, event: projectedEvent }), + hwlabEvent: { schema: "hwlab.event.v1", eventId: `hwlab:${sourceEventId}`, traceId, sessionId, sourceEventId, sourceSeq, valuesPrinted: false }, + hwlabTopic: "hwlab.event.v1", + partitionKey: traceId, + headers: { sourceEventId } + }); +} + +function faultInjectingPool(pool, pattern) { + return { + async connect() { + const client = await pool.connect(); + let injected = false; + return { + async query(sql, params) { + if (!injected && pattern.test(String(sql))) { + injected = true; + const error = new Error("injected projection outbox failure"); + error.code = "XX2464"; + throw error; + } + return client.query(sql, params); + }, + release() { + client.release(); + } + }; + } + }; +} + +async function count(pool, table, where, params) { + if (!/^[a-z_]+$/u.test(table)) throw new Error("unsafe test table"); + const result = await pool.query(`SELECT COUNT(*)::int AS count FROM ${table} WHERE ${where}`, params); + return Number(result.rows[0].count); +} diff --git a/internal/db/runtime-store-postgres-kafka.test.ts b/internal/db/runtime-store-postgres-kafka.test.ts new file mode 100644 index 00000000..f1a6eee7 --- /dev/null +++ b/internal/db/runtime-store-postgres-kafka.test.ts @@ -0,0 +1,244 @@ +import assert from "node:assert/strict"; +import { test } from "bun:test"; + +import { buildWorkbenchProjectionEventFacts } from "../cloud/workbench-projection-writer.ts"; +import { projectionOutboxRealtimeEvents } from "../cloud/server-workbench-realtime-http.ts"; +import { + claimHwlabKafkaOutbox, + commitAgentRunKafkaProjection, + completeHwlabKafkaOutbox, + projectionOutboxEventId, + recordFailedAgentRunKafkaMessage, + recordIgnoredAgentRunKafkaMessage, + retryHwlabKafkaOutbox +} from "./runtime-store-postgres-kafka.ts"; + +test("malformed duplicate source event at a new offset is durably DLQed without mutating projected inbox", async () => { + const queries = []; + const client = { + async query(sql, params) { + queries.push({ sql, params }); + if (sql.startsWith("SELECT pg_advisory_xact_lock")) return { rows: [] }; + if (sql.startsWith("SELECT source_topic")) return { rows: [{ source_topic: "agentrun.event.v1", source_partition: 0, source_offset: "10", input_sha256: "prior-hash", status: "projected", projected_seq: 7 }] }; + if (sql.startsWith("INSERT INTO workbench_kafka_dlq")) return { rows: [] }; + throw new Error(`unexpected SQL: ${sql}`); + } + }; + const store = transactionStore(client); + const result = await recordFailedAgentRunKafkaMessage(store, { + transport: { sourceTopic: "agentrun.event.v1", sourcePartition: 0, sourceOffset: "11", sourceKey: "run-1", inputSha256: "new-hash" }, + sourceEventId: "evt-duplicate", + envelope: { eventId: "evt-duplicate", valuesPrinted: false }, + error: { code: "workbench_kafka_schema_invalid", message: "malformed" } + }); + + assert.equal(result.recorded, true); + assert.equal(result.conflict, true); + assert.equal(result.priorStatus, "projected"); + assert.equal(queries[0].params[0], "workbench-kafka-inbox:agentrun.event.v1:transport:0:11"); + assert.equal(queries[1].params[0], "workbench-kafka-inbox:agentrun.event.v1:event:evt-duplicate"); + assert.equal(queries.some(({ sql }) => sql.startsWith("UPDATE workbench_kafka_inbox")), false); + assert.equal(queries.some(({ sql }) => sql.startsWith("INSERT INTO workbench_kafka_dlq")), true); +}); + +test("ignored Kafka messages acquire transport then event identity locks", async () => { + const queries = []; + const client = { + async query(sql, params) { + queries.push({ sql, params }); + if (sql.startsWith("SELECT pg_advisory_xact_lock")) return { rows: [] }; + if (sql.startsWith("SELECT input_sha256")) return { rows: [] }; + if (sql.startsWith("INSERT INTO workbench_kafka_inbox")) return { rows: [] }; + throw new Error(`unexpected SQL: ${sql}`); + } + }; + const result = await recordIgnoredAgentRunKafkaMessage(transactionStore(client), { + transport: { sourceTopic: "agentrun.event.v1", sourcePartition: 2, sourceOffset: "31", sourceKey: "run-ignored", inputSha256: "hash-ignored" }, + envelope: { eventId: "evt-ignored", sourceSeq: 1, runId: "run-ignored" }, + reason: "hwlab-correlation-absent" + }); + + assert.equal(result.ignored, true); + assert.equal(result.duplicate, false); + assert.equal(queries[0].params[0], "workbench-kafka-inbox:agentrun.event.v1:transport:2:31"); + assert.equal(queries[1].params[0], "workbench-kafka-inbox:agentrun.event.v1:event:evt-ignored"); +}); + +test("projection outbox derives a stable non-null identity when a fact has no source event", () => { + const input = { + fact: { traceId: "trc-derived", projectedSeq: 3, sourceSeq: 2 }, + event: { aggregateId: "trc-derived", aggregateSeq: 3, projectionRevision: 3 }, + family: "traceEvents", + entityId: "wte-derived", + commitType: "event" + }; + const first = projectionOutboxEventId(input); + assert.match(first, /^projection:[a-f0-9]{64}$/u); + assert.equal(projectionOutboxEventId(input), first); + assert.notEqual(projectionOutboxEventId({ ...input, event: { ...input.event, projectionRevision: 4 } }), first); +}); + +test("Kafka projection preserves admission owner and scope fields", async () => { + let persistedSession = null; + const client = { + async query(sql) { + if (sql.startsWith("SELECT pg_advisory_xact_lock")) return { rows: [] }; + if (sql.startsWith("SELECT source_topic")) return { rows: [] }; + if (sql.startsWith("INSERT INTO workbench_kafka_inbox")) return { rows: [] }; + if (sql.startsWith("SELECT checkpoint_json")) return { rows: [] }; + if (sql.startsWith("SELECT owner_user_id")) return { rows: [{ owner_user_id: "usr-owner", project_id: "prj-owner", conversation_id: "cnv-owner", thread_id: "thread-owner", session_json: { sessionId: "ses-owner", ownerUserId: "usr-owner", ownerRole: "user", launchContext: { source: "workbench" } } }] }; + if (sql.startsWith("SELECT GREATEST")) return { rows: [{ max_projected_seq: 0 }] }; + if (sql.startsWith("INSERT INTO hwlab_kafka_outbox")) return { rows: [] }; + if (sql.startsWith("UPDATE workbench_kafka_inbox")) return { rows: [] }; + throw new Error(`unexpected SQL: ${sql}`); + } + }; + const store = { + ...transactionStore(client), + memory: { writeWorkbenchFacts() {} }, + async persistWorkbenchAggregateEvent(event) { return { ...event, eventSeq: 1, aggregateSeq: 1, projectionRevision: 1 }; }, + async persistWorkbenchSessionInputFact() {}, + async persistWorkbenchSessionFact(fact) { persistedSession = fact; }, + async persistWorkbenchMessageFact() {}, + async persistWorkbenchPartFact() {}, + async persistWorkbenchTurnFact() {}, + async persistWorkbenchTraceEventFact() {}, + async persistWorkbenchProjectionCheckpoint() {}, + async persistWorkbenchProjectionOutbox() {} + }; + await commitAgentRunKafkaProjection(store, { + transport: { sourceTopic: "agentrun.event.v1", sourcePartition: 0, sourceOffset: "12", sourceKey: "run-owner", inputSha256: "hash-owner" }, + canonicalEvent: { eventId: "evt-owner", sourceSeq: 1, traceId: "trc-owner", hwlabSessionId: "ses-owner", runId: "run-owner", commandId: "cmd-owner" }, + projectedEvent: { traceId: "trc-owner", sessionId: "ses-owner" }, + factsFactory: ({ projectedSeq, projectedAt }) => ({ + written: true, + facts: { + sessions: [{ sessionId: "ses-owner", status: "running", lastTraceId: "trc-owner", projectedSeq, sourceSeq: 1, sourceEventId: "evt-owner", terminal: false, sealed: false, updatedAt: projectedAt }], + messages: [], parts: [], turns: [], traceEvents: [], checkpoints: [] + } + }), + hwlabEvent: { eventId: "hwlab:evt-owner" }, + hwlabTopic: "hwlab.event.v1", + partitionKey: "trc-owner", + headers: {} + }); + + assert.equal(persistedSession.ownerUserId, "usr-owner"); + assert.equal(persistedSession.ownerRole, "user"); + assert.equal(persistedSession.projectId, "prj-owner"); + assert.equal(persistedSession.conversationId, "cnv-owner"); + assert.equal(persistedSession.threadId, "thread-owner"); + assert.deepEqual(persistedSession.launchContext, { source: "workbench" }); +}); + +test("terminal Kafka transaction persists sealed turn and immutable SSE snapshot", async () => { + let persistedTurn = null; + const outbox = []; + const client = { + async query(sql) { + if (sql.startsWith("SELECT pg_advisory_xact_lock")) return { rows: [] }; + if (sql.startsWith("SELECT source_topic")) return { rows: [] }; + if (sql.startsWith("INSERT INTO workbench_kafka_inbox")) return { rows: [] }; + if (sql.startsWith("SELECT checkpoint_json")) return { rows: [{ checkpoint_json: { traceId: "trc_terminal", sessionId: "ses_terminal", assistantText: "final body", finalResponse: { text: "final body", status: "running" }, projectedSeq: 2, sourceSeq: 2, terminal: false, sealed: false } }] }; + if (sql.startsWith("SELECT owner_user_id")) return { rows: [{ owner_user_id: "usr-owner", session_json: { sessionId: "ses_terminal", ownerUserId: "usr-owner" } }] }; + if (sql.startsWith("SELECT GREATEST")) return { rows: [{ max_projected_seq: 2 }] }; + if (sql.startsWith("INSERT INTO hwlab_kafka_outbox") || sql.startsWith("UPDATE workbench_kafka_inbox")) return { rows: [] }; + throw new Error(`unexpected SQL: ${sql}`); + } + }; + const store = { + ...transactionStore(client), + memory: { writeWorkbenchFacts() {} }, + async persistWorkbenchAggregateEvent(event) { return { ...event, eventSeq: 3, aggregateSeq: 3, projectionRevision: 3 }; }, + async persistWorkbenchSessionInputFact() {}, async persistWorkbenchSessionFact() {}, async persistWorkbenchMessageFact() {}, async persistWorkbenchPartFact() {}, + async persistWorkbenchTurnFact(fact) { persistedTurn = fact; }, async persistWorkbenchTraceEventFact() {}, async persistWorkbenchProjectionCheckpoint() {}, + async persistWorkbenchProjectionOutbox(record) { outbox.push(record); } + }; + await commitAgentRunKafkaProjection(store, { + transport: { sourceTopic: "agentrun.event.v1", sourcePartition: 0, sourceOffset: "20", inputSha256: "hash-terminal" }, + canonicalEvent: { eventId: "evt-terminal", sourceSeq: 3, traceId: "trc_terminal", hwlabSessionId: "ses_terminal", runId: "run-terminal", commandId: "cmd-terminal" }, + projectedEvent: { traceId: "trc_terminal", sessionId: "ses_terminal", sourceEventId: "evt-terminal", sourceSeq: 3, type: "result", eventType: "terminal", status: "completed", terminal: true, createdAt: "2026-07-10T10:00:03.000Z" }, + factsFactory: ({ projectedSeq, previousCheckpoint, projectedAt }) => buildWorkbenchProjectionEventFacts({ projectedSeq, previousCheckpoint, projectedAt, event: { traceId: "trc_terminal", sessionId: "ses_terminal", sourceEventId: "evt-terminal", sourceSeq: 3, type: "result", eventType: "terminal", status: "completed", terminal: true, createdAt: "2026-07-10T10:00:03.000Z" } }), + hwlabEvent: { eventId: "hwlab:evt-terminal" }, hwlabTopic: "hwlab.event.v1", partitionKey: "trc_terminal", headers: {} + }); + + assert.equal(persistedTurn.terminal, true); + assert.equal(persistedTurn.sealed, true); + assert.equal(persistedTurn.finalResponse.text, "final body"); + const turnRow = outbox.find((row) => row.entityFamily === "turns"); + assert.equal(turnRow.payload.fact.assistantText, "final body"); + const [sse] = projectionOutboxRealtimeEvents({ events: [{ ...turnRow, outboxSeq: 9 }], facts: {} }); + assert.equal(sse.name, "workbench.turn.snapshot"); + assert.equal(sse.payload.turn.finalResponse.text, "final body"); +}); + +test("late second terminal is acknowledged without facts or either projection outbox", async () => { + const queries = []; + const client = { + async query(sql, params) { + queries.push({ sql, params }); + if (sql.startsWith("SELECT pg_advisory_xact_lock")) return { rows: [] }; + if (sql.startsWith("SELECT source_topic")) return { rows: [] }; + if (sql.startsWith("INSERT INTO workbench_kafka_inbox")) return { rows: [] }; + if (sql.startsWith("SELECT checkpoint_json")) return { rows: [{ checkpoint_json: { traceId: "trc_sealed", projectedSeq: 9, terminal: true, sealed: true } }] }; + if (sql.startsWith("SELECT owner_user_id")) return { rows: [] }; + if (sql.startsWith("SELECT GREATEST")) return { rows: [{ max_projected_seq: 9 }] }; + if (sql.startsWith("INSERT INTO hwlab_kafka_outbox") || sql.startsWith("UPDATE workbench_kafka_inbox")) return { rows: [] }; + throw new Error(`unexpected SQL: ${sql}`); + } + }; + const store = { ...transactionStore(client), memory: { writeWorkbenchFacts() { throw new Error("suppressed facts must not reach memory"); } } }; + const result = await commitAgentRunKafkaProjection(store, { + transport: { sourceTopic: "agentrun.event.v1", sourcePartition: 0, sourceOffset: "21", inputSha256: "hash-late" }, + canonicalEvent: { eventId: "evt-late", sourceSeq: 10, traceId: "trc_sealed", hwlabSessionId: "ses_sealed" }, + projectedEvent: { traceId: "trc_sealed", sessionId: "ses_sealed" }, + factsFactory: ({ projectedSeq, previousCheckpoint, projectedAt }) => buildWorkbenchProjectionEventFacts({ + projectedSeq, + previousCheckpoint, + projectedAt, + event: { traceId: "trc_sealed", sessionId: "ses_sealed", sourceEventId: "evt-late", sourceSeq: 10, type: "result", eventType: "terminal", status: "failed", terminal: true } + }), + hwlabEvent: { eventId: "hwlab:evt-late" }, + hwlabTopic: "hwlab.event.v1", + partitionKey: "trc_sealed", + headers: {} + }); + + assert.equal(result.suppressedAfterSeal, true); + assert.equal(result.projectionWritten, false); + assert.equal(result.projectedSeq, 9); + assert.equal(queries.some(({ sql }) => sql.startsWith("INSERT INTO hwlab_kafka_outbox")), false); + const inboxUpdate = queries.find(({ sql }) => sql.startsWith("UPDATE workbench_kafka_inbox")); + assert.equal(inboxUpdate.params[0], 9); +}); + +test("HWLAB outbox claim and completion use attempt and lease fencing", async () => { + const calls = []; + const row = { outbox_seq: 3, event_id: "evt-3", topic: "hwlab.event.v1", partition_key: "trc-3", payload_json: {}, headers_json: {}, attempt_count: 4, lease_owner: "relay-a", lease_expires_at: "2026-07-10T10:05:00.000Z", next_attempt_at: "2026-07-10T10:00:00.000Z", created_at: "2026-07-10T10:00:00.000Z" }; + const store = { + now: () => "2026-07-10T10:04:00.000Z", + async withDurableTransaction(_label, operation) { return operation({ query: async (sql, params) => { calls.push({ sql, params }); return { rows: [row] }; } }); }, + async query(sql, params) { calls.push({ sql, params }); return { rows: [{ outbox_seq: 3 }] }; } + }; + const [item] = await claimHwlabKafkaOutbox(store, { owner: "relay-a", leaseMs: 30000, limit: 1 }); + await completeHwlabKafkaOutbox(store, item); + await retryHwlabKafkaOutbox(store, item, new Error("retry"), "2026-07-10T10:04:30.000Z"); + + assert.match(calls[0].sql, /attempt_count = o\.attempt_count \+ 1/u); + assert.match(calls[0].sql, /NOT EXISTS/u); + assert.match(calls[1].sql, /lease_owner = \$3 AND attempt_count = \$4 AND lease_expires_at = \$5 AND lease_expires_at > \$1/u); + assert.deepEqual(calls[1].params.slice(2), ["relay-a", 4, "2026-07-10T10:05:00.000Z"]); + assert.match(calls[2].sql, /lease_owner = \$5 AND attempt_count = \$6 AND lease_expires_at = \$7 AND lease_expires_at > \$3/u); +}); + +test("stale HWLAB outbox fencing token cannot complete a claim", async () => { + const store = { now: () => "2026-07-10T10:06:00.000Z", async query() { return { rows: [] }; } }; + await assert.rejects(completeHwlabKafkaOutbox(store, { outboxSeq: 3, leaseOwner: "relay-old", attemptCount: 1, leaseExpiresAt: "2026-07-10T10:05:00.000Z" }), (error) => error?.code === "hwlab_kafka_outbox_lease_conflict"); +}); + +function transactionStore(client) { + return { + now: () => "2026-07-10T10:00:00.000Z", + async withDurableTransaction(_label, operation) { return operation(client); } + }; +} diff --git a/internal/db/runtime-store-postgres-kafka.ts b/internal/db/runtime-store-postgres-kafka.ts new file mode 100644 index 00000000..a3744719 --- /dev/null +++ b/internal/db/runtime-store-postgres-kafka.ts @@ -0,0 +1,418 @@ +/* + * Durable AgentRun Kafka projector storage operations. + * Kafka I/O is intentionally outside every transaction in this module. + */ +import { createHash } from "node:crypto"; + +import { + indexWorkbenchAggregateEvents, + inputFactWithAggregateSeq, + nonNegativeInteger, + normalizeWorkbenchAggregateEventsForFacts, + normalizeWorkbenchFacts, + parseJsonColumn, + stableJson, + textOr +} from "./runtime-store-core.ts"; + +export async function commitAgentRunKafkaProjection(store, params = {}) { + const transport = normalizeTransport(params.transport); + const canonicalEvent = objectValue(params.canonicalEvent); + const projectedEvent = objectValue(params.projectedEvent); + const sourceEventId = requiredText(canonicalEvent.eventId, "canonical AgentRun eventId"); + const traceId = requiredText(projectedEvent.traceId ?? canonicalEvent.traceId, "canonical AgentRun traceId"); + const sessionId = text(projectedEvent.sessionId ?? canonicalEvent.hwlabSessionId); + const sourceSeq = nonNegativeInteger(canonicalEvent.sourceSeq ?? canonicalEvent.event?.seq); + const inputSha256 = requiredText(transport.inputSha256, "Kafka input sha256"); + const now = store.now(); + const result = await store.withDurableTransaction("workbench.kafka-projection.commit", async (client) => { + await lockKafkaInboxIdentity(client, transport, sourceEventId); + await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`workbench-projection:${traceId}`]); + const existing = await client.query( + "SELECT source_topic, source_partition, source_offset, source_event_id, input_sha256, status, projected_seq FROM workbench_kafka_inbox WHERE (source_topic = $1 AND source_partition = $2 AND source_offset = $3) OR (source_topic = $1 AND source_event_id = $4) ORDER BY created_at ASC LIMIT 1", + [transport.sourceTopic, transport.sourcePartition, transport.sourceOffset, sourceEventId] + ); + const prior = existing.rows?.[0] ?? null; + if (prior) { + if (prior.input_sha256 !== inputSha256) { + const diagnostic = { code: "workbench_kafka_inbox_hash_conflict", message: "Kafka transport or source event identity was reused with a different payload hash.", valuesRedacted: true }; + if (prior.status !== "projected") { + await client.query( + "UPDATE workbench_kafka_inbox SET status = 'failed', error_json = $1, updated_at = $2 WHERE source_topic = $3 AND source_partition = $4 AND source_offset = $5", + [stableJson(diagnostic), now, prior.source_topic, prior.source_partition, prior.source_offset] + ); + } + await insertDlq(client, { ...transport, sourceEventId, inputSha256, error: diagnostic, envelope: redactedEnvelope(canonicalEvent), createdAt: now }); + return { conflict: true, diagnostic, duplicate: false, projectedSeq: nonNegativeInteger(prior.projected_seq) }; + } + return { duplicate: true, conflict: false, status: prior.status, projectedSeq: nonNegativeInteger(prior.projected_seq) }; + } + await client.query( + "INSERT INTO workbench_kafka_inbox (source_topic, source_partition, source_offset, source_key, source_event_id, input_sha256, trace_id, session_id, run_id, command_id, source_seq, status, projected_seq, envelope_json, error_json, created_at, updated_at, projected_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,'processing',NULL,$12,'{}',$13,$13,NULL)", + [transport.sourceTopic, transport.sourcePartition, transport.sourceOffset, transport.sourceKey, sourceEventId, inputSha256, traceId, sessionId, text(canonicalEvent.runId), text(canonicalEvent.commandId), sourceSeq, stableJson(redactedEnvelope(canonicalEvent)), now] + ); + const checkpointResult = await client.query("SELECT checkpoint_json FROM workbench_projection_checkpoints WHERE trace_id = $1 FOR UPDATE", [traceId]); + const previousCheckpoint = parseJsonColumn(checkpointResult.rows?.[0]?.checkpoint_json, null); + const sessionResult = sessionId + ? await client.query("SELECT owner_user_id, project_id, conversation_id, thread_id, session_json FROM workbench_sessions WHERE session_id = $1 FOR UPDATE", [sessionId]) + : { rows: [] }; + const existingSession = sessionResult.rows?.[0] ?? null; + const maxResult = await client.query( + "SELECT GREATEST(COALESCE((SELECT MAX(projected_seq) FROM workbench_trace_events WHERE trace_id = $1), 0), COALESCE((SELECT projected_seq FROM workbench_projection_checkpoints WHERE trace_id = $1), 0))::int AS max_projected_seq", + [traceId] + ); + const projectedSeq = nonNegativeInteger(maxResult.rows?.[0]?.max_projected_seq) + 1; + if (typeof params.factsFactory !== "function") throw codedError("workbench_kafka_facts_factory_required", "Kafka projection requires a pure facts factory."); + const built = await params.factsFactory({ projectedSeq, previousCheckpoint, projectedAt: now }); + let facts = null; + let persistedEvents = []; + if (built?.written !== false) { + facts = normalizeWorkbenchFacts(mergeKafkaSessionIdentity(built?.facts ?? {}, existingSession), params.requestMeta ?? {}, now); + const persisted = await persistFacts(store, client, facts, params.requestMeta ?? {}); + persistedEvents = persisted.events; + } + const committedProjectedSeq = built?.written === false + ? nonNegativeInteger(built?.projectedSeq) || nonNegativeInteger(previousCheckpoint?.projectedSeq) || projectedSeq + : projectedSeq; + const projectionWritten = built?.written !== false; + if (projectionWritten) { + const hwlabEvent = objectValue(params.hwlabEvent); + const hwlabEventId = requiredText(hwlabEvent.eventId, "HWLAB Kafka eventId"); + await client.query( + "INSERT INTO hwlab_kafka_outbox (event_id, topic, partition_key, payload_json, headers_json, attempt_count, next_attempt_at, lease_owner, lease_expires_at, published_at, last_error, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,0,$6,NULL,NULL,NULL,NULL,$6,$6) ON CONFLICT (event_id) DO NOTHING", + [hwlabEventId, requiredText(params.hwlabTopic, "HWLAB Kafka topic"), requiredText(params.partitionKey, "HWLAB Kafka partition key"), stableJson(hwlabEvent), stableJson(objectValue(params.headers)), now] + ); + } + await client.query( + "UPDATE workbench_kafka_inbox SET status = 'projected', projected_seq = $1, updated_at = $2, projected_at = $2 WHERE source_topic = $3 AND source_partition = $4 AND source_offset = $5", + [committedProjectedSeq, now, transport.sourceTopic, transport.sourcePartition, transport.sourceOffset] + ); + return { duplicate: false, conflict: false, projectedSeq: committedProjectedSeq, facts, events: persistedEvents, projectionWritten, suppressedAfterSeal: built?.suppressedAfterSeal === true }; + }); + if (result.facts) store.memory.writeWorkbenchFacts({ facts: result.facts }, params.requestMeta ?? {}); + return { ...result, transport, sourceEventId, traceId, sessionId, sourceSeq, valuesPrinted: false }; +} + +export async function recordFailedAgentRunKafkaMessage(store, params = {}) { + const transport = normalizeTransport(params.transport); + const now = store.now(); + const error = redactedError(params.error); + const sourceEventId = text(params.sourceEventId); + const inputSha256 = requiredText(transport.inputSha256, "Kafka input sha256"); + const result = await store.withDurableTransaction("workbench.kafka-projection.dlq", async (client) => { + await lockKafkaInboxIdentity(client, transport, sourceEventId); + const existing = await client.query("SELECT source_topic, source_partition, source_offset, input_sha256, status, projected_seq FROM workbench_kafka_inbox WHERE (source_topic = $1 AND source_partition = $2 AND source_offset = $3) OR ($4::text IS NOT NULL AND source_topic = $1 AND source_event_id = $4) ORDER BY created_at ASC LIMIT 1 FOR UPDATE", [transport.sourceTopic, transport.sourcePartition, transport.sourceOffset, sourceEventId]); + const prior = existing.rows?.[0] ?? null; + if (prior) { + const sameTransport = prior.source_partition === transport.sourcePartition && String(prior.source_offset) === transport.sourceOffset; + const diagnostic = prior.input_sha256 !== inputSha256 + ? { ...error, code: "workbench_kafka_inbox_hash_conflict" } + : sameTransport ? error : { ...error, code: "workbench_kafka_inbox_source_event_duplicate" }; + if (prior.status !== "projected") { + await client.query("UPDATE workbench_kafka_inbox SET status = 'failed', error_json = $1, updated_at = $2 WHERE source_topic = $3 AND source_partition = $4 AND source_offset = $5", [stableJson(diagnostic), now, prior.source_topic, prior.source_partition, prior.source_offset]); + } + await insertDlq(client, { ...transport, sourceEventId, inputSha256, error: diagnostic, envelope: redactedEnvelope(params.envelope), createdAt: now }); + return { duplicate: sameTransport, conflict: !sameTransport || prior.input_sha256 !== inputSha256, priorStatus: prior.status, projectedSeq: nonNegativeInteger(prior.projected_seq), diagnostic }; + } + await client.query( + "INSERT INTO workbench_kafka_inbox (source_topic, source_partition, source_offset, source_key, source_event_id, input_sha256, trace_id, session_id, run_id, command_id, source_seq, status, projected_seq, envelope_json, error_json, created_at, updated_at, projected_at) VALUES ($1,$2,$3,$4,$5,$6,NULL,NULL,NULL,NULL,0,'failed',NULL,$7,$8,$9,$9,NULL) ON CONFLICT (source_topic, source_partition, source_offset) DO UPDATE SET status = 'failed', error_json = EXCLUDED.error_json, updated_at = EXCLUDED.updated_at", + [transport.sourceTopic, transport.sourcePartition, transport.sourceOffset, transport.sourceKey, sourceEventId, inputSha256, stableJson(redactedEnvelope(params.envelope)), stableJson(error), now] + ); + await insertDlq(client, { ...transport, sourceEventId, inputSha256, error, envelope: redactedEnvelope(params.envelope), createdAt: now }); + return { duplicate: false, conflict: false, priorStatus: null, diagnostic: error }; + }); + return { recorded: true, transport, sourceEventId, error: result.diagnostic, ...result, valuesPrinted: false }; +} + +export async function recordIgnoredAgentRunKafkaMessage(store, params = {}) { + const transport = normalizeTransport(params.transport); + const envelope = objectValue(params.envelope); + const sourceEventId = requiredText(envelope.eventId, "canonical AgentRun eventId"); + const inputSha256 = requiredText(transport.inputSha256, "Kafka input sha256"); + const now = store.now(); + return store.withDurableTransaction("workbench.kafka-projection.ignore", async (client) => { + await lockKafkaInboxIdentity(client, transport, sourceEventId); + const existing = await client.query("SELECT input_sha256, status FROM workbench_kafka_inbox WHERE (source_topic = $1 AND source_partition = $2 AND source_offset = $3) OR (source_topic = $1 AND source_event_id = $4) ORDER BY created_at ASC LIMIT 1", [transport.sourceTopic, transport.sourcePartition, transport.sourceOffset, sourceEventId]); + const prior = existing.rows?.[0] ?? null; + if (prior) { + if (prior.input_sha256 !== inputSha256) { + const diagnostic = { code: "workbench_kafka_inbox_hash_conflict", message: "Ignored Kafka identity was reused with a different payload hash.", valuesRedacted: true }; + await insertDlq(client, { ...transport, sourceEventId, inputSha256, error: diagnostic, envelope: redactedEnvelope(envelope), createdAt: now }); + return { ignored: true, duplicate: false, conflict: true, status: prior.status, diagnostic, valuesPrinted: false }; + } + return { ignored: true, duplicate: true, status: prior.status, valuesPrinted: false }; + } + await client.query( + "INSERT INTO workbench_kafka_inbox (source_topic, source_partition, source_offset, source_key, source_event_id, input_sha256, trace_id, session_id, run_id, command_id, source_seq, status, projected_seq, envelope_json, error_json, created_at, updated_at, projected_at) VALUES ($1,$2,$3,$4,$5,$6,NULL,NULL,$7,$8,$9,'ignored',NULL,$10,'{}',$11,$11,NULL)", + [transport.sourceTopic, transport.sourcePartition, transport.sourceOffset, transport.sourceKey, sourceEventId, inputSha256, text(envelope.runId), text(envelope.commandId), nonNegativeInteger(envelope.sourceSeq), stableJson(redactedEnvelope(envelope)), now] + ); + return { ignored: true, duplicate: false, status: "ignored", valuesPrinted: false }; + }); +} + +export async function claimHwlabKafkaOutbox(store, { owner, leaseMs, limit } = {}) { + const leaseOwner = requiredText(owner, "HWLAB Kafka relay owner"); + const boundedLimit = boundedInteger(limit, 100, 1, 500); + const now = store.now(); + const leaseExpiresAt = new Date(Date.parse(now) + boundedInteger(leaseMs, 60_000, 1_000, 3_600_000)).toISOString(); + const result = await store.withDurableTransaction("hwlab.kafka-outbox.claim", (client) => client.query( + `WITH candidates AS ( + SELECT candidate.outbox_seq FROM hwlab_kafka_outbox candidate + WHERE candidate.published_at IS NULL + AND candidate.next_attempt_at <= $1 + AND (candidate.lease_expires_at IS NULL OR candidate.lease_expires_at <= $1) + AND NOT EXISTS ( + SELECT 1 FROM hwlab_kafka_outbox earlier + WHERE earlier.published_at IS NULL + AND earlier.partition_key = candidate.partition_key + AND earlier.outbox_seq < candidate.outbox_seq + ) + ORDER BY candidate.outbox_seq ASC + FOR UPDATE SKIP LOCKED + LIMIT $2 + ) + UPDATE hwlab_kafka_outbox o SET attempt_count = o.attempt_count + 1, lease_owner = $3, lease_expires_at = $4, updated_at = $1 + FROM candidates WHERE o.outbox_seq = candidates.outbox_seq + RETURNING o.*`, + [now, boundedLimit, leaseOwner, leaseExpiresAt] + )); + return (result.rows ?? []).map(outboxRow); +} + +export async function completeHwlabKafkaOutbox(store, item = {}) { + const result = await store.query( + "UPDATE hwlab_kafka_outbox SET published_at = $1, lease_owner = NULL, lease_expires_at = NULL, last_error = NULL, updated_at = $1 WHERE outbox_seq = $2 AND lease_owner = $3 AND attempt_count = $4 AND lease_expires_at = $5 AND lease_expires_at > $1 AND published_at IS NULL RETURNING outbox_seq", + [store.now(), item.outboxSeq, item.leaseOwner, item.attemptCount, item.leaseExpiresAt] + ); + if (!result.rows?.[0]) throw codedError("hwlab_kafka_outbox_lease_conflict", "HWLAB Kafka outbox claim is stale."); + return { completed: true, outboxSeq: Number(result.rows[0].outbox_seq), valuesPrinted: false }; +} + +export async function retryHwlabKafkaOutbox(store, item = {}, error = null, retryAt = null) { + const now = store.now(); + const nextAttemptAt = text(retryAt) ?? now; + const result = await store.query( + "UPDATE hwlab_kafka_outbox SET next_attempt_at = $1, lease_owner = NULL, lease_expires_at = NULL, last_error = $2, updated_at = $3 WHERE outbox_seq = $4 AND lease_owner = $5 AND attempt_count = $6 AND lease_expires_at = $7 AND lease_expires_at > $3 AND published_at IS NULL RETURNING outbox_seq", + [nextAttemptAt, stableJson(redactedError(error)), now, item.outboxSeq, item.leaseOwner, item.attemptCount, item.leaseExpiresAt] + ); + if (!result.rows?.[0]) throw codedError("hwlab_kafka_outbox_lease_conflict", "HWLAB Kafka outbox claim is stale."); + return { retried: true, outboxSeq: Number(result.rows[0].outbox_seq), nextAttemptAt, valuesPrinted: false }; +} + +export async function hwlabKafkaProjectorStatus(store) { + const result = await store.query( + `SELECT + COUNT(*) FILTER (WHERE published_at IS NULL)::int AS backlog_count, + COUNT(*) FILTER (WHERE published_at IS NULL AND attempt_count > 0)::int AS retry_count, + MIN(created_at) FILTER (WHERE published_at IS NULL) AS oldest_pending_at, + (SELECT COUNT(*)::int FROM workbench_kafka_inbox WHERE status = 'failed') AS failed_inbox_count, + (SELECT COUNT(*)::int FROM workbench_kafka_dlq) AS dlq_count + FROM hwlab_kafka_outbox`, [] + ); + const row = result.rows?.[0] ?? {}; + return { backlogCount: Number(row.backlog_count ?? 0), retryCount: Number(row.retry_count ?? 0), oldestPendingAt: row.oldest_pending_at ?? null, failedInboxCount: Number(row.failed_inbox_count ?? 0), dlqCount: Number(row.dlq_count ?? 0), valuesPrinted: false }; +} + +export async function readAtomicWorkbenchProjectionSync(store, params = {}) { + const since = nonNegativeInteger(params.afterOutboxSeq ?? params.afterSeq ?? params.since); + const limit = boundedInteger(params.limit, 100, 1, 500); + const traceId = text(params.traceId); + const sessionId = text(params.sessionId); + const snapshotOnly = params.snapshotOnly === true; + const deltaOnly = params.deltaOnly === true; + if (!traceId && !sessionId) throw codedError("workbench_sync_scope_required", "Atomic Workbench sync requires sessionId or traceId."); + if (snapshotOnly && deltaOnly) throw codedError("workbench_sync_mode_invalid", "snapshotOnly and deltaOnly are mutually exclusive."); + return store.withDurableTransaction("workbench.projection-sync.read", async (client) => { + await client.query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY"); + const filter = traceId ? { column: "trace_id", value: traceId } : { column: "session_id", value: sessionId }; + const cutoffResult = await client.query(`SELECT COALESCE(MAX(outbox_seq), 0)::bigint AS cutoff FROM workbench_projection_outbox WHERE ${filter.column} = $1`, [filter.value]); + const cutoff = Number(cutoffResult.rows?.[0]?.cutoff ?? 0); + const outboxResult = snapshotOnly ? { rows: [] } : await client.query( + `SELECT outbox_seq, outbox_event_id, entity_family, entity_id, event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at FROM workbench_projection_outbox WHERE outbox_seq > $1 AND outbox_seq <= $2 AND ${filter.column} = $3 ORDER BY outbox_seq ASC LIMIT $4`, + [since, cutoff, filter.value, limit + 1] + ); + const allRows = (outboxResult.rows ?? []).map(projectionOutboxRow); + const events = allRows.slice(0, limit); + const facts = deltaOnly ? {} : await readSyncFacts(client, { traceId, sessionId, limit: 500, families: snapshotOnly ? ["sessions", "messages", "turns"] : null }); + const hasMore = allRows.length > limit; + return { facts, events, cutoffOutboxSeq: cutoff, cursorOutboxSeq: snapshotOnly ? cutoff : hasMore ? (events.at(-1)?.outboxSeq ?? since) : cutoff, hasMore, valuesPrinted: false }; + }); +} + +async function persistFacts(store, client, facts, requestMeta) { + const aggregateEvents = normalizeWorkbenchAggregateEventsForFacts(facts, requestMeta, store.now()); + const events = []; + for (const event of aggregateEvents) events.push(await store.persistWorkbenchAggregateEvent(event, client)); + const eventIndex = indexWorkbenchAggregateEvents(events); + const inputFacts = facts.inputs.map((fact) => inputFactWithAggregateSeq(fact, eventIndex)); + for (const fact of inputFacts) await store.persistWorkbenchSessionInputFact(fact, client); + for (const fact of facts.sessions) await store.persistWorkbenchSessionFact(fact, client); + for (const fact of facts.messages) await store.persistWorkbenchMessageFact(fact, client); + for (const fact of facts.parts) await store.persistWorkbenchPartFact(fact, client); + for (const fact of facts.turns) await store.persistWorkbenchTurnFact(fact, client); + for (const fact of facts.traceEvents) await store.persistWorkbenchTraceEventFact(fact, client); + for (const fact of facts.checkpoints) await store.persistWorkbenchProjectionCheckpoint(fact, client); + for (const fact of facts.messages) { + const event = eventIndex.get(`message:${fact.messageId}`) ?? eventIndex.get(`messages:${fact.messageId}`) ?? eventIndex.get(`sourceEvent:${fact.sourceEventId}`) ?? null; + await store.persistWorkbenchProjectionOutbox(outboxRecord(fact, event, "messages", "message", { role: fact.role, status: fact.status }), client); + } + for (const fact of facts.traceEvents) { + const event = eventIndex.get(`traceEvent:${fact.id}`) ?? eventIndex.get(`sourceEvent:${fact.sourceEventId}`) ?? null; + await store.persistWorkbenchProjectionOutbox(outboxRecord(fact, event, "traceEvents", fact.terminal ? "terminal" : "event", { type: fact.eventType }), client); + } + for (const fact of facts.turns.filter((item) => item.terminal || item.sealed)) { + const event = eventIndex.get(`turn:${fact.turnId}`) ?? eventIndex.get(`sourceEvent:${fact.sourceEventId}`) ?? null; + await store.persistWorkbenchProjectionOutbox(outboxRecord(fact, event, "turns", "terminal", { status: fact.status, failureKind: fact.failureKind }), client); + } + return { events, inputFacts }; +} + +function outboxRecord(fact, event, family, commitType, metadata) { + const entityId = text(family === "messages" ? fact.messageId : family === "turns" ? fact.turnId : fact.id) ?? text(fact.traceId) ?? "projection"; + return { + outboxEventId: projectionOutboxEventId({ fact, event, family, entityId, commitType }), + entityFamily: family, + entityId, + eventSeq: event?.eventSeq ?? null, + aggregateId: event?.aggregateId ?? null, + aggregateSeq: event?.aggregateSeq ?? 0, + projectionRevision: event?.projectionRevision ?? fact.projectedSeq, + traceId: fact.traceId, + sessionId: fact.sessionId, + turnId: fact.turnId, + messageId: fact.messageId, + projectedSeq: fact.projectedSeq, + sourceSeq: fact.sourceSeq, + sourceEventId: fact.sourceEventId, + commitType, + terminal: Boolean(fact.terminal), + sealed: Boolean(fact.sealed), + payload: { + family, + fact, + metadata: { ...metadata, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq, valuesRedacted: true }, + valuesRedacted: true + }, + createdAt: fact.occurredAt ?? fact.updatedAt ?? fact.createdAt + }; +} + +export function projectionOutboxEventId({ fact = {}, event = null, family, entityId, commitType }) { + const sourceEventId = text(fact.sourceEventId); + if (sourceEventId) return `${sourceEventId}:${family}:${entityId}:${commitType}`; + const identity = stableJson({ + family, + entityId, + commitType, + eventSeq: event?.eventSeq ?? null, + aggregateId: event?.aggregateId ?? null, + aggregateSeq: event?.aggregateSeq ?? 0, + projectionRevision: event?.projectionRevision ?? fact.projectedSeq ?? 0, + projectedSeq: fact.projectedSeq ?? 0, + sourceSeq: fact.sourceSeq ?? 0 + }); + return `projection:${createHash("sha256").update(identity).digest("hex")}`; +} + +function mergeKafkaSessionIdentity(facts, row) { + if (!row) return facts; + const existing = parseJsonColumn(row.session_json, {}); + const sessions = Array.isArray(facts?.sessions) ? facts.sessions.map((fact) => ({ + ...existing, + ...fact, + ownerUserId: fact.ownerUserId ?? row.owner_user_id ?? existing.ownerUserId ?? null, + projectId: fact.projectId ?? row.project_id ?? existing.projectId ?? null, + conversationId: fact.conversationId ?? row.conversation_id ?? existing.conversationId ?? null, + threadId: fact.threadId ?? row.thread_id ?? existing.threadId ?? null, + ownerRole: fact.ownerRole ?? existing.ownerRole ?? null + })) : []; + return { ...facts, sessions }; +} + +async function readSyncFacts(client, { traceId, sessionId, limit, families = null }) { + const familySet = Array.isArray(families) ? new Set(families) : null; + const specs = [ + ["sessions", "workbench_sessions", "session_json", traceId ? "last_trace_id" : "session_id"], + ["messages", "workbench_messages", "message_json", traceId ? "trace_id" : "session_id"], + ["parts", "workbench_parts", "part_json", traceId ? "trace_id" : "session_id"], + ["turns", "workbench_turns", "turn_json", traceId ? "trace_id" : "session_id"], + ["traceEvents", "workbench_trace_events", "event_json", traceId ? "trace_id" : "session_id"], + ["checkpoints", "workbench_projection_checkpoints", "checkpoint_json", traceId ? "trace_id" : "session_id"] + ].filter(([family]) => !familySet || familySet.has(family)); + const value = traceId ?? sessionId; + const facts = {}; + for (const [family, table, jsonColumn, column] of specs) { + const result = await client.query(`SELECT ${jsonColumn} FROM ${table} WHERE ${column} = $1 ORDER BY updated_at ASC LIMIT $2`, [value, limit]); + facts[family] = (result.rows ?? []).map((row) => parseJsonColumn(row[jsonColumn], null)).filter(Boolean); + } + return facts; +} + +async function insertDlq(client, record) { + await client.query( + "INSERT INTO workbench_kafka_dlq (source_topic, source_partition, source_offset, source_event_id, input_sha256, error_code, error_message, envelope_json, created_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (source_topic, source_partition, source_offset) DO UPDATE SET error_code = EXCLUDED.error_code, error_message = EXCLUDED.error_message, envelope_json = EXCLUDED.envelope_json", + [record.sourceTopic, record.sourcePartition, record.sourceOffset, record.sourceEventId, record.inputSha256, record.error.code, record.error.message, stableJson(record.envelope), record.createdAt] + ); +} + +async function lockKafkaInboxIdentity(client, transport, sourceEventId) { + const transportIdentity = `${transport.sourceTopic}:transport:${transport.sourcePartition}:${transport.sourceOffset}`; + await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`workbench-kafka-inbox:${transportIdentity}`]); + if (sourceEventId) { + const eventIdentity = `${transport.sourceTopic}:event:${sourceEventId}`; + await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`workbench-kafka-inbox:${eventIdentity}`]); + } +} + +function normalizeTransport(value = {}) { + const input = objectValue(value); + const sourceTopic = requiredText(input.sourceTopic ?? input.topic, "Kafka source topic"); + const sourcePartition = Number(input.sourcePartition ?? input.partition); + const sourceOffset = text(input.sourceOffset ?? input.offset); + if (!Number.isInteger(sourcePartition) || sourcePartition < 0) throw codedError("workbench_kafka_partition_invalid", "Kafka source partition must be a non-negative integer."); + if (!/^\d+$/u.test(sourceOffset ?? "")) throw codedError("workbench_kafka_offset_invalid", "Kafka source offset must be a non-negative integer string."); + return { sourceTopic, sourcePartition, sourceOffset, inputSha256: text(input.inputSha256), sourceKey: text(input.sourceKey), valuesPrinted: false }; +} + +function projectionOutboxRow(row) { + return { outboxSeq: Number(row.outbox_seq), outboxEventId: row.outbox_event_id, entityFamily: row.entity_family, entityId: row.entity_id, eventSeq: row.event_seq == null ? null : Number(row.event_seq), aggregateId: row.aggregate_id, aggregateSeq: Number(row.aggregate_seq ?? 0), projectionRevision: Number(row.projection_revision ?? 0), traceId: row.trace_id, sessionId: row.session_id, turnId: row.turn_id, messageId: row.message_id, projectedSeq: Number(row.projected_seq ?? 0), sourceSeq: Number(row.source_seq ?? 0), sourceEventId: row.source_event_id, commitType: row.commit_type, terminal: Boolean(row.terminal), sealed: Boolean(row.sealed), payload: parseJsonColumn(row.payload_json, {}), createdAt: row.created_at, valuesPrinted: false }; +} + +function outboxRow(row) { + return { outboxSeq: Number(row.outbox_seq), eventId: row.event_id, topic: row.topic, partitionKey: row.partition_key, payload: parseJsonColumn(row.payload_json, {}), headers: parseJsonColumn(row.headers_json, {}), attemptCount: Number(row.attempt_count ?? 0), nextAttemptAt: row.next_attempt_at, leaseOwner: row.lease_owner, leaseExpiresAt: row.lease_expires_at, createdAt: row.created_at, valuesPrinted: false }; +} + +function redactedEnvelope(value) { + const input = objectValue(value); + return { schema: text(input.schema), eventType: text(input.eventType), eventId: text(input.eventId), outboxSeq: nonNegativeInteger(input.outboxSeq), sourceSeq: nonNegativeInteger(input.sourceSeq), traceId: text(input.traceId), sessionId: text(input.sessionId), hwlabSessionId: text(input.hwlabSessionId), runId: text(input.runId), commandId: text(input.commandId), valuesRedacted: true }; +} + +function redactedError(error) { + return { code: text(error?.code) ?? "workbench_kafka_message_invalid", message: String(error?.message ?? error ?? "Kafka message is invalid.").slice(0, 500), valuesRedacted: true }; +} + +function objectValue(value) { + return value && typeof value === "object" && !Array.isArray(value) ? value : {}; +} + +function text(value) { + const normalized = textOr(value, ""); + return normalized || null; +} + +function requiredText(value, label) { + const normalized = text(value); + if (!normalized) throw codedError("workbench_kafka_contract_invalid", `${label} is required.`); + return normalized; +} + +function boundedInteger(value, fallback, min, max) { + const parsed = Number(value); + return Number.isInteger(parsed) && parsed >= min && parsed <= max ? parsed : fallback; +} + +function codedError(code, message) { + const error = new Error(message); + error.code = code; + return error; +} diff --git a/internal/db/runtime-store-postgres-notify.test.ts b/internal/db/runtime-store-postgres-notify.test.ts new file mode 100644 index 00000000..4239b7b2 --- /dev/null +++ b/internal/db/runtime-store-postgres-notify.test.ts @@ -0,0 +1,26 @@ +import assert from "node:assert/strict"; +import { EventEmitter } from "node:events"; +import { test } from "bun:test"; + +import { subscribeWorkbenchProjectionCommits, WORKBENCH_PROJECTION_CHANNEL } from "./runtime-store-postgres-notify.ts"; + +test("Postgres LISTEN fans committed projection scope out and releases its dedicated connection", async () => { + const queries = []; + let released = 0; + const client = new EventEmitter(); + client.query = async (sql) => { queries.push(sql); return { rows: [] }; }; + client.release = () => { released += 1; }; + const pool = { async connect() { return client; } }; + const store = { async getQueryClient() { return pool; }, logger: null }; + const signals = []; + + const unsubscribe = await subscribeWorkbenchProjectionCommits(store, (signal) => signals.push(signal)); + assert.deepEqual(queries, [`LISTEN ${WORKBENCH_PROJECTION_CHANNEL}`]); + + client.emit("notification", { channel: WORKBENCH_PROJECTION_CHANNEL, payload: JSON.stringify({ sessionId: "ses_notify", traceId: "trc_notify", outboxSeq: 12 }) }); + assert.deepEqual(signals, [{ sessionId: "ses_notify", traceId: "trc_notify", outboxSeq: 12, valuesRedacted: true }]); + + await unsubscribe(); + assert.equal(queries.at(-1), `UNLISTEN ${WORKBENCH_PROJECTION_CHANNEL}`); + assert.equal(released, 1); +}); diff --git a/internal/db/runtime-store-postgres-notify.ts b/internal/db/runtime-store-postgres-notify.ts new file mode 100644 index 00000000..4969bcbc --- /dev/null +++ b/internal/db/runtime-store-postgres-notify.ts @@ -0,0 +1,145 @@ +/* PostgreSQL projection notifications wake SSE readers; the outbox remains the payload authority. */ + +const WORKBENCH_PROJECTION_CHANNEL = "hwlab_workbench_projection"; +const RECONNECT_DELAY_MS = 500; +const hubs = new WeakMap(); + +export async function subscribeWorkbenchProjectionCommits(store, listener) { + if (typeof listener !== "function") throw codedError("workbench_projection_listener_invalid", "Projection notification listener must be a function."); + let hub = hubs.get(store); + if (!hub) { + hub = { store, listeners: new Set(), client: null, connectPromise: null, reconnectTimer: null, connectedOnce: false, closing: false }; + hubs.set(store, hub); + } + hub.listeners.add(listener); + hub.closing = false; + await ensureConnected(hub); + let subscribed = true; + return async () => { + if (!subscribed) return; + subscribed = false; + hub.listeners.delete(listener); + if (hub.listeners.size === 0) await closeHub(hub); + }; +} + +async function ensureConnected(hub) { + if (hub.client || hub.connectPromise || hub.closing || hub.listeners.size === 0) return hub.connectPromise; + hub.connectPromise = (async () => { + const queryClient = await hub.store.getQueryClient(); + const client = typeof queryClient?.connect === "function" ? await queryClient.connect() : queryClient; + if (!client || typeof client.query !== "function" || typeof client.on !== "function") { + client?.release?.(); + throw codedError("workbench_projection_notify_unsupported", "Postgres runtime adapter requires a dedicated LISTEN connection."); + } + const onNotification = (message) => { + if (message?.channel !== WORKBENCH_PROJECTION_CHANNEL) return; + const signal = parseSignal(message.payload); + for (const notify of [...hub.listeners]) { + try { notify(signal); } catch {} + } + }; + const onDisconnect = (error) => disconnectHubClient(hub, client, error); + client.on("notification", onNotification); + client.on("error", onDisconnect); + client.on("end", onDisconnect); + client.__hwlabProjectionNotifyHandlers = { onNotification, onDisconnect }; + try { + await client.query(`LISTEN ${WORKBENCH_PROJECTION_CHANNEL}`); + } catch (error) { + detachClient(client, error); + throw error; + } + if (hub.closing || hub.listeners.size === 0) { + detachClient(client); + return; + } + hub.client = client; + if (hub.connectedOnce) emitRecovery(hub); + hub.connectedOnce = true; + })().finally(() => { + hub.connectPromise = null; + }); + return hub.connectPromise; +} + +function disconnectHubClient(hub, client, error) { + if (client && hub.client !== client) return; + if (client) { + hub.client = null; + detachClient(client, error); + } + scheduleReconnect(hub, error); +} + +function scheduleReconnect(hub, error) { + if (hub.closing || hub.listeners.size === 0 || hub.reconnectTimer) return; + hub.store.logger?.warn?.({ event: "workbench_projection_notify_disconnected", errorCode: error?.code ?? "UNKNOWN", valuesRedacted: true }); + hub.reconnectTimer = setTimeout(() => { + hub.reconnectTimer = null; + void ensureConnected(hub).catch((connectError) => scheduleReconnect(hub, connectError)); + }, RECONNECT_DELAY_MS); + hub.reconnectTimer.unref?.(); +} + +async function closeHub(hub) { + hub.closing = true; + if (hub.reconnectTimer) clearTimeout(hub.reconnectTimer); + hub.reconnectTimer = null; + const client = hub.client; + hub.client = null; + if (client) { + try { await client.query(`UNLISTEN ${WORKBENCH_PROJECTION_CHANNEL}`); } catch {} + detachClient(client); + } + hubs.delete(hub.store); +} + +function detachClient(client, error = null) { + const handlers = client?.__hwlabProjectionNotifyHandlers; + if (handlers) { + client.off?.("notification", handlers.onNotification); + client.off?.("error", handlers.onDisconnect); + client.off?.("end", handlers.onDisconnect); + delete client.__hwlabProjectionNotifyHandlers; + } + client?.release?.(error || undefined); +} + +function emitRecovery(hub) { + for (const notify of [...hub.listeners]) { + try { notify({ recovery: true, valuesRedacted: true }); } catch {} + } +} + +function parseSignal(payload) { + try { + const value = JSON.parse(String(payload ?? "{}")); + return { + sessionId: text(value.sessionId), + traceId: text(value.traceId), + outboxSeq: nonNegativeInteger(value.outboxSeq), + valuesRedacted: true + }; + } catch { + return { recovery: true, valuesRedacted: true }; + } +} + +function text(value) { + const normalized = String(value ?? "").trim(); + return normalized || null; +} + +function nonNegativeInteger(value) { + const number = Number(value); + return Number.isInteger(number) && number >= 0 ? number : 0; +} + +function codedError(code, message) { + const error = new Error(message); + error.code = code; + return error; +} + +export { WORKBENCH_PROJECTION_CHANNEL }; diff --git a/internal/db/runtime-store-postgres.ts b/internal/db/runtime-store-postgres.ts index 8bb5bebc..395208d0 100644 --- a/internal/db/runtime-store-postgres.ts +++ b/internal/db/runtime-store-postgres.ts @@ -19,12 +19,26 @@ import { CLOUD_CORE_MIGRATION_ID, CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, - CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS + CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS, + CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID, + CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION, + CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS } from "./schema.ts"; import { CloudRuntimeStore } from "./runtime-store-memory.ts"; import * as runtimeCore from "./runtime-store-core.ts"; +import { + claimHwlabKafkaOutbox as claimHwlabKafkaOutboxOperation, + commitAgentRunKafkaProjection as commitAgentRunKafkaProjectionOperation, + completeHwlabKafkaOutbox as completeHwlabKafkaOutboxOperation, + hwlabKafkaProjectorStatus as hwlabKafkaProjectorStatusOperation, + readAtomicWorkbenchProjectionSync as readAtomicWorkbenchProjectionSyncOperation, + recordFailedAgentRunKafkaMessage as recordFailedAgentRunKafkaMessageOperation, + recordIgnoredAgentRunKafkaMessage as recordIgnoredAgentRunKafkaMessageOperation, + retryHwlabKafkaOutbox as retryHwlabKafkaOutboxOperation +} from "./runtime-store-postgres-kafka.ts"; +import { subscribeWorkbenchProjectionCommits as subscribeWorkbenchProjectionCommitsOperation } from "./runtime-store-postgres-notify.ts"; const { RUNTIME_STORE_KIND, @@ -697,7 +711,7 @@ export class PostgresCloudRuntimeStore { commitType: "message", terminal: Boolean(fact.terminal), sealed: Boolean(fact.sealed), - payload: { role: fact.role, status: fact.status, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq, valuesRedacted: true }, + payload: immutableProjectionOutboxPayload("messages", fact, { role: fact.role, status: fact.status, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq }), createdAt: fact.updatedAt ?? fact.createdAt ?? this.now() }, client); } @@ -718,7 +732,7 @@ export class PostgresCloudRuntimeStore { commitType: fact.terminal ? "terminal" : "event", terminal: Boolean(fact.terminal), sealed: Boolean(fact.sealed), - payload: { type: fact.eventType, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq, valuesRedacted: true }, + payload: immutableProjectionOutboxPayload("traceEvents", fact, { type: fact.eventType, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq }), createdAt: fact.occurredAt ?? fact.updatedAt ?? this.now() }, client); } @@ -740,7 +754,7 @@ export class PostgresCloudRuntimeStore { commitType: "terminal", terminal: true, sealed: true, - payload: { status: fact.status, failureKind: fact.failureKind, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, valuesRedacted: true }, + payload: immutableProjectionOutboxPayload("turns", fact, { status: fact.status, failureKind: fact.failureKind, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null }), createdAt: fact.updatedAt ?? this.now() }, client); } @@ -752,6 +766,135 @@ export class PostgresCloudRuntimeStore { return withPersistence({ written: true, facts: persistedFacts, events: txResult.events }, this.summary()); } + async writeWorkbenchSessionAdmissionFact(params = {}, requestMeta = {}) { + const readiness = this.summary(); + if (readiness.ready !== true || readiness.durable !== true || readiness.liveRuntimeEvidence !== true) { + await this.assertReadyForWrites(); + } + const facts = normalizeWorkbenchFacts({ sessions: [params.fact ?? params.session ?? params] }, requestMeta, this.now()); + const fact = facts.sessions[0]; + if (!fact?.sessionId) throw new Error("Workbench session admission fact requires sessionId."); + await this.withDurableTransaction("workbench.session-admission.write", (client) => this.persistWorkbenchSessionAdmissionFact(fact, client)); + const memoryResult = this.memory.writeWorkbenchSessionAdmissionFact({ fact }, requestMeta); + return withPersistence({ written: true, admissionOnly: true, fact: memoryResult.fact, valuesPrinted: false }, this.summary()); + } + + async promoteWorkbenchTurnAdmission(params = {}, requestMeta = {}) { + const readiness = this.summary(); + if (readiness.ready !== true || readiness.durable !== true || readiness.liveRuntimeEvidence !== true) { + await this.assertReadyForWrites(); + } + const facts = normalizeWorkbenchFacts(params.facts ?? params, requestMeta, this.now()); + const sessionFact = facts.sessions[0]; + const inputFact = facts.inputs[0]; + const turnFact = facts.turns[0]; + if (!sessionFact?.sessionId || !inputFact?.inputId || !turnFact?.turnId) throw new Error("Workbench turn admission promotion requires session, input, and turn facts."); + const aggregateEvents = normalizeWorkbenchAggregateEventsForFacts(facts, requestMeta, this.now()); + const transaction = await this.withDurableTransaction("workbench.turn-admission.promote", async (client) => { + await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`workbench-projection:${turnFact.traceId}`]); + const existingTurnResult = await client.query( + "SELECT projected_seq, sealed FROM workbench_turns WHERE turn_id = $1 FOR UPDATE", + [turnFact.turnId] + ); + const existingTurn = existingTurnResult.rows?.[0] ?? null; + const supersededByProjection = nonNegativeInteger(existingTurn?.projected_seq) > 0 || existingTurn?.sealed === true; + const persistedEvents = []; + const transactionEvents = supersededByProjection + ? aggregateEvents.filter((event) => event.factFamily === "inputs") + : aggregateEvents; + for (const event of transactionEvents) persistedEvents.push(await this.persistWorkbenchAggregateEvent(event, client)); + const eventIndex = indexWorkbenchAggregateEvents(persistedEvents); + const promotedInput = inputFactWithAggregateSeq(inputFact, eventIndex); + await this.persistWorkbenchSessionInputFact(promotedInput, client); + if (supersededByProjection) { + return { persistedEvents, promotedInput, outbox: null, supersededByProjection: true }; + } + await this.persistWorkbenchSessionAdmissionFact(sessionFact, client); + for (const fact of facts.messages) await this.persistWorkbenchMessageAdmissionFact(fact, client); + for (const fact of facts.parts) await this.persistWorkbenchPartAdmissionFact(fact, client); + for (const fact of facts.turns) await this.persistWorkbenchTurnAdmissionFact(fact, client); + const turnEvent = eventIndex.get(`turn:${turnFact.turnId}`) ?? eventIndex.get(`sourceEvent:${turnFact.sourceEventId}`) ?? null; + const outbox = { + entityFamily: "turns", + entityId: turnFact.turnId, + eventSeq: turnEvent?.eventSeq ?? null, + aggregateId: turnEvent?.aggregateId ?? null, + aggregateSeq: turnEvent?.aggregateSeq ?? 0, + projectionRevision: turnEvent?.projectionRevision ?? turnFact.projectedSeq, + traceId: turnFact.traceId, + sessionId: turnFact.sessionId, + turnId: turnFact.turnId, + messageId: turnFact.messageId, + projectedSeq: turnFact.projectedSeq, + sourceSeq: turnFact.sourceSeq, + sourceEventId: turnFact.sourceEventId, + commitType: "admission", + terminal: false, + sealed: false, + payload: immutableProjectionOutboxPayload("turns", turnFact, { status: turnFact.status, admissionState: "promoted", eventSeq: turnEvent?.eventSeq ?? null, aggregateSeq: turnEvent?.aggregateSeq ?? null }), + createdAt: turnFact.updatedAt ?? this.now() + }; + await this.persistWorkbenchProjectionOutbox(outbox, client); + return { persistedEvents, promotedInput, outbox, supersededByProjection: false }; + }); + const memoryResult = transaction.supersededByProjection + ? this.memory.writeWorkbenchFacts({ facts: { inputs: [transaction.promotedInput] } }, requestMeta) + : this.memory.promoteWorkbenchTurnAdmission({ facts: { ...facts, inputs: [transaction.promotedInput] } }, requestMeta); + return withPersistence({ + written: true, + promoted: true, + outboxCommitted: !transaction.supersededByProjection, + outbox: transaction.outbox, + supersededByProjection: transaction.supersededByProjection, + facts: memoryResult.facts, + events: transaction.persistedEvents, + valuesPrinted: false + }, this.summary()); + } + + async commitAgentRunKafkaProjection(params = {}) { + await this.assertReadyForWrites(); + return commitAgentRunKafkaProjectionOperation(this, params); + } + + async recordFailedAgentRunKafkaMessage(params = {}) { + await this.assertReadyForWrites(); + return recordFailedAgentRunKafkaMessageOperation(this, params); + } + + async recordIgnoredAgentRunKafkaMessage(params = {}) { + await this.assertReadyForWrites(); + return recordIgnoredAgentRunKafkaMessageOperation(this, params); + } + + async claimHwlabKafkaOutbox(params = {}) { + await this.assertReadyForWrites(); + return claimHwlabKafkaOutboxOperation(this, params); + } + + async completeHwlabKafkaOutbox(item = {}) { + return completeHwlabKafkaOutboxOperation(this, item); + } + + async retryHwlabKafkaOutbox(item = {}, error = null, retryAt = null) { + return retryHwlabKafkaOutboxOperation(this, item, error, retryAt); + } + + async hwlabKafkaProjectorStatus() { + await this.assertReadyForDurableReads("hwlab.kafka-projector.status"); + return hwlabKafkaProjectorStatusOperation(this); + } + + async readAtomicWorkbenchProjectionSync(params = {}) { + await this.assertReadyForDurableReads("workbench.projection-sync.read"); + return readAtomicWorkbenchProjectionSyncOperation(this, params); + } + + async subscribeWorkbenchProjectionCommits(listener) { + await this.assertReadyForDurableReads("workbench.projection-notify.listen"); + return subscribeWorkbenchProjectionCommitsOperation(this, listener); + } + async queryWorkbenchFacts(params = {}) { await this.assertReadyForDurableReads("workbench.facts.query"); const families = workbenchFactFamilySet(params); @@ -1122,7 +1265,14 @@ export class PostgresCloudRuntimeStore { async persistWorkbenchSessionFact(record, client = this) { await client.query( - "INSERT INTO workbench_sessions (session_id, owner_user_id, project_id, conversation_id, thread_id, status, last_trace_id, projected_seq, source_seq, source_event_id, terminal, sealed, session_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (session_id) DO UPDATE SET owner_user_id = EXCLUDED.owner_user_id, project_id = EXCLUDED.project_id, conversation_id = EXCLUDED.conversation_id, thread_id = EXCLUDED.thread_id, status = EXCLUDED.status, last_trace_id = EXCLUDED.last_trace_id, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, session_json = EXCLUDED.session_json, updated_at = EXCLUDED.updated_at", + "INSERT INTO workbench_sessions (session_id, owner_user_id, project_id, conversation_id, thread_id, status, last_trace_id, projected_seq, source_seq, source_event_id, terminal, sealed, session_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (session_id) DO UPDATE SET owner_user_id = COALESCE(EXCLUDED.owner_user_id, workbench_sessions.owner_user_id), project_id = COALESCE(EXCLUDED.project_id, workbench_sessions.project_id), conversation_id = COALESCE(EXCLUDED.conversation_id, workbench_sessions.conversation_id), thread_id = COALESCE(EXCLUDED.thread_id, workbench_sessions.thread_id), status = EXCLUDED.status, last_trace_id = EXCLUDED.last_trace_id, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, session_json = EXCLUDED.session_json, updated_at = EXCLUDED.updated_at", + [record.sessionId, record.ownerUserId, record.projectId, record.conversationId, record.threadId, record.status, record.lastTraceId, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record), record.createdAt, record.updatedAt] + ); + } + + async persistWorkbenchSessionAdmissionFact(record, client = this) { + await client.query( + "INSERT INTO workbench_sessions (session_id, owner_user_id, project_id, conversation_id, thread_id, status, last_trace_id, projected_seq, source_seq, source_event_id, terminal, sealed, session_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (session_id) DO UPDATE SET owner_user_id = COALESCE(workbench_sessions.owner_user_id, EXCLUDED.owner_user_id), project_id = COALESCE(workbench_sessions.project_id, EXCLUDED.project_id), conversation_id = COALESCE(workbench_sessions.conversation_id, EXCLUDED.conversation_id), thread_id = COALESCE(workbench_sessions.thread_id, EXCLUDED.thread_id), status = CASE WHEN workbench_sessions.projected_seq <= 0 AND NOT workbench_sessions.sealed THEN EXCLUDED.status ELSE workbench_sessions.status END, last_trace_id = CASE WHEN workbench_sessions.projected_seq <= 0 AND NOT workbench_sessions.sealed THEN COALESCE(EXCLUDED.last_trace_id, workbench_sessions.last_trace_id) ELSE workbench_sessions.last_trace_id END, session_json = CASE WHEN workbench_sessions.projected_seq <= 0 AND NOT workbench_sessions.sealed THEN EXCLUDED.session_json ELSE workbench_sessions.session_json END, updated_at = CASE WHEN workbench_sessions.projected_seq <= 0 AND NOT workbench_sessions.sealed THEN EXCLUDED.updated_at ELSE workbench_sessions.updated_at END", [record.sessionId, record.ownerUserId, record.projectId, record.conversationId, record.threadId, record.status, record.lastTraceId, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record), record.createdAt, record.updatedAt] ); } @@ -1141,6 +1291,13 @@ export class PostgresCloudRuntimeStore { ); } + async persistWorkbenchMessageAdmissionFact(record, client = this) { + await client.query( + "INSERT INTO workbench_messages (message_id, session_id, turn_id, trace_id, role, status, projected_seq, source_seq, source_event_id, terminal, sealed, message_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT (message_id) DO UPDATE SET session_id = EXCLUDED.session_id, turn_id = EXCLUDED.turn_id, trace_id = EXCLUDED.trace_id, role = EXCLUDED.role, status = CASE WHEN workbench_messages.projected_seq <= 0 AND NOT workbench_messages.sealed THEN EXCLUDED.status ELSE workbench_messages.status END, source_event_id = CASE WHEN workbench_messages.projected_seq <= 0 AND NOT workbench_messages.sealed THEN EXCLUDED.source_event_id ELSE workbench_messages.source_event_id END, message_json = CASE WHEN workbench_messages.projected_seq <= 0 AND NOT workbench_messages.sealed THEN EXCLUDED.message_json ELSE workbench_messages.message_json END, updated_at = CASE WHEN workbench_messages.projected_seq <= 0 AND NOT workbench_messages.sealed THEN EXCLUDED.updated_at ELSE workbench_messages.updated_at END", + [record.messageId, record.sessionId, record.turnId, record.traceId, record.role, record.status, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record), record.createdAt, record.updatedAt] + ); + } + async persistWorkbenchPartFact(record, client = this) { await client.query( "INSERT INTO workbench_parts (part_id, message_id, session_id, turn_id, trace_id, part_index, part_type, status, projected_seq, source_seq, source_event_id, terminal, sealed, part_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) ON CONFLICT (part_id) DO UPDATE SET message_id = EXCLUDED.message_id, session_id = EXCLUDED.session_id, turn_id = EXCLUDED.turn_id, trace_id = EXCLUDED.trace_id, part_index = EXCLUDED.part_index, part_type = EXCLUDED.part_type, status = EXCLUDED.status, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, part_json = EXCLUDED.part_json, updated_at = EXCLUDED.updated_at", @@ -1148,6 +1305,13 @@ export class PostgresCloudRuntimeStore { ); } + async persistWorkbenchPartAdmissionFact(record, client = this) { + await client.query( + "INSERT INTO workbench_parts (part_id, message_id, session_id, turn_id, trace_id, part_index, part_type, status, projected_seq, source_seq, source_event_id, terminal, sealed, part_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) ON CONFLICT (part_id) DO UPDATE SET message_id = EXCLUDED.message_id, session_id = EXCLUDED.session_id, turn_id = EXCLUDED.turn_id, trace_id = EXCLUDED.trace_id, part_index = EXCLUDED.part_index, part_type = EXCLUDED.part_type, status = CASE WHEN workbench_parts.projected_seq <= 0 AND NOT workbench_parts.sealed THEN EXCLUDED.status ELSE workbench_parts.status END, source_event_id = CASE WHEN workbench_parts.projected_seq <= 0 AND NOT workbench_parts.sealed THEN EXCLUDED.source_event_id ELSE workbench_parts.source_event_id END, part_json = CASE WHEN workbench_parts.projected_seq <= 0 AND NOT workbench_parts.sealed THEN EXCLUDED.part_json ELSE workbench_parts.part_json END, updated_at = CASE WHEN workbench_parts.projected_seq <= 0 AND NOT workbench_parts.sealed THEN EXCLUDED.updated_at ELSE workbench_parts.updated_at END", + [record.partId, record.messageId, record.sessionId, record.turnId, record.traceId, record.partIndex, record.partType, record.status, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record), record.createdAt, record.updatedAt] + ); + } + async persistWorkbenchTurnFact(record, client = this) { await client.query( "INSERT INTO workbench_turns (turn_id, session_id, trace_id, message_id, status, projected_seq, source_seq, source_event_id, terminal, sealed, final_response_json, diagnostic_json, turn_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (turn_id) DO UPDATE SET session_id = EXCLUDED.session_id, trace_id = EXCLUDED.trace_id, message_id = EXCLUDED.message_id, status = EXCLUDED.status, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, final_response_json = EXCLUDED.final_response_json, diagnostic_json = EXCLUDED.diagnostic_json, turn_json = EXCLUDED.turn_json, updated_at = EXCLUDED.updated_at", @@ -1155,6 +1319,13 @@ export class PostgresCloudRuntimeStore { ); } + async persistWorkbenchTurnAdmissionFact(record, client = this) { + await client.query( + "INSERT INTO workbench_turns (turn_id, session_id, trace_id, message_id, status, projected_seq, source_seq, source_event_id, terminal, sealed, final_response_json, diagnostic_json, turn_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (turn_id) DO UPDATE SET session_id = EXCLUDED.session_id, trace_id = EXCLUDED.trace_id, message_id = CASE WHEN workbench_turns.projected_seq <= 0 AND NOT workbench_turns.sealed THEN EXCLUDED.message_id ELSE workbench_turns.message_id END, status = CASE WHEN workbench_turns.projected_seq <= 0 AND NOT workbench_turns.sealed THEN EXCLUDED.status ELSE workbench_turns.status END, source_event_id = CASE WHEN workbench_turns.projected_seq <= 0 AND NOT workbench_turns.sealed THEN EXCLUDED.source_event_id ELSE workbench_turns.source_event_id END, diagnostic_json = CASE WHEN workbench_turns.projected_seq <= 0 AND NOT workbench_turns.sealed THEN EXCLUDED.diagnostic_json ELSE workbench_turns.diagnostic_json END, turn_json = CASE WHEN workbench_turns.projected_seq <= 0 AND NOT workbench_turns.sealed THEN EXCLUDED.turn_json ELSE workbench_turns.turn_json END, updated_at = CASE WHEN workbench_turns.projected_seq <= 0 AND NOT workbench_turns.sealed THEN EXCLUDED.updated_at ELSE workbench_turns.updated_at END", + [record.turnId, record.sessionId, record.traceId, record.messageId, record.status, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record.finalResponse), stableJson(record.diagnostic), stableJson(record), record.createdAt, record.updatedAt] + ); + } + async persistWorkbenchTraceEventFact(record, client = this) { await client.query( "INSERT INTO workbench_trace_events (id, trace_id, session_id, turn_id, message_id, source_seq, source_event_id, projected_seq, event_type, terminal, sealed, event_json, occurred_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT (id) DO UPDATE SET trace_id = EXCLUDED.trace_id, session_id = EXCLUDED.session_id, turn_id = EXCLUDED.turn_id, message_id = EXCLUDED.message_id, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, projected_seq = EXCLUDED.projected_seq, event_type = EXCLUDED.event_type, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, event_json = EXCLUDED.event_json, occurred_at = EXCLUDED.occurred_at, updated_at = EXCLUDED.updated_at", @@ -1164,8 +1335,8 @@ export class PostgresCloudRuntimeStore { async persistWorkbenchProjectionOutbox(record, client = this) { await client.query( - "INSERT INTO workbench_projection_outbox (event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)", - [record.eventSeq, record.aggregateId, nonNegativeInteger(record.aggregateSeq), nonNegativeInteger(record.projectionRevision), record.traceId, record.sessionId, record.turnId, record.messageId, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.commitType ?? "event", Boolean(record.terminal), Boolean(record.sealed), stableJson(record.payload ?? record), record.createdAt ?? this.now()] + "INSERT INTO workbench_projection_outbox (outbox_event_id, entity_family, entity_id, event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19) ON CONFLICT (outbox_event_id) DO NOTHING", + [record.outboxEventId ?? projectionOutboxEventId(record), record.entityFamily ?? projectionOutboxEntity(record).family, record.entityId ?? projectionOutboxEntity(record).id, record.eventSeq, record.aggregateId, nonNegativeInteger(record.aggregateSeq), nonNegativeInteger(record.projectionRevision), record.traceId, record.sessionId, record.turnId, record.messageId, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.commitType ?? "event", Boolean(record.terminal), Boolean(record.sealed), stableJson(record.payload ?? record), record.createdAt ?? this.now()] ); } @@ -1186,11 +1357,14 @@ export class PostgresCloudRuntimeStore { } params.push(Math.min(Math.max(Number(limit) || 100, 1), 500)); const result = await this.query( - `SELECT outbox_seq, event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at FROM workbench_projection_outbox WHERE ${where} ORDER BY outbox_seq ASC LIMIT $${paramIdx}`, + `SELECT outbox_seq, outbox_event_id, entity_family, entity_id, event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at FROM workbench_projection_outbox WHERE ${where} ORDER BY outbox_seq ASC LIMIT $${paramIdx}`, params ); return (result.rows ?? []).map((row) => ({ outboxSeq: Number(row.outbox_seq), + outboxEventId: row.outbox_event_id, + entityFamily: row.entity_family, + entityId: row.entity_id, eventSeq: row.event_seq === null || row.event_seq === undefined ? null : Number(row.event_seq), aggregateId: row.aggregate_id, aggregateSeq: Number(row.aggregate_seq ?? 0), @@ -1248,6 +1422,45 @@ export class PostgresCloudRuntimeStore { }; } + async workbenchTransactionalRealtimeReadiness() { + const schemaResult = await this.query( + "SELECT table_name, column_name FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = ANY($1::text[])", + [Object.keys(CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS)] + ); + const schema = summarizeRuntimeSchema(schemaResult?.rows ?? [], CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS); + const migrationResult = await this.query( + `SELECT id, schema_version FROM ${CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE} WHERE id = $1 LIMIT 1`, + [CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID] + ); + const row = migrationResult.rows?.[0] ?? null; + const migration = { + checked: true, + ready: row?.id === CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID && row?.schema_version === CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION, + table: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, + requiredMigrationId: CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID, + requiredSchemaVersion: CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION, + appliedMigrationId: row?.id ?? null, + appliedSchemaVersion: row?.schema_version ?? null, + missing: !row + }; + return { + ready: schema.ready && migration.ready, + schema, + migration, + valuesRedacted: true + }; + } + + async assertWorkbenchTransactionalRealtimeReady() { + const readiness = await this.workbenchTransactionalRealtimeReadiness(); + if (readiness.ready) return readiness; + const error = new Error("Workbench transactional realtime schema or migration is not ready."); + error.code = readiness.schema.ready ? "workbench_transactional_realtime_migration_blocked" : "workbench_transactional_realtime_schema_blocked"; + error.readiness = readiness; + error.valuesRedacted = true; + throw error; + } + async ensureRuntimeReadIndexes() { if (this.runtimeReadIndexesReady) return; if (!this.runtimeReadIndexesReadyPromise) { @@ -1535,3 +1748,28 @@ export class PostgresCloudRuntimeStore { }); } } + +function projectionOutboxEventId(record = {}) { + const sourceEventId = textOr(record.sourceEventId, ""); + const commitType = textOr(record.commitType, "event"); + const family = textOr(record.entityFamily ?? record.payload?.family, "") || (commitType === "message" ? "messages" : record.payload?.type ? "traceEvents" : "turns"); + const entityId = family === "traceEvents" + ? textOr(record.entityId ?? record.payload?.fact?.id ?? record.aggregateId ?? record.sourceEventId ?? record.traceId, "projection") + : textOr(record.entityId ?? record.payload?.fact?.messageId ?? record.payload?.fact?.turnId ?? record.messageId ?? record.turnId ?? record.traceId ?? record.aggregateId, "projection"); + if (sourceEventId) return `${sourceEventId}:${family}:${entityId}:${commitType}`; + const derived = stableJson({ family, entityId, commitType, eventSeq: record.eventSeq ?? null, aggregateId: record.aggregateId ?? null, aggregateSeq: record.aggregateSeq ?? 0, projectionRevision: record.projectionRevision ?? record.projectedSeq ?? 0, traceId: record.traceId ?? null, sessionId: record.sessionId ?? null }); + return `derived:${createHash("sha256").update(derived).digest("hex")}`; +} + +function projectionOutboxEntity(record = {}) { + const commitType = textOr(record.commitType, "event"); + const family = textOr(record.entityFamily ?? record.payload?.family, "") || (commitType === "message" ? "messages" : record.payload?.type ? "traceEvents" : "turns"); + const id = family === "traceEvents" + ? textOr(record.entityId ?? record.payload?.fact?.id ?? record.aggregateId ?? record.sourceEventId ?? record.traceId, "projection") + : textOr(record.entityId ?? record.payload?.fact?.messageId ?? record.payload?.fact?.turnId ?? record.messageId ?? record.turnId ?? record.traceId ?? record.aggregateId, "projection"); + return { family, id }; +} + +function immutableProjectionOutboxPayload(family, fact, metadata = {}) { + return { family, fact, metadata: { ...metadata, valuesRedacted: true }, valuesRedacted: true }; +} diff --git a/internal/db/runtime-store.test.ts b/internal/db/runtime-store.test.ts index d46f4a98..51370388 100644 --- a/internal/db/runtime-store.test.ts +++ b/internal/db/runtime-store.test.ts @@ -1605,7 +1605,7 @@ function createFakePostgresClient({ if (beforeWorkbenchFactRead) await beforeWorkbenchFactRead(sql, params); return workbenchFactRows(state.workbench_projection_checkpoints, "checkpoint_json", sql, params, readErrorCode); } - if (sql.startsWith("SELECT outbox_seq, event_seq, aggregate_id")) { + if (sql.startsWith("SELECT outbox_seq, outbox_event_id, entity_family, entity_id")) { const afterSeq = Number(params[0] ?? 0); const traceFilter = sql.includes("trace_id = $2") ? params[1] : null; const sessionFilter = sql.includes("session_id = $2") ? params[1] : sql.includes("session_id = $3") ? params[2] : null; @@ -1770,25 +1770,30 @@ function createFakePostgresClient({ error.code = outboxErrorCode; throw error; } + const existing = [...state.workbench_projection_outbox.values()].find((record) => record.outbox_event_id === params[0]); + if (existing) return { rows: [] }; const outboxSeq = state.workbench_projection_outbox.size + 1; state.workbench_projection_outbox.set(outboxSeq, { outbox_seq: outboxSeq, - event_seq: params[0], - aggregate_id: params[1], - aggregate_seq: params[2], - projection_revision: params[3], - trace_id: params[4], - session_id: params[5], - turn_id: params[6], - message_id: params[7], - projected_seq: params[8], - source_seq: params[9], - source_event_id: params[10], - commit_type: params[11], - terminal: params[12], - sealed: params[13], - payload_json: params[14], - created_at: params[15] + outbox_event_id: params[0], + entity_family: params[1], + entity_id: params[2], + event_seq: params[3], + aggregate_id: params[4], + aggregate_seq: params[5], + projection_revision: params[6], + trace_id: params[7], + session_id: params[8], + turn_id: params[9], + message_id: params[10], + projected_seq: params[11], + source_seq: params[12], + source_event_id: params[13], + commit_type: params[14], + terminal: params[15], + sealed: params[16], + payload_json: params[17], + created_at: params[18] }); return { rows: [] }; } diff --git a/internal/db/schema.test.ts b/internal/db/schema.test.ts index 44827f0f..8814efe8 100644 --- a/internal/db/schema.test.ts +++ b/internal/db/schema.test.ts @@ -12,6 +12,8 @@ import { CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE_COLUMNS, CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS, + CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID, + CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION, FROZEN_CLOUD_CORE_TABLES, assertFrozenCloudCoreTables, requiredRuntimeDurableColumns @@ -41,15 +43,17 @@ test("initial migration skeleton declares every frozen table", async () => { assert.match(sql, /\btimestamp\b/); }); -test("initial migration exposes columns required by the durable runtime adapter", async () => { - const sql = await readFile(path.join(repoRoot, "internal/db/migrations/0001_cloud_core_skeleton.sql"), "utf8"); +test("versioned migration chain exposes columns required by the durable runtime adapter", async () => { + const sql = await readMigrationChain(); assert.ok(requiredRuntimeDurableColumns().length > 0); for (const [table, columns] of Object.entries(CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS)) { const tableMatch = sql.match(new RegExp(`CREATE TABLE IF NOT EXISTS ${table}\\s*\\(([\\s\\S]*?)\\);`, "u")); assert.ok(tableMatch, `missing ${table}`); for (const column of columns) { - assert.match(tableMatch[1], new RegExp(`\\b${column}\\b`), `missing ${table}.${column}`); + const declared = new RegExp(`\\b${column}\\b`).test(tableMatch[1]); + const added = new RegExp(`ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS ${column}\\b`, "u").test(sql); + assert.equal(declared || added, true, `missing ${table}.${column}`); } } @@ -92,6 +96,36 @@ test("initial migration declares Workbench fact backfill sources", async () => { assert.match(sql, /CREATE UNIQUE INDEX idx_workbench_trace_events_trace_projected_seq/u); }); +test("v8 migration upgrades the existing Workbench outbox before enforcing realtime identity", async () => { + const sql = await readFile(path.join(repoRoot, "internal/db/migrations/0008_workbench_kafka_realtime.sql"), "utf8"); + + for (const column of ["outbox_event_id", "entity_family", "entity_id"]) { + assert.match( + sql, + new RegExp(`ALTER TABLE workbench_projection_outbox ADD COLUMN IF NOT EXISTS ${column} TEXT`, "u"), + `missing additive upgrade for workbench_projection_outbox.${column}` + ); + assert.match( + sql, + new RegExp(`ALTER TABLE workbench_projection_outbox ALTER COLUMN ${column} SET NOT NULL`, "u"), + `missing post-backfill constraint for workbench_projection_outbox.${column}` + ); + } + + assert.match(sql, /SET outbox_event_id = COALESCE\(outbox_event_id, 'legacy:' \|\| outbox_seq::text\)/u); + assert.match(sql, /entity_family = COALESCE\(entity_family, 'legacy'\)/u); + assert.match(sql, /CREATE UNIQUE INDEX idx_workbench_projection_outbox_event_id ON workbench_projection_outbox\(outbox_event_id\)/u); + assert.match(sql, /CREATE TRIGGER trg_notify_hwlab_workbench_projection/u); + assert.match(sql, new RegExp(`'${CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID}'`, "u")); + assert.match(sql, new RegExp(`'${CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION}'`, "u")); + assert.match(sql, /ON CONFLICT \(id\) DO UPDATE SET\s+schema_version = EXCLUDED\.schema_version/u); +}); + +async function readMigrationChain() { + const sources = await Promise.all(CLOUD_CORE_MIGRATIONS.map((migration) => readFile(path.join(repoRoot, migration.path), "utf8"))); + return sources.join("\n"); +} + test("protocol record guards catch schema drift before runtime writes", () => { assertProtocolRecord("gatewaySession", { gatewaySessionId: "gws_01J00000000000000000000000", diff --git a/internal/db/schema.ts b/internal/db/schema.ts index fe57c0d2..5cd9f440 100644 --- a/internal/db/schema.ts +++ b/internal/db/schema.ts @@ -6,6 +6,8 @@ import { TABLES } from "../protocol/index.mjs"; export const CLOUD_CORE_MIGRATION_ID = "0001_cloud_core_skeleton"; export const CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION = "runtime-durable-postgres-v7"; +export const CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID = "0008_workbench_kafka_realtime"; +export const CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION = "runtime-durable-postgres-v8"; export const CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE = "hwlab_schema_migrations"; export const FROZEN_CLOUD_CORE_TABLES = Object.freeze([...TABLES]); @@ -337,13 +339,86 @@ export const CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS = Object.freeze({ ]) }); +export const CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS = Object.freeze({ + workbench_projection_outbox: Object.freeze([ + "outbox_event_id", + "entity_family", + "entity_id" + ]), + workbench_kafka_inbox: Object.freeze([ + "source_topic", + "source_partition", + "source_offset", + "source_key", + "source_event_id", + "input_sha256", + "trace_id", + "session_id", + "run_id", + "command_id", + "source_seq", + "status", + "projected_seq", + "envelope_json", + "error_json", + "created_at", + "updated_at", + "projected_at" + ]), + workbench_kafka_dlq: Object.freeze([ + "dlq_seq", + "source_topic", + "source_partition", + "source_offset", + "source_event_id", + "input_sha256", + "error_code", + "error_message", + "envelope_json", + "created_at" + ]), + hwlab_kafka_outbox: Object.freeze([ + "outbox_seq", + "event_id", + "topic", + "partition_key", + "payload_json", + "headers_json", + "attempt_count", + "next_attempt_at", + "lease_owner", + "lease_expires_at", + "published_at", + "last_error", + "created_at", + "updated_at" + ]) +}); + +export const CLOUD_TRANSACTIONAL_REALTIME_REQUIRED_TABLE_COLUMNS = Object.freeze({ + ...CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS, + ...CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS, + workbench_projection_outbox: Object.freeze([ + ...CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS.workbench_projection_outbox, + ...CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS.workbench_projection_outbox + ]) +}); + export const CLOUD_CORE_MIGRATIONS = Object.freeze([ { id: CLOUD_CORE_MIGRATION_ID, path: "internal/db/migrations/0001_cloud_core_skeleton.sql", tables: FROZEN_CLOUD_CORE_TABLES, connectsToDatabase: false, - runtimeDurableAdapterSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + runtimeDurableAdapterSchemaVersion: "runtime-durable-postgres-v7", + runtimeDurableMigrationTable: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE + }, + { + id: CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID, + path: "internal/db/migrations/0008_workbench_kafka_realtime.sql", + tables: Object.freeze(["workbench_projection_outbox", "workbench_kafka_inbox", "workbench_kafka_dlq", "hwlab_kafka_outbox"]), + connectsToDatabase: false, + runtimeDurableAdapterSchemaVersion: CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION, runtimeDurableMigrationTable: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE } ]); @@ -360,3 +435,9 @@ export function requiredRuntimeDurableColumns() { columns.map((column) => `${table}.${column}`) ); } + +export function requiredTransactionalRealtimeColumns() { + return Object.entries(CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS).flatMap(([table, columns]) => + columns.map((column) => `${table}.${column}`) + ); +} diff --git a/internal/dev-entrypoint/cloud-web-runtime.mjs b/internal/dev-entrypoint/cloud-web-runtime.mjs index 52e30f80..8fbb4a73 100644 --- a/internal/dev-entrypoint/cloud-web-runtime.mjs +++ b/internal/dev-entrypoint/cloud-web-runtime.mjs @@ -390,7 +390,12 @@ function validateDisplayLocale(locale) { function workbenchRuntimeConfigFromEnv() { const traceTimeline = {}; - const result = {}; + const result = { + realtimeFeatures: { + liveKafkaSse: requiredWorkbenchRealtimeFeature(process.env.HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED, "HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED"), + projectionRealtime: requiredWorkbenchRealtimeFeature(process.env.HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED, "HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED") + } + }; const autoExpandRunning = parseEnvBoolean(process.env.HWLAB_WORKBENCH_TRACE_AUTO_EXPAND_RUNNING); const autoCollapseTerminal = parseEnvBoolean(process.env.HWLAB_WORKBENCH_TRACE_AUTO_COLLAPSE_TERMINAL); const traceExplorerUrlTemplate = traceExplorerUrlTemplateFromEnv(process.env.HWLAB_WORKBENCH_TRACE_EXPLORER_URL_TEMPLATE); @@ -401,6 +406,13 @@ function workbenchRuntimeConfigFromEnv() { return Object.keys(result).length > 0 ? result : null; } +function requiredWorkbenchRealtimeFeature(value, name) { + const normalized = String(value ?? "").trim().toLowerCase(); + if (normalized === "true" || normalized === "1") return true; + if (normalized === "false" || normalized === "0") return false; + throw new Error(`${name} is required and must be explicitly true or false`); +} + function traceExplorerUrlTemplateFromEnv(value) { if (value === undefined || value === null || value === "") return null; const template = String(value).trim(); diff --git a/internal/dev-entrypoint/cloud-web-runtime.test.mjs b/internal/dev-entrypoint/cloud-web-runtime.test.mjs index 5f0512b2..d16bc98c 100644 --- a/internal/dev-entrypoint/cloud-web-runtime.test.mjs +++ b/internal/dev-entrypoint/cloud-web-runtime.test.mjs @@ -429,7 +429,9 @@ test("cloud web serves client deep links through the Vue shell", async () => { const restoreEnv = withEnv({ HWLAB_CLOUD_WEB_DISPLAY_TIME_ZONE: "Asia/Shanghai", HWLAB_CLOUD_WEB_DISPLAY_TIME_LOCALE: "zh-CN", - HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: "北京时间" + HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: "北京时间", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" }); const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-cloud-web-runtime-")); await writeFile(path.join(root, "index.html"), "
\n", "utf8"); @@ -484,7 +486,9 @@ test("cloud web injects trace explorer runtime config from env", async () => { HWLAB_CLOUD_WEB_DISPLAY_TIME_ZONE: "Asia/Shanghai", HWLAB_CLOUD_WEB_DISPLAY_TIME_LOCALE: "zh-CN", HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: "北京时间", - HWLAB_WORKBENCH_TRACE_EXPLORER_URL_TEMPLATE: "/v1/workbench/traces/{trace_id}/events" + HWLAB_WORKBENCH_TRACE_EXPLORER_URL_TEMPLATE: "/v1/workbench/traces/{trace_id}/events", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" }); const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-cloud-web-runtime-")); await writeFile(path.join(root, "index.html"), "
\n", "utf8"); @@ -588,7 +592,9 @@ test("cloud web OpenCode frame-url endpoint mints traced tickets", async () => { HWLAB_CLOUD_WEB_DISPLAY_TIME_ZONE: "Asia/Shanghai", HWLAB_CLOUD_WEB_DISPLAY_TIME_LOCALE: "zh-CN", HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: "北京时间", - HWLAB_CLOUD_WEB_OPENCODE_URL: "https://opencode.example.test" + HWLAB_CLOUD_WEB_OPENCODE_URL: "https://opencode.example.test", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" }); const cloudApiRequests = []; const opencodeRequests = []; @@ -1028,7 +1034,9 @@ test("cloud web OpenCode proxy accepts short-lived tickets minted by the shell", HWLAB_CLOUD_WEB_DISPLAY_TIME_ZONE: "Asia/Shanghai", HWLAB_CLOUD_WEB_DISPLAY_TIME_LOCALE: "zh-CN", HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: "北京时间", - HWLAB_CLOUD_WEB_OPENCODE_URL: "https://opencode.example.test" + HWLAB_CLOUD_WEB_OPENCODE_URL: "https://opencode.example.test", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" }); const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-cloud-web-runtime-")); await writeFile(path.join(root, "index.html"), "
\n", "utf8"); diff --git a/internal/workbenchruntime/realtime_sync.go b/internal/workbenchruntime/realtime_sync.go new file mode 100644 index 00000000..96b5b4bf --- /dev/null +++ b/internal/workbenchruntime/realtime_sync.go @@ -0,0 +1,493 @@ +package workbenchruntime + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" +) + +const ( + realtimeSyncContractVersion = "workbench-runtime-sync-v1" + realtimeSyncDefaultLimit = 100 + realtimeSyncMaxLimit = 500 + realtimeSyncMaxIdentityLen = 512 +) + +type realtimeSyncQuery struct { + SessionID string `json:"sessionId"` + TraceID string `json:"traceId"` + AfterOutboxSeq int64 `json:"afterOutboxSeq"` + Limit int `json:"limit"` + SnapshotOnly bool `json:"snapshotOnly,omitempty"` + DeltaOnly bool `json:"deltaOnly,omitempty"` + Actor sessionActor `json:"actor"` +} + +type realtimeSyncSnapshot struct { + Facts map[string][]any `json:"facts"` + Events []realtimeSyncEvent `json:"events"` + CutoffOutboxSeq int64 `json:"cutoffOutboxSeq"` + CursorOutboxSeq int64 `json:"cursorOutboxSeq"` + HasMore bool `json:"hasMore"` +} + +type realtimeSyncEvent struct { + OutboxSeq int64 `json:"outboxSeq"` + OutboxEventID string `json:"outboxEventId"` + EntityFamily string `json:"entityFamily"` + EntityID string `json:"entityId"` + EventSeq any `json:"eventSeq"` + AggregateID any `json:"aggregateId"` + AggregateSeq int64 `json:"aggregateSeq"` + ProjectionRevision int64 `json:"projectionRevision"` + TraceID string `json:"traceId"` + SessionID any `json:"sessionId"` + TurnID any `json:"turnId"` + MessageID any `json:"messageId"` + ProjectedSeq int64 `json:"projectedSeq"` + SourceSeq int64 `json:"sourceSeq"` + SourceEventID any `json:"sourceEventId"` + CommitType string `json:"commitType"` + Terminal bool `json:"terminal"` + Sealed bool `json:"sealed"` + Payload any `json:"payload"` + CreatedAt string `json:"createdAt"` + ValuesRedacted bool `json:"valuesRedacted"` +} + +type realtimeSyncFactSpec struct { + Family string + Table string + JSONColumn string + OrderBy string +} + +var realtimeSyncFactSpecs = []realtimeSyncFactSpec{ + {Family: "sessions", Table: "workbench_sessions", JSONColumn: "session_json", OrderBy: "updated_at ASC, session_id ASC"}, + {Family: "messages", Table: "workbench_messages", JSONColumn: "message_json", OrderBy: "updated_at ASC, message_id ASC"}, + {Family: "parts", Table: "workbench_parts", JSONColumn: "part_json", OrderBy: "updated_at ASC, part_index ASC, part_id ASC"}, + {Family: "turns", Table: "workbench_turns", JSONColumn: "turn_json", OrderBy: "updated_at ASC, turn_id ASC"}, + {Family: "traceEvents", Table: "workbench_trace_events", JSONColumn: "event_json", OrderBy: "projected_seq ASC, id ASC"}, + {Family: "checkpoints", Table: "workbench_projection_checkpoints", JSONColumn: "checkpoint_json", OrderBy: "updated_at ASC, trace_id ASC"}, +} + +var realtimeSyncSnapshotFactSpecs = []realtimeSyncFactSpec{ + realtimeSyncFactSpecs[0], + realtimeSyncFactSpecs[1], + realtimeSyncFactSpecs[3], +} + +const realtimeSyncResolveScopeSQL = ` +SELECT sessions.session_id +FROM workbench_sessions AS sessions +WHERE ($1 = '' OR sessions.session_id = $1) + AND ( + $2 = '' + OR sessions.last_trace_id = $2 + OR EXISTS ( + SELECT 1 + FROM workbench_projection_outbox AS scoped_outbox + WHERE scoped_outbox.session_id = sessions.session_id + AND scoped_outbox.trace_id = $2 + ) + ) + AND ($3 OR sessions.owner_user_id = $4) +ORDER BY + CASE WHEN sessions.session_id = NULLIF($1, '') THEN 0 ELSE 1 END, + sessions.updated_at DESC, + sessions.session_id ASC +LIMIT 1` + +const realtimeSyncCutoffSQL = ` +SELECT COALESCE(MAX(outbox_seq), 0)::bigint +FROM workbench_projection_outbox +WHERE session_id = $1 + AND ($2 = '' OR trace_id = $2)` + +const realtimeSyncEventsSQL = ` +SELECT + outbox_seq, + outbox_event_id, + entity_family, + entity_id, + event_seq, + aggregate_id, + aggregate_seq, + projection_revision, + trace_id, + session_id, + turn_id, + message_id, + projected_seq, + source_seq, + source_event_id, + commit_type, + terminal, + sealed, + payload_json, + created_at +FROM workbench_projection_outbox +WHERE session_id = $1 + AND ($2 = '' OR trace_id = $2) + AND outbox_seq > $3 + AND outbox_seq <= $4 +ORDER BY outbox_seq ASC +LIMIT $5` + +func (s *Server) handleRealtimeSync(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + methodNotAllowed(w, "POST") + return + } + query, ok := decodeRealtimeSyncQuery(w, r) + if !ok { + return + } + query, validation := normalizeRealtimeSyncQuery(query) + if validation != nil { + writeAPIError(w, validation.Status, validation.Code, validation.Message) + return + } + ctx, cancel := context.WithTimeout(r.Context(), s.config.QueryTimeout) + defer cancel() + snapshot, err := s.readRealtimeSync(ctx, query) + if errors.Is(err, sql.ErrNoRows) { + writeAPIError(w, http.StatusNotFound, "workbench_sync_scope_not_found", "workbench sync scope was not found") + return + } + if err != nil { + writeQueryError(w, err) + return + } + writeJSON(w, http.StatusOK, map[string]any{ + "ok": true, + "status": "succeeded", + "contractVersion": realtimeSyncContractVersion, + "facts": snapshot.Facts, + "events": snapshot.Events, + "cutoffOutboxSeq": snapshot.CutoffOutboxSeq, + "cursorOutboxSeq": snapshot.CursorOutboxSeq, + "hasMore": snapshot.HasMore, + "persistence": s.persistenceSummary(), + "servedBy": serviceID, + "valuesRedacted": true, + "secretMaterialStored": false, + }) +} + +type realtimeSyncValidationError struct { + Status int + Code string + Message string +} + +func decodeRealtimeSyncQuery(w http.ResponseWriter, r *http.Request) (realtimeSyncQuery, bool) { + defer r.Body.Close() + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)) + decoder.DisallowUnknownFields() + var query realtimeSyncQuery + if err := decoder.Decode(&query); err != nil { + writeAPIError(w, http.StatusBadRequest, "invalid_json", "request body must be a closed workbench sync JSON object") + return realtimeSyncQuery{}, false + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + writeAPIError(w, http.StatusBadRequest, "invalid_json", "request body must contain exactly one JSON object") + return realtimeSyncQuery{}, false + } + return query, true +} + +func normalizeRealtimeSyncQuery(query realtimeSyncQuery) (realtimeSyncQuery, *realtimeSyncValidationError) { + query.SessionID = strings.TrimSpace(query.SessionID) + query.TraceID = strings.TrimSpace(query.TraceID) + query.Actor.ID = strings.TrimSpace(query.Actor.ID) + query.Actor.Role = strings.ToLower(strings.TrimSpace(query.Actor.Role)) + if query.SessionID == "" && query.TraceID == "" { + return query, &realtimeSyncValidationError{Status: http.StatusBadRequest, Code: "workbench_sync_scope_required", Message: "sessionId or traceId is required"} + } + if len(query.SessionID) > realtimeSyncMaxIdentityLen || len(query.TraceID) > realtimeSyncMaxIdentityLen || len(query.Actor.ID) > realtimeSyncMaxIdentityLen { + return query, &realtimeSyncValidationError{Status: http.StatusBadRequest, Code: "workbench_sync_identity_invalid", Message: "workbench sync identifiers are too long"} + } + if query.AfterOutboxSeq < 0 { + return query, &realtimeSyncValidationError{Status: http.StatusBadRequest, Code: "workbench_sync_cursor_invalid", Message: "afterOutboxSeq must be non-negative"} + } + if query.Limit < 0 { + return query, &realtimeSyncValidationError{Status: http.StatusBadRequest, Code: "workbench_sync_limit_invalid", Message: "limit must be non-negative"} + } + if query.SnapshotOnly && query.DeltaOnly { + return query, &realtimeSyncValidationError{Status: http.StatusBadRequest, Code: "workbench_sync_mode_invalid", Message: "snapshotOnly and deltaOnly are mutually exclusive"} + } + query.Limit = boundedLimit(query.Limit, realtimeSyncDefaultLimit, realtimeSyncMaxLimit) + if query.Actor.Role != "admin" && query.Actor.Role != "user" { + return query, &realtimeSyncValidationError{Status: http.StatusBadRequest, Code: "workbench_sync_actor_invalid", Message: "actor.role must be user or admin"} + } + if query.Actor.Role != "admin" && query.Actor.ID == "" { + return query, &realtimeSyncValidationError{Status: http.StatusForbidden, Code: "workbench_sync_actor_required", Message: "ordinary users require an actor id"} + } + return query, nil +} + +func realtimeSyncTxOptions() *sql.TxOptions { + return &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true} +} + +func (s *Server) readRealtimeSync(ctx context.Context, query realtimeSyncQuery) (realtimeSyncSnapshot, error) { + tx, err := s.db.BeginTx(ctx, realtimeSyncTxOptions()) + if err != nil { + return realtimeSyncSnapshot{}, wrapQueryStage("workbench_runtime.db.sync.begin", err) + } + committed := false + defer func() { + if !committed { + _ = tx.Rollback() + } + }() + snapshot, err := readRealtimeSyncSnapshot(ctx, sqlRealtimeSyncQueryer{tx: tx}, query) + if err != nil { + return realtimeSyncSnapshot{}, err + } + if err := tx.Commit(); err != nil { + return realtimeSyncSnapshot{}, wrapQueryStage("workbench_runtime.db.sync.commit", err) + } + committed = true + return snapshot, nil +} + +type realtimeSyncRow interface { + Scan(...any) error +} + +type realtimeSyncRows interface { + Close() error + Err() error + Next() bool + Scan(...any) error +} + +type realtimeSyncQueryer interface { + QueryContext(context.Context, string, ...any) (realtimeSyncRows, error) + QueryRowContext(context.Context, string, ...any) realtimeSyncRow +} + +type sqlRealtimeSyncQueryer struct { + tx *sql.Tx +} + +func (queryer sqlRealtimeSyncQueryer) QueryContext(ctx context.Context, query string, args ...any) (realtimeSyncRows, error) { + return queryer.tx.QueryContext(ctx, query, args...) +} + +func (queryer sqlRealtimeSyncQueryer) QueryRowContext(ctx context.Context, query string, args ...any) realtimeSyncRow { + return queryer.tx.QueryRowContext(ctx, query, args...) +} + +func readRealtimeSyncSnapshot(ctx context.Context, tx realtimeSyncQueryer, query realtimeSyncQuery) (realtimeSyncSnapshot, error) { + sessionID, err := resolveRealtimeSyncSession(ctx, tx, query) + if err != nil { + return realtimeSyncSnapshot{}, err + } + cutoff, err := readRealtimeSyncCutoff(ctx, tx, sessionID, query.TraceID) + if err != nil { + return realtimeSyncSnapshot{}, err + } + events := make([]realtimeSyncEvent, 0) + hasMore := false + if !query.SnapshotOnly { + allEvents, readErr := readRealtimeSyncEvents(ctx, tx, sessionID, query.TraceID, query.AfterOutboxSeq, cutoff, query.Limit+1) + if readErr != nil { + return realtimeSyncSnapshot{}, readErr + } + hasMore = len(allEvents) > query.Limit + events = allEvents + if hasMore { + events = allEvents[:query.Limit] + } + } + facts := make(map[string][]any) + if !query.DeltaOnly { + specs := realtimeSyncFactSpecs + if query.SnapshotOnly { + specs = realtimeSyncSnapshotFactSpecs + } + facts, err = readRealtimeSyncFacts(ctx, tx, sessionID, query.TraceID, specs) + if err != nil { + return realtimeSyncSnapshot{}, err + } + } + cursor := query.AfterOutboxSeq + if query.SnapshotOnly || cursor > cutoff { + cursor = cutoff + } + if len(events) > 0 { + cursor = events[len(events)-1].OutboxSeq + } + return realtimeSyncSnapshot{Facts: facts, Events: events, CutoffOutboxSeq: cutoff, CursorOutboxSeq: cursor, HasMore: hasMore}, nil +} + +func resolveRealtimeSyncSession(ctx context.Context, tx realtimeSyncQueryer, query realtimeSyncQuery) (string, error) { + var sessionID string + err := tx.QueryRowContext(ctx, realtimeSyncResolveScopeSQL, query.SessionID, query.TraceID, query.Actor.Role == "admin", query.Actor.ID).Scan(&sessionID) + if err != nil { + return "", wrapQueryStage("workbench_runtime.db.sync.scope", err) + } + return sessionID, nil +} + +func readRealtimeSyncCutoff(ctx context.Context, tx realtimeSyncQueryer, sessionID string, traceID string) (int64, error) { + var cutoff int64 + if err := tx.QueryRowContext(ctx, realtimeSyncCutoffSQL, sessionID, traceID).Scan(&cutoff); err != nil { + return 0, wrapQueryStage("workbench_runtime.db.sync.cutoff", err) + } + return cutoff, nil +} + +func readRealtimeSyncEvents(ctx context.Context, tx realtimeSyncQueryer, sessionID string, traceID string, after int64, cutoff int64, limit int) ([]realtimeSyncEvent, error) { + rows, err := tx.QueryContext(ctx, realtimeSyncEventsSQL, sessionID, traceID, after, cutoff, limit) + if err != nil { + return nil, wrapQueryStage("workbench_runtime.db.sync.events", err) + } + defer rows.Close() + events := make([]realtimeSyncEvent, 0) + for rows.Next() { + event, err := scanRealtimeSyncEvent(rows) + if err != nil { + return nil, wrapQueryStage("workbench_runtime.db.sync.events", err) + } + events = append(events, event) + } + if err := rows.Err(); err != nil { + return nil, wrapQueryStage("workbench_runtime.db.sync.events", err) + } + return events, nil +} + +type realtimeSyncScanner interface { + Scan(...any) error +} + +func scanRealtimeSyncEvent(scanner realtimeSyncScanner) (realtimeSyncEvent, error) { + var event realtimeSyncEvent + var outboxEventID, entityFamily, entityID, traceID, commitType, payloadJSON, createdAt sql.NullString + var eventSeq, aggregateSeq, projectionRevision, projectedSeq, sourceSeq sql.NullInt64 + var aggregateID, sessionID, turnID, messageID, sourceEventID sql.NullString + var terminal, sealed sql.NullBool + if err := scanner.Scan( + &event.OutboxSeq, + &outboxEventID, + &entityFamily, + &entityID, + &eventSeq, + &aggregateID, + &aggregateSeq, + &projectionRevision, + &traceID, + &sessionID, + &turnID, + &messageID, + &projectedSeq, + &sourceSeq, + &sourceEventID, + &commitType, + &terminal, + &sealed, + &payloadJSON, + &createdAt, + ); err != nil { + return realtimeSyncEvent{}, err + } + payload := any(map[string]any{}) + if payloadJSON.Valid && strings.TrimSpace(payloadJSON.String) != "" { + if err := json.Unmarshal([]byte(payloadJSON.String), &payload); err != nil { + return realtimeSyncEvent{}, fmt.Errorf("decode projection outbox payload: %w", err) + } + } + event.OutboxEventID = nullableStringValue(outboxEventID) + event.EntityFamily = nullableStringValue(entityFamily) + event.EntityID = nullableStringValue(entityID) + event.EventSeq = realtimeSyncNullableInt(eventSeq) + event.AggregateID = nullableString(aggregateID) + event.AggregateSeq = realtimeSyncInt(aggregateSeq) + event.ProjectionRevision = realtimeSyncInt(projectionRevision) + event.TraceID = nullableStringValue(traceID) + event.SessionID = nullableString(sessionID) + event.TurnID = nullableString(turnID) + event.MessageID = nullableString(messageID) + event.ProjectedSeq = realtimeSyncInt(projectedSeq) + event.SourceSeq = realtimeSyncInt(sourceSeq) + event.SourceEventID = nullableString(sourceEventID) + event.CommitType = nullableStringValue(commitType) + event.Terminal = nullableBool(terminal) + event.Sealed = nullableBool(sealed) + event.Payload = payload + event.CreatedAt = nullableTimeText(createdAt) + event.ValuesRedacted = true + return event, nil +} + +func readRealtimeSyncFacts(ctx context.Context, tx realtimeSyncQueryer, sessionID string, traceID string, specs []realtimeSyncFactSpec) (map[string][]any, error) { + facts := make(map[string][]any, len(specs)) + for _, spec := range specs { + args := []any{sessionID} + where := "session_id = $1" + if spec.Family != "sessions" { + args = append(args, traceID) + where += " AND ($2 = '' OR trace_id = $2)" + } + sqlText := fmt.Sprintf("SELECT %s FROM %s WHERE %s ORDER BY %s", spec.JSONColumn, spec.Table, where, spec.OrderBy) + rows, err := tx.QueryContext(ctx, sqlText, args...) + if err != nil { + return nil, wrapQueryStage("workbench_runtime.db.sync.facts."+spec.Family, err) + } + items, err := scanRealtimeSyncFacts(rows) + if err != nil { + return nil, wrapQueryStage("workbench_runtime.db.sync.facts."+spec.Family, err) + } + facts[spec.Family] = items + } + return facts, nil +} + +func scanRealtimeSyncFacts(rows realtimeSyncRows) ([]any, error) { + defer rows.Close() + items := make([]any, 0) + for rows.Next() { + var raw sql.NullString + if err := rows.Scan(&raw); err != nil { + return nil, err + } + if !raw.Valid || strings.TrimSpace(raw.String) == "" { + continue + } + var item any + if err := json.Unmarshal([]byte(raw.String), &item); err != nil { + return nil, fmt.Errorf("decode workbench fact: %w", err) + } + if item != nil { + items = append(items, item) + } + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +func realtimeSyncNullableInt(value sql.NullInt64) any { + if !value.Valid { + return nil + } + return value.Int64 +} + +func realtimeSyncInt(value sql.NullInt64) int64 { + if !value.Valid { + return 0 + } + return value.Int64 +} diff --git a/internal/workbenchruntime/realtime_sync_test.go b/internal/workbenchruntime/realtime_sync_test.go new file mode 100644 index 00000000..ad52d4f4 --- /dev/null +++ b/internal/workbenchruntime/realtime_sync_test.go @@ -0,0 +1,409 @@ +package workbenchruntime + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestRealtimeSyncTxOptionsAreReadOnlyRepeatableRead(t *testing.T) { + options := realtimeSyncTxOptions() + if options.Isolation != sql.LevelRepeatableRead { + t.Fatalf("isolation=%v, want repeatable read", options.Isolation) + } + if !options.ReadOnly { + t.Fatal("realtime sync transaction must be read only") + } +} + +func TestNormalizeRealtimeSyncQueryRequiresScopedActorAndBoundsLimit(t *testing.T) { + query, validation := normalizeRealtimeSyncQuery(realtimeSyncQuery{ + SessionID: " ses_sync ", + Limit: 900, + Actor: sessionActor{ID: " usr_owner ", Role: " USER "}, + }) + if validation != nil { + t.Fatalf("unexpected validation: %#v", validation) + } + if query.SessionID != "ses_sync" || query.Actor.ID != "usr_owner" || query.Actor.Role != "user" { + t.Fatalf("query was not normalized: %#v", query) + } + if query.Limit != realtimeSyncMaxLimit { + t.Fatalf("limit=%d, want %d", query.Limit, realtimeSyncMaxLimit) + } + + _, validation = normalizeRealtimeSyncQuery(realtimeSyncQuery{TraceID: "trc_sync", Actor: sessionActor{Role: "user"}}) + if validation == nil || validation.Status != http.StatusForbidden || validation.Code != "workbench_sync_actor_required" { + t.Fatalf("ordinary user without owner id validation=%#v", validation) + } + + admin, validation := normalizeRealtimeSyncQuery(realtimeSyncQuery{TraceID: "trc_sync", Actor: sessionActor{Role: "admin"}}) + if validation != nil || admin.Actor.ID != "" { + t.Fatalf("admin should not require owner id: query=%#v validation=%#v", admin, validation) + } + + _, validation = normalizeRealtimeSyncQuery(realtimeSyncQuery{Actor: sessionActor{ID: "usr_owner", Role: "user"}}) + if validation == nil || validation.Code != "workbench_sync_scope_required" { + t.Fatalf("missing scope validation=%#v", validation) + } + + _, validation = normalizeRealtimeSyncQuery(realtimeSyncQuery{SessionID: "ses_sync", AfterOutboxSeq: -1, Actor: sessionActor{ID: "usr_owner", Role: "user"}}) + if validation == nil || validation.Code != "workbench_sync_cursor_invalid" { + t.Fatalf("negative cursor validation=%#v", validation) + } + + _, validation = normalizeRealtimeSyncQuery(realtimeSyncQuery{SessionID: "ses_sync", SnapshotOnly: true, DeltaOnly: true, Actor: sessionActor{ID: "usr_owner", Role: "user"}}) + if validation == nil || validation.Code != "workbench_sync_mode_invalid" { + t.Fatalf("conflicting sync mode validation=%#v", validation) + } +} + +func TestRealtimeSyncRouteIsPostOnlyAndRejectsUnknownFields(t *testing.T) { + server := &Server{config: Config{}, mux: http.NewServeMux()} + server.routes() + + methodRecorder := httptest.NewRecorder() + server.mux.ServeHTTP(methodRecorder, httptest.NewRequest(http.MethodGet, "/v1/workbench-runtime/sync", nil)) + if methodRecorder.Code != http.StatusMethodNotAllowed { + t.Fatalf("GET status=%d, want %d", methodRecorder.Code, http.StatusMethodNotAllowed) + } + if methodRecorder.Header().Get("Allow") != "POST" { + t.Fatalf("Allow=%q, want POST", methodRecorder.Header().Get("Allow")) + } + + unknownRecorder := httptest.NewRecorder() + unknownRequest := httptest.NewRequest(http.MethodPost, "/v1/workbench-runtime/sync", strings.NewReader(`{"sessionId":"ses_sync","actor":{"id":"usr_owner","role":"user"},"unknown":true}`)) + server.mux.ServeHTTP(unknownRecorder, unknownRequest) + if unknownRecorder.Code != http.StatusBadRequest || !strings.Contains(unknownRecorder.Body.String(), "invalid_json") { + t.Fatalf("unknown field response=%d %s", unknownRecorder.Code, unknownRecorder.Body.String()) + } + + trailingRecorder := httptest.NewRecorder() + trailingRequest := httptest.NewRequest(http.MethodPost, "/v1/workbench-runtime/sync", strings.NewReader(`{"sessionId":"ses_sync","actor":{"id":"usr_owner","role":"user"}} {}`)) + server.mux.ServeHTTP(trailingRecorder, trailingRequest) + if trailingRecorder.Code != http.StatusBadRequest || !strings.Contains(trailingRecorder.Body.String(), "invalid_json") { + t.Fatalf("trailing JSON response=%d %s", trailingRecorder.Code, trailingRecorder.Body.String()) + } +} + +func TestReadRealtimeSyncSnapshotUsesOwnerScopeAndSingleCutoff(t *testing.T) { + queryer := newRealtimeSyncFakeQueryer("usr_owner") + query := realtimeSyncQuery{ + SessionID: "ses_sync", + TraceID: "trc_sync", + AfterOutboxSeq: 0, + Limit: 2, + Actor: sessionActor{ID: "usr_owner", Role: "user"}, + } + snapshot, err := readRealtimeSyncSnapshot(context.Background(), queryer, query) + if err != nil { + t.Fatalf("read sync snapshot: %v", err) + } + if snapshot.CutoffOutboxSeq != 3 || snapshot.CursorOutboxSeq != 2 || !snapshot.HasMore { + t.Fatalf("unexpected cursor boundary: %#v", snapshot) + } + if len(snapshot.Events) != 2 { + t.Fatalf("events=%d, want 2", len(snapshot.Events)) + } + first := snapshot.Events[0] + if first.OutboxEventID != "obe_1" || first.EntityFamily != "traceEvents" || first.EntityID != "evt_1" { + t.Fatalf("event identity fields were not returned: %#v", first) + } + if first.EventSeq != int64(101) || first.AggregateID != "trc_sync" || first.AggregateSeq != 1 || first.ProjectionRevision != 11 { + t.Fatalf("event aggregate fields were not returned: %#v", first) + } + if len(snapshot.Facts) != len(realtimeSyncFactSpecs) { + t.Fatalf("fact families=%d, want %d: %#v", len(snapshot.Facts), len(realtimeSyncFactSpecs), snapshot.Facts) + } + for _, spec := range realtimeSyncFactSpecs { + if len(snapshot.Facts[spec.Family]) != 1 { + t.Fatalf("facts[%s]=%#v, want one fact", spec.Family, snapshot.Facts[spec.Family]) + } + } + if len(queryer.calls) != 3+len(realtimeSyncFactSpecs) { + t.Fatalf("query calls=%v", queryer.calls) + } + if queryer.calls[0].kind != "scope" || queryer.calls[1].kind != "cutoff" || queryer.calls[2].kind != "events" { + t.Fatalf("snapshot boundary query order=%v", queryer.calls) + } + if got := queryer.calls[0].args; len(got) != 4 || got[2] != false || got[3] != "usr_owner" { + t.Fatalf("owner scope args=%#v", got) + } + if got := queryer.calls[2].args; len(got) != 5 || got[3] != int64(3) || got[4] != 3 { + t.Fatalf("events must use cutoff and limit+1: %#v", got) + } +} + +func TestReadRealtimeSyncSnapshotRejectsOtherOwnerBeforeReadingFacts(t *testing.T) { + if !strings.Contains(realtimeSyncResolveScopeSQL, "sessions.owner_user_id = $4") { + t.Fatalf("ordinary user scope must be constrained by workbench_sessions.owner_user_id: %s", realtimeSyncResolveScopeSQL) + } + queryer := newRealtimeSyncFakeQueryer("usr_owner") + _, err := readRealtimeSyncSnapshot(context.Background(), queryer, realtimeSyncQuery{ + SessionID: "ses_sync", + Limit: 100, + Actor: sessionActor{ID: "usr_other", Role: "user"}, + }) + if !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("error=%v, want sql.ErrNoRows", err) + } + if len(queryer.calls) != 1 || queryer.calls[0].kind != "scope" { + t.Fatalf("unauthorized scope must stop before outbox/facts reads: %v", queryer.calls) + } +} + +func TestReadRealtimeSyncSnapshotOnlySkipsHistoryAndReadsBoundedCurrentFacts(t *testing.T) { + queryer := newRealtimeSyncFakeQueryer("usr_owner") + snapshot, err := readRealtimeSyncSnapshot(context.Background(), queryer, realtimeSyncQuery{ + SessionID: "ses_sync", + Limit: 100, + SnapshotOnly: true, + Actor: sessionActor{ID: "usr_owner", Role: "user"}, + }) + if err != nil { + t.Fatalf("read snapshot-only sync: %v", err) + } + if len(snapshot.Events) != 0 || snapshot.HasMore || snapshot.CursorOutboxSeq != snapshot.CutoffOutboxSeq { + t.Fatalf("snapshot-only boundary=%#v", snapshot) + } + if len(snapshot.Facts) != len(realtimeSyncSnapshotFactSpecs) { + t.Fatalf("snapshot-only facts=%#v", snapshot.Facts) + } + for _, call := range queryer.calls { + if call.kind == "events" || call.kind == "facts.parts" || call.kind == "facts.traceEvents" || call.kind == "facts.checkpoints" { + t.Fatalf("snapshot-only performed unbounded read: %v", queryer.calls) + } + } +} + +func TestReadRealtimeSyncDeltaOnlyReadsOutboxWithoutFacts(t *testing.T) { + queryer := newRealtimeSyncFakeQueryer("usr_owner") + snapshot, err := readRealtimeSyncSnapshot(context.Background(), queryer, realtimeSyncQuery{ + SessionID: "ses_sync", + AfterOutboxSeq: 1, + Limit: 100, + DeltaOnly: true, + Actor: sessionActor{ID: "usr_owner", Role: "user"}, + }) + if err != nil { + t.Fatalf("read delta-only sync: %v", err) + } + if len(snapshot.Events) == 0 || len(snapshot.Facts) != 0 { + t.Fatalf("delta-only result=%#v", snapshot) + } + for _, call := range queryer.calls { + if strings.HasPrefix(call.kind, "facts.") { + t.Fatalf("delta-only read facts: %v", queryer.calls) + } + } +} + +func TestReadRealtimeSyncSnapshotClampsForeignCursorToScopedCutoff(t *testing.T) { + queryer := newRealtimeSyncFakeQueryer("usr_owner") + queryer.events = [][]any{} + snapshot, err := readRealtimeSyncSnapshot(context.Background(), queryer, realtimeSyncQuery{ + SessionID: "ses_sync", + AfterOutboxSeq: 999, + Limit: 100, + Actor: sessionActor{ID: "usr_owner", Role: "user"}, + }) + if err != nil { + t.Fatalf("read sync snapshot: %v", err) + } + if snapshot.CutoffOutboxSeq != 3 || snapshot.CursorOutboxSeq != 3 || snapshot.HasMore || len(snapshot.Events) != 0 { + t.Fatalf("foreign cursor was not clamped to scoped cutoff: %#v", snapshot) + } +} + +func TestReadRealtimeSyncSnapshotAllowsAdminWithoutOwnerFilter(t *testing.T) { + queryer := newRealtimeSyncFakeQueryer("usr_owner") + snapshot, err := readRealtimeSyncSnapshot(context.Background(), queryer, realtimeSyncQuery{ + TraceID: "trc_sync", + Limit: 3, + Actor: sessionActor{Role: "admin"}, + }) + if err != nil { + t.Fatalf("admin sync: %v", err) + } + if len(queryer.calls) == 0 || queryer.calls[0].args[2] != true || queryer.calls[0].args[3] != "" { + t.Fatalf("admin scope args=%#v", queryer.calls) + } + if snapshot.CursorOutboxSeq != snapshot.CutoffOutboxSeq || snapshot.HasMore { + t.Fatalf("admin complete page boundary=%#v", snapshot) + } +} + +type realtimeSyncFakeCall struct { + kind string + query string + args []any +} + +type realtimeSyncFakeQueryer struct { + ownerID string + sessionID string + cutoff int64 + events [][]any + facts map[string][][]any + calls []realtimeSyncFakeCall +} + +func newRealtimeSyncFakeQueryer(ownerID string) *realtimeSyncFakeQueryer { + facts := map[string][][]any{} + for _, spec := range realtimeSyncFactSpecs { + facts[spec.Table] = [][]any{{fmt.Sprintf(`{"family":%q,"sessionId":"ses_sync"}`, spec.Family)}} + } + return &realtimeSyncFakeQueryer{ + ownerID: ownerID, + sessionID: "ses_sync", + cutoff: 3, + events: [][]any{realtimeSyncTestEventRow(1), realtimeSyncTestEventRow(2), realtimeSyncTestEventRow(3)}, + facts: facts, + } +} + +func (queryer *realtimeSyncFakeQueryer) QueryRowContext(_ context.Context, query string, args ...any) realtimeSyncRow { + if query == realtimeSyncResolveScopeSQL { + queryer.record("scope", query, args) + isAdmin, _ := args[2].(bool) + actorID, _ := args[3].(string) + if !isAdmin && actorID != queryer.ownerID { + return realtimeSyncFakeRow{err: sql.ErrNoRows} + } + return realtimeSyncFakeRow{values: []any{queryer.sessionID}} + } + if query == realtimeSyncCutoffSQL { + queryer.record("cutoff", query, args) + return realtimeSyncFakeRow{values: []any{queryer.cutoff}} + } + return realtimeSyncFakeRow{err: fmt.Errorf("unexpected QueryRowContext: %s", query)} +} + +func (queryer *realtimeSyncFakeQueryer) QueryContext(_ context.Context, query string, args ...any) (realtimeSyncRows, error) { + if query == realtimeSyncEventsSQL { + queryer.record("events", query, args) + return &realtimeSyncFakeRows{values: queryer.events}, nil + } + for _, spec := range realtimeSyncFactSpecs { + if strings.Contains(query, "FROM "+spec.Table+" ") { + queryer.record("facts."+spec.Family, query, args) + return &realtimeSyncFakeRows{values: queryer.facts[spec.Table]}, nil + } + } + return nil, fmt.Errorf("unexpected QueryContext: %s", query) +} + +func (queryer *realtimeSyncFakeQueryer) record(kind string, query string, args []any) { + cloned := append([]any(nil), args...) + queryer.calls = append(queryer.calls, realtimeSyncFakeCall{kind: kind, query: query, args: cloned}) +} + +type realtimeSyncFakeRow struct { + values []any + err error +} + +func (row realtimeSyncFakeRow) Scan(destinations ...any) error { + if row.err != nil { + return row.err + } + return assignRealtimeSyncFakeValues(destinations, row.values) +} + +type realtimeSyncFakeRows struct { + values [][]any + index int + err error +} + +func (rows *realtimeSyncFakeRows) Close() error { return nil } + +func (rows *realtimeSyncFakeRows) Err() error { return rows.err } + +func (rows *realtimeSyncFakeRows) Next() bool { return rows.index < len(rows.values) } + +func (rows *realtimeSyncFakeRows) Scan(destinations ...any) error { + if rows.index >= len(rows.values) { + return errors.New("realtime sync fake rows exhausted") + } + values := rows.values[rows.index] + rows.index++ + return assignRealtimeSyncFakeValues(destinations, values) +} + +func assignRealtimeSyncFakeValues(destinations []any, values []any) error { + if len(destinations) != len(values) { + return fmt.Errorf("scan destinations=%d values=%d", len(destinations), len(values)) + } + for index, destination := range destinations { + value := values[index] + switch target := destination.(type) { + case *string: + if value == nil { + return fmt.Errorf("cannot scan nil into string") + } + *target = fmt.Sprint(value) + case *int64: + integer, ok := value.(int64) + if !ok { + return fmt.Errorf("cannot scan %T into int64", value) + } + *target = integer + case *sql.NullString: + if value == nil { + *target = sql.NullString{} + } else { + *target = sql.NullString{String: fmt.Sprint(value), Valid: true} + } + case *sql.NullInt64: + if value == nil { + *target = sql.NullInt64{} + } else if integer, ok := value.(int64); ok { + *target = sql.NullInt64{Int64: integer, Valid: true} + } else { + return fmt.Errorf("cannot scan %T into NullInt64", value) + } + case *sql.NullBool: + if value == nil { + *target = sql.NullBool{} + } else if boolean, ok := value.(bool); ok { + *target = sql.NullBool{Bool: boolean, Valid: true} + } else { + return fmt.Errorf("cannot scan %T into NullBool", value) + } + default: + return fmt.Errorf("unsupported scan destination %T", destination) + } + } + return nil +} + +func realtimeSyncTestEventRow(sequence int64) []any { + return []any{ + sequence, + fmt.Sprintf("obe_%d", sequence), + "traceEvents", + fmt.Sprintf("evt_%d", sequence), + int64(100) + sequence, + "trc_sync", + sequence, + int64(10) + sequence, + "trc_sync", + "ses_sync", + "turn_sync", + nil, + sequence, + sequence, + fmt.Sprintf("source_%d", sequence), + "event", + false, + false, + fmt.Sprintf(`{"sequence":%d}`, sequence), + "2026-07-10T12:00:00Z", + } +} diff --git a/internal/workbenchruntime/service.go b/internal/workbenchruntime/service.go index b81912a8..6db67401 100644 --- a/internal/workbenchruntime/service.go +++ b/internal/workbenchruntime/service.go @@ -258,6 +258,7 @@ func (s *Server) routes() { s.mux.HandleFunc("/v1/workbench-runtime/trace-events", s.handleTraceEvents) s.mux.HandleFunc("/v1/workbench-runtime/sessions", s.handleSessions) s.mux.HandleFunc("/v1/workbench-runtime/projection-outbox", s.handleProjectionOutbox) + s.mux.HandleFunc("/v1/workbench-runtime/sync", s.handleRealtimeSync) } func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request) { diff --git a/scripts/gitops-render.mjs b/scripts/gitops-render.mjs index ebf13b4d..001a1153 100644 --- a/scripts/gitops-render.mjs +++ b/scripts/gitops-render.mjs @@ -1,5827 +1,82 @@ #!/usr/bin/env node // SPEC: PJ2026-01060505 Workbench Performance; PJ2026-0106 平台运维 import assert from "node:assert/strict"; -import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import path from "node:path"; -import { promisify } from "node:util"; -import { fileURLToPath } from "node:url"; import { applyRuntimeLaneDeployConfig, - defaultEndpointForRuntimeLane, - defaultPortForRuntimeLane, + argoApplicationName, + artifactCatalogSkeleton, + attachAccessControlEnv, + annotate, + cloneJson, + defaultBranch, + defaultCatalogPath, + defaultGitopsBranch, + defaultOutDir, + defaultRuntimeEndpoint, + defaultSourceRepo, + defaultV02RuntimeEndpoint, + defaultWebEndpoint, + ensureObject, + generatedPath, + gitopsPathForProfile, + imageTagForSource, isRuntimeLane, - runtimeLaneMirrorSpecs, - runtimeLaneConfig, - versionNameForRuntimeLane -} from "./src/runtime-lane.ts"; -import { readStructuredFile } from "./src/structured-config.mjs"; - -const execFileAsync = promisify(execFile); -const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); -const defaultOutDir = "deploy/gitops/node"; -const defaultRegistryPrefix = process.env.HWLAB_NODE_REGISTRY_PREFIX || "127.0.0.1:5000/hwlab"; -const defaultSourceRepo = process.env.HWLAB_NODE_GIT_URL || "git@github.com:pikasTech/HWLAB.git"; -const defaultGitReadUrl = process.env.HWLAB_NODE_GIT_READ_URL || null; -const defaultV02GitReadUrl = process.env.HWLAB_V02_GIT_READ_URL || "http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git"; -const defaultV02GitWriteUrl = process.env.HWLAB_V02_GIT_WRITE_URL || "http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/HWLAB.git"; -const defaultRuntimeLaneGitReadUrl = process.env.HWLAB_RUNTIME_LANE_GIT_READ_URL || "http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikasTech-HWLAB.git"; -const defaultRuntimeLaneGitWriteUrl = process.env.HWLAB_RUNTIME_LANE_GIT_WRITE_URL || defaultV02GitWriteUrl; -const defaultBranch = process.env.HWLAB_NODE_BRANCH || "node"; -const defaultGitopsBranch = process.env.HWLAB_NODE_GITOPS_BRANCH || "node-gitops"; -const defaultCatalogPath = process.env.HWLAB_ARTIFACT_CATALOG_PATH || "deploy/artifact-catalog.dev.json"; -const defaultRuntimeEndpoint = process.env.HWLAB_NODE_RUNTIME_ENDPOINT || "http://74.48.78.17:17667"; -const defaultWebEndpoint = process.env.HWLAB_NODE_WEB_ENDPOINT || "http://74.48.78.17:17666"; -const defaultProdRuntimeEndpoint = process.env.HWLAB_NODE_PROD_RUNTIME_ENDPOINT || "http://74.48.78.17:18667"; -const defaultProdWebEndpoint = process.env.HWLAB_NODE_PROD_WEB_ENDPOINT || "http://74.48.78.17:18666"; -const defaultV02RuntimeEndpoint = process.env.HWLAB_V02_RUNTIME_ENDPOINT || "https://hwlab.74-48-78-17.nip.io"; -const defaultV02WebEndpoint = process.env.HWLAB_V02_WEB_ENDPOINT || "https://hwlab.74-48-78-17.nip.io"; -const defaultProxyUrl = process.env.HWLAB_NODE_PROXY_URL || "http://127.0.0.1:10808"; -const defaultAllProxyUrl = process.env.HWLAB_NODE_ALL_PROXY_URL || "socks5h://127.0.0.1:10808"; -const defaultNoProxy = process.env.HWLAB_NODE_NO_PROXY || "hyueapi.com,.hyueapi.com,127.0.0.1,localhost,::1,10.0.0.0/8,10.42.0.0/16,10.43.0.0/16,192.168.0.0/16,.svc,.cluster.local,kubernetes.default.svc,registry.hwlab-ci.svc,hwlab-registry.hwlab-ci.svc,git-mirror-http,git-mirror-http.devops-infra,git-mirror-http.devops-infra.svc,git-mirror-http.devops-infra.svc.cluster.local,git-mirror-write,git-mirror-write.devops-infra,git-mirror-write.devops-infra.svc,git-mirror-write.devops-infra.svc.cluster.local"; -const ciToolsRunnerImage = process.env.HWLAB_NODE_CI_TOOLS_IMAGE || "127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1"; -const buildkitRunnerImage = process.env.HWLAB_NODE_BUILDKIT_IMAGE || "moby/buildkit:rootless"; -const defaultDevBaseImage = process.env.HWLAB_NODE_DEV_BASE_IMAGE || "127.0.0.1:5000/hwlab/hwlab-node20-base:20-bookworm-slim"; -const defaultServiceIds = Object.freeze([ - "hwlab-cloud-api", - "hwlab-cloud-web", - "hwlab-gateway", - "hwlab-edge-proxy", - "hwlab-agent-skills" -]); -const v02ExtraObservableServices = Object.freeze([ - { serviceId: "hwlab-deepseek-proxy", port: 4000, healthPath: "/health/live" } -]); -const v02RemovedServiceIds = new Set([ - "hwlab-agent-mgr", - "hwlab-agent-worker", - "hwlab-agent-worker-template", - "hwlab-code-agent-workspace", - "hwlab-router", - "hwlab-tunnel-client", - "hwlab-gateway-simu", - "hwlab-box-simu", - "hwlab-patch-panel" -]); -const v02RemovedServiceEnvNames = new Set([ - "CODEX_HOME", - "OPENAI_API_KEY", - "HWLAB_CODE_AGENT_PROVIDER", - "HWLAB_CODE_AGENT_MODEL", - "HWLAB_CODE_AGENT_OPENAI_BASE_URL", - "HWLAB_CODE_AGENT_WORKSPACE", - "HWLAB_CODE_AGENT_CODEX_HOME", - "HWLAB_CODE_AGENT_CODEX_WORKSPACE", - "HWLAB_CODE_AGENT_CODEX_SANDBOX", - "HWLAB_CODE_AGENT_CODEX_COMMAND", - "HWLAB_CODE_AGENT_CODEX_STDIO_ENABLED", - "HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR", - "HWLAB_M3_GATEWAY_SIMU_1_URL", - "HWLAB_M3_GATEWAY_SIMU_2_URL", - "HWLAB_M3_PATCH_PANEL_URL", - "HWLAB_GATEWAY_SIMU_URL", - "HWLAB_BOX_SIMU_URL", - "HWLAB_PATCH_PANEL_URL", - "HWLAB_ROUTER_URL", - "HWLAB_TUNNEL_CLIENT_URL" -]); -const moonBridgeSourceRef = process.env.HWLAB_NODE_MOONBRIDGE_REF || "1b99888d3dae889b79ee602cb875c7907f7e76f2"; -const moonBridgeImage = process.env.HWLAB_NODE_MOONBRIDGE_IMAGE || `${defaultRegistryPrefix}/moonbridge:${moonBridgeSourceRef.slice(0, 12)}`; -const deepSeekProfileModel = process.env.HWLAB_NODE_DEEPSEEK_PROFILE_MODEL || "deepseek-chat"; -const codexApiProfileModel = process.env.HWLAB_NODE_CODEX_API_PROFILE_MODEL || "gpt-5.5"; -const codexApiProfileBaseUrl = process.env.HWLAB_NODE_CODEX_API_PROFILE_BASE_URL || "http://127.0.0.1:49280/responses"; -const codexApiForwarderPort = process.env.HWLAB_NODE_CODEX_API_FORWARDER_PORT || "49280"; -const codexApiForwarderUpstreamBaseUrl = process.env.HWLAB_NODE_CODEX_API_UPSTREAM_BASE_URL || "https://hyueapi.com"; -const sourceCommitPattern = /^[a-f0-9]{7,40}$/u; -const v02EnvReuseRuntimeMode = "env-reuse-git-mirror-checkout"; - -function k8sLabelHash(value) { - return String(value || "").slice(0, 16); -} - -const primitiveValidationTasks = Object.freeze([ - { - name: "repo-reports-guard", - commands: ["node scripts/repo-reports-guard.mjs"] - }, - { - name: "node-contract-check", - commands: [ - "node --check scripts/gitops-render.mjs", - `node --input-type=module <<'NODE' -import { readdirSync, readFileSync, statSync } from "node:fs"; -import path from "node:path"; - -const root = process.cwd(); -const ciFileName = "CI" + ".json"; -const forbiddenCiFragments = [ - "ci" + "-json", - "HWLAB_NODE_TEKTON" + "_SINGLE_IMAGE_PUBLISH", - "single" + "-dind", - "docker" + ":29", - "DOCKER" + "_HOST", - "Docker" + "-in-Docker", - "D" + "IND", - "docker" + " push", - "--build" + "-backend", - "HWLAB_ARTIFACT" + "_BUILD_BACKEND", - "--legacy" + "-source-images", - "HWLAB_NODE_USE_DEPLOY_IMAGES" + "=0" -]; -const scanTargets = [ - "scripts/gitops-render.mjs", - "scripts/src/runtime-lane.ts", - "scripts/artifact-publish.mjs", - "scripts/src/ci-plan-lib.mjs", - "deploy/gitops/node/tekton" -]; -const skipDirs = new Set([".git", "node_modules", ".worktree"]); - -function walkFiles(relativePath) { - const absolutePath = path.join(root, relativePath); - let stat; - try { - stat = statSync(absolutePath); - } catch { - return []; - } - if (stat.isFile()) return [relativePath]; - if (!stat.isDirectory()) return []; - const files = []; - for (const entry of readdirSync(absolutePath, { withFileTypes: true })) { - if (entry.isDirectory() && skipDirs.has(entry.name)) continue; - files.push(...walkFiles(path.join(relativePath, entry.name))); - } - return files; -} - -const ciFiles = walkFiles(".").filter((filePath) => path.basename(filePath) === ciFileName); -const violations = []; -for (const filePath of scanTargets.flatMap(walkFiles)) { - const text = readFileSync(path.join(root, filePath), "utf8"); - for (const fragment of forbiddenCiFragments) { - if (text.includes(fragment)) violations.push({ filePath, fragment }); - } -} -if (ciFiles.length || violations.length) { - console.error(JSON.stringify({ status: "failed", ciFiles, violations }, null, 2)); - process.exit(1); -} -NODE`, - "node --check scripts/artifact-publish.mjs" - ] - }, - { - name: "codex-api-forwarder-check", - commands: [ - "node scripts/run-bun.mjs test cmd/hwlab-codex-api-responses-forwarder/main.test.ts" - ] - } -]); - -function proxyEnv() { - return [ - { name: "HTTP_PROXY", value: defaultProxyUrl }, - { name: "HTTPS_PROXY", value: defaultProxyUrl }, - { name: "ALL_PROXY", value: defaultAllProxyUrl }, - { name: "NO_PROXY", value: defaultNoProxy }, - { name: "http_proxy", value: defaultProxyUrl }, - { name: "https_proxy", value: defaultProxyUrl }, - { name: "all_proxy", value: defaultAllProxyUrl }, - { name: "no_proxy", value: defaultNoProxy } - ]; -} - -function parseArgs(argv) { - const args = { - lane: process.env.HWLAB_GITOPS_LANE || "node", - nodeId: process.env.HWLAB_NODE_ID || "node", - outDir: defaultOutDir, - gitopsRoot: defaultOutDir, - catalogPath: defaultCatalogPath, - imageTagMode: process.env.HWLAB_ARTIFACT_IMAGE_TAG_MODE || "short", - registryPrefix: defaultRegistryPrefix, - sourceRevision: null, - sourceBranch: defaultBranch, - gitopsBranch: defaultGitopsBranch, - sourceRepo: defaultSourceRepo, - gitReadUrl: defaultGitReadUrl, - gitWriteUrl: null, - runtimeEndpoint: defaultRuntimeEndpoint, - webEndpoint: defaultWebEndpoint, - prodRuntimeEndpoint: defaultProdRuntimeEndpoint, - prodWebEndpoint: defaultProdWebEndpoint, - useDeployImages: true, - check: false, - write: true, - help: false - }; - for (let index = 0; index < argv.length; index += 1) { - const arg = argv[index]; - if (arg === "--lane") args.lane = readOption(argv, ++index, arg); - else if (arg === "--node") args.nodeId = readOption(argv, ++index, arg); - else if (arg === "--out") args.outDir = readOption(argv, ++index, arg); - else if (arg === "--gitops-root") args.gitopsRoot = readOption(argv, ++index, arg); - else if (arg === "--catalog-path") args.catalogPath = readOption(argv, ++index, arg); - else if (arg === "--image-tag-mode") args.imageTagMode = readOption(argv, ++index, arg); - else if (arg === "--registry-prefix") args.registryPrefix = readOption(argv, ++index, arg).replace(/\/+$/u, ""); - else if (arg === "--source-revision") args.sourceRevision = readOption(argv, ++index, arg); - else if (arg === "--source-branch") args.sourceBranch = readOption(argv, ++index, arg); - else if (arg === "--gitops-branch") args.gitopsBranch = readOption(argv, ++index, arg); - else if (arg === "--source-repo") args.sourceRepo = readOption(argv, ++index, arg); - else if (arg === "--git-read-url") args.gitReadUrl = readOption(argv, ++index, arg); - else if (arg === "--git-write-url") args.gitWriteUrl = readOption(argv, ++index, arg); - else if (arg === "--runtime-endpoint") args.runtimeEndpoint = readOption(argv, ++index, arg); - else if (arg === "--web-endpoint") args.webEndpoint = readOption(argv, ++index, arg); - else if (arg === "--prod-runtime-endpoint") args.prodRuntimeEndpoint = readOption(argv, ++index, arg); - else if (arg === "--prod-web-endpoint") args.prodWebEndpoint = readOption(argv, ++index, arg); - else if (arg === "--use-deploy-images") args.useDeployImages = true; - else if (arg === "--check") { - args.check = true; - args.write = false; - } else if (arg === "--no-write") args.write = false; - else if (arg === "--help" || arg === "-h") args.help = true; - else throw new Error(`unknown argument ${arg}`); - } - if (isRuntimeLane(args.lane)) { - const versionName = versionNameForRuntimeLane(args.lane); - const defaultLaneEndpoint = defaultEndpointForRuntimeLane(args.lane); - if (args.sourceBranch === defaultBranch) args.sourceBranch = versionName; - if (args.gitopsBranch === defaultGitopsBranch) args.gitopsBranch = `${versionName}-gitops`; - if (args.catalogPath === defaultCatalogPath) args.catalogPath = `deploy/artifact-catalog.${args.lane}.json`; - if (args.runtimeEndpoint === defaultRuntimeEndpoint) args.runtimeEndpoint = defaultLaneEndpoint; - if (args.webEndpoint === defaultWebEndpoint) args.webEndpoint = defaultLaneEndpoint; - if (args.imageTagMode === "short") args.imageTagMode = "full"; - if (!args.gitReadUrl) args.gitReadUrl = defaultRuntimeLaneGitReadUrl; - if (!args.gitWriteUrl) args.gitWriteUrl = defaultRuntimeLaneGitWriteUrl; - } - if (!args.gitReadUrl) args.gitReadUrl = args.sourceRepo; - if (!args.gitWriteUrl) args.gitWriteUrl = args.sourceRepo; - assert.ok(args.lane === "node" || isRuntimeLane(args.lane), `unknown lane ${args.lane}`); - assert.ok(["short", "full"].includes(args.imageTagMode), `unknown image tag mode ${args.imageTagMode}`); - if (isRuntimeLane(args.lane)) { - const versionName = versionNameForRuntimeLane(args.lane); - assert.equal(args.sourceBranch, versionName, `${args.lane} source branch must be ${versionName}`); - assert.equal(args.gitopsBranch, `${versionName}-gitops`, `${args.lane} GitOps branch must be ${versionName}-gitops`); - } - return args; -} - -function readOption(argv, index, name) { - const value = argv[index]; - if (!value || value.startsWith("--")) throw new Error(`${name} requires a value`); - return value; -} - -function usage() { - return [ - "usage: node scripts/run-bun.mjs scripts/gitops-render.mjs [options]", - "", - "Render node-scoped Kubernetes-native CI/CD manifests from built-in Tekton CI primitives, deploy/deploy.yaml, and deploy/k8s/*.", - "", - "options:", - " --lane LANE node or runtime lane such as v02/v03/v04; default: node", - ` --node NODE deployment node id from deploy.nodes; runtime lanes default from deploy.lanes..node`, - ` --out DIR default: ${defaultOutDir}`, - ` --gitops-root DIR path inside GitOps repo; default comes from deploy.nodes..gitopsRoot`, - ` --catalog-path PATH default: ${defaultCatalogPath}`, - " --image-tag-mode MODE short or full; v02 uses full source commit IDs", - ` --source-repo URL default: ${defaultSourceRepo}`, - " --git-read-url URL read-only source URL; runtime lanes default to devops-infra git mirror", - " --git-write-url URL GitOps write URL; runtime lanes default to devops-infra git mirror write relay", - ` --source-branch BRANCH default: ${defaultBranch}`, - ` --gitops-branch BRANCH default: ${defaultGitopsBranch}`, - " --source-revision SHA source commit used for image tags and runtime annotations; default: git rev-parse HEAD", - ` --registry-prefix PREFIX default: ${defaultRegistryPrefix}`, - ` --runtime-endpoint URL default: ${defaultRuntimeEndpoint}; runtime lane default is derived from deploy.yaml/public URL`, - ` --web-endpoint URL default: ${defaultWebEndpoint}; runtime lane default is derived from deploy.yaml/public URL`, - ` --prod-runtime-endpoint URL default: ${defaultProdRuntimeEndpoint}`, - ` --prod-web-endpoint URL default: ${defaultProdWebEndpoint}`, - " --use-deploy-images default and only supported mode: render workload images from deploy/artifact-catalog.dev.json per service", - " --check verify generated files are current without writing", - " --no-write plan and print generated file list without writing" - ].join("\n"); -} - -function generatedPath(filePath) { - return path.isAbsolute(filePath) ? filePath : path.join(repoRoot, filePath); -} - -async function readJson(relativePath) { - return readStructuredFile(repoRoot, relativePath); -} - -async function readJsonIfPresent(relativePath, fallback = null) { - try { - return await readJson(relativePath); - } catch (error) { - if (error && error.code === "ENOENT") return fallback; - throw error; - } -} - -async function attachAccessControlEnv(deploy) { - const byProfile = {}; - for (const [profile, laneConfig] of Object.entries(deploy?.lanes ?? {})) { - if (!isRuntimeLane(profile)) continue; - const accessControl = laneConfig?.accessControl; - if (!accessControl || typeof accessControl !== "object" || Array.isArray(accessControl)) continue; - const users = await readConfigRef(accessControl.usersRef, `${profile}.accessControl.usersRef`); - const navProfiles = await readConfigRef(accessControl.navProfilesRef, `${profile}.accessControl.navProfilesRef`); - const env = accessControlEnvFromConfig({ users, navProfiles }); - if (Object.keys(env).length > 0) byProfile[profile] = { "hwlab-cloud-api": env }; - } - Object.defineProperty(deploy, "__accessControlEnvByProfile", { value: byProfile, enumerable: false, configurable: true }); -} - -async function readConfigRef(ref, label) { - assert.equal(typeof ref, "string", `${label} must be a file reference`); - const [filePath, fragment = ""] = ref.split("#"); - assert.ok(filePath, `${label} must include a file path`); - const root = await readJson(filePath); - return configFragment(root, fragment || "", label); -} - -function configFragment(value, fragment, label) { - const pathText = String(fragment ?? "").replace(/^\//u, ""); - if (!pathText) return value; - return pathText.split(/[./]/u).filter(Boolean).reduce((current, segment) => { - assert.ok(current && typeof current === "object" && Object.hasOwn(current, segment), `${label} fragment is missing ${segment}`); - return current[segment]; - }, value); -} - -function accessControlEnvFromConfig({ users, navProfiles }) { - assert.ok(Array.isArray(users), "accessControl usersRef must resolve to an array"); - assert.ok(Array.isArray(navProfiles), "accessControl navProfilesRef must resolve to an array"); - const env = {}; - const renderedUsers = users.map((user, index) => normalizeAccessControlUser(user, index, env)); - const renderedProfiles = navProfiles.map(normalizeNavProfile); - env.HWLAB_ACCESS_CONTROL_USERS_JSON = JSON.stringify(renderedUsers); - env.HWLAB_ACCESS_CONTROL_NAV_PROFILES_JSON = JSON.stringify(renderedProfiles); - return env; -} - -function normalizeAccessControlUser(user, index, env) { - assert.ok(user && typeof user === "object" && !Array.isArray(user), `accessControl user ${index} must be an object`); - const passwordHashRef = user.passwordHashRef && typeof user.passwordHashRef === "object" && !Array.isArray(user.passwordHashRef) ? user.passwordHashRef : null; - const passwordHashEnv = configString(user.passwordHashEnv) || configString(passwordHashRef?.env); - const secretRef = configString(passwordHashRef?.secretRef) || secretRefFromParts(passwordHashRef); - if (passwordHashEnv && secretRef) env[passwordHashEnv] = `secretRef:${secretRef}`; - return { - id: requiredAccessConfigString(user.id, `accessControl user ${index}.id`), - username: requiredAccessConfigString(user.username, `accessControl user ${index}.username`), - displayName: configString(user.displayName) || configString(user.username), - email: configString(user.email) || null, - role: configString(user.role) === "admin" ? "admin" : "user", - status: configString(user.status) === "disabled" ? "disabled" : "active", - navProfile: requiredAccessConfigString(user.navProfile, `accessControl user ${index}.navProfile`), - passwordHashEnv: passwordHashEnv || null - }; -} - -function normalizeNavProfile(profile, index) { - assert.ok(profile && typeof profile === "object" && !Array.isArray(profile), `accessControl nav profile ${index} must be an object`); - const allowedIds = Array.isArray(profile.allowedIds) ? profile.allowedIds.map(configString).filter(Boolean) : []; - assert.ok(allowedIds.length > 0, `accessControl nav profile ${index}.allowedIds must not be empty`); - return { - id: requiredAccessConfigString(profile.id, `accessControl nav profile ${index}.id`), - displayName: configString(profile.displayName) || configString(profile.id), - allowedIds - }; -} - -function secretRefFromParts(value) { - if (!value || typeof value !== "object" || Array.isArray(value)) return ""; - const sourceRef = configString(value.sourceRef ?? value.secretName); - const targetKey = configString(value.targetKey ?? value.key); - return sourceRef && targetKey ? `${sourceRef}/${targetKey}` : ""; -} - -function configString(value) { - return typeof value === "string" ? value.trim() : ""; -} - -function requiredAccessConfigString(value, label) { - const text = configString(value); - assert.ok(text, `${label} is required`); - return text; -} - -function optionalObject(value) { - return value && typeof value === "object" && !Array.isArray(value) ? value : {}; -} - -async function gitValue(args) { - const result = await execFileAsync("git", args, { cwd: repoRoot, timeout: 10000, maxBuffer: 1024 * 1024 }); - return result.stdout.trim(); -} - -async function resolveSourceRevision(value) { - const revision = value || await gitValue(["rev-parse", "HEAD"]); - const full = sourceCommitPattern.test(revision) && revision.length === 40 - ? revision - : await gitValue(["rev-parse", revision]); - assert.match(full, /^[a-f0-9]{40}$/u, "source revision must resolve to a 40-char git commit"); - return { full, short: full.slice(0, 7) }; -} - -function jsonManifest(value) { - return `${JSON.stringify(value, null, 2)}\n`; -} - -function textFile(value) { - return value.endsWith("\n") ? value : `${value}\n`; -} - -function cloneJson(value) { - return JSON.parse(JSON.stringify(value)); -} - -function ensureObject(value, name) { - assert.ok(value && typeof value === "object" && !Array.isArray(value), `${name} must be an object`); - return value; -} - -function asItems(list, name) { - assert.equal(list?.kind, "List", `${name} must be a Kubernetes List`); - assert.ok(Array.isArray(list.items), `${name}.items must be an array`); - return list.items; -} - -function serviceIdForWorkload(item, container) { - return ( - item?.metadata?.labels?.["hwlab.pikastech.local/service-id"] || - item?.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/service-id"] || - container?.name - ); -} - -function upsertEnv(envList, name, value) { - if (!Array.isArray(envList)) return; - const existing = envList.find((entry) => entry?.name === name); - if (existing) { - delete existing.valueFrom; - existing.value = value; - } else { - envList.push({ name, value }); - } -} - -function upsertEnvEntry(envList, entry) { - if (!Array.isArray(envList) || !entry?.name) return; - const existing = envList.find((item) => item?.name === entry.name); - if (existing) { - delete existing.value; - delete existing.valueFrom; - Object.assign(existing, cloneJson(entry)); - } else { - envList.push(cloneJson(entry)); - } -} - -function syncCodeAgentWorkspaceInitContainer(initContainers, { image, runtimeCommit, runtimeImageTag }) { - const initContainer = Array.isArray(initContainers) - ? initContainers.find((entry) => entry?.name === "hwlab-code-agent-workspace-init") - : null; - if (!initContainer) return; - initContainer.image = image; - initContainer.imagePullPolicy = "IfNotPresent"; - initContainer.env ??= []; - upsertEnv(initContainer.env, "HWLAB_WORKSPACE_SOURCE_COMMIT", runtimeCommit); - upsertEnv(initContainer.env, "HWLAB_IMAGE", image); - upsertEnv(initContainer.env, "HWLAB_IMAGE_TAG", runtimeImageTag); -} - -function annotate(metadata, annotations) { - metadata.annotations = { ...(metadata.annotations ?? {}), ...annotations }; -} - -function label(metadata, labels) { - metadata.labels = { ...(metadata.labels ?? {}), ...labels }; -} - -function imageFor(registryPrefix, serviceId, imageTag) { - return `${registryPrefix}/${serviceId}:${imageTag}`; -} - -function imageTagFromReference(image) { - const value = String(image ?? ""); - const slashIndex = value.lastIndexOf("/"); - const colonIndex = value.lastIndexOf(":"); - if (colonIndex <= slashIndex) return null; - return value.slice(colonIndex + 1).split("@", 1)[0] || null; -} - -function repositoryFromImage(image) { - const value = String(image ?? "").split("@", 1)[0]; - const slashIndex = value.lastIndexOf("/"); - const colonIndex = value.lastIndexOf(":"); - return colonIndex > slashIndex ? value.slice(0, colonIndex) : value; -} - -function digestPinnedImage(image, digest) { - if (!/^sha256:[a-f0-9]{64}$/u.test(String(digest ?? ""))) return image; - const repository = repositoryFromImage(image); - return repository ? `${repository}@${digest}` : image; -} - -function catalogServiceById(catalog, serviceId) { - return (catalog?.services ?? []).find((service) => service?.serviceId === serviceId) ?? null; -} - -function v02EnvReuseEnabled(catalog, serviceId, envReuseServiceIds = null) { - if (envReuseServiceIds?.has(serviceId)) return true; - const service = catalogServiceById(catalog, serviceId); - return service?.runtimeMode === v02EnvReuseRuntimeMode || service?.envReuse === true; -} - -function bootShForService(deployService, serviceId) { - assert.ok(deployService?.bootSh, `deploy lane bootScripts.${serviceId} is required`); - const value = deployService.bootSh; - const text = String(value).replaceAll("\\", "/").replace(/^\.\//u, ""); - assert.ok(text && !text.startsWith("/") && !text.split("/").includes(".."), `${serviceId} boot script must be repo-relative`); - return text; -} - -function envReuseServiceIdsForLane(deploy, lane) { - if (!isRuntimeLane(lane)) return new Set(); - const configured = deploy?.lanes?.[lane]?.envReuseServices; - const allowed = new Set(serviceIdsForLane(lane, deploy)); - return new Set((Array.isArray(configured) ? configured : []).filter((serviceId) => allowed.has(serviceId))); -} - -function deployServiceForBoot(deploy, serviceId, lane = "v02") { - return { - serviceId, - bootSh: deploy?.lanes?.[lane]?.bootScripts?.[serviceId] - }; -} - -function bootMetadataForService({ args, catalog, deployService, serviceId, source }) { - const catalogService = catalogServiceById(catalog, serviceId); - const environmentImage = catalogService?.environmentImage ?? imageFor(args.registryPrefix, `${serviceId}-env`, `env-${String(catalogService?.environmentInputHash ?? source.full).slice(0, 12)}`); - const environmentDigest = catalogService?.environmentDigest ?? catalogService?.digest ?? null; - return { - runtimeMode: v02EnvReuseRuntimeMode, - environmentImage, - environmentDigest, - environmentInputHash: catalogService?.environmentInputHash ?? null, - codeInputHash: catalogService?.codeInputHash ?? null, - bootRepo: catalogService?.bootRepo ?? args.sourceRepo, - bootCommit: source.full, - bootSh: catalogService?.bootSh ?? bootShForService(deployService, serviceId) - }; -} - -function envReuseBootShForContainer({ serviceId, container, defaultBootSh }) { - if (serviceId === "hwlab-cloud-api" && container?.name === "hwlab-codex-api-forwarder") { - return "deploy/runtime/boot/hwlab-codex-api-responses-forwarder.sh"; - } - return defaultBootSh; -} - -function applyEnvReuseBootEnv(container, bootMetadata, { sourceBranch, gitReadUrl, bootSh = bootMetadata?.bootSh } = {}) { - if (!container || !bootMetadata) return; - delete container.command; - delete container.args; - container.env ??= []; - upsertEnv(container.env, "HWLAB_RUNTIME_MODE", bootMetadata.runtimeMode); - upsertEnv(container.env, "HWLAB_BOOT_REPO", bootMetadata.bootRepo); - upsertEnv(container.env, "HWLAB_BOOT_COMMIT", bootMetadata.bootCommit); - upsertEnv(container.env, "HWLAB_BOOT_REF", sourceBranch); - upsertEnv(container.env, "HWLAB_BOOT_SH", bootSh); - upsertEnv(container.env, "HWLAB_BOOT_READ_URL", gitReadUrl ?? defaultV02GitReadUrl); - upsertEnv(container.env, "HWLAB_ENVIRONMENT_IMAGE", bootMetadata.environmentImage ?? ""); - upsertEnv(container.env, "HWLAB_ENVIRONMENT_DIGEST", bootMetadata.environmentDigest ?? ""); - upsertEnv(container.env, "HWLAB_ENVIRONMENT_INPUT_HASH", bootMetadata.environmentInputHash ?? ""); - upsertEnv(container.env, "HWLAB_CODE_INPUT_HASH", bootMetadata.codeInputHash ?? ""); - upsertEnv(container.env, "HWLAB_IMAGE_DIGEST", bootMetadata.environmentDigest ?? "unknown"); - applyEnvReuseStartupProbe(container); -} - -function applyEnvReuseStartupProbe(container) { - const probe = container.readinessProbe ?? container.livenessProbe; - if (!probe?.httpGet && !probe?.tcpSocket && !probe?.exec) return; - container.startupProbe ??= { - ...cloneJson(probe), - periodSeconds: 10, - timeoutSeconds: Math.max(Number(probe.timeoutSeconds ?? 1), 2), - failureThreshold: 30, - successThreshold: 1 - }; -} - -function applyServiceHealthProbe(container, healthProbe) { - if (!healthProbe || typeof healthProbe !== "object" || Array.isArray(healthProbe)) return; - for (const [probeName, key] of [["readinessProbe", "readiness"], ["livenessProbe", "liveness"], ["startupProbe", "startup"]]) { - const probe = container?.[probeName]; - const timing = normalizeProbeTiming(healthProbe[key]); - if (!probe || Object.keys(timing).length === 0) continue; - Object.assign(probe, timing); - } -} - -function applyServiceShutdownDrainLifecycle(container, shutdownDrain) { - if (!container || !shutdownDrain || shutdownDrain.enabled !== true) return; - const pathValue = typeof shutdownDrain.path === "string" && /^\/[A-Za-z0-9/_:.-]+$/u.test(shutdownDrain.path) ? shutdownDrain.path : "/internal/workbench/drain"; - const sleepSeconds = boundedInteger(shutdownDrain.sleepSeconds, 3, 0, 30); - const timeoutMs = boundedInteger(shutdownDrain.timeoutMs, 2500, 100, 30000); - container.env ??= []; - upsertEnv(container.env, "HWLAB_WORKBENCH_SSE_DRAIN_TIMEOUT_MS", String(timeoutMs)); - const command = [ - 'port="${PORT:-${HWLAB_PORT:-${HWLAB_CLOUD_API_PORT:-6667}}}"', - `url="http://127.0.0.1:\${port}${pathValue}"`, - '/usr/local/bin/bun -e \'const [url, hostname] = process.argv.slice(2); await fetch(url, { method: "POST", headers: { "x-hwlab-drain-hostname": hostname || "" } }).catch(() => {});\' "$url" "${HOSTNAME:-}" >/dev/null 2>&1 || true', - `sleep ${sleepSeconds}` - ].join("; "); - container.lifecycle ??= {}; - container.lifecycle.preStop = { exec: { command: ["sh", "-c", command] } }; -} - -function boundedInteger(value, fallback, minimum, maximum) { - const parsed = Number(value); - if (!Number.isInteger(parsed)) return fallback; - return Math.max(minimum, Math.min(maximum, parsed)); -} - -function normalizeProbeTiming(value) { - if (!value || typeof value !== "object" || Array.isArray(value)) return {}; - const result = {}; - for (const field of ["initialDelaySeconds", "periodSeconds", "timeoutSeconds", "failureThreshold", "successThreshold"]) { - const raw = value[field]; - const minimum = field === "initialDelaySeconds" ? 0 : 1; - if (Number.isInteger(raw) && raw >= minimum) result[field] = raw; - } - return result; -} - -function useLocalHealthProbe(container, pathValue = "/health") { - if (!container) return; - for (const probeName of ["readinessProbe", "livenessProbe"]) { - const probe = container[probeName]; - if (!probe?.httpGet) continue; - probe.httpGet.path = pathValue; - } -} - -function annotateEnvReusePodTemplate(podMetadata, bootMetadata) { - if (!podMetadata || !bootMetadata) return; - annotate(podMetadata, { - "hwlab.pikastech.local/runtime-mode": bootMetadata.runtimeMode, - "hwlab.pikastech.local/boot-repo": bootMetadata.bootRepo, - "hwlab.pikastech.local/boot-commit": bootMetadata.bootCommit, - "hwlab.pikastech.local/boot-sh": bootMetadata.bootSh, - "hwlab.pikastech.local/environment-digest": bootMetadata.environmentDigest ?? "not_published", - "hwlab.pikastech.local/environment-input-hash": bootMetadata.environmentInputHash ?? "unknown", - "hwlab.pikastech.local/code-input-hash": bootMetadata.codeInputHash ?? "unknown" - }); - label(podMetadata, { - "hwlab.pikastech.local/runtime-mode": bootMetadata.runtimeMode, - "hwlab.pikastech.local/boot-commit": bootMetadata.bootCommit - }); -} - -function runtimeArtifactForService({ catalog, deploy, serviceId, source, registryPrefix, useDeployImages = false, digestPin = false, envReuseServiceIds = null }) { - const catalogService = catalogServiceById(catalog, serviceId); - if (v02EnvReuseEnabled(catalog, serviceId, envReuseServiceIds)) { - const environmentImage = catalogService?.environmentImage ?? imageFor(registryPrefix, `${serviceId}-env`, `env-${String(catalogService?.environmentInputHash ?? source.full).slice(0, 12)}`); - const runtimeImage = digestPin && useDeployImages ? digestPinnedImage(environmentImage, catalogService?.environmentDigest ?? catalogService?.digest) : environmentImage; - return { - image: runtimeImage, - imageTag: imageTagFromReference(environmentImage) ?? source.imageTag ?? source.short, - commit: source.full, - runtimeMode: v02EnvReuseRuntimeMode, - environmentDigest: catalogService?.environmentDigest ?? catalogService?.digest ?? null - }; - } - const fallbackImageTag = source.imageTag ?? source.short; - const image = useDeployImages && catalogService?.image - ? catalogService.image - : imageFor(registryPrefix, serviceId, fallbackImageTag); - const runtimeImage = digestPin && useDeployImages ? digestPinnedImage(image, catalogService?.digest) : image; - const imageTag = useDeployImages && catalogService?.imageTag - ? catalogService.imageTag - : imageTagFromReference(image) ?? fallbackImageTag; - const commit = useDeployImages - ? catalogService?.sourceCommitId ?? catalogService?.commitId ?? imageTag ?? source.full - : source.full; - return { image: runtimeImage, imageTag, commit }; -} - -function runtimeImageForService(options) { - return runtimeArtifactForService(options).image; -} - -function runtimeCommitForService(options) { - return runtimeArtifactForService(options).commit; -} - -function runtimeImageTagForService(options) { - return runtimeArtifactForService(options).imageTag; -} - -function artifactEnvironment(args) { - return isRuntimeLane(args.lane) ? args.lane : "dev"; -} - -function artifactEndpoint(args) { - return isRuntimeLane(args.lane) ? args.runtimeEndpoint : defaultRuntimeEndpoint; -} - -function serviceIdsForLane(lane, deploy = null) { - return isRuntimeLane(lane) ? runtimeLaneServiceIdsFromDeploy(deploy, lane) : defaultServiceIds; -} - -function serviceIdsForProfile(profile, deploy = null) { - return isRuntimeLane(profile) ? runtimeLaneServiceIdsFromDeploy(deploy, profile) : defaultServiceIds; -} - -function runtimeLaneServiceIdsFromDeploy(deploy, lane) { - const laneConfig = runtimeLaneConfig(deploy, lane); - const serviceIds = uniquePreserveOrder((Array.isArray(laneConfig.envReuseServices) ? laneConfig.envReuseServices : []) - .map((item) => String(item ?? "").trim()) - .filter(Boolean)); - assert.ok(serviceIds.length > 0, `deploy.lanes.${lane}.envReuseServices must not be empty`); - return serviceIds; -} - -function v02ServiceIdsFromDeploy(deploy) { - return runtimeLaneServiceIdsFromDeploy(deploy, "v02"); -} - -function runtimeLaneObservableServicesForDeploy(deploy, lane) { - const laneConfig = runtimeLaneConfig(deploy, lane); - const declarations = laneConfig.serviceDeclarations ?? {}; - const serviceIds = new Set(runtimeLaneServiceIdsFromDeploy(deploy, lane)); - const declared = Object.entries(declarations) - .filter(([serviceId, declaration]) => serviceIds.has(serviceId) && declaration?.observable !== false && Number.isInteger(declaration?.healthPort)) - .map(([serviceId, declaration]) => ({ - serviceId, - port: declaration.healthPort, - healthPath: declaration.healthPath || "/health/live" - })); - return [...declared, ...v02ExtraObservableServices]; -} - -function v02ObservableServicesForDeploy(deploy) { - return runtimeLaneObservableServicesForDeploy(deploy, "v02"); -} - -function v02ObservableService(deploy, serviceId) { - return v02ObservableServicesForDeploy(deploy).find((item) => item.serviceId === serviceId) ?? null; -} - -function runtimeLaneObservableService(deploy, lane, serviceId) { - return runtimeLaneObservableServicesForDeploy(deploy, lane).find((item) => item.serviceId === serviceId) ?? null; -} - -function runtimeLaneObservableServiceIdsForDeploy(deploy, lane) { - return new Set(runtimeLaneObservableServicesForDeploy(deploy, lane).map((item) => item.serviceId)); -} - -function v02ObservableServiceIdsForDeploy(deploy) { - return runtimeLaneObservableServiceIdsForDeploy(deploy, "v02"); -} - -function servicesParamForLane(lane, deploy = null) { - return serviceIdsForLane(lane, deploy).join(","); -} - -function uniquePreserveOrder(values) { - const seen = new Set(); - const result = []; - for (const value of values) { - if (seen.has(value)) continue; - seen.add(value); - result.push(value); - } - return result; -} - -function isV02RemovedServiceId(serviceId) { - return v02RemovedServiceIds.has(serviceId); -} - -function artifactCatalogSkeleton({ args, deploy, source }) { - const environment = artifactEnvironment(args); - const namespace = isRuntimeLane(args.lane) ? namespaceNameForProfile(args.lane) : "hwlab-dev"; - const tag = imageTagForSource(args, source); - return { - catalogVersion: "v1", - kind: "hwlab-artifact-catalog", - environment, - profile: environment, - namespace, - endpoint: artifactEndpoint(args), - commitId: tag, - artifactState: "contract-skeleton", - publish: { - registryPrefix: args.registryPrefix, - sourceCommitId: source.full, - imageTag: tag, - publishedAt: null - }, - allowedProfiles: [environment], - forbiddenProfiles: isRuntimeLane(args.lane) ? ["dev", "prod"] : ["prod"], - services: serviceIdsForLane(args.lane, deploy).map((serviceId) => artifactCatalogSkeletonService({ args, deploy, source, serviceId, environment, namespace, tag })) - }; -} - -function artifactCatalogSkeletonService({ args, deploy, source, serviceId, environment, namespace, tag }) { - const base = { - serviceId, - profile: environment, - namespace, - commitId: tag, - sourceCommitId: source.full, - image: imageFor(args.registryPrefix, serviceId, tag), - imageTag: tag, - digest: null, - buildBackend: "contract-skeleton" - }; - const envReuseServiceIds = envReuseServiceIdsForLane(deploy, args.lane); - if (!envReuseServiceIds.has(serviceId)) return base; - const bootSh = bootShForService(deployServiceForBoot(deploy, serviceId, args.lane), serviceId); - const environmentImage = imageFor(args.registryPrefix, `${serviceId}-env`, `env-${String(source.full).slice(0, 12)}`); - return { - ...base, - image: environmentImage, - imageTag: imageTagFromReference(environmentImage), - runtimeMode: v02EnvReuseRuntimeMode, - envReuse: true, - environmentImage, - environmentDigest: null, - environmentInputHash: null, - bootRepo: args.sourceRepo, - bootCommit: source.full, - bootSh, - codeInputHash: null, - bootEnvEvidence: { - env: { - HWLAB_BOOT_REPO: args.sourceRepo, - HWLAB_BOOT_COMMIT: source.full, - HWLAB_BOOT_SH: bootSh - } - } - }; -} - -function namespaceNameForProfile(profile) { - if (isRuntimeLane(profile)) return `hwlab-${profile}`; - return profile === "prod" ? "hwlab-prod" : "hwlab-dev"; -} - -function runtimePathForProfile(profile) { - if (isRuntimeLane(profile)) return `runtime-${profile}`; - return profile === "prod" ? "runtime-prod" : "runtime-dev"; -} - -function runtimeLabelForProfile(profile) { - if (isRuntimeLane(profile)) return profile; - return profile === "prod" ? "prod" : "dev"; -} - -function gitopsTargetForProfile(profile) { - return isRuntimeLane(profile) ? profile : "node"; -} - -function gitopsRootForArgs(args) { - return String(args.gitopsRoot || defaultOutDir).replace(/\/+$/u, ""); -} - -function gitopsPathForProfile(args, profile) { - return `${gitopsRootForArgs(args)}/${runtimePathForProfile(profile)}`; -} - -function gitopsPathFor(args, relativePath) { - return `${gitopsRootForArgs(args)}/${String(relativePath).replace(/^\/+/, "")}`; -} - -function profileEndpoint(args, profile) { - if (isRuntimeLane(profile)) return { runtimeEndpoint: args.runtimeEndpoint, webEndpoint: args.webEndpoint }; - return profile === "prod" - ? { runtimeEndpoint: args.prodRuntimeEndpoint, webEndpoint: args.prodWebEndpoint } - : { runtimeEndpoint: args.runtimeEndpoint, webEndpoint: args.webEndpoint }; -} - -function laneRuntimeProfiles(args) { - return isRuntimeLane(args.lane) ? [args.lane] : ["dev", "prod"]; -} - -function laneNames(args) { - if (isRuntimeLane(args.lane)) { - const namespace = namespaceNameForProfile(args.lane); - return { - gitopsTarget: args.lane, - pipeline: `${namespace}-ci-image-publish`, - poller: `${namespace}-branch-poller`, - reconciler: `${namespace}-control-plane-reconciler`, - serviceAccount: `${namespace}-tekton-runner`, - pipelineRunPrefix: `${namespace}-ci-poll-`, - runtimeNamespace: namespace, - runtimePath: gitopsPathForProfile(args, args.lane), - argoApplications: [`argocd/hwlab-node-${args.lane}`] - }; - } - return { - gitopsTarget: "node", - pipeline: "hwlab-node-ci-image-publish", - poller: "hwlab-node-branch-poller", - reconciler: "hwlab-node-control-plane-reconciler", - serviceAccount: "hwlab-tekton-runner", - pipelineRunPrefix: "hwlab-node-ci-poll-", - runtimeNamespace: "hwlab-dev", - runtimePath: gitopsPathForProfile(args, "dev"), - argoApplications: ["argocd/hwlab-node-dev", "argocd/hwlab-node-prod"] - }; -} - -function argoApplicationName(profile) { - return `hwlab-node-${profile}`; -} - -function imageTagForSource(args, source) { - return args.imageTagMode === "full" ? source.full : source.short; -} - -function deepSeekBaseUrl(namespace) { - return `http://hwlab-deepseek-proxy.${namespace}.svc.cluster.local:4000/v1/responses`; -} - -function codeAgentNoProxy(namespace) { - return [ - `${namespace}.svc.cluster.local`, - ".svc", - ".cluster.local", - "hyueapi.com", - ".hyueapi.com", - "hyui.com", - ".hyui.com", - "127.0.0.1", - "localhost", - "::1", - "10.0.0.0/8", - "10.42.0.0/16", - "10.43.0.0/16", - "api.minimaxi.com", - ".minimaxi.com" - ].join(","); -} - -function namespaceScopedHost(value, namespace) { - if (typeof value !== "string") return value; - return value.replace(/\.hwlab-dev\.svc\.cluster\.local/gu, `.${namespace}.svc.cluster.local`); -} - -function secretNameForProfile(secretName, profile) { - if (!isRuntimeLane(profile)) return secretName; - if (secretName === "hwlab-cloud-api-dev-db") return `hwlab-cloud-api-${profile}-db`; - if (secretName === "hwlab-code-agent-provider") return `hwlab-${profile}-code-agent-provider`; - if (secretName === "hwlab-code-agent-codex-auth") return `hwlab-${profile}-code-agent-codex-auth`; - if (secretName === "hwlab-opencode-server-auth") return `hwlab-${profile}-opencode-server-auth`; - if (secretName === "hwlab-bootstrap-admin") return `hwlab-${profile}-bootstrap-admin`; - return secretName; -} - -function secretRefObjectForProfile(secretRef, profile) { - const ref = configString(secretRef); - const slashIndex = ref.indexOf("/"); - assert.ok(slashIndex > 0 && slashIndex < ref.length - 1, `secretRef ${JSON.stringify(secretRef)} must be name/key`); - return { name: secretNameForProfile(ref.slice(0, slashIndex), profile), key: ref.slice(slashIndex + 1), optional: true }; -} - -function rewritePodSecretRefs(podSpec, profile) { - if (!podSpec || !isRuntimeLane(profile)) return; - for (const volume of podSpec.volumes ?? []) { - if (volume?.secret?.secretName) volume.secret.secretName = secretNameForProfile(volume.secret.secretName, profile); - } -} - -function deployEnvEntry(name, value, namespace, profile = "dev") { - if (typeof value === "string" && value.startsWith("secretRef:")) { - return { name, valueFrom: { secretKeyRef: secretRefObjectForProfile(value.slice("secretRef:".length), profile) } }; - } - return { name, value: namespaceScopedHost(value, namespace) }; -} - -function applyDeployEnv(envList, deployService, namespace, profile = "dev") { - if (!deployService?.env || typeof deployService.env !== "object" || Array.isArray(deployService.env)) return; - for (const [name, value] of Object.entries(deployService.env)) { - upsertEnvEntry(envList, deployEnvEntry(name, value, namespace, profile)); - } -} - -function applyDeployConfigMapMounts(podSpec, container, deployService) { - const mounts = Array.isArray(deployService?.configMapMounts) ? deployService.configMapMounts : []; - if (mounts.length === 0 || !podSpec || !container) return; - podSpec.volumes ??= []; - container.volumeMounts ??= []; - for (const mount of mounts) { - const name = String(mount?.name ?? "").trim(); - const configMapName = String(mount?.configMapName ?? "").trim(); - const mountPath = String(mount?.mountPath ?? "").trim(); - if (!name || !configMapName || !mountPath) continue; - const items = Array.isArray(mount.items) - ? mount.items - .map((item) => ({ key: String(item?.key ?? "").trim(), path: String(item?.path ?? "").trim() })) - .filter((item) => item.key && item.path) - : []; - const volume = { name, configMap: { name: configMapName } }; - if (items.length > 0) volume.configMap.items = items; - const existingVolume = podSpec.volumes.find((entry) => entry?.name === name); - if (existingVolume) Object.assign(existingVolume, volume); - else podSpec.volumes.push(volume); - const volumeMount = { name, mountPath, readOnly: mount.readOnly !== false }; - const existingMount = container.volumeMounts.find((entry) => entry?.name === name); - if (existingMount) Object.assign(existingMount, volumeMount); - else container.volumeMounts.push(volumeMount); - } -} - -function removeV02LegacySimulatorEnv(envList) { - if (!Array.isArray(envList)) return envList; - return envList.filter((entry) => !v02RemovedServiceEnvNames.has(entry?.name)); -} - -function removeV02RepoOwnedCodeAgentPodConfig(deploymentSpec, podSpec) { - if (!podSpec) return; - podSpec.initContainers = (podSpec.initContainers ?? []).filter((entry) => entry?.name !== "hwlab-code-agent-workspace-init"); - const removedVolumeNames = new Set([ - "hwlab-code-agent-workspace", - "hwlab-code-agent-codex-home", - "hwlab-code-agent-codex-config", - "hwlab-code-agent-codex-auth" - ]); - podSpec.volumes = (podSpec.volumes ?? []).filter((volume) => !removedVolumeNames.has(volume?.name)); - for (const container of podSpec.containers ?? []) { - container.volumeMounts = (container.volumeMounts ?? []).filter((mount) => !removedVolumeNames.has(mount?.name)); - } - if (deploymentSpec?.strategy?.type === "Recreate") delete deploymentSpec.strategy; -} - -function applyDeployServiceReplicas(item, deployService) { - if (!(item?.kind === "Deployment" || item?.kind === "StatefulSet")) return; - const replicas = deployService?.replicas; - if (!Number.isInteger(replicas) || replicas < 0) return; - item.spec ??= {}; - item.spec.replicas = replicas; -} - -function deployServicesForProfile(deploy, profile) { - const allowed = new Set(serviceIdsForProfile(profile, deploy)); - const services = new Map((deploy.services ?? []) - .filter((service) => allowed.has(service.serviceId)) - .map((service) => { - const copy = cloneJson(service); - copy.env = mergeEnvMaps( - copy.env, - serviceDeclarationEnvForProfile(deploy, profile, service.serviceId), - accessControlEnvForProfile(deploy, profile, service.serviceId), - runtimeStoreEnvForProfile(deploy, profile, service.serviceId), - workbenchRuntimeClientEnvForProfile(deploy, profile, service.serviceId), - workbenchRuntimeFactsQueryEnvForProfile(deploy, profile, service.serviceId), - workbenchRuntimeSessionsSummaryEnvForProfile(deploy, profile, service.serviceId), - workbenchRuntimePostgresEnvForProfile(deploy, profile, service.serviceId), - workbenchRuntimeRedisEnvForProfile(deploy, profile, service.serviceId) - ); - return [service.serviceId, copy]; - })); - const laneServices = isRuntimeLane(profile) && Array.isArray(deploy.lanes?.[profile]?.services) - ? deploy.lanes[profile].services - : []; - for (const override of laneServices) { - const existing = services.get(override.serviceId) ?? { - serviceId: override.serviceId, - env: mergeEnvMaps( - serviceDeclarationEnvForProfile(deploy, profile, override.serviceId), - accessControlEnvForProfile(deploy, profile, override.serviceId), - runtimeStoreEnvForProfile(deploy, profile, override.serviceId), - workbenchRuntimeClientEnvForProfile(deploy, profile, override.serviceId), - workbenchRuntimeFactsQueryEnvForProfile(deploy, profile, override.serviceId), - workbenchRuntimeSessionsSummaryEnvForProfile(deploy, profile, override.serviceId), - workbenchRuntimePostgresEnvForProfile(deploy, profile, override.serviceId), - workbenchRuntimeRedisEnvForProfile(deploy, profile, override.serviceId) - ) - }; - services.set(override.serviceId, { - ...existing, - ...cloneJson(override), - env: mergeEnvMaps(existing.env, override.env) - }); - } - return services; -} - -function accessControlEnvForProfile(deploy, profile, serviceId) { - return deploy?.__accessControlEnvByProfile?.[profile]?.[serviceId] ?? {}; -} - -function serviceDeclarationEnvForProfile(deploy, profile, serviceId) { - if (!isRuntimeLane(profile)) return {}; - const env = deploy?.lanes?.[profile]?.serviceDeclarations?.[serviceId]?.env; - if (!env || typeof env !== "object" || Array.isArray(env)) return {}; - return cloneJson(env); -} - -function runtimeWorkbenchTraceTimelinePolicy(deploy, profile) { - if (!isRuntimeLane(profile)) return null; - const traceTimeline = runtimeLaneConfig(deploy, profile)?.workbench?.traceTimeline; - if (!traceTimeline || typeof traceTimeline !== "object" || Array.isArray(traceTimeline)) return null; - const result = {}; - if (typeof traceTimeline.autoExpandRunning === "boolean") result.autoExpandRunning = traceTimeline.autoExpandRunning; - if (typeof traceTimeline.autoCollapseTerminal === "boolean") result.autoCollapseTerminal = traceTimeline.autoCollapseTerminal; - return Object.keys(result).length > 0 ? result : null; -} - -function runtimeTraceExplorerUrlTemplate(deploy, profile) { - if (!isRuntimeLane(profile)) return null; - const template = runtimeLaneConfig(deploy, profile)?.observability?.traceExplorerUrlTemplate; - return typeof template === "string" && template.trim().length > 0 ? template.trim() : null; -} - -function runtimePrometheusOperatorResourcesEnabled(deploy, profile) { - if (!isRuntimeLane(profile)) return true; - const enabled = runtimeLaneConfig(deploy, profile)?.observability?.prometheusOperatorResources; - return enabled !== false; -} - -function runtimeFrpConfigForProfile(deploy, profile, args = {}) { - const fallbackWebRemotePort = isRuntimeLane(profile) ? defaultPortForRuntimeLane(profile) : profile === "prod" ? 18666 : 17666; - const fallbackEdgeRemotePort = isRuntimeLane(profile) ? fallbackWebRemotePort + 1 : profile === "prod" ? 18667 : 17667; - const baseFrp = isRuntimeLane(profile) ? runtimeLaneConfig(deploy, profile)?.frp : deploy?.frp; - const nodeId = effectiveSecretPlaneNodeId(args); - const nodeFrp = nodeId ? baseFrp?.nodes?.[nodeId] : null; - const frp = nodeFrp && typeof nodeFrp === "object" && !Array.isArray(nodeFrp) ? { ...baseFrp, ...nodeFrp } : baseFrp; - const server = frp && typeof frp === "object" && !Array.isArray(frp) ? frp.server : null; - const proxies = Array.isArray(frp?.proxies) ? frp.proxies : []; - const proxyByEndpoint = new Map(proxies - .filter((proxy) => proxy && typeof proxy === "object" && !Array.isArray(proxy)) - .map((proxy) => [String(proxy.endpointId || proxy.name || ""), proxy])); - const proxyValue = (endpointId, key, fallback) => { - const value = proxyByEndpoint.get(endpointId)?.[key]; - return typeof value === "string" && value.trim().length > 0 ? value.trim() : fallback; - }; - const proxyPort = (endpointId, fallback) => { - const value = proxyByEndpoint.get(endpointId)?.remotePort; - return Number.isInteger(value) ? value : fallback; - }; - return { - serverAddr: typeof server?.address === "string" && server.address.trim().length > 0 ? server.address.trim() : "74.48.78.17", - serverPort: Number.isInteger(server?.bindPort) ? server.bindPort : 7000, - webRemotePort: proxyPort("frontend", fallbackWebRemotePort), - edgeRemotePort: proxyPort("api", fallbackEdgeRemotePort), - webProxyName: proxyValue("frontend", "name", null), - edgeProxyName: proxyValue("api", "name", null), - authSecretName: typeof frp?.auth?.secretName === "string" && frp.auth.secretName.trim().length > 0 ? frp.auth.secretName.trim() : null, - authSecretKey: typeof frp?.auth?.secretKey === "string" && frp.auth.secretKey.trim().length > 0 ? frp.auth.secretKey.trim() : null - }; -} - -function runtimeStoreEnvForProfile(deploy, profile, serviceId) { - if (!isRuntimeLane(profile) || serviceId !== "hwlab-cloud-api") return {}; - const postgres = runtimeLaneConfig(deploy, profile)?.runtimeStore?.postgres; - if (!postgres || typeof postgres !== "object" || Array.isArray(postgres)) return {}; - const env = {}; - if (postgres.sslMode === "disable" || postgres.sslMode === "require") env.HWLAB_CLOUD_DB_SSL_MODE = postgres.sslMode; - if (Number.isInteger(postgres.poolMax)) env.HWLAB_CLOUD_DB_POOL_MAX = String(postgres.poolMax); - if (Number.isInteger(postgres.connectionTimeoutMs)) { - env.HWLAB_CLOUD_DB_PROBE_TIMEOUT_MS = String(postgres.connectionTimeoutMs); - } - if (Number.isInteger(postgres.queryTimeoutMs)) { - env.HWLAB_CLOUD_DB_QUERY_TIMEOUT_MS = String(postgres.queryTimeoutMs); - } - if (Number.isInteger(postgres.queryRetryMaxAttempts)) { - env.HWLAB_CLOUD_DB_QUERY_RETRY_MAX_ATTEMPTS = String(postgres.queryRetryMaxAttempts); - } - if (Number.isInteger(postgres.queryRetryInitialDelayMs)) { - env.HWLAB_CLOUD_DB_QUERY_RETRY_INITIAL_DELAY_MS = String(postgres.queryRetryInitialDelayMs); - } - if (Number.isInteger(postgres.queryRetryMaxDelayMs)) { - env.HWLAB_CLOUD_DB_QUERY_RETRY_MAX_DELAY_MS = String(postgres.queryRetryMaxDelayMs); - } - return env; -} - -function workbenchRuntimeRedisEnvForProfile(deploy, profile, serviceId) { - if (!isRuntimeLane(profile) || serviceId !== "hwlab-workbench-runtime") return {}; - const redis = workbenchRuntimeRedisConfigForProfile(deploy, profile); - if (!redis || typeof redis !== "object" || Array.isArray(redis)) return {}; - const env = { - HWLAB_WORKBENCH_CACHE_REDIS_ENABLED: booleanEnv(Boolean(redis.enabled)) - }; - const namespace = typeof redis.namespace === "string" ? redis.namespace.trim() : ""; - const serviceName = typeof redis.serviceName === "string" ? redis.serviceName.trim() : ""; - const port = Number.isInteger(redis.port) ? redis.port : null; - if (namespace) env.HWLAB_WORKBENCH_CACHE_REDIS_NAMESPACE = namespace; - if (serviceName) env.HWLAB_WORKBENCH_CACHE_REDIS_SERVICE_NAME = serviceName; - if (port !== null) env.HWLAB_WORKBENCH_CACHE_REDIS_PORT = String(port); - if (namespace && serviceName && port !== null) { - env.HWLAB_WORKBENCH_CACHE_REDIS_URL = `redis://${serviceName}.${namespace}.svc.cluster.local:${port}/0`; - } - const ttl = redis.ttl ?? {}; - if (Number.isInteger(ttl.sessionsSummaryMs)) env.HWLAB_WORKBENCH_CACHE_REDIS_TTL_SESSIONS_SUMMARY_MS = String(ttl.sessionsSummaryMs); - if (Number.isInteger(ttl.terminalTurnMs)) env.HWLAB_WORKBENCH_CACHE_REDIS_TTL_TERMINAL_TURN_MS = String(ttl.terminalTurnMs); - if (Number.isInteger(ttl.terminalTracePageMs)) env.HWLAB_WORKBENCH_CACHE_REDIS_TTL_TERMINAL_TRACE_PAGE_MS = String(ttl.terminalTracePageMs); - if (Number.isInteger(ttl.runningPageMs)) env.HWLAB_WORKBENCH_CACHE_REDIS_TTL_RUNNING_PAGE_MS = String(ttl.runningPageMs); - const limits = redis.limits ?? {}; - if (Number.isInteger(limits.maxKeyBytes)) env.HWLAB_WORKBENCH_CACHE_REDIS_MAX_KEY_BYTES = String(limits.maxKeyBytes); - if (Number.isInteger(limits.maxPayloadBytes)) env.HWLAB_WORKBENCH_CACHE_REDIS_MAX_PAYLOAD_BYTES = String(limits.maxPayloadBytes); - const timeouts = redis.timeouts ?? {}; - if (Number.isInteger(timeouts.connectMs)) env.HWLAB_WORKBENCH_CACHE_REDIS_CONNECT_TIMEOUT_MS = String(timeouts.connectMs); - if (Number.isInteger(timeouts.operationMs)) env.HWLAB_WORKBENCH_CACHE_REDIS_OPERATION_TIMEOUT_MS = String(timeouts.operationMs); - const labels = redis.metrics?.labels; - if (labels && typeof labels === "object" && !Array.isArray(labels)) { - env.HWLAB_WORKBENCH_CACHE_REDIS_METRICS_LABELS = JSON.stringify(labels); - } - const behavior = redis.unavailableBehavior ?? {}; - if (typeof behavior.livenessFailure === "boolean") env.HWLAB_WORKBENCH_CACHE_REDIS_UNAVAILABLE_LIVENESS_FAILURE = booleanEnv(behavior.livenessFailure); - if (typeof behavior.readMode === "string") env.HWLAB_WORKBENCH_CACHE_REDIS_UNAVAILABLE_READ_MODE = behavior.readMode; - if (typeof behavior.fallbackStateSource === "boolean") env.HWLAB_WORKBENCH_CACHE_REDIS_UNAVAILABLE_FALLBACK_STATE_SOURCE = booleanEnv(behavior.fallbackStateSource); - if (typeof redis.disableSwitchEnv === "string" && redis.disableSwitchEnv.trim()) { - env[redis.disableSwitchEnv.trim()] = booleanEnv(Boolean(redis.enabled)); - } - return env; -} - -function workbenchRuntimeClientEnvForProfile(deploy, profile, serviceId) { - if (!isRuntimeLane(profile) || serviceId !== "hwlab-cloud-api") return {}; - const client = runtimeLaneConfig(deploy, profile)?.workbenchRuntime?.client; - if (!client || typeof client !== "object" || Array.isArray(client)) return {}; - const env = {}; - if (Number.isInteger(client.timeoutMs)) env.HWLAB_WORKBENCH_RUNTIME_TIMEOUT_MS = String(client.timeoutMs); - return env; -} - -function workbenchRuntimeFactsQueryEnvForProfile(deploy, profile, serviceId) { - if (!isRuntimeLane(profile) || serviceId !== "hwlab-cloud-api") return {}; - const factsQuery = runtimeLaneConfig(deploy, profile)?.workbenchRuntime?.factsQuery; - if (!factsQuery || typeof factsQuery !== "object" || Array.isArray(factsQuery)) return {}; - const env = {}; - if (Number.isInteger(factsQuery.maxAttempts)) env.HWLAB_WORKBENCH_FACTS_QUERY_MAX_ATTEMPTS = String(factsQuery.maxAttempts); - if (Number.isInteger(factsQuery.backoffBaseMs)) env.HWLAB_WORKBENCH_FACTS_QUERY_BACKOFF_BASE_MS = String(factsQuery.backoffBaseMs); - if (Number.isInteger(factsQuery.backoffMaxMs)) env.HWLAB_WORKBENCH_FACTS_QUERY_BACKOFF_MAX_MS = String(factsQuery.backoffMaxMs); - return env; -} - -function workbenchRuntimeSessionsSummaryEnvForProfile(deploy, profile, serviceId) { - if (!isRuntimeLane(profile) || serviceId !== "hwlab-workbench-runtime") return {}; - const sessionsSummary = runtimeLaneConfig(deploy, profile)?.workbenchRuntime?.sessionsSummary; - if (!sessionsSummary || typeof sessionsSummary !== "object" || Array.isArray(sessionsSummary)) return {}; - const env = {}; - if (Number.isInteger(sessionsSummary.maxAttempts)) env.HWLAB_WORKBENCH_SESSIONS_SUMMARY_QUERY_MAX_ATTEMPTS = String(sessionsSummary.maxAttempts); - if (Number.isInteger(sessionsSummary.attemptTimeoutMs)) env.HWLAB_WORKBENCH_SESSIONS_SUMMARY_QUERY_ATTEMPT_TIMEOUT_MS = String(sessionsSummary.attemptTimeoutMs); - if (Number.isInteger(sessionsSummary.backoffMs)) env.HWLAB_WORKBENCH_SESSIONS_SUMMARY_QUERY_BACKOFF_MS = String(sessionsSummary.backoffMs); - return env; -} - -function workbenchRuntimePostgresEnvForProfile(deploy, profile, serviceId) { - if (!isRuntimeLane(profile) || serviceId !== "hwlab-workbench-runtime") return {}; - const postgres = runtimeLaneConfig(deploy, profile)?.workbenchRuntime?.postgres; - if (!postgres || typeof postgres !== "object" || Array.isArray(postgres)) return {}; - const env = {}; - if (Number.isInteger(postgres.poolMax)) env.HWLAB_WORKBENCH_RUNTIME_DB_POOL_MAX = String(postgres.poolMax); - if (Number.isInteger(postgres.queryTimeoutMs)) env.HWLAB_WORKBENCH_RUNTIME_QUERY_TIMEOUT_MS = String(postgres.queryTimeoutMs); - if (Number.isInteger(postgres.readyTimeoutMs)) env.HWLAB_WORKBENCH_RUNTIME_READY_TIMEOUT_MS = String(postgres.readyTimeoutMs); - return env; -} - -function workbenchRuntimeRedisConfigForProfile(deploy, profile) { - if (!isRuntimeLane(profile)) return null; - const label = `deploy.lanes.${profile}.workbenchRuntime.cache.redis`; - const redis = runtimeLaneConfig(deploy, profile)?.workbenchRuntime?.cache?.redis; - if (!redis || redis.enabled !== true) return null; - configObject(redis, label); - const namespace = requiredConfigString(redis, "namespace", label); - assert.equal(namespace, namespaceNameForProfile(profile), `${label}.namespace must match ${namespaceNameForProfile(profile)}`); - const serviceName = requiredConfigString(redis, "serviceName", label); - const image = requiredConfigString(redis, "image", label); - const port = requiredConfigPositiveInteger(redis, "port", label); - assert.ok(port <= 65535, `${label}.port must be a valid TCP port`); - const resources = configObject(redis.resources, `${label}.resources`); - configObject(resources.requests, `${label}.resources.requests`); - configObject(resources.limits, `${label}.resources.limits`); - const memoryPolicy = configObject(redis.memoryPolicy, `${label}.memoryPolicy`); - assert.equal(memoryPolicy.persistence, "disabled", `${label}.memoryPolicy.persistence must be disabled`); - requiredConfigString(memoryPolicy, "maxMemory", `${label}.memoryPolicy`); - requiredConfigString(memoryPolicy, "eviction", `${label}.memoryPolicy`); - return redis; -} - -function booleanEnv(value) { - return value ? "1" : "0"; -} - -function mergeEnvMaps(...values) { - const result = {}; - for (const value of values) { - if (!value || typeof value !== "object" || Array.isArray(value)) continue; - Object.assign(result, value); - } - return result; -} - -function runtimeNodeIdForProfile(deploy, profile, fallback = "node") { - if (!isRuntimeLane(profile)) return fallback; - if (typeof fallback === "string" && fallback.trim() && fallback !== "node") return fallback; - const nodeId = deploy?.lanes?.[profile]?.node; - return typeof nodeId === "string" && nodeId.length > 0 ? nodeId : fallback; -} - -function v02MetricsLabels(serviceId) { - return { - "hwlab.pikastech.local/monitoring": "enabled", - "hwlab.pikastech.local/service-id": serviceId - }; -} - -function v02MetricsSidecarVolume(profile = "v02") { - return { name: "hwlab-metrics-sidecar", configMap: { name: `hwlab-${profile}-metrics-sidecar` } }; -} - -function v02MetricsSidecarAnnotations(metricsSidecarSha256) { - return metricsSidecarSha256 ? { "hwlab.pikastech.local/metrics-sidecar-sha256": metricsSidecarSha256 } : {}; -} - -function v02MetricsSidecarContainer({ deploy, serviceId, namespace, gitopsTarget, profile = "v02" }) { - const service = runtimeLaneObservableService(deploy, profile, serviceId); - assert.ok(service, `unknown ${profile} observable service ${serviceId}`); - const extraMetricsEnv = serviceId === "hwlab-cloud-api" - ? [ - { name: "HWLAB_METRICS_EXTRA_URL", value: `http://127.0.0.1:${service.port}/v1/web-performance/metrics` }, - { name: "HWLAB_METRICS_EXTRA_TIMEOUT_MS", value: "2000" } - ] - : []; - return { - name: "hwlab-metrics", - image: ciToolsRunnerImage, - imagePullPolicy: "IfNotPresent", - command: ["node", "/metrics/metrics-sidecar.mjs"], - env: [ - { name: "HWLAB_METRICS_SERVICE_ID", value: serviceId }, - { name: "HWLAB_METRICS_NAMESPACE", value: namespace }, - { name: "HWLAB_METRICS_GITOPS_TARGET", value: gitopsTarget }, - { name: "HWLAB_METRICS_PORT", value: "9100" }, - { name: "HWLAB_METRICS_TARGET_URL", value: `http://127.0.0.1:${service.port}${service.healthPath}` }, - { name: "HWLAB_METRICS_TARGET_TIMEOUT_MS", value: "2000" }, - ...extraMetricsEnv - ], - ports: [{ name: "metrics", containerPort: 9100 }], - readinessProbe: { httpGet: { path: "/health/live", port: "metrics" }, initialDelaySeconds: 3, periodSeconds: 10 }, - livenessProbe: { httpGet: { path: "/health/live", port: "metrics" }, initialDelaySeconds: 10, periodSeconds: 20 }, - volumeMounts: [{ name: "hwlab-metrics-sidecar", mountPath: "/metrics", readOnly: true }] - }; -} - -function upsertV02MetricsSidecar(podSpec, options) { - if (!podSpec || !Array.isArray(podSpec.containers)) return; - podSpec.containers = podSpec.containers.filter((container) => container?.name !== "hwlab-metrics"); - podSpec.containers.push(v02MetricsSidecarContainer(options)); - podSpec.volumes ??= []; - podSpec.volumes = podSpec.volumes.filter((volume) => volume?.name !== "hwlab-metrics-sidecar"); - podSpec.volumes.push(v02MetricsSidecarVolume(options.profile)); -} - -function upsertV02MetricsPort(service) { - service.spec ??= {}; - service.spec.ports ??= []; - service.spec.ports = service.spec.ports.filter((port) => port?.name !== "metrics"); - service.spec.ports.push({ name: "metrics", port: 9100, targetPort: "metrics" }); -} - -function transformWorkloads({ workloads, deploy, catalog, source, sourceBranch = defaultBranch, gitReadUrl, registryPrefix, runtimeEndpoint, webEndpoint, profile = "dev", nodeId = "node", useDeployImages = false, metricsSidecarSha256 = null }) { - const result = cloneJson(workloads); - const deployServices = deployServicesForProfile(deploy, profile); - const namespace = namespaceNameForProfile(profile); - const profileLabel = runtimeLabelForProfile(profile); - const gitopsTarget = gitopsTargetForProfile(profile); - const runtimeLane = isRuntimeLane(profile); - const digestPin = runtimeLane; - const envReuseServiceIds = envReuseServiceIdsForLane(deploy, profile); - const runtimeObservableServiceIds = runtimeLane ? runtimeLaneObservableServiceIdsForDeploy(deploy, profile) : new Set(); - const stableRuntimeLabels = { - "app.kubernetes.io/part-of": "hwlab", - "hwlab.pikastech.local/environment": profileLabel, - "hwlab.pikastech.local/profile": profileLabel - }; - result.items = asItems(result, "workloads").filter((item) => !(runtimeLane && isV02RemovedServiceId(serviceIdForWorkload(item, null)))); - for (const item of asItems(result, "workloads")) { - item.metadata.namespace = namespace; - label(item.metadata, { - ...stableRuntimeLabels, - "hwlab.pikastech.local/gitops-target": gitopsTarget, - "hwlab.pikastech.local/gitops-render-source-commit": source.full, - "hwlab.pikastech.local/source-commit": source.full, - "unidesk.ai/node-staging": profile === "dev" ? "true" : "false" - }); - annotate(item.metadata, { - "hwlab.pikastech.local/gitops-note": profile === "v02" ? "HWLAB v0.2 GitOps target." : profile === "prod" ? "node PROD GitOps target." : "node DEV GitOps target.", - "hwlab.pikastech.local/gitops-render-source-commit": source.full, - "hwlab.pikastech.local/source-commit": source.full - }); - if (item.kind === "Job") { - annotate(item.metadata, { - "argocd.argoproj.io/sync-options": "Force=true,Replace=true" - }); - if (item.spec?.suspend === true) { - annotate(item.metadata, { - "argocd.argoproj.io/ignore-healthcheck": "true" - }); - } - } - const podTemplate = item?.spec?.template; - if (!podTemplate?.spec?.containers) continue; - const templateServiceId = serviceIdForWorkload(item, podTemplate.spec.containers[0]); - const templateArtifact = runtimeArtifactForService({ catalog, deploy, serviceId: templateServiceId, source, registryPrefix, useDeployImages, digestPin, envReuseServiceIds }); - const templateSourceCommit = runtimeLane ? (templateArtifact.commit ?? source.full) : source.full; - const templateArtifactCommit = templateArtifact.commit; - label(podTemplate.metadata ??= {}, { - ...stableRuntimeLabels, - "hwlab.pikastech.local/gitops-target": gitopsTarget, - "hwlab.pikastech.local/gitops-render-source-commit": source.full, - "hwlab.pikastech.local/source-commit": templateSourceCommit, - "hwlab.pikastech.local/artifact-source-commit": templateArtifactCommit - }); - if (runtimeLane && runtimeObservableServiceIds.has(templateServiceId)) { - label(item.metadata, v02MetricsLabels(templateServiceId)); - label(podTemplate.metadata, v02MetricsLabels(templateServiceId)); - annotate(podTemplate.metadata, v02MetricsSidecarAnnotations(metricsSidecarSha256)); - upsertV02MetricsSidecar(podTemplate.spec, { deploy, serviceId: templateServiceId, namespace, gitopsTarget, profile }); - } - if ((item.kind === "Deployment" || item.kind === "StatefulSet") && item.spec?.selector?.matchLabels) { - item.spec.selector.matchLabels = { - ...item.spec.selector.matchLabels, - ...stableRuntimeLabels - }; - } - annotate(podTemplate.metadata, { - "hwlab.pikastech.local/gitops-render-source-commit": source.full, - "hwlab.pikastech.local/source-commit": templateSourceCommit, - "hwlab.pikastech.local/artifact-source-commit": templateArtifactCommit, - "hwlab.pikastech.local/rendered-by": "scripts/gitops-render.mjs" - }); - for (const container of podTemplate.spec.containers) { - if (container.name === "hwlab-metrics") continue; - const serviceId = serviceIdForWorkload(item, container); - const deployService = deployServices.get(serviceId); - if (!deployService) continue; - applyDeployServiceReplicas(item, deployService); - const artifact = runtimeArtifactForService({ catalog, deploy, serviceId, source, registryPrefix, useDeployImages, digestPin, envReuseServiceIds }); - const envReuse = runtimeLane && v02EnvReuseEnabled(catalog, serviceId, envReuseServiceIds); - const bootDeployService = envReuse ? deployServiceForBoot(deploy, serviceId, profile) : null; - const bootMetadata = envReuse ? bootMetadataForService({ args: { sourceRepo: deploy?.lanes?.[profile]?.sourceRepo || defaultSourceRepo, registryPrefix }, catalog, deployService: bootDeployService, serviceId, source }) : null; - const image = artifact.image; - const runtimeCommit = artifact.commit; - const runtimeImageTag = artifact.imageTag; - container.image = image; - container.imagePullPolicy = "IfNotPresent"; - container.env ??= []; - for (const entry of container.env) { - if (Object.hasOwn(entry, "value")) entry.value = namespaceScopedHost(entry.value, namespace); - } - applyDeployEnv(container.env, deployService, namespace, profile); - applyDeployConfigMapMounts(podTemplate.spec, container, deployService); - if (runtimeLane) container.env = removeV02LegacySimulatorEnv(container.env); - upsertEnv(container.env, "HWLAB_ENVIRONMENT", profileLabel); - upsertEnv(container.env, "HWLAB_COMMIT_ID", runtimeCommit); - upsertEnv(container.env, "HWLAB_IMAGE", image); - upsertEnv(container.env, "HWLAB_IMAGE_TAG", runtimeImageTag); - if (serviceId === "hwlab-agent-skills") { - upsertEnv(container.env, "HWLAB_SKILLS_COMMIT_ID", runtimeCommit); - } - upsertEnv(container.env, "HWLAB_GITOPS_TARGET", gitopsTarget); - upsertEnv(container.env, "HWLAB_GITOPS_PROFILE", profileLabel); - upsertEnv(container.env, "HWLAB_GITOPS_SOURCE_COMMIT", runtimeLane ? runtimeCommit : source.full); - if (runtimeLane && serviceId === "hwlab-cloud-api") { - upsertEnv(container.env, "HWLAB_CODE_AGENT_AGENTRUN_SOURCE_COMMIT", runtimeCommit); - } - if (bootMetadata) { - applyEnvReuseBootEnv(container, bootMetadata, { - sourceBranch, - gitReadUrl, - bootSh: envReuseBootShForContainer({ serviceId, container, defaultBootSh: bootMetadata.bootSh }) - }); - if (container.name === serviceId) annotateEnvReusePodTemplate(podTemplate.metadata, bootMetadata); - } - if (runtimeLane && container.name === serviceId) { - applyServiceHealthProbe(container, deploy?.lanes?.[profile]?.serviceDeclarations?.[serviceId]?.healthProbe); - } - if (serviceId === "hwlab-cloud-api" || serviceId === "hwlab-edge-proxy") { - upsertEnv(container.env, "HWLAB_PUBLIC_ENDPOINT", runtimeEndpoint); - } - if (runtimeLane && serviceId === "hwlab-edge-proxy") { - useLocalHealthProbe(container, "/health"); - } - if (serviceId === "hwlab-cloud-api") { - if (runtimeLane) { - removeV02RepoOwnedCodeAgentPodConfig(item.spec, podTemplate.spec); - applyServiceShutdownDrainLifecycle(container, deployService.shutdownDrain); - upsertEnv(container.env, "HWLAB_M3_IO_CONTROL_ENABLED", "false"); - upsertEnv(container.env, "HWLAB_CODE_AGENT_AGENTRUN_REPO_URL", gitReadUrl ?? defaultRuntimeLaneGitReadUrl); - upsertEnv(container.env, "HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID", runtimeNodeIdForProfile(deploy, profile, nodeId)); - upsertEnv(container.env, "HWLAB_OPENFGA_MODE", "enforce"); - upsertEnv(container.env, "HWLAB_OPENFGA_API_URL", `http://hwlab-openfga.${namespace}.svc.cluster.local:8080`); - upsertEnvEntry(container.env, deployEnvEntry("HWLAB_OPENFGA_AUTHN_TOKEN", `secretRef:hwlab-${profile}-openfga/authn-preshared-key`, namespace, profile)); - upsertEnv(container.env, "UNIDESK_MAIN_SERVER_IP", "http://74.48.78.17:18081"); - } else { - syncCodeAgentWorkspaceInitContainer(podTemplate.spec.initContainers, { image, runtimeCommit, runtimeImageTag }); - } - upsertEnv(container.env, "HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE", runtimeLane ? (configString(runtimeLaneConfig(deploy, profile)?.opencode?.providerProfile) || "dsflash-go") : "deepseek"); - if (runtimeLane) upsertEnv(container.env, "HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR", "repo-owned"); - upsertEnv(container.env, "HWLAB_CODE_AGENT_DEEPSEEK_MODEL", deepSeekProfileModel); - upsertEnv(container.env, "HWLAB_CODE_AGENT_DEEPSEEK_BASE_URL", deepSeekBaseUrl(namespace)); - upsertEnv(container.env, "HWLAB_CODE_AGENT_CODEX_API_MODEL", codexApiProfileModel); - upsertEnv(container.env, "HWLAB_CODE_AGENT_CODEX_API_BASE_URL", codexApiProfileBaseUrl); - upsertEnv(container.env, "HWLAB_CODE_AGENT_CODEX_API_UPSTREAM_BASE_URL", codexApiForwarderUpstreamBaseUrl); - upsertEnv(container.env, "HWLAB_CODE_AGENT_CODEX_API_FORWARDER_PORT", codexApiForwarderPort); - upsertEnv(container.env, "HWLAB_PREINSTALLED_SKILLS_DIR", "/app/skills"); - upsertEnv(container.env, "HWLAB_USER_SKILLS_DIR", "/data/user-skills"); - upsertEnv(container.env, "HWLAB_CODE_AGENT_SKILLS_DIRS", "/app/skills:/data/user-skills"); - upsertEnv(container.env, "NO_PROXY", codeAgentNoProxy(namespace)); - upsertEnv(container.env, "no_proxy", codeAgentNoProxy(namespace)); - } - if (serviceId === "hwlab-cloud-web") { - upsertEnv(container.env, "HWLAB_API_BASE_URL", `http://hwlab-cloud-api.${namespace}.svc.cluster.local:6667`); - upsertEnv(container.env, "HWLAB_PUBLIC_ENDPOINT", webEndpoint); - const traceTimelinePolicy = runtimeWorkbenchTraceTimelinePolicy(deploy, profile); - if (typeof traceTimelinePolicy?.autoExpandRunning === "boolean") { - upsertEnv(container.env, "HWLAB_WORKBENCH_TRACE_AUTO_EXPAND_RUNNING", booleanEnv(traceTimelinePolicy.autoExpandRunning)); - } - if (typeof traceTimelinePolicy?.autoCollapseTerminal === "boolean") { - upsertEnv(container.env, "HWLAB_WORKBENCH_TRACE_AUTO_COLLAPSE_TERMINAL", booleanEnv(traceTimelinePolicy.autoCollapseTerminal)); - } - const traceExplorerUrlTemplate = runtimeTraceExplorerUrlTemplate(deploy, profile); - if (traceExplorerUrlTemplate) { - upsertEnv(container.env, "HWLAB_WORKBENCH_TRACE_EXPLORER_URL_TEMPLATE", traceExplorerUrlTemplate); - } - } - } - rewritePodSecretRefs(podTemplate.spec, profile); - } - return result; -} - -function transformHealthContract(value, namespace, labels, annotations, runtimeEndpoint, webEndpoint, profile = "dev") { - const result = transformListNamespace(value, namespace, labels, annotations); - if (result.metadata?.name === "hwlab-dev-health-contract") { - result.metadata.name = `hwlab-${profile}-health-contract`; - } - if (result.metadata?.labels && Object.hasOwn(result.metadata.labels, "app.kubernetes.io/name")) { - result.metadata.labels["app.kubernetes.io/name"] = `hwlab-${profile}-health-contract`; - } - if (result.data && typeof result.data === "object") { - if (Object.hasOwn(result.data, "endpoint")) result.data.endpoint = runtimeEndpoint; - if (Object.hasOwn(result.data, "cloud-web")) { - result.data["cloud-web"] = `GET /health/live on ${webEndpoint}; consumes cloud-api only`; - } - if (Object.hasOwn(result.data, "cloud-api")) { - result.data["cloud-api"] = `GET /health/live through ${runtimeEndpoint}`; - } - if (Object.hasOwn(result.data, "tunnel-client")) { - result.data["tunnel-client"] = "disabled for node GitOps; no FRP or production traffic tunnel is required"; - } - } - return result; -} - -function transformListNamespace(value, namespace, labels, annotations) { - const result = cloneJson(value); - if (result.kind === "List") { - for (const item of result.items ?? []) { - item.metadata ??= {}; - item.metadata.namespace = namespace; - label(item.metadata, labels); - annotate(item.metadata, annotations); - } - } else { - result.metadata ??= {}; - result.metadata.namespace = namespace; - label(result.metadata, labels); - annotate(result.metadata, annotations); - } - return result; -} - -function transformServices({ services, namespace, labels, annotations, profile = "dev", deploy = null }) { - const observableServiceIds = isRuntimeLane(profile) ? runtimeLaneObservableServiceIdsForDeploy(deploy, profile) : new Set(); - const result = transformListNamespace(services, namespace, labels, annotations); - if (result.kind === "List") { - result.items = (result.items ?? []).filter((item) => !(isRuntimeLane(profile) && isV02RemovedServiceId(serviceIdForWorkload(item, null)))); - for (const item of result.items) { - const serviceId = serviceIdForWorkload(item, null); - if (!isRuntimeLane(profile) || !observableServiceIds.has(serviceId)) continue; - label(item.metadata ??= {}, v02MetricsLabels(serviceId)); - upsertV02MetricsPort(item); - } - } - return result; -} - -function observabilityManifest({ deploy, profile, namespace, labels, annotations, metricsSidecarScript }) { - assert.ok(isRuntimeLane(profile), `observability profile must be a runtime lane, got ${profile}`); - const includePrometheusOperatorResources = runtimePrometheusOperatorResourcesEnabled(deploy, profile); - const baseLabels = { - ...labels, - "app.kubernetes.io/part-of": "hwlab", - "hwlab.pikastech.local/monitoring": "enabled" - }; - const serviceMonitors = runtimeLaneObservableServicesForDeploy(deploy, profile).map((service) => ({ - apiVersion: "monitoring.coreos.com/v1", - kind: "ServiceMonitor", - metadata: { - name: `hwlab-${profile}-${service.serviceId}`, - namespace, - labels: { ...baseLabels, "hwlab.pikastech.local/service-id": service.serviceId }, - annotations - }, - spec: { - namespaceSelector: { matchNames: [namespace] }, - selector: { - matchLabels: { - "hwlab.pikastech.local/gitops-target": profile, - "hwlab.pikastech.local/monitoring": "enabled", - "hwlab.pikastech.local/service-id": service.serviceId - } - }, - targetLabels: [ - "hwlab.pikastech.local/gitops-target", - "hwlab.pikastech.local/service-id" - ], - endpoints: [{ port: "metrics", path: "/metrics", interval: "30s", scrapeTimeout: "10s" }] - } - })); - return { - apiVersion: "v1", - kind: "List", - items: [ - { - apiVersion: "v1", - kind: "ConfigMap", - metadata: { name: `hwlab-${profile}-metrics-sidecar`, namespace, labels: baseLabels, annotations }, - data: { "metrics-sidecar.mjs": metricsSidecarScript } - }, - ...(includePrometheusOperatorResources ? serviceMonitors : []), - ...(includePrometheusOperatorResources ? [{ - apiVersion: "monitoring.coreos.com/v1", - kind: "PrometheusRule", - metadata: { name: `hwlab-${profile}-observability`, namespace, labels: baseLabels, annotations }, - spec: { - groups: [{ - name: `hwlab-${profile}-observability`, - rules: [ - { - alert: `HWLAB${profile.toUpperCase()}MetricsTargetDown`, - expr: `up{namespace="${namespace}"} == 0`, - for: "5m", - labels: { severity: "warning", lane: profile }, - annotations: { summary: `HWLAB ${versionNameForRuntimeLane(profile)} metrics target is down`, description: `Prometheus cannot scrape a HWLAB ${versionNameForRuntimeLane(profile)} metrics target.` } - }, - { - alert: `HWLAB${profile.toUpperCase()}ServiceHealthProbeFailed`, - expr: `hwlab_service_health_probe_success{namespace="${namespace}"} == 0`, - for: "5m", - labels: { severity: "warning", lane: profile }, - annotations: { summary: "HWLAB v0.2 service health probe failed", description: "The metrics sidecar cannot reach the service health endpoint inside the pod network." } - } - ] - }] - } - }] : []) - ] - }; -} - -function shellSingleQuote(value) { - return `'${String(value).replace(/'/g, `'"'"'`)}'`; -} - -function ciTimingShellFunction() { - return `ci_now_ms() { - node -e 'console.log(Date.now())' -} - -ci_timing_emit() { - stage="$1" - status="$2" - started_ms="$3" - finished_ms="$(ci_now_ms)" - duration_ms="$((finished_ms - started_ms))" - service_id="\${HWLAB_TIMING_SERVICE_ID:-}" - node -e 'const [stage,status,durationMs,serviceId]=process.argv.slice(1); const payload={event:"node-cicd-timing",schemaVersion:"v1",stage,status,durationMs:Number(durationMs),pipelineRun:process.env.HWLAB_TEKTON_PIPELINERUN||null,taskRun:process.env.HWLAB_TEKTON_TASKRUN||null,task:process.env.HWLAB_TEKTON_TASK||null,revision:process.env.HWLAB_SOURCE_REVISION||null,serviceId:serviceId||null,source:"scripts/gitops-render.mjs",at:new Date().toISOString()}; console.log(JSON.stringify(payload));' "$stage" "$status" "$duration_ms" "$service_id" -} -`; -} - -function dependencyProxyProbeScript(phase, targetUrl) { - return `HWLAB_PROXY_PROBE_PHASE=${shellSingleQuote(phase)} HWLAB_PROXY_PROBE_URL=${shellSingleQuote(targetUrl)} node <<'NODE' || echo '{"event":"dependency-proxy-probe-nonblocking","phase":"${phase}","ok":false,"reason":"probe-failed-but-continuing"}' -const net = require("node:net"); - -const phase = process.env.HWLAB_PROXY_PROBE_PHASE || "unknown"; -const target = new URL(process.env.HWLAB_PROXY_PROBE_URL || "http://deb.debian.org/debian/dists/bookworm/InRelease"); -const proxyRaw = process.env.HTTP_PROXY || process.env.http_proxy || ""; - -function redactProxy(value) { - if (!value) return ""; - try { - const parsed = new URL(value); - if (parsed.username || parsed.password) { - parsed.username = "***"; - parsed.password = ""; - } - return parsed.toString(); - } catch { - return ""; - } -} - -function emit(payload, exitCode = 0) { - console.log(JSON.stringify({ - event: "dependency-proxy-probe", - phase, - target: target.href, - proxy: redactProxy(proxyRaw), - ...payload - })); - if (exitCode) process.exit(exitCode); -} - -if (!proxyRaw) emit({ ok: false, reason: "proxy-env-missing" }, 21); - -let proxy; -try { - proxy = new URL(proxyRaw); -} catch (error) { - emit({ ok: false, reason: "proxy-url-invalid", error: error.message }, 22); -} - -if (proxy.protocol !== "http:" && proxy.protocol !== "https:") { - emit({ ok: false, reason: "http-proxy-required", protocol: proxy.protocol }, 23); -} - -const startedAt = Date.now(); -const socket = net.createConnection({ host: proxy.hostname, port: Number(proxy.port || (proxy.protocol === "https:" ? 443 : 80)) }); -let bytes = 0; -let firstByteMs = null; -let responseHead = ""; -let finished = false; - -const timeout = setTimeout(() => { - if (finished) return; - finished = true; - socket.destroy(); - emit({ ok: false, reason: "timeout", timeoutMs: 15000 }, 24); -}, 15000); - -socket.on("connect", () => { - socket.write("GET " + target.href + " HTTP/1.1\\r\\nHost: " + target.host + "\\r\\nUser-Agent: hwlab-node-proxy-probe\\r\\nConnection: close\\r\\n\\r\\n"); -}); - -socket.on("data", (chunk) => { - if (firstByteMs === null) firstByteMs = Date.now() - startedAt; - bytes += chunk.length; - if (responseHead.length < 240) responseHead += chunk.toString("utf8", 0, Math.min(chunk.length, 240 - responseHead.length)); -}); - -socket.on("end", () => { - if (finished) return; - finished = true; - clearTimeout(timeout); - const totalMs = Date.now() - startedAt; - const statusLine = responseHead.split("\\r\\n", 1)[0] || ""; - const statusCode = Number(statusLine.split(" ")[1] || 0); - const ok = statusLine.startsWith("HTTP/") && statusCode >= 200 && statusCode < 400; - emit({ - ok, - reason: ok ? null : "bad-http-status", - statusLine, - statusCode, - firstByteMs, - totalMs, - bytes, - speedBytesPerSecond: totalMs > 0 ? Math.round(bytes * 1000 / totalMs) : 0 - }, ok ? 0 : 25); -}); - -socket.on("error", (error) => { - if (finished) return; - finished = true; - clearTimeout(timeout); - emit({ ok: false, reason: "proxy-connect-failed", error: error.message, totalMs: Date.now() - startedAt }, 26); -}); -NODE`; -} - -function curlDownloadProbeFunction() { - return `redact_proxy_value() { - printf '%s' "\${1:-}" | sed -E 's#(https?://)[^/@]+@#\\1***@#; s#(socks5h?://)[^/@]+@#\\1***@#' -} - -curl_dependency_probe() { - phase="$1" - url="$2" - proxy="\${HTTPS_PROXY:-\${https_proxy:-\${HTTP_PROXY:-\${http_proxy:-}}}}" - if [ -z "$proxy" ]; then - echo '{"event":"dependency-curl-probe","phase":"'"$phase"'","ok":false,"reason":"proxy-env-missing"}' - return 21 - fi - safe_proxy="$(redact_proxy_value "$proxy")" - echo '{"event":"dependency-curl-probe-start","phase":"'"$phase"'","proxy":"'"$safe_proxy"'","target":"'"$url"'"}' - curl -sS -L --range 0-1048575 -o /dev/null --connect-timeout 15 --max-time 60 --proxy "$proxy" \ - -w '{"event":"dependency-curl-probe","phase":"'"$phase"'","http_code":"%{http_code}","time_namelookup":%{time_namelookup},"time_connect":%{time_connect},"time_starttransfer":%{time_starttransfer},"time_total":%{time_total},"speed_download":%{speed_download},"size_download":%{size_download},"remote_ip":"%{remote_ip}"}\n' \ - "$url" -}`; -} - -function gitSshShellFunction() { - return `git_ssh_setup() { - mkdir -p /root/.ssh - cp /workspace/git-ssh/ssh-privatekey /root/.ssh/id_rsa - chmod 600 /root/.ssh/id_rsa - timeout 10 ssh-keyscan github.com >> /root/.ssh/known_hosts 2>/dev/null || true - timeout 10 ssh-keyscan -p 443 ssh.github.com >> /root/.ssh/known_hosts 2>/dev/null || true - write_github_proxy_command - export GIT_SSH_COMMAND="ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ServerAliveInterval=10 -o ServerAliveCountMax=2 -o ProxyCommand='node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808 %h %p'" - git config --global url."ssh://git@ssh.github.com:443/".insteadOf "git@github.com:" - git config --global url."ssh://git@ssh.github.com:443/".insteadOf "ssh://git@github.com/" -} - -write_github_proxy_command() { - cat > /tmp/hwlab-github-proxy-connect.mjs <<'NODE_PROXY' -#!/usr/bin/env node -import net from "node:net"; - -const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2); -const proxyPort = Number.parseInt(proxyPortRaw || "", 10); -const targetPort = Number.parseInt(targetPortRaw || "", 10); -if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) { - console.error("usage: hwlab-github-proxy-connect "); - process.exit(64); -} - -const socket = net.createConnection({ host: proxyHost, port: proxyPort }); -let buffer = Buffer.alloc(0); - -socket.setTimeout(10000, () => { - console.error("proxy connect timeout " + proxyHost + ":" + proxyPort + " -> " + targetHost + ":" + targetPort); - socket.destroy(); - process.exit(65); -}); - -socket.on("connect", () => { - socket.write("CONNECT " + targetHost + ":" + targetPort + " HTTP/1.1\\r\\nHost: " + targetHost + ":" + targetPort + "\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n"); -}); - -socket.on("error", (error) => { - console.error("proxy connect failed: " + error.message); - process.exit(66); -}); - -function onData(chunk) { - buffer = Buffer.concat([buffer, chunk]); - const headerEnd = buffer.indexOf("\\r\\n\\r\\n"); - if (headerEnd === -1 && buffer.length < 8192) return; - const head = buffer.slice(0, headerEnd + 4).toString("latin1"); - const statusLine = head.split("\\r\\n", 1)[0] || ""; - const statusCode = Number.parseInt(statusLine.split(" ")[1] || "", 10); - if (!statusLine.startsWith("HTTP/1.") || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) { - console.error("proxy CONNECT rejected: " + (statusLine || "missing-status")); - socket.destroy(); - process.exit(67); - } - socket.off("data", onData); - socket.setTimeout(0); - const rest = buffer.slice(headerEnd + 4); - if (rest.length) process.stdout.write(rest); - process.stdin.pipe(socket); - socket.pipe(process.stdout); -} - -socket.on("data", onData); -socket.on("close", () => process.exit(0)); -NODE_PROXY - chmod 0700 /tmp/hwlab-github-proxy-connect.mjs -} - -git_now_ms() { - node -e 'console.log(Date.now())' 2>/dev/null || date +%s000 -} - -git_timed() { - phase="$1" - timeout_seconds="$2" - shift 2 - started_ms="$(git_now_ms)" - echo '{"event":"git-operation","phase":"'"$phase"'","status":"started","timeoutSeconds":'"$timeout_seconds"'}' >&2 - set +e - timeout "$timeout_seconds" "$@" - status="$?" - set -e - finished_ms="$(git_now_ms)" - duration_ms="$((finished_ms - started_ms))" - if [ "$status" -eq 0 ]; then - echo '{"event":"git-operation","phase":"'"$phase"'","status":"succeeded","durationMs":'"$duration_ms"'}' >&2 - return 0 - fi - if [ "$status" -eq 124 ] || [ "$status" -eq 137 ]; then - echo '{"event":"git-operation","phase":"'"$phase"'","status":"failed","reason":"timeout","durationMs":'"$duration_ms"',"timeoutSeconds":'"$timeout_seconds"'}' >&2 - else - echo '{"event":"git-operation","phase":"'"$phase"'","status":"failed","exitCode":'"$status"',"durationMs":'"$duration_ms"'}' >&2 - fi - return "$status" -} -`; -} - -function prepareSourceScript() { - const lines = [ - "#!/bin/sh", - "set -eu", - ciTimingShellFunction(), - "export HWLAB_TEKTON_PIPELINERUN=\"$(context.pipelineRun.name)\"", - "export HWLAB_TEKTON_TASKRUN=\"$(context.taskRun.name)\"", - "export HWLAB_TEKTON_TASK=\"prepare-source\"", - "export HWLAB_SOURCE_REVISION=\"$(params.revision)\"", - `echo '{"event":"ci-base-image","phase":"prepare-source","image":"${ciToolsRunnerImage}","policy":"no-runtime-apk"}'`, - "for tool in node git timeout; do command -v \"$tool\" >/dev/null 2>&1 || { echo '{\"event\":\"ci-base-image\",\"ok\":false,\"reason\":\"missing-tool\",\"tool\":\"'\"$tool\"'\"}'; exit 31; }; done", - "node -e 'console.log(JSON.stringify({event:\"ci-base-image\",phase:\"prepare-source\",ok:true,node:process.version}))'", - gitSshShellFunction(), - "git_url_requires_ssh() { case \"$1\" in git@*|ssh://*) return 0 ;; *) return 1 ;; esac; }", - "git_read_url=\"$(params.git-read-url)\"", - "if git_url_requires_ssh \"$git_read_url\"; then", - " for tool in ssh ssh-keyscan; do command -v \"$tool\" >/dev/null 2>&1 || { echo '{\"event\":\"ci-base-image\",\"ok\":false,\"reason\":\"missing-tool\",\"tool\":\"'\"$tool\"'\"}'; exit 31; }; done", - " git_ssh_setup", - "else", - " echo '{\"event\":\"git-ssh-setup\",\"phase\":\"prepare-source\",\"status\":\"skipped\",\"reason\":\"non-ssh-git-read-url\"}'", - "fi", - "rm -rf /workspace/source/repo", - "source_clone_started_ms=\"$(ci_now_ms)\"", - "git_timed source-clone 180 git clone --branch \"$(params.source-branch)\" \"$git_read_url\" /workspace/source/repo", - "cd /workspace/source/repo", - "git config --global --add safe.directory /workspace/source/repo", - "git remote set-url origin \"$git_read_url\"", - "git checkout \"$(params.revision)\"", - "test \"$(git rev-parse HEAD)\" = \"$(params.revision)\"", - "git merge-base --is-ancestor \"$(params.revision)\" \"origin/$(params.source-branch)\" || { echo '{\"event\":\"source-ancestry\",\"status\":\"failed\",\"revision\":\"'\"$(params.revision)\"'\",\"sourceBranch\":\"'\"$(params.source-branch)\"'\"}'; exit 32; }", - "ci_timing_emit source-clone succeeded \"$source_clone_started_ms\"", - "prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"", - "echo '{\"event\":\"prepare-source-dependencies\",\"status\":\"skipped\",\"reason\":\"renderer-dependency-install-disabled\",\"dependency\":\"yaml\"}'", - "ci_timing_emit prepare-source-dependencies succeeded \"$prepare_source_dependencies_started_ms\"", - "catalog_path=\"$(params.catalog-path)\"", - "mkdir -p \"$(dirname \"$catalog_path\")\"", - "catalog_fetch_started_ms=\"$(ci_now_ms)\"", - "if git_timed catalog-ls-remote 45 git ls-remote --exit-code --heads \"$git_read_url\" \"$(params.gitops-branch)\" >/dev/null; then", - " git remote set-url gitops-catalog \"$git_read_url\" 2>/dev/null || git remote add gitops-catalog \"$git_read_url\"", - " git_timed catalog-fetch 90 git fetch --depth 1 gitops-catalog \"$(params.gitops-branch)\" >/dev/null || true", - " if git cat-file -e FETCH_HEAD:\"$catalog_path\" 2>/dev/null; then", - " git show FETCH_HEAD:\"$catalog_path\" > /tmp/hwlab-gitops-artifact-catalog.json", - " if node --input-type=module - /tmp/hwlab-gitops-artifact-catalog.json deploy/deploy.yaml \"$(params.services)\" <<'NODE'", - "import fs from \"node:fs\";", - "const [catalogPath, deployPath, selectedServices] = process.argv.slice(2);", - "const ids = (doc) => (doc.services || []).map((service) => service.serviceId).filter(Boolean);", - "const topLevelServiceIds = (text) => {", - " const values = [];", - " let inServices = false;", - " for (const line of text.split(/\\r?\\n/u)) {", - " if (/^services:\\s*$/u.test(line)) { inServices = true; continue; }", - " if (!inServices) continue;", - " if (/^\\S/u.test(line)) break;", - " const match = line.match(/^\\s*-\\s+serviceId:\\s*['\"]?([^'\"#\\s]+)['\"]?/u);", - " if (match) values.push(match[1]);", - " }", - " return values;", - "};", - "const selected = (selectedServices || \"\").split(\",\").map((item) => item.trim()).filter(Boolean);", - "const uniqueSorted = (items) => [...new Set(items)].sort();", - "const gitops = JSON.parse(fs.readFileSync(catalogPath, \"utf8\"));", - "const expected = selected.length ? selected : topLevelServiceIds(fs.readFileSync(deployPath, \"utf8\"));", - "const actual = ids(gitops);", - "const expectedSet = uniqueSorted(expected);", - "const actualSet = uniqueSorted(actual);", - "const sameServices = expectedSet.length === actualSet.length && expectedSet.every((item, index) => item === actualSet[index]);", - "if (!sameServices) {", - " const missing = expectedSet.filter((item) => !actualSet.includes(item));", - " const extra = actualSet.filter((item) => !expectedSet.includes(item));", - " console.error(JSON.stringify({ event: \"gitops-artifact-catalog\", phase: \"prepare-source\", status: \"ignored-stale\", reason: \"service-ids-mismatch\", expected, actual, missing, extra }));", - " process.exit(42);", - "}", - "NODE", - " then", - " cp /tmp/hwlab-gitops-artifact-catalog.json \"$catalog_path\"", - " echo '{\"event\":\"gitops-artifact-catalog\",\"phase\":\"prepare-source\",\"status\":\"loaded\",\"branch\":\"'\"$(params.gitops-branch)\"'\"}'", - " else", - " echo '{\"event\":\"gitops-artifact-catalog\",\"phase\":\"prepare-source\",\"status\":\"ignored-stale\",\"reason\":\"service-ids-mismatch\",\"branch\":\"'\"$(params.gitops-branch)\"'\"}'", - " fi", - " else", - " echo '{\"event\":\"gitops-artifact-catalog\",\"phase\":\"prepare-source\",\"status\":\"seed\",\"reason\":\"missing-on-gitops-branch\"}'", - " fi", - "else", - " echo '{\"event\":\"gitops-artifact-catalog\",\"phase\":\"prepare-source\",\"status\":\"seed\",\"reason\":\"gitops-branch-missing\"}'", - "fi", - "if [ ! -s \"$catalog_path\" ] && [ \"$(params.lane)\" = \"v02\" ]; then", - " node --input-type=module - \"$catalog_path\" \"$(params.revision)\" \"$(params.registry-prefix)\" \"$(params.image-tag-mode)\" \"$(params.services)\" <<'NODE'", - "import fs from 'node:fs';", - "import path from 'node:path';", - "const [catalogPath, revision, registryPrefix, imageTagMode, selectedServices] = process.argv.slice(2);", - "const topLevelServiceIds = (text) => {", - " const values = [];", - " let inServices = false;", - " for (const line of text.split(/\\r?\\n/u)) {", - " if (/^services:\\s*$/u.test(line)) { inServices = true; continue; }", - " if (!inServices) continue;", - " if (/^\\S/u.test(line)) break;", - " const match = line.match(/^\\s*-\\s+serviceId:\\s*['\"]?([^'\"#\\s]+)['\"]?/u);", - " if (match) values.push(match[1]);", - " }", - " return values;", - "};", - "const tag = imageTagMode === 'full' ? revision : revision.slice(0, 7);", - "const namespace = 'hwlab-v02';", - "const selected = (selectedServices || '').split(',').filter(Boolean);", - "const serviceIds = selected.length ? selected : topLevelServiceIds(fs.readFileSync('deploy/deploy.yaml', 'utf8'));", - "const services = serviceIds.map((serviceId) => {", - " const service = { serviceId, profile: 'v02', namespace, commitId: tag, sourceCommitId: revision, image: `${registryPrefix}/${serviceId}:${tag}`, imageTag: tag, digest: null, buildBackend: 'contract-skeleton' };", - " return service;", - "});", - "fs.mkdirSync(path.dirname(catalogPath), { recursive: true });", - `fs.writeFileSync(catalogPath, JSON.stringify({ catalogVersion: 'v1', kind: 'hwlab-artifact-catalog', environment: 'v02', profile: 'v02', namespace, endpoint: ${JSON.stringify(defaultV02RuntimeEndpoint)}, commitId: tag, artifactState: 'contract-skeleton', publish: { registryPrefix, sourceCommitId: revision, imageTag: tag, publishedAt: null }, allowedProfiles: ['v02'], forbiddenProfiles: ['dev', 'prod'], services }, null, 2) + '\\n');`, - "NODE", - " echo '{\"event\":\"gitops-artifact-catalog\",\"phase\":\"prepare-source\",\"status\":\"seeded-v02-skeleton\"}'", - "fi", - "ci_timing_emit catalog-fetch succeeded \"$catalog_fetch_started_ms\"", - `echo ${shellSingleQuote(`node prepare-source complete; validation task count=${primitiveValidationTasks.length}`)}` - ]; - return `${lines.join("\n")}\n`; -} - -function sourceValidationScript(commands) { - return [ - "#!/bin/sh", - "set -eu", - "git config --global --add safe.directory /workspace/source/repo", - "cd /workspace/source/repo", - "test \"$(git rev-parse HEAD)\" = \"$(params.revision)\"", - ...commands - ].join("\n") + "\n"; -} - -function primitiveValidationTaskNames() { - return primitiveValidationTasks.map((task) => task.name); -} - -function primitiveValidationTask(task) { - return { - name: task.name, - runAfter: ["prepare-source"], - workspaces: [{ name: "source", workspace: "source" }], - taskSpec: { - params: [ - { name: "revision" }, - { name: "lane" }, - { name: "catalog-path" }, - { name: "image-tag-mode" }, - { name: "source-branch" }, - { name: "gitops-branch" } - ], - workspaces: [{ name: "source" }], - steps: [{ name: task.name, image: ciToolsRunnerImage, env: proxyEnv(), script: sourceValidationScript(task.commands) }] - }, - params: [ - { name: "revision", value: "$(params.revision)" }, - { name: "lane", value: "$(params.lane)" }, - { name: "catalog-path", value: "$(params.catalog-path)" }, - { name: "image-tag-mode", value: "$(params.image-tag-mode)" }, - { name: "source-branch", value: "$(params.source-branch)" }, - { name: "gitops-branch", value: "$(params.gitops-branch)" } - ] - }; -} - -function planArtifactsScript() { - return `#!/bin/sh -set -eu -echo '{"event":"ci-base-image","phase":"plan-artifacts","image":"${ciToolsRunnerImage}","policy":"no-runtime-apk"}' -for tool in node git; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"plan-artifacts","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done -cd /workspace/source/repo -ci_node_deps="\${HWLAB_CI_NODE_DEPS:-/opt/hwlab-ci-node-deps/node_modules}" -if [ -d "$ci_node_deps/yaml" ] && [ ! -e node_modules/yaml ]; then - mkdir -p node_modules - ln -s "$ci_node_deps/yaml" node_modules/yaml -fi -test "$(git rev-parse HEAD)" = "$(params.revision)" -HWLAB_CATALOG_PATH="$(params.catalog-path)" \ -HWLAB_GIT_URL="$(params.git-url)" \ -HWLAB_GIT_READ_URL="$(params.git-read-url)" \ -HWLAB_GITOPS_BRANCH="$(params.gitops-branch)" \ -node scripts/ci/restore-artifact-catalog.mjs -node scripts/ci-plan.mjs --lane "$(params.lane)" --target-ref HEAD --deploy-config deploy/deploy.yaml --artifact-catalog "$(params.catalog-path)" --registry-prefix "$(params.registry-prefix)" --services "$(params.services)" --verify-reuse-registry > /workspace/source/ci-plan.json -node - <<'NODE' -const fs = require("node:fs"); -const plan = JSON.parse(fs.readFileSync("/workspace/source/ci-plan.json", "utf8")); -const selected = String(process.env.HWLAB_SELECTED_SERVICES || "").split(",").map((item) => item.trim()).filter(Boolean); -const allServices = selected.length > 0 ? selected : ${JSON.stringify(defaultServiceIds)}; -const affected = new Set(plan.affectedServices || []); -const plannedBuildServices = Array.isArray(plan.buildServices) ? new Set(plan.buildServices) : null; -const selectedSet = new Set(selected); -const byService = new Map((plan.services || []).map((service) => [service.serviceId, service])); -const entries = allServices.map((serviceId) => { - const service = byService.get(serviceId) || {}; - const serviceSelected = selectedSet.has(serviceId); - const rolloutAffected = serviceSelected && affected.has(serviceId); - const envReuse = service.runtimeMode === "env-reuse-git-mirror-checkout" || service.envReuse === true; - const buildRequired = serviceSelected && (plannedBuildServices ? plannedBuildServices.has(serviceId) : (envReuse ? service.envChanged === true : rolloutAffected)); - return { - serviceId, - selected: serviceSelected, - affected: rolloutAffected, - buildRequired, - rolloutAffected, - runtimeMode: service.runtimeMode || "service-image", - envChanged: service.envChanged ?? null, - codeChanged: service.codeChanged ?? null - }; -}); -for (const entry of entries) { - fs.writeFileSync("/tekton/results/affected-" + entry.serviceId, entry.affected ? "true" : "false"); - fs.writeFileSync("/tekton/results/build-" + entry.serviceId, entry.buildRequired ? "true" : "false"); -} -fs.writeFileSync("/workspace/source/affected-services.json", JSON.stringify({ - sourceCommitId: plan.sourceCommitId, - affectedServices: plan.affectedServices || [], - rolloutServices: plan.rolloutServices || plan.affectedServices || [], - buildServices: plan.buildServices || entries.filter((entry) => entry.buildRequired).map((entry) => entry.serviceId), - reusedServices: plan.reusedServices || [], - buildSkippedCount: plan.buildSkippedCount || 0, - envArtifactGroups: plan.envArtifactGroups || [], - changedPathSummary: plan.changedPathSummary || null, - ciCdPlan: plan.ciCdPlan || null, - services: plan.services || [], - entries -}, null, 2) + String.fromCharCode(10)); -console.log(JSON.stringify({ event: "g14-ci-plan", sourceCommitId: plan.sourceCommitId, affectedServices: plan.affectedServices || [], rolloutServices: plan.rolloutServices || plan.affectedServices || [], buildServices: plan.buildServices || [], reusedServices: plan.reusedServices || [], buildSkippedCount: plan.buildSkippedCount || 0, envArtifactGroups: plan.envArtifactGroups || [] })); -NODE -`; -} - -function perServicePublishScript() { - return `#!/bin/sh -set -eu -${ciTimingShellFunction()} -export HWLAB_TEKTON_PIPELINERUN="$(context.pipelineRun.name)" -export HWLAB_TEKTON_TASKRUN="$(context.taskRun.name)" -export HWLAB_TEKTON_TASK="build-$(params.service-id)" -export HWLAB_SOURCE_REVISION="$(params.revision)" -export HWLAB_TIMING_SERVICE_ID="$(params.service-id)" -${dependencyProxyProbeScript("service-image-publish-proxy-preflight", "http://deb.debian.org/debian/dists/bookworm/InRelease")} -echo '{"event":"ci-base-image","phase":"service-image-publish","serviceId":"'"$(params.service-id)"'","image":"${ciToolsRunnerImage}","policy":"no-runtime-apk"}' -for tool in node npm git python3 ssh curl; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"service-image-publish","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done -mkdir -p /workspace/service-work/home -export HOME=/workspace/service-work/home -git config --global --add safe.directory /workspace/source/repo -cd /workspace/source/repo -test "$(git rev-parse HEAD)" = "$(params.revision)" -export HWLAB_ARTIFACT_LANE="$(params.lane)" -export HWLAB_ARTIFACT_CATALOG_PATH="$(params.catalog-path)" -export HWLAB_DEPLOY_MANIFEST_PATH="deploy/deploy.yaml" -export HWLAB_ARTIFACT_IMAGE_TAG_MODE="$(params.image-tag-mode)" -mkdir -p /workspace/source/service-results -if [ -s /workspace/source/affected-services.json ] && ! node -e 'const fs=require("node:fs"); const p=JSON.parse(fs.readFileSync("/workspace/source/affected-services.json","utf8")); const service=process.argv[1]; const item=(p.entries||[]).find((entry)=>entry.serviceId===service); process.exit(item && item.buildRequired ? 0 : 1);' "$(params.service-id)"; then - node - "$(params.service-id)" <<'NODE' -const fs = require("node:fs"); -const serviceId = process.argv[2]; -const catalog = JSON.parse(fs.readFileSync(process.env.HWLAB_ARTIFACT_CATALOG_PATH, "utf8")); -const plan = fs.existsSync("/workspace/source/affected-services.json") ? JSON.parse(fs.readFileSync("/workspace/source/affected-services.json", "utf8")) : {}; -const service = (catalog.services || []).find((item) => item.serviceId === serviceId) || {}; -const planned = (plan.services || []).find((item) => item.serviceId === serviceId) || {}; -const envReuse = planned.runtimeMode === "env-reuse-git-mirror-checkout" || service.runtimeMode === "env-reuse-git-mirror-checkout" || planned.envReuse === true || service.envReuse === true; -const image = envReuse ? (service.environmentImage || service.image || "") : (service.image || ""); -const digest = envReuse ? (service.environmentDigest || service.digest || "not_published") : (service.digest || "not_published"); -const imageTag = service.imageTag || (image.includes(":") ? image.slice(image.lastIndexOf(":") + 1) : ""); -const repository = image.includes(":") ? image.slice(0, image.lastIndexOf(":")) : image; -const revision = process.env.HWLAB_SOURCE_REVISION || planned.bootCommit || service.bootCommit || service.sourceCommitId || service.commitId || imageTag; -const componentInputHash = envReuse ? (planned.componentInputHash || service.componentInputHash || "") : (service.componentInputHash || ""); -const environmentInputHash = envReuse ? (service.environmentInputHash || planned.environmentInputHash || "") : (service.environmentInputHash || ""); -const codeInputHash = envReuse ? (planned.codeInputHash || service.codeInputHash || "") : (service.codeInputHash || ""); -const values = { - "service-id": serviceId, - status: /^sha256:[a-f0-9]{64}$/.test(digest) ? "reused" : "blocked_reuse_unavailable", - image, - "image-tag": imageTag, - digest, - "repository-digest": /^sha256:[a-f0-9]{64}$/.test(digest) && repository ? repository + "@" + digest : "", - "source-commit-id": envReuse ? revision : (service.sourceCommitId || service.commitId || imageTag), - "component-input-hash": componentInputHash, - "environment-input-hash": environmentInputHash, - "code-input-hash": codeInputHash, - "runtime-mode": envReuse ? "env-reuse-git-mirror-checkout" : "service-image", - "boot-repo": planned.bootRepo || service.bootRepo || "", - "boot-commit": envReuse ? revision : (service.bootCommit || ""), - "boot-sh": planned.bootSh || service.bootSh || "", - "build-created-at": service.buildCreatedAt || "", - "build-backend": envReuse ? "reused-env-catalog" : "reused-catalog", - "reused-from": (envReuse ? service.environmentInputHash : service.componentInputHash) || service.commitId || imageTag || "catalog" -}; -for (const [name, value] of Object.entries(values)) fs.writeFileSync("/tekton/results/" + name, String(value || "")); -NODE - echo '{"event":"service-build-skip","serviceId":"'"$(params.service-id)"'","reason":"component-inputs-unchanged"}' - exit 0 -fi -rm -rf /workspace/service-work/repo -mkdir -p /workspace/service-work/repo -tar -C /workspace/source/repo -cf - . | tar -C /workspace/service-work/repo -xo -f - -cd /workspace/service-work/repo -export HWLAB_CI_ARTIFACT_RUN_ID="$(context.pipelineRun.name)-$(params.service-id)" -export HWLAB_CI_ARTIFACT_RUN_OWNER="tekton" -export HWLAB_DEV_REGISTRY_PREFIX="$(params.registry-prefix)" -export HWLAB_DEV_REGISTRY_K8S_NATIVE=1 -export HWLAB_NODE_CICD_TIMING=1 -export HWLAB_TEKTON_PIPELINERUN -export HWLAB_TEKTON_TASKRUN -export HWLAB_TEKTON_TASK -export HWLAB_SOURCE_REVISION -export PATH="/workspace/buildkit-bin:$PATH" -export HWLAB_BUILDKIT_ADDR="unix:///workspace/buildkit-run/buildkitd.sock" -if [ -n "$(params.base-image)" ]; then export HWLAB_DEV_BASE_IMAGE="$(params.base-image)"; fi -if [ -n "\${HWLAB_DEV_BASE_IMAGE:-}" ]; then - echo '{"event":"dependency-local-registry-probe-start","phase":"service-image-publish-pre-base-image","serviceId":"'"$(params.service-id)"'","target":"http://127.0.0.1:5000/v2/"}' - registry_started_at="$(date +%s)" - curl -fsS --max-time 10 http://127.0.0.1:5000/v2/ >/dev/null - registry_finished_at="$(date +%s)" - echo '{"event":"dependency-local-registry-probe","phase":"service-image-publish-pre-base-image","serviceId":"'"$(params.service-id)"'","ok":true,"durationSeconds":'"$((registry_finished_at - registry_started_at))"'}' -fi -buildkit_ready=0 -for attempt in $(seq 1 60); do - if /workspace/buildkit-bin/buildctl --addr "$HWLAB_BUILDKIT_ADDR" debug workers >/dev/null 2>&1; then - buildkit_ready=1 - break - fi - sleep 1 -done -if [ "$buildkit_ready" != "1" ]; then - echo '{"event":"buildkit-sidecar-not-ready","serviceId":"'"$(params.service-id)"'","addr":"'"$HWLAB_BUILDKIT_ADDR"'"}' >&2 - exit 1 -fi -HWLAB_CI_PLAN_VERIFY_REUSE_REGISTRY=1 node scripts/artifact-publish.mjs --publish --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --registry-prefix "$(params.registry-prefix)" --report "/workspace/source/service-results/$(params.service-id).json" --tekton-results-dir /tekton/results --quiet-build --concurrency 1 --services "$(params.service-id)" --buildkit-command /workspace/buildkit-bin/buildctl --buildkit-addr "$HWLAB_BUILDKIT_ADDR" --build-cache-mode "$(params.build-cache-mode)" -`; -} - -function collectArtifactsScript() { - return `#!/bin/sh -set -eu -echo '{"event":"ci-base-image","phase":"collect-artifacts","image":"${ciToolsRunnerImage}","policy":"no-runtime-apk"}' -for tool in node git; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"collect-artifacts","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done -cd /workspace/source/repo -test "$(git rev-parse HEAD)" = "$(params.revision)" -export HWLAB_CI_ARTIFACT_RUN_ID="$(context.pipelineRun.name)-collect" -export HWLAB_CI_ARTIFACT_RUN_OWNER="tekton" -export HWLAB_DEV_REGISTRY_PREFIX="$(params.registry-prefix)" -export HWLAB_DEV_REGISTRY_K8S_NATIVE=1 -if [ -n "$(params.base-image)" ]; then export HWLAB_DEV_BASE_IMAGE="$(params.base-image)"; fi -export HWLAB_ARTIFACT_LANE="$(params.lane)" -export HWLAB_ARTIFACT_CATALOG_PATH="$(params.catalog-path)" -export HWLAB_DEPLOY_MANIFEST_PATH="deploy/deploy.yaml" -export HWLAB_ARTIFACT_IMAGE_TAG_MODE="$(params.image-tag-mode)" -if [ -s /workspace/source/affected-services.json ] && node - /workspace/source/affected-services.json <<'NODE' -const fs = require("node:fs"); -const plan = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); -const buildServices = Array.isArray(plan.buildServices) ? plan.buildServices : []; -const affectedServices = Array.isArray(plan.affectedServices) ? plan.affectedServices : []; -const willRunGitopsPromote = plan.ciCdPlan && plan.ciCdPlan.willRunGitopsPromote === true; -process.exit(!willRunGitopsPromote && buildServices.length === 0 && affectedServices.length === 0 ? 0 : 1); -NODE -then - rm -f /workspace/source/dev-artifacts.json - echo '{"event":"collect-artifacts","status":"skipped","reason":"no-build-no-rollout-plan"}' - exit 0 -fi -HWLAB_CI_PLAN_VERIFY_REUSE_REGISTRY=1 node scripts/artifact-publish.mjs --publish --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --registry-prefix "$(params.registry-prefix)" --report /workspace/source/dev-artifacts.json --quiet-build --concurrency 1 --services "$(params.services)" --external-service-report-dir /workspace/source/service-results --ci-plan-path /workspace/source/affected-services.json -node scripts/refresh-artifact-catalog.mjs --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --target-ref HEAD --publish-report /workspace/source/dev-artifacts.json --no-write -`; -} - -function gitopsPromoteScript() { - return `#!/bin/sh -set -eu -${ciTimingShellFunction()} -export HWLAB_TEKTON_PIPELINERUN="$(context.pipelineRun.name)" -export HWLAB_TEKTON_TASKRUN="$(context.taskRun.name)" -export HWLAB_TEKTON_TASK="gitops-promote" -export HWLAB_SOURCE_REVISION="$(params.revision)" -echo '{"event":"ci-base-image","phase":"gitops-promote","image":"${ciToolsRunnerImage}","policy":"no-runtime-apt"}' -for tool in node git timeout; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"gitops-promote","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done -${gitSshShellFunction()} -git_url_requires_ssh() { case "$1" in git@*|ssh://*) return 0 ;; *) return 1 ;; esac; } -lane="$(params.lane)" -case "$lane" in - v[0-9][0-9]*) runtime_lane=true ;; - *) runtime_lane=false ;; -esac -if [ "$runtime_lane" = "true" ]; then - printf 'false' > /tekton/results/runtime-ready-required -else - printf 'true' > /tekton/results/runtime-ready-required -fi -source_head_url_for_setup="$(params.git-url)" -gitops_read_url_for_setup="$(params.git-url)" -if [ "$runtime_lane" = "true" ]; then - source_head_url_for_setup="$(params.git-read-url)" - gitops_read_url_for_setup="$(params.git-read-url)" -fi -gitops_write_url_for_setup="$(params.git-write-url)" -if git_url_requires_ssh "$source_head_url_for_setup" || git_url_requires_ssh "$gitops_read_url_for_setup" || git_url_requires_ssh "$gitops_write_url_for_setup"; then - for tool in ssh ssh-keyscan; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"gitops-promote","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done - git_ssh_setup -else - echo '{"event":"git-ssh-setup","phase":"gitops-promote","status":"skipped","reason":"non-ssh-git-url"}' -fi -cd /workspace/source/repo -test "$(git rev-parse HEAD)" = "$(params.revision)" - -check_source_head() { - phase="$1" - expected="\${2:-$(params.revision)}" - source_head_url="$(params.git-url)" - if [ "$runtime_lane" = "true" ]; then - source_head_url="$(params.git-read-url)" - fi - latest_file="$(mktemp)" - if ! git_timed "source-head-$phase" 45 git ls-remote "$source_head_url" "refs/heads/$(params.source-branch)" > "$latest_file"; then - echo '{"status":"failed","reason":"source-head-check-failed","phase":"'"$phase"'","sourceBranch":"'"$(params.source-branch)"'","sourceRevision":"'"$(params.revision)"'"}' - exit 1 - fi - latest="$(cut -f1 "$latest_file" | head -n 1)" - if [ -z "$latest" ]; then - echo '{"status":"failed","reason":"source-head-unresolved","phase":"'"$phase"'","sourceBranch":"'"$(params.source-branch)"'","sourceRevision":"'"$(params.revision)"'"}' - exit 1 - fi - if [ "$latest" != "$expected" ]; then - printf 'false' > /tekton/results/runtime-ready-required || true - echo '{"status":"skipped-stale-source","verdict":"superseded","phase":"'"$phase"'","sourceBranch":"'"$(params.source-branch)"'","sourceRevision":"'"$(params.revision)"'","expectedRevision":"'"$expected"'","latestRevision":"'"$latest"'"}' - exit 0 - fi -} - -check_source_head before-render -if [ -s /workspace/source/affected-services.json ] && node - /workspace/source/affected-services.json <<'NODE' -const fs = require("node:fs"); -const plan = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); -const buildServices = Array.isArray(plan.buildServices) ? plan.buildServices : []; -const affectedServices = Array.isArray(plan.affectedServices) ? plan.affectedServices : []; -const willRunGitopsPromote = plan.ciCdPlan && plan.ciCdPlan.willRunGitopsPromote === true; -process.exit(!willRunGitopsPromote && buildServices.length === 0 && affectedServices.length === 0 ? 0 : 1); -NODE -then - printf 'false' > /tekton/results/runtime-ready-required || true - echo '{"event":"gitops-promote","status":"skipped","reason":"no-build-no-rollout-plan"}' - exit 0 -fi -git config --global user.name "HWLAB node GitOps Bot" -git config --global user.email "hwlab-node-gitops-bot@users.noreply.github.com" -catalog_path="$(params.catalog-path)" -runtime_path="$(params.runtime-path)" -gitops_root_from_runtime_path() { - path="$1" - case "$path" in - */*) printf '%s\n' "\${path%/*}" ;; - *) printf '.\n' ;; - esac -} -gitops_root="$(gitops_root_from_runtime_path "$runtime_path")" - -argo_hard_refresh_runtime_lane() { - if [ "$runtime_lane" != "true" ]; then return 0; fi - ARGO_APPLICATION="hwlab-node-$lane" ARGO_FIELD_MANAGER="hwlab-$lane-gitops-promote" node <<'NODE' -const fs = require("node:fs"); -const https = require("node:https"); - -const host = process.env.KUBERNETES_SERVICE_HOST; -const port = process.env.KUBERNETES_SERVICE_PORT || "443"; -const startedAt = Date.now(); -const application = process.env.ARGO_APPLICATION || "hwlab-node-v02"; -const fieldManager = process.env.ARGO_FIELD_MANAGER || "hwlab-v02-gitops-promote"; -const pipelineRun = process.env.HWLAB_TEKTON_PIPELINERUN || null; -const taskRun = process.env.HWLAB_TEKTON_TASKRUN || null; -const task = process.env.HWLAB_TEKTON_TASK || null; -const revision = process.env.HWLAB_SOURCE_REVISION || null; - -function emit(payload) { - console.log(JSON.stringify({ - event: "node-cicd-timing", - schemaVersion: "v1", - stage: "argo-hard-refresh", - pipelineRun, - taskRun, - task, - revision, - application, - source: "scripts/gitops-render.mjs", - at: new Date().toISOString(), - ...payload - })); -} - -function safeJson(text) { - try { - return JSON.parse(text); - } catch { - return null; - } -} - -function request(method, path, body, contentType = "application/merge-patch+json") { - const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8"); - const ca = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); - const payload = body ? JSON.stringify(body) : ""; - const headers = { Authorization: "Bearer " + token }; - if (payload) { - headers["Content-Type"] = contentType; - headers["Content-Length"] = Buffer.byteLength(payload); - } - return new Promise((resolve, reject) => { - const req = https.request({ host, port, method, path, ca, headers }, (res) => { - let data = ""; - res.setEncoding("utf8"); - res.on("data", (chunk) => { data += chunk; }); - res.on("end", () => resolve({ statusCode: res.statusCode || 0, body: data, json: safeJson(data) })); - }); - req.on("error", reject); - if (payload) req.write(payload); - req.end(); - }); -} - -(async () => { - if (!host) { - emit({ status: "skipped", reason: "kubernetes-service-host-missing", durationMs: Date.now() - startedAt }); - return; - } - const response = await request( - "PATCH", - "/apis/argoproj.io/v1alpha1/namespaces/argocd/applications/" + encodeURIComponent(application) + "?fieldManager=" + encodeURIComponent(fieldManager), - { metadata: { annotations: { "argocd.argoproj.io/refresh": "hard" } } } - ); - if (response.statusCode >= 200 && response.statusCode < 300) { - emit({ status: "succeeded", durationMs: Date.now() - startedAt, statusCode: response.statusCode }); - return; - } - emit({ status: "degraded", reason: "argo-refresh-patch-failed", durationMs: Date.now() - startedAt, statusCode: response.statusCode, body: response.body.slice(0, 1000) }); -})().catch((error) => { - emit({ status: "degraded", reason: "argo-refresh-patch-error", durationMs: Date.now() - startedAt, error: error.message }); -}); -NODE -} - -if [ -s /workspace/source/dev-artifacts.json ]; then - node scripts/refresh-artifact-catalog.mjs --lane "$lane" --catalog-path "$catalog_path" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --target-ref "$(params.revision)" --publish-report /workspace/source/dev-artifacts.json --write -fi -gitops_render_started_ms="$(ci_now_ms)" -node scripts/run-bun.mjs scripts/gitops-render.mjs --lane "$lane" --catalog-path "$catalog_path" --image-tag-mode "$(params.image-tag-mode)" --source-revision "$(params.revision)" --source-repo "$(params.git-url)" --source-branch "$(params.source-branch)" --gitops-branch "$(params.gitops-branch)" --gitops-root "$gitops_root" --out "$gitops_root" --registry-prefix "$(params.registry-prefix)" --use-deploy-images -node scripts/run-bun.mjs scripts/gitops-render.mjs --lane "$lane" --catalog-path "$catalog_path" --image-tag-mode "$(params.image-tag-mode)" --source-revision "$(params.revision)" --source-repo "$(params.git-url)" --source-branch "$(params.source-branch)" --gitops-branch "$(params.gitops-branch)" --gitops-root "$gitops_root" --out "$gitops_root" --registry-prefix "$(params.registry-prefix)" --use-deploy-images --check -ci_timing_emit gitops-render succeeded "$gitops_render_started_ms" -check_source_head before-push -workdir="$(mktemp -d)" -gitops_read_url="$(params.git-url)" -gitops_write_url="$(params.git-write-url)" -if [ "$runtime_lane" = "true" ]; then - gitops_read_url="$(params.git-read-url)" -fi -gitops_clone_started_ms="$(ci_now_ms)" -git_timed gitops-clone 180 git clone --no-checkout "$gitops_read_url" "$workdir/gitops" -cd "$workdir/gitops" -git remote set-url origin "$gitops_write_url" -old_runtime_snapshot="$workdir/old-runtime-snapshot" -if git_timed gitops-branch-ls 45 git ls-remote --exit-code --heads origin "$(params.gitops-branch)" >/dev/null; then - git_timed gitops-fetch 90 git fetch origin "$(params.gitops-branch)" - git checkout -B "$(params.gitops-branch)" "origin/$(params.gitops-branch)" - if [ "$runtime_lane" = "true" ] && [ -d "$runtime_path" ]; then - mkdir -p "$old_runtime_snapshot" - cp -a "$runtime_path"/. "$old_runtime_snapshot"/ - fi - if [ "$runtime_lane" = "true" ]; then rm -rf "$runtime_path" "$catalog_path"; else rm -rf deploy/gitops/node "$catalog_path"; fi -else - git checkout --orphan "$(params.gitops-branch)" - git rm -rf . >/dev/null 2>&1 || true -fi -ci_timing_emit gitops-clone succeeded "$gitops_clone_started_ms" -mkdir -p "$(dirname "$runtime_path")" "$(dirname "$catalog_path")" -if [ "$runtime_lane" = "true" ]; then - cp -a "/workspace/source/repo/$runtime_path" "$runtime_path" - cp "/workspace/source/repo/$catalog_path" "$catalog_path" - git add "$catalog_path" "$runtime_path" -else - mkdir -p deploy/gitops - cp -a /workspace/source/repo/deploy/gitops/node deploy/gitops/node - cp "/workspace/source/repo/$catalog_path" "$catalog_path" - git add "$catalog_path" deploy/gitops/node -fi -if [ "$runtime_lane" = "true" ] && [ -s /workspace/source/affected-services.json ]; then - runtime_noop_decision="$(node - "$runtime_path" "$old_runtime_snapshot" /workspace/source/affected-services.json <<'NODE' -const fs = require("node:fs"); -const path = require("node:path"); - -const [runtimePath, oldRuntimePath, planPath] = process.argv.slice(2); - -function readJson(filePath) { - return JSON.parse(fs.readFileSync(filePath, "utf8")); -} - -function stable(value) { - if (Array.isArray(value)) return "[" + value.map(stable).join(",") + "]"; - if (value && typeof value === "object") { - return "{" + Object.keys(value).sort().map((key) => JSON.stringify(key) + ":" + stable(value[key])).join(",") + "}"; - } - return JSON.stringify(value); -} - -function scrub(value) { - if (Array.isArray(value)) return value.map(scrub); - if (!value || typeof value !== "object") return value; - const result = {}; - for (const [key, child] of Object.entries(value)) { - if (key === "hwlab.pikastech.local/source-commit" || - key === "hwlab.pikastech.local/artifact-source-commit" || - key === "hwlab.pikastech.local/boot-commit" || - key === "sourceCommitId" || - key === "bootCommit") { - result[key] = ""; - continue; - } - const envName = String(value.name || ""); - if (key === "value" && /^HWLAB_.*COMMIT/.test(envName)) { - result[key] = ""; - continue; - } - result[key] = scrub(child); - } - return result; -} - -function listFiles(rootDir) { - if (!rootDir || !fs.existsSync(rootDir)) return null; - const files = []; - function walk(current) { - for (const entry of fs.readdirSync(current, { withFileTypes: true })) { - const fullPath = path.join(current, entry.name); - if (entry.isDirectory()) walk(fullPath); - else if (entry.isFile()) files.push(path.relative(rootDir, fullPath).replaceAll(path.sep, "/")); - } - } - walk(rootDir); - return files.sort(); -} - -function normalizedTree(rootDir) { - const files = listFiles(rootDir); - if (!files) return null; - const entries = {}; - for (const relativePath of files) { - const filePath = path.join(rootDir, relativePath); - const text = fs.readFileSync(filePath, "utf8"); - try { - entries[relativePath] = stable(scrub(JSON.parse(text))); - } catch { - entries[relativePath] = text.replace(/[a-f0-9]{40}/gu, ""); - } - } - return stable(entries); -} - -const plan = readJson(planPath); -const buildServices = Array.isArray(plan.buildServices) ? plan.buildServices : []; -const rolloutServices = Array.isArray(plan.rolloutServices) ? plan.rolloutServices : []; -if (buildServices.length > 0 || rolloutServices.length > 0) { - process.stdout.write("runtime-required"); - process.exit(0); -} - -const oldTree = normalizedTree(oldRuntimePath); -const newTree = normalizedTree(runtimePath); -process.stdout.write(oldTree && newTree && oldTree === newTree ? "runtime-identity-only" : "runtime-required"); -NODE -)" - if [ "$runtime_noop_decision" = "runtime-identity-only" ]; then - printf 'false' > /tekton/results/runtime-ready-required - echo '{"status":"skipped-runtime-unchanged","reason":"runtime-identity-only","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}' - ci_timing_emit gitops-commit skipped "$(ci_now_ms)" - ci_timing_emit gitops-push skipped "$(ci_now_ms)" - exit 0 - fi -fi -if git diff --cached --quiet; then - printf 'false' > /tekton/results/runtime-ready-required - echo '{"status":"unchanged","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}' - ci_timing_emit gitops-commit unchanged "$(ci_now_ms)" - ci_timing_emit gitops-push unchanged "$(ci_now_ms)" - exit 0 -fi -short="$(printf '%.7s' "$(params.revision)")" -gitops_commit_started_ms="$(ci_now_ms)" -git commit -m "chore: promote node GitOps source $short" -ci_timing_emit gitops-commit succeeded "$gitops_commit_started_ms" -gitops_push_started_ms="$(ci_now_ms)" -git_timed gitops-push 120 git push origin "HEAD:$(params.gitops-branch)" -ci_timing_emit gitops-push succeeded "$gitops_push_started_ms" -if [ "$runtime_lane" = "true" ]; then - printf 'false' > /tekton/results/runtime-ready-required - echo '{"event":"runtime-ready","phase":"gitops-promote","status":"delegated-post-flush-closeout","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}' -fi -argo_hard_refresh_runtime_lane -echo '{"status":"pushed","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'","gitopsWriteUrl":"'"$gitops_write_url"'"}' -`; -} - -function controlPlaneReconcileScript() { - return `#!/bin/sh -set -eu -${dependencyProxyProbeScript("control-plane-proxy-preflight", "http://deb.debian.org/debian/dists/bookworm/InRelease")} -echo '{"event":"ci-base-image","phase":"control-plane-reconciler","image":"${ciToolsRunnerImage}","policy":"no-runtime-apk"}' -for tool in node npm git python3 ssh curl timeout; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"control-plane-reconciler","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done -${gitSshShellFunction()} -git_ssh_setup -workdir="$(mktemp -d)" -git_timed control-plane-clone 180 git clone --depth 1 --branch "$SOURCE_BRANCH" "$GIT_READ_URL" "$workdir/repo" -cd "$workdir/repo" -git remote set-url origin "$GIT_READ_URL" -revision="$(git rev-parse HEAD)" -: "\${LANE:?LANE is required}" -: "\${CATALOG_PATH:?CATALOG_PATH is required}" -: "\${IMAGE_TAG_MODE:?IMAGE_TAG_MODE is required}" -: "\${RUNTIME_PATH:?RUNTIME_PATH is required}" -gitops_root="\${RUNTIME_PATH%/*}" -if [ "$gitops_root" = "$RUNTIME_PATH" ]; then gitops_root="."; fi -node scripts/run-bun.mjs scripts/gitops-render.mjs --lane "$LANE" --catalog-path "$CATALOG_PATH" --image-tag-mode "$IMAGE_TAG_MODE" --source-revision "$revision" --source-repo "$GIT_URL" --source-branch "$SOURCE_BRANCH" --gitops-branch "$GITOPS_BRANCH" --gitops-root "$gitops_root" --out "$gitops_root" --registry-prefix "$REGISTRY_PREFIX" -node scripts/run-bun.mjs scripts/gitops-render.mjs --lane "$LANE" --catalog-path "$CATALOG_PATH" --image-tag-mode "$IMAGE_TAG_MODE" --source-revision "$revision" --source-repo "$GIT_URL" --source-branch "$SOURCE_BRANCH" --gitops-branch "$GITOPS_BRANCH" --gitops-root "$gitops_root" --out "$gitops_root" --registry-prefix "$REGISTRY_PREFIX" --check -node <<'NODE' -const fs = require("node:fs"); -const https = require("node:https"); - -function requiredEnv(name) { - const value = process.env[name]; - if (!value) throw new Error(name + " is required"); - return value; -} - -const namespace = requiredEnv("POD_NAMESPACE"); -const host = process.env.KUBERNETES_SERVICE_HOST; -const port = process.env.KUBERNETES_SERVICE_PORT || "443"; -if (!host) throw new Error("KUBERNETES_SERVICE_HOST is not available"); - -const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8"); -const ca = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); -const tektonDir = requiredEnv("TEKTON_DIR"); -const argoFiles = String(process.env.ARGO_FILES || "").split(",").map((item) => item.trim()).filter(Boolean); -const fieldManager = requiredEnv("FIELD_MANAGER"); -const eventName = requiredEnv("RECONCILE_EVENT"); -const manifestFiles = [ - "deploy/gitops/node/" + tektonDir + "/rbac.yaml", - "deploy/gitops/node/" + tektonDir + "/pipeline.yaml", - ...(process.env.LANE === "v02" ? [] : [ - "deploy/gitops/node/" + tektonDir + "/poller.yaml", - "deploy/gitops/node/" + tektonDir + "/control-plane-reconciler.yaml" - ]), - ...argoFiles -]; -const plurals = new Map([ - ["ServiceAccount", "serviceaccounts"], - ["Role", "roles"], - ["RoleBinding", "rolebindings"], - ["Pipeline", "pipelines"], - ["CronJob", "cronjobs"], - ["Application", "applications"], - ["AppProject", "appprojects"] -]); - -function encode(value) { - return encodeURIComponent(String(value)); -} - -function itemsFrom(file) { - const parsed = JSON.parse(fs.readFileSync(file, "utf8")); - return parsed.kind === "List" ? parsed.items || [] : [parsed]; -} - -function apiPath(item) { - if (item.kind === "Namespace") return null; - const name = item.metadata?.name; - if (!name) throw new Error("manifest item is missing metadata.name"); - const ns = item.metadata?.namespace || namespace; - const plural = plurals.get(item.kind); - if (!plural) throw new Error("unsupported reconciler manifest kind " + item.kind); - if (item.apiVersion === "v1") return "/api/v1/namespaces/" + encode(ns) + "/" + plural + "/" + encode(name); - const parts = String(item.apiVersion || "").split("/"); - if (parts.length !== 2) throw new Error("unsupported apiVersion " + item.apiVersion + " for " + item.kind); - return "/apis/" + encode(parts[0]) + "/" + encode(parts[1]) + "/namespaces/" + encode(ns) + "/" + plural + "/" + encode(name); -} - -function reconcilePolicy(item) { - const ns = item.metadata?.namespace || namespace; - const crossNamespaceRbac = (item.kind === "Role" || item.kind === "RoleBinding") && ns !== namespace; - if (crossNamespaceRbac && process.env.HWLAB_RECONCILE_CROSS_NAMESPACE_RBAC !== "1") { - return { action: "skip", reason: "cross-namespace-rbac-bootstrap-managed", namespace: ns }; - } - return { action: "apply", namespace: ns }; -} - -function request(method, path, body) { - const payload = body ? JSON.stringify(body) : ""; - const headers = { Authorization: "Bearer " + token }; - if (payload) { - headers["Content-Type"] = "application/apply-patch+yaml"; - headers["Content-Length"] = Buffer.byteLength(payload); - } - return new Promise((resolve, reject) => { - const req = https.request({ host, port, method, path, ca, headers }, (res) => { - let data = ""; - res.setEncoding("utf8"); - res.on("data", (chunk) => { data += chunk; }); - res.on("end", () => resolve({ statusCode: res.statusCode || 0, body: data })); - }); - req.on("error", reject); - if (payload) req.write(payload); - req.end(); - }); -} - -(async () => { - const results = []; - for (const file of manifestFiles) { - for (const item of itemsFrom(file)) { - const policy = reconcilePolicy(item); - if (policy.action === "skip") { - results.push({ file, kind: item.kind, name: item.metadata?.name || null, namespace: policy.namespace, status: "skipped", reason: policy.reason }); - continue; - } - const path = apiPath(item); - if (!path) { - results.push({ file, kind: item.kind, name: item.metadata?.name || null, status: "skipped-cluster-scoped" }); - continue; - } - const response = await request("PATCH", path + "?fieldManager=" + encode(fieldManager) + "&force=true", item); - if (response.statusCode < 200 || response.statusCode > 299) { - throw new Error("apply failed for " + item.kind + "/" + item.metadata.name + " status=" + response.statusCode + " body=" + response.body.slice(0, 1000)); - } - results.push({ file, kind: item.kind, name: item.metadata.name, status: "applied", statusCode: response.statusCode }); - } - } - console.log(JSON.stringify({ event: eventName, sourceRevision: process.env.RECONCILE_REVISION || null, results }, null, 2)); -})().catch((error) => { - console.error(error.stack || error.message); - process.exitCode = 1; -}); -NODE -`; -} - -function pollerScript() { - return `#!/bin/sh -set -eu -${dependencyProxyProbeScript("poller-proxy-preflight", "http://deb.debian.org/debian/dists/bookworm/InRelease")} -echo '{"event":"ci-base-image","phase":"poller","image":"${ciToolsRunnerImage}","policy":"no-runtime-apk"}' -for tool in node git ssh curl timeout; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"poller","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done -${gitSshShellFunction()} -git_ssh_setup -workdir="$(mktemp -d)" -git_timed poller-clone 180 git clone --depth 1 --branch "$SOURCE_BRANCH" "$GIT_READ_URL" "$workdir/repo" -cd "$workdir/repo" -git remote set-url origin "$GIT_READ_URL" -revision="$(git rev-parse HEAD)" -subject="$(git log -1 --pretty=%s)" -case "$subject" in - "chore: promote node GitOps source "*) - echo "Skipping generated GitOps promotion commit $revision" - exit 0 - ;; -esac -short="$(printf '%.12s' "$revision")" -: "\${PIPELINERUN_PREFIX:?PIPELINERUN_PREFIX is required}" -: "\${GITOPS_TARGET:?GITOPS_TARGET is required}" -prefix="$PIPELINERUN_PREFIX" -name="$prefix-$short" -export POLLER_REVISION="$revision" -export POLLER_PIPELINERUN="$name" -export POLLER_SOURCE_SUBJECT="$subject" -echo "Checking $GITOPS_TARGET source $revision with PipelineRun $name" -node <<'NODE' -const fs = require("node:fs"); -const https = require("node:https"); - -function requiredEnv(name) { - const value = process.env[name]; - if (!value) throw new Error(name + " is required"); - return value; -} - -function optionalEnv(name) { - return process.env[name] || ""; -} - -const namespace = requiredEnv("POD_NAMESPACE"); -const name = requiredEnv("POLLER_PIPELINERUN"); -const revision = requiredEnv("POLLER_REVISION"); -const host = process.env.KUBERNETES_SERVICE_HOST; -const port = process.env.KUBERNETES_SERVICE_PORT || "443"; -if (!host) throw new Error("KUBERNETES_SERVICE_HOST is not available"); - -const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8"); -const ca = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); - -function request(method, path, body) { - const payload = body ? JSON.stringify(body) : ""; - const headers = { Authorization: "Bearer " + token }; - if (payload) { - headers["Content-Type"] = "application/json"; - headers["Content-Length"] = Buffer.byteLength(payload); - } - return new Promise((resolve, reject) => { - const req = https.request({ host, port, method, path, ca, headers }, (res) => { - let data = ""; - res.setEncoding("utf8"); - res.on("data", (chunk) => { data += chunk; }); - res.on("end", () => resolve({ statusCode: res.statusCode || 0, body: data })); - }); - req.on("error", reject); - if (payload) req.write(payload); - req.end(); - }); -} - -const labels = { - "app.kubernetes.io/part-of": "hwlab", - "hwlab.pikastech.local/gitops-target": requiredEnv("GITOPS_TARGET"), - "hwlab.pikastech.local/source-commit": revision, - "hwlab.pikastech.local/trigger": "polling" -}; - -const pipelineRun = { - apiVersion: "tekton.dev/v1", - kind: "PipelineRun", - metadata: { - name, - namespace, - labels, - annotations: { - "hwlab.pikastech.local/source-subject": optionalEnv("POLLER_SOURCE_SUBJECT"), - "hwlab.pikastech.local/source-branch": requiredEnv("SOURCE_BRANCH"), - "hwlab.pikastech.local/gitops-branch": requiredEnv("GITOPS_BRANCH") - } - }, - spec: { - pipelineRef: { name: requiredEnv("PIPELINE_NAME") }, - taskRunTemplate: { - serviceAccountName: requiredEnv("SERVICE_ACCOUNT_NAME"), - podTemplate: { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet", securityContext: { fsGroup: 1000 } } - }, - params: [ - { name: "git-url", value: requiredEnv("GIT_URL") }, - { name: "git-read-url", value: requiredEnv("GIT_READ_URL") }, - { name: "source-branch", value: requiredEnv("SOURCE_BRANCH") }, - { name: "gitops-branch", value: requiredEnv("GITOPS_BRANCH") }, - { name: "lane", value: requiredEnv("LANE") }, - { name: "catalog-path", value: requiredEnv("CATALOG_PATH") }, - { name: "image-tag-mode", value: requiredEnv("IMAGE_TAG_MODE") }, - { name: "runtime-path", value: requiredEnv("RUNTIME_PATH") }, - { name: "revision", value: revision }, - { name: "registry-prefix", value: requiredEnv("REGISTRY_PREFIX") }, - { name: "services", value: requiredEnv("SERVICES") }, - { name: "base-image", value: requiredEnv("BASE_IMAGE") } - ], - workspaces: [ - { name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } }, - { name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } } - ] - } -}; - -const base = "/apis/tekton.dev/v1/namespaces/" + encodeURIComponent(namespace) + "/pipelineruns"; -const item = base + "/" + encodeURIComponent(name); - -(async () => { - const existing = await request("GET", item); - if (existing.statusCode === 200) { - console.log(JSON.stringify({ status: "exists", pipelineRun: name, revision })); - return; - } - if (existing.statusCode !== 404) { - throw new Error("unexpected PipelineRun lookup status " + existing.statusCode + ": " + existing.body.slice(0, 500)); - } - const created = await request("POST", base, pipelineRun); - if (created.statusCode < 200 || created.statusCode > 299) { - throw new Error("PipelineRun create failed with status " + created.statusCode + ": " + created.body.slice(0, 1000)); - } - console.log(JSON.stringify({ status: "created", pipelineRun: name, revision })); -})().catch((error) => { - console.error(error.stack || error.message); - process.exitCode = 1; -}); -NODE -`; -} - -function ciLaneSettings(argsOrLane = "node") { - const lane = typeof argsOrLane === "string" ? argsOrLane : argsOrLane.lane; - if (isRuntimeLane(lane)) { - const namespace = namespaceNameForProfile(lane); - return { - lane, - profile: lane, - gitopsTarget: lane, - pipelineName: `${namespace}-ci-image-publish`, - pipelineRunPrefix: `${namespace}-ci-poll`, - serviceAccountName: `${namespace}-tekton-runner`, - pollerName: `${namespace}-branch-poller`, - controlPlaneReconcilerName: `${namespace}-control-plane-reconciler`, - tektonDir: `tekton-${lane}`, - catalogPath: typeof argsOrLane === "object" ? argsOrLane.catalogPath : `deploy/artifact-catalog.${lane}.json`, - imageTagMode: typeof argsOrLane === "object" ? argsOrLane.imageTagMode : "full", - runtimeNamespace: namespace - }; - } - return { - lane: "node", - profile: "dev", - gitopsTarget: "node", - pipelineName: "hwlab-node-ci-image-publish", - pipelineRunPrefix: "hwlab-node-ci-poll", - serviceAccountName: "hwlab-tekton-runner", - pollerName: "hwlab-node-branch-poller", - controlPlaneReconcilerName: "hwlab-node-control-plane-reconciler", - tektonDir: "tekton", - catalogPath: "deploy/artifact-catalog.dev.json", - imageTagMode: "short", - runtimeNamespace: "hwlab-dev" - }; -} - -function tektonRbac(args = { lane: "node" }) { - const settings = ciLaneSettings(args); - if (isRuntimeLane(settings.lane)) { - const runtimeObserverName = `${settings.runtimeNamespace}-runtime-observer`; - const pipelineRunWriterName = `${settings.runtimeNamespace}-pipelinerun-writer`; - const argoReconcilerName = `${settings.runtimeNamespace}-argocd-reconciler`; - return { - apiVersion: "v1", - kind: "List", - items: [ - { apiVersion: "v1", kind: "Namespace", metadata: { name: "hwlab-ci" } }, - { apiVersion: "v1", kind: "ServiceAccount", metadata: { name: settings.serviceAccountName, namespace: "hwlab-ci" } }, - { - apiVersion: "rbac.authorization.k8s.io/v1", - kind: "Role", - metadata: { name: runtimeObserverName, namespace: settings.runtimeNamespace }, - rules: [ - { apiGroups: [""], resources: ["pods", "services", "configmaps"], verbs: ["get", "list", "watch"] }, - { apiGroups: ["apps"], resources: ["deployments", "statefulsets"], verbs: ["get", "list", "watch"] }, - { apiGroups: ["batch"], resources: ["jobs"], verbs: ["get", "list", "watch"] } - ] - }, - { - apiVersion: "rbac.authorization.k8s.io/v1", - kind: "RoleBinding", - metadata: { name: runtimeObserverName, namespace: settings.runtimeNamespace }, - subjects: [{ kind: "ServiceAccount", name: settings.serviceAccountName, namespace: "hwlab-ci" }], - roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: runtimeObserverName } - }, - { - apiVersion: "rbac.authorization.k8s.io/v1", - kind: "Role", - metadata: { name: pipelineRunWriterName, namespace: "hwlab-ci" }, - rules: [ - { apiGroups: ["tekton.dev"], resources: ["pipelineruns"], verbs: ["get", "list", "create"] }, - { apiGroups: ["tekton.dev"], resources: ["pipelines"], verbs: ["get", "patch", "create"] }, - { apiGroups: ["batch"], resources: ["cronjobs"], verbs: ["get", "patch", "create"] }, - { apiGroups: ["rbac.authorization.k8s.io"], resources: ["roles", "rolebindings"], verbs: ["get", "patch", "create"] }, - { apiGroups: [""], resources: ["serviceaccounts"], verbs: ["get", "patch", "create"] } - ] - }, - { - apiVersion: "rbac.authorization.k8s.io/v1", - kind: "RoleBinding", - metadata: { name: pipelineRunWriterName, namespace: "hwlab-ci" }, - subjects: [{ kind: "ServiceAccount", name: settings.serviceAccountName, namespace: "hwlab-ci" }], - roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: pipelineRunWriterName } - }, - { - apiVersion: "rbac.authorization.k8s.io/v1", - kind: "Role", - metadata: { name: argoReconcilerName, namespace: "argocd" }, - rules: [ - { apiGroups: ["argoproj.io"], resources: ["applications", "appprojects"], verbs: ["get", "patch", "create"] }, - { apiGroups: ["rbac.authorization.k8s.io"], resources: ["roles", "rolebindings"], verbs: ["get", "patch", "create"] } - ] - }, - { - apiVersion: "rbac.authorization.k8s.io/v1", - kind: "RoleBinding", - metadata: { name: argoReconcilerName, namespace: "argocd" }, - subjects: [{ kind: "ServiceAccount", name: settings.serviceAccountName, namespace: "hwlab-ci" }], - roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: argoReconcilerName } - } - ] - }; - } - return { - apiVersion: "v1", - kind: "List", - items: [ - { apiVersion: "v1", kind: "Namespace", metadata: { name: "hwlab-ci" } }, - { apiVersion: "v1", kind: "ServiceAccount", metadata: { name: "hwlab-tekton-runner", namespace: "hwlab-ci" } }, - { - apiVersion: "rbac.authorization.k8s.io/v1", - kind: "Role", - metadata: { name: "hwlab-node-control-plane-reconciler", namespace: "hwlab-dev" }, - rules: [ - { apiGroups: ["rbac.authorization.k8s.io"], resources: ["roles", "rolebindings"], verbs: ["get", "patch", "create"] } - ] - }, - { - apiVersion: "rbac.authorization.k8s.io/v1", - kind: "RoleBinding", - metadata: { name: "hwlab-node-control-plane-reconciler", namespace: "hwlab-dev" }, - subjects: [{ kind: "ServiceAccount", name: "hwlab-tekton-runner", namespace: "hwlab-ci" }], - roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: "hwlab-node-control-plane-reconciler" } - }, - { - apiVersion: "rbac.authorization.k8s.io/v1", - kind: "Role", - metadata: { name: "hwlab-tekton-runtime-observer", namespace: "hwlab-dev" }, - rules: [ - { apiGroups: [""], resources: ["pods", "services", "configmaps"], verbs: ["get", "list", "watch"] }, - { apiGroups: ["apps"], resources: ["deployments", "statefulsets"], verbs: ["get", "list", "watch"] }, - { apiGroups: ["batch"], resources: ["jobs"], verbs: ["get", "list", "watch"] } - ] - }, - { - apiVersion: "rbac.authorization.k8s.io/v1", - kind: "RoleBinding", - metadata: { name: "hwlab-tekton-runtime-observer", namespace: "hwlab-dev" }, - subjects: [{ kind: "ServiceAccount", name: "hwlab-tekton-runner", namespace: "hwlab-ci" }], - roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: "hwlab-tekton-runtime-observer" } - }, - { - apiVersion: "rbac.authorization.k8s.io/v1", - kind: "Role", - metadata: { name: "hwlab-tekton-pipelinerun-writer", namespace: "hwlab-ci" }, - rules: [ - { apiGroups: ["tekton.dev"], resources: ["pipelineruns"], verbs: ["get", "list", "create"] }, - { apiGroups: ["tekton.dev"], resources: ["pipelines"], verbs: ["get", "patch", "create"] }, - { apiGroups: ["batch"], resources: ["cronjobs"], verbs: ["get", "patch", "create"] }, - { apiGroups: ["rbac.authorization.k8s.io"], resources: ["roles", "rolebindings"], verbs: ["get", "patch", "create"] }, - { apiGroups: [""], resources: ["serviceaccounts"], verbs: ["get", "patch", "create"] } - ] - }, - { - apiVersion: "rbac.authorization.k8s.io/v1", - kind: "RoleBinding", - metadata: { name: "hwlab-tekton-pipelinerun-writer", namespace: "hwlab-ci" }, - subjects: [{ kind: "ServiceAccount", name: "hwlab-tekton-runner", namespace: "hwlab-ci" }], - roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: "hwlab-tekton-pipelinerun-writer" } - } - ] - }; -} - -function buildTaskName(serviceId) { - return `build-${serviceId}`; -} - -function affectedResultName(serviceId) { - return `affected-${serviceId}`; -} - -function buildResultName(serviceId) { - return `build-${serviceId}`; -} - -const serviceResultFields = Object.freeze([ - "status", - "service-id", - "image", - "image-tag", - "digest", - "repository-digest", - "source-commit-id", - "component-input-hash", - "environment-input-hash", - "code-input-hash", - "runtime-mode", - "boot-repo", - "boot-commit", - "boot-sh", - "build-created-at", - "build-backend", - "reused-from", - "go-base-image", - "go-base-image-status", - "go-base-digest", - "go-base-build-duration-ms", - "go-base-buildkit-cache-ref" -]); - -function serviceResultParamName(serviceId, field) { - return `${serviceId}-${field}`; -} - -function serviceResultEnvName(serviceId, field) { - return `HWLAB_SERVICE_RESULT_${serviceId.toUpperCase().replace(/[^A-Z0-9]+/gu, "_")}_${field.toUpperCase().replace(/[^A-Z0-9]+/gu, "_")}`; -} - -function serviceResultParams(serviceIds = defaultServiceIds) { - return serviceIds.flatMap((serviceId) => serviceResultFields.map((field) => ({ name: serviceResultParamName(serviceId, field), default: "" }))); -} - -function serviceResultParamValues(serviceIds = defaultServiceIds) { - return serviceIds.flatMap((serviceId) => serviceResultFields.map((field) => ({ - name: serviceResultParamName(serviceId, field), - value: `$(tasks.${buildTaskName(serviceId)}.results.${field})` - }))); -} - -function serviceResultEnv(serviceIds = defaultServiceIds) { - return serviceIds.flatMap((serviceId) => serviceResultFields.map((field) => ({ - name: serviceResultEnvName(serviceId, field), - value: `$(params.${serviceResultParamName(serviceId, field)})` - }))); -} - -function serviceWorkVolume() { - return { name: "service-work", emptyDir: {} }; -} - -function buildkitBinVolume() { - return { name: "buildkit-bin", emptyDir: {} }; -} - -function buildkitRunVolume() { - return { name: "buildkit-run", emptyDir: {} }; -} - -function prepareBuildkitClientStep() { - return { - name: "prepare-buildkit-client", - image: buildkitRunnerImage, - securityContext: { runAsUser: 0, runAsGroup: 0 }, - volumeMounts: [{ name: "buildkit-bin", mountPath: "/workspace/buildkit-bin" }], - script: `#!/bin/sh -set -eu -mkdir -p /workspace/buildkit-bin -cp /usr/bin/buildctl /workspace/buildkit-bin/ -chmod +x /workspace/buildkit-bin/* -` - }; -} - -function buildkitSidecar() { - return { - name: "buildkitd", - image: buildkitRunnerImage, - args: ["--addr", "unix:///workspace/buildkit-run/buildkitd.sock", "--oci-worker-no-process-sandbox"], - env: proxyEnv(), - volumeMounts: [{ name: "buildkit-run", mountPath: "/workspace/buildkit-run" }], - securityContext: { - runAsUser: 1000, - runAsGroup: 1000, - allowPrivilegeEscalation: true, - appArmorProfile: { type: "Unconfined" }, - seccompProfile: { type: "Unconfined" } - } - }; -} - -function planArtifactsTask({ serviceIds = defaultServiceIds } = {}) { - return { - name: "plan-artifacts", - runAfter: primitiveValidationTaskNames(), - workspaces: [{ name: "source", workspace: "source" }], - taskSpec: { - params: [ - { name: "git-url" }, - { name: "git-read-url" }, - { name: "gitops-branch" }, - { name: "lane" }, - { name: "revision" }, - { name: "catalog-path" }, - { name: "registry-prefix" }, - { name: "services" } - ], - results: serviceIds.flatMap((serviceId) => [ - { name: affectedResultName(serviceId), description: `${serviceId} rollout affected according to ci-plan` }, - { name: buildResultName(serviceId), description: `${serviceId} image build required according to ci-plan` } - ]), - workspaces: [{ name: "source" }], - steps: [{ - name: "plan", - image: ciToolsRunnerImage, - env: [ - { name: "HWLAB_SELECTED_SERVICES", value: "$(params.services)" }, - { name: "HWLAB_DEV_REGISTRY_K8S_NATIVE", value: "1" }, - { name: "HWLAB_TEKTON_PIPELINERUN", value: "$(context.pipelineRun.name)" }, - ...proxyEnv() - ], - script: planArtifactsScript() - }] - }, - params: [ - { name: "git-url", value: "$(params.git-url)" }, - { name: "git-read-url", value: "$(params.git-read-url)" }, - { name: "gitops-branch", value: "$(params.gitops-branch)" }, - { name: "lane", value: "$(params.lane)" }, - { name: "revision", value: "$(params.revision)" }, - { name: "catalog-path", value: "$(params.catalog-path)" }, - { name: "registry-prefix", value: "$(params.registry-prefix)" }, - { name: "services", value: "$(params.services)" } - ] - }; -} - -function perServiceBuildTask(serviceId, { runAfter = ["plan-artifacts"] } = {}) { - return { - name: buildTaskName(serviceId), - runAfter, - workspaces: [{ name: "source", workspace: "source" }], - when: [{ input: `$(tasks.plan-artifacts.results.${buildResultName(serviceId)})`, operator: "in", values: ["true"] }], - taskSpec: { - params: [ - { name: "revision" }, - { name: "lane" }, - { name: "catalog-path" }, - { name: "image-tag-mode" }, - { name: "registry-prefix" }, - { name: "service-id" }, - { name: "base-image" }, - { name: "build-cache-mode" } - ], - results: serviceResultFields.map((field) => ({ name: field, description: `${serviceId} ${field}` })), - workspaces: [{ name: "source" }], - volumes: [serviceWorkVolume(), buildkitBinVolume(), buildkitRunVolume()], - sidecars: [buildkitSidecar()], - steps: [ - prepareBuildkitClientStep(), - { - name: "publish", - image: ciToolsRunnerImage, - securityContext: { runAsUser: 1000, runAsGroup: 1000 }, - env: proxyEnv(), - volumeMounts: [{ name: "service-work", mountPath: "/workspace/service-work" }, { name: "buildkit-bin", mountPath: "/workspace/buildkit-bin" }, { name: "buildkit-run", mountPath: "/workspace/buildkit-run" }], - script: perServicePublishScript() - } - ] - }, - params: [ - { name: "revision", value: "$(params.revision)" }, - { name: "lane", value: "$(params.lane)" }, - { name: "catalog-path", value: "$(params.catalog-path)" }, - { name: "image-tag-mode", value: "$(params.image-tag-mode)" }, - { name: "registry-prefix", value: "$(params.registry-prefix)" }, - { name: "service-id", value: serviceId }, - { name: "base-image", value: "$(params.base-image)" }, - { name: "build-cache-mode", value: "$(params.build-cache-mode)" } - ] - }; -} - -function collectArtifactsTask({ serviceIds = defaultServiceIds } = {}) { - return { - name: "collect-artifacts", - runAfter: serviceIds.map(buildTaskName), - workspaces: [{ name: "source", workspace: "source" }], - taskSpec: { - params: [ - { name: "revision" }, - { name: "lane" }, - { name: "catalog-path" }, - { name: "image-tag-mode" }, - { name: "registry-prefix" }, - { name: "services" }, - { name: "base-image" } - ], - workspaces: [{ name: "source" }], - steps: [{ name: "collect", image: ciToolsRunnerImage, env: proxyEnv(), script: collectArtifactsScript() }] - }, - params: [ - { name: "revision", value: "$(params.revision)" }, - { name: "lane", value: "$(params.lane)" }, - { name: "catalog-path", value: "$(params.catalog-path)" }, - { name: "image-tag-mode", value: "$(params.image-tag-mode)" }, - { name: "registry-prefix", value: "$(params.registry-prefix)" }, - { name: "services", value: "$(params.services)" }, - { name: "base-image", value: "$(params.base-image)" } - ] - }; -} - -function runtimeReadyScript() { - return `#!/bin/sh -set -eu -${ciTimingShellFunction()} -export HWLAB_TEKTON_PIPELINERUN="$(context.pipelineRun.name)" -export HWLAB_TEKTON_TASKRUN="$(context.taskRun.name)" -export HWLAB_TEKTON_TASK="runtime-ready" -export HWLAB_SOURCE_REVISION="$(params.revision)" -export HWLAB_RUNTIME_READY_TIMEOUT_MS="$(params.timeout-ms)" -cd /workspace/source/repo -node scripts/ci/runtime-ready.mjs -`; -} - -function runtimeReadyTask({ profile = "dev" } = {}) { - return { - name: "runtime-ready", - runAfter: ["gitops-promote"], - workspaces: [{ name: "source", workspace: "source" }], - taskSpec: { - params: [{ name: "revision" }, { name: "runtime-namespace" }, { name: "gitops-target" }, { name: "argo-application" }, { name: "timeout-ms" }], - workspaces: [{ name: "source" }], - steps: [{ - name: "runtime-ready", - image: ciToolsRunnerImage, - env: [ - ...proxyEnv(), - { name: "HWLAB_RUNTIME_NAMESPACE", value: "$(params.runtime-namespace)" }, - { name: "HWLAB_GITOPS_TARGET", value: "$(params.gitops-target)" }, - { name: "HWLAB_ARGO_APPLICATION", value: "$(params.argo-application)" } - ], - script: runtimeReadyScript() - }] - }, - params: [ - { name: "revision", value: "$(params.revision)" }, - { name: "runtime-namespace", value: namespaceNameForProfile(profile) }, - { name: "gitops-target", value: gitopsTargetForProfile(profile) }, - { name: "argo-application", value: argoApplicationName(profile) }, - { name: "timeout-ms", value: "$(params.runtime-ready-timeout-ms)" } - ] - }; -} - -function imagePublishTaskSet(args = { lane: "node" }, deploy = null) { - const serviceIds = serviceIdsForLane(args.lane, deploy); - const buildTasks = serviceIds.map((serviceId) => perServiceBuildTask(serviceId, { - runAfter: ["plan-artifacts"] - })); - return [ - planArtifactsTask({ serviceIds }), - ...buildTasks, - collectArtifactsTask({ serviceIds }) - ]; -} - -function tektonPipeline(args = { lane: "node" }, deploy = null) { - const settings = ciLaneSettings(args); - const runtimePath = gitopsPathForProfile(args, settings.profile); - const preFlushRuntimeReadyTasks = isRuntimeLane(settings.lane) - ? [] - : [{ - ...runtimeReadyTask({ profile: settings.profile }), - when: [{ input: "$(tasks.gitops-promote.results.runtime-ready-required)", operator: "in", values: ["true"] }] - }]; - return { - apiVersion: "tekton.dev/v1", - kind: "Pipeline", - metadata: { - name: settings.pipelineName, - namespace: "hwlab-ci", - labels: { - "app.kubernetes.io/part-of": "hwlab", - "hwlab.pikastech.local/gitops-target": settings.gitopsTarget - }, - annotations: { - "hwlab.pikastech.local/source-config": "scripts/gitops-render.mjs#tekton-native-primitive-ci", - "hwlab.pikastech.local/ci-contract": "tekton-native-primitive-tasks", - "hwlab.pikastech.local/policy": "native-per-service-taskrun-image-publish" - } - }, - spec: { - params: [ - { name: "git-url", type: "string", default: defaultSourceRepo }, - { name: "git-read-url", type: "string", default: args.gitReadUrl }, - { name: "git-write-url", type: "string", default: args.gitWriteUrl }, - { name: "source-branch", type: "string", default: args.sourceBranch }, - { name: "gitops-branch", type: "string", default: args.gitopsBranch }, - { name: "lane", type: "string", default: settings.lane }, - { name: "catalog-path", type: "string", default: args.catalogPath || settings.catalogPath }, - { name: "image-tag-mode", type: "string", default: args.imageTagMode || settings.imageTagMode }, - { name: "runtime-path", type: "string", default: runtimePath }, - { name: "revision", type: "string", description: "Full git commit SHA; the only source truth for image tags." }, - { name: "registry-prefix", type: "string", default: defaultRegistryPrefix }, - { name: "services", type: "string", default: servicesParamForLane(args.lane, deploy) }, - { name: "base-image", type: "string", default: defaultDevBaseImage }, - { name: "build-cache-mode", type: "string", default: "registry" }, - { name: "runtime-ready-timeout-ms", type: "string", default: "60000" } - ], - workspaces: [ - { name: "source" }, - { name: "git-ssh" } - ], - tasks: [ - { - name: "prepare-source", - workspaces: [ - { name: "source", workspace: "source" }, - { name: "git-ssh", workspace: "git-ssh" } - ], - taskSpec: { - params: [ - { name: "git-url" }, - { name: "git-read-url" }, - { name: "source-branch" }, - { name: "gitops-branch" }, - { name: "lane" }, - { name: "catalog-path" }, - { name: "image-tag-mode" }, - { name: "registry-prefix" }, - { name: "services" }, - { name: "revision" } - ], - workspaces: [{ name: "source" }, { name: "git-ssh" }], - steps: [{ name: "prepare-source", image: ciToolsRunnerImage, env: proxyEnv(), script: prepareSourceScript() }] - }, - params: [ - { name: "git-url", value: "$(params.git-url)" }, - { name: "git-read-url", value: "$(params.git-read-url)" }, - { name: "source-branch", value: "$(params.source-branch)" }, - { name: "gitops-branch", value: "$(params.gitops-branch)" }, - { name: "lane", value: "$(params.lane)" }, - { name: "catalog-path", value: "$(params.catalog-path)" }, - { name: "image-tag-mode", value: "$(params.image-tag-mode)" }, - { name: "registry-prefix", value: "$(params.registry-prefix)" }, - { name: "services", value: "$(params.services)" }, - { name: "revision", value: "$(params.revision)" } - ] - }, - ...primitiveValidationTasks.map(primitiveValidationTask), - ...imagePublishTaskSet(args, deploy), - { - name: "gitops-promote", - runAfter: ["collect-artifacts"], - workspaces: [ - { name: "source", workspace: "source" }, - { name: "git-ssh", workspace: "git-ssh" } - ], - taskSpec: { - params: [ - { name: "git-url" }, - { name: "git-read-url" }, - { name: "git-write-url" }, - { name: "source-branch" }, - { name: "gitops-branch" }, - { name: "lane" }, - { name: "catalog-path" }, - { name: "image-tag-mode" }, - { name: "runtime-path" }, - { name: "revision" }, - { name: "registry-prefix" } - ], - results: [{ name: "runtime-ready-required", description: "true when GitOps promotion changed runtime desired state and runtime readiness must be observed" }], - workspaces: [{ name: "source" }, { name: "git-ssh" }], - steps: [{ name: "promote", image: ciToolsRunnerImage, env: proxyEnv(), script: gitopsPromoteScript() }] - }, - params: [ - { name: "git-url", value: "$(params.git-url)" }, - { name: "git-read-url", value: "$(params.git-read-url)" }, - { name: "git-write-url", value: "$(params.git-write-url)" }, - { name: "source-branch", value: "$(params.source-branch)" }, - { name: "gitops-branch", value: "$(params.gitops-branch)" }, - { name: "lane", value: "$(params.lane)" }, - { name: "catalog-path", value: "$(params.catalog-path)" }, - { name: "image-tag-mode", value: "$(params.image-tag-mode)" }, - { name: "runtime-path", value: "$(params.runtime-path)" }, - { name: "revision", value: "$(params.revision)" }, - { name: "registry-prefix", value: "$(params.registry-prefix)" } - ] - }, - ...preFlushRuntimeReadyTasks - ] - } - }; -} - -function tektonPipelineRunTemplate({ source, args }) { - const settings = ciLaneSettings(args); - const runtimePath = gitopsPathForProfile(args, settings.profile); - return { - apiVersion: "tekton.dev/v1", - kind: "PipelineRun", - metadata: { - generateName: `${settings.pipelineRunPrefix}-manual-`, - namespace: "hwlab-ci", - labels: { - "app.kubernetes.io/part-of": "hwlab", - "hwlab.pikastech.local/gitops-target": settings.gitopsTarget, - "hwlab.pikastech.local/source-commit": source.full - } - }, - spec: { - pipelineRef: { name: settings.pipelineName }, - taskRunTemplate: { - serviceAccountName: settings.serviceAccountName, - podTemplate: { - hostNetwork: true, - dnsPolicy: "ClusterFirstWithHostNet", - securityContext: { fsGroup: 1000 } - } - }, - params: [ - { name: "git-url", value: args.sourceRepo }, - { name: "git-read-url", value: args.gitReadUrl }, - { name: "git-write-url", value: args.gitWriteUrl }, - { name: "source-branch", value: args.sourceBranch }, - { name: "gitops-branch", value: args.gitopsBranch }, - { name: "lane", value: settings.lane }, - { name: "catalog-path", value: args.catalogPath || settings.catalogPath }, - { name: "image-tag-mode", value: args.imageTagMode || settings.imageTagMode }, - { name: "runtime-path", value: runtimePath }, - { name: "revision", value: source.full }, - { name: "registry-prefix", value: args.registryPrefix }, - { name: "base-image", value: defaultDevBaseImage }, - { name: "build-cache-mode", value: "registry" } - ], - workspaces: [ - { name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } }, - { name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } } - ] - } - }; -} - -function tektonPollerCronJob(args, deploy) { - const settings = ciLaneSettings(args); - if (settings.lane === "v02") throw new Error("v02 CI/CD is manually triggered by UniDesk CLI and must not render a branch poller CronJob"); - return { - apiVersion: "batch/v1", - kind: "CronJob", - metadata: { - name: settings.pollerName, - namespace: "hwlab-ci", - labels: { - "app.kubernetes.io/part-of": "hwlab", - "hwlab.pikastech.local/gitops-target": settings.gitopsTarget - }, - annotations: { - "hwlab.pikastech.local/source-branch": args.sourceBranch, - "hwlab.pikastech.local/gitops-branch": args.gitopsBranch - } - }, - spec: { - schedule: settings.lane === "v02" ? "*/10 * * * *" : "* * * * *", - concurrencyPolicy: "Forbid", - successfulJobsHistoryLimit: 3, - failedJobsHistoryLimit: 3, - jobTemplate: { - spec: { - backoffLimit: 1, - template: { - metadata: { - labels: { - "app.kubernetes.io/part-of": "hwlab", - "hwlab.pikastech.local/gitops-target": settings.gitopsTarget - } - }, - spec: { - serviceAccountName: settings.serviceAccountName, - restartPolicy: "Never", - hostNetwork: true, - dnsPolicy: "ClusterFirstWithHostNet", - containers: [{ - name: "poll", - image: ciToolsRunnerImage, - imagePullPolicy: "IfNotPresent", - env: [ - ...proxyEnv(), - { name: "POD_NAMESPACE", valueFrom: { fieldRef: { fieldPath: "metadata.namespace" } } }, - { name: "PIPELINE_NAME", value: settings.pipelineName }, - { name: "PIPELINERUN_PREFIX", value: settings.pipelineRunPrefix }, - { name: "SERVICE_ACCOUNT_NAME", value: settings.serviceAccountName }, - { name: "GITOPS_TARGET", value: settings.gitopsTarget }, - { name: "LANE", value: settings.lane }, - { name: "CATALOG_PATH", value: args.catalogPath || settings.catalogPath }, - { name: "IMAGE_TAG_MODE", value: args.imageTagMode || settings.imageTagMode }, - { name: "RUNTIME_PATH", value: gitopsPathForProfile(args, settings.profile) }, - { name: "GIT_URL", value: args.sourceRepo }, - { name: "GIT_READ_URL", value: args.gitReadUrl }, - { name: "SOURCE_BRANCH", value: args.sourceBranch }, - { name: "GITOPS_BRANCH", value: args.gitopsBranch }, - { name: "REGISTRY_PREFIX", value: args.registryPrefix }, - { name: "SERVICES", value: servicesParamForLane(args.lane, deploy) }, - { name: "BASE_IMAGE", value: defaultDevBaseImage } - ], - volumeMounts: [{ name: "git-ssh", mountPath: "/workspace/git-ssh", readOnly: true }], - command: ["/bin/sh", "-c"], - args: [pollerScript()] - }], - volumes: [{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }] - } - } - } - } - } - }; -} - -function tektonControlPlaneReconcilerCronJob(args) { - const settings = ciLaneSettings(args); - if (settings.lane === "v02") throw new Error("v02 CI/CD is manually triggered by UniDesk CLI and must not render a control-plane reconciler CronJob"); - return { - apiVersion: "batch/v1", - kind: "CronJob", - metadata: { - name: settings.controlPlaneReconcilerName, - namespace: "hwlab-ci", - labels: { - "app.kubernetes.io/part-of": "hwlab", - "hwlab.pikastech.local/gitops-target": settings.gitopsTarget - }, - annotations: { - "hwlab.pikastech.local/source-branch": args.sourceBranch, - "hwlab.pikastech.local/gitops-branch": args.gitopsBranch, - "hwlab.pikastech.local/purpose": "render-and-apply-node-ci-control-plane" - } - }, - spec: { - schedule: "*/10 * * * *", - concurrencyPolicy: "Forbid", - successfulJobsHistoryLimit: 3, - failedJobsHistoryLimit: 3, - jobTemplate: { - spec: { - backoffLimit: 1, - template: { - metadata: { - labels: { - "app.kubernetes.io/part-of": "hwlab", - "hwlab.pikastech.local/gitops-target": settings.gitopsTarget - } - }, - spec: { - serviceAccountName: settings.serviceAccountName, - restartPolicy: "Never", - hostNetwork: true, - dnsPolicy: "ClusterFirstWithHostNet", - containers: [{ - name: "reconcile", - image: ciToolsRunnerImage, - imagePullPolicy: "IfNotPresent", - env: [ - ...proxyEnv(), - { name: "POD_NAMESPACE", valueFrom: { fieldRef: { fieldPath: "metadata.namespace" } } }, - { name: "LANE", value: settings.lane }, - { name: "CATALOG_PATH", value: args.catalogPath || settings.catalogPath }, - { name: "IMAGE_TAG_MODE", value: args.imageTagMode || settings.imageTagMode }, - { name: "TEKTON_DIR", value: settings.tektonDir }, - { name: "ARGO_FILES", value: settings.lane === "v02" ? `${gitopsPathFor(args, "argocd/project.yaml")},${gitopsPathFor(args, "argocd/application-v02.yaml")}` : "" }, - { name: "FIELD_MANAGER", value: settings.controlPlaneReconcilerName }, - { name: "RECONCILE_EVENT", value: `${settings.gitopsTarget}-control-plane-reconciled` }, - { name: "GIT_URL", value: args.sourceRepo }, - { name: "GIT_READ_URL", value: args.gitReadUrl }, - { name: "SOURCE_BRANCH", value: args.sourceBranch }, - { name: "GITOPS_BRANCH", value: args.gitopsBranch }, - { name: "REGISTRY_PREFIX", value: args.registryPrefix } - ], - volumeMounts: [{ name: "git-ssh", mountPath: "/workspace/git-ssh", readOnly: true }], - command: ["/bin/sh", "-c"], - args: [controlPlaneReconcileScript()] - }], - volumes: [{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }] - } - } - } - } - } - }; -} - -function registryManifest() { - return { - apiVersion: "v1", - kind: "List", - items: [ - { apiVersion: "v1", kind: "Namespace", metadata: { name: "hwlab-ci" } }, - { - apiVersion: "apps/v1", - kind: "Deployment", - metadata: { name: "hwlab-registry", namespace: "hwlab-ci", labels: { "app.kubernetes.io/name": "hwlab-registry" } }, - spec: { - replicas: 1, - selector: { matchLabels: { "app.kubernetes.io/name": "hwlab-registry" } }, - template: { - metadata: { labels: { "app.kubernetes.io/name": "hwlab-registry" } }, - spec: { - hostNetwork: true, - dnsPolicy: "ClusterFirstWithHostNet", - containers: [{ - name: "registry", - image: "registry:2.8.3", - imagePullPolicy: "IfNotPresent", - ports: [{ name: "registry", containerPort: 5000, hostPort: 5000 }], - env: [{ name: "REGISTRY_STORAGE_DELETE_ENABLED", value: "true" }], - volumeMounts: [{ name: "storage", mountPath: "/var/lib/registry" }], - readinessProbe: { httpGet: { path: "/v2/", port: "registry" } }, - livenessProbe: { httpGet: { path: "/v2/", port: "registry" } } - }], - volumes: [{ name: "storage", hostPath: { path: "/var/lib/hwlab/registry", type: "DirectoryOrCreate" } }] - } - } - } - }, - { - apiVersion: "v1", - kind: "Service", - metadata: { name: "hwlab-registry", namespace: "hwlab-ci", labels: { "app.kubernetes.io/name": "hwlab-registry" } }, - spec: { selector: { "app.kubernetes.io/name": "hwlab-registry" }, ports: [{ name: "registry", port: 5000, targetPort: "registry" }] } - } - ] - }; -} - -function uniqueStrings(values) { - return Array.from(new Set(values.filter((value) => typeof value === "string" && value.length > 0))); -} - -function configObject(value, label) { - assert.ok(value && typeof value === "object" && !Array.isArray(value), `${label} must be an object`); - return value; -} - -function requiredConfigString(record, key, label) { - const value = record[key]; - assert.ok(typeof value === "string" && value.length > 0, `${label}.${key} must be a non-empty string`); - return value; -} - -function optionalConfigString(record, key, label) { - const value = record[key]; - if (value === undefined || value === null || value === "") return null; - assert.ok(typeof value === "string", `${label}.${key} must be a string when set`); - return value; -} - -function requiredConfigPositiveInteger(record, key, label) { - const value = record[key]; - assert.ok(Number.isInteger(value) && value > 0, `${label}.${key} must be a positive integer`); - return value; -} - -function devopsInfraGitMirrorManifest(args, deploy) { - const mirrorSpecs = runtimeLaneMirrorSpecs(deploy, { defaultNodeId: args.nodeId, defaultGitopsRoot: defaultOutDir }); - assert.ok(mirrorSpecs.length > 0, "runtime lane git mirror requires at least one deploy.lanes entry"); - const sourceBranches = uniqueStrings(mirrorSpecs.map((spec) => spec.sourceBranch)); - const gitopsBranches = uniqueStrings(mirrorSpecs.map((spec) => spec.gitopsBranch)); - const mirrorBranches = uniqueStrings([...sourceBranches, ...gitopsBranches]); - const fetchConfigCommands = mirrorBranches - .map((branch) => `git -C "$repo_path" config --add remote.origin.fetch '+refs/heads/${branch}:refs/mirror-stage/heads/${branch}'`) - .join("\n"); - const requiredRefArgs = mirrorBranches.map((branch) => shellSingleQuote(`refs/mirror-stage/heads/${branch}`)).join(" "); - const sourceBranchArgs = sourceBranches.map((branch) => shellSingleQuote(branch)).join(" "); - const gitopsBranchArgs = gitopsBranches.map((branch) => shellSingleQuote(branch)).join(" "); - const mirrorBranchesJson = JSON.stringify(mirrorBranches); - const sourceBranchesJson = JSON.stringify(sourceBranches); - const gitopsBranchesJson = JSON.stringify(gitopsBranches); - const sourceSnapshotRefPrefix = "refs/unidesk/snapshots/hwlab-node-runtime"; - const laneConfig = configObject(configObject(deploy.lanes, "deploy.lanes")[args.lane], `deploy.lanes.${args.lane}`); - const gitMirrorConfig = configObject(laneConfig.gitMirror, `deploy.lanes.${args.lane}.gitMirror`); - const gitMirrorProxy = configObject(gitMirrorConfig.egressProxy, `deploy.lanes.${args.lane}.gitMirror.egressProxy`); - const gitMirrorProxyMode = requiredConfigString(gitMirrorProxy, "mode", `deploy.lanes.${args.lane}.gitMirror.egressProxy`); - assert.ok(gitMirrorProxyMode === "node-global" || gitMirrorProxyMode === "host-proxy-client" || gitMirrorProxyMode === "direct", `deploy.lanes.${args.lane}.gitMirror.egressProxy.mode must be node-global, host-proxy-client, or direct`); - const useDirectGitMirrorProxy = gitMirrorProxyMode === "direct"; - const useHostGitMirrorProxy = gitMirrorProxyMode === "host-proxy-client"; - const gitMirrorNamespace = requiredConfigString(gitMirrorConfig, "namespace", `deploy.lanes.${args.lane}.gitMirror`); - const gitMirrorReadName = requiredConfigString(gitMirrorConfig, "serviceReadName", `deploy.lanes.${args.lane}.gitMirror`); - const gitMirrorWriteName = requiredConfigString(gitMirrorConfig, "serviceWriteName", `deploy.lanes.${args.lane}.gitMirror`); - const gitMirrorCachePvcName = requiredConfigString(gitMirrorConfig, "cachePvcName", `deploy.lanes.${args.lane}.gitMirror`); - const gitMirrorCachePvcStorage = requiredConfigString(gitMirrorConfig, "cachePvcStorage", `deploy.lanes.${args.lane}.gitMirror`); - const gitMirrorCacheHostPath = optionalConfigString(gitMirrorConfig, "cacheHostPath", `deploy.lanes.${args.lane}.gitMirror`); - const gitMirrorServicePort = requiredConfigPositiveInteger(gitMirrorConfig, "servicePort", `deploy.lanes.${args.lane}.gitMirror`); - const gitMirrorConfigMapName = requiredConfigString(gitMirrorConfig, "syncConfigMapName", `deploy.lanes.${args.lane}.gitMirror`); - const proxyNamespace = useDirectGitMirrorProxy || useHostGitMirrorProxy ? "" : requiredConfigString(gitMirrorProxy, "namespace", `deploy.lanes.${args.lane}.gitMirror.egressProxy`); - const proxyServiceName = useDirectGitMirrorProxy || useHostGitMirrorProxy ? "" : requiredConfigString(gitMirrorProxy, "serviceName", `deploy.lanes.${args.lane}.gitMirror.egressProxy`); - const proxyPort = useDirectGitMirrorProxy ? 0 : requiredConfigPositiveInteger(gitMirrorProxy, "port", `deploy.lanes.${args.lane}.gitMirror.egressProxy`); - const proxyClientName = useDirectGitMirrorProxy ? "" : requiredConfigString(gitMirrorProxy, "clientName", `deploy.lanes.${args.lane}.gitMirror.egressProxy`); - const proxyNoProxy = !useDirectGitMirrorProxy && Array.isArray(gitMirrorProxy.noProxy) ? gitMirrorProxy.noProxy.filter((value) => typeof value === "string" && value.length > 0).join(",") : ""; - const proxyHost = useHostGitMirrorProxy - ? requiredConfigString(gitMirrorProxy, "host", `deploy.lanes.${args.lane}.gitMirror.egressProxy`) - : `${proxyServiceName}.${proxyNamespace}.svc.cluster.local`; - const proxyUrl = `http://${proxyHost}:${proxyPort}`; - const proxySummary = useDirectGitMirrorProxy - ? "git-mirror-egress-proxy mode=direct required=false transport=ssh source=yaml" - : `git-mirror-egress-proxy client=${proxyClientName} mode=${gitMirrorProxyMode} required=true host=${proxyHost} port=${proxyPort} ssh=GIT_SSH-wrapper source=yaml`; - const proxyCommand = useDirectGitMirrorProxy ? "" : `ProxyCommand=node /tmp/hwlab-github-proxy-connect.mjs ${proxyHost} ${proxyPort} %h %p`; - const directGitMirrorSshPrelude = `printf '%s\\n' ${shellSingleQuote(proxySummary)} >&2 -cat > /tmp/hwlab-git-ssh-proxy.sh <<'SH_PROXY' -#!/bin/sh -exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 "$@" -SH_PROXY -chmod 0700 /tmp/hwlab-git-ssh-proxy.sh -unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy -export NO_PROXY='*' -export no_proxy='*' -export GIT_SSH=/tmp/hwlab-git-ssh-proxy.sh -unset GIT_SSH_COMMAND`; - const proxiedGitMirrorSshPrelude = `printf '%s\\n' ${shellSingleQuote(proxySummary)} >&2 -cat > /tmp/hwlab-github-proxy-connect.mjs <<'NODE_PROXY' -#!/usr/bin/env node -import net from "node:net"; -const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2); -const proxyPort = Number.parseInt(proxyPortRaw || "", 10); -const targetPort = Number.parseInt(targetPortRaw || "", 10); -if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) { - console.error("hwlab git-mirror proxy-connect: invalid ProxyCommand arguments"); - process.exit(64); -} -let settled = false; -let tunnelEstablished = false; -function finish(code, message) { - if (settled) return; - settled = true; - if (message) console.error("hwlab git-mirror proxy-connect: " + message); - process.exit(code); -} -const socket = net.createConnection({ host: proxyHost, port: proxyPort }); -let buffer = Buffer.alloc(0); -socket.setTimeout(15000, () => { socket.destroy(); finish(65, "timeout connecting via " + proxyHost + ":" + proxyPort + " to " + targetHost + ":" + targetPort); }); -socket.on("connect", () => socket.write("CONNECT " + targetHost + ":" + targetPort + " HTTP/1.1\\r\\nHost: " + targetHost + ":" + targetPort + "\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n")); -socket.on("error", (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? "tunnel socket error: " : "tcp error connecting to proxy: ") + (error && error.message ? error.message : String(error)))); -socket.on("close", () => { if (!tunnelEstablished) finish(68, "proxy closed before CONNECT completed via " + proxyHost + ":" + proxyPort + " to " + targetHost + ":" + targetPort); else finish(0); }); -function onData(chunk) { - buffer = Buffer.concat([buffer, chunk]); - const headerEnd = buffer.indexOf("\\r\\n\\r\\n"); - if (headerEnd === -1 && buffer.length < 8192) return; - if (headerEnd === -1) { socket.destroy(); finish(68, "proxy response header exceeded 8192 bytes before CONNECT status via " + proxyHost + ":" + proxyPort + " to " + targetHost + ":" + targetPort); return; } - const head = buffer.slice(0, headerEnd + 4).toString("latin1"); - const statusLine = head.split("\\r\\n", 1)[0] || ""; - const statusCode = Number.parseInt(statusLine.split(" ")[1] || "", 10); - if (!statusLine.startsWith("HTTP/1.") || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) { - const safeStatus = statusLine.replace(/[^\\x20-\\x7e]/g, "?").slice(0, 160); - socket.destroy(); - finish(67, "proxy CONNECT failed via " + proxyHost + ":" + proxyPort + " to " + targetHost + ":" + targetPort + ": " + safeStatus); - return; - } - socket.off("data", onData); - socket.setTimeout(0); - tunnelEstablished = true; - const rest = buffer.slice(headerEnd + 4); - if (rest.length) process.stdout.write(rest); - process.stdin.on("error", () => {}); - process.stdout.on("error", () => {}); - process.stdin.pipe(socket); - socket.pipe(process.stdout); -} -socket.on("data", onData); -NODE_PROXY -chmod 0700 /tmp/hwlab-github-proxy-connect.mjs -cat > /tmp/hwlab-git-ssh-proxy.sh <<'SH_PROXY' -#!/bin/sh -exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ${shellSingleQuote(proxyCommand)} "$@" -SH_PROXY -chmod 0700 /tmp/hwlab-git-ssh-proxy.sh -export HTTP_PROXY=${shellSingleQuote(proxyUrl)} -export HTTPS_PROXY=${shellSingleQuote(proxyUrl)} -export ALL_PROXY=${shellSingleQuote(proxyUrl)} -export http_proxy=${shellSingleQuote(proxyUrl)} -export https_proxy=${shellSingleQuote(proxyUrl)} -export all_proxy=${shellSingleQuote(proxyUrl)} -export NO_PROXY=${shellSingleQuote(proxyNoProxy)} -export no_proxy=${shellSingleQuote(proxyNoProxy)} -export GIT_SSH=/tmp/hwlab-git-ssh-proxy.sh -unset GIT_SSH_COMMAND`; - const gitMirrorSshPrelude = useDirectGitMirrorProxy ? directGitMirrorSshPrelude : proxiedGitMirrorSshPrelude; - const cacheVolume = gitMirrorCacheHostPath === null - ? { name: "cache", persistentVolumeClaim: { claimName: gitMirrorCachePvcName } } - : { name: "cache", hostPath: { path: gitMirrorCacheHostPath, type: "DirectoryOrCreate" } }; - const labels = { - "app.kubernetes.io/part-of": "hwlab-node-control-plane", - "hwlab.pikastech.local/node": args.nodeId, - "hwlab.pikastech.local/lane": args.lane - }; - const configLabels = { ...labels, "app.kubernetes.io/name": "git-mirror" }; - const readLabels = { ...labels, "app.kubernetes.io/name": gitMirrorReadName, "hwlab.pikastech.local/git-mirror-mode": "read" }; - const writeLabels = { ...labels, "app.kubernetes.io/name": gitMirrorWriteName, "hwlab.pikastech.local/git-mirror-mode": "write" }; - return { - apiVersion: "v1", - kind: "List", - items: [ - { apiVersion: "v1", kind: "Namespace", metadata: { name: gitMirrorNamespace } }, - ...(gitMirrorCacheHostPath === null ? [{ - apiVersion: "v1", - kind: "PersistentVolumeClaim", - metadata: { name: gitMirrorCachePvcName, namespace: gitMirrorNamespace, labels: configLabels }, - spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: gitMirrorCachePvcStorage } } } - }] : []), - { - apiVersion: "v1", - kind: "ConfigMap", - metadata: { name: gitMirrorConfigMapName, namespace: gitMirrorNamespace, labels: configLabels }, - data: { - "sync.sh": `#!/bin/sh -set -eu -repo_url="ssh://git@ssh.github.com:443/pikasTech/HWLAB.git" -repo_path="/cache/pikasTech/HWLAB.git" -source_snapshot_ref_prefix=${shellSingleQuote(sourceSnapshotRefPrefix)} -lock_dir="/cache/.hwlab-sync.lock" -sync_started_ms=$(node -e 'console.log(Date.now())') -now_ms() { node -e 'console.log(Date.now())'; } -emit_timing() { - phase="$1" - status="$2" - started_ms="$3" - finished_ms=$(now_ms) - duration_ms=$((finished_ms - started_ms)) - printf '{"event":"git-mirror-sync-timing","phase":"%s","status":"%s","durationMs":%s,"repository":"pikasTech/HWLAB"}\n' "$phase" "$status" "$duration_ms" -} -mkdir -p /cache/pikasTech /root/.ssh -if ! mkdir "$lock_dir" 2>/dev/null; then - echo "git mirror sync already running" - exit 0 -fi -trap 'rmdir "$lock_dir" 2>/dev/null || true' EXIT INT TERM -cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa -chmod 0400 /root/.ssh/id_rsa -${gitMirrorSshPrelude} -if [ -d "$repo_path/objects" ] && [ -f "$repo_path/HEAD" ]; then - git -C "$repo_path" remote set-url origin "$repo_url" || git -C "$repo_path" remote add origin "$repo_url" -else - rm -rf "$repo_path" - git init --bare "$repo_path" - git -C "$repo_path" remote add origin "$repo_url" -fi -/script/install-hooks.sh -git -C "$repo_path" config uploadpack.hideRefs refs/mirror-stage -git -C "$repo_path" config uploadpack.allowReachableSHA1InWant true -git -C "$repo_path" config uploadpack.allowAnySHA1InWant true -git -C "$repo_path" config --unset-all remote.origin.fetch 2>/dev/null || true -${fetchConfigCommands} -fetch_started_ms=$(now_ms) -timeout 180 git -C "$repo_path" fetch --quiet --prune origin -emit_timing fetch succeeded "$fetch_started_ms" -git -C "$repo_path" for-each-ref --format='%(objectname) %(refname)' refs/mirror-stage/heads > /tmp/hwlab-stage-heads -git -C "$repo_path" for-each-ref --format='%(refname)' refs/mirror-stage/heads > /tmp/hwlab-stage-head-refs -git -C "$repo_path" for-each-ref --format='%(objectname) %(refname)' refs/mirror-stage/tags > /tmp/hwlab-stage-tags -git -C "$repo_path" for-each-ref --format='%(refname)' refs/mirror-stage/tags > /tmp/hwlab-stage-tag-refs -if [ ! -s /tmp/hwlab-stage-heads ]; then - echo "git mirror sync fetched no branch refs" >&2 - exit 41 -fi -for required_ref in ${requiredRefArgs}; do - git -C "$repo_path" show-ref --verify --quiet "$required_ref" || { echo "git mirror sync missing required ref $required_ref" >&2; exit 43; } -done -validate_started_ms=$(now_ms) -while read -r sha ref; do - [ -n "$sha" ] || continue - git -C "$repo_path" cat-file -e "$sha^{commit}" - git -C "$repo_path" cat-file -e "$sha^{tree}" - if git -C "$repo_path" rev-list --objects --missing=print "$sha" | grep -q '^?'; then - echo "git mirror sync found missing objects for $ref $sha" >&2 - exit 42 - fi -done < /tmp/hwlab-stage-heads -emit_timing validate succeeded "$validate_started_ms" -publish_started_ms=$(now_ms) -while read -r sha ref; do - [ -n "$sha" ] || continue - name=$(printf '%s\n' "$ref" | sed 's#^refs/mirror-stage/heads/##') - is_gitops_branch=false - for gitops_branch in ${gitopsBranchArgs}; do - if [ "$name" = "$gitops_branch" ]; then is_gitops_branch=true; break; fi - done - if [ "$is_gitops_branch" = true ]; then - local_sha=$(git -C "$repo_path" show-ref --verify --hash "refs/heads/$name" 2>/dev/null || true) - if [ -n "$local_sha" ] && [ "$local_sha" != "$sha" ]; then - if git -C "$repo_path" merge-base --is-ancestor "$sha" "$local_sha"; then - echo "git mirror sync keeps local $name ahead of GitHub" - continue - fi - if ! git -C "$repo_path" merge-base --is-ancestor "$local_sha" "$sha"; then - echo "git mirror sync found divergent $name local=$local_sha github=$sha" >&2 - exit 44 - fi - fi - fi - git -C "$repo_path" update-ref "refs/heads/$name" "$sha" -done < /tmp/hwlab-stage-heads -for source_branch in ${sourceBranchArgs}; do - source_sha=$(git -C "$repo_path" show-ref --verify --hash "refs/heads/$source_branch" 2>/dev/null || true) - if [ -n "$source_sha" ]; then - source_stage_ref="\${source_snapshot_ref_prefix%/}/$source_branch/$source_sha" - git -C "$repo_path" update-ref "$source_stage_ref" "$source_sha" - fi -done -while read -r sha ref; do - [ -n "$sha" ] || continue - name=$(printf '%s\n' "$ref" | sed 's#^refs/mirror-stage/tags/##') - git -C "$repo_path" update-ref "refs/tags/$name" "$sha" -done < /tmp/hwlab-stage-tags -git -C "$repo_path" for-each-ref --format='%(refname)' refs/heads > /tmp/hwlab-public-heads -while read -r ref; do - [ -n "$ref" ] || continue - name=$(printf '%s\n' "$ref" | sed 's#^refs/heads/##') - if ! grep -Fxq "refs/mirror-stage/heads/$name" /tmp/hwlab-stage-head-refs; then - git -C "$repo_path" update-ref -d "$ref" - fi -done < /tmp/hwlab-public-heads -git -C "$repo_path" for-each-ref --format='%(refname)' refs/tags > /tmp/hwlab-public-tags -while read -r ref; do - [ -n "$ref" ] || continue - name=$(printf '%s\n' "$ref" | sed 's#^refs/tags/##') - if ! grep -Fxq "refs/mirror-stage/tags/$name" /tmp/hwlab-stage-tag-refs; then - git -C "$repo_path" update-ref -d "$ref" - fi -done < /tmp/hwlab-public-tags -emit_timing publish succeeded "$publish_started_ms" -for head_branch in ${sourceBranchArgs}; do - if git -C "$repo_path" show-ref --verify --quiet "refs/heads/$head_branch"; then - git -C "$repo_path" symbolic-ref HEAD "refs/heads/$head_branch" - break - fi -done -fsck_started_ms=$(now_ms) -git -C "$repo_path" fsck --connectivity-only --no-dangling >/tmp/hwlab-git-fsck.out -emit_timing fsck succeeded "$fsck_started_ms" -git -C "$repo_path" update-server-info -published_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) -for source_branch in ${sourceBranchArgs}; do - git -C "$repo_path" rev-parse "refs/heads/$source_branch" 2>/dev/null | tee "/cache/HWLAB-$source_branch.head" >/dev/null || true -done -printf '%s\n' "$published_at" > /cache/HWLAB.last-sync -export published_at -export source_snapshot_ref_prefix -export mirror_branches_json=${shellSingleQuote(mirrorBranchesJson)} -export source_branches_json=${shellSingleQuote(sourceBranchesJson)} -export gitops_branches_json=${shellSingleQuote(gitopsBranchesJson)} -node - <<'NODE' | tee /cache/HWLAB.last-sync.json -const { execFileSync } = require("node:child_process"); -const repo = "/cache/pikasTech/HWLAB.git"; -const mirrorBranches = JSON.parse(process.env.mirror_branches_json || "[]"); -const sourceBranches = JSON.parse(process.env.source_branches_json || "[]"); -const gitopsBranches = JSON.parse(process.env.gitops_branches_json || "[]"); -const sourceSnapshotRefPrefix = (process.env.source_snapshot_ref_prefix || "refs/unidesk/snapshots/hwlab-node-runtime").replace(/\/+$/u, ""); -function rev(ref) { - try { return execFileSync("git", ["-C", repo, "rev-parse", ref], { encoding: "utf8" }).trim() || null; } catch { return null; } -} -const refs = {}; -for (const branch of mirrorBranches) { - refs["refs/heads/" + branch] = rev("refs/heads/" + branch); - refs["refs/mirror-stage/heads/" + branch] = rev("refs/mirror-stage/heads/" + branch); -} -const sourceSnapshots = {}; -for (const branch of sourceBranches) { - const source = refs["refs/heads/" + branch] || null; - const stageRef = source ? sourceSnapshotRefPrefix + "/" + branch + "/" + source : null; - const sourceSnapshot = stageRef ? rev(stageRef) : null; - if (stageRef) refs[stageRef] = sourceSnapshot; - sourceSnapshots[branch] = { stageRef, sourceSnapshot }; -} -const sourceInSync = sourceBranches.every((branch) => refs["refs/heads/" + branch] && refs["refs/heads/" + branch] === refs["refs/mirror-stage/heads/" + branch] && sourceSnapshots[branch]?.sourceSnapshot === refs["refs/mirror-stage/heads/" + branch]); -const gitopsInSync = gitopsBranches.every((branch) => refs["refs/heads/" + branch] && refs["refs/heads/" + branch] === refs["refs/mirror-stage/heads/" + branch]); -const pendingFlush = gitopsBranches.some((branch) => refs["refs/heads/" + branch] && refs["refs/mirror-stage/heads/" + branch] && refs["refs/heads/" + branch] !== refs["refs/mirror-stage/heads/" + branch]); -const payload = { - event: "git-mirror-sync", - status: "published", - repository: "pikasTech/HWLAB", - publishedAt: process.env.published_at, - pendingFlush, - sourceInSync, - gitopsInSync, - sourceSnapshots, - refs -}; -console.log(JSON.stringify(payload)); -NODE -emit_timing total succeeded "$sync_started_ms" -`, - "install-hooks.sh": `#!/bin/sh -set -eu -repo_path="/cache/pikasTech/HWLAB.git" -mkdir -p "$repo_path/hooks" -cat > "$repo_path/hooks/pre-receive" <<'HOOK' -#!/bin/sh -set -eu -zero="0000000000000000000000000000000000000000" -status=0 -while read -r old new ref; do - if [ "$new" = "$zero" ]; then - echo "git mirror write rejected: deleting $ref is forbidden" >&2 - status=1 - continue - fi - git cat-file -e "$new^{commit}" || status=1 - git cat-file -e "$new^{tree}" || status=1 - if git rev-list --objects --missing=print "$new" | grep -q '^?'; then - echo "git mirror write rejected: missing objects for $new" >&2 - status=1 - continue - fi - if [ "$old" != "$zero" ] && ! git merge-base --is-ancestor "$old" "$new"; then - echo "git mirror write rejected: non-fast-forward $ref" >&2 - status=1 - continue - fi -done -exit "$status" -HOOK -cat > "$repo_path/hooks/post-receive" <<'HOOK' -#!/bin/sh -set -eu -repo_path=$(git rev-parse --git-dir) -git -C "$repo_path" update-server-info -written_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) -export repo_path written_at -export gitops_branches_json=${shellSingleQuote(gitopsBranchesJson)} -node - <<'NODE' > /cache/HWLAB.last-write.json -const { execFileSync } = require("node:child_process"); -const repo = process.env.repo_path || "/cache/pikasTech/HWLAB.git"; -const gitopsBranches = JSON.parse(process.env.gitops_branches_json || "[]"); -function rev(ref) { - try { return execFileSync("git", ["-C", repo, "rev-parse", ref], { encoding: "utf8" }).trim() || null; } catch { return null; } -} -const refs = {}; -for (const branch of gitopsBranches) { - refs["refs/heads/" + branch] = rev("refs/heads/" + branch); - refs["refs/mirror-stage/heads/" + branch] = rev("refs/mirror-stage/heads/" + branch); -} -const pendingFlush = gitopsBranches.some((branch) => refs["refs/heads/" + branch] && refs["refs/heads/" + branch] !== refs["refs/mirror-stage/heads/" + branch]); -console.log(JSON.stringify({ - event: "git-mirror-write", - status: "received", - repository: "pikasTech/HWLAB", - writtenAt: process.env.written_at, - pendingFlush, - refs -})); -NODE -HOOK -chmod 0755 "$repo_path/hooks/pre-receive" "$repo_path/hooks/post-receive" -git -C "$repo_path" config http.receivepack true -git -C "$repo_path" config receive.denyDeletes true -git -C "$repo_path" config receive.denyNonFastForwards true -`, - "flush.sh": `#!/bin/sh -set -eu -repo_url="ssh://git@ssh.github.com:443/pikasTech/HWLAB.git" -repo_path="/cache/pikasTech/HWLAB.git" -lock_dir="/cache/.hwlab-flush.lock" -started_ms=$(node -e 'console.log(Date.now())') -now_ms() { node -e 'console.log(Date.now())'; } -emit_timing() { - phase="$1" - status="$2" - started="$3" - finished=$(now_ms) - duration=$((finished - started)) - printf '{"event":"git-mirror-flush-timing","phase":"%s","status":"%s","durationMs":%s,"repository":"pikasTech/HWLAB"}\n' "$phase" "$status" "$duration" -} -if ! mkdir "$lock_dir" 2>/dev/null; then - echo "git mirror flush already running" - exit 0 -fi -trap 'rmdir "$lock_dir" 2>/dev/null || true' EXIT INT TERM -mkdir -p /root/.ssh -cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa -chmod 0400 /root/.ssh/id_rsa -${gitMirrorSshPrelude} -/script/install-hooks.sh -for gitops_branch in ${gitopsBranchArgs}; do - local_head=$(git -C "$repo_path" show-ref --verify --hash "refs/heads/$gitops_branch" 2>/dev/null || true) - if [ -z "$local_head" ]; then - echo "git mirror flush skips missing local refs/heads/$gitops_branch" - continue - fi - fetch_started=$(now_ms) - timeout 90 git -C "$repo_path" fetch --quiet "$repo_url" "refs/heads/$gitops_branch:refs/mirror-stage/heads/$gitops_branch" - emit_timing "fetch-$gitops_branch" succeeded "$fetch_started" - github_head=$(git -C "$repo_path" show-ref --verify --hash "refs/mirror-stage/heads/$gitops_branch" 2>/dev/null || true) - if [ -n "$github_head" ] && [ "$github_head" != "$local_head" ] && ! git -C "$repo_path" merge-base --is-ancestor "$github_head" "$local_head"; then - echo "git mirror flush rejected divergent $gitops_branch local=$local_head github=$github_head" >&2 - exit 52 - fi - if [ "$local_head" != "$github_head" ]; then - push_started=$(now_ms) - timeout 120 git -C "$repo_path" push "$repo_url" "$local_head:refs/heads/$gitops_branch" - emit_timing "push-$gitops_branch" succeeded "$push_started" - else - echo "git mirror flush $gitops_branch already matches GitHub" - fi - git -C "$repo_path" update-ref "refs/mirror-stage/heads/$gitops_branch" "$local_head" -done -git -C "$repo_path" update-server-info -flushed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) -export flushed_at -export gitops_branches_json=${shellSingleQuote(gitopsBranchesJson)} -node - <<'NODE' | tee /cache/HWLAB.last-flush.json -const { execFileSync } = require("node:child_process"); -const repo = "/cache/pikasTech/HWLAB.git"; -const gitopsBranches = JSON.parse(process.env.gitops_branches_json || "[]"); -function rev(ref) { - try { return execFileSync("git", ["-C", repo, "rev-parse", ref], { encoding: "utf8" }).trim() || null; } catch { return null; } -} -const refs = {}; -for (const branch of gitopsBranches) { - refs["refs/heads/" + branch] = rev("refs/heads/" + branch); - refs["refs/mirror-stage/heads/" + branch] = rev("refs/mirror-stage/heads/" + branch); -} -const pendingFlush = gitopsBranches.some((branch) => refs["refs/heads/" + branch] && refs["refs/mirror-stage/heads/" + branch] && refs["refs/heads/" + branch] !== refs["refs/mirror-stage/heads/" + branch]); -console.log(JSON.stringify({ - event: "git-mirror-flush", - status: "flushed", - repository: "pikasTech/HWLAB", - flushedAt: process.env.flushed_at, - pendingFlush, - refs -})); -NODE -emit_timing total succeeded "$started_ms" -`, - "http.mjs": `import { createServer } from "node:http"; -import { spawn } from "node:child_process"; -import { createReadStream, statSync } from "node:fs"; -import path from "node:path"; -import { createGunzip, createInflate } from "node:zlib"; - -const cacheRoot = process.env.GIT_MIRROR_CACHE_ROOT || "/cache"; -const port = Number(process.env.PORT || 8080); - -createServer((req, res) => { - handle(req, res).catch((error) => { - if (!res.headersSent) res.writeHead(500, { "content-type": "application/json" }); - res.end(JSON.stringify({ ok: false, error: error.message })); - }); -}).listen(port, "0.0.0.0", () => { - console.log(JSON.stringify({ event: "git-mirror-smart-http-listening", port, cacheRoot })); -}); - -async function handle(req, res) { - const url = new URL(req.url || "/", "http://git-mirror-http"); - const writeHost = isWriteHost(req.headers.host || ""); - if (req.method === "GET" && (url.pathname === "/" || url.pathname === "/health")) { - res.writeHead(200, { "content-type": "application/json", "cache-control": "no-cache" }); - res.end(JSON.stringify({ ok: true, service: "git-mirror-smart-http", writeHost })); - return; - } - if (req.method === "GET" && url.pathname.endsWith("/info/refs") && url.searchParams.get("service") === "git-upload-pack") { - await runUploadPackAdvertisement(repoPathFromGitServicePath(url.pathname, "/info/refs"), res); - return; - } - if (req.method === "POST" && url.pathname.endsWith("/git-upload-pack")) { - await runUploadPackRpc(repoPathFromGitServicePath(url.pathname, "/git-upload-pack"), req, res); - return; - } - if (req.method === "GET" && url.pathname.endsWith("/info/refs") && url.searchParams.get("service") === "git-receive-pack") { - if (!writeHost) return rejectReadOnly(res); - await runReceivePackAdvertisement(repoPathFromGitServicePath(url.pathname, "/info/refs"), res); - return; - } - if (req.method === "POST" && url.pathname.endsWith("/git-receive-pack")) { - if (!writeHost) return rejectReadOnly(res); - await runReceivePackRpc(repoPathFromGitServicePath(url.pathname, "/git-receive-pack"), req, res); - return; - } - if (req.method === "GET" || req.method === "HEAD") { - serveStaticFile(url.pathname, req, res); - return; - } - res.writeHead(405, { "content-type": "text/plain" }); - res.end("method not allowed\\n"); -} - -function isWriteHost(hostHeader) { - const host = String(hostHeader || "").split(":", 1)[0].toLowerCase(); - return host === "git-mirror-write" || host.startsWith("git-mirror-write."); -} - -function rejectReadOnly(res) { - res.writeHead(403, { "content-type": "application/json", "cache-control": "no-cache" }); - res.end(JSON.stringify({ ok: false, error: "git mirror receive-pack is only available through git-mirror-write" })); -} - -function repoPathFromGitServicePath(urlPath, suffix) { - const repoUrlPath = urlPath.slice(0, -suffix.length); - if (!repoUrlPath.endsWith(".git")) throw new Error(` + "`" + `unsupported git repo path: \${urlPath}` + "`" + `); - return safePath(repoUrlPath); -} - -function safePath(urlPath) { - const decoded = decodeURIComponent(urlPath.replace(/^[/]+/u, "")); - const fullPath = path.resolve(cacheRoot, decoded); - const normalizedRoot = path.resolve(cacheRoot); - if (fullPath !== normalizedRoot && !fullPath.startsWith(` + "`" + `\${normalizedRoot}\${path.sep}` + "`" + `)) throw new Error("path escapes cache root"); - return fullPath; -} - -function smartHeaders(contentType) { - return { - "content-type": contentType, - "cache-control": "no-cache, max-age=0, must-revalidate", - expires: "Fri, 01 Jan 1980 00:00:00 GMT", - pragma: "no-cache" - }; -} - -function pktLine(text) { - const length = Buffer.byteLength(text) + 4; - return ` + "`" + `\${length.toString(16).padStart(4, "0")}\${text}` + "`" + `; -} - -function requestBodyStream(req) { - const encoding = String(req.headers["content-encoding"] || "identity").toLowerCase(); - if (encoding === "identity" || encoding === "") return req; - if (encoding === "gzip" || encoding === "x-gzip") return req.pipe(createGunzip()); - if (encoding === "deflate") return req.pipe(createInflate()); - throw new Error(` + "`" + `unsupported git upload-pack content-encoding: \${encoding}` + "`" + `); -} - -function runUploadPackAdvertisement(repoPath, res) { - return new Promise((resolve, reject) => { - const child = spawn("git-upload-pack", ["--stateless-rpc", "--advertise-refs", repoPath], { stdio: ["ignore", "pipe", "pipe"] }); - let stderr = ""; - child.stderr.on("data", (chunk) => { stderr += chunk; process.stderr.write(chunk); }); - child.on("error", reject); - child.on("close", (code) => { - if (code !== 0 && !res.headersSent) reject(new Error(` + "`" + `git-upload-pack advertise failed: \${stderr.trim()}` + "`" + `)); - else resolve(); - }); - res.writeHead(200, smartHeaders("application/x-git-upload-pack-advertisement")); - res.write(pktLine("# service=git-upload-pack\\n")); - res.write("0000"); - child.stdout.pipe(res, { end: false }); - child.stdout.on("end", () => res.end()); - }); -} - -function runUploadPackRpc(repoPath, req, res) { - return new Promise((resolve, reject) => { - const child = spawn("git-upload-pack", ["--stateless-rpc", repoPath], { stdio: ["pipe", "pipe", "pipe"] }); - let stderr = ""; - child.stderr.on("data", (chunk) => { stderr += chunk; process.stderr.write(chunk); }); - child.on("error", reject); - child.on("close", (code) => { - if (code !== 0 && !res.headersSent) reject(new Error(` + "`" + `git-upload-pack rpc failed: \${stderr.trim()}` + "`" + `)); - else resolve(); - }); - res.writeHead(200, smartHeaders("application/x-git-upload-pack-result")); - const body = requestBodyStream(req); - body.on("error", (error) => child.stdin.destroy(error)); - body.pipe(child.stdin); - child.stdout.pipe(res); - }); -} - -function runReceivePackAdvertisement(repoPath, res) { - return new Promise((resolve, reject) => { - const child = spawn("git-receive-pack", ["--stateless-rpc", "--advertise-refs", repoPath], { stdio: ["ignore", "pipe", "pipe"] }); - let stderr = ""; - child.stderr.on("data", (chunk) => { stderr += chunk; process.stderr.write(chunk); }); - child.on("error", reject); - child.on("close", (code) => { - if (code !== 0 && !res.headersSent) reject(new Error(` + "`" + `git-receive-pack advertise failed: \${stderr.trim()}` + "`" + `)); - else resolve(); - }); - res.writeHead(200, smartHeaders("application/x-git-receive-pack-advertisement")); - res.write(pktLine("# service=git-receive-pack\\n")); - res.write("0000"); - child.stdout.pipe(res, { end: false }); - child.stdout.on("end", () => res.end()); - }); -} - -function runReceivePackRpc(repoPath, req, res) { - return new Promise((resolve, reject) => { - const child = spawn("git-receive-pack", ["--stateless-rpc", repoPath], { stdio: ["pipe", "pipe", "pipe"] }); - let stderr = ""; - child.stderr.on("data", (chunk) => { stderr += chunk; process.stderr.write(chunk); }); - child.on("error", reject); - child.on("close", (code) => { - if (code !== 0 && !res.headersSent) reject(new Error(` + "`" + `git-receive-pack rpc failed: \${stderr.trim()}` + "`" + `)); - else resolve(); - }); - res.writeHead(200, smartHeaders("application/x-git-receive-pack-result")); - const body = requestBodyStream(req); - body.on("error", (error) => child.stdin.destroy(error)); - body.pipe(child.stdin); - child.stdout.pipe(res); - }); -} - -function serveStaticFile(urlPath, req, res) { - const fullPath = safePath(urlPath); - let stat; - try { stat = statSync(fullPath); } catch { - res.writeHead(404, { "content-type": "text/plain" }); - res.end("not found\\n"); - return; - } - if (!stat.isFile()) { - res.writeHead(404, { "content-type": "text/plain" }); - res.end("not found\\n"); - return; - } - res.writeHead(200, { "content-length": stat.size, "content-type": contentType(fullPath), "cache-control": "no-cache" }); - if (req.method === "HEAD") { - res.end(); - return; - } - createReadStream(fullPath).pipe(res); -} - -function contentType(filePath) { - if (filePath.endsWith(".pack")) return "application/x-git-packed-objects"; - if (filePath.endsWith(".idx")) return "application/x-git-packed-objects-toc"; - return "text/plain"; -} -` - } - }, - { - apiVersion: "apps/v1", - kind: "Deployment", - metadata: { name: gitMirrorReadName, namespace: gitMirrorNamespace, labels: readLabels }, - spec: { - replicas: 1, - selector: { matchLabels: { "app.kubernetes.io/name": gitMirrorReadName } }, - template: { - metadata: { labels: readLabels }, - spec: { - containers: [{ - name: "git-mirror", - image: ciToolsRunnerImage, - imagePullPolicy: "IfNotPresent", - command: ["node"], - args: ["/etc/git-mirror/http.mjs"], - ports: [{ name: "http", containerPort: 8080 }], - readinessProbe: { httpGet: { path: "/pikasTech/HWLAB.git/HEAD", port: "http" }, initialDelaySeconds: 5, periodSeconds: 10 }, - livenessProbe: { httpGet: { path: "/", port: "http" }, initialDelaySeconds: 10, periodSeconds: 30 }, - volumeMounts: [ - { name: "cache", mountPath: "/cache" }, - { name: "config", mountPath: "/etc/git-mirror", readOnly: true } - ] - }], - volumes: [ - cacheVolume, - { name: "config", configMap: { name: gitMirrorConfigMapName, defaultMode: 0o555 } } - ] - } - } - } - }, - { - apiVersion: "v1", - kind: "Service", - metadata: { name: gitMirrorReadName, namespace: gitMirrorNamespace, labels: readLabels }, - spec: { selector: { "app.kubernetes.io/name": gitMirrorReadName }, ports: [{ name: "http", port: gitMirrorServicePort, targetPort: "http" }] } - }, - { - apiVersion: "v1", - kind: "Service", - metadata: { name: gitMirrorWriteName, namespace: gitMirrorNamespace, labels: writeLabels }, - spec: { selector: { "app.kubernetes.io/name": gitMirrorReadName }, ports: [{ name: "http", port: gitMirrorServicePort, targetPort: "http" }] } - } - ] - }; -} - -function argoProject(args = { lane: "node" }) { - if (isRuntimeLane(args.lane)) { - const namespace = namespaceNameForProfile(args.lane); - return { - apiVersion: "argoproj.io/v1alpha1", - kind: "AppProject", - metadata: { name: namespace, namespace: "argocd" }, - spec: { - description: `HWLAB ${versionNameForRuntimeLane(args.lane)} additive GitOps project on target node; DEV/PROD remain outside this lane.`, - sourceRepos: [args.gitReadUrl], - destinations: [{ server: "https://kubernetes.default.svc", namespace }], - clusterResourceWhitelist: [{ group: "", kind: "Namespace" }], - namespaceResourceWhitelist: [{ group: "*", kind: "*" }] - } - }; - } - return { - apiVersion: "argoproj.io/v1alpha1", - kind: "AppProject", - metadata: { name: "hwlab-node", namespace: "argocd" }, - spec: { - description: "HWLAB node GitOps project; D601 remains outside this project.", - sourceRepos: [defaultSourceRepo], - destinations: [ - { server: "https://kubernetes.default.svc", namespace: "hwlab-dev" }, - { server: "https://kubernetes.default.svc", namespace: "hwlab-prod" } - ], - clusterResourceWhitelist: [{ group: "", kind: "Namespace" }], - namespaceResourceWhitelist: [{ group: "*", kind: "*" }] - } - }; -} - -function argoApplication(args, profile = "dev") { - const namespace = namespaceNameForProfile(profile); - const runtimePath = runtimePathForProfile(profile); - const gitopsTarget = gitopsTargetForProfile(profile); - const project = isRuntimeLane(profile) ? namespace : "hwlab-node"; - const repoURL = isRuntimeLane(profile) ? args.gitReadUrl : args.sourceRepo; - return { - apiVersion: "argoproj.io/v1alpha1", - kind: "Application", - metadata: { - name: argoApplicationName(profile), - namespace: "argocd", - labels: { "app.kubernetes.io/part-of": "hwlab", "hwlab.pikastech.local/gitops-target": gitopsTarget } - }, - spec: { - project, - source: { repoURL, targetRevision: args.gitopsBranch, path: gitopsPathForProfile(args, profile) }, - destination: { server: "https://kubernetes.default.svc", namespace }, - syncPolicy: { automated: { prune: isRuntimeLane(profile), selfHeal: true }, syncOptions: ["CreateNamespace=true", "ApplyOutOfSyncOnly=true"] } - } - }; -} - -function nodeFrpcManifest({ profile = "dev", source = null, deploy = null, args = {} } = {}) { - const namespace = namespaceNameForProfile(profile); - const profileLabel = runtimeLabelForProfile(profile); - const deploymentName = isRuntimeLane(profile) ? `hwlab-${profile}-frpc` : profile === "prod" ? "hwlab-node-prod-frpc" : "hwlab-node-frpc"; - const configName = `${deploymentName}-config`; - const proxyPrefix = isRuntimeLane(profile) ? `hwlab-${profile}` : profile === "prod" ? "hwlab-node-prod" : "hwlab-node"; - const frpConfig = runtimeFrpConfigForProfile(deploy, profile, args); - const webProxyName = frpConfig.webProxyName || `${proxyPrefix}-cloud-web`; - const edgeProxyName = frpConfig.edgeProxyName || `${proxyPrefix}-edge-proxy`; - const labels = { - "app.kubernetes.io/name": deploymentName, - "app.kubernetes.io/part-of": "hwlab", - "hwlab.pikastech.local/environment": profileLabel, - "hwlab.pikastech.local/gitops-target": gitopsTargetForProfile(profile), - "hwlab.pikastech.local/profile": profileLabel - }; - const sourceCommitLabel = source?.full ? { "hwlab.pikastech.local/source-commit": source.full } : {}; - const renderedLabels = { ...labels, ...sourceCommitLabel }; - const podNetwork = isRuntimeLane(profile) ? { - hostNetwork: true, - dnsPolicy: "ClusterFirstWithHostNet" - } : {}; - const authConfigText = frpConfig.authSecretName && frpConfig.authSecretKey ? `auth.token = "{{ .Envs.FRPC_AUTH_TOKEN }}"\n` : ""; - const configText = `serverAddr = "${frpConfig.serverAddr}" -serverPort = ${frpConfig.serverPort} -loginFailExit = true -${authConfigText} - -[[proxies]] -name = "${webProxyName}" -type = "tcp" -localIP = "hwlab-cloud-web.${namespace}.svc.cluster.local" -localPort = 8080 -remotePort = ${frpConfig.webRemotePort} - -[[proxies]] -name = "${edgeProxyName}" -type = "tcp" -localIP = "hwlab-edge-proxy.${namespace}.svc.cluster.local" -localPort = 6667 -remotePort = ${frpConfig.edgeRemotePort} -`; - const configSha256 = createHash("sha256").update(configText).digest("hex"); - const templateLabels = { ...labels, "hwlab.pikastech.local/config-sha256": k8sLabelHash(configSha256) }; - const templateAnnotations = { "hwlab.pikastech.local/config-sha256": configSha256 }; - return { - apiVersion: "v1", - kind: "List", - items: [ - { - apiVersion: "v1", - kind: "ConfigMap", - metadata: { - name: configName, - namespace, - labels: renderedLabels - }, - data: { - "frpc.toml": configText - } - }, - { - apiVersion: "apps/v1", - kind: "Deployment", - metadata: { - name: deploymentName, - namespace, - labels: renderedLabels - }, - spec: { - replicas: 1, - selector: { matchLabels: { "app.kubernetes.io/name": deploymentName } }, - template: { - metadata: { labels: templateLabels, annotations: templateAnnotations }, - spec: { - ...podNetwork, - containers: [{ - name: "frpc", - image: "fatedier/frpc:v0.68.1", - imagePullPolicy: "IfNotPresent", - ...(frpConfig.authSecretName && frpConfig.authSecretKey ? { env: [{ name: "FRPC_AUTH_TOKEN", valueFrom: { secretKeyRef: { name: frpConfig.authSecretName, key: frpConfig.authSecretKey } } }] } : {}), - args: ["-c", "/etc/frp/frpc.toml"], - volumeMounts: [{ name: "config", mountPath: "/etc/frp", readOnly: true }] - }], - volumes: [{ name: "config", configMap: { name: configName } }] - } - } - } - } - ] - }; -} - -function deepSeekProxyManifest({ profile = "dev", source, sourceBranch = defaultBranch, sourceRepo = defaultSourceRepo, gitReadUrl, deploy = null, registryPrefix = defaultRegistryPrefix, catalog = null, useDeployImages = false, metricsSidecarSha256 = null } = {}) { - const namespace = namespaceNameForProfile(profile); - const bridgeServiceId = "hwlab-cloud-api"; - const runtimeLane = isRuntimeLane(profile); - const digestPin = runtimeLane; - const envReuseServiceIds = runtimeLane ? envReuseServiceIdsForLane(deploy, profile) : null; - const bridgeImage = runtimeImageForService({ catalog, deploy, serviceId: bridgeServiceId, source, registryPrefix, useDeployImages, digestPin, envReuseServiceIds }); - const bridgeCommit = runtimeCommitForService({ catalog, deploy, serviceId: bridgeServiceId, source, registryPrefix, useDeployImages, digestPin, envReuseServiceIds }); - const bridgeImageTag = runtimeImageTagForService({ catalog, deploy, serviceId: bridgeServiceId, source, registryPrefix, useDeployImages, digestPin, envReuseServiceIds }); - const bridgeBootMetadata = runtimeLane && v02EnvReuseEnabled(catalog, bridgeServiceId, envReuseServiceIds) - ? bootMetadataForService({ args: { sourceRepo: deploy?.lanes?.[profile]?.sourceRepo || sourceRepo, registryPrefix }, catalog, deployService: deployServiceForBoot(deploy, bridgeServiceId, profile), serviceId: bridgeServiceId, source }) - : null; - const labels = { - "app.kubernetes.io/name": "hwlab-deepseek-proxy", - "app.kubernetes.io/part-of": "hwlab", - "hwlab.pikastech.local/environment": runtimeLabelForProfile(profile), - "hwlab.pikastech.local/gitops-target": gitopsTargetForProfile(profile), - "hwlab.pikastech.local/service-id": "hwlab-deepseek-proxy", - "hwlab.pikastech.local/source-commit": source.full - }; - if (runtimeLane) Object.assign(labels, v02MetricsLabels("hwlab-deepseek-proxy")); - const templateLabels = { - ...labels, - "hwlab.pikastech.local/source-commit": bridgeCommit, - "hwlab.pikastech.local/bridge-source-commit": bridgeCommit - }; - const templateAnnotations = { - "hwlab.pikastech.local/bridge-service-id": bridgeServiceId, - "hwlab.pikastech.local/source-commit": bridgeCommit, - "hwlab.pikastech.local/bridge-source-commit": bridgeCommit, - "hwlab.pikastech.local/bridge-image-tag": bridgeImageTag, - "hwlab.pikastech.local/bridge-image": bridgeImage, - ...v02MetricsSidecarAnnotations(runtimeLane ? metricsSidecarSha256 : null) - }; - const responsesBridgeContainer = { - name: "responses-bridge", - image: bridgeImage, - imagePullPolicy: "IfNotPresent", - command: ["/usr/local/bin/bun", "run", "/app/cmd/hwlab-deepseek-responses-bridge/main.ts"], - env: [ - { name: "PORT", value: "4000" }, - { name: "HWLAB_COMMIT_ID", value: bridgeCommit }, - { name: "HWLAB_IMAGE", value: bridgeImage }, - { name: "HWLAB_IMAGE_TAG", value: bridgeImageTag }, - { name: "HWLAB_DEEPSEEK_BRIDGE_UPSTREAM", value: "http://127.0.0.1:4001" }, - { name: "HWLAB_DEEPSEEK_BRIDGE_MODEL", value: deepSeekProfileModel } - ], - ports: [{ name: "http", containerPort: 4000 }], - readinessProbe: { httpGet: { path: "/health/readiness", port: "http" }, initialDelaySeconds: 10, periodSeconds: 10 }, - livenessProbe: { httpGet: { path: "/health/liveliness", port: "http" }, initialDelaySeconds: 20, periodSeconds: 20 } - }; - if (bridgeBootMetadata) { - applyEnvReuseBootEnv(responsesBridgeContainer, bridgeBootMetadata, { sourceBranch, gitReadUrl, bootSh: "deploy/runtime/boot/hwlab-deepseek-responses-bridge.sh" }); - Object.assign(templateAnnotations, { - "hwlab.pikastech.local/bridge-runtime-mode": bridgeBootMetadata.runtimeMode, - "hwlab.pikastech.local/bridge-boot-repo": bridgeBootMetadata.bootRepo, - "hwlab.pikastech.local/bridge-boot-commit": bridgeBootMetadata.bootCommit, - "hwlab.pikastech.local/bridge-boot-sh": "deploy/runtime/boot/hwlab-deepseek-responses-bridge.sh", - "hwlab.pikastech.local/bridge-environment-digest": bridgeBootMetadata.environmentDigest ?? "not_published" - }); - Object.assign(templateLabels, { - "hwlab.pikastech.local/bridge-runtime-mode": bridgeBootMetadata.runtimeMode, - "hwlab.pikastech.local/bridge-boot-commit": bridgeBootMetadata.bootCommit - }); - } - return { - apiVersion: "v1", - kind: "List", - items: [ - { - apiVersion: "v1", - kind: "ConfigMap", - metadata: { name: "hwlab-deepseek-proxy-config", namespace, labels }, - data: { - "render-config.sh": moonBridgeConfigRenderScript() - } - }, - { - apiVersion: "apps/v1", - kind: "Deployment", - metadata: { name: "hwlab-deepseek-proxy", namespace, labels, annotations: { "hwlab.pikastech.local/moonbridge-source-ref": moonBridgeSourceRef, ...templateAnnotations } }, - spec: { - replicas: 1, - selector: { matchLabels: { "app.kubernetes.io/name": "hwlab-deepseek-proxy" } }, - template: { - metadata: { labels: templateLabels, annotations: templateAnnotations }, - spec: { - initContainers: [{ - name: "moonbridge-config", - image: bridgeImage, - imagePullPolicy: "IfNotPresent", - command: ["/bin/sh", "-eu", "/scripts/render-config.sh"], - env: [ - { name: "DEEPSEEK_API_KEY", valueFrom: { secretKeyRef: { name: secretNameForProfile("hwlab-code-agent-provider", profile), key: "openai-api-key", optional: true } } }, - { name: "OPENCODE_API_KEY", valueFrom: { secretKeyRef: { name: secretNameForProfile("hwlab-code-agent-provider", profile), key: "opencode-api-key", optional: true } } } - ], - volumeMounts: [ - { name: "moonbridge-scripts", mountPath: "/scripts", readOnly: true }, - { name: "moonbridge-config", mountPath: "/config" } - ] - }], - containers: [responsesBridgeContainer, { - name: "moonbridge", - image: moonBridgeImage, - imagePullPolicy: "IfNotPresent", - args: ["-config", "/config/config.yml"], - ports: [{ name: "moonbridge-http", containerPort: 4001 }], - readinessProbe: { httpGet: { path: "/v1/models", port: "moonbridge-http" }, initialDelaySeconds: 10, periodSeconds: 10 }, - livenessProbe: { httpGet: { path: "/v1/models", port: "moonbridge-http" }, initialDelaySeconds: 20, periodSeconds: 20 }, - volumeMounts: [ - { name: "moonbridge-config", mountPath: "/config", readOnly: true }, - { name: "moonbridge-data", mountPath: "/data" } - ] - }, ...(runtimeLane ? [v02MetricsSidecarContainer({ deploy, serviceId: "hwlab-deepseek-proxy", namespace, gitopsTarget: gitopsTargetForProfile(profile) })] : [])], - volumes: [ - { name: "moonbridge-scripts", configMap: { name: "hwlab-deepseek-proxy-config" } }, - { name: "moonbridge-config", emptyDir: {} }, - { name: "moonbridge-data", emptyDir: {} }, - ...(runtimeLane ? [v02MetricsSidecarVolume(profile)] : []) - ] - } - } - } - }, - { - apiVersion: "v1", - kind: "Service", - metadata: { name: "hwlab-deepseek-proxy", namespace, labels }, - spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": "hwlab-deepseek-proxy" }, ports: [{ name: "http", port: 4000, targetPort: "http" }, ...(runtimeLane ? [{ name: "metrics", port: 9100, targetPort: "metrics" }] : [])] } - } - ] - }; -} - -function moonBridgeConfigRenderScript() { - return `#!/bin/sh -if [ -z "\${DEEPSEEK_API_KEY:-}" ] && [ -z "\${OPENCODE_API_KEY:-}" ]; then - echo "at least one of DEEPSEEK_API_KEY or OPENCODE_API_KEY is required" >&2 - exit 1 -fi -cat > /config/config.yml < {", - " let body = \"\";", - " req.setEncoding(\"utf8\");", - " req.on(\"data\", (chunk) => {", - " body += chunk;", - " if (body.length > 1024 * 1024) req.destroy(new Error(\"request body too large\"));", - " });", - " req.on(\"end\", () => resolve(body ? JSON.parse(body) : {}));", - " req.on(\"error\", reject);", - " });", - "}", - "", - "function postJson(url, payload, timeoutMs) {", - " return new Promise((resolve, reject) => {", - " const parsed = new URL(url);", - " const body = JSON.stringify(payload);", - " const request = http.request({", - " hostname: parsed.hostname,", - " port: parsed.port || 80,", - " path: parsed.pathname + parsed.search,", - " method: \"POST\",", - " headers: { \"content-type\": \"application/json\", \"content-length\": Buffer.byteLength(body) },", - " timeout: timeoutMs", - " }, (response) => {", - " let responseBody = \"\";", - " response.setEncoding(\"utf8\");", - " response.on(\"data\", (chunk) => { responseBody += chunk; });", - " response.on(\"end\", () => {", - " try {", - " const parsedBody = responseBody ? JSON.parse(responseBody) : null;", - " resolve({ statusCode: response.statusCode, body: parsedBody });", - " } catch (error) {", - " reject(error);", - " }", - " });", - " });", - " request.on(\"timeout\", () => request.destroy(new Error(\"request timeout after \" + timeoutMs + \"ms\")));", - " request.on(\"error\", reject);", - " request.end(body);", - " });", - "}", - "", - "function gatewayPayload(input) {", - " const traceId = input.traceId || \"trc_device_agent_71_freq_\" + Date.now();", - " return {", - " jsonrpc: \"2.0\",", - " id: input.id || \"req_device_agent_71_freq_\" + Date.now(),", - " method: \"hardware.invoke.shell\",", - " meta: { serviceId: \"hwlab-cloud-api\", actorId: \"device-agent-71-freq\", environment: \"dev\", traceId, deviceId, workspaceRoot },", - " params: {", - " gatewaySessionId,", - " resourceId,", - " capabilityId,", - " input: {", - " command: input.command,", - " cwd: input.cwd || workspaceRoot,", - " timeoutMs: input.timeoutMs || 30000", - " }", - " }", - " };", - "}", - "", - "const server = http.createServer(async (req, res) => {", - " try {", - " const url = new URL(req.url || \"/\", \"http://device-agent-71-freq\");", - " if (req.method === \"GET\" && url.pathname === \"/health\") {", - " return sendJson(res, 200, { ok: true, deviceId, workspaceRoot, gatewaySessionId, resourceId, capabilityId, cloudApiUrl });", - " }", - " if (req.method === \"GET\" && url.pathname === \"/skills\") {", - " return sendJson(res, 200, { ok: true, deviceId, skills: [{ name: \"cmd\", description: \"Run a bounded Windows cmd command through hwlab-gateway.\" }] });", - " }", - " if (req.method === \"POST\" && url.pathname === \"/run\") {", - " const input = await readBody(req);", - " if (!input.command || typeof input.command !== \"string\") return sendJson(res, 400, { ok: false, error: \"command is required\" });", - " const timeoutMs = Number(input.timeoutMs || 30000);", - " const upstream = await postJson(cloudApiUrl + \"/json-rpc\", gatewayPayload({ ...input, timeoutMs }), timeoutMs + 5000);", - " return sendJson(res, upstream.statusCode || 502, { ok: upstream.statusCode >= 200 && upstream.statusCode < 300, deviceId, gatewaySessionId, upstream: upstream.body });", - " }", - " return sendJson(res, 404, { ok: false, error: \"not found\" });", - " } catch (error) {", - " return sendJson(res, 500, { ok: false, error: error.message });", - " }", - "});", - "", - "server.listen(port, \"0.0.0.0\", () => {", - " console.log(JSON.stringify({ ok: true, service: \"device-agent-71-freq\", port, deviceId, workspaceRoot, cloudApiUrl }));", - "});" - ].join("\n") + "\n"; -} - -function deviceAgent71FreqManifest({ profile = "dev", source, registryPrefix, catalog = null, useDeployImages = false }) { - assert.equal(profile, "dev", "71-freq device-agent is dev-only"); - const namespace = namespaceNameForProfile(profile); - const name = "device-agent-71-freq"; - const bridgeServiceId = "hwlab-cloud-api"; - const bridgeImage = runtimeImageForService({ catalog, serviceId: bridgeServiceId, source, registryPrefix, useDeployImages }); - const bridgeCommit = runtimeCommitForService({ catalog, serviceId: bridgeServiceId, source, registryPrefix, useDeployImages }); - const bridgeImageTag = runtimeImageTagForService({ catalog, serviceId: bridgeServiceId, source, registryPrefix, useDeployImages }); - const labels = { - "app.kubernetes.io/name": name, - "app.kubernetes.io/part-of": "hwlab", - "hwlab.pikastech.local/component": "device-agent", - "hwlab.pikastech.local/device-id": "71-freq", - "hwlab.pikastech.local/environment": runtimeLabelForProfile(profile), - "hwlab.pikastech.local/gitops-target": "node", - "hwlab.pikastech.local/profile": runtimeLabelForProfile(profile), - "hwlab.pikastech.local/service-id": name, - "hwlab.pikastech.local/source-commit": source.full - }; - const annotations = { "hwlab.pikastech.local/rendered-by": "scripts/gitops-render.mjs" }; - const selector = { "app.kubernetes.io/name": name }; - const script = deviceAgent71FreqServerScript(); - const scriptSha256 = createHash("sha256").update(script).digest("hex"); - const templateLabels = { ...labels, ...selector, "hwlab.pikastech.local/source-commit": bridgeCommit }; - const templateAnnotations = { - ...annotations, - "hwlab.pikastech.local/script-sha256": scriptSha256, - "hwlab.pikastech.local/bridge-service-id": bridgeServiceId, - "hwlab.pikastech.local/bridge-source-commit": bridgeCommit, - "hwlab.pikastech.local/bridge-image-tag": bridgeImageTag, - "hwlab.pikastech.local/bridge-image": bridgeImage - }; - return { - apiVersion: "v1", - kind: "List", - items: [ - { - apiVersion: "v1", - kind: "ConfigMap", - metadata: { name: `${name}-script`, namespace, labels, annotations: templateAnnotations }, - data: { "server.mjs": script } - }, - { - apiVersion: "apps/v1", - kind: "Deployment", - metadata: { name, namespace, labels, annotations }, - spec: { - replicas: 1, - selector: { matchLabels: selector }, - template: { - metadata: { labels: templateLabels, annotations: templateAnnotations }, - spec: { - containers: [{ - name: "device-agent", - image: bridgeImage, - imagePullPolicy: "IfNotPresent", - command: ["node", "/opt/device-agent/server.mjs"], - env: [ - { name: "PORT", value: "7601" }, - { name: "HWLAB_COMMIT_ID", value: bridgeCommit }, - { name: "HWLAB_IMAGE", value: bridgeImage }, - { name: "HWLAB_IMAGE_TAG", value: bridgeImageTag }, - { name: "DEVICE_ID", value: "71-freq" }, - { name: "DEVICE_WORKSPACE_ROOT", value: "F:\\Work\\ConStart" }, - { name: "HWLAB_CLOUD_API_URL", value: `http://hwlab-cloud-api.${namespace}.svc.cluster.local:6667` }, - { name: "HWLAB_GATEWAY_SESSION_ID", value: "gws_d601_win_71_freq" }, - { name: "HWLAB_GATEWAY_RESOURCE_ID", value: "res_d601_windows_host" }, - { name: "HWLAB_GATEWAY_CAPABILITY_ID", value: "cap_d601_windows_cmd_exec" } - ], - ports: [{ name: "http", containerPort: 7601 }], - readinessProbe: { httpGet: { path: "/health", port: "http" }, initialDelaySeconds: 3, periodSeconds: 10 }, - livenessProbe: { httpGet: { path: "/health", port: "http" }, initialDelaySeconds: 10, periodSeconds: 20 }, - resources: { requests: { cpu: "10m", memory: "64Mi" }, limits: { cpu: "200m", memory: "256Mi" } }, - volumeMounts: [ - { name: "script", mountPath: "/opt/device-agent", readOnly: true }, - { name: "hwlab-code-agent-workspace", mountPath: "/workspace" } - ] - }], - volumes: [ - { name: "script", configMap: { name: `${name}-script` } }, - { name: "hwlab-code-agent-workspace", persistentVolumeClaim: { claimName: "hwlab-code-agent-workspace" } } - ] - } - } - } - }, - { - apiVersion: "v1", - kind: "Service", - metadata: { name, namespace, labels, annotations }, - spec: { type: "ClusterIP", selector, ports: [{ name: "http", port: 7601, targetPort: "http" }] } - } - ] - }; -} - -function runtimePostgresImageForProfile(deploy, profile) { - const postgres = runtimeLaneConfig(deploy, profile)?.runtimeStore?.postgres; - const image = postgres && typeof postgres === "object" && !Array.isArray(postgres) ? postgres.image : null; - return typeof image === "string" && image.trim().length > 0 ? image.trim() : "postgres:16-alpine"; -} - -function v02PostgresManifest({ profile = "v02", migrationSql, source, image = "postgres:16-alpine" }) { - const namespace = namespaceNameForProfile(profile); - const name = `${namespace}-postgres`; - const dbName = `hwlab_${profile}`; - const labels = { - "app.kubernetes.io/name": name, - "app.kubernetes.io/part-of": "hwlab", - "hwlab.pikastech.local/environment": profile, - "hwlab.pikastech.local/gitops-target": profile, - "hwlab.pikastech.local/profile": profile, - "hwlab.pikastech.local/source-commit": source.full - }; - const migrationSha256 = createHash("sha256").update(migrationSql).digest("hex"); - const templateLabels = { - "app.kubernetes.io/name": name, - "app.kubernetes.io/part-of": "hwlab", - "hwlab.pikastech.local/environment": profile, - "hwlab.pikastech.local/gitops-target": profile, - "hwlab.pikastech.local/profile": profile, - "hwlab.pikastech.local/migration-sha256": k8sLabelHash(migrationSha256) - }; - const templateAnnotations = { "hwlab.pikastech.local/migration-sha256": migrationSha256 }; - return { - apiVersion: "v1", - kind: "List", - items: [ - { - apiVersion: "v1", - kind: "ConfigMap", - metadata: { name: `${name}-init`, namespace, labels }, - data: { "0001_cloud_core_skeleton.sql": migrationSql } - }, - { - apiVersion: "v1", - kind: "Service", - metadata: { name, namespace, labels }, - spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": name }, ports: [{ name: "postgres", port: 5432, targetPort: "postgres" }] } - }, - { - apiVersion: "apps/v1", - kind: "StatefulSet", - metadata: { name, namespace, labels }, - spec: { - serviceName: name, - replicas: 1, - selector: { matchLabels: { "app.kubernetes.io/name": name } }, - template: { - metadata: { labels: templateLabels, annotations: templateAnnotations }, - spec: { - containers: [{ - name: "postgres", - image, - imagePullPolicy: "IfNotPresent", - env: [ - { name: "POSTGRES_DB", value: dbName }, - { name: "POSTGRES_USER", value: dbName }, - { name: "POSTGRES_PASSWORD", valueFrom: { secretKeyRef: { name, key: "POSTGRES_PASSWORD" } } } - ], - ports: [{ name: "postgres", containerPort: 5432 }], - readinessProbe: { tcpSocket: { port: "postgres" }, initialDelaySeconds: 5, periodSeconds: 10 }, - livenessProbe: { tcpSocket: { port: "postgres" }, initialDelaySeconds: 30, periodSeconds: 20 }, - volumeMounts: [ - { name: "data", mountPath: "/var/lib/postgresql/data" }, - { name: "init", mountPath: "/docker-entrypoint-initdb.d", readOnly: true } - ] - }], - volumes: [{ name: "init", configMap: { name: `${name}-init` } }] - } - }, - volumeClaimTemplates: [{ - metadata: { name: "data" }, - spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } - }] - } - } - ] - }; -} - -function externalPostgresConfigForLane(deploy, profile, args = {}) { - if (!isRuntimeLane(profile)) return null; - const config = deploy?.lanes?.[profile]?.externalPostgres; - if (!config || config.enabled !== true) return null; - const nodeId = effectiveSecretPlaneNodeId(args); - const nodeConfig = nodeId ? config?.nodes?.[nodeId] : null; - const effective = nodeConfig && typeof nodeConfig === "object" && !Array.isArray(nodeConfig) ? { ...config, ...nodeConfig } : config; - assert.ok(typeof effective.serviceName === "string" && effective.serviceName.length > 0, `deploy.lanes.${profile}.externalPostgres.serviceName is required`); - assert.ok(typeof effective.endpointAddress === "string" && effective.endpointAddress.length > 0, `deploy.lanes.${profile}.externalPostgres.endpointAddress is required`); - const port = Number(effective.port ?? 5432); - assert.ok(Number.isInteger(port) && port > 0 && port <= 65535, `deploy.lanes.${profile}.externalPostgres.port must be a valid TCP port`); - return { serviceName: effective.serviceName, endpointAddress: effective.endpointAddress, port }; -} - -function externalPostgresManifest({ profile = "v03", config, source }) { - const namespace = namespaceNameForProfile(profile); - const labels = { - "app.kubernetes.io/name": config.serviceName, - "app.kubernetes.io/part-of": "hwlab", - "hwlab.pikastech.local/component": "platform-db-bridge", - "hwlab.pikastech.local/environment": profile, - "hwlab.pikastech.local/gitops-target": profile, - "hwlab.pikastech.local/profile": profile, - "hwlab.pikastech.local/source-commit": source.full - }; - const annotations = { "hwlab.pikastech.local/rendered-by": "scripts/gitops-render.mjs" }; - return { - apiVersion: "v1", - kind: "List", - items: [ - { - apiVersion: "v1", - kind: "Service", - metadata: { name: config.serviceName, namespace, labels, annotations }, - spec: { type: "ClusterIP", ports: [{ name: "postgres", port: config.port, targetPort: config.port, protocol: "TCP" }] } - }, - { - apiVersion: "discovery.k8s.io/v1", - kind: "EndpointSlice", - metadata: { name: `${config.serviceName}-host`, namespace, labels: { ...labels, "kubernetes.io/service-name": config.serviceName }, annotations }, - addressType: "IPv4", - ports: [{ name: "postgres", port: config.port, protocol: "TCP" }], - endpoints: [{ addresses: [config.endpointAddress], conditions: { ready: true } }] - } - ] - }; -} - -function v02OpenFgaManifest({ profile = "v02", source }) { - const namespace = namespaceNameForProfile(profile); - const name = "hwlab-openfga"; - const image = "127.0.0.1:5000/hwlab/openfga:v1.17.0"; - const labels = { - "app.kubernetes.io/name": name, - "app.kubernetes.io/part-of": "hwlab", - "hwlab.pikastech.local/component": "authorization", - "hwlab.pikastech.local/environment": profile, - "hwlab.pikastech.local/gitops-target": profile, - "hwlab.pikastech.local/profile": profile, - "hwlab.pikastech.local/service-id": name, - "hwlab.pikastech.local/source-commit": source.full - }; - const annotations = { "hwlab.pikastech.local/rendered-by": "scripts/gitops-render.mjs" }; - const selector = { "app.kubernetes.io/name": name }; - const env = [ - { name: "OPENFGA_DATASTORE_ENGINE", value: "postgres" }, - { name: "OPENFGA_DATASTORE_URI", valueFrom: { secretKeyRef: { name: `hwlab-${profile}-openfga`, key: "datastore-uri" } } }, - { name: "OPENFGA_AUTHN_METHOD", value: "preshared" }, - { name: "OPENFGA_AUTHN_PRESHARED_KEYS", valueFrom: { secretKeyRef: { name: `hwlab-${profile}-openfga`, key: "authn-preshared-key" } } }, - { name: "OPENFGA_PLAYGROUND_ENABLED", value: "false" }, - { name: "OPENFGA_HTTP_ADDR", value: "0.0.0.0:8080" }, - { name: "OPENFGA_GRPC_ADDR", value: "0.0.0.0:8081" } - ]; - const templateLabels = { ...labels, ...selector }; - const runtimeAnnotations = { ...annotations, "argocd.argoproj.io/sync-wave": "2" }; - return { - apiVersion: "v1", - kind: "List", - items: [ - { - apiVersion: "batch/v1", - kind: "Job", - metadata: { - name: `${name}-migrate`, - namespace, - labels, - annotations: { - ...annotations, - "argocd.argoproj.io/hook": "Sync", - "argocd.argoproj.io/sync-wave": "1", - "argocd.argoproj.io/hook-delete-policy": "BeforeHookCreation,HookSucceeded" - } - }, - spec: { - backoffLimit: 3, - template: { - metadata: { labels: templateLabels, annotations }, - spec: { - restartPolicy: "OnFailure", - containers: [{ name: "openfga-migrate", image, imagePullPolicy: "IfNotPresent", args: ["migrate"], env }] - } - } - } - }, - { - apiVersion: "apps/v1", - kind: "Deployment", - metadata: { name, namespace, labels, annotations: runtimeAnnotations }, - spec: { - replicas: 1, - selector: { matchLabels: selector }, - template: { - metadata: { labels: templateLabels, annotations }, - spec: { - containers: [{ - name: "openfga", - image, - imagePullPolicy: "IfNotPresent", - args: ["run"], - env, - ports: [{ name: "http", containerPort: 8080 }, { name: "grpc", containerPort: 8081 }], - readinessProbe: { httpGet: { path: "/healthz", port: "http" }, initialDelaySeconds: 3, periodSeconds: 10 }, - livenessProbe: { httpGet: { path: "/healthz", port: "http" }, initialDelaySeconds: 15, periodSeconds: 20 }, - resources: { requests: { cpu: "50m", memory: "128Mi" }, limits: { cpu: "500m", memory: "512Mi" } } - }] - } - } - } - }, - { - apiVersion: "v1", - kind: "Service", - metadata: { name, namespace, labels, annotations: runtimeAnnotations }, - spec: { type: "ClusterIP", selector, ports: [{ name: "http", port: 8080, targetPort: "http" }, { name: "grpc", port: 8081, targetPort: "grpc" }] } - } - ] - }; -} - -function workbenchRuntimeRedisConf(config) { - return [ - "save \"\"", - "appendonly no", - `maxmemory ${config.memoryPolicy.maxMemory}`, - `maxmemory-policy ${config.memoryPolicy.eviction}`, - "" - ].join("\n"); -} - -function workbenchRuntimeRedisManifest({ profile = "v03", config, source }) { - const namespace = config.namespace; - const name = config.serviceName; - const port = config.port; - const selector = { "app.kubernetes.io/name": name }; - const labels = { - ...selector, - "app.kubernetes.io/part-of": "hwlab", - "hwlab.pikastech.local/cache-role": "workbench-derived-read", - "hwlab.pikastech.local/component": "workbench-runtime-cache", - "hwlab.pikastech.local/environment": profile, - "hwlab.pikastech.local/gitops-target": profile, - "hwlab.pikastech.local/profile": profile, - "hwlab.pikastech.local/service-id": name, - "hwlab.pikastech.local/source-commit": source.full - }; - const annotations = { "hwlab.pikastech.local/rendered-by": "scripts/gitops-render.mjs" }; - const runtimeAnnotations = { ...annotations, "argocd.argoproj.io/sync-wave": "2" }; - const templateLabels = { ...labels, ...selector }; - return { - apiVersion: "v1", - kind: "List", - items: [ - { - apiVersion: "v1", - kind: "ConfigMap", - metadata: { name: `${name}-config`, namespace, labels, annotations: runtimeAnnotations }, - data: { "redis.conf": workbenchRuntimeRedisConf(config) } - }, - { - apiVersion: "apps/v1", - kind: "Deployment", - metadata: { name, namespace, labels, annotations: runtimeAnnotations }, - spec: { - replicas: 1, - selector: { matchLabels: selector }, - template: { - metadata: { labels: templateLabels, annotations: runtimeAnnotations }, - spec: { - containers: [{ - name: "redis", - image: config.image, - imagePullPolicy: "IfNotPresent", - args: ["redis-server", "/usr/local/etc/redis/redis.conf"], - ports: [{ name: "redis", containerPort: port }], - readinessProbe: { exec: { command: ["redis-cli", "-p", String(port), "ping"] }, initialDelaySeconds: 3, periodSeconds: 10, timeoutSeconds: 2, failureThreshold: 3 }, - livenessProbe: { exec: { command: ["redis-cli", "-p", String(port), "ping"] }, initialDelaySeconds: 15, periodSeconds: 20, timeoutSeconds: 3, failureThreshold: 3 }, - resources: cloneJson(config.resources), - volumeMounts: [{ name: "config", mountPath: "/usr/local/etc/redis", readOnly: true }] - }], - volumes: [{ name: "config", configMap: { name: `${name}-config` } }] - } - } - } - }, - { - apiVersion: "v1", - kind: "Service", - metadata: { name, namespace, labels, annotations: runtimeAnnotations }, - spec: { type: "ClusterIP", selector, ports: [{ name: "redis", port, targetPort: "redis", protocol: "TCP" }] } - }, - { - apiVersion: "networking.k8s.io/v1", - kind: "NetworkPolicy", - metadata: { name: `${name}-ingress`, namespace, labels, annotations: runtimeAnnotations }, - spec: { - podSelector: { matchLabels: selector }, - policyTypes: ["Ingress"], - ingress: [{ - from: [{ podSelector: { matchLabels: { "app.kubernetes.io/name": "hwlab-workbench-runtime" } } }], - ports: [{ protocol: "TCP", port }] - }] - } - } - ] - }; -} - -function runtimeConfigMapsForProfile(deploy, profile) { - if (!isRuntimeLane(profile)) return []; - const configMaps = deploy?.lanes?.[profile]?.configMaps; - if (!Array.isArray(configMaps)) return []; - return configMaps - .map((configMap) => cloneJson(configMap)) - .filter((configMap) => typeof configMap?.name === "string" && configMap.name.trim() && configMap.data && typeof configMap.data === "object" && !Array.isArray(configMap.data)); -} - -function runtimeConfigMapsManifest({ configMaps, namespace, labels, annotations }) { - return { - apiVersion: "v1", - kind: "List", - items: configMaps.map((configMap) => ({ - apiVersion: "v1", - kind: "ConfigMap", - metadata: { - name: configMap.name, - namespace, - labels: { ...labels, ...(configMap.labels ?? {}) }, - annotations: { ...annotations, ...(configMap.annotations ?? {}) } - }, - data: Object.fromEntries(Object.entries(configMap.data).map(([key, value]) => [key, String(value)])) - })) - }; -} - -async function runtimeSecretPlaneConfigForProfile(deploy, profile, args = {}) { - if (!isRuntimeLane(profile)) return null; - const laneConfig = deploy?.lanes?.[profile]; - const secretPlane = laneConfig?.secretPlaneRef - ? await readConfigRef(laneConfig.secretPlaneRef, `${profile}.secretPlaneRef`) - : laneConfig?.secretPlane; - if (!secretPlane || typeof secretPlane !== "object" || Array.isArray(secretPlane)) return null; - if (secretPlane.enabled !== true) return null; - if (!runtimeSecretPlaneEnabledForNode(secretPlane, args)) return null; - return cloneJson(secretPlane); -} - -function runtimeSecretPlaneEnabledForNode(secretPlane, args = {}) { - const enabledOnNodes = normalizeSecretPlaneNodeList(secretPlane.enabledOnNodes, "secretPlane.enabledOnNodes"); - if (enabledOnNodes.length === 0) return true; - const nodeId = effectiveSecretPlaneNodeId(args); - assert.ok(nodeId, "secretPlane.enabledOnNodes requires --node or a node-scoped --gitops-root"); - return enabledOnNodes.includes(nodeId); -} - -function normalizeSecretPlaneNodeList(value, label) { - if (value === undefined) return []; - assert.ok(Array.isArray(value), `${label} must be an array when set`); - return value.map((item, index) => { - assert.equal(typeof item, "string", `${label}[${index}] must be a string`); - const nodeId = item.trim().toUpperCase(); - assert.ok(/^[A-Z0-9][A-Z0-9-]*$/u.test(nodeId), `${label}[${index}] must be a node id`); - return nodeId; - }); -} - -function effectiveSecretPlaneNodeId(args = {}) { - return gitopsRootNodeId(args.gitopsRoot) ?? String(args.nodeId ?? "").trim().toUpperCase(); -} - -function gitopsRootNodeId(gitopsRoot) { - const normalized = String(gitopsRoot ?? "").replaceAll("\\", "/").replace(/\/+$/u, ""); - const match = normalized.match(/(?:^|\/)deploy\/gitops\/node\/([^/]+)$/u); - return match?.[1] ? match[1].trim().toUpperCase() : null; -} - -function runtimeSecretPlaneManifest({ config, namespace, labels, annotations }) { - const store = config?.store; - assert.ok(store && typeof store === "object" && !Array.isArray(store), "secretPlane.store must be an object"); - assert.equal(store.kind, "ClusterSecretStore", "secretPlane.store.kind must be ClusterSecretStore for node-scoped v0.3"); - assert.ok(typeof store.name === "string" && store.name.trim(), "secretPlane.store.name must be set"); - const secrets = Array.isArray(config.secrets) ? config.secrets : []; - assert.ok(secrets.length > 0, "secretPlane.secrets must not be empty"); - return { - apiVersion: "v1", - kind: "List", - items: secrets.map((secret, index) => runtimeSecretPlaneExternalSecret({ config, secret, index, namespace, labels, annotations, store })) - }; -} - -function runtimeSecretPlaneExternalSecret({ config, secret, index, namespace, labels, annotations, store }) { - const externalSecretName = requiredSecretPlaneString(secret?.externalSecretName, `secretPlane.secrets[${index}].externalSecretName`); - const targetSecretName = requiredSecretPlaneString(secret?.targetSecretName, `secretPlane.secrets[${index}].targetSecretName`); - const data = Array.isArray(secret?.data) ? secret.data : []; - assert.ok(data.length > 0, `secretPlane.secrets[${index}].data must not be empty`); - const sourceRefs = data.map((item) => requiredSecretPlaneString(item?.remoteRef, `secretPlane.secrets[${index}].data.remoteRef`)); - const issueRef = String(config.issue ?? "pikasTech/HWLAB#2234"); - return { - apiVersion: "external-secrets.io/v1", - kind: "ExternalSecret", - metadata: { - name: externalSecretName, - namespace, - labels: { - ...labels, - "app.kubernetes.io/name": externalSecretName, - "app.kubernetes.io/component": "external-secret", - "hwlab.pikastech.local/secret-plane": secretPlaneIssueLabelValue(issueRef) - }, - annotations: { - ...annotations, - "hwlab.pikastech.local/secret-plane-issue": issueRef, - "hwlab.pikastech.local/source-ref": sourceRefs.join(","), - "hwlab.pikastech.local/values-printed": "false" - } - }, - spec: { - refreshInterval: requiredSecretPlaneString(config.refreshInterval, "secretPlane.refreshInterval"), - secretStoreRef: { - name: requiredSecretPlaneString(store.name, "secretPlane.store.name"), - kind: store.kind - }, - target: { - name: targetSecretName, - creationPolicy: "Owner" - }, - data: data.map((item, itemIndex) => ({ - secretKey: requiredSecretPlaneString(item?.targetKey, `secretPlane.secrets[${index}].data[${itemIndex}].targetKey`), - remoteRef: { - key: requiredSecretPlaneString(item?.remoteRef, `secretPlane.secrets[${index}].data[${itemIndex}].remoteRef`), - property: requiredSecretPlaneString(item?.property, `secretPlane.secrets[${index}].data[${itemIndex}].property`) - } - })) - } - }; -} - -function secretPlaneIssueLabelValue(issueRef) { - const text = String(issueRef ?? "").trim(); - const issueNumber = text.match(/#([0-9]+)$/u)?.[1]; - const raw = issueNumber ? `issue-${issueNumber}` : text; - const normalized = raw.replace(/[^A-Za-z0-9_.-]+/gu, "-").replace(/^[^A-Za-z0-9]+|[^A-Za-z0-9]+$/gu, ""); - const label = normalized.slice(0, 63).replace(/[^A-Za-z0-9]+$/u, ""); - assert.ok(label.length > 0, "secretPlane.issue must produce a Kubernetes label-safe value"); - return label; -} - -function opencodeRuntimeConfigForProfile(deploy, profile) { - const config = optionalObject(deploy?.lanes?.[profile]?.opencode); - const providerProfile = configString(config.providerProfile) || "dsflash-go"; - const providerId = configString(config.providerId) || providerProfile; - const model = configString(config.model) || "deepseek-v4-flash"; - const smallModel = configString(config.smallModel) || model; - const apiKeyEnv = configString(config.apiKeyEnv) || opencodeApiKeyEnvName(providerId); - const contextLimit = opencodePositiveInteger(config.contextLimit, 1000000, `deploy.lanes.${profile}.opencode.contextLimit`); - const outputLimit = opencodePositiveInteger(config.outputLimit, 384000, `deploy.lanes.${profile}.opencode.outputLimit`); - const reasoning = opencodeConfigBoolean(config.reasoning, false, `deploy.lanes.${profile}.opencode.reasoning`); - const toolCall = opencodeConfigBoolean(config.toolCall, true, `deploy.lanes.${profile}.opencode.toolCall`); - const upstreamBaseURL = configString(config.upstreamBaseURL) || configString(config.baseURL) || "https://opencode.ai/zen/go/v1"; - const providerProxyPort = opencodePositiveInteger(config.providerProxyPort, 4097, `deploy.lanes.${profile}.opencode.providerProxyPort`); - const providerProxyBasePath = configString(config.providerProxyBasePath) || "/v1"; - const providerProxyBaseURL = configString(config.providerProxyBaseURL) || `http://127.0.0.1:${providerProxyPort}${providerProxyBasePath}`; - return { - image: configString(config.image) || "ghcr.io/anomalyco/opencode:1.17.7", - providerProfile, - providerId, - displayName: configString(config.displayName) || `AgentRun ${providerProfile}`, - model, - smallModel, - baseURL: providerProxyBaseURL, - upstreamBaseURL, - apkProxyURL: configString(config.apkProxyURL) || "", - providerProxyPort, - providerProxyBasePath, - npm: configString(config.npm) || "@ai-sdk/openai-compatible", - apiKeyEnv, - apiKeySecretRef: configString(config.apiKeySecretRef) || "hwlab-code-agent-provider/opencode-api-key", - contextLimit, - outputLimit, - reasoning, - toolCall, - agentrunSecretNamespace: configString(config.agentrunSecretNamespace) || "agentrun-v02", - agentrunSecretName: configString(config.agentrunSecretName) || `agentrun-v01-provider-${providerProfile}`, - agentrunSecretKeys: Array.isArray(config.agentrunSecretKeys) && config.agentrunSecretKeys.length > 0 - ? config.agentrunSecretKeys.map(configString).filter(Boolean) - : ["auth.json", "config.toml", "model-catalog.json"] - }; -} - -function opencodePositiveInteger(value, fallback, label) { - if (value === undefined || value === null) return fallback; - assert.ok(Number.isInteger(value) && value > 0, `${label} must be a positive integer`); - return value; -} - -function opencodeConfigBoolean(value, fallback, label) { - if (value === undefined || value === null) return fallback; - assert.equal(typeof value, "boolean", `${label} must be a boolean`); - return value; -} - -function opencodeEgressProxyUrlForProfile(deploy, profile) { - const proxy = deploy?.lanes?.[profile]?.gitMirror?.egressProxy; - if (!proxy || proxy.required === false) return ""; - const proxyUrl = configString(proxy.proxyUrl); - if (proxyUrl) return proxyUrl; - const serviceName = configString(proxy.serviceName); - const namespace = configString(proxy.namespace) || "platform-infra"; - const port = Number(proxy.port); - if (!serviceName || !Number.isInteger(port) || port <= 0) return ""; - return `http://${serviceName}.${namespace}.svc.cluster.local:${port}`; -} - -function opencodeEgressNoProxyForProfile(deploy, profile) { - const proxy = deploy?.lanes?.[profile]?.gitMirror?.egressProxy; - const entries = Array.isArray(proxy?.noProxy) ? proxy.noProxy.map(configString).filter(Boolean) : []; - return entries.length > 0 ? entries.join(",") : "localhost,127.0.0.1,::1,.svc,.svc.cluster.local,.cluster.local"; -} - -function opencodeApiKeyEnvName(providerId) { - const normalized = String(providerId || "provider").toUpperCase().replace(/[^A-Z0-9]+/gu, "_").replace(/^_+|_+$/gu, ""); - return `OPENCODE_${normalized || "PROVIDER"}_API_KEY`; -} - -function opencodeConfigContent(config) { - return JSON.stringify({ - $schema: "https://opencode.ai/config.json", - autoupdate: false, - share: "disabled", - model: `${config.providerId}/${config.model}`, - small_model: `${config.providerId}/${config.smallModel}`, - provider: { - [config.providerId]: { - npm: config.npm, - name: config.displayName, - options: { - baseURL: config.baseURL, - apiKey: `{env:${config.apiKeyEnv}}`, - timeout: 600000, - headerTimeout: 600000, - chunkTimeout: 300000 - }, - models: { - [config.model]: { - name: config.model, - reasoning: config.reasoning, - tool_call: config.toolCall, - limit: { - context: config.contextLimit, - output: config.outputLimit - } - } - } - } - } - }); -} - -function opencodeServerManifest({ profile = "v03", source, deploy = null, catalog = null, registryPrefix = defaultRegistryPrefix, useDeployImages = false, sourceBranch = defaultBranch, gitReadUrl = defaultV02GitReadUrl, sourceRepo = defaultSourceRepo }) { - assert.ok(isRuntimeLane(profile), `opencode-server profile must be a runtime lane, got ${profile}`); - const namespace = namespaceNameForProfile(profile); - const name = "opencode-server"; - const helperServiceId = "hwlab-cloud-web"; - const envReuseServiceIds = envReuseServiceIdsForLane(deploy, profile); - const helperImage = runtimeImageForService({ catalog, deploy, serviceId: helperServiceId, source, registryPrefix, useDeployImages, digestPin: true, envReuseServiceIds }); - const helperCommit = runtimeCommitForService({ catalog, deploy, serviceId: helperServiceId, source, registryPrefix, useDeployImages, digestPin: true, envReuseServiceIds }); - const helperImageTag = runtimeImageTagForService({ catalog, deploy, serviceId: helperServiceId, source, registryPrefix, useDeployImages, digestPin: true, envReuseServiceIds }); - const providerProxyBootSh = "deploy/runtime/boot/opencode-provider-proxy.sh"; - const providerProxyBootMetadata = v02EnvReuseEnabled(catalog, helperServiceId, envReuseServiceIds) - ? bootMetadataForService({ args: { sourceRepo: deploy?.lanes?.[profile]?.sourceRepo || sourceRepo, registryPrefix }, catalog, deployService: deployServiceForBoot(deploy, helperServiceId, profile), serviceId: helperServiceId, source }) - : null; - const selector = { - "app.kubernetes.io/name": name, - "app.kubernetes.io/part-of": "hwlab", - "hwlab.pikastech.local/environment": profile, - "hwlab.pikastech.local/profile": profile - }; - const labels = { - ...selector, - "hwlab.pikastech.local/component": "opencode", - "hwlab.pikastech.local/gitops-target": profile, - "hwlab.pikastech.local/service-id": name, - "hwlab.pikastech.local/source-commit": source.full - }; - const opencodeConfig = opencodeRuntimeConfigForProfile(deploy, profile); - const configContent = opencodeConfigContent(opencodeConfig); - const configSha256 = createHash("sha256").update(configContent).digest("hex"); - const apiKeySecretRef = secretRefObjectForProfile(opencodeConfig.apiKeySecretRef, profile); - const opencodeApkProxyUrl = opencodeConfig.apkProxyURL || opencodeEgressProxyUrlForProfile(deploy, profile); - const opencodeApkNoProxy = opencodeEgressNoProxyForProfile(deploy, profile); - const annotations = { - "hwlab.pikastech.local/rendered-by": "scripts/gitops-render.mjs", - "hwlab.pikastech.local/opencode-provider-profile": opencodeConfig.providerProfile, - "hwlab.pikastech.local/opencode-provider-id": opencodeConfig.providerId, - "hwlab.pikastech.local/opencode-model": opencodeConfig.model, - "hwlab.pikastech.local/opencode-provider-base-url": opencodeConfig.baseURL, - "hwlab.pikastech.local/opencode-provider-upstream-base-url": opencodeConfig.upstreamBaseURL, - "hwlab.pikastech.local/opencode-image": opencodeConfig.image, - "hwlab.pikastech.local/opencode-config-sha256": configSha256, - "hwlab.pikastech.local/opencode-api-key-secret-ref": `${apiKeySecretRef.name}/${apiKeySecretRef.key}`, - "hwlab.pikastech.local/agentrun-provider-secret-ref": `${opencodeConfig.agentrunSecretNamespace}/${opencodeConfig.agentrunSecretName}`, - "hwlab.pikastech.local/agentrun-provider-secret-keys": opencodeConfig.agentrunSecretKeys.join(","), - "hwlab.pikastech.local/values-printed": "false" - }; - if (providerProxyBootMetadata) { - Object.assign(annotations, { - "hwlab.pikastech.local/opencode-provider-proxy-runtime-mode": providerProxyBootMetadata.runtimeMode, - "hwlab.pikastech.local/opencode-provider-proxy-boot-repo": providerProxyBootMetadata.bootRepo, - "hwlab.pikastech.local/opencode-provider-proxy-boot-commit": providerProxyBootMetadata.bootCommit, - "hwlab.pikastech.local/opencode-provider-proxy-boot-sh": providerProxyBootSh, - "hwlab.pikastech.local/opencode-provider-proxy-environment-digest": providerProxyBootMetadata.environmentDigest ?? "not_published" - }); - } - const authSecretName = secretNameForProfile("hwlab-opencode-server-auth", profile); - return { - apiVersion: "v1", - kind: "List", - items: [ - { - apiVersion: "v1", - kind: "ServiceAccount", - metadata: { name, namespace, labels, annotations } - }, - { - apiVersion: "v1", - kind: "PersistentVolumeClaim", - metadata: { name: `${name}-data`, namespace, labels, annotations }, - spec: { - accessModes: ["ReadWriteOnce"], - resources: { requests: { storage: "8Gi" } } - } - }, - { - apiVersion: "v1", - kind: "Service", - metadata: { name, namespace, labels, annotations }, - spec: { - type: "ClusterIP", - selector, - ports: [{ name: "http", port: 4096, targetPort: "http" }] - } - }, - { - apiVersion: "apps/v1", - kind: "Deployment", - metadata: { name, namespace, labels, annotations }, - spec: { - replicas: 1, - strategy: { type: "Recreate" }, - selector: { matchLabels: selector }, - template: { - metadata: { labels, annotations }, - spec: { - serviceAccountName: name, - securityContext: { fsGroup: 1000, fsGroupChangePolicy: "OnRootMismatch" }, - initContainers: [{ - name: "opencode-workspace-git-init", - image: helperImage, - imagePullPolicy: "IfNotPresent", - command: ["/bin/sh", "-ec"], - args: [ - [ - "set -eu", - "mkdir -p /workspace", - "if [ ! -d /workspace/.git ]; then", - " git -C /workspace init", - "fi", - "git -C /workspace config user.name 'HWLAB OpenCode' || true", - "git -C /workspace config user.email 'opencode@hwlab.local' || true", - "chgrp -R 1000 /workspace/.git 2>/dev/null || true", - "chmod -R g+rwX /workspace/.git 2>/dev/null || true" - ].join("\n") - ], - resources: { requests: { cpu: "25m", memory: "64Mi" }, limits: { cpu: "250m", memory: "256Mi" } }, - volumeMounts: [{ name: "workspace", mountPath: "/workspace" }] - }], - containers: [{ - name, - image: opencodeConfig.image, - imagePullPolicy: "IfNotPresent", - command: ["/bin/sh", "-ec"], - args: ["exec opencode serve --hostname 0.0.0.0 --port 4096"], - workingDir: "/workspace", - env: [ - { name: "HOME", value: "/workspace" }, - { name: "XDG_CONFIG_HOME", value: "/workspace/.config" }, - { name: "XDG_DATA_HOME", value: "/workspace/.local/share" }, - ...(opencodeApkProxyUrl ? [ - { name: "HTTP_PROXY", value: opencodeApkProxyUrl }, - { name: "HTTPS_PROXY", value: opencodeApkProxyUrl }, - { name: "http_proxy", value: opencodeApkProxyUrl }, - { name: "https_proxy", value: opencodeApkProxyUrl }, - { name: "NO_PROXY", value: opencodeApkNoProxy }, - { name: "no_proxy", value: opencodeApkNoProxy } - ] : []), - { name: "OPENCODE_CONFIG_CONTENT", value: configContent }, - { name: opencodeConfig.apiKeyEnv, valueFrom: { secretKeyRef: apiKeySecretRef } }, - { name: "OPENCODE_SERVER_USERNAME", valueFrom: { secretKeyRef: { name: authSecretName, key: "username" } } }, - { name: "OPENCODE_SERVER_PASSWORD", valueFrom: { secretKeyRef: { name: authSecretName, key: "password" } } } - ], - ports: [{ name: "http", containerPort: 4096 }], - startupProbe: { tcpSocket: { port: "http" }, periodSeconds: 5, failureThreshold: 60 }, - readinessProbe: { tcpSocket: { port: "http" }, periodSeconds: 10, failureThreshold: 3 }, - livenessProbe: { tcpSocket: { port: "http" }, periodSeconds: 20, failureThreshold: 3 }, - resources: { requests: { cpu: "100m", memory: "256Mi" }, limits: { cpu: "1", memory: "1Gi" } }, - volumeMounts: [{ name: "workspace", mountPath: "/workspace" }] - }, (() => { - const container = { - name: "opencode-provider-proxy", - image: helperImage, - imagePullPolicy: "IfNotPresent", - command: ["node", "/app/internal/dev-entrypoint/opencode-provider-proxy.mjs"], - env: [ - { name: "HWLAB_SERVICE_ID", value: "opencode-provider-proxy" }, - { name: "HWLAB_ENVIRONMENT", value: profile }, - { name: "HWLAB_COMMIT_ID", value: helperCommit }, - { name: "HWLAB_IMAGE", value: helperImage }, - { name: "HWLAB_IMAGE_TAG", value: helperImageTag }, - { name: "PORT", value: String(opencodeConfig.providerProxyPort) }, - { name: "HWLAB_OPENCODE_PROVIDER_PROXY_UPSTREAM_BASE_URL", value: opencodeConfig.upstreamBaseURL }, - { name: "HWLAB_OPENCODE_PROVIDER_PROXY_PUBLIC_BASE_PATH", value: opencodeConfig.providerProxyBasePath }, - { name: "HWLAB_OPENCODE_PROVIDER_PROXY_TIMEOUT_MS", value: "600000" }, - { name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", value: "http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces" }, - { name: "OTEL_SERVICE_NAME", value: "opencode-provider-proxy" } - ], - ports: [{ name: "provider", containerPort: opencodeConfig.providerProxyPort }], - readinessProbe: { httpGet: { path: "/health/readiness", port: "provider" }, periodSeconds: 10, failureThreshold: 3 }, - livenessProbe: { httpGet: { path: "/health/live", port: "provider" }, periodSeconds: 20, failureThreshold: 3 }, - resources: { requests: { cpu: "25m", memory: "64Mi" }, limits: { cpu: "250m", memory: "256Mi" } } - }; - if (providerProxyBootMetadata) { - applyEnvReuseBootEnv(container, providerProxyBootMetadata, { sourceBranch, gitReadUrl, bootSh: providerProxyBootSh }); - } - return container; - })()], - volumes: [{ name: "workspace", persistentVolumeClaim: { claimName: `${name}-data` } }] - } - } - } - } - ] - }; -} - -function requiredSecretPlaneString(value, label) { - assert.equal(typeof value, "string", `${label} must be a string`); - const trimmed = value.trim(); - assert.ok(trimmed, `${label} must not be empty`); - return trimmed; -} + jsonManifest, + label, + laneRuntimeProfiles, + namespaceNameForProfile, + observabilityManifest, + parseArgs, + profileEndpoint, + readJson, + readJsonIfPresent, + repoRoot, + resolveSourceRevision, + runtimeLabelForProfile, + runtimePathForProfile, + textFile, + transformHealthContract, + transformListNamespace, + transformServices, + transformWorkloads, + usage, + workbenchRuntimeRedisConfigForProfile +} from "./src/gitops-render/core.mjs"; +import { + devopsInfraGitMirrorManifest, + registryManifest +} from "./src/gitops-render/infra-manifests.mjs"; +import { + argoApplication, + argoProject, + deepSeekProxyManifest, + deviceAgent71FreqManifest, + externalPostgresConfigForLane, + externalPostgresManifest, + nodeFrpcManifest, + opencodeServerManifest, + runtimeConfigMapsForProfile, + runtimeConfigMapsManifest, + runtimePostgresImageForProfile, + runtimeSecretPlaneConfigForProfile, + runtimeSecretPlaneManifest, + v02OpenFgaManifest, + v02PostgresManifest, + workbenchRuntimeRedisManifest +} from "./src/gitops-render/runtime-manifests.mjs"; +import { + ciLaneSettings, + tektonControlPlaneReconcilerCronJob, + tektonPipeline, + tektonPipelineRunTemplate, + tektonPollerCronJob, + tektonRbac +} from "./src/gitops-render/tekton-manifests.mjs"; +import { CLOUD_CORE_MIGRATIONS } from "../internal/db/schema.ts"; function runtimeKustomization({ profile = "dev", includeDeviceAgent = profile === "dev", externalPostgres = false, workbenchRedis = false, runtimeConfigMaps = false, externalSecrets = false } = {}) { const resources = ["namespace.yaml", "services.yaml", "health-contract.yaml", "workloads.yaml", "deepseek-proxy.yaml"]; @@ -5907,9 +162,13 @@ async function plannedFiles(args) { if (!artifactCatalog) artifactCatalog = await readJson(args.catalogPath); const settings = ciLaneSettings(args); const profiles = laneRuntimeProfiles(args); - const migrationSql = isRuntimeLane(args.lane) - ? await readFile(path.join(repoRoot, "internal/db/migrations/0001_cloud_core_skeleton.sql"), "utf8") - : null; + const migrationSources = isRuntimeLane(args.lane) + ? await Promise.all(CLOUD_CORE_MIGRATIONS.map(async (migration) => ({ + id: migration.id, + path: migration.path, + sql: await readFile(path.join(repoRoot, migration.path), "utf8") + }))) + : []; const metricsSidecarScript = isRuntimeLane(args.lane) ? await readFile(path.join(repoRoot, "internal/dev-entrypoint/metrics-sidecar.mjs"), "utf8") : null; @@ -5987,7 +246,7 @@ async function plannedFiles(args) { putJson(`${runtimePath}/workloads.yaml`, transformWorkloads({ workloads, deploy, catalog: artifactCatalog, source, sourceBranch: args.sourceBranch, gitReadUrl: args.gitReadUrl, registryPrefix: args.registryPrefix, runtimeEndpoint: endpoints.runtimeEndpoint, webEndpoint: endpoints.webEndpoint, profile, nodeId: args.nodeId, useDeployImages: args.useDeployImages, metricsSidecarSha256 })); putJson(`${runtimePath}/deepseek-proxy.yaml`, deepSeekProxyManifest({ profile, source, sourceBranch: args.sourceBranch, sourceRepo: args.sourceRepo, gitReadUrl: args.gitReadUrl, deploy, registryPrefix: args.registryPrefix, catalog: artifactCatalog, useDeployImages: args.useDeployImages, metricsSidecarSha256 })); if (isRuntimeLane(profile) && externalPostgres) putJson(`${runtimePath}/external-postgres.yaml`, externalPostgresManifest({ profile, config: externalPostgres, source })); - if (isRuntimeLane(profile) && !externalPostgres) putJson(`${runtimePath}/postgres.yaml`, v02PostgresManifest({ profile, migrationSql, source, image: runtimePostgresImageForProfile(deploy, profile) })); + if (isRuntimeLane(profile) && !externalPostgres) putJson(`${runtimePath}/postgres.yaml`, v02PostgresManifest({ profile, migrationSources, source, image: runtimePostgresImageForProfile(deploy, profile) })); if (workbenchRedis) putJson(`${runtimePath}/workbench-redis.yaml`, workbenchRuntimeRedisManifest({ profile, config: workbenchRedis, source })); if (isRuntimeLane(profile)) putJson(`${runtimePath}/openfga.yaml`, v02OpenFgaManifest({ profile, source })); if (isRuntimeLane(profile)) putJson(`${runtimePath}/observability.yaml`, observabilityManifest({ deploy, profile, namespace, labels: profileLabels, annotations, metricsSidecarScript })); diff --git a/scripts/gitops-render.test.ts b/scripts/gitops-render.test.ts index 53480965..fd07f93f 100644 --- a/scripts/gitops-render.test.ts +++ b/scripts/gitops-render.test.ts @@ -1,10 +1,15 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test from "node:test"; -import vm from "node:vm"; + +import { configString } from "./src/gitops-render/core.mjs"; +import { opencodeEgressProxyUrlForProfile } from "./src/gitops-render/runtime-manifests.mjs"; + +import { CLOUD_CORE_MIGRATIONS, CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID } from "../internal/db/schema.ts"; function taskByName(pipeline, name) { const task = pipeline.spec.tasks.find((item) => item.name === name); @@ -98,36 +103,11 @@ async function runtimeNoopDecision(gitopsPromoteScript, { oldRuntime, newRuntime } } -function extractNamedFunction(source, name) { - const start = source.indexOf(`function ${name}`); - assert.notEqual(start, -1, `missing function ${name}`); - const bodyStart = source.indexOf("{", start); - assert.notEqual(bodyStart, -1, `missing function body for ${name}`); - let depth = 0; - for (let index = bodyStart; index < source.length; index += 1) { - const char = source[index]; - if (char === "{") depth += 1; - else if (char === "}") { - depth -= 1; - if (depth === 0) return source.slice(start, index + 1); - } - } - assert.fail(`unterminated function ${name}`); -} - test("opencode APK proxy prefers explicit gitMirror proxyUrl", async () => { - const source = await readFile("scripts/gitops-render.mjs", "utf8"); - const context = {}; - vm.runInNewContext(`${extractNamedFunction(source, "configString")} -${extractNamedFunction(source, "opencodeEgressProxyUrlForProfile")} -globalThis.result = { - hostRoute: opencodeEgressProxyUrlForProfile({ lanes: { v03: { gitMirror: { egressProxy: { required: true, proxyUrl: "http://10.42.0.1:10808", namespace: "platform-infra", serviceName: "jd01-host-proxy", port: 10808 } } } } }, "v03"), - serviceFallback: opencodeEgressProxyUrlForProfile({ lanes: { v03: { gitMirror: { egressProxy: { required: true, namespace: "platform-infra", serviceName: "sub2api-egress-proxy", port: 10808 } } } } }, "v03"), - disabled: opencodeEgressProxyUrlForProfile({ lanes: { v03: { gitMirror: { egressProxy: { required: false, proxyUrl: "http://10.42.0.1:10808" } } } } }, "v03") -};`, context, { filename: "gitops-render-opencode-proxy.test.mjs" }); - assert.equal(context.result.hostRoute, "http://10.42.0.1:10808"); - assert.equal(context.result.serviceFallback, "http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808"); - assert.equal(context.result.disabled, ""); + assert.equal(configString(" value "), "value"); + assert.equal(opencodeEgressProxyUrlForProfile({ lanes: { v03: { gitMirror: { egressProxy: { required: true, proxyUrl: "http://10.42.0.1:10808", namespace: "platform-infra", serviceName: "jd01-host-proxy", port: 10808 } } } } }, "v03"), "http://10.42.0.1:10808"); + assert.equal(opencodeEgressProxyUrlForProfile({ lanes: { v03: { gitMirror: { egressProxy: { required: true, namespace: "platform-infra", serviceName: "sub2api-egress-proxy", port: 10808 } } } } }, "v03"), "http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808"); + assert.equal(opencodeEgressProxyUrlForProfile({ lanes: { v03: { gitMirror: { egressProxy: { required: false, proxyUrl: "http://10.42.0.1:10808" } } } } }, "v03"), ""); }); test("v02 render follows TypeScript runtime checks and does not self-patch bootstrap RBAC", async () => { @@ -499,12 +479,23 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots assert.equal(frpcDeployment.spec.template.metadata.labels["hwlab.pikastech.local/source-commit"], undefined); const postgres = JSON.parse(await readFile(path.join(outDir, "runtime-v02", "postgres.yaml"), "utf8")); + const postgresMigrationConfig = postgres.items.find((item) => item.kind === "ConfigMap" && item.metadata?.name === "hwlab-v02-postgres-init"); const postgresStatefulSet = postgres.items.find((item) => item.kind === "StatefulSet"); const postgresMigrationLabel = postgresStatefulSet.spec.template.metadata.labels["hwlab.pikastech.local/migration-sha256"]; const postgresMigrationAnnotation = postgresStatefulSet.spec.template.metadata.annotations["hwlab.pikastech.local/migration-sha256"]; + const migrationSources = await Promise.all(CLOUD_CORE_MIGRATIONS.map(async (migration) => ({ + ...migration, + sql: await readFile(path.join(process.cwd(), migration.path), "utf8") + }))); + const migrationData = Object.fromEntries(migrationSources.map((migration) => [path.basename(migration.path), migration.sql])); + const migrationSha256 = createHash("sha256").update(migrationSources.map((migration) => migration.sql).join("\n")).digest("hex"); + assert.deepEqual(postgresMigrationConfig.data, migrationData); + assert.deepEqual(Object.keys(postgresMigrationConfig.data), CLOUD_CORE_MIGRATIONS.map((migration) => path.basename(migration.path))); + assert.match(postgresMigrationConfig.data["0008_workbench_kafka_realtime.sql"], new RegExp(`'${CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID}'`, "u")); assert.ok(postgresMigrationLabel); assert.equal(postgresMigrationLabel.length <= 63, true); assert.match(postgresMigrationAnnotation, /^[a-f0-9]{64}$/u); + assert.equal(postgresMigrationAnnotation, migrationSha256); assert.equal(postgresMigrationAnnotation.startsWith(postgresMigrationLabel), true); assert.equal(postgresStatefulSet.spec.template.metadata.labels["hwlab.pikastech.local/source-commit"], undefined); diff --git a/scripts/src/dev-runtime-migration.mjs b/scripts/src/dev-runtime-migration.mjs index b3b11958..f0362bcc 100644 --- a/scripts/src/dev-runtime-migration.mjs +++ b/scripts/src/dev-runtime-migration.mjs @@ -4,11 +4,12 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { - CLOUD_CORE_MIGRATION_ID, - CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + CLOUD_CORE_MIGRATIONS, + CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID as CLOUD_RUNTIME_DURABLE_MIGRATION_ID, + CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION as CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE_COLUMNS, - CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS + CLOUD_TRANSACTIONAL_REALTIME_REQUIRED_TABLE_COLUMNS as CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS } from "../../internal/db/schema.ts"; import { PostgresCloudRuntimeStore, @@ -25,9 +26,10 @@ import { ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs"; import { ensureNotRepoReportsPath, tempReportPath } from "./report-paths.mjs"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); -const migrationPath = "internal/db/migrations/0001_cloud_core_skeleton.sql"; +const migrationPaths = CLOUD_CORE_MIGRATIONS.map((migration) => migration.path); +const latestMigrationPath = migrationPaths.at(-1); const defaultReportPath = tempReportPath("dev-runtime-migration-report.json"); -const issue = "pikasTech/HWLAB#164"; +const issue = "pikasTech/HWLAB#2464"; export async function runDevRuntimeMigrationCli(argv = process.argv.slice(2), options = {}) { const args = parseArgs(argv); @@ -55,7 +57,11 @@ export async function buildDevRuntimeMigrationReport(args = {}, options = {}) { const env = options.env ?? process.env; const now = options.now ?? (() => new Date().toISOString()); const root = options.repoRoot ?? repoRoot; - const sql = await readFile(path.join(root, migrationPath), "utf8"); + const migrationSources = await Promise.all(migrationPaths.map(async (migrationPath) => ({ + path: migrationPath, + sql: await readFile(path.join(root, migrationPath), "utf8") + }))); + const sql = migrationSources.map((migration) => migration.sql).join("\n"); const sourceCheck = validateMigrationSource(sql); const db = summarizeDbEnv(env); const report = { @@ -71,8 +77,9 @@ export async function buildDevRuntimeMigrationReport(args = {}, options = {}) { dbEndpointAuthority: DEV_DB_ENV_CONTRACT.endpointAuthority.source }, migration: { - id: CLOUD_CORE_MIGRATION_ID, - path: migrationPath, + id: CLOUD_RUNTIME_DURABLE_MIGRATION_ID, + path: latestMigrationPath, + paths: migrationPaths, sha256: sha256(sql), schemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, ledgerTable: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, @@ -195,7 +202,9 @@ export function validateMigrationSource(sql) { continue; } for (const column of columns) { - if (!new RegExp(`\\b${column}\\b`, "u").test(tableMatch[1])) { + const created = new RegExp(`\\b${column}\\b`, "u").test(tableMatch[1]); + const added = new RegExp(`ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS ${column}\\b`, "u").test(sql); + if (!created && !added) { missing.push(`column:${table}.${column}`); } } @@ -214,8 +223,8 @@ export function validateMigrationSource(sql) { } } - if (!new RegExp(`'${CLOUD_CORE_MIGRATION_ID}'`, "u").test(sql)) { - missing.push(`ledger-id:${CLOUD_CORE_MIGRATION_ID}`); + if (!new RegExp(`'${CLOUD_RUNTIME_DURABLE_MIGRATION_ID}'`, "u").test(sql)) { + missing.push(`ledger-id:${CLOUD_RUNTIME_DURABLE_MIGRATION_ID}`); } if (!new RegExp(`'${CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION}'`, "u").test(sql)) { missing.push(`schema-version:${CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION}`); @@ -227,8 +236,9 @@ export function validateMigrationSource(sql) { return { ready: missing.length === 0, checked: true, - migrationPath, - requiredMigrationId: CLOUD_CORE_MIGRATION_ID, + migrationPath: latestMigrationPath, + migrationPaths, + requiredMigrationId: CLOUD_RUNTIME_DURABLE_MIGRATION_ID, requiredSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, requiredTables: Object.keys(CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS), requiredLedgerTable: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, @@ -257,6 +267,10 @@ async function verifyRuntimeReadiness(report, env, options) { let runtime; try { runtime = await runtimeStore.readiness(); + if (runtime.gates?.schema?.ready === true && runtime.gates?.migration?.ready === true) { + const transactionalReadiness = await runtimeStore.workbenchTransactionalRealtimeReadiness(); + runtime = mergeTransactionalRealtimeReadiness(runtime, transactionalReadiness); + } } finally { if (!options.queryClient && typeof runtimeStore.pool?.end === "function") { await runtimeStore.pool.end(); @@ -280,6 +294,56 @@ async function verifyRuntimeReadiness(report, env, options) { } } +function mergeTransactionalRealtimeReadiness(runtime, readiness = {}) { + const schemaReady = readiness.schema?.ready === true; + const migrationReady = readiness.migration?.ready === true; + const transactionalReady = readiness.ready === true && schemaReady && migrationReady; + const ready = runtime.ready === true && transactionalReady; + const blocker = ready + ? null + : !schemaReady + ? RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED + : !migrationReady + ? RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED + : runtime.blocker; + return { + ...runtime, + ready, + status: ready ? "ready" : "blocked", + blocker, + reason: ready + ? "Postgres transactional Workbench realtime schema and migration are ready" + : !schemaReady + ? "Postgres transactional Workbench realtime schema is incomplete" + : !migrationReady + ? "Postgres transactional Workbench realtime migration is not recorded" + : runtime.reason, + liveRuntimeEvidence: runtime.liveRuntimeEvidence === true && ready, + connection: { + ...runtime.connection, + queryResult: ready + ? "transactional_realtime_readiness_ready" + : !schemaReady + ? "schema_blocked" + : !migrationReady + ? "migration_blocked" + : runtime.connection?.queryResult + }, + schema: readiness.schema, + migration: readiness.migration, + gates: { + ...runtime.gates, + schema: schemaReady ? readyGate() : blockedGate(RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED), + migration: !schemaReady + ? notCheckedGate() + : migrationReady + ? readyGate() + : blockedGate(RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED), + durability: transactionalReady ? runtime.gates?.durability ?? notCheckedGate() : notCheckedGate() + } + }; +} + async function applyMigration(report, sql, env, options) { report.actions.liveDbWriteAttempted = true; let client; @@ -456,7 +520,7 @@ function summarizeRuntimeMigration(migration = {}) { checked: Boolean(migration.checked), ready: Boolean(migration.ready), table: migration.table ?? CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, - requiredMigrationId: migration.requiredMigrationId ?? CLOUD_CORE_MIGRATION_ID, + requiredMigrationId: migration.requiredMigrationId ?? CLOUD_RUNTIME_DURABLE_MIGRATION_ID, requiredSchemaVersion: migration.requiredSchemaVersion ?? CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, appliedMigrationId: migration.appliedMigrationId ?? null, appliedSchemaVersion: migration.appliedSchemaVersion ?? null, @@ -485,6 +549,12 @@ function summarizeDbEnv(env = {}) { } function buildApplyBoundary(args) { + const schemaWriteScope = migrationPaths.map((migrationPath) => + `DEV Postgres schema objects declared by ${migrationPath}` + ); + const ledgerWriteScope = CLOUD_CORE_MIGRATIONS.map((migration) => + `DEV ${CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE} ledger row ${migration.id}` + ); return { defaultMode: "check", mode: args.mode, @@ -493,10 +563,7 @@ function buildApplyBoundary(args) { requiresForLiveDbRead: ["--confirm-dev", "HWLAB_CLOUD_DB_URL from hwlab-cloud-api-dev-db/database-url"], requiresForApply: ["--apply", "--confirm-dev", "--confirmed-non-production", "HWLAB_CLOUD_DB_URL from hwlab-cloud-api-dev-db/database-url"], writeScope: args.mode === "apply" - ? [ - "DEV Postgres schema objects declared by internal/db/migrations/0001_cloud_core_skeleton.sql", - "DEV hwlab_schema_migrations ledger row 0001_cloud_core_skeleton" - ] + ? [...schemaWriteScope, ...ledgerWriteScope] : [], noWriteScope: [ "Kubernetes Secret resources", diff --git a/scripts/src/dev-runtime-migration.test.mjs b/scripts/src/dev-runtime-migration.test.mjs index fc81c8af..126a6caa 100644 --- a/scripts/src/dev-runtime-migration.test.mjs +++ b/scripts/src/dev-runtime-migration.test.mjs @@ -10,8 +10,12 @@ import { } from "./dev-runtime-migration.mjs"; import { CLOUD_CORE_MIGRATION_ID, - CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, - CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS + CLOUD_CORE_MIGRATIONS, + CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION as CLOUD_CORE_SCHEMA_VERSION, + CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID as CLOUD_RUNTIME_DURABLE_MIGRATION_ID, + CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION as CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, + CLOUD_TRANSACTIONAL_REALTIME_REQUIRED_TABLE_COLUMNS as CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS } from "../../internal/db/schema.ts"; const execFileAsync = promisify(execFile); @@ -34,6 +38,8 @@ test("source check is non-secret and does not attempt live DB access", async () assert.equal(report.gates.schema.status, "ready"); assert.equal(report.gates.migration.status, "ready"); assert.equal(report.gates.readiness.status, "not_checked"); + assert.equal(report.issue, "pikasTech/HWLAB#2464"); + assert.deepEqual(report.applyBoundary.writeScope, []); assert.equal(report.safety.sourceStoresSecretValues, false); assert.equal(report.safety.printsSecretValues, false); assert.equal(JSON.stringify(report).includes("secret-password"), false); @@ -60,7 +66,7 @@ test("cloud-api image migration entrypoint exposes the same non-secret source ch assert.equal(report.actions.liveDbWriteAttempted, false); assert.equal(report.safety.printsSecretValues, false); assert.equal(report.safety.dbUrlValueRedacted, true); - assert.equal(report.migration.id, CLOUD_CORE_MIGRATION_ID); + assert.equal(report.migration.id, CLOUD_RUNTIME_DURABLE_MIGRATION_ID); assert.equal(JSON.stringify(report).includes("secret-password"), false); assert.equal(JSON.stringify(report).includes("db.example.test"), false); }); @@ -120,6 +126,7 @@ test("live dry-run separates migration ledger blocker from auth and schema", asy assert.equal(report.gates.readiness.status, "not_checked"); assert.equal(report.blockers[0].scope, "runtime-migration-ledger"); assert.equal(report.runtime.migration.missing, true); + assert.equal(report.runtime.migration.requiredMigrationId, CLOUD_RUNTIME_DURABLE_MIGRATION_ID); }); test("live dry-run separates SSL negotiation blocker from auth and schema", async () => { @@ -306,8 +313,17 @@ test("apply mode records ledger and verifies durable runtime readiness", async ( assert.equal(report.gates.schema.status, "ready"); assert.equal(report.gates.migration.status, "ready"); assert.equal(report.gates.readiness.status, "ready"); - assert.equal(report.runtime.migration.appliedMigrationId, CLOUD_CORE_MIGRATION_ID); + assert.equal(report.runtime.migration.appliedMigrationId, CLOUD_RUNTIME_DURABLE_MIGRATION_ID); assert.equal(report.runtime.migration.appliedSchemaVersion, CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION); + assert.ok(queryClient.calls.some((call) => call.params[0] === CLOUD_CORE_MIGRATION_ID)); + assert.ok(queryClient.calls.some((call) => call.params[0] === CLOUD_RUNTIME_DURABLE_MIGRATION_ID)); + assert.deepEqual(report.applyBoundary.writeScope, [ + ...CLOUD_CORE_MIGRATIONS.map((migration) => `DEV Postgres schema objects declared by ${migration.path}`), + ...CLOUD_CORE_MIGRATIONS.map((migration) => `DEV ${CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE} ledger row ${migration.id}`) + ]); + assert.equal(report.applyBoundary.writeScope.length, CLOUD_CORE_MIGRATIONS.length * 2); + assert.ok(report.applyBoundary.writeScope.some((entry) => entry.includes("0008_workbench_kafka_realtime.sql"))); + assert.ok(report.applyBoundary.writeScope.some((entry) => entry.endsWith(`ledger row ${CLOUD_RUNTIME_DURABLE_MIGRATION_ID}`))); assert.ok(queryClient.calls.some((call) => call.sql === "BEGIN")); assert.ok(queryClient.calls.some((call) => call.sql.includes("INSERT INTO hwlab_schema_migrations"))); assert.ok(queryClient.calls.some((call) => call.sql === "COMMIT")); @@ -444,11 +460,17 @@ function createFakeQueryClient({ migrationReady, countErrorCode = null, migratio return { rows: schemaRows() }; } if (sql.startsWith("SELECT id, schema_version FROM hwlab_schema_migrations")) { + const requestedMigrationId = params[0]; + if (requestedMigrationId === CLOUD_CORE_MIGRATION_ID) { + return { + rows: [{ id: CLOUD_CORE_MIGRATION_ID, schema_version: CLOUD_CORE_SCHEMA_VERSION }] + }; + } return { - rows: state.migrationReady + rows: requestedMigrationId === CLOUD_RUNTIME_DURABLE_MIGRATION_ID && state.migrationReady ? [ { - id: CLOUD_CORE_MIGRATION_ID, + id: CLOUD_RUNTIME_DURABLE_MIGRATION_ID, schema_version: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION } ] diff --git a/scripts/src/gitops-render/core.mjs b/scripts/src/gitops-render/core.mjs new file mode 100644 index 00000000..eccbf06a --- /dev/null +++ b/scripts/src/gitops-render/core.mjs @@ -0,0 +1,1782 @@ +#!/usr/bin/env node +// SPEC: PJ2026-01060505 Workbench Performance; PJ2026-0106 平台运维 +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; + +import { + applyRuntimeLaneDeployConfig, + defaultEndpointForRuntimeLane, + defaultPortForRuntimeLane, + isRuntimeLane, + runtimeLaneMirrorSpecs, + runtimeLaneConfig, + versionNameForRuntimeLane +} from "../runtime-lane.ts"; +import { readStructuredFile } from "../structured-config.mjs"; + +const execFileAsync = promisify(execFile); +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../.."); +const defaultOutDir = "deploy/gitops/node"; +const defaultRegistryPrefix = process.env.HWLAB_NODE_REGISTRY_PREFIX || "127.0.0.1:5000/hwlab"; +const defaultSourceRepo = process.env.HWLAB_NODE_GIT_URL || "git@github.com:pikasTech/HWLAB.git"; +const defaultGitReadUrl = process.env.HWLAB_NODE_GIT_READ_URL || null; +const defaultV02GitReadUrl = process.env.HWLAB_V02_GIT_READ_URL || "http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git"; +const defaultV02GitWriteUrl = process.env.HWLAB_V02_GIT_WRITE_URL || "http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/HWLAB.git"; +const defaultRuntimeLaneGitReadUrl = process.env.HWLAB_RUNTIME_LANE_GIT_READ_URL || "http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikasTech-HWLAB.git"; +const defaultRuntimeLaneGitWriteUrl = process.env.HWLAB_RUNTIME_LANE_GIT_WRITE_URL || defaultV02GitWriteUrl; +const defaultBranch = process.env.HWLAB_NODE_BRANCH || "node"; +const defaultGitopsBranch = process.env.HWLAB_NODE_GITOPS_BRANCH || "node-gitops"; +const defaultCatalogPath = process.env.HWLAB_ARTIFACT_CATALOG_PATH || "deploy/artifact-catalog.dev.json"; +const defaultRuntimeEndpoint = process.env.HWLAB_NODE_RUNTIME_ENDPOINT || "http://74.48.78.17:17667"; +const defaultWebEndpoint = process.env.HWLAB_NODE_WEB_ENDPOINT || "http://74.48.78.17:17666"; +const defaultProdRuntimeEndpoint = process.env.HWLAB_NODE_PROD_RUNTIME_ENDPOINT || "http://74.48.78.17:18667"; +const defaultProdWebEndpoint = process.env.HWLAB_NODE_PROD_WEB_ENDPOINT || "http://74.48.78.17:18666"; +const defaultV02RuntimeEndpoint = process.env.HWLAB_V02_RUNTIME_ENDPOINT || "https://hwlab.74-48-78-17.nip.io"; +const defaultV02WebEndpoint = process.env.HWLAB_V02_WEB_ENDPOINT || "https://hwlab.74-48-78-17.nip.io"; +const defaultProxyUrl = process.env.HWLAB_NODE_PROXY_URL || "http://127.0.0.1:10808"; +const defaultAllProxyUrl = process.env.HWLAB_NODE_ALL_PROXY_URL || "socks5h://127.0.0.1:10808"; +const defaultNoProxy = process.env.HWLAB_NODE_NO_PROXY || "hyueapi.com,.hyueapi.com,127.0.0.1,localhost,::1,10.0.0.0/8,10.42.0.0/16,10.43.0.0/16,192.168.0.0/16,.svc,.cluster.local,kubernetes.default.svc,registry.hwlab-ci.svc,hwlab-registry.hwlab-ci.svc,git-mirror-http,git-mirror-http.devops-infra,git-mirror-http.devops-infra.svc,git-mirror-http.devops-infra.svc.cluster.local,git-mirror-write,git-mirror-write.devops-infra,git-mirror-write.devops-infra.svc,git-mirror-write.devops-infra.svc.cluster.local"; +const ciToolsRunnerImage = process.env.HWLAB_NODE_CI_TOOLS_IMAGE || "127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1"; +const buildkitRunnerImage = process.env.HWLAB_NODE_BUILDKIT_IMAGE || "moby/buildkit:rootless"; +const defaultDevBaseImage = process.env.HWLAB_NODE_DEV_BASE_IMAGE || "127.0.0.1:5000/hwlab/hwlab-node20-base:20-bookworm-slim"; +const defaultServiceIds = Object.freeze([ + "hwlab-cloud-api", + "hwlab-cloud-web", + "hwlab-gateway", + "hwlab-edge-proxy", + "hwlab-agent-skills" +]); +const v02ExtraObservableServices = Object.freeze([ + { serviceId: "hwlab-deepseek-proxy", port: 4000, healthPath: "/health/live" } +]); +const v02RemovedServiceIds = new Set([ + "hwlab-agent-mgr", + "hwlab-agent-worker", + "hwlab-agent-worker-template", + "hwlab-code-agent-workspace", + "hwlab-router", + "hwlab-tunnel-client", + "hwlab-gateway-simu", + "hwlab-box-simu", + "hwlab-patch-panel" +]); +const v02RemovedServiceEnvNames = new Set([ + "CODEX_HOME", + "OPENAI_API_KEY", + "HWLAB_CODE_AGENT_PROVIDER", + "HWLAB_CODE_AGENT_MODEL", + "HWLAB_CODE_AGENT_OPENAI_BASE_URL", + "HWLAB_CODE_AGENT_WORKSPACE", + "HWLAB_CODE_AGENT_CODEX_HOME", + "HWLAB_CODE_AGENT_CODEX_WORKSPACE", + "HWLAB_CODE_AGENT_CODEX_SANDBOX", + "HWLAB_CODE_AGENT_CODEX_COMMAND", + "HWLAB_CODE_AGENT_CODEX_STDIO_ENABLED", + "HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR", + "HWLAB_M3_GATEWAY_SIMU_1_URL", + "HWLAB_M3_GATEWAY_SIMU_2_URL", + "HWLAB_M3_PATCH_PANEL_URL", + "HWLAB_GATEWAY_SIMU_URL", + "HWLAB_BOX_SIMU_URL", + "HWLAB_PATCH_PANEL_URL", + "HWLAB_ROUTER_URL", + "HWLAB_TUNNEL_CLIENT_URL" +]); +const moonBridgeSourceRef = process.env.HWLAB_NODE_MOONBRIDGE_REF || "1b99888d3dae889b79ee602cb875c7907f7e76f2"; +const moonBridgeImage = process.env.HWLAB_NODE_MOONBRIDGE_IMAGE || `${defaultRegistryPrefix}/moonbridge:${moonBridgeSourceRef.slice(0, 12)}`; +const deepSeekProfileModel = process.env.HWLAB_NODE_DEEPSEEK_PROFILE_MODEL || "deepseek-chat"; +const codexApiProfileModel = process.env.HWLAB_NODE_CODEX_API_PROFILE_MODEL || "gpt-5.5"; +const codexApiProfileBaseUrl = process.env.HWLAB_NODE_CODEX_API_PROFILE_BASE_URL || "http://127.0.0.1:49280/responses"; +const codexApiForwarderPort = process.env.HWLAB_NODE_CODEX_API_FORWARDER_PORT || "49280"; +const codexApiForwarderUpstreamBaseUrl = process.env.HWLAB_NODE_CODEX_API_UPSTREAM_BASE_URL || "https://hyueapi.com"; +const sourceCommitPattern = /^[a-f0-9]{7,40}$/u; +const v02EnvReuseRuntimeMode = "env-reuse-git-mirror-checkout"; + +function k8sLabelHash(value) { + return String(value || "").slice(0, 16); +} + +const primitiveValidationTasks = Object.freeze([ + { + name: "repo-reports-guard", + commands: ["node scripts/repo-reports-guard.mjs"] + }, + { + name: "node-contract-check", + commands: [ + "node --check scripts/gitops-render.mjs", + `node --input-type=module <<'NODE' +import { readdirSync, readFileSync, statSync } from "node:fs"; +import path from "node:path"; + +const root = process.cwd(); +const ciFileName = "CI" + ".json"; +const forbiddenCiFragments = [ + "ci" + "-json", + "HWLAB_NODE_TEKTON" + "_SINGLE_IMAGE_PUBLISH", + "single" + "-dind", + "docker" + ":29", + "DOCKER" + "_HOST", + "Docker" + "-in-Docker", + "D" + "IND", + "docker" + " push", + "--build" + "-backend", + "HWLAB_ARTIFACT" + "_BUILD_BACKEND", + "--legacy" + "-source-images", + "HWLAB_NODE_USE_DEPLOY_IMAGES" + "=0" +]; +const scanTargets = [ + "scripts/gitops-render.mjs", + "scripts/src/runtime-lane.ts", + "scripts/artifact-publish.mjs", + "scripts/src/ci-plan-lib.mjs", + "deploy/gitops/node/tekton" +]; +const skipDirs = new Set([".git", "node_modules", ".worktree"]); + +function walkFiles(relativePath) { + const absolutePath = path.join(root, relativePath); + let stat; + try { + stat = statSync(absolutePath); + } catch { + return []; + } + if (stat.isFile()) return [relativePath]; + if (!stat.isDirectory()) return []; + const files = []; + for (const entry of readdirSync(absolutePath, { withFileTypes: true })) { + if (entry.isDirectory() && skipDirs.has(entry.name)) continue; + files.push(...walkFiles(path.join(relativePath, entry.name))); + } + return files; +} + +const ciFiles = walkFiles(".").filter((filePath) => path.basename(filePath) === ciFileName); +const violations = []; +for (const filePath of scanTargets.flatMap(walkFiles)) { + const text = readFileSync(path.join(root, filePath), "utf8"); + for (const fragment of forbiddenCiFragments) { + if (text.includes(fragment)) violations.push({ filePath, fragment }); + } +} +if (ciFiles.length || violations.length) { + console.error(JSON.stringify({ status: "failed", ciFiles, violations }, null, 2)); + process.exit(1); +} +NODE`, + "node --check scripts/artifact-publish.mjs" + ] + }, + { + name: "codex-api-forwarder-check", + commands: [ + "node scripts/run-bun.mjs test cmd/hwlab-codex-api-responses-forwarder/main.test.ts" + ] + } +]); + +function proxyEnv() { + return [ + { name: "HTTP_PROXY", value: defaultProxyUrl }, + { name: "HTTPS_PROXY", value: defaultProxyUrl }, + { name: "ALL_PROXY", value: defaultAllProxyUrl }, + { name: "NO_PROXY", value: defaultNoProxy }, + { name: "http_proxy", value: defaultProxyUrl }, + { name: "https_proxy", value: defaultProxyUrl }, + { name: "all_proxy", value: defaultAllProxyUrl }, + { name: "no_proxy", value: defaultNoProxy } + ]; +} + +function parseArgs(argv) { + const args = { + lane: process.env.HWLAB_GITOPS_LANE || "node", + nodeId: process.env.HWLAB_NODE_ID || "node", + outDir: defaultOutDir, + gitopsRoot: defaultOutDir, + catalogPath: defaultCatalogPath, + imageTagMode: process.env.HWLAB_ARTIFACT_IMAGE_TAG_MODE || "short", + registryPrefix: defaultRegistryPrefix, + sourceRevision: null, + sourceBranch: defaultBranch, + gitopsBranch: defaultGitopsBranch, + sourceRepo: defaultSourceRepo, + gitReadUrl: defaultGitReadUrl, + gitWriteUrl: null, + runtimeEndpoint: defaultRuntimeEndpoint, + webEndpoint: defaultWebEndpoint, + prodRuntimeEndpoint: defaultProdRuntimeEndpoint, + prodWebEndpoint: defaultProdWebEndpoint, + useDeployImages: true, + check: false, + write: true, + help: false + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--lane") args.lane = readOption(argv, ++index, arg); + else if (arg === "--node") args.nodeId = readOption(argv, ++index, arg); + else if (arg === "--out") args.outDir = readOption(argv, ++index, arg); + else if (arg === "--gitops-root") args.gitopsRoot = readOption(argv, ++index, arg); + else if (arg === "--catalog-path") args.catalogPath = readOption(argv, ++index, arg); + else if (arg === "--image-tag-mode") args.imageTagMode = readOption(argv, ++index, arg); + else if (arg === "--registry-prefix") args.registryPrefix = readOption(argv, ++index, arg).replace(/\/+$/u, ""); + else if (arg === "--source-revision") args.sourceRevision = readOption(argv, ++index, arg); + else if (arg === "--source-branch") args.sourceBranch = readOption(argv, ++index, arg); + else if (arg === "--gitops-branch") args.gitopsBranch = readOption(argv, ++index, arg); + else if (arg === "--source-repo") args.sourceRepo = readOption(argv, ++index, arg); + else if (arg === "--git-read-url") args.gitReadUrl = readOption(argv, ++index, arg); + else if (arg === "--git-write-url") args.gitWriteUrl = readOption(argv, ++index, arg); + else if (arg === "--runtime-endpoint") args.runtimeEndpoint = readOption(argv, ++index, arg); + else if (arg === "--web-endpoint") args.webEndpoint = readOption(argv, ++index, arg); + else if (arg === "--prod-runtime-endpoint") args.prodRuntimeEndpoint = readOption(argv, ++index, arg); + else if (arg === "--prod-web-endpoint") args.prodWebEndpoint = readOption(argv, ++index, arg); + else if (arg === "--use-deploy-images") args.useDeployImages = true; + else if (arg === "--check") { + args.check = true; + args.write = false; + } else if (arg === "--no-write") args.write = false; + else if (arg === "--help" || arg === "-h") args.help = true; + else throw new Error(`unknown argument ${arg}`); + } + if (isRuntimeLane(args.lane)) { + const versionName = versionNameForRuntimeLane(args.lane); + const defaultLaneEndpoint = defaultEndpointForRuntimeLane(args.lane); + if (args.sourceBranch === defaultBranch) args.sourceBranch = versionName; + if (args.gitopsBranch === defaultGitopsBranch) args.gitopsBranch = `${versionName}-gitops`; + if (args.catalogPath === defaultCatalogPath) args.catalogPath = `deploy/artifact-catalog.${args.lane}.json`; + if (args.runtimeEndpoint === defaultRuntimeEndpoint) args.runtimeEndpoint = defaultLaneEndpoint; + if (args.webEndpoint === defaultWebEndpoint) args.webEndpoint = defaultLaneEndpoint; + if (args.imageTagMode === "short") args.imageTagMode = "full"; + if (!args.gitReadUrl) args.gitReadUrl = defaultRuntimeLaneGitReadUrl; + if (!args.gitWriteUrl) args.gitWriteUrl = defaultRuntimeLaneGitWriteUrl; + } + if (!args.gitReadUrl) args.gitReadUrl = args.sourceRepo; + if (!args.gitWriteUrl) args.gitWriteUrl = args.sourceRepo; + assert.ok(args.lane === "node" || isRuntimeLane(args.lane), `unknown lane ${args.lane}`); + assert.ok(["short", "full"].includes(args.imageTagMode), `unknown image tag mode ${args.imageTagMode}`); + if (isRuntimeLane(args.lane)) { + const versionName = versionNameForRuntimeLane(args.lane); + assert.equal(args.sourceBranch, versionName, `${args.lane} source branch must be ${versionName}`); + assert.equal(args.gitopsBranch, `${versionName}-gitops`, `${args.lane} GitOps branch must be ${versionName}-gitops`); + } + return args; +} + +function readOption(argv, index, name) { + const value = argv[index]; + if (!value || value.startsWith("--")) throw new Error(`${name} requires a value`); + return value; +} + +function usage() { + return [ + "usage: node scripts/run-bun.mjs scripts/gitops-render.mjs [options]", + "", + "Render node-scoped Kubernetes-native CI/CD manifests from built-in Tekton CI primitives, deploy/deploy.yaml, and deploy/k8s/*.", + "", + "options:", + " --lane LANE node or runtime lane such as v02/v03/v04; default: node", + ` --node NODE deployment node id from deploy.nodes; runtime lanes default from deploy.lanes..node`, + ` --out DIR default: ${defaultOutDir}`, + ` --gitops-root DIR path inside GitOps repo; default comes from deploy.nodes..gitopsRoot`, + ` --catalog-path PATH default: ${defaultCatalogPath}`, + " --image-tag-mode MODE short or full; v02 uses full source commit IDs", + ` --source-repo URL default: ${defaultSourceRepo}`, + " --git-read-url URL read-only source URL; runtime lanes default to devops-infra git mirror", + " --git-write-url URL GitOps write URL; runtime lanes default to devops-infra git mirror write relay", + ` --source-branch BRANCH default: ${defaultBranch}`, + ` --gitops-branch BRANCH default: ${defaultGitopsBranch}`, + " --source-revision SHA source commit used for image tags and runtime annotations; default: git rev-parse HEAD", + ` --registry-prefix PREFIX default: ${defaultRegistryPrefix}`, + ` --runtime-endpoint URL default: ${defaultRuntimeEndpoint}; runtime lane default is derived from deploy.yaml/public URL`, + ` --web-endpoint URL default: ${defaultWebEndpoint}; runtime lane default is derived from deploy.yaml/public URL`, + ` --prod-runtime-endpoint URL default: ${defaultProdRuntimeEndpoint}`, + ` --prod-web-endpoint URL default: ${defaultProdWebEndpoint}`, + " --use-deploy-images default and only supported mode: render workload images from deploy/artifact-catalog.dev.json per service", + " --check verify generated files are current without writing", + " --no-write plan and print generated file list without writing" + ].join("\n"); +} + +function generatedPath(filePath) { + return path.isAbsolute(filePath) ? filePath : path.join(repoRoot, filePath); +} + +async function readJson(relativePath) { + return readStructuredFile(repoRoot, relativePath); +} + +async function readJsonIfPresent(relativePath, fallback = null) { + try { + return await readJson(relativePath); + } catch (error) { + if (error && error.code === "ENOENT") return fallback; + throw error; + } +} + +async function attachAccessControlEnv(deploy) { + const byProfile = {}; + for (const [profile, laneConfig] of Object.entries(deploy?.lanes ?? {})) { + if (!isRuntimeLane(profile)) continue; + const accessControl = laneConfig?.accessControl; + if (!accessControl || typeof accessControl !== "object" || Array.isArray(accessControl)) continue; + const users = await readConfigRef(accessControl.usersRef, `${profile}.accessControl.usersRef`); + const navProfiles = await readConfigRef(accessControl.navProfilesRef, `${profile}.accessControl.navProfilesRef`); + const env = accessControlEnvFromConfig({ users, navProfiles }); + if (Object.keys(env).length > 0) byProfile[profile] = { "hwlab-cloud-api": env }; + } + Object.defineProperty(deploy, "__accessControlEnvByProfile", { value: byProfile, enumerable: false, configurable: true }); +} + +async function readConfigRef(ref, label) { + assert.equal(typeof ref, "string", `${label} must be a file reference`); + const [filePath, fragment = ""] = ref.split("#"); + assert.ok(filePath, `${label} must include a file path`); + const root = await readJson(filePath); + return configFragment(root, fragment || "", label); +} + +function configFragment(value, fragment, label) { + const pathText = String(fragment ?? "").replace(/^\//u, ""); + if (!pathText) return value; + return pathText.split(/[./]/u).filter(Boolean).reduce((current, segment) => { + assert.ok(current && typeof current === "object" && Object.hasOwn(current, segment), `${label} fragment is missing ${segment}`); + return current[segment]; + }, value); +} + +function accessControlEnvFromConfig({ users, navProfiles }) { + assert.ok(Array.isArray(users), "accessControl usersRef must resolve to an array"); + assert.ok(Array.isArray(navProfiles), "accessControl navProfilesRef must resolve to an array"); + const env = {}; + const renderedUsers = users.map((user, index) => normalizeAccessControlUser(user, index, env)); + const renderedProfiles = navProfiles.map(normalizeNavProfile); + env.HWLAB_ACCESS_CONTROL_USERS_JSON = JSON.stringify(renderedUsers); + env.HWLAB_ACCESS_CONTROL_NAV_PROFILES_JSON = JSON.stringify(renderedProfiles); + return env; +} + +function normalizeAccessControlUser(user, index, env) { + assert.ok(user && typeof user === "object" && !Array.isArray(user), `accessControl user ${index} must be an object`); + const passwordHashRef = user.passwordHashRef && typeof user.passwordHashRef === "object" && !Array.isArray(user.passwordHashRef) ? user.passwordHashRef : null; + const passwordHashEnv = configString(user.passwordHashEnv) || configString(passwordHashRef?.env); + const secretRef = configString(passwordHashRef?.secretRef) || secretRefFromParts(passwordHashRef); + if (passwordHashEnv && secretRef) env[passwordHashEnv] = `secretRef:${secretRef}`; + return { + id: requiredAccessConfigString(user.id, `accessControl user ${index}.id`), + username: requiredAccessConfigString(user.username, `accessControl user ${index}.username`), + displayName: configString(user.displayName) || configString(user.username), + email: configString(user.email) || null, + role: configString(user.role) === "admin" ? "admin" : "user", + status: configString(user.status) === "disabled" ? "disabled" : "active", + navProfile: requiredAccessConfigString(user.navProfile, `accessControl user ${index}.navProfile`), + passwordHashEnv: passwordHashEnv || null + }; +} + +function normalizeNavProfile(profile, index) { + assert.ok(profile && typeof profile === "object" && !Array.isArray(profile), `accessControl nav profile ${index} must be an object`); + const allowedIds = Array.isArray(profile.allowedIds) ? profile.allowedIds.map(configString).filter(Boolean) : []; + assert.ok(allowedIds.length > 0, `accessControl nav profile ${index}.allowedIds must not be empty`); + return { + id: requiredAccessConfigString(profile.id, `accessControl nav profile ${index}.id`), + displayName: configString(profile.displayName) || configString(profile.id), + allowedIds + }; +} + +function secretRefFromParts(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) return ""; + const sourceRef = configString(value.sourceRef ?? value.secretName); + const targetKey = configString(value.targetKey ?? value.key); + return sourceRef && targetKey ? `${sourceRef}/${targetKey}` : ""; +} + +function configString(value) { + return typeof value === "string" ? value.trim() : ""; +} + +function requiredAccessConfigString(value, label) { + const text = configString(value); + assert.ok(text, `${label} is required`); + return text; +} + +function optionalObject(value) { + return value && typeof value === "object" && !Array.isArray(value) ? value : {}; +} + +async function gitValue(args) { + const result = await execFileAsync("git", args, { cwd: repoRoot, timeout: 10000, maxBuffer: 1024 * 1024 }); + return result.stdout.trim(); +} + +async function resolveSourceRevision(value) { + const revision = value || await gitValue(["rev-parse", "HEAD"]); + const full = sourceCommitPattern.test(revision) && revision.length === 40 + ? revision + : await gitValue(["rev-parse", revision]); + assert.match(full, /^[a-f0-9]{40}$/u, "source revision must resolve to a 40-char git commit"); + return { full, short: full.slice(0, 7) }; +} + +function jsonManifest(value) { + return `${JSON.stringify(value, null, 2)}\n`; +} + +function textFile(value) { + return value.endsWith("\n") ? value : `${value}\n`; +} + +function cloneJson(value) { + return JSON.parse(JSON.stringify(value)); +} + +function ensureObject(value, name) { + assert.ok(value && typeof value === "object" && !Array.isArray(value), `${name} must be an object`); + return value; +} + +function asItems(list, name) { + assert.equal(list?.kind, "List", `${name} must be a Kubernetes List`); + assert.ok(Array.isArray(list.items), `${name}.items must be an array`); + return list.items; +} + +function serviceIdForWorkload(item, container) { + return ( + item?.metadata?.labels?.["hwlab.pikastech.local/service-id"] || + item?.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/service-id"] || + container?.name + ); +} + +function upsertEnv(envList, name, value) { + if (!Array.isArray(envList)) return; + const existing = envList.find((entry) => entry?.name === name); + if (existing) { + delete existing.valueFrom; + existing.value = value; + } else { + envList.push({ name, value }); + } +} + +function upsertEnvEntry(envList, entry) { + if (!Array.isArray(envList) || !entry?.name) return; + const existing = envList.find((item) => item?.name === entry.name); + if (existing) { + delete existing.value; + delete existing.valueFrom; + Object.assign(existing, cloneJson(entry)); + } else { + envList.push(cloneJson(entry)); + } +} + +function syncCodeAgentWorkspaceInitContainer(initContainers, { image, runtimeCommit, runtimeImageTag }) { + const initContainer = Array.isArray(initContainers) + ? initContainers.find((entry) => entry?.name === "hwlab-code-agent-workspace-init") + : null; + if (!initContainer) return; + initContainer.image = image; + initContainer.imagePullPolicy = "IfNotPresent"; + initContainer.env ??= []; + upsertEnv(initContainer.env, "HWLAB_WORKSPACE_SOURCE_COMMIT", runtimeCommit); + upsertEnv(initContainer.env, "HWLAB_IMAGE", image); + upsertEnv(initContainer.env, "HWLAB_IMAGE_TAG", runtimeImageTag); +} + +function annotate(metadata, annotations) { + metadata.annotations = { ...(metadata.annotations ?? {}), ...annotations }; +} + +function label(metadata, labels) { + metadata.labels = { ...(metadata.labels ?? {}), ...labels }; +} + +function imageFor(registryPrefix, serviceId, imageTag) { + return `${registryPrefix}/${serviceId}:${imageTag}`; +} + +function imageTagFromReference(image) { + const value = String(image ?? ""); + const slashIndex = value.lastIndexOf("/"); + const colonIndex = value.lastIndexOf(":"); + if (colonIndex <= slashIndex) return null; + return value.slice(colonIndex + 1).split("@", 1)[0] || null; +} + +function repositoryFromImage(image) { + const value = String(image ?? "").split("@", 1)[0]; + const slashIndex = value.lastIndexOf("/"); + const colonIndex = value.lastIndexOf(":"); + return colonIndex > slashIndex ? value.slice(0, colonIndex) : value; +} + +function digestPinnedImage(image, digest) { + if (!/^sha256:[a-f0-9]{64}$/u.test(String(digest ?? ""))) return image; + const repository = repositoryFromImage(image); + return repository ? `${repository}@${digest}` : image; +} + +function catalogServiceById(catalog, serviceId) { + return (catalog?.services ?? []).find((service) => service?.serviceId === serviceId) ?? null; +} + +function v02EnvReuseEnabled(catalog, serviceId, envReuseServiceIds = null) { + if (envReuseServiceIds?.has(serviceId)) return true; + const service = catalogServiceById(catalog, serviceId); + return service?.runtimeMode === v02EnvReuseRuntimeMode || service?.envReuse === true; +} + +function bootShForService(deployService, serviceId) { + assert.ok(deployService?.bootSh, `deploy lane bootScripts.${serviceId} is required`); + const value = deployService.bootSh; + const text = String(value).replaceAll("\\", "/").replace(/^\.\//u, ""); + assert.ok(text && !text.startsWith("/") && !text.split("/").includes(".."), `${serviceId} boot script must be repo-relative`); + return text; +} + +function envReuseServiceIdsForLane(deploy, lane) { + if (!isRuntimeLane(lane)) return new Set(); + const configured = deploy?.lanes?.[lane]?.envReuseServices; + const allowed = new Set(serviceIdsForLane(lane, deploy)); + return new Set((Array.isArray(configured) ? configured : []).filter((serviceId) => allowed.has(serviceId))); +} + +function deployServiceForBoot(deploy, serviceId, lane = "v02") { + return { + serviceId, + bootSh: deploy?.lanes?.[lane]?.bootScripts?.[serviceId] + }; +} + +function bootMetadataForService({ args, catalog, deployService, serviceId, source }) { + const catalogService = catalogServiceById(catalog, serviceId); + const environmentImage = catalogService?.environmentImage ?? imageFor(args.registryPrefix, `${serviceId}-env`, `env-${String(catalogService?.environmentInputHash ?? source.full).slice(0, 12)}`); + const environmentDigest = catalogService?.environmentDigest ?? catalogService?.digest ?? null; + return { + runtimeMode: v02EnvReuseRuntimeMode, + environmentImage, + environmentDigest, + environmentInputHash: catalogService?.environmentInputHash ?? null, + codeInputHash: catalogService?.codeInputHash ?? null, + bootRepo: catalogService?.bootRepo ?? args.sourceRepo, + bootCommit: source.full, + bootSh: catalogService?.bootSh ?? bootShForService(deployService, serviceId) + }; +} + +function envReuseBootShForContainer({ serviceId, container, defaultBootSh }) { + if (serviceId === "hwlab-cloud-api" && container?.name === "hwlab-codex-api-forwarder") { + return "deploy/runtime/boot/hwlab-codex-api-responses-forwarder.sh"; + } + return defaultBootSh; +} + +function applyEnvReuseBootEnv(container, bootMetadata, { sourceBranch, gitReadUrl, bootSh = bootMetadata?.bootSh } = {}) { + if (!container || !bootMetadata) return; + delete container.command; + delete container.args; + container.env ??= []; + upsertEnv(container.env, "HWLAB_RUNTIME_MODE", bootMetadata.runtimeMode); + upsertEnv(container.env, "HWLAB_BOOT_REPO", bootMetadata.bootRepo); + upsertEnv(container.env, "HWLAB_BOOT_COMMIT", bootMetadata.bootCommit); + upsertEnv(container.env, "HWLAB_BOOT_REF", sourceBranch); + upsertEnv(container.env, "HWLAB_BOOT_SH", bootSh); + upsertEnv(container.env, "HWLAB_BOOT_READ_URL", gitReadUrl ?? defaultV02GitReadUrl); + upsertEnv(container.env, "HWLAB_ENVIRONMENT_IMAGE", bootMetadata.environmentImage ?? ""); + upsertEnv(container.env, "HWLAB_ENVIRONMENT_DIGEST", bootMetadata.environmentDigest ?? ""); + upsertEnv(container.env, "HWLAB_ENVIRONMENT_INPUT_HASH", bootMetadata.environmentInputHash ?? ""); + upsertEnv(container.env, "HWLAB_CODE_INPUT_HASH", bootMetadata.codeInputHash ?? ""); + upsertEnv(container.env, "HWLAB_IMAGE_DIGEST", bootMetadata.environmentDigest ?? "unknown"); + applyEnvReuseStartupProbe(container); +} + +function applyEnvReuseStartupProbe(container) { + const probe = container.readinessProbe ?? container.livenessProbe; + if (!probe?.httpGet && !probe?.tcpSocket && !probe?.exec) return; + container.startupProbe ??= { + ...cloneJson(probe), + periodSeconds: 10, + timeoutSeconds: Math.max(Number(probe.timeoutSeconds ?? 1), 2), + failureThreshold: 30, + successThreshold: 1 + }; +} + +function applyServiceHealthProbe(container, healthProbe) { + if (!healthProbe || typeof healthProbe !== "object" || Array.isArray(healthProbe)) return; + for (const [probeName, key] of [["readinessProbe", "readiness"], ["livenessProbe", "liveness"], ["startupProbe", "startup"]]) { + const probe = container?.[probeName]; + const timing = normalizeProbeTiming(healthProbe[key]); + if (!probe || Object.keys(timing).length === 0) continue; + Object.assign(probe, timing); + } +} + +function applyServiceShutdownDrainLifecycle(container, shutdownDrain) { + if (!container || !shutdownDrain || shutdownDrain.enabled !== true) return; + const pathValue = typeof shutdownDrain.path === "string" && /^\/[A-Za-z0-9/_:.-]+$/u.test(shutdownDrain.path) ? shutdownDrain.path : "/internal/workbench/drain"; + const sleepSeconds = boundedInteger(shutdownDrain.sleepSeconds, 3, 0, 30); + const timeoutMs = boundedInteger(shutdownDrain.timeoutMs, 2500, 100, 30000); + container.env ??= []; + upsertEnv(container.env, "HWLAB_WORKBENCH_SSE_DRAIN_TIMEOUT_MS", String(timeoutMs)); + const command = [ + 'port="${PORT:-${HWLAB_PORT:-${HWLAB_CLOUD_API_PORT:-6667}}}"', + `url="http://127.0.0.1:\${port}${pathValue}"`, + '/usr/local/bin/bun -e \'const [url, hostname] = process.argv.slice(2); await fetch(url, { method: "POST", headers: { "x-hwlab-drain-hostname": hostname || "" } }).catch(() => {});\' "$url" "${HOSTNAME:-}" >/dev/null 2>&1 || true', + `sleep ${sleepSeconds}` + ].join("; "); + container.lifecycle ??= {}; + container.lifecycle.preStop = { exec: { command: ["sh", "-c", command] } }; +} + +function boundedInteger(value, fallback, minimum, maximum) { + const parsed = Number(value); + if (!Number.isInteger(parsed)) return fallback; + return Math.max(minimum, Math.min(maximum, parsed)); +} + +function normalizeProbeTiming(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) return {}; + const result = {}; + for (const field of ["initialDelaySeconds", "periodSeconds", "timeoutSeconds", "failureThreshold", "successThreshold"]) { + const raw = value[field]; + const minimum = field === "initialDelaySeconds" ? 0 : 1; + if (Number.isInteger(raw) && raw >= minimum) result[field] = raw; + } + return result; +} + +function useLocalHealthProbe(container, pathValue = "/health") { + if (!container) return; + for (const probeName of ["readinessProbe", "livenessProbe"]) { + const probe = container[probeName]; + if (!probe?.httpGet) continue; + probe.httpGet.path = pathValue; + } +} + +function annotateEnvReusePodTemplate(podMetadata, bootMetadata) { + if (!podMetadata || !bootMetadata) return; + annotate(podMetadata, { + "hwlab.pikastech.local/runtime-mode": bootMetadata.runtimeMode, + "hwlab.pikastech.local/boot-repo": bootMetadata.bootRepo, + "hwlab.pikastech.local/boot-commit": bootMetadata.bootCommit, + "hwlab.pikastech.local/boot-sh": bootMetadata.bootSh, + "hwlab.pikastech.local/environment-digest": bootMetadata.environmentDigest ?? "not_published", + "hwlab.pikastech.local/environment-input-hash": bootMetadata.environmentInputHash ?? "unknown", + "hwlab.pikastech.local/code-input-hash": bootMetadata.codeInputHash ?? "unknown" + }); + label(podMetadata, { + "hwlab.pikastech.local/runtime-mode": bootMetadata.runtimeMode, + "hwlab.pikastech.local/boot-commit": bootMetadata.bootCommit + }); +} + +function runtimeArtifactForService({ catalog, deploy, serviceId, source, registryPrefix, useDeployImages = false, digestPin = false, envReuseServiceIds = null }) { + const catalogService = catalogServiceById(catalog, serviceId); + if (v02EnvReuseEnabled(catalog, serviceId, envReuseServiceIds)) { + const environmentImage = catalogService?.environmentImage ?? imageFor(registryPrefix, `${serviceId}-env`, `env-${String(catalogService?.environmentInputHash ?? source.full).slice(0, 12)}`); + const runtimeImage = digestPin && useDeployImages ? digestPinnedImage(environmentImage, catalogService?.environmentDigest ?? catalogService?.digest) : environmentImage; + return { + image: runtimeImage, + imageTag: imageTagFromReference(environmentImage) ?? source.imageTag ?? source.short, + commit: source.full, + runtimeMode: v02EnvReuseRuntimeMode, + environmentDigest: catalogService?.environmentDigest ?? catalogService?.digest ?? null + }; + } + const fallbackImageTag = source.imageTag ?? source.short; + const image = useDeployImages && catalogService?.image + ? catalogService.image + : imageFor(registryPrefix, serviceId, fallbackImageTag); + const runtimeImage = digestPin && useDeployImages ? digestPinnedImage(image, catalogService?.digest) : image; + const imageTag = useDeployImages && catalogService?.imageTag + ? catalogService.imageTag + : imageTagFromReference(image) ?? fallbackImageTag; + const commit = useDeployImages + ? catalogService?.sourceCommitId ?? catalogService?.commitId ?? imageTag ?? source.full + : source.full; + return { image: runtimeImage, imageTag, commit }; +} + +function runtimeImageForService(options) { + return runtimeArtifactForService(options).image; +} + +function runtimeCommitForService(options) { + return runtimeArtifactForService(options).commit; +} + +function runtimeImageTagForService(options) { + return runtimeArtifactForService(options).imageTag; +} + +function artifactEnvironment(args) { + return isRuntimeLane(args.lane) ? args.lane : "dev"; +} + +function artifactEndpoint(args) { + return isRuntimeLane(args.lane) ? args.runtimeEndpoint : defaultRuntimeEndpoint; +} + +function serviceIdsForLane(lane, deploy = null) { + return isRuntimeLane(lane) ? runtimeLaneServiceIdsFromDeploy(deploy, lane) : defaultServiceIds; +} + +function serviceIdsForProfile(profile, deploy = null) { + return isRuntimeLane(profile) ? runtimeLaneServiceIdsFromDeploy(deploy, profile) : defaultServiceIds; +} + +function runtimeLaneServiceIdsFromDeploy(deploy, lane) { + const laneConfig = runtimeLaneConfig(deploy, lane); + const serviceIds = uniquePreserveOrder((Array.isArray(laneConfig.envReuseServices) ? laneConfig.envReuseServices : []) + .map((item) => String(item ?? "").trim()) + .filter(Boolean)); + assert.ok(serviceIds.length > 0, `deploy.lanes.${lane}.envReuseServices must not be empty`); + return serviceIds; +} + +function v02ServiceIdsFromDeploy(deploy) { + return runtimeLaneServiceIdsFromDeploy(deploy, "v02"); +} + +function runtimeLaneObservableServicesForDeploy(deploy, lane) { + const laneConfig = runtimeLaneConfig(deploy, lane); + const declarations = laneConfig.serviceDeclarations ?? {}; + const serviceIds = new Set(runtimeLaneServiceIdsFromDeploy(deploy, lane)); + const declared = Object.entries(declarations) + .filter(([serviceId, declaration]) => serviceIds.has(serviceId) && declaration?.observable !== false && Number.isInteger(declaration?.healthPort)) + .map(([serviceId, declaration]) => ({ + serviceId, + port: declaration.healthPort, + healthPath: declaration.healthPath || "/health/live" + })); + return [...declared, ...v02ExtraObservableServices]; +} + +function v02ObservableServicesForDeploy(deploy) { + return runtimeLaneObservableServicesForDeploy(deploy, "v02"); +} + +function v02ObservableService(deploy, serviceId) { + return v02ObservableServicesForDeploy(deploy).find((item) => item.serviceId === serviceId) ?? null; +} + +function runtimeLaneObservableService(deploy, lane, serviceId) { + return runtimeLaneObservableServicesForDeploy(deploy, lane).find((item) => item.serviceId === serviceId) ?? null; +} + +function runtimeLaneObservableServiceIdsForDeploy(deploy, lane) { + return new Set(runtimeLaneObservableServicesForDeploy(deploy, lane).map((item) => item.serviceId)); +} + +function v02ObservableServiceIdsForDeploy(deploy) { + return runtimeLaneObservableServiceIdsForDeploy(deploy, "v02"); +} + +function servicesParamForLane(lane, deploy = null) { + return serviceIdsForLane(lane, deploy).join(","); +} + +function uniquePreserveOrder(values) { + const seen = new Set(); + const result = []; + for (const value of values) { + if (seen.has(value)) continue; + seen.add(value); + result.push(value); + } + return result; +} + +function isV02RemovedServiceId(serviceId) { + return v02RemovedServiceIds.has(serviceId); +} + +function artifactCatalogSkeleton({ args, deploy, source }) { + const environment = artifactEnvironment(args); + const namespace = isRuntimeLane(args.lane) ? namespaceNameForProfile(args.lane) : "hwlab-dev"; + const tag = imageTagForSource(args, source); + return { + catalogVersion: "v1", + kind: "hwlab-artifact-catalog", + environment, + profile: environment, + namespace, + endpoint: artifactEndpoint(args), + commitId: tag, + artifactState: "contract-skeleton", + publish: { + registryPrefix: args.registryPrefix, + sourceCommitId: source.full, + imageTag: tag, + publishedAt: null + }, + allowedProfiles: [environment], + forbiddenProfiles: isRuntimeLane(args.lane) ? ["dev", "prod"] : ["prod"], + services: serviceIdsForLane(args.lane, deploy).map((serviceId) => artifactCatalogSkeletonService({ args, deploy, source, serviceId, environment, namespace, tag })) + }; +} + +function artifactCatalogSkeletonService({ args, deploy, source, serviceId, environment, namespace, tag }) { + const base = { + serviceId, + profile: environment, + namespace, + commitId: tag, + sourceCommitId: source.full, + image: imageFor(args.registryPrefix, serviceId, tag), + imageTag: tag, + digest: null, + buildBackend: "contract-skeleton" + }; + const envReuseServiceIds = envReuseServiceIdsForLane(deploy, args.lane); + if (!envReuseServiceIds.has(serviceId)) return base; + const bootSh = bootShForService(deployServiceForBoot(deploy, serviceId, args.lane), serviceId); + const environmentImage = imageFor(args.registryPrefix, `${serviceId}-env`, `env-${String(source.full).slice(0, 12)}`); + return { + ...base, + image: environmentImage, + imageTag: imageTagFromReference(environmentImage), + runtimeMode: v02EnvReuseRuntimeMode, + envReuse: true, + environmentImage, + environmentDigest: null, + environmentInputHash: null, + bootRepo: args.sourceRepo, + bootCommit: source.full, + bootSh, + codeInputHash: null, + bootEnvEvidence: { + env: { + HWLAB_BOOT_REPO: args.sourceRepo, + HWLAB_BOOT_COMMIT: source.full, + HWLAB_BOOT_SH: bootSh + } + } + }; +} + +function namespaceNameForProfile(profile) { + if (isRuntimeLane(profile)) return `hwlab-${profile}`; + return profile === "prod" ? "hwlab-prod" : "hwlab-dev"; +} + +function runtimePathForProfile(profile) { + if (isRuntimeLane(profile)) return `runtime-${profile}`; + return profile === "prod" ? "runtime-prod" : "runtime-dev"; +} + +function runtimeLabelForProfile(profile) { + if (isRuntimeLane(profile)) return profile; + return profile === "prod" ? "prod" : "dev"; +} + +function gitopsTargetForProfile(profile) { + return isRuntimeLane(profile) ? profile : "node"; +} + +function gitopsRootForArgs(args) { + return String(args.gitopsRoot || defaultOutDir).replace(/\/+$/u, ""); +} + +function gitopsPathForProfile(args, profile) { + return `${gitopsRootForArgs(args)}/${runtimePathForProfile(profile)}`; +} + +function gitopsPathFor(args, relativePath) { + return `${gitopsRootForArgs(args)}/${String(relativePath).replace(/^\/+/, "")}`; +} + +function profileEndpoint(args, profile) { + if (isRuntimeLane(profile)) return { runtimeEndpoint: args.runtimeEndpoint, webEndpoint: args.webEndpoint }; + return profile === "prod" + ? { runtimeEndpoint: args.prodRuntimeEndpoint, webEndpoint: args.prodWebEndpoint } + : { runtimeEndpoint: args.runtimeEndpoint, webEndpoint: args.webEndpoint }; +} + +function laneRuntimeProfiles(args) { + return isRuntimeLane(args.lane) ? [args.lane] : ["dev", "prod"]; +} + +function laneNames(args) { + if (isRuntimeLane(args.lane)) { + const namespace = namespaceNameForProfile(args.lane); + return { + gitopsTarget: args.lane, + pipeline: `${namespace}-ci-image-publish`, + poller: `${namespace}-branch-poller`, + reconciler: `${namespace}-control-plane-reconciler`, + serviceAccount: `${namespace}-tekton-runner`, + pipelineRunPrefix: `${namespace}-ci-poll-`, + runtimeNamespace: namespace, + runtimePath: gitopsPathForProfile(args, args.lane), + argoApplications: [`argocd/hwlab-node-${args.lane}`] + }; + } + return { + gitopsTarget: "node", + pipeline: "hwlab-node-ci-image-publish", + poller: "hwlab-node-branch-poller", + reconciler: "hwlab-node-control-plane-reconciler", + serviceAccount: "hwlab-tekton-runner", + pipelineRunPrefix: "hwlab-node-ci-poll-", + runtimeNamespace: "hwlab-dev", + runtimePath: gitopsPathForProfile(args, "dev"), + argoApplications: ["argocd/hwlab-node-dev", "argocd/hwlab-node-prod"] + }; +} + +function argoApplicationName(profile) { + return `hwlab-node-${profile}`; +} + +function imageTagForSource(args, source) { + return args.imageTagMode === "full" ? source.full : source.short; +} + +function deepSeekBaseUrl(namespace) { + return `http://hwlab-deepseek-proxy.${namespace}.svc.cluster.local:4000/v1/responses`; +} + +function codeAgentNoProxy(namespace) { + return [ + `${namespace}.svc.cluster.local`, + ".svc", + ".cluster.local", + "hyueapi.com", + ".hyueapi.com", + "hyui.com", + ".hyui.com", + "127.0.0.1", + "localhost", + "::1", + "10.0.0.0/8", + "10.42.0.0/16", + "10.43.0.0/16", + "api.minimaxi.com", + ".minimaxi.com" + ].join(","); +} + +function namespaceScopedHost(value, namespace) { + if (typeof value !== "string") return value; + return value.replace(/\.hwlab-dev\.svc\.cluster\.local/gu, `.${namespace}.svc.cluster.local`); +} + +function secretNameForProfile(secretName, profile) { + if (!isRuntimeLane(profile)) return secretName; + if (secretName === "hwlab-cloud-api-dev-db") return `hwlab-cloud-api-${profile}-db`; + if (secretName === "hwlab-code-agent-provider") return `hwlab-${profile}-code-agent-provider`; + if (secretName === "hwlab-code-agent-codex-auth") return `hwlab-${profile}-code-agent-codex-auth`; + if (secretName === "hwlab-opencode-server-auth") return `hwlab-${profile}-opencode-server-auth`; + if (secretName === "hwlab-bootstrap-admin") return `hwlab-${profile}-bootstrap-admin`; + return secretName; +} + +function secretRefObjectForProfile(secretRef, profile) { + const ref = configString(secretRef); + const slashIndex = ref.indexOf("/"); + assert.ok(slashIndex > 0 && slashIndex < ref.length - 1, `secretRef ${JSON.stringify(secretRef)} must be name/key`); + return { name: secretNameForProfile(ref.slice(0, slashIndex), profile), key: ref.slice(slashIndex + 1), optional: true }; +} + +function rewritePodSecretRefs(podSpec, profile) { + if (!podSpec || !isRuntimeLane(profile)) return; + for (const volume of podSpec.volumes ?? []) { + if (volume?.secret?.secretName) volume.secret.secretName = secretNameForProfile(volume.secret.secretName, profile); + } +} + +function deployEnvEntry(name, value, namespace, profile = "dev") { + if (typeof value === "string" && value.startsWith("secretRef:")) { + return { name, valueFrom: { secretKeyRef: secretRefObjectForProfile(value.slice("secretRef:".length), profile) } }; + } + return { name, value: namespaceScopedHost(value, namespace) }; +} + +function applyDeployEnv(envList, deployService, namespace, profile = "dev") { + if (!deployService?.env || typeof deployService.env !== "object" || Array.isArray(deployService.env)) return; + for (const [name, value] of Object.entries(deployService.env)) { + upsertEnvEntry(envList, deployEnvEntry(name, value, namespace, profile)); + } +} + +function applyDeployConfigMapMounts(podSpec, container, deployService) { + const mounts = Array.isArray(deployService?.configMapMounts) ? deployService.configMapMounts : []; + if (mounts.length === 0 || !podSpec || !container) return; + podSpec.volumes ??= []; + container.volumeMounts ??= []; + for (const mount of mounts) { + const name = String(mount?.name ?? "").trim(); + const configMapName = String(mount?.configMapName ?? "").trim(); + const mountPath = String(mount?.mountPath ?? "").trim(); + if (!name || !configMapName || !mountPath) continue; + const items = Array.isArray(mount.items) + ? mount.items + .map((item) => ({ key: String(item?.key ?? "").trim(), path: String(item?.path ?? "").trim() })) + .filter((item) => item.key && item.path) + : []; + const volume = { name, configMap: { name: configMapName } }; + if (items.length > 0) volume.configMap.items = items; + const existingVolume = podSpec.volumes.find((entry) => entry?.name === name); + if (existingVolume) Object.assign(existingVolume, volume); + else podSpec.volumes.push(volume); + const volumeMount = { name, mountPath, readOnly: mount.readOnly !== false }; + const existingMount = container.volumeMounts.find((entry) => entry?.name === name); + if (existingMount) Object.assign(existingMount, volumeMount); + else container.volumeMounts.push(volumeMount); + } +} + +function removeV02LegacySimulatorEnv(envList) { + if (!Array.isArray(envList)) return envList; + return envList.filter((entry) => !v02RemovedServiceEnvNames.has(entry?.name)); +} + +function removeV02RepoOwnedCodeAgentPodConfig(deploymentSpec, podSpec) { + if (!podSpec) return; + podSpec.initContainers = (podSpec.initContainers ?? []).filter((entry) => entry?.name !== "hwlab-code-agent-workspace-init"); + const removedVolumeNames = new Set([ + "hwlab-code-agent-workspace", + "hwlab-code-agent-codex-home", + "hwlab-code-agent-codex-config", + "hwlab-code-agent-codex-auth" + ]); + podSpec.volumes = (podSpec.volumes ?? []).filter((volume) => !removedVolumeNames.has(volume?.name)); + for (const container of podSpec.containers ?? []) { + container.volumeMounts = (container.volumeMounts ?? []).filter((mount) => !removedVolumeNames.has(mount?.name)); + } + if (deploymentSpec?.strategy?.type === "Recreate") delete deploymentSpec.strategy; +} + +function applyDeployServiceReplicas(item, deployService) { + if (!(item?.kind === "Deployment" || item?.kind === "StatefulSet")) return; + const replicas = deployService?.replicas; + if (!Number.isInteger(replicas) || replicas < 0) return; + item.spec ??= {}; + item.spec.replicas = replicas; +} + +function deployServicesForProfile(deploy, profile) { + const allowed = new Set(serviceIdsForProfile(profile, deploy)); + const services = new Map((deploy.services ?? []) + .filter((service) => allowed.has(service.serviceId)) + .map((service) => { + const copy = cloneJson(service); + copy.env = mergeEnvMaps( + copy.env, + serviceDeclarationEnvForProfile(deploy, profile, service.serviceId), + accessControlEnvForProfile(deploy, profile, service.serviceId), + runtimeStoreEnvForProfile(deploy, profile, service.serviceId), + workbenchRuntimeClientEnvForProfile(deploy, profile, service.serviceId), + workbenchRuntimeFactsQueryEnvForProfile(deploy, profile, service.serviceId), + workbenchRuntimeSessionsSummaryEnvForProfile(deploy, profile, service.serviceId), + workbenchRuntimePostgresEnvForProfile(deploy, profile, service.serviceId), + workbenchRuntimeRedisEnvForProfile(deploy, profile, service.serviceId) + ); + return [service.serviceId, copy]; + })); + const laneServices = isRuntimeLane(profile) && Array.isArray(deploy.lanes?.[profile]?.services) + ? deploy.lanes[profile].services + : []; + for (const override of laneServices) { + const existing = services.get(override.serviceId) ?? { + serviceId: override.serviceId, + env: mergeEnvMaps( + serviceDeclarationEnvForProfile(deploy, profile, override.serviceId), + accessControlEnvForProfile(deploy, profile, override.serviceId), + runtimeStoreEnvForProfile(deploy, profile, override.serviceId), + workbenchRuntimeClientEnvForProfile(deploy, profile, override.serviceId), + workbenchRuntimeFactsQueryEnvForProfile(deploy, profile, override.serviceId), + workbenchRuntimeSessionsSummaryEnvForProfile(deploy, profile, override.serviceId), + workbenchRuntimePostgresEnvForProfile(deploy, profile, override.serviceId), + workbenchRuntimeRedisEnvForProfile(deploy, profile, override.serviceId) + ) + }; + services.set(override.serviceId, { + ...existing, + ...cloneJson(override), + env: mergeEnvMaps(existing.env, override.env) + }); + } + return services; +} + +function accessControlEnvForProfile(deploy, profile, serviceId) { + return deploy?.__accessControlEnvByProfile?.[profile]?.[serviceId] ?? {}; +} + +function serviceDeclarationEnvForProfile(deploy, profile, serviceId) { + if (!isRuntimeLane(profile)) return {}; + const env = deploy?.lanes?.[profile]?.serviceDeclarations?.[serviceId]?.env; + if (!env || typeof env !== "object" || Array.isArray(env)) return {}; + return cloneJson(env); +} + +function runtimeWorkbenchTraceTimelinePolicy(deploy, profile) { + if (!isRuntimeLane(profile)) return null; + const traceTimeline = runtimeLaneConfig(deploy, profile)?.workbench?.traceTimeline; + if (!traceTimeline || typeof traceTimeline !== "object" || Array.isArray(traceTimeline)) return null; + const result = {}; + if (typeof traceTimeline.autoExpandRunning === "boolean") result.autoExpandRunning = traceTimeline.autoExpandRunning; + if (typeof traceTimeline.autoCollapseTerminal === "boolean") result.autoCollapseTerminal = traceTimeline.autoCollapseTerminal; + return Object.keys(result).length > 0 ? result : null; +} + +function runtimeTraceExplorerUrlTemplate(deploy, profile) { + if (!isRuntimeLane(profile)) return null; + const template = runtimeLaneConfig(deploy, profile)?.observability?.traceExplorerUrlTemplate; + return typeof template === "string" && template.trim().length > 0 ? template.trim() : null; +} + +function runtimePrometheusOperatorResourcesEnabled(deploy, profile) { + if (!isRuntimeLane(profile)) return true; + const enabled = runtimeLaneConfig(deploy, profile)?.observability?.prometheusOperatorResources; + return enabled !== false; +} + +function effectiveSecretPlaneNodeId(args = {}) { + return gitopsRootNodeId(args.gitopsRoot) ?? String(args.nodeId ?? "").trim().toUpperCase(); +} + +function gitopsRootNodeId(gitopsRoot) { + const normalized = String(gitopsRoot ?? "").replaceAll("\\", "/").replace(/\/+$/u, ""); + const match = normalized.match(/(?:^|\/)deploy\/gitops\/node\/([^/]+)$/u); + return match?.[1] ? match[1].trim().toUpperCase() : null; +} + +function runtimeFrpConfigForProfile(deploy, profile, args = {}) { + const fallbackWebRemotePort = isRuntimeLane(profile) ? defaultPortForRuntimeLane(profile) : profile === "prod" ? 18666 : 17666; + const fallbackEdgeRemotePort = isRuntimeLane(profile) ? fallbackWebRemotePort + 1 : profile === "prod" ? 18667 : 17667; + const baseFrp = isRuntimeLane(profile) ? runtimeLaneConfig(deploy, profile)?.frp : deploy?.frp; + const nodeId = effectiveSecretPlaneNodeId(args); + const nodeFrp = nodeId ? baseFrp?.nodes?.[nodeId] : null; + const frp = nodeFrp && typeof nodeFrp === "object" && !Array.isArray(nodeFrp) ? { ...baseFrp, ...nodeFrp } : baseFrp; + const server = frp && typeof frp === "object" && !Array.isArray(frp) ? frp.server : null; + const proxies = Array.isArray(frp?.proxies) ? frp.proxies : []; + const proxyByEndpoint = new Map(proxies + .filter((proxy) => proxy && typeof proxy === "object" && !Array.isArray(proxy)) + .map((proxy) => [String(proxy.endpointId || proxy.name || ""), proxy])); + const proxyValue = (endpointId, key, fallback) => { + const value = proxyByEndpoint.get(endpointId)?.[key]; + return typeof value === "string" && value.trim().length > 0 ? value.trim() : fallback; + }; + const proxyPort = (endpointId, fallback) => { + const value = proxyByEndpoint.get(endpointId)?.remotePort; + return Number.isInteger(value) ? value : fallback; + }; + return { + serverAddr: typeof server?.address === "string" && server.address.trim().length > 0 ? server.address.trim() : "74.48.78.17", + serverPort: Number.isInteger(server?.bindPort) ? server.bindPort : 7000, + webRemotePort: proxyPort("frontend", fallbackWebRemotePort), + edgeRemotePort: proxyPort("api", fallbackEdgeRemotePort), + webProxyName: proxyValue("frontend", "name", null), + edgeProxyName: proxyValue("api", "name", null), + authSecretName: typeof frp?.auth?.secretName === "string" && frp.auth.secretName.trim().length > 0 ? frp.auth.secretName.trim() : null, + authSecretKey: typeof frp?.auth?.secretKey === "string" && frp.auth.secretKey.trim().length > 0 ? frp.auth.secretKey.trim() : null + }; +} + +function runtimeStoreEnvForProfile(deploy, profile, serviceId) { + if (!isRuntimeLane(profile) || serviceId !== "hwlab-cloud-api") return {}; + const postgres = runtimeLaneConfig(deploy, profile)?.runtimeStore?.postgres; + if (!postgres || typeof postgres !== "object" || Array.isArray(postgres)) return {}; + const env = {}; + if (postgres.sslMode === "disable" || postgres.sslMode === "require") env.HWLAB_CLOUD_DB_SSL_MODE = postgres.sslMode; + if (Number.isInteger(postgres.poolMax)) env.HWLAB_CLOUD_DB_POOL_MAX = String(postgres.poolMax); + if (Number.isInteger(postgres.connectionTimeoutMs)) { + env.HWLAB_CLOUD_DB_PROBE_TIMEOUT_MS = String(postgres.connectionTimeoutMs); + } + if (Number.isInteger(postgres.queryTimeoutMs)) { + env.HWLAB_CLOUD_DB_QUERY_TIMEOUT_MS = String(postgres.queryTimeoutMs); + } + if (Number.isInteger(postgres.queryRetryMaxAttempts)) { + env.HWLAB_CLOUD_DB_QUERY_RETRY_MAX_ATTEMPTS = String(postgres.queryRetryMaxAttempts); + } + if (Number.isInteger(postgres.queryRetryInitialDelayMs)) { + env.HWLAB_CLOUD_DB_QUERY_RETRY_INITIAL_DELAY_MS = String(postgres.queryRetryInitialDelayMs); + } + if (Number.isInteger(postgres.queryRetryMaxDelayMs)) { + env.HWLAB_CLOUD_DB_QUERY_RETRY_MAX_DELAY_MS = String(postgres.queryRetryMaxDelayMs); + } + return env; +} + +function workbenchRuntimeRedisEnvForProfile(deploy, profile, serviceId) { + if (!isRuntimeLane(profile) || serviceId !== "hwlab-workbench-runtime") return {}; + const redis = workbenchRuntimeRedisConfigForProfile(deploy, profile); + if (!redis || typeof redis !== "object" || Array.isArray(redis)) return {}; + const env = { + HWLAB_WORKBENCH_CACHE_REDIS_ENABLED: booleanEnv(Boolean(redis.enabled)) + }; + const namespace = typeof redis.namespace === "string" ? redis.namespace.trim() : ""; + const serviceName = typeof redis.serviceName === "string" ? redis.serviceName.trim() : ""; + const port = Number.isInteger(redis.port) ? redis.port : null; + if (namespace) env.HWLAB_WORKBENCH_CACHE_REDIS_NAMESPACE = namespace; + if (serviceName) env.HWLAB_WORKBENCH_CACHE_REDIS_SERVICE_NAME = serviceName; + if (port !== null) env.HWLAB_WORKBENCH_CACHE_REDIS_PORT = String(port); + if (namespace && serviceName && port !== null) { + env.HWLAB_WORKBENCH_CACHE_REDIS_URL = `redis://${serviceName}.${namespace}.svc.cluster.local:${port}/0`; + } + const ttl = redis.ttl ?? {}; + if (Number.isInteger(ttl.sessionsSummaryMs)) env.HWLAB_WORKBENCH_CACHE_REDIS_TTL_SESSIONS_SUMMARY_MS = String(ttl.sessionsSummaryMs); + if (Number.isInteger(ttl.terminalTurnMs)) env.HWLAB_WORKBENCH_CACHE_REDIS_TTL_TERMINAL_TURN_MS = String(ttl.terminalTurnMs); + if (Number.isInteger(ttl.terminalTracePageMs)) env.HWLAB_WORKBENCH_CACHE_REDIS_TTL_TERMINAL_TRACE_PAGE_MS = String(ttl.terminalTracePageMs); + if (Number.isInteger(ttl.runningPageMs)) env.HWLAB_WORKBENCH_CACHE_REDIS_TTL_RUNNING_PAGE_MS = String(ttl.runningPageMs); + const limits = redis.limits ?? {}; + if (Number.isInteger(limits.maxKeyBytes)) env.HWLAB_WORKBENCH_CACHE_REDIS_MAX_KEY_BYTES = String(limits.maxKeyBytes); + if (Number.isInteger(limits.maxPayloadBytes)) env.HWLAB_WORKBENCH_CACHE_REDIS_MAX_PAYLOAD_BYTES = String(limits.maxPayloadBytes); + const timeouts = redis.timeouts ?? {}; + if (Number.isInteger(timeouts.connectMs)) env.HWLAB_WORKBENCH_CACHE_REDIS_CONNECT_TIMEOUT_MS = String(timeouts.connectMs); + if (Number.isInteger(timeouts.operationMs)) env.HWLAB_WORKBENCH_CACHE_REDIS_OPERATION_TIMEOUT_MS = String(timeouts.operationMs); + const labels = redis.metrics?.labels; + if (labels && typeof labels === "object" && !Array.isArray(labels)) { + env.HWLAB_WORKBENCH_CACHE_REDIS_METRICS_LABELS = JSON.stringify(labels); + } + const behavior = redis.unavailableBehavior ?? {}; + if (typeof behavior.livenessFailure === "boolean") env.HWLAB_WORKBENCH_CACHE_REDIS_UNAVAILABLE_LIVENESS_FAILURE = booleanEnv(behavior.livenessFailure); + if (typeof behavior.readMode === "string") env.HWLAB_WORKBENCH_CACHE_REDIS_UNAVAILABLE_READ_MODE = behavior.readMode; + if (typeof behavior.fallbackStateSource === "boolean") env.HWLAB_WORKBENCH_CACHE_REDIS_UNAVAILABLE_FALLBACK_STATE_SOURCE = booleanEnv(behavior.fallbackStateSource); + if (typeof redis.disableSwitchEnv === "string" && redis.disableSwitchEnv.trim()) { + env[redis.disableSwitchEnv.trim()] = booleanEnv(Boolean(redis.enabled)); + } + return env; +} + +function workbenchRuntimeClientEnvForProfile(deploy, profile, serviceId) { + if (!isRuntimeLane(profile) || serviceId !== "hwlab-cloud-api") return {}; + const client = runtimeLaneConfig(deploy, profile)?.workbenchRuntime?.client; + if (!client || typeof client !== "object" || Array.isArray(client)) return {}; + const env = {}; + if (Number.isInteger(client.timeoutMs)) env.HWLAB_WORKBENCH_RUNTIME_TIMEOUT_MS = String(client.timeoutMs); + return env; +} + +function workbenchRuntimeFactsQueryEnvForProfile(deploy, profile, serviceId) { + if (!isRuntimeLane(profile) || serviceId !== "hwlab-cloud-api") return {}; + const factsQuery = runtimeLaneConfig(deploy, profile)?.workbenchRuntime?.factsQuery; + if (!factsQuery || typeof factsQuery !== "object" || Array.isArray(factsQuery)) return {}; + const env = {}; + if (Number.isInteger(factsQuery.maxAttempts)) env.HWLAB_WORKBENCH_FACTS_QUERY_MAX_ATTEMPTS = String(factsQuery.maxAttempts); + if (Number.isInteger(factsQuery.backoffBaseMs)) env.HWLAB_WORKBENCH_FACTS_QUERY_BACKOFF_BASE_MS = String(factsQuery.backoffBaseMs); + if (Number.isInteger(factsQuery.backoffMaxMs)) env.HWLAB_WORKBENCH_FACTS_QUERY_BACKOFF_MAX_MS = String(factsQuery.backoffMaxMs); + return env; +} + +function workbenchRuntimeSessionsSummaryEnvForProfile(deploy, profile, serviceId) { + if (!isRuntimeLane(profile) || serviceId !== "hwlab-workbench-runtime") return {}; + const sessionsSummary = runtimeLaneConfig(deploy, profile)?.workbenchRuntime?.sessionsSummary; + if (!sessionsSummary || typeof sessionsSummary !== "object" || Array.isArray(sessionsSummary)) return {}; + const env = {}; + if (Number.isInteger(sessionsSummary.maxAttempts)) env.HWLAB_WORKBENCH_SESSIONS_SUMMARY_QUERY_MAX_ATTEMPTS = String(sessionsSummary.maxAttempts); + if (Number.isInteger(sessionsSummary.attemptTimeoutMs)) env.HWLAB_WORKBENCH_SESSIONS_SUMMARY_QUERY_ATTEMPT_TIMEOUT_MS = String(sessionsSummary.attemptTimeoutMs); + if (Number.isInteger(sessionsSummary.backoffMs)) env.HWLAB_WORKBENCH_SESSIONS_SUMMARY_QUERY_BACKOFF_MS = String(sessionsSummary.backoffMs); + return env; +} + +function workbenchRuntimePostgresEnvForProfile(deploy, profile, serviceId) { + if (!isRuntimeLane(profile) || serviceId !== "hwlab-workbench-runtime") return {}; + const postgres = runtimeLaneConfig(deploy, profile)?.workbenchRuntime?.postgres; + if (!postgres || typeof postgres !== "object" || Array.isArray(postgres)) return {}; + const env = {}; + if (Number.isInteger(postgres.poolMax)) env.HWLAB_WORKBENCH_RUNTIME_DB_POOL_MAX = String(postgres.poolMax); + if (Number.isInteger(postgres.queryTimeoutMs)) env.HWLAB_WORKBENCH_RUNTIME_QUERY_TIMEOUT_MS = String(postgres.queryTimeoutMs); + if (Number.isInteger(postgres.readyTimeoutMs)) env.HWLAB_WORKBENCH_RUNTIME_READY_TIMEOUT_MS = String(postgres.readyTimeoutMs); + return env; +} + +function configObject(value, label) { + assert.ok(value && typeof value === "object" && !Array.isArray(value), `${label} must be an object`); + return value; +} + +function requiredConfigString(record, key, label) { + const value = record[key]; + assert.ok(typeof value === "string" && value.length > 0, `${label}.${key} must be a non-empty string`); + return value; +} + +function optionalConfigString(record, key, label) { + const value = record[key]; + if (value === undefined || value === null || value === "") return null; + assert.ok(typeof value === "string", `${label}.${key} must be a string when set`); + return value; +} + +function requiredConfigPositiveInteger(record, key, label) { + const value = record[key]; + assert.ok(Number.isInteger(value) && value > 0, `${label}.${key} must be a positive integer`); + return value; +} + +function workbenchRuntimeRedisConfigForProfile(deploy, profile) { + if (!isRuntimeLane(profile)) return null; + const label = `deploy.lanes.${profile}.workbenchRuntime.cache.redis`; + const redis = runtimeLaneConfig(deploy, profile)?.workbenchRuntime?.cache?.redis; + if (!redis || redis.enabled !== true) return null; + configObject(redis, label); + const namespace = requiredConfigString(redis, "namespace", label); + assert.equal(namespace, namespaceNameForProfile(profile), `${label}.namespace must match ${namespaceNameForProfile(profile)}`); + const serviceName = requiredConfigString(redis, "serviceName", label); + const image = requiredConfigString(redis, "image", label); + const port = requiredConfigPositiveInteger(redis, "port", label); + assert.ok(port <= 65535, `${label}.port must be a valid TCP port`); + const resources = configObject(redis.resources, `${label}.resources`); + configObject(resources.requests, `${label}.resources.requests`); + configObject(resources.limits, `${label}.resources.limits`); + const memoryPolicy = configObject(redis.memoryPolicy, `${label}.memoryPolicy`); + assert.equal(memoryPolicy.persistence, "disabled", `${label}.memoryPolicy.persistence must be disabled`); + requiredConfigString(memoryPolicy, "maxMemory", `${label}.memoryPolicy`); + requiredConfigString(memoryPolicy, "eviction", `${label}.memoryPolicy`); + return redis; +} + +function booleanEnv(value) { + return value ? "1" : "0"; +} + +function mergeEnvMaps(...values) { + const result = {}; + for (const value of values) { + if (!value || typeof value !== "object" || Array.isArray(value)) continue; + Object.assign(result, value); + } + return result; +} + +function runtimeNodeIdForProfile(deploy, profile, fallback = "node") { + if (!isRuntimeLane(profile)) return fallback; + if (typeof fallback === "string" && fallback.trim() && fallback !== "node") return fallback; + const nodeId = deploy?.lanes?.[profile]?.node; + return typeof nodeId === "string" && nodeId.length > 0 ? nodeId : fallback; +} + +function v02MetricsLabels(serviceId) { + return { + "hwlab.pikastech.local/monitoring": "enabled", + "hwlab.pikastech.local/service-id": serviceId + }; +} + +function v02MetricsSidecarVolume(profile = "v02") { + return { name: "hwlab-metrics-sidecar", configMap: { name: `hwlab-${profile}-metrics-sidecar` } }; +} + +function v02MetricsSidecarAnnotations(metricsSidecarSha256) { + return metricsSidecarSha256 ? { "hwlab.pikastech.local/metrics-sidecar-sha256": metricsSidecarSha256 } : {}; +} + +function v02MetricsSidecarContainer({ deploy, serviceId, namespace, gitopsTarget, profile = "v02" }) { + const service = runtimeLaneObservableService(deploy, profile, serviceId); + assert.ok(service, `unknown ${profile} observable service ${serviceId}`); + const extraMetricsEnv = serviceId === "hwlab-cloud-api" + ? [ + { name: "HWLAB_METRICS_EXTRA_URL", value: `http://127.0.0.1:${service.port}/v1/web-performance/metrics` }, + { name: "HWLAB_METRICS_EXTRA_TIMEOUT_MS", value: "2000" } + ] + : []; + return { + name: "hwlab-metrics", + image: ciToolsRunnerImage, + imagePullPolicy: "IfNotPresent", + command: ["node", "/metrics/metrics-sidecar.mjs"], + env: [ + { name: "HWLAB_METRICS_SERVICE_ID", value: serviceId }, + { name: "HWLAB_METRICS_NAMESPACE", value: namespace }, + { name: "HWLAB_METRICS_GITOPS_TARGET", value: gitopsTarget }, + { name: "HWLAB_METRICS_PORT", value: "9100" }, + { name: "HWLAB_METRICS_TARGET_URL", value: `http://127.0.0.1:${service.port}${service.healthPath}` }, + { name: "HWLAB_METRICS_TARGET_TIMEOUT_MS", value: "2000" }, + ...extraMetricsEnv + ], + ports: [{ name: "metrics", containerPort: 9100 }], + readinessProbe: { httpGet: { path: "/health/live", port: "metrics" }, initialDelaySeconds: 3, periodSeconds: 10 }, + livenessProbe: { httpGet: { path: "/health/live", port: "metrics" }, initialDelaySeconds: 10, periodSeconds: 20 }, + volumeMounts: [{ name: "hwlab-metrics-sidecar", mountPath: "/metrics", readOnly: true }] + }; +} + +function upsertV02MetricsSidecar(podSpec, options) { + if (!podSpec || !Array.isArray(podSpec.containers)) return; + podSpec.containers = podSpec.containers.filter((container) => container?.name !== "hwlab-metrics"); + podSpec.containers.push(v02MetricsSidecarContainer(options)); + podSpec.volumes ??= []; + podSpec.volumes = podSpec.volumes.filter((volume) => volume?.name !== "hwlab-metrics-sidecar"); + podSpec.volumes.push(v02MetricsSidecarVolume(options.profile)); +} + +function upsertV02MetricsPort(service) { + service.spec ??= {}; + service.spec.ports ??= []; + service.spec.ports = service.spec.ports.filter((port) => port?.name !== "metrics"); + service.spec.ports.push({ name: "metrics", port: 9100, targetPort: "metrics" }); +} + +function transformWorkloads({ workloads, deploy, catalog, source, sourceBranch = defaultBranch, gitReadUrl, registryPrefix, runtimeEndpoint, webEndpoint, profile = "dev", nodeId = "node", useDeployImages = false, metricsSidecarSha256 = null }) { + const result = cloneJson(workloads); + const deployServices = deployServicesForProfile(deploy, profile); + const namespace = namespaceNameForProfile(profile); + const profileLabel = runtimeLabelForProfile(profile); + const gitopsTarget = gitopsTargetForProfile(profile); + const runtimeLane = isRuntimeLane(profile); + const digestPin = runtimeLane; + const envReuseServiceIds = envReuseServiceIdsForLane(deploy, profile); + const runtimeObservableServiceIds = runtimeLane ? runtimeLaneObservableServiceIdsForDeploy(deploy, profile) : new Set(); + const stableRuntimeLabels = { + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/environment": profileLabel, + "hwlab.pikastech.local/profile": profileLabel + }; + result.items = asItems(result, "workloads").filter((item) => !(runtimeLane && isV02RemovedServiceId(serviceIdForWorkload(item, null)))); + for (const item of asItems(result, "workloads")) { + item.metadata.namespace = namespace; + label(item.metadata, { + ...stableRuntimeLabels, + "hwlab.pikastech.local/gitops-target": gitopsTarget, + "hwlab.pikastech.local/gitops-render-source-commit": source.full, + "hwlab.pikastech.local/source-commit": source.full, + "unidesk.ai/node-staging": profile === "dev" ? "true" : "false" + }); + annotate(item.metadata, { + "hwlab.pikastech.local/gitops-note": profile === "v02" ? "HWLAB v0.2 GitOps target." : profile === "prod" ? "node PROD GitOps target." : "node DEV GitOps target.", + "hwlab.pikastech.local/gitops-render-source-commit": source.full, + "hwlab.pikastech.local/source-commit": source.full + }); + if (item.kind === "Job") { + annotate(item.metadata, { + "argocd.argoproj.io/sync-options": "Force=true,Replace=true" + }); + if (item.spec?.suspend === true) { + annotate(item.metadata, { + "argocd.argoproj.io/ignore-healthcheck": "true" + }); + } + } + const podTemplate = item?.spec?.template; + if (!podTemplate?.spec?.containers) continue; + const templateServiceId = serviceIdForWorkload(item, podTemplate.spec.containers[0]); + const templateArtifact = runtimeArtifactForService({ catalog, deploy, serviceId: templateServiceId, source, registryPrefix, useDeployImages, digestPin, envReuseServiceIds }); + const templateSourceCommit = runtimeLane ? (templateArtifact.commit ?? source.full) : source.full; + const templateArtifactCommit = templateArtifact.commit; + label(podTemplate.metadata ??= {}, { + ...stableRuntimeLabels, + "hwlab.pikastech.local/gitops-target": gitopsTarget, + "hwlab.pikastech.local/gitops-render-source-commit": source.full, + "hwlab.pikastech.local/source-commit": templateSourceCommit, + "hwlab.pikastech.local/artifact-source-commit": templateArtifactCommit + }); + if (runtimeLane && runtimeObservableServiceIds.has(templateServiceId)) { + label(item.metadata, v02MetricsLabels(templateServiceId)); + label(podTemplate.metadata, v02MetricsLabels(templateServiceId)); + annotate(podTemplate.metadata, v02MetricsSidecarAnnotations(metricsSidecarSha256)); + upsertV02MetricsSidecar(podTemplate.spec, { deploy, serviceId: templateServiceId, namespace, gitopsTarget, profile }); + } + if ((item.kind === "Deployment" || item.kind === "StatefulSet") && item.spec?.selector?.matchLabels) { + item.spec.selector.matchLabels = { + ...item.spec.selector.matchLabels, + ...stableRuntimeLabels + }; + } + annotate(podTemplate.metadata, { + "hwlab.pikastech.local/gitops-render-source-commit": source.full, + "hwlab.pikastech.local/source-commit": templateSourceCommit, + "hwlab.pikastech.local/artifact-source-commit": templateArtifactCommit, + "hwlab.pikastech.local/rendered-by": "scripts/gitops-render.mjs" + }); + for (const container of podTemplate.spec.containers) { + if (container.name === "hwlab-metrics") continue; + const serviceId = serviceIdForWorkload(item, container); + const deployService = deployServices.get(serviceId); + if (!deployService) continue; + applyDeployServiceReplicas(item, deployService); + const artifact = runtimeArtifactForService({ catalog, deploy, serviceId, source, registryPrefix, useDeployImages, digestPin, envReuseServiceIds }); + const envReuse = runtimeLane && v02EnvReuseEnabled(catalog, serviceId, envReuseServiceIds); + const bootDeployService = envReuse ? deployServiceForBoot(deploy, serviceId, profile) : null; + const bootMetadata = envReuse ? bootMetadataForService({ args: { sourceRepo: deploy?.lanes?.[profile]?.sourceRepo || defaultSourceRepo, registryPrefix }, catalog, deployService: bootDeployService, serviceId, source }) : null; + const image = artifact.image; + const runtimeCommit = artifact.commit; + const runtimeImageTag = artifact.imageTag; + container.image = image; + container.imagePullPolicy = "IfNotPresent"; + container.env ??= []; + for (const entry of container.env) { + if (Object.hasOwn(entry, "value")) entry.value = namespaceScopedHost(entry.value, namespace); + } + applyDeployEnv(container.env, deployService, namespace, profile); + applyDeployConfigMapMounts(podTemplate.spec, container, deployService); + if (runtimeLane) container.env = removeV02LegacySimulatorEnv(container.env); + upsertEnv(container.env, "HWLAB_ENVIRONMENT", profileLabel); + upsertEnv(container.env, "HWLAB_COMMIT_ID", runtimeCommit); + upsertEnv(container.env, "HWLAB_IMAGE", image); + upsertEnv(container.env, "HWLAB_IMAGE_TAG", runtimeImageTag); + if (serviceId === "hwlab-agent-skills") { + upsertEnv(container.env, "HWLAB_SKILLS_COMMIT_ID", runtimeCommit); + } + upsertEnv(container.env, "HWLAB_GITOPS_TARGET", gitopsTarget); + upsertEnv(container.env, "HWLAB_GITOPS_PROFILE", profileLabel); + upsertEnv(container.env, "HWLAB_GITOPS_SOURCE_COMMIT", runtimeLane ? runtimeCommit : source.full); + if (runtimeLane && serviceId === "hwlab-cloud-api") { + upsertEnv(container.env, "HWLAB_CODE_AGENT_AGENTRUN_SOURCE_COMMIT", runtimeCommit); + } + if (bootMetadata) { + applyEnvReuseBootEnv(container, bootMetadata, { + sourceBranch, + gitReadUrl, + bootSh: envReuseBootShForContainer({ serviceId, container, defaultBootSh: bootMetadata.bootSh }) + }); + if (container.name === serviceId) annotateEnvReusePodTemplate(podTemplate.metadata, bootMetadata); + } + if (runtimeLane && container.name === serviceId) { + applyServiceHealthProbe(container, deploy?.lanes?.[profile]?.serviceDeclarations?.[serviceId]?.healthProbe); + } + if (serviceId === "hwlab-cloud-api" || serviceId === "hwlab-edge-proxy") { + upsertEnv(container.env, "HWLAB_PUBLIC_ENDPOINT", runtimeEndpoint); + } + if (runtimeLane && serviceId === "hwlab-edge-proxy") { + useLocalHealthProbe(container, "/health"); + } + if (serviceId === "hwlab-cloud-api") { + if (runtimeLane) { + removeV02RepoOwnedCodeAgentPodConfig(item.spec, podTemplate.spec); + applyServiceShutdownDrainLifecycle(container, deployService.shutdownDrain); + upsertEnv(container.env, "HWLAB_M3_IO_CONTROL_ENABLED", "false"); + upsertEnv(container.env, "HWLAB_CODE_AGENT_AGENTRUN_REPO_URL", gitReadUrl ?? defaultRuntimeLaneGitReadUrl); + upsertEnv(container.env, "HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID", runtimeNodeIdForProfile(deploy, profile, nodeId)); + upsertEnv(container.env, "HWLAB_OPENFGA_MODE", "enforce"); + upsertEnv(container.env, "HWLAB_OPENFGA_API_URL", `http://hwlab-openfga.${namespace}.svc.cluster.local:8080`); + upsertEnvEntry(container.env, deployEnvEntry("HWLAB_OPENFGA_AUTHN_TOKEN", `secretRef:hwlab-${profile}-openfga/authn-preshared-key`, namespace, profile)); + upsertEnv(container.env, "UNIDESK_MAIN_SERVER_IP", "http://74.48.78.17:18081"); + } else { + syncCodeAgentWorkspaceInitContainer(podTemplate.spec.initContainers, { image, runtimeCommit, runtimeImageTag }); + } + upsertEnv(container.env, "HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE", runtimeLane ? (configString(runtimeLaneConfig(deploy, profile)?.opencode?.providerProfile) || "dsflash-go") : "deepseek"); + if (runtimeLane) upsertEnv(container.env, "HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR", "repo-owned"); + upsertEnv(container.env, "HWLAB_CODE_AGENT_DEEPSEEK_MODEL", deepSeekProfileModel); + upsertEnv(container.env, "HWLAB_CODE_AGENT_DEEPSEEK_BASE_URL", deepSeekBaseUrl(namespace)); + upsertEnv(container.env, "HWLAB_CODE_AGENT_CODEX_API_MODEL", codexApiProfileModel); + upsertEnv(container.env, "HWLAB_CODE_AGENT_CODEX_API_BASE_URL", codexApiProfileBaseUrl); + upsertEnv(container.env, "HWLAB_CODE_AGENT_CODEX_API_UPSTREAM_BASE_URL", codexApiForwarderUpstreamBaseUrl); + upsertEnv(container.env, "HWLAB_CODE_AGENT_CODEX_API_FORWARDER_PORT", codexApiForwarderPort); + upsertEnv(container.env, "HWLAB_PREINSTALLED_SKILLS_DIR", "/app/skills"); + upsertEnv(container.env, "HWLAB_USER_SKILLS_DIR", "/data/user-skills"); + upsertEnv(container.env, "HWLAB_CODE_AGENT_SKILLS_DIRS", "/app/skills:/data/user-skills"); + upsertEnv(container.env, "NO_PROXY", codeAgentNoProxy(namespace)); + upsertEnv(container.env, "no_proxy", codeAgentNoProxy(namespace)); + } + if (serviceId === "hwlab-cloud-web") { + upsertEnv(container.env, "HWLAB_API_BASE_URL", `http://hwlab-cloud-api.${namespace}.svc.cluster.local:6667`); + upsertEnv(container.env, "HWLAB_PUBLIC_ENDPOINT", webEndpoint); + const traceTimelinePolicy = runtimeWorkbenchTraceTimelinePolicy(deploy, profile); + if (typeof traceTimelinePolicy?.autoExpandRunning === "boolean") { + upsertEnv(container.env, "HWLAB_WORKBENCH_TRACE_AUTO_EXPAND_RUNNING", booleanEnv(traceTimelinePolicy.autoExpandRunning)); + } + if (typeof traceTimelinePolicy?.autoCollapseTerminal === "boolean") { + upsertEnv(container.env, "HWLAB_WORKBENCH_TRACE_AUTO_COLLAPSE_TERMINAL", booleanEnv(traceTimelinePolicy.autoCollapseTerminal)); + } + const traceExplorerUrlTemplate = runtimeTraceExplorerUrlTemplate(deploy, profile); + if (traceExplorerUrlTemplate) { + upsertEnv(container.env, "HWLAB_WORKBENCH_TRACE_EXPLORER_URL_TEMPLATE", traceExplorerUrlTemplate); + } + } + } + rewritePodSecretRefs(podTemplate.spec, profile); + } + return result; +} + +function transformHealthContract(value, namespace, labels, annotations, runtimeEndpoint, webEndpoint, profile = "dev") { + const result = transformListNamespace(value, namespace, labels, annotations); + if (result.metadata?.name === "hwlab-dev-health-contract") { + result.metadata.name = `hwlab-${profile}-health-contract`; + } + if (result.metadata?.labels && Object.hasOwn(result.metadata.labels, "app.kubernetes.io/name")) { + result.metadata.labels["app.kubernetes.io/name"] = `hwlab-${profile}-health-contract`; + } + if (result.data && typeof result.data === "object") { + if (Object.hasOwn(result.data, "endpoint")) result.data.endpoint = runtimeEndpoint; + if (Object.hasOwn(result.data, "cloud-web")) { + result.data["cloud-web"] = `GET /health/live on ${webEndpoint}; consumes cloud-api only`; + } + if (Object.hasOwn(result.data, "cloud-api")) { + result.data["cloud-api"] = `GET /health/live through ${runtimeEndpoint}`; + } + if (Object.hasOwn(result.data, "tunnel-client")) { + result.data["tunnel-client"] = "disabled for node GitOps; no FRP or production traffic tunnel is required"; + } + } + return result; +} + +function transformListNamespace(value, namespace, labels, annotations) { + const result = cloneJson(value); + if (result.kind === "List") { + for (const item of result.items ?? []) { + item.metadata ??= {}; + item.metadata.namespace = namespace; + label(item.metadata, labels); + annotate(item.metadata, annotations); + } + } else { + result.metadata ??= {}; + result.metadata.namespace = namespace; + label(result.metadata, labels); + annotate(result.metadata, annotations); + } + return result; +} + +function transformServices({ services, namespace, labels, annotations, profile = "dev", deploy = null }) { + const observableServiceIds = isRuntimeLane(profile) ? runtimeLaneObservableServiceIdsForDeploy(deploy, profile) : new Set(); + const result = transformListNamespace(services, namespace, labels, annotations); + if (result.kind === "List") { + result.items = (result.items ?? []).filter((item) => !(isRuntimeLane(profile) && isV02RemovedServiceId(serviceIdForWorkload(item, null)))); + for (const item of result.items) { + const serviceId = serviceIdForWorkload(item, null); + if (!isRuntimeLane(profile) || !observableServiceIds.has(serviceId)) continue; + label(item.metadata ??= {}, v02MetricsLabels(serviceId)); + upsertV02MetricsPort(item); + } + } + return result; +} + +function observabilityManifest({ deploy, profile, namespace, labels, annotations, metricsSidecarScript }) { + assert.ok(isRuntimeLane(profile), `observability profile must be a runtime lane, got ${profile}`); + const includePrometheusOperatorResources = runtimePrometheusOperatorResourcesEnabled(deploy, profile); + const baseLabels = { + ...labels, + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/monitoring": "enabled" + }; + const serviceMonitors = runtimeLaneObservableServicesForDeploy(deploy, profile).map((service) => ({ + apiVersion: "monitoring.coreos.com/v1", + kind: "ServiceMonitor", + metadata: { + name: `hwlab-${profile}-${service.serviceId}`, + namespace, + labels: { ...baseLabels, "hwlab.pikastech.local/service-id": service.serviceId }, + annotations + }, + spec: { + namespaceSelector: { matchNames: [namespace] }, + selector: { + matchLabels: { + "hwlab.pikastech.local/gitops-target": profile, + "hwlab.pikastech.local/monitoring": "enabled", + "hwlab.pikastech.local/service-id": service.serviceId + } + }, + targetLabels: [ + "hwlab.pikastech.local/gitops-target", + "hwlab.pikastech.local/service-id" + ], + endpoints: [{ port: "metrics", path: "/metrics", interval: "30s", scrapeTimeout: "10s" }] + } + })); + return { + apiVersion: "v1", + kind: "List", + items: [ + { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { name: `hwlab-${profile}-metrics-sidecar`, namespace, labels: baseLabels, annotations }, + data: { "metrics-sidecar.mjs": metricsSidecarScript } + }, + ...(includePrometheusOperatorResources ? serviceMonitors : []), + ...(includePrometheusOperatorResources ? [{ + apiVersion: "monitoring.coreos.com/v1", + kind: "PrometheusRule", + metadata: { name: `hwlab-${profile}-observability`, namespace, labels: baseLabels, annotations }, + spec: { + groups: [{ + name: `hwlab-${profile}-observability`, + rules: [ + { + alert: `HWLAB${profile.toUpperCase()}MetricsTargetDown`, + expr: `up{namespace="${namespace}"} == 0`, + for: "5m", + labels: { severity: "warning", lane: profile }, + annotations: { summary: `HWLAB ${versionNameForRuntimeLane(profile)} metrics target is down`, description: `Prometheus cannot scrape a HWLAB ${versionNameForRuntimeLane(profile)} metrics target.` } + }, + { + alert: `HWLAB${profile.toUpperCase()}ServiceHealthProbeFailed`, + expr: `hwlab_service_health_probe_success{namespace="${namespace}"} == 0`, + for: "5m", + labels: { severity: "warning", lane: profile }, + annotations: { summary: "HWLAB v0.2 service health probe failed", description: "The metrics sidecar cannot reach the service health endpoint inside the pod network." } + } + ] + }] + } + }] : []) + ] + }; +} + +export { + applyDeployConfigMapMounts, applyDeployEnv, applyDeployServiceReplicas, + applyEnvReuseBootEnv, applyEnvReuseStartupProbe, applyRuntimeLaneDeployConfig, + argoApplicationName, artifactCatalogSkeleton, artifactEndpoint, artifactEnvironment, + asItems, attachAccessControlEnv, annotate, booleanEnv, bootMetadataForService, + bootShForService, boundedInteger, buildkitRunnerImage, catalogServiceById, + ciToolsRunnerImage, cloneJson, codeAgentNoProxy, codexApiForwarderPort, + codexApiForwarderUpstreamBaseUrl, codexApiProfileBaseUrl, codexApiProfileModel, + configObject, configString, deepSeekBaseUrl, deepSeekProfileModel, defaultAllProxyUrl, + defaultBranch, defaultCatalogPath, defaultDevBaseImage, defaultEndpointForRuntimeLane, + defaultGitopsBranch, defaultGitReadUrl, defaultNoProxy, defaultOutDir, + defaultPortForRuntimeLane, defaultProdRuntimeEndpoint, defaultProdWebEndpoint, + defaultProxyUrl, defaultRegistryPrefix, defaultRuntimeEndpoint, + defaultRuntimeLaneGitReadUrl, defaultRuntimeLaneGitWriteUrl, defaultServiceIds, + defaultSourceRepo, defaultV02GitReadUrl, defaultV02GitWriteUrl, + defaultV02RuntimeEndpoint, defaultV02WebEndpoint, defaultWebEndpoint, + deployEnvEntry, deployServiceForBoot, deployServicesForProfile, digestPinnedImage, + effectiveSecretPlaneNodeId, ensureObject, envReuseBootShForContainer, envReuseServiceIdsForLane, + generatedPath, gitopsPathFor, gitopsPathForProfile, gitopsRootForArgs, gitopsRootNodeId, + gitopsTargetForProfile, gitValue, imageFor, imageTagForSource, + imageTagFromReference, isRuntimeLane, isV02RemovedServiceId, jsonManifest, + k8sLabelHash, label, laneNames, laneRuntimeProfiles, mergeEnvMaps, + moonBridgeImage, moonBridgeSourceRef, namespaceNameForProfile, namespaceScopedHost, + normalizeAccessControlUser, normalizeNavProfile, normalizeProbeTiming, + observabilityManifest, optionalConfigString, optionalObject, parseArgs, primitiveValidationTasks, + profileEndpoint, proxyEnv, readConfigRef, readJson, readJsonIfPresent, + readOption, readStructuredFile, removeV02LegacySimulatorEnv, repoRoot, + removeV02RepoOwnedCodeAgentPodConfig, repositoryFromImage, + requiredAccessConfigString, requiredConfigPositiveInteger, requiredConfigString, + resolveSourceRevision, rewritePodSecretRefs, + runtimeArtifactForService, runtimeCommitForService, runtimeFrpConfigForProfile, + runtimeImageForService, runtimeImageTagForService, runtimeLabelForProfile, + runtimeLaneConfig, runtimeLaneMirrorSpecs, runtimeLaneObservableService, + runtimeLaneObservableServiceIdsForDeploy, runtimeLaneObservableServicesForDeploy, + runtimeLaneServiceIdsFromDeploy, runtimeNodeIdForProfile, + runtimePathForProfile, runtimePrometheusOperatorResourcesEnabled, + runtimeStoreEnvForProfile, runtimeTraceExplorerUrlTemplate, + runtimeWorkbenchTraceTimelinePolicy, secretNameForProfile, + secretRefFromParts, secretRefObjectForProfile, serviceDeclarationEnvForProfile, + serviceIdForWorkload, serviceIdsForLane, serviceIdsForProfile, + servicesParamForLane, sourceCommitPattern, syncCodeAgentWorkspaceInitContainer, + textFile, transformHealthContract, transformListNamespace, transformServices, + transformWorkloads, uniquePreserveOrder, upsertEnv, upsertEnvEntry, + upsertV02MetricsPort, upsertV02MetricsSidecar, usage, useLocalHealthProbe, + v02EnvReuseEnabled, v02EnvReuseRuntimeMode, v02ExtraObservableServices, + v02MetricsLabels, v02MetricsSidecarAnnotations, v02MetricsSidecarContainer, + v02MetricsSidecarVolume, v02ObservableService, v02ObservableServiceIdsForDeploy, + v02ObservableServicesForDeploy, v02RemovedServiceEnvNames, v02RemovedServiceIds, + v02ServiceIdsFromDeploy, versionNameForRuntimeLane, workbenchRuntimeClientEnvForProfile, + workbenchRuntimeFactsQueryEnvForProfile, workbenchRuntimePostgresEnvForProfile, + workbenchRuntimeRedisConfigForProfile, workbenchRuntimeRedisEnvForProfile, + workbenchRuntimeSessionsSummaryEnvForProfile +}; diff --git a/scripts/src/gitops-render/infra-manifests.mjs b/scripts/src/gitops-render/infra-manifests.mjs new file mode 100644 index 00000000..ab9fd877 --- /dev/null +++ b/scripts/src/gitops-render/infra-manifests.mjs @@ -0,0 +1,254 @@ +import assert from "node:assert/strict"; + +import { + ciToolsRunnerImage, + configObject, + defaultOutDir, + label, + optionalConfigString, + requiredConfigPositiveInteger, + requiredConfigString, + runtimeLaneMirrorSpecs +} from "./core.mjs"; +import { renderTemplate, shellSingleQuote } from "./tekton-scripts.mjs"; + +function registryManifest() { + return { + apiVersion: "v1", + kind: "List", + items: [ + { apiVersion: "v1", kind: "Namespace", metadata: { name: "hwlab-ci" } }, + { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { name: "hwlab-registry", namespace: "hwlab-ci", labels: { "app.kubernetes.io/name": "hwlab-registry" } }, + spec: { + replicas: 1, + selector: { matchLabels: { "app.kubernetes.io/name": "hwlab-registry" } }, + template: { + metadata: { labels: { "app.kubernetes.io/name": "hwlab-registry" } }, + spec: { + hostNetwork: true, + dnsPolicy: "ClusterFirstWithHostNet", + containers: [{ + name: "registry", + image: "registry:2.8.3", + imagePullPolicy: "IfNotPresent", + ports: [{ name: "registry", containerPort: 5000, hostPort: 5000 }], + env: [{ name: "REGISTRY_STORAGE_DELETE_ENABLED", value: "true" }], + volumeMounts: [{ name: "storage", mountPath: "/var/lib/registry" }], + readinessProbe: { httpGet: { path: "/v2/", port: "registry" } }, + livenessProbe: { httpGet: { path: "/v2/", port: "registry" } } + }], + volumes: [{ name: "storage", hostPath: { path: "/var/lib/hwlab/registry", type: "DirectoryOrCreate" } }] + } + } + } + }, + { + apiVersion: "v1", + kind: "Service", + metadata: { name: "hwlab-registry", namespace: "hwlab-ci", labels: { "app.kubernetes.io/name": "hwlab-registry" } }, + spec: { selector: { "app.kubernetes.io/name": "hwlab-registry" }, ports: [{ name: "registry", port: 5000, targetPort: "registry" }] } + } + ] + }; +} + +function uniqueStrings(values) { + return Array.from(new Set(values.filter((value) => typeof value === "string" && value.length > 0))); +} + +function renderGitMirrorSshPrelude({ direct, proxySummary, proxyCommand, proxyUrl, proxyNoProxy }) { + if (direct) { + return renderTemplate("git-mirror-ssh-direct.sh", { + "__HWLAB_PROXY_SUMMARY_SHELL__": shellSingleQuote(proxySummary) + }); + } + return renderTemplate("git-mirror-ssh-proxied.sh", { + "// __HWLAB_GIT_MIRROR_PROXY_CONNECT_MJS__": renderTemplate("git-mirror-proxy-connect.mjs"), + "__HWLAB_PROXY_SUMMARY_SHELL__": shellSingleQuote(proxySummary), + "__HWLAB_PROXY_COMMAND_SHELL__": shellSingleQuote(proxyCommand), + "__HWLAB_PROXY_URL_SHELL__": shellSingleQuote(proxyUrl), + "__HWLAB_PROXY_NO_PROXY_SHELL__": shellSingleQuote(proxyNoProxy) + }); +} + +function gitMirrorConfigScripts({ + fetchConfigCommands, + gitMirrorSshPrelude, + gitopsBranchArgs, + gitopsBranchesJson, + mirrorBranchesJson, + requiredRefArgs, + sourceBranchArgs, + sourceBranchesJson, + sourceSnapshotRefPrefix +}) { + const gitopsBranchesJsonShell = shellSingleQuote(gitopsBranchesJson); + return { + "sync.sh": renderTemplate("git-mirror-sync.sh", { + "# __HWLAB_GIT_MIRROR_SSH_PRELUDE__": gitMirrorSshPrelude, + "__HWLAB_SOURCE_SNAPSHOT_REF_PREFIX_SHELL__": shellSingleQuote(sourceSnapshotRefPrefix), + "# __HWLAB_FETCH_CONFIG_COMMANDS__": fetchConfigCommands, + "__HWLAB_REQUIRED_REF_ARGS__": requiredRefArgs, + "__HWLAB_SOURCE_BRANCH_ARGS__": sourceBranchArgs, + "__HWLAB_GITOPS_BRANCH_ARGS__": gitopsBranchArgs, + "__HWLAB_MIRROR_BRANCHES_JSON_SHELL__": shellSingleQuote(mirrorBranchesJson), + "__HWLAB_SOURCE_BRANCHES_JSON_SHELL__": shellSingleQuote(sourceBranchesJson), + "__HWLAB_GITOPS_BRANCHES_JSON_SHELL__": gitopsBranchesJsonShell + }), + "install-hooks.sh": renderTemplate("git-mirror-install-hooks.sh", { + "__HWLAB_GITOPS_BRANCHES_JSON_SHELL__": gitopsBranchesJsonShell + }), + "flush.sh": renderTemplate("git-mirror-flush.sh", { + "# __HWLAB_GIT_MIRROR_SSH_PRELUDE__": gitMirrorSshPrelude, + "__HWLAB_GITOPS_BRANCH_ARGS__": gitopsBranchArgs, + "__HWLAB_GITOPS_BRANCHES_JSON_SHELL__": gitopsBranchesJsonShell + }), + "http.mjs": renderTemplate("git-mirror-http.mjs") + }; +} + +function devopsInfraGitMirrorManifest(args, deploy) { + const mirrorSpecs = runtimeLaneMirrorSpecs(deploy, { defaultNodeId: args.nodeId, defaultGitopsRoot: defaultOutDir }); + assert.ok(mirrorSpecs.length > 0, "runtime lane git mirror requires at least one deploy.lanes entry"); + const sourceBranches = uniqueStrings(mirrorSpecs.map((spec) => spec.sourceBranch)); + const gitopsBranches = uniqueStrings(mirrorSpecs.map((spec) => spec.gitopsBranch)); + const mirrorBranches = uniqueStrings([...sourceBranches, ...gitopsBranches]); + const fetchConfigCommands = mirrorBranches + .map((branch) => `git -C "$repo_path" config --add remote.origin.fetch '+refs/heads/${branch}:refs/mirror-stage/heads/${branch}'`) + .join("\n"); + const requiredRefArgs = mirrorBranches.map((branch) => shellSingleQuote(`refs/mirror-stage/heads/${branch}`)).join(" "); + const sourceBranchArgs = sourceBranches.map((branch) => shellSingleQuote(branch)).join(" "); + const gitopsBranchArgs = gitopsBranches.map((branch) => shellSingleQuote(branch)).join(" "); + const mirrorBranchesJson = JSON.stringify(mirrorBranches); + const sourceBranchesJson = JSON.stringify(sourceBranches); + const gitopsBranchesJson = JSON.stringify(gitopsBranches); + const sourceSnapshotRefPrefix = "refs/unidesk/snapshots/hwlab-node-runtime"; + const laneConfig = configObject(configObject(deploy.lanes, "deploy.lanes")[args.lane], `deploy.lanes.${args.lane}`); + const gitMirrorConfig = configObject(laneConfig.gitMirror, `deploy.lanes.${args.lane}.gitMirror`); + const gitMirrorProxy = configObject(gitMirrorConfig.egressProxy, `deploy.lanes.${args.lane}.gitMirror.egressProxy`); + const gitMirrorProxyMode = requiredConfigString(gitMirrorProxy, "mode", `deploy.lanes.${args.lane}.gitMirror.egressProxy`); + assert.ok(gitMirrorProxyMode === "node-global" || gitMirrorProxyMode === "host-proxy-client" || gitMirrorProxyMode === "direct", `deploy.lanes.${args.lane}.gitMirror.egressProxy.mode must be node-global, host-proxy-client, or direct`); + const useDirectGitMirrorProxy = gitMirrorProxyMode === "direct"; + const useHostGitMirrorProxy = gitMirrorProxyMode === "host-proxy-client"; + const gitMirrorNamespace = requiredConfigString(gitMirrorConfig, "namespace", `deploy.lanes.${args.lane}.gitMirror`); + const gitMirrorReadName = requiredConfigString(gitMirrorConfig, "serviceReadName", `deploy.lanes.${args.lane}.gitMirror`); + const gitMirrorWriteName = requiredConfigString(gitMirrorConfig, "serviceWriteName", `deploy.lanes.${args.lane}.gitMirror`); + const gitMirrorCachePvcName = requiredConfigString(gitMirrorConfig, "cachePvcName", `deploy.lanes.${args.lane}.gitMirror`); + const gitMirrorCachePvcStorage = requiredConfigString(gitMirrorConfig, "cachePvcStorage", `deploy.lanes.${args.lane}.gitMirror`); + const gitMirrorCacheHostPath = optionalConfigString(gitMirrorConfig, "cacheHostPath", `deploy.lanes.${args.lane}.gitMirror`); + const gitMirrorServicePort = requiredConfigPositiveInteger(gitMirrorConfig, "servicePort", `deploy.lanes.${args.lane}.gitMirror`); + const gitMirrorConfigMapName = requiredConfigString(gitMirrorConfig, "syncConfigMapName", `deploy.lanes.${args.lane}.gitMirror`); + const proxyNamespace = useDirectGitMirrorProxy || useHostGitMirrorProxy ? "" : requiredConfigString(gitMirrorProxy, "namespace", `deploy.lanes.${args.lane}.gitMirror.egressProxy`); + const proxyServiceName = useDirectGitMirrorProxy || useHostGitMirrorProxy ? "" : requiredConfigString(gitMirrorProxy, "serviceName", `deploy.lanes.${args.lane}.gitMirror.egressProxy`); + const proxyPort = useDirectGitMirrorProxy ? 0 : requiredConfigPositiveInteger(gitMirrorProxy, "port", `deploy.lanes.${args.lane}.gitMirror.egressProxy`); + const proxyClientName = useDirectGitMirrorProxy ? "" : requiredConfigString(gitMirrorProxy, "clientName", `deploy.lanes.${args.lane}.gitMirror.egressProxy`); + const proxyNoProxy = !useDirectGitMirrorProxy && Array.isArray(gitMirrorProxy.noProxy) ? gitMirrorProxy.noProxy.filter((value) => typeof value === "string" && value.length > 0).join(",") : ""; + const proxyHost = useHostGitMirrorProxy + ? requiredConfigString(gitMirrorProxy, "host", `deploy.lanes.${args.lane}.gitMirror.egressProxy`) + : `${proxyServiceName}.${proxyNamespace}.svc.cluster.local`; + const proxyUrl = `http://${proxyHost}:${proxyPort}`; + const proxySummary = useDirectGitMirrorProxy + ? "git-mirror-egress-proxy mode=direct required=false transport=ssh source=yaml" + : `git-mirror-egress-proxy client=${proxyClientName} mode=${gitMirrorProxyMode} required=true host=${proxyHost} port=${proxyPort} ssh=GIT_SSH-wrapper source=yaml`; + const proxyCommand = useDirectGitMirrorProxy ? "" : `ProxyCommand=node /tmp/hwlab-github-proxy-connect.mjs ${proxyHost} ${proxyPort} %h %p`; + const gitMirrorSshPrelude = renderGitMirrorSshPrelude({ + direct: useDirectGitMirrorProxy, + proxySummary, + proxyCommand, + proxyUrl, + proxyNoProxy + }); + const cacheVolume = gitMirrorCacheHostPath === null + ? { name: "cache", persistentVolumeClaim: { claimName: gitMirrorCachePvcName } } + : { name: "cache", hostPath: { path: gitMirrorCacheHostPath, type: "DirectoryOrCreate" } }; + const labels = { + "app.kubernetes.io/part-of": "hwlab-node-control-plane", + "hwlab.pikastech.local/node": args.nodeId, + "hwlab.pikastech.local/lane": args.lane + }; + const configLabels = { ...labels, "app.kubernetes.io/name": "git-mirror" }; + const readLabels = { ...labels, "app.kubernetes.io/name": gitMirrorReadName, "hwlab.pikastech.local/git-mirror-mode": "read" }; + const writeLabels = { ...labels, "app.kubernetes.io/name": gitMirrorWriteName, "hwlab.pikastech.local/git-mirror-mode": "write" }; + return { + apiVersion: "v1", + kind: "List", + items: [ + { apiVersion: "v1", kind: "Namespace", metadata: { name: gitMirrorNamespace } }, + ...(gitMirrorCacheHostPath === null ? [{ + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { name: gitMirrorCachePvcName, namespace: gitMirrorNamespace, labels: configLabels }, + spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: gitMirrorCachePvcStorage } } } + }] : []), + { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { name: gitMirrorConfigMapName, namespace: gitMirrorNamespace, labels: configLabels }, + data: gitMirrorConfigScripts({ + fetchConfigCommands, + gitMirrorSshPrelude, + gitopsBranchArgs, + gitopsBranchesJson, + mirrorBranchesJson, + requiredRefArgs, + sourceBranchArgs, + sourceBranchesJson, + sourceSnapshotRefPrefix + }) + }, + { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { name: gitMirrorReadName, namespace: gitMirrorNamespace, labels: readLabels }, + spec: { + replicas: 1, + selector: { matchLabels: { "app.kubernetes.io/name": gitMirrorReadName } }, + template: { + metadata: { labels: readLabels }, + spec: { + containers: [{ + name: "git-mirror", + image: ciToolsRunnerImage, + imagePullPolicy: "IfNotPresent", + command: ["node"], + args: ["/etc/git-mirror/http.mjs"], + ports: [{ name: "http", containerPort: 8080 }], + readinessProbe: { httpGet: { path: "/pikasTech/HWLAB.git/HEAD", port: "http" }, initialDelaySeconds: 5, periodSeconds: 10 }, + livenessProbe: { httpGet: { path: "/", port: "http" }, initialDelaySeconds: 10, periodSeconds: 30 }, + volumeMounts: [ + { name: "cache", mountPath: "/cache" }, + { name: "config", mountPath: "/etc/git-mirror", readOnly: true } + ] + }], + volumes: [ + cacheVolume, + { name: "config", configMap: { name: gitMirrorConfigMapName, defaultMode: 0o555 } } + ] + } + } + } + }, + { + apiVersion: "v1", + kind: "Service", + metadata: { name: gitMirrorReadName, namespace: gitMirrorNamespace, labels: readLabels }, + spec: { selector: { "app.kubernetes.io/name": gitMirrorReadName }, ports: [{ name: "http", port: gitMirrorServicePort, targetPort: "http" }] } + }, + { + apiVersion: "v1", + kind: "Service", + metadata: { name: gitMirrorWriteName, namespace: gitMirrorNamespace, labels: writeLabels }, + spec: { selector: { "app.kubernetes.io/name": gitMirrorReadName }, ports: [{ name: "http", port: gitMirrorServicePort, targetPort: "http" }] } + } + ] + }; +} + +export { + devopsInfraGitMirrorManifest, + registryManifest, + uniqueStrings +}; diff --git a/scripts/src/gitops-render/runtime-manifests.mjs b/scripts/src/gitops-render/runtime-manifests.mjs new file mode 100644 index 00000000..13685771 --- /dev/null +++ b/scripts/src/gitops-render/runtime-manifests.mjs @@ -0,0 +1,1418 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import path from "node:path"; + +import { + applyEnvReuseBootEnv, + argoApplicationName, + bootMetadataForService, + cloneJson, + configString, + deepSeekProfileModel, + defaultBranch, + defaultRegistryPrefix, + defaultSourceRepo, + defaultV02GitReadUrl, + deployServiceForBoot, + effectiveSecretPlaneNodeId, + envReuseServiceIdsForLane, + gitopsPathForProfile, + gitopsRootNodeId, + gitopsTargetForProfile, + isRuntimeLane, + k8sLabelHash, + label, + moonBridgeImage, + moonBridgeSourceRef, + namespaceNameForProfile, + optionalObject, + readConfigRef, + runtimeCommitForService, + runtimeFrpConfigForProfile, + runtimeImageForService, + runtimeImageTagForService, + runtimeLabelForProfile, + runtimeLaneConfig, + runtimePathForProfile, + secretNameForProfile, + secretRefObjectForProfile, + v02EnvReuseEnabled, + v02MetricsLabels, + v02MetricsSidecarAnnotations, + v02MetricsSidecarContainer, + v02MetricsSidecarVolume, + versionNameForRuntimeLane +} from "./core.mjs"; + +function argoProject(args = { lane: "node" }) { + if (isRuntimeLane(args.lane)) { + const namespace = namespaceNameForProfile(args.lane); + return { + apiVersion: "argoproj.io/v1alpha1", + kind: "AppProject", + metadata: { name: namespace, namespace: "argocd" }, + spec: { + description: `HWLAB ${versionNameForRuntimeLane(args.lane)} additive GitOps project on target node; DEV/PROD remain outside this lane.`, + sourceRepos: [args.gitReadUrl], + destinations: [{ server: "https://kubernetes.default.svc", namespace }], + clusterResourceWhitelist: [{ group: "", kind: "Namespace" }], + namespaceResourceWhitelist: [{ group: "*", kind: "*" }] + } + }; + } + return { + apiVersion: "argoproj.io/v1alpha1", + kind: "AppProject", + metadata: { name: "hwlab-node", namespace: "argocd" }, + spec: { + description: "HWLAB node GitOps project; D601 remains outside this project.", + sourceRepos: [defaultSourceRepo], + destinations: [ + { server: "https://kubernetes.default.svc", namespace: "hwlab-dev" }, + { server: "https://kubernetes.default.svc", namespace: "hwlab-prod" } + ], + clusterResourceWhitelist: [{ group: "", kind: "Namespace" }], + namespaceResourceWhitelist: [{ group: "*", kind: "*" }] + } + }; +} + +function argoApplication(args, profile = "dev") { + const namespace = namespaceNameForProfile(profile); + const runtimePath = runtimePathForProfile(profile); + const gitopsTarget = gitopsTargetForProfile(profile); + const project = isRuntimeLane(profile) ? namespace : "hwlab-node"; + const repoURL = isRuntimeLane(profile) ? args.gitReadUrl : args.sourceRepo; + return { + apiVersion: "argoproj.io/v1alpha1", + kind: "Application", + metadata: { + name: argoApplicationName(profile), + namespace: "argocd", + labels: { "app.kubernetes.io/part-of": "hwlab", "hwlab.pikastech.local/gitops-target": gitopsTarget } + }, + spec: { + project, + source: { repoURL, targetRevision: args.gitopsBranch, path: gitopsPathForProfile(args, profile) }, + destination: { server: "https://kubernetes.default.svc", namespace }, + syncPolicy: { automated: { prune: isRuntimeLane(profile), selfHeal: true }, syncOptions: ["CreateNamespace=true", "ApplyOutOfSyncOnly=true"] } + } + }; +} + +function nodeFrpcManifest({ profile = "dev", source = null, deploy = null, args = {} } = {}) { + const namespace = namespaceNameForProfile(profile); + const profileLabel = runtimeLabelForProfile(profile); + const deploymentName = isRuntimeLane(profile) ? `hwlab-${profile}-frpc` : profile === "prod" ? "hwlab-node-prod-frpc" : "hwlab-node-frpc"; + const configName = `${deploymentName}-config`; + const proxyPrefix = isRuntimeLane(profile) ? `hwlab-${profile}` : profile === "prod" ? "hwlab-node-prod" : "hwlab-node"; + const frpConfig = runtimeFrpConfigForProfile(deploy, profile, args); + const webProxyName = frpConfig.webProxyName || `${proxyPrefix}-cloud-web`; + const edgeProxyName = frpConfig.edgeProxyName || `${proxyPrefix}-edge-proxy`; + const labels = { + "app.kubernetes.io/name": deploymentName, + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/environment": profileLabel, + "hwlab.pikastech.local/gitops-target": gitopsTargetForProfile(profile), + "hwlab.pikastech.local/profile": profileLabel + }; + const sourceCommitLabel = source?.full ? { "hwlab.pikastech.local/source-commit": source.full } : {}; + const renderedLabels = { ...labels, ...sourceCommitLabel }; + const podNetwork = isRuntimeLane(profile) ? { + hostNetwork: true, + dnsPolicy: "ClusterFirstWithHostNet" + } : {}; + const authConfigText = frpConfig.authSecretName && frpConfig.authSecretKey ? `auth.token = "{{ .Envs.FRPC_AUTH_TOKEN }}"\n` : ""; + const configText = `serverAddr = "${frpConfig.serverAddr}" +serverPort = ${frpConfig.serverPort} +loginFailExit = true +${authConfigText} + +[[proxies]] +name = "${webProxyName}" +type = "tcp" +localIP = "hwlab-cloud-web.${namespace}.svc.cluster.local" +localPort = 8080 +remotePort = ${frpConfig.webRemotePort} + +[[proxies]] +name = "${edgeProxyName}" +type = "tcp" +localIP = "hwlab-edge-proxy.${namespace}.svc.cluster.local" +localPort = 6667 +remotePort = ${frpConfig.edgeRemotePort} +`; + const configSha256 = createHash("sha256").update(configText).digest("hex"); + const templateLabels = { ...labels, "hwlab.pikastech.local/config-sha256": k8sLabelHash(configSha256) }; + const templateAnnotations = { "hwlab.pikastech.local/config-sha256": configSha256 }; + return { + apiVersion: "v1", + kind: "List", + items: [ + { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + name: configName, + namespace, + labels: renderedLabels + }, + data: { + "frpc.toml": configText + } + }, + { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { + name: deploymentName, + namespace, + labels: renderedLabels + }, + spec: { + replicas: 1, + selector: { matchLabels: { "app.kubernetes.io/name": deploymentName } }, + template: { + metadata: { labels: templateLabels, annotations: templateAnnotations }, + spec: { + ...podNetwork, + containers: [{ + name: "frpc", + image: "fatedier/frpc:v0.68.1", + imagePullPolicy: "IfNotPresent", + ...(frpConfig.authSecretName && frpConfig.authSecretKey ? { env: [{ name: "FRPC_AUTH_TOKEN", valueFrom: { secretKeyRef: { name: frpConfig.authSecretName, key: frpConfig.authSecretKey } } }] } : {}), + args: ["-c", "/etc/frp/frpc.toml"], + volumeMounts: [{ name: "config", mountPath: "/etc/frp", readOnly: true }] + }], + volumes: [{ name: "config", configMap: { name: configName } }] + } + } + } + } + ] + }; +} + +function deepSeekProxyManifest({ profile = "dev", source, sourceBranch = defaultBranch, sourceRepo = defaultSourceRepo, gitReadUrl, deploy = null, registryPrefix = defaultRegistryPrefix, catalog = null, useDeployImages = false, metricsSidecarSha256 = null } = {}) { + const namespace = namespaceNameForProfile(profile); + const bridgeServiceId = "hwlab-cloud-api"; + const runtimeLane = isRuntimeLane(profile); + const digestPin = runtimeLane; + const envReuseServiceIds = runtimeLane ? envReuseServiceIdsForLane(deploy, profile) : null; + const bridgeImage = runtimeImageForService({ catalog, deploy, serviceId: bridgeServiceId, source, registryPrefix, useDeployImages, digestPin, envReuseServiceIds }); + const bridgeCommit = runtimeCommitForService({ catalog, deploy, serviceId: bridgeServiceId, source, registryPrefix, useDeployImages, digestPin, envReuseServiceIds }); + const bridgeImageTag = runtimeImageTagForService({ catalog, deploy, serviceId: bridgeServiceId, source, registryPrefix, useDeployImages, digestPin, envReuseServiceIds }); + const bridgeBootMetadata = runtimeLane && v02EnvReuseEnabled(catalog, bridgeServiceId, envReuseServiceIds) + ? bootMetadataForService({ args: { sourceRepo: deploy?.lanes?.[profile]?.sourceRepo || sourceRepo, registryPrefix }, catalog, deployService: deployServiceForBoot(deploy, bridgeServiceId, profile), serviceId: bridgeServiceId, source }) + : null; + const labels = { + "app.kubernetes.io/name": "hwlab-deepseek-proxy", + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/environment": runtimeLabelForProfile(profile), + "hwlab.pikastech.local/gitops-target": gitopsTargetForProfile(profile), + "hwlab.pikastech.local/service-id": "hwlab-deepseek-proxy", + "hwlab.pikastech.local/source-commit": source.full + }; + if (runtimeLane) Object.assign(labels, v02MetricsLabels("hwlab-deepseek-proxy")); + const templateLabels = { + ...labels, + "hwlab.pikastech.local/source-commit": bridgeCommit, + "hwlab.pikastech.local/bridge-source-commit": bridgeCommit + }; + const templateAnnotations = { + "hwlab.pikastech.local/bridge-service-id": bridgeServiceId, + "hwlab.pikastech.local/source-commit": bridgeCommit, + "hwlab.pikastech.local/bridge-source-commit": bridgeCommit, + "hwlab.pikastech.local/bridge-image-tag": bridgeImageTag, + "hwlab.pikastech.local/bridge-image": bridgeImage, + ...v02MetricsSidecarAnnotations(runtimeLane ? metricsSidecarSha256 : null) + }; + const responsesBridgeContainer = { + name: "responses-bridge", + image: bridgeImage, + imagePullPolicy: "IfNotPresent", + command: ["/usr/local/bin/bun", "run", "/app/cmd/hwlab-deepseek-responses-bridge/main.ts"], + env: [ + { name: "PORT", value: "4000" }, + { name: "HWLAB_COMMIT_ID", value: bridgeCommit }, + { name: "HWLAB_IMAGE", value: bridgeImage }, + { name: "HWLAB_IMAGE_TAG", value: bridgeImageTag }, + { name: "HWLAB_DEEPSEEK_BRIDGE_UPSTREAM", value: "http://127.0.0.1:4001" }, + { name: "HWLAB_DEEPSEEK_BRIDGE_MODEL", value: deepSeekProfileModel } + ], + ports: [{ name: "http", containerPort: 4000 }], + readinessProbe: { httpGet: { path: "/health/readiness", port: "http" }, initialDelaySeconds: 10, periodSeconds: 10 }, + livenessProbe: { httpGet: { path: "/health/liveliness", port: "http" }, initialDelaySeconds: 20, periodSeconds: 20 } + }; + if (bridgeBootMetadata) { + applyEnvReuseBootEnv(responsesBridgeContainer, bridgeBootMetadata, { sourceBranch, gitReadUrl, bootSh: "deploy/runtime/boot/hwlab-deepseek-responses-bridge.sh" }); + Object.assign(templateAnnotations, { + "hwlab.pikastech.local/bridge-runtime-mode": bridgeBootMetadata.runtimeMode, + "hwlab.pikastech.local/bridge-boot-repo": bridgeBootMetadata.bootRepo, + "hwlab.pikastech.local/bridge-boot-commit": bridgeBootMetadata.bootCommit, + "hwlab.pikastech.local/bridge-boot-sh": "deploy/runtime/boot/hwlab-deepseek-responses-bridge.sh", + "hwlab.pikastech.local/bridge-environment-digest": bridgeBootMetadata.environmentDigest ?? "not_published" + }); + Object.assign(templateLabels, { + "hwlab.pikastech.local/bridge-runtime-mode": bridgeBootMetadata.runtimeMode, + "hwlab.pikastech.local/bridge-boot-commit": bridgeBootMetadata.bootCommit + }); + } + return { + apiVersion: "v1", + kind: "List", + items: [ + { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { name: "hwlab-deepseek-proxy-config", namespace, labels }, + data: { + "render-config.sh": moonBridgeConfigRenderScript() + } + }, + { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { name: "hwlab-deepseek-proxy", namespace, labels, annotations: { "hwlab.pikastech.local/moonbridge-source-ref": moonBridgeSourceRef, ...templateAnnotations } }, + spec: { + replicas: 1, + selector: { matchLabels: { "app.kubernetes.io/name": "hwlab-deepseek-proxy" } }, + template: { + metadata: { labels: templateLabels, annotations: templateAnnotations }, + spec: { + initContainers: [{ + name: "moonbridge-config", + image: bridgeImage, + imagePullPolicy: "IfNotPresent", + command: ["/bin/sh", "-eu", "/scripts/render-config.sh"], + env: [ + { name: "DEEPSEEK_API_KEY", valueFrom: { secretKeyRef: { name: secretNameForProfile("hwlab-code-agent-provider", profile), key: "openai-api-key", optional: true } } }, + { name: "OPENCODE_API_KEY", valueFrom: { secretKeyRef: { name: secretNameForProfile("hwlab-code-agent-provider", profile), key: "opencode-api-key", optional: true } } } + ], + volumeMounts: [ + { name: "moonbridge-scripts", mountPath: "/scripts", readOnly: true }, + { name: "moonbridge-config", mountPath: "/config" } + ] + }], + containers: [responsesBridgeContainer, { + name: "moonbridge", + image: moonBridgeImage, + imagePullPolicy: "IfNotPresent", + args: ["-config", "/config/config.yml"], + ports: [{ name: "moonbridge-http", containerPort: 4001 }], + readinessProbe: { httpGet: { path: "/v1/models", port: "moonbridge-http" }, initialDelaySeconds: 10, periodSeconds: 10 }, + livenessProbe: { httpGet: { path: "/v1/models", port: "moonbridge-http" }, initialDelaySeconds: 20, periodSeconds: 20 }, + volumeMounts: [ + { name: "moonbridge-config", mountPath: "/config", readOnly: true }, + { name: "moonbridge-data", mountPath: "/data" } + ] + }, ...(runtimeLane ? [v02MetricsSidecarContainer({ deploy, serviceId: "hwlab-deepseek-proxy", namespace, gitopsTarget: gitopsTargetForProfile(profile) })] : [])], + volumes: [ + { name: "moonbridge-scripts", configMap: { name: "hwlab-deepseek-proxy-config" } }, + { name: "moonbridge-config", emptyDir: {} }, + { name: "moonbridge-data", emptyDir: {} }, + ...(runtimeLane ? [v02MetricsSidecarVolume(profile)] : []) + ] + } + } + } + }, + { + apiVersion: "v1", + kind: "Service", + metadata: { name: "hwlab-deepseek-proxy", namespace, labels }, + spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": "hwlab-deepseek-proxy" }, ports: [{ name: "http", port: 4000, targetPort: "http" }, ...(runtimeLane ? [{ name: "metrics", port: 9100, targetPort: "metrics" }] : [])] } + } + ] + }; +} + +function moonBridgeConfigRenderScript() { + return `#!/bin/sh +if [ -z "\${DEEPSEEK_API_KEY:-}" ] && [ -z "\${OPENCODE_API_KEY:-}" ]; then + echo "at least one of DEEPSEEK_API_KEY or OPENCODE_API_KEY is required" >&2 + exit 1 +fi +cat > /config/config.yml < {", + " let body = \"\";", + " req.setEncoding(\"utf8\");", + " req.on(\"data\", (chunk) => {", + " body += chunk;", + " if (body.length > 1024 * 1024) req.destroy(new Error(\"request body too large\"));", + " });", + " req.on(\"end\", () => resolve(body ? JSON.parse(body) : {}));", + " req.on(\"error\", reject);", + " });", + "}", + "", + "function postJson(url, payload, timeoutMs) {", + " return new Promise((resolve, reject) => {", + " const parsed = new URL(url);", + " const body = JSON.stringify(payload);", + " const request = http.request({", + " hostname: parsed.hostname,", + " port: parsed.port || 80,", + " path: parsed.pathname + parsed.search,", + " method: \"POST\",", + " headers: { \"content-type\": \"application/json\", \"content-length\": Buffer.byteLength(body) },", + " timeout: timeoutMs", + " }, (response) => {", + " let responseBody = \"\";", + " response.setEncoding(\"utf8\");", + " response.on(\"data\", (chunk) => { responseBody += chunk; });", + " response.on(\"end\", () => {", + " try {", + " const parsedBody = responseBody ? JSON.parse(responseBody) : null;", + " resolve({ statusCode: response.statusCode, body: parsedBody });", + " } catch (error) {", + " reject(error);", + " }", + " });", + " });", + " request.on(\"timeout\", () => request.destroy(new Error(\"request timeout after \" + timeoutMs + \"ms\")));", + " request.on(\"error\", reject);", + " request.end(body);", + " });", + "}", + "", + "function gatewayPayload(input) {", + " const traceId = input.traceId || \"trc_device_agent_71_freq_\" + Date.now();", + " return {", + " jsonrpc: \"2.0\",", + " id: input.id || \"req_device_agent_71_freq_\" + Date.now(),", + " method: \"hardware.invoke.shell\",", + " meta: { serviceId: \"hwlab-cloud-api\", actorId: \"device-agent-71-freq\", environment: \"dev\", traceId, deviceId, workspaceRoot },", + " params: {", + " gatewaySessionId,", + " resourceId,", + " capabilityId,", + " input: {", + " command: input.command,", + " cwd: input.cwd || workspaceRoot,", + " timeoutMs: input.timeoutMs || 30000", + " }", + " }", + " };", + "}", + "", + "const server = http.createServer(async (req, res) => {", + " try {", + " const url = new URL(req.url || \"/\", \"http://device-agent-71-freq\");", + " if (req.method === \"GET\" && url.pathname === \"/health\") {", + " return sendJson(res, 200, { ok: true, deviceId, workspaceRoot, gatewaySessionId, resourceId, capabilityId, cloudApiUrl });", + " }", + " if (req.method === \"GET\" && url.pathname === \"/skills\") {", + " return sendJson(res, 200, { ok: true, deviceId, skills: [{ name: \"cmd\", description: \"Run a bounded Windows cmd command through hwlab-gateway.\" }] });", + " }", + " if (req.method === \"POST\" && url.pathname === \"/run\") {", + " const input = await readBody(req);", + " if (!input.command || typeof input.command !== \"string\") return sendJson(res, 400, { ok: false, error: \"command is required\" });", + " const timeoutMs = Number(input.timeoutMs || 30000);", + " const upstream = await postJson(cloudApiUrl + \"/json-rpc\", gatewayPayload({ ...input, timeoutMs }), timeoutMs + 5000);", + " return sendJson(res, upstream.statusCode || 502, { ok: upstream.statusCode >= 200 && upstream.statusCode < 300, deviceId, gatewaySessionId, upstream: upstream.body });", + " }", + " return sendJson(res, 404, { ok: false, error: \"not found\" });", + " } catch (error) {", + " return sendJson(res, 500, { ok: false, error: error.message });", + " }", + "});", + "", + "server.listen(port, \"0.0.0.0\", () => {", + " console.log(JSON.stringify({ ok: true, service: \"device-agent-71-freq\", port, deviceId, workspaceRoot, cloudApiUrl }));", + "});" + ].join("\n") + "\n"; +} + +function deviceAgent71FreqManifest({ profile = "dev", source, registryPrefix, catalog = null, useDeployImages = false }) { + assert.equal(profile, "dev", "71-freq device-agent is dev-only"); + const namespace = namespaceNameForProfile(profile); + const name = "device-agent-71-freq"; + const bridgeServiceId = "hwlab-cloud-api"; + const bridgeImage = runtimeImageForService({ catalog, serviceId: bridgeServiceId, source, registryPrefix, useDeployImages }); + const bridgeCommit = runtimeCommitForService({ catalog, serviceId: bridgeServiceId, source, registryPrefix, useDeployImages }); + const bridgeImageTag = runtimeImageTagForService({ catalog, serviceId: bridgeServiceId, source, registryPrefix, useDeployImages }); + const labels = { + "app.kubernetes.io/name": name, + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/component": "device-agent", + "hwlab.pikastech.local/device-id": "71-freq", + "hwlab.pikastech.local/environment": runtimeLabelForProfile(profile), + "hwlab.pikastech.local/gitops-target": "node", + "hwlab.pikastech.local/profile": runtimeLabelForProfile(profile), + "hwlab.pikastech.local/service-id": name, + "hwlab.pikastech.local/source-commit": source.full + }; + const annotations = { "hwlab.pikastech.local/rendered-by": "scripts/gitops-render.mjs" }; + const selector = { "app.kubernetes.io/name": name }; + const script = deviceAgent71FreqServerScript(); + const scriptSha256 = createHash("sha256").update(script).digest("hex"); + const templateLabels = { ...labels, ...selector, "hwlab.pikastech.local/source-commit": bridgeCommit }; + const templateAnnotations = { + ...annotations, + "hwlab.pikastech.local/script-sha256": scriptSha256, + "hwlab.pikastech.local/bridge-service-id": bridgeServiceId, + "hwlab.pikastech.local/bridge-source-commit": bridgeCommit, + "hwlab.pikastech.local/bridge-image-tag": bridgeImageTag, + "hwlab.pikastech.local/bridge-image": bridgeImage + }; + return { + apiVersion: "v1", + kind: "List", + items: [ + { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { name: `${name}-script`, namespace, labels, annotations: templateAnnotations }, + data: { "server.mjs": script } + }, + { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { name, namespace, labels, annotations }, + spec: { + replicas: 1, + selector: { matchLabels: selector }, + template: { + metadata: { labels: templateLabels, annotations: templateAnnotations }, + spec: { + containers: [{ + name: "device-agent", + image: bridgeImage, + imagePullPolicy: "IfNotPresent", + command: ["node", "/opt/device-agent/server.mjs"], + env: [ + { name: "PORT", value: "7601" }, + { name: "HWLAB_COMMIT_ID", value: bridgeCommit }, + { name: "HWLAB_IMAGE", value: bridgeImage }, + { name: "HWLAB_IMAGE_TAG", value: bridgeImageTag }, + { name: "DEVICE_ID", value: "71-freq" }, + { name: "DEVICE_WORKSPACE_ROOT", value: "F:\\Work\\ConStart" }, + { name: "HWLAB_CLOUD_API_URL", value: `http://hwlab-cloud-api.${namespace}.svc.cluster.local:6667` }, + { name: "HWLAB_GATEWAY_SESSION_ID", value: "gws_d601_win_71_freq" }, + { name: "HWLAB_GATEWAY_RESOURCE_ID", value: "res_d601_windows_host" }, + { name: "HWLAB_GATEWAY_CAPABILITY_ID", value: "cap_d601_windows_cmd_exec" } + ], + ports: [{ name: "http", containerPort: 7601 }], + readinessProbe: { httpGet: { path: "/health", port: "http" }, initialDelaySeconds: 3, periodSeconds: 10 }, + livenessProbe: { httpGet: { path: "/health", port: "http" }, initialDelaySeconds: 10, periodSeconds: 20 }, + resources: { requests: { cpu: "10m", memory: "64Mi" }, limits: { cpu: "200m", memory: "256Mi" } }, + volumeMounts: [ + { name: "script", mountPath: "/opt/device-agent", readOnly: true }, + { name: "hwlab-code-agent-workspace", mountPath: "/workspace" } + ] + }], + volumes: [ + { name: "script", configMap: { name: `${name}-script` } }, + { name: "hwlab-code-agent-workspace", persistentVolumeClaim: { claimName: "hwlab-code-agent-workspace" } } + ] + } + } + } + }, + { + apiVersion: "v1", + kind: "Service", + metadata: { name, namespace, labels, annotations }, + spec: { type: "ClusterIP", selector, ports: [{ name: "http", port: 7601, targetPort: "http" }] } + } + ] + }; +} + +function runtimePostgresImageForProfile(deploy, profile) { + const postgres = runtimeLaneConfig(deploy, profile)?.runtimeStore?.postgres; + const image = postgres && typeof postgres === "object" && !Array.isArray(postgres) ? postgres.image : null; + return typeof image === "string" && image.trim().length > 0 ? image.trim() : "postgres:16-alpine"; +} + +function v02PostgresManifest({ profile = "v02", migrationSources, source, image = "postgres:16-alpine" }) { + const namespace = namespaceNameForProfile(profile); + const name = `${namespace}-postgres`; + const dbName = `hwlab_${profile}`; + const labels = { + "app.kubernetes.io/name": name, + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/environment": profile, + "hwlab.pikastech.local/gitops-target": profile, + "hwlab.pikastech.local/profile": profile, + "hwlab.pikastech.local/source-commit": source.full + }; + assert.ok(Array.isArray(migrationSources) && migrationSources.length > 0, "runtime Postgres migration chain is required"); + const migrationSql = migrationSources.map((migration) => migration.sql).join("\n"); + const migrationData = Object.fromEntries(migrationSources.map((migration) => [path.basename(migration.path), migration.sql])); + const migrationSha256 = createHash("sha256").update(migrationSql).digest("hex"); + const templateLabels = { + "app.kubernetes.io/name": name, + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/environment": profile, + "hwlab.pikastech.local/gitops-target": profile, + "hwlab.pikastech.local/profile": profile, + "hwlab.pikastech.local/migration-sha256": k8sLabelHash(migrationSha256) + }; + const templateAnnotations = { "hwlab.pikastech.local/migration-sha256": migrationSha256 }; + return { + apiVersion: "v1", + kind: "List", + items: [ + { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { name: `${name}-init`, namespace, labels }, + data: migrationData + }, + { + apiVersion: "v1", + kind: "Service", + metadata: { name, namespace, labels }, + spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": name }, ports: [{ name: "postgres", port: 5432, targetPort: "postgres" }] } + }, + { + apiVersion: "apps/v1", + kind: "StatefulSet", + metadata: { name, namespace, labels }, + spec: { + serviceName: name, + replicas: 1, + selector: { matchLabels: { "app.kubernetes.io/name": name } }, + template: { + metadata: { labels: templateLabels, annotations: templateAnnotations }, + spec: { + containers: [{ + name: "postgres", + image, + imagePullPolicy: "IfNotPresent", + env: [ + { name: "POSTGRES_DB", value: dbName }, + { name: "POSTGRES_USER", value: dbName }, + { name: "POSTGRES_PASSWORD", valueFrom: { secretKeyRef: { name, key: "POSTGRES_PASSWORD" } } } + ], + ports: [{ name: "postgres", containerPort: 5432 }], + readinessProbe: { tcpSocket: { port: "postgres" }, initialDelaySeconds: 5, periodSeconds: 10 }, + livenessProbe: { tcpSocket: { port: "postgres" }, initialDelaySeconds: 30, periodSeconds: 20 }, + volumeMounts: [ + { name: "data", mountPath: "/var/lib/postgresql/data" }, + { name: "init", mountPath: "/docker-entrypoint-initdb.d", readOnly: true } + ] + }], + volumes: [{ name: "init", configMap: { name: `${name}-init` } }] + } + }, + volumeClaimTemplates: [{ + metadata: { name: "data" }, + spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } + }] + } + } + ] + }; +} + +function externalPostgresConfigForLane(deploy, profile, args = {}) { + if (!isRuntimeLane(profile)) return null; + const config = deploy?.lanes?.[profile]?.externalPostgres; + if (!config || config.enabled !== true) return null; + const nodeId = effectiveSecretPlaneNodeId(args); + const nodeConfig = nodeId ? config?.nodes?.[nodeId] : null; + const effective = nodeConfig && typeof nodeConfig === "object" && !Array.isArray(nodeConfig) ? { ...config, ...nodeConfig } : config; + assert.ok(typeof effective.serviceName === "string" && effective.serviceName.length > 0, `deploy.lanes.${profile}.externalPostgres.serviceName is required`); + assert.ok(typeof effective.endpointAddress === "string" && effective.endpointAddress.length > 0, `deploy.lanes.${profile}.externalPostgres.endpointAddress is required`); + const port = Number(effective.port ?? 5432); + assert.ok(Number.isInteger(port) && port > 0 && port <= 65535, `deploy.lanes.${profile}.externalPostgres.port must be a valid TCP port`); + return { serviceName: effective.serviceName, endpointAddress: effective.endpointAddress, port }; +} + +function externalPostgresManifest({ profile = "v03", config, source }) { + const namespace = namespaceNameForProfile(profile); + const labels = { + "app.kubernetes.io/name": config.serviceName, + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/component": "platform-db-bridge", + "hwlab.pikastech.local/environment": profile, + "hwlab.pikastech.local/gitops-target": profile, + "hwlab.pikastech.local/profile": profile, + "hwlab.pikastech.local/source-commit": source.full + }; + const annotations = { "hwlab.pikastech.local/rendered-by": "scripts/gitops-render.mjs" }; + return { + apiVersion: "v1", + kind: "List", + items: [ + { + apiVersion: "v1", + kind: "Service", + metadata: { name: config.serviceName, namespace, labels, annotations }, + spec: { type: "ClusterIP", ports: [{ name: "postgres", port: config.port, targetPort: config.port, protocol: "TCP" }] } + }, + { + apiVersion: "discovery.k8s.io/v1", + kind: "EndpointSlice", + metadata: { name: `${config.serviceName}-host`, namespace, labels: { ...labels, "kubernetes.io/service-name": config.serviceName }, annotations }, + addressType: "IPv4", + ports: [{ name: "postgres", port: config.port, protocol: "TCP" }], + endpoints: [{ addresses: [config.endpointAddress], conditions: { ready: true } }] + } + ] + }; +} + +function v02OpenFgaManifest({ profile = "v02", source }) { + const namespace = namespaceNameForProfile(profile); + const name = "hwlab-openfga"; + const image = "127.0.0.1:5000/hwlab/openfga:v1.17.0"; + const labels = { + "app.kubernetes.io/name": name, + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/component": "authorization", + "hwlab.pikastech.local/environment": profile, + "hwlab.pikastech.local/gitops-target": profile, + "hwlab.pikastech.local/profile": profile, + "hwlab.pikastech.local/service-id": name, + "hwlab.pikastech.local/source-commit": source.full + }; + const annotations = { "hwlab.pikastech.local/rendered-by": "scripts/gitops-render.mjs" }; + const selector = { "app.kubernetes.io/name": name }; + const env = [ + { name: "OPENFGA_DATASTORE_ENGINE", value: "postgres" }, + { name: "OPENFGA_DATASTORE_URI", valueFrom: { secretKeyRef: { name: `hwlab-${profile}-openfga`, key: "datastore-uri" } } }, + { name: "OPENFGA_AUTHN_METHOD", value: "preshared" }, + { name: "OPENFGA_AUTHN_PRESHARED_KEYS", valueFrom: { secretKeyRef: { name: `hwlab-${profile}-openfga`, key: "authn-preshared-key" } } }, + { name: "OPENFGA_PLAYGROUND_ENABLED", value: "false" }, + { name: "OPENFGA_HTTP_ADDR", value: "0.0.0.0:8080" }, + { name: "OPENFGA_GRPC_ADDR", value: "0.0.0.0:8081" } + ]; + const templateLabels = { ...labels, ...selector }; + const runtimeAnnotations = { ...annotations, "argocd.argoproj.io/sync-wave": "2" }; + return { + apiVersion: "v1", + kind: "List", + items: [ + { + apiVersion: "batch/v1", + kind: "Job", + metadata: { + name: `${name}-migrate`, + namespace, + labels, + annotations: { + ...annotations, + "argocd.argoproj.io/hook": "Sync", + "argocd.argoproj.io/sync-wave": "1", + "argocd.argoproj.io/hook-delete-policy": "BeforeHookCreation,HookSucceeded" + } + }, + spec: { + backoffLimit: 3, + template: { + metadata: { labels: templateLabels, annotations }, + spec: { + restartPolicy: "OnFailure", + containers: [{ name: "openfga-migrate", image, imagePullPolicy: "IfNotPresent", args: ["migrate"], env }] + } + } + } + }, + { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { name, namespace, labels, annotations: runtimeAnnotations }, + spec: { + replicas: 1, + selector: { matchLabels: selector }, + template: { + metadata: { labels: templateLabels, annotations }, + spec: { + containers: [{ + name: "openfga", + image, + imagePullPolicy: "IfNotPresent", + args: ["run"], + env, + ports: [{ name: "http", containerPort: 8080 }, { name: "grpc", containerPort: 8081 }], + readinessProbe: { httpGet: { path: "/healthz", port: "http" }, initialDelaySeconds: 3, periodSeconds: 10 }, + livenessProbe: { httpGet: { path: "/healthz", port: "http" }, initialDelaySeconds: 15, periodSeconds: 20 }, + resources: { requests: { cpu: "50m", memory: "128Mi" }, limits: { cpu: "500m", memory: "512Mi" } } + }] + } + } + } + }, + { + apiVersion: "v1", + kind: "Service", + metadata: { name, namespace, labels, annotations: runtimeAnnotations }, + spec: { type: "ClusterIP", selector, ports: [{ name: "http", port: 8080, targetPort: "http" }, { name: "grpc", port: 8081, targetPort: "grpc" }] } + } + ] + }; +} + +function workbenchRuntimeRedisConf(config) { + return [ + "save \"\"", + "appendonly no", + `maxmemory ${config.memoryPolicy.maxMemory}`, + `maxmemory-policy ${config.memoryPolicy.eviction}`, + "" + ].join("\n"); +} + +function workbenchRuntimeRedisManifest({ profile = "v03", config, source }) { + const namespace = config.namespace; + const name = config.serviceName; + const port = config.port; + const selector = { "app.kubernetes.io/name": name }; + const labels = { + ...selector, + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/cache-role": "workbench-derived-read", + "hwlab.pikastech.local/component": "workbench-runtime-cache", + "hwlab.pikastech.local/environment": profile, + "hwlab.pikastech.local/gitops-target": profile, + "hwlab.pikastech.local/profile": profile, + "hwlab.pikastech.local/service-id": name, + "hwlab.pikastech.local/source-commit": source.full + }; + const annotations = { "hwlab.pikastech.local/rendered-by": "scripts/gitops-render.mjs" }; + const runtimeAnnotations = { ...annotations, "argocd.argoproj.io/sync-wave": "2" }; + const templateLabels = { ...labels, ...selector }; + return { + apiVersion: "v1", + kind: "List", + items: [ + { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { name: `${name}-config`, namespace, labels, annotations: runtimeAnnotations }, + data: { "redis.conf": workbenchRuntimeRedisConf(config) } + }, + { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { name, namespace, labels, annotations: runtimeAnnotations }, + spec: { + replicas: 1, + selector: { matchLabels: selector }, + template: { + metadata: { labels: templateLabels, annotations: runtimeAnnotations }, + spec: { + containers: [{ + name: "redis", + image: config.image, + imagePullPolicy: "IfNotPresent", + args: ["redis-server", "/usr/local/etc/redis/redis.conf"], + ports: [{ name: "redis", containerPort: port }], + readinessProbe: { exec: { command: ["redis-cli", "-p", String(port), "ping"] }, initialDelaySeconds: 3, periodSeconds: 10, timeoutSeconds: 2, failureThreshold: 3 }, + livenessProbe: { exec: { command: ["redis-cli", "-p", String(port), "ping"] }, initialDelaySeconds: 15, periodSeconds: 20, timeoutSeconds: 3, failureThreshold: 3 }, + resources: cloneJson(config.resources), + volumeMounts: [{ name: "config", mountPath: "/usr/local/etc/redis", readOnly: true }] + }], + volumes: [{ name: "config", configMap: { name: `${name}-config` } }] + } + } + } + }, + { + apiVersion: "v1", + kind: "Service", + metadata: { name, namespace, labels, annotations: runtimeAnnotations }, + spec: { type: "ClusterIP", selector, ports: [{ name: "redis", port, targetPort: "redis", protocol: "TCP" }] } + }, + { + apiVersion: "networking.k8s.io/v1", + kind: "NetworkPolicy", + metadata: { name: `${name}-ingress`, namespace, labels, annotations: runtimeAnnotations }, + spec: { + podSelector: { matchLabels: selector }, + policyTypes: ["Ingress"], + ingress: [{ + from: [{ podSelector: { matchLabels: { "app.kubernetes.io/name": "hwlab-workbench-runtime" } } }], + ports: [{ protocol: "TCP", port }] + }] + } + } + ] + }; +} + +function runtimeConfigMapsForProfile(deploy, profile) { + if (!isRuntimeLane(profile)) return []; + const configMaps = deploy?.lanes?.[profile]?.configMaps; + if (!Array.isArray(configMaps)) return []; + return configMaps + .map((configMap) => cloneJson(configMap)) + .filter((configMap) => typeof configMap?.name === "string" && configMap.name.trim() && configMap.data && typeof configMap.data === "object" && !Array.isArray(configMap.data)); +} + +function runtimeConfigMapsManifest({ configMaps, namespace, labels, annotations }) { + return { + apiVersion: "v1", + kind: "List", + items: configMaps.map((configMap) => ({ + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + name: configMap.name, + namespace, + labels: { ...labels, ...(configMap.labels ?? {}) }, + annotations: { ...annotations, ...(configMap.annotations ?? {}) } + }, + data: Object.fromEntries(Object.entries(configMap.data).map(([key, value]) => [key, String(value)])) + })) + }; +} + +async function runtimeSecretPlaneConfigForProfile(deploy, profile, args = {}) { + if (!isRuntimeLane(profile)) return null; + const laneConfig = deploy?.lanes?.[profile]; + const secretPlane = laneConfig?.secretPlaneRef + ? await readConfigRef(laneConfig.secretPlaneRef, `${profile}.secretPlaneRef`) + : laneConfig?.secretPlane; + if (!secretPlane || typeof secretPlane !== "object" || Array.isArray(secretPlane)) return null; + if (secretPlane.enabled !== true) return null; + if (!runtimeSecretPlaneEnabledForNode(secretPlane, args)) return null; + return cloneJson(secretPlane); +} + +function runtimeSecretPlaneEnabledForNode(secretPlane, args = {}) { + const enabledOnNodes = normalizeSecretPlaneNodeList(secretPlane.enabledOnNodes, "secretPlane.enabledOnNodes"); + if (enabledOnNodes.length === 0) return true; + const nodeId = effectiveSecretPlaneNodeId(args); + assert.ok(nodeId, "secretPlane.enabledOnNodes requires --node or a node-scoped --gitops-root"); + return enabledOnNodes.includes(nodeId); +} + +function normalizeSecretPlaneNodeList(value, label) { + if (value === undefined) return []; + assert.ok(Array.isArray(value), `${label} must be an array when set`); + return value.map((item, index) => { + assert.equal(typeof item, "string", `${label}[${index}] must be a string`); + const nodeId = item.trim().toUpperCase(); + assert.ok(/^[A-Z0-9][A-Z0-9-]*$/u.test(nodeId), `${label}[${index}] must be a node id`); + return nodeId; + }); +} + +function requiredSecretPlaneString(value, label) { + assert.equal(typeof value, "string", `${label} must be a string`); + const trimmed = value.trim(); + assert.ok(trimmed, `${label} must not be empty`); + return trimmed; +} + +function runtimeSecretPlaneManifest({ config, namespace, labels, annotations }) { + const store = config?.store; + assert.ok(store && typeof store === "object" && !Array.isArray(store), "secretPlane.store must be an object"); + assert.equal(store.kind, "ClusterSecretStore", "secretPlane.store.kind must be ClusterSecretStore for node-scoped v0.3"); + assert.ok(typeof store.name === "string" && store.name.trim(), "secretPlane.store.name must be set"); + const secrets = Array.isArray(config.secrets) ? config.secrets : []; + assert.ok(secrets.length > 0, "secretPlane.secrets must not be empty"); + return { + apiVersion: "v1", + kind: "List", + items: secrets.map((secret, index) => runtimeSecretPlaneExternalSecret({ config, secret, index, namespace, labels, annotations, store })) + }; +} + +function runtimeSecretPlaneExternalSecret({ config, secret, index, namespace, labels, annotations, store }) { + const externalSecretName = requiredSecretPlaneString(secret?.externalSecretName, `secretPlane.secrets[${index}].externalSecretName`); + const targetSecretName = requiredSecretPlaneString(secret?.targetSecretName, `secretPlane.secrets[${index}].targetSecretName`); + const data = Array.isArray(secret?.data) ? secret.data : []; + assert.ok(data.length > 0, `secretPlane.secrets[${index}].data must not be empty`); + const sourceRefs = data.map((item) => requiredSecretPlaneString(item?.remoteRef, `secretPlane.secrets[${index}].data.remoteRef`)); + const issueRef = String(config.issue ?? "pikasTech/HWLAB#2234"); + return { + apiVersion: "external-secrets.io/v1", + kind: "ExternalSecret", + metadata: { + name: externalSecretName, + namespace, + labels: { + ...labels, + "app.kubernetes.io/name": externalSecretName, + "app.kubernetes.io/component": "external-secret", + "hwlab.pikastech.local/secret-plane": secretPlaneIssueLabelValue(issueRef) + }, + annotations: { + ...annotations, + "hwlab.pikastech.local/secret-plane-issue": issueRef, + "hwlab.pikastech.local/source-ref": sourceRefs.join(","), + "hwlab.pikastech.local/values-printed": "false" + } + }, + spec: { + refreshInterval: requiredSecretPlaneString(config.refreshInterval, "secretPlane.refreshInterval"), + secretStoreRef: { + name: requiredSecretPlaneString(store.name, "secretPlane.store.name"), + kind: store.kind + }, + target: { + name: targetSecretName, + creationPolicy: "Owner" + }, + data: data.map((item, itemIndex) => ({ + secretKey: requiredSecretPlaneString(item?.targetKey, `secretPlane.secrets[${index}].data[${itemIndex}].targetKey`), + remoteRef: { + key: requiredSecretPlaneString(item?.remoteRef, `secretPlane.secrets[${index}].data[${itemIndex}].remoteRef`), + property: requiredSecretPlaneString(item?.property, `secretPlane.secrets[${index}].data[${itemIndex}].property`) + } + })) + } + }; +} + +function secretPlaneIssueLabelValue(issueRef) { + const text = String(issueRef ?? "").trim(); + const issueNumber = text.match(/#([0-9]+)$/u)?.[1]; + const raw = issueNumber ? `issue-${issueNumber}` : text; + const normalized = raw.replace(/[^A-Za-z0-9_.-]+/gu, "-").replace(/^[^A-Za-z0-9]+|[^A-Za-z0-9]+$/gu, ""); + const label = normalized.slice(0, 63).replace(/[^A-Za-z0-9]+$/u, ""); + assert.ok(label.length > 0, "secretPlane.issue must produce a Kubernetes label-safe value"); + return label; +} + +function opencodeRuntimeConfigForProfile(deploy, profile) { + const config = optionalObject(deploy?.lanes?.[profile]?.opencode); + const providerProfile = configString(config.providerProfile) || "dsflash-go"; + const providerId = configString(config.providerId) || providerProfile; + const model = configString(config.model) || "deepseek-v4-flash"; + const smallModel = configString(config.smallModel) || model; + const apiKeyEnv = configString(config.apiKeyEnv) || opencodeApiKeyEnvName(providerId); + const contextLimit = opencodePositiveInteger(config.contextLimit, 1000000, `deploy.lanes.${profile}.opencode.contextLimit`); + const outputLimit = opencodePositiveInteger(config.outputLimit, 384000, `deploy.lanes.${profile}.opencode.outputLimit`); + const reasoning = opencodeConfigBoolean(config.reasoning, false, `deploy.lanes.${profile}.opencode.reasoning`); + const toolCall = opencodeConfigBoolean(config.toolCall, true, `deploy.lanes.${profile}.opencode.toolCall`); + const upstreamBaseURL = configString(config.upstreamBaseURL) || configString(config.baseURL) || "https://opencode.ai/zen/go/v1"; + const providerProxyPort = opencodePositiveInteger(config.providerProxyPort, 4097, `deploy.lanes.${profile}.opencode.providerProxyPort`); + const providerProxyBasePath = configString(config.providerProxyBasePath) || "/v1"; + const providerProxyBaseURL = configString(config.providerProxyBaseURL) || `http://127.0.0.1:${providerProxyPort}${providerProxyBasePath}`; + return { + image: configString(config.image) || "ghcr.io/anomalyco/opencode:1.17.7", + providerProfile, + providerId, + displayName: configString(config.displayName) || `AgentRun ${providerProfile}`, + model, + smallModel, + baseURL: providerProxyBaseURL, + upstreamBaseURL, + apkProxyURL: configString(config.apkProxyURL) || "", + providerProxyPort, + providerProxyBasePath, + npm: configString(config.npm) || "@ai-sdk/openai-compatible", + apiKeyEnv, + apiKeySecretRef: configString(config.apiKeySecretRef) || "hwlab-code-agent-provider/opencode-api-key", + contextLimit, + outputLimit, + reasoning, + toolCall, + agentrunSecretNamespace: configString(config.agentrunSecretNamespace) || "agentrun-v02", + agentrunSecretName: configString(config.agentrunSecretName) || `agentrun-v01-provider-${providerProfile}`, + agentrunSecretKeys: Array.isArray(config.agentrunSecretKeys) && config.agentrunSecretKeys.length > 0 + ? config.agentrunSecretKeys.map(configString).filter(Boolean) + : ["auth.json", "config.toml", "model-catalog.json"] + }; +} + +function opencodePositiveInteger(value, fallback, label) { + if (value === undefined || value === null) return fallback; + assert.ok(Number.isInteger(value) && value > 0, `${label} must be a positive integer`); + return value; +} + +function opencodeConfigBoolean(value, fallback, label) { + if (value === undefined || value === null) return fallback; + assert.equal(typeof value, "boolean", `${label} must be a boolean`); + return value; +} + +function opencodeEgressProxyUrlForProfile(deploy, profile) { + const proxy = deploy?.lanes?.[profile]?.gitMirror?.egressProxy; + if (!proxy || proxy.required === false) return ""; + const proxyUrl = configString(proxy.proxyUrl); + if (proxyUrl) return proxyUrl; + const serviceName = configString(proxy.serviceName); + const namespace = configString(proxy.namespace) || "platform-infra"; + const port = Number(proxy.port); + if (!serviceName || !Number.isInteger(port) || port <= 0) return ""; + return `http://${serviceName}.${namespace}.svc.cluster.local:${port}`; +} + +function opencodeEgressNoProxyForProfile(deploy, profile) { + const proxy = deploy?.lanes?.[profile]?.gitMirror?.egressProxy; + const entries = Array.isArray(proxy?.noProxy) ? proxy.noProxy.map(configString).filter(Boolean) : []; + return entries.length > 0 ? entries.join(",") : "localhost,127.0.0.1,::1,.svc,.svc.cluster.local,.cluster.local"; +} + +function opencodeApiKeyEnvName(providerId) { + const normalized = String(providerId || "provider").toUpperCase().replace(/[^A-Z0-9]+/gu, "_").replace(/^_+|_+$/gu, ""); + return `OPENCODE_${normalized || "PROVIDER"}_API_KEY`; +} + +function opencodeConfigContent(config) { + return JSON.stringify({ + $schema: "https://opencode.ai/config.json", + autoupdate: false, + share: "disabled", + model: `${config.providerId}/${config.model}`, + small_model: `${config.providerId}/${config.smallModel}`, + provider: { + [config.providerId]: { + npm: config.npm, + name: config.displayName, + options: { + baseURL: config.baseURL, + apiKey: `{env:${config.apiKeyEnv}}`, + timeout: 600000, + headerTimeout: 600000, + chunkTimeout: 300000 + }, + models: { + [config.model]: { + name: config.model, + reasoning: config.reasoning, + tool_call: config.toolCall, + limit: { + context: config.contextLimit, + output: config.outputLimit + } + } + } + } + } + }); +} + +function opencodeServerManifest({ profile = "v03", source, deploy = null, catalog = null, registryPrefix = defaultRegistryPrefix, useDeployImages = false, sourceBranch = defaultBranch, gitReadUrl = defaultV02GitReadUrl, sourceRepo = defaultSourceRepo }) { + assert.ok(isRuntimeLane(profile), `opencode-server profile must be a runtime lane, got ${profile}`); + const namespace = namespaceNameForProfile(profile); + const name = "opencode-server"; + const helperServiceId = "hwlab-cloud-web"; + const envReuseServiceIds = envReuseServiceIdsForLane(deploy, profile); + const helperImage = runtimeImageForService({ catalog, deploy, serviceId: helperServiceId, source, registryPrefix, useDeployImages, digestPin: true, envReuseServiceIds }); + const helperCommit = runtimeCommitForService({ catalog, deploy, serviceId: helperServiceId, source, registryPrefix, useDeployImages, digestPin: true, envReuseServiceIds }); + const helperImageTag = runtimeImageTagForService({ catalog, deploy, serviceId: helperServiceId, source, registryPrefix, useDeployImages, digestPin: true, envReuseServiceIds }); + const providerProxyBootSh = "deploy/runtime/boot/opencode-provider-proxy.sh"; + const providerProxyBootMetadata = v02EnvReuseEnabled(catalog, helperServiceId, envReuseServiceIds) + ? bootMetadataForService({ args: { sourceRepo: deploy?.lanes?.[profile]?.sourceRepo || sourceRepo, registryPrefix }, catalog, deployService: deployServiceForBoot(deploy, helperServiceId, profile), serviceId: helperServiceId, source }) + : null; + const selector = { + "app.kubernetes.io/name": name, + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/environment": profile, + "hwlab.pikastech.local/profile": profile + }; + const labels = { + ...selector, + "hwlab.pikastech.local/component": "opencode", + "hwlab.pikastech.local/gitops-target": profile, + "hwlab.pikastech.local/service-id": name, + "hwlab.pikastech.local/source-commit": source.full + }; + const opencodeConfig = opencodeRuntimeConfigForProfile(deploy, profile); + const configContent = opencodeConfigContent(opencodeConfig); + const configSha256 = createHash("sha256").update(configContent).digest("hex"); + const apiKeySecretRef = secretRefObjectForProfile(opencodeConfig.apiKeySecretRef, profile); + const opencodeApkProxyUrl = opencodeConfig.apkProxyURL || opencodeEgressProxyUrlForProfile(deploy, profile); + const opencodeApkNoProxy = opencodeEgressNoProxyForProfile(deploy, profile); + const annotations = { + "hwlab.pikastech.local/rendered-by": "scripts/gitops-render.mjs", + "hwlab.pikastech.local/opencode-provider-profile": opencodeConfig.providerProfile, + "hwlab.pikastech.local/opencode-provider-id": opencodeConfig.providerId, + "hwlab.pikastech.local/opencode-model": opencodeConfig.model, + "hwlab.pikastech.local/opencode-provider-base-url": opencodeConfig.baseURL, + "hwlab.pikastech.local/opencode-provider-upstream-base-url": opencodeConfig.upstreamBaseURL, + "hwlab.pikastech.local/opencode-image": opencodeConfig.image, + "hwlab.pikastech.local/opencode-config-sha256": configSha256, + "hwlab.pikastech.local/opencode-api-key-secret-ref": `${apiKeySecretRef.name}/${apiKeySecretRef.key}`, + "hwlab.pikastech.local/agentrun-provider-secret-ref": `${opencodeConfig.agentrunSecretNamespace}/${opencodeConfig.agentrunSecretName}`, + "hwlab.pikastech.local/agentrun-provider-secret-keys": opencodeConfig.agentrunSecretKeys.join(","), + "hwlab.pikastech.local/values-printed": "false" + }; + if (providerProxyBootMetadata) { + Object.assign(annotations, { + "hwlab.pikastech.local/opencode-provider-proxy-runtime-mode": providerProxyBootMetadata.runtimeMode, + "hwlab.pikastech.local/opencode-provider-proxy-boot-repo": providerProxyBootMetadata.bootRepo, + "hwlab.pikastech.local/opencode-provider-proxy-boot-commit": providerProxyBootMetadata.bootCommit, + "hwlab.pikastech.local/opencode-provider-proxy-boot-sh": providerProxyBootSh, + "hwlab.pikastech.local/opencode-provider-proxy-environment-digest": providerProxyBootMetadata.environmentDigest ?? "not_published" + }); + } + const authSecretName = secretNameForProfile("hwlab-opencode-server-auth", profile); + return { + apiVersion: "v1", + kind: "List", + items: [ + { + apiVersion: "v1", + kind: "ServiceAccount", + metadata: { name, namespace, labels, annotations } + }, + { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { name: `${name}-data`, namespace, labels, annotations }, + spec: { + accessModes: ["ReadWriteOnce"], + resources: { requests: { storage: "8Gi" } } + } + }, + { + apiVersion: "v1", + kind: "Service", + metadata: { name, namespace, labels, annotations }, + spec: { + type: "ClusterIP", + selector, + ports: [{ name: "http", port: 4096, targetPort: "http" }] + } + }, + { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { name, namespace, labels, annotations }, + spec: { + replicas: 1, + strategy: { type: "Recreate" }, + selector: { matchLabels: selector }, + template: { + metadata: { labels, annotations }, + spec: { + serviceAccountName: name, + securityContext: { fsGroup: 1000, fsGroupChangePolicy: "OnRootMismatch" }, + initContainers: [{ + name: "opencode-workspace-git-init", + image: helperImage, + imagePullPolicy: "IfNotPresent", + command: ["/bin/sh", "-ec"], + args: [ + [ + "set -eu", + "mkdir -p /workspace", + "if [ ! -d /workspace/.git ]; then", + " git -C /workspace init", + "fi", + "git -C /workspace config user.name 'HWLAB OpenCode' || true", + "git -C /workspace config user.email 'opencode@hwlab.local' || true", + "chgrp -R 1000 /workspace/.git 2>/dev/null || true", + "chmod -R g+rwX /workspace/.git 2>/dev/null || true" + ].join("\n") + ], + resources: { requests: { cpu: "25m", memory: "64Mi" }, limits: { cpu: "250m", memory: "256Mi" } }, + volumeMounts: [{ name: "workspace", mountPath: "/workspace" }] + }], + containers: [{ + name, + image: opencodeConfig.image, + imagePullPolicy: "IfNotPresent", + command: ["/bin/sh", "-ec"], + args: ["exec opencode serve --hostname 0.0.0.0 --port 4096"], + workingDir: "/workspace", + env: [ + { name: "HOME", value: "/workspace" }, + { name: "XDG_CONFIG_HOME", value: "/workspace/.config" }, + { name: "XDG_DATA_HOME", value: "/workspace/.local/share" }, + ...(opencodeApkProxyUrl ? [ + { name: "HTTP_PROXY", value: opencodeApkProxyUrl }, + { name: "HTTPS_PROXY", value: opencodeApkProxyUrl }, + { name: "http_proxy", value: opencodeApkProxyUrl }, + { name: "https_proxy", value: opencodeApkProxyUrl }, + { name: "NO_PROXY", value: opencodeApkNoProxy }, + { name: "no_proxy", value: opencodeApkNoProxy } + ] : []), + { name: "OPENCODE_CONFIG_CONTENT", value: configContent }, + { name: opencodeConfig.apiKeyEnv, valueFrom: { secretKeyRef: apiKeySecretRef } }, + { name: "OPENCODE_SERVER_USERNAME", valueFrom: { secretKeyRef: { name: authSecretName, key: "username" } } }, + { name: "OPENCODE_SERVER_PASSWORD", valueFrom: { secretKeyRef: { name: authSecretName, key: "password" } } } + ], + ports: [{ name: "http", containerPort: 4096 }], + startupProbe: { tcpSocket: { port: "http" }, periodSeconds: 5, failureThreshold: 60 }, + readinessProbe: { tcpSocket: { port: "http" }, periodSeconds: 10, failureThreshold: 3 }, + livenessProbe: { tcpSocket: { port: "http" }, periodSeconds: 20, failureThreshold: 3 }, + resources: { requests: { cpu: "100m", memory: "256Mi" }, limits: { cpu: "1", memory: "1Gi" } }, + volumeMounts: [{ name: "workspace", mountPath: "/workspace" }] + }, (() => { + const container = { + name: "opencode-provider-proxy", + image: helperImage, + imagePullPolicy: "IfNotPresent", + command: ["node", "/app/internal/dev-entrypoint/opencode-provider-proxy.mjs"], + env: [ + { name: "HWLAB_SERVICE_ID", value: "opencode-provider-proxy" }, + { name: "HWLAB_ENVIRONMENT", value: profile }, + { name: "HWLAB_COMMIT_ID", value: helperCommit }, + { name: "HWLAB_IMAGE", value: helperImage }, + { name: "HWLAB_IMAGE_TAG", value: helperImageTag }, + { name: "PORT", value: String(opencodeConfig.providerProxyPort) }, + { name: "HWLAB_OPENCODE_PROVIDER_PROXY_UPSTREAM_BASE_URL", value: opencodeConfig.upstreamBaseURL }, + { name: "HWLAB_OPENCODE_PROVIDER_PROXY_PUBLIC_BASE_PATH", value: opencodeConfig.providerProxyBasePath }, + { name: "HWLAB_OPENCODE_PROVIDER_PROXY_TIMEOUT_MS", value: "600000" }, + { name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", value: "http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces" }, + { name: "OTEL_SERVICE_NAME", value: "opencode-provider-proxy" } + ], + ports: [{ name: "provider", containerPort: opencodeConfig.providerProxyPort }], + readinessProbe: { httpGet: { path: "/health/readiness", port: "provider" }, periodSeconds: 10, failureThreshold: 3 }, + livenessProbe: { httpGet: { path: "/health/live", port: "provider" }, periodSeconds: 20, failureThreshold: 3 }, + resources: { requests: { cpu: "25m", memory: "64Mi" }, limits: { cpu: "250m", memory: "256Mi" } } + }; + if (providerProxyBootMetadata) { + applyEnvReuseBootEnv(container, providerProxyBootMetadata, { sourceBranch, gitReadUrl, bootSh: providerProxyBootSh }); + } + return container; + })()], + volumes: [{ name: "workspace", persistentVolumeClaim: { claimName: `${name}-data` } }] + } + } + } + } + ] + }; +} + +export { + argoApplication, + argoProject, + deepSeekProxyManifest, + deviceAgent71FreqManifest, + deviceAgent71FreqServerScript, + externalPostgresConfigForLane, + externalPostgresManifest, + moonBridgeConfigRenderScript, + nodeFrpcManifest, + normalizeSecretPlaneNodeList, + opencodeApiKeyEnvName, + opencodeConfigBoolean, + opencodeConfigContent, + opencodeEgressNoProxyForProfile, + opencodeEgressProxyUrlForProfile, + opencodePositiveInteger, + opencodeRuntimeConfigForProfile, + opencodeServerManifest, + runtimeConfigMapsForProfile, + runtimeConfigMapsManifest, + runtimePostgresImageForProfile, + runtimeSecretPlaneConfigForProfile, + runtimeSecretPlaneEnabledForNode, + runtimeSecretPlaneExternalSecret, + runtimeSecretPlaneManifest, + secretPlaneIssueLabelValue, + v02OpenFgaManifest, + v02PostgresManifest, + workbenchRuntimeRedisConf, + workbenchRuntimeRedisManifest +}; diff --git a/scripts/src/gitops-render/tekton-manifests.mjs b/scripts/src/gitops-render/tekton-manifests.mjs new file mode 100644 index 00000000..5d555130 --- /dev/null +++ b/scripts/src/gitops-render/tekton-manifests.mjs @@ -0,0 +1,830 @@ +import { + argoApplicationName, + buildkitRunnerImage, + ciToolsRunnerImage, + defaultDevBaseImage, + defaultRegistryPrefix, + defaultServiceIds, + defaultSourceRepo, + gitopsPathFor, + gitopsPathForProfile, + gitopsTargetForProfile, + isRuntimeLane, + namespaceNameForProfile, + primitiveValidationTasks, + proxyEnv, + serviceIdsForLane, + servicesParamForLane +} from "./core.mjs"; +import { + ciTimingShellFunction, + collectArtifactsScript, + controlPlaneReconcileScript, + gitopsPromoteScript, + perServicePublishScript, + planArtifactsScript, + pollerScript, + prepareSourceScript, + primitiveValidationTask, + primitiveValidationTaskNames +} from "./tekton-scripts.mjs"; + +function ciLaneSettings(argsOrLane = "node") { + const lane = typeof argsOrLane === "string" ? argsOrLane : argsOrLane.lane; + if (isRuntimeLane(lane)) { + const namespace = namespaceNameForProfile(lane); + return { + lane, + profile: lane, + gitopsTarget: lane, + pipelineName: `${namespace}-ci-image-publish`, + pipelineRunPrefix: `${namespace}-ci-poll`, + serviceAccountName: `${namespace}-tekton-runner`, + pollerName: `${namespace}-branch-poller`, + controlPlaneReconcilerName: `${namespace}-control-plane-reconciler`, + tektonDir: `tekton-${lane}`, + catalogPath: typeof argsOrLane === "object" ? argsOrLane.catalogPath : `deploy/artifact-catalog.${lane}.json`, + imageTagMode: typeof argsOrLane === "object" ? argsOrLane.imageTagMode : "full", + runtimeNamespace: namespace + }; + } + return { + lane: "node", + profile: "dev", + gitopsTarget: "node", + pipelineName: "hwlab-node-ci-image-publish", + pipelineRunPrefix: "hwlab-node-ci-poll", + serviceAccountName: "hwlab-tekton-runner", + pollerName: "hwlab-node-branch-poller", + controlPlaneReconcilerName: "hwlab-node-control-plane-reconciler", + tektonDir: "tekton", + catalogPath: "deploy/artifact-catalog.dev.json", + imageTagMode: "short", + runtimeNamespace: "hwlab-dev" + }; +} + +function tektonRbac(args = { lane: "node" }) { + const settings = ciLaneSettings(args); + if (isRuntimeLane(settings.lane)) { + const runtimeObserverName = `${settings.runtimeNamespace}-runtime-observer`; + const pipelineRunWriterName = `${settings.runtimeNamespace}-pipelinerun-writer`; + const argoReconcilerName = `${settings.runtimeNamespace}-argocd-reconciler`; + return { + apiVersion: "v1", + kind: "List", + items: [ + { apiVersion: "v1", kind: "Namespace", metadata: { name: "hwlab-ci" } }, + { apiVersion: "v1", kind: "ServiceAccount", metadata: { name: settings.serviceAccountName, namespace: "hwlab-ci" } }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "Role", + metadata: { name: runtimeObserverName, namespace: settings.runtimeNamespace }, + rules: [ + { apiGroups: [""], resources: ["pods", "services", "configmaps"], verbs: ["get", "list", "watch"] }, + { apiGroups: ["apps"], resources: ["deployments", "statefulsets"], verbs: ["get", "list", "watch"] }, + { apiGroups: ["batch"], resources: ["jobs"], verbs: ["get", "list", "watch"] } + ] + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "RoleBinding", + metadata: { name: runtimeObserverName, namespace: settings.runtimeNamespace }, + subjects: [{ kind: "ServiceAccount", name: settings.serviceAccountName, namespace: "hwlab-ci" }], + roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: runtimeObserverName } + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "Role", + metadata: { name: pipelineRunWriterName, namespace: "hwlab-ci" }, + rules: [ + { apiGroups: ["tekton.dev"], resources: ["pipelineruns"], verbs: ["get", "list", "create"] }, + { apiGroups: ["tekton.dev"], resources: ["pipelines"], verbs: ["get", "patch", "create"] }, + { apiGroups: ["batch"], resources: ["cronjobs"], verbs: ["get", "patch", "create"] }, + { apiGroups: ["rbac.authorization.k8s.io"], resources: ["roles", "rolebindings"], verbs: ["get", "patch", "create"] }, + { apiGroups: [""], resources: ["serviceaccounts"], verbs: ["get", "patch", "create"] } + ] + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "RoleBinding", + metadata: { name: pipelineRunWriterName, namespace: "hwlab-ci" }, + subjects: [{ kind: "ServiceAccount", name: settings.serviceAccountName, namespace: "hwlab-ci" }], + roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: pipelineRunWriterName } + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "Role", + metadata: { name: argoReconcilerName, namespace: "argocd" }, + rules: [ + { apiGroups: ["argoproj.io"], resources: ["applications", "appprojects"], verbs: ["get", "patch", "create"] }, + { apiGroups: ["rbac.authorization.k8s.io"], resources: ["roles", "rolebindings"], verbs: ["get", "patch", "create"] } + ] + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "RoleBinding", + metadata: { name: argoReconcilerName, namespace: "argocd" }, + subjects: [{ kind: "ServiceAccount", name: settings.serviceAccountName, namespace: "hwlab-ci" }], + roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: argoReconcilerName } + } + ] + }; + } + return { + apiVersion: "v1", + kind: "List", + items: [ + { apiVersion: "v1", kind: "Namespace", metadata: { name: "hwlab-ci" } }, + { apiVersion: "v1", kind: "ServiceAccount", metadata: { name: "hwlab-tekton-runner", namespace: "hwlab-ci" } }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "Role", + metadata: { name: "hwlab-node-control-plane-reconciler", namespace: "hwlab-dev" }, + rules: [ + { apiGroups: ["rbac.authorization.k8s.io"], resources: ["roles", "rolebindings"], verbs: ["get", "patch", "create"] } + ] + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "RoleBinding", + metadata: { name: "hwlab-node-control-plane-reconciler", namespace: "hwlab-dev" }, + subjects: [{ kind: "ServiceAccount", name: "hwlab-tekton-runner", namespace: "hwlab-ci" }], + roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: "hwlab-node-control-plane-reconciler" } + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "Role", + metadata: { name: "hwlab-tekton-runtime-observer", namespace: "hwlab-dev" }, + rules: [ + { apiGroups: [""], resources: ["pods", "services", "configmaps"], verbs: ["get", "list", "watch"] }, + { apiGroups: ["apps"], resources: ["deployments", "statefulsets"], verbs: ["get", "list", "watch"] }, + { apiGroups: ["batch"], resources: ["jobs"], verbs: ["get", "list", "watch"] } + ] + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "RoleBinding", + metadata: { name: "hwlab-tekton-runtime-observer", namespace: "hwlab-dev" }, + subjects: [{ kind: "ServiceAccount", name: "hwlab-tekton-runner", namespace: "hwlab-ci" }], + roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: "hwlab-tekton-runtime-observer" } + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "Role", + metadata: { name: "hwlab-tekton-pipelinerun-writer", namespace: "hwlab-ci" }, + rules: [ + { apiGroups: ["tekton.dev"], resources: ["pipelineruns"], verbs: ["get", "list", "create"] }, + { apiGroups: ["tekton.dev"], resources: ["pipelines"], verbs: ["get", "patch", "create"] }, + { apiGroups: ["batch"], resources: ["cronjobs"], verbs: ["get", "patch", "create"] }, + { apiGroups: ["rbac.authorization.k8s.io"], resources: ["roles", "rolebindings"], verbs: ["get", "patch", "create"] }, + { apiGroups: [""], resources: ["serviceaccounts"], verbs: ["get", "patch", "create"] } + ] + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "RoleBinding", + metadata: { name: "hwlab-tekton-pipelinerun-writer", namespace: "hwlab-ci" }, + subjects: [{ kind: "ServiceAccount", name: "hwlab-tekton-runner", namespace: "hwlab-ci" }], + roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: "hwlab-tekton-pipelinerun-writer" } + } + ] + }; +} + +function buildTaskName(serviceId) { + return `build-${serviceId}`; +} + +function affectedResultName(serviceId) { + return `affected-${serviceId}`; +} + +function buildResultName(serviceId) { + return `build-${serviceId}`; +} + +const serviceResultFields = Object.freeze([ + "status", + "service-id", + "image", + "image-tag", + "digest", + "repository-digest", + "source-commit-id", + "component-input-hash", + "environment-input-hash", + "code-input-hash", + "runtime-mode", + "boot-repo", + "boot-commit", + "boot-sh", + "build-created-at", + "build-backend", + "reused-from", + "go-base-image", + "go-base-image-status", + "go-base-digest", + "go-base-build-duration-ms", + "go-base-buildkit-cache-ref" +]); + +function serviceResultParamName(serviceId, field) { + return `${serviceId}-${field}`; +} + +function serviceResultEnvName(serviceId, field) { + return `HWLAB_SERVICE_RESULT_${serviceId.toUpperCase().replace(/[^A-Z0-9]+/gu, "_")}_${field.toUpperCase().replace(/[^A-Z0-9]+/gu, "_")}`; +} + +function serviceResultParams(serviceIds = defaultServiceIds) { + return serviceIds.flatMap((serviceId) => serviceResultFields.map((field) => ({ name: serviceResultParamName(serviceId, field), default: "" }))); +} + +function serviceResultParamValues(serviceIds = defaultServiceIds) { + return serviceIds.flatMap((serviceId) => serviceResultFields.map((field) => ({ + name: serviceResultParamName(serviceId, field), + value: `$(tasks.${buildTaskName(serviceId)}.results.${field})` + }))); +} + +function serviceResultEnv(serviceIds = defaultServiceIds) { + return serviceIds.flatMap((serviceId) => serviceResultFields.map((field) => ({ + name: serviceResultEnvName(serviceId, field), + value: `$(params.${serviceResultParamName(serviceId, field)})` + }))); +} + +function serviceWorkVolume() { + return { name: "service-work", emptyDir: {} }; +} + +function buildkitBinVolume() { + return { name: "buildkit-bin", emptyDir: {} }; +} + +function buildkitRunVolume() { + return { name: "buildkit-run", emptyDir: {} }; +} + +function prepareBuildkitClientStep() { + return { + name: "prepare-buildkit-client", + image: buildkitRunnerImage, + securityContext: { runAsUser: 0, runAsGroup: 0 }, + volumeMounts: [{ name: "buildkit-bin", mountPath: "/workspace/buildkit-bin" }], + script: `#!/bin/sh +set -eu +mkdir -p /workspace/buildkit-bin +cp /usr/bin/buildctl /workspace/buildkit-bin/ +chmod +x /workspace/buildkit-bin/* +` + }; +} + +function buildkitSidecar() { + return { + name: "buildkitd", + image: buildkitRunnerImage, + args: ["--addr", "unix:///workspace/buildkit-run/buildkitd.sock", "--oci-worker-no-process-sandbox"], + env: proxyEnv(), + volumeMounts: [{ name: "buildkit-run", mountPath: "/workspace/buildkit-run" }], + securityContext: { + runAsUser: 1000, + runAsGroup: 1000, + allowPrivilegeEscalation: true, + appArmorProfile: { type: "Unconfined" }, + seccompProfile: { type: "Unconfined" } + } + }; +} + +function planArtifactsTask({ serviceIds = defaultServiceIds } = {}) { + return { + name: "plan-artifacts", + runAfter: primitiveValidationTaskNames(), + workspaces: [{ name: "source", workspace: "source" }], + taskSpec: { + params: [ + { name: "git-url" }, + { name: "git-read-url" }, + { name: "gitops-branch" }, + { name: "lane" }, + { name: "revision" }, + { name: "catalog-path" }, + { name: "registry-prefix" }, + { name: "services" } + ], + results: serviceIds.flatMap((serviceId) => [ + { name: affectedResultName(serviceId), description: `${serviceId} rollout affected according to ci-plan` }, + { name: buildResultName(serviceId), description: `${serviceId} image build required according to ci-plan` } + ]), + workspaces: [{ name: "source" }], + steps: [{ + name: "plan", + image: ciToolsRunnerImage, + env: [ + { name: "HWLAB_SELECTED_SERVICES", value: "$(params.services)" }, + { name: "HWLAB_DEV_REGISTRY_K8S_NATIVE", value: "1" }, + { name: "HWLAB_TEKTON_PIPELINERUN", value: "$(context.pipelineRun.name)" }, + ...proxyEnv() + ], + script: planArtifactsScript() + }] + }, + params: [ + { name: "git-url", value: "$(params.git-url)" }, + { name: "git-read-url", value: "$(params.git-read-url)" }, + { name: "gitops-branch", value: "$(params.gitops-branch)" }, + { name: "lane", value: "$(params.lane)" }, + { name: "revision", value: "$(params.revision)" }, + { name: "catalog-path", value: "$(params.catalog-path)" }, + { name: "registry-prefix", value: "$(params.registry-prefix)" }, + { name: "services", value: "$(params.services)" } + ] + }; +} + +function perServiceBuildTask(serviceId, { runAfter = ["plan-artifacts"] } = {}) { + return { + name: buildTaskName(serviceId), + runAfter, + workspaces: [{ name: "source", workspace: "source" }], + when: [{ input: `$(tasks.plan-artifacts.results.${buildResultName(serviceId)})`, operator: "in", values: ["true"] }], + taskSpec: { + params: [ + { name: "revision" }, + { name: "lane" }, + { name: "catalog-path" }, + { name: "image-tag-mode" }, + { name: "registry-prefix" }, + { name: "service-id" }, + { name: "base-image" }, + { name: "build-cache-mode" } + ], + results: serviceResultFields.map((field) => ({ name: field, description: `${serviceId} ${field}` })), + workspaces: [{ name: "source" }], + volumes: [serviceWorkVolume(), buildkitBinVolume(), buildkitRunVolume()], + sidecars: [buildkitSidecar()], + steps: [ + prepareBuildkitClientStep(), + { + name: "publish", + image: ciToolsRunnerImage, + securityContext: { runAsUser: 1000, runAsGroup: 1000 }, + env: proxyEnv(), + volumeMounts: [{ name: "service-work", mountPath: "/workspace/service-work" }, { name: "buildkit-bin", mountPath: "/workspace/buildkit-bin" }, { name: "buildkit-run", mountPath: "/workspace/buildkit-run" }], + script: perServicePublishScript() + } + ] + }, + params: [ + { name: "revision", value: "$(params.revision)" }, + { name: "lane", value: "$(params.lane)" }, + { name: "catalog-path", value: "$(params.catalog-path)" }, + { name: "image-tag-mode", value: "$(params.image-tag-mode)" }, + { name: "registry-prefix", value: "$(params.registry-prefix)" }, + { name: "service-id", value: serviceId }, + { name: "base-image", value: "$(params.base-image)" }, + { name: "build-cache-mode", value: "$(params.build-cache-mode)" } + ] + }; +} + +function collectArtifactsTask({ serviceIds = defaultServiceIds } = {}) { + return { + name: "collect-artifacts", + runAfter: serviceIds.map(buildTaskName), + workspaces: [{ name: "source", workspace: "source" }], + taskSpec: { + params: [ + { name: "revision" }, + { name: "lane" }, + { name: "catalog-path" }, + { name: "image-tag-mode" }, + { name: "registry-prefix" }, + { name: "services" }, + { name: "base-image" } + ], + workspaces: [{ name: "source" }], + steps: [{ name: "collect", image: ciToolsRunnerImage, env: proxyEnv(), script: collectArtifactsScript() }] + }, + params: [ + { name: "revision", value: "$(params.revision)" }, + { name: "lane", value: "$(params.lane)" }, + { name: "catalog-path", value: "$(params.catalog-path)" }, + { name: "image-tag-mode", value: "$(params.image-tag-mode)" }, + { name: "registry-prefix", value: "$(params.registry-prefix)" }, + { name: "services", value: "$(params.services)" }, + { name: "base-image", value: "$(params.base-image)" } + ] + }; +} + +function runtimeReadyScript() { + return `#!/bin/sh +set -eu +${ciTimingShellFunction()} +export HWLAB_TEKTON_PIPELINERUN="$(context.pipelineRun.name)" +export HWLAB_TEKTON_TASKRUN="$(context.taskRun.name)" +export HWLAB_TEKTON_TASK="runtime-ready" +export HWLAB_SOURCE_REVISION="$(params.revision)" +export HWLAB_RUNTIME_READY_TIMEOUT_MS="$(params.timeout-ms)" +cd /workspace/source/repo +node scripts/ci/runtime-ready.mjs +`; +} + +function runtimeReadyTask({ profile = "dev" } = {}) { + return { + name: "runtime-ready", + runAfter: ["gitops-promote"], + workspaces: [{ name: "source", workspace: "source" }], + taskSpec: { + params: [{ name: "revision" }, { name: "runtime-namespace" }, { name: "gitops-target" }, { name: "argo-application" }, { name: "timeout-ms" }], + workspaces: [{ name: "source" }], + steps: [{ + name: "runtime-ready", + image: ciToolsRunnerImage, + env: [ + ...proxyEnv(), + { name: "HWLAB_RUNTIME_NAMESPACE", value: "$(params.runtime-namespace)" }, + { name: "HWLAB_GITOPS_TARGET", value: "$(params.gitops-target)" }, + { name: "HWLAB_ARGO_APPLICATION", value: "$(params.argo-application)" } + ], + script: runtimeReadyScript() + }] + }, + params: [ + { name: "revision", value: "$(params.revision)" }, + { name: "runtime-namespace", value: namespaceNameForProfile(profile) }, + { name: "gitops-target", value: gitopsTargetForProfile(profile) }, + { name: "argo-application", value: argoApplicationName(profile) }, + { name: "timeout-ms", value: "$(params.runtime-ready-timeout-ms)" } + ] + }; +} + +function imagePublishTaskSet(args = { lane: "node" }, deploy = null) { + const serviceIds = serviceIdsForLane(args.lane, deploy); + const buildTasks = serviceIds.map((serviceId) => perServiceBuildTask(serviceId, { + runAfter: ["plan-artifacts"] + })); + return [ + planArtifactsTask({ serviceIds }), + ...buildTasks, + collectArtifactsTask({ serviceIds }) + ]; +} + +function tektonPipeline(args = { lane: "node" }, deploy = null) { + const settings = ciLaneSettings(args); + const runtimePath = gitopsPathForProfile(args, settings.profile); + const preFlushRuntimeReadyTasks = isRuntimeLane(settings.lane) + ? [] + : [{ + ...runtimeReadyTask({ profile: settings.profile }), + when: [{ input: "$(tasks.gitops-promote.results.runtime-ready-required)", operator: "in", values: ["true"] }] + }]; + return { + apiVersion: "tekton.dev/v1", + kind: "Pipeline", + metadata: { + name: settings.pipelineName, + namespace: "hwlab-ci", + labels: { + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/gitops-target": settings.gitopsTarget + }, + annotations: { + "hwlab.pikastech.local/source-config": "scripts/gitops-render.mjs#tekton-native-primitive-ci", + "hwlab.pikastech.local/ci-contract": "tekton-native-primitive-tasks", + "hwlab.pikastech.local/policy": "native-per-service-taskrun-image-publish" + } + }, + spec: { + params: [ + { name: "git-url", type: "string", default: defaultSourceRepo }, + { name: "git-read-url", type: "string", default: args.gitReadUrl }, + { name: "git-write-url", type: "string", default: args.gitWriteUrl }, + { name: "source-branch", type: "string", default: args.sourceBranch }, + { name: "gitops-branch", type: "string", default: args.gitopsBranch }, + { name: "lane", type: "string", default: settings.lane }, + { name: "catalog-path", type: "string", default: args.catalogPath || settings.catalogPath }, + { name: "image-tag-mode", type: "string", default: args.imageTagMode || settings.imageTagMode }, + { name: "runtime-path", type: "string", default: runtimePath }, + { name: "revision", type: "string", description: "Full git commit SHA; the only source truth for image tags." }, + { name: "registry-prefix", type: "string", default: defaultRegistryPrefix }, + { name: "services", type: "string", default: servicesParamForLane(args.lane, deploy) }, + { name: "base-image", type: "string", default: defaultDevBaseImage }, + { name: "build-cache-mode", type: "string", default: "registry" }, + { name: "runtime-ready-timeout-ms", type: "string", default: "60000" } + ], + workspaces: [ + { name: "source" }, + { name: "git-ssh" } + ], + tasks: [ + { + name: "prepare-source", + workspaces: [ + { name: "source", workspace: "source" }, + { name: "git-ssh", workspace: "git-ssh" } + ], + taskSpec: { + params: [ + { name: "git-url" }, + { name: "git-read-url" }, + { name: "source-branch" }, + { name: "gitops-branch" }, + { name: "lane" }, + { name: "catalog-path" }, + { name: "image-tag-mode" }, + { name: "registry-prefix" }, + { name: "services" }, + { name: "revision" } + ], + workspaces: [{ name: "source" }, { name: "git-ssh" }], + steps: [{ name: "prepare-source", image: ciToolsRunnerImage, env: proxyEnv(), script: prepareSourceScript() }] + }, + params: [ + { name: "git-url", value: "$(params.git-url)" }, + { name: "git-read-url", value: "$(params.git-read-url)" }, + { name: "source-branch", value: "$(params.source-branch)" }, + { name: "gitops-branch", value: "$(params.gitops-branch)" }, + { name: "lane", value: "$(params.lane)" }, + { name: "catalog-path", value: "$(params.catalog-path)" }, + { name: "image-tag-mode", value: "$(params.image-tag-mode)" }, + { name: "registry-prefix", value: "$(params.registry-prefix)" }, + { name: "services", value: "$(params.services)" }, + { name: "revision", value: "$(params.revision)" } + ] + }, + ...primitiveValidationTasks.map(primitiveValidationTask), + ...imagePublishTaskSet(args, deploy), + { + name: "gitops-promote", + runAfter: ["collect-artifacts"], + workspaces: [ + { name: "source", workspace: "source" }, + { name: "git-ssh", workspace: "git-ssh" } + ], + taskSpec: { + params: [ + { name: "git-url" }, + { name: "git-read-url" }, + { name: "git-write-url" }, + { name: "source-branch" }, + { name: "gitops-branch" }, + { name: "lane" }, + { name: "catalog-path" }, + { name: "image-tag-mode" }, + { name: "runtime-path" }, + { name: "revision" }, + { name: "registry-prefix" } + ], + results: [{ name: "runtime-ready-required", description: "true when GitOps promotion changed runtime desired state and runtime readiness must be observed" }], + workspaces: [{ name: "source" }, { name: "git-ssh" }], + steps: [{ name: "promote", image: ciToolsRunnerImage, env: proxyEnv(), script: gitopsPromoteScript() }] + }, + params: [ + { name: "git-url", value: "$(params.git-url)" }, + { name: "git-read-url", value: "$(params.git-read-url)" }, + { name: "git-write-url", value: "$(params.git-write-url)" }, + { name: "source-branch", value: "$(params.source-branch)" }, + { name: "gitops-branch", value: "$(params.gitops-branch)" }, + { name: "lane", value: "$(params.lane)" }, + { name: "catalog-path", value: "$(params.catalog-path)" }, + { name: "image-tag-mode", value: "$(params.image-tag-mode)" }, + { name: "runtime-path", value: "$(params.runtime-path)" }, + { name: "revision", value: "$(params.revision)" }, + { name: "registry-prefix", value: "$(params.registry-prefix)" } + ] + }, + ...preFlushRuntimeReadyTasks + ] + } + }; +} + +function tektonPipelineRunTemplate({ source, args }) { + const settings = ciLaneSettings(args); + const runtimePath = gitopsPathForProfile(args, settings.profile); + return { + apiVersion: "tekton.dev/v1", + kind: "PipelineRun", + metadata: { + generateName: `${settings.pipelineRunPrefix}-manual-`, + namespace: "hwlab-ci", + labels: { + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/gitops-target": settings.gitopsTarget, + "hwlab.pikastech.local/source-commit": source.full + } + }, + spec: { + pipelineRef: { name: settings.pipelineName }, + taskRunTemplate: { + serviceAccountName: settings.serviceAccountName, + podTemplate: { + hostNetwork: true, + dnsPolicy: "ClusterFirstWithHostNet", + securityContext: { fsGroup: 1000 } + } + }, + params: [ + { name: "git-url", value: args.sourceRepo }, + { name: "git-read-url", value: args.gitReadUrl }, + { name: "git-write-url", value: args.gitWriteUrl }, + { name: "source-branch", value: args.sourceBranch }, + { name: "gitops-branch", value: args.gitopsBranch }, + { name: "lane", value: settings.lane }, + { name: "catalog-path", value: args.catalogPath || settings.catalogPath }, + { name: "image-tag-mode", value: args.imageTagMode || settings.imageTagMode }, + { name: "runtime-path", value: runtimePath }, + { name: "revision", value: source.full }, + { name: "registry-prefix", value: args.registryPrefix }, + { name: "base-image", value: defaultDevBaseImage }, + { name: "build-cache-mode", value: "registry" } + ], + workspaces: [ + { name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } }, + { name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } } + ] + } + }; +} + +function tektonPollerCronJob(args, deploy) { + const settings = ciLaneSettings(args); + if (settings.lane === "v02") throw new Error("v02 CI/CD is manually triggered by UniDesk CLI and must not render a branch poller CronJob"); + return { + apiVersion: "batch/v1", + kind: "CronJob", + metadata: { + name: settings.pollerName, + namespace: "hwlab-ci", + labels: { + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/gitops-target": settings.gitopsTarget + }, + annotations: { + "hwlab.pikastech.local/source-branch": args.sourceBranch, + "hwlab.pikastech.local/gitops-branch": args.gitopsBranch + } + }, + spec: { + schedule: settings.lane === "v02" ? "*/10 * * * *" : "* * * * *", + concurrencyPolicy: "Forbid", + successfulJobsHistoryLimit: 3, + failedJobsHistoryLimit: 3, + jobTemplate: { + spec: { + backoffLimit: 1, + template: { + metadata: { + labels: { + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/gitops-target": settings.gitopsTarget + } + }, + spec: { + serviceAccountName: settings.serviceAccountName, + restartPolicy: "Never", + hostNetwork: true, + dnsPolicy: "ClusterFirstWithHostNet", + containers: [{ + name: "poll", + image: ciToolsRunnerImage, + imagePullPolicy: "IfNotPresent", + env: [ + ...proxyEnv(), + { name: "POD_NAMESPACE", valueFrom: { fieldRef: { fieldPath: "metadata.namespace" } } }, + { name: "PIPELINE_NAME", value: settings.pipelineName }, + { name: "PIPELINERUN_PREFIX", value: settings.pipelineRunPrefix }, + { name: "SERVICE_ACCOUNT_NAME", value: settings.serviceAccountName }, + { name: "GITOPS_TARGET", value: settings.gitopsTarget }, + { name: "LANE", value: settings.lane }, + { name: "CATALOG_PATH", value: args.catalogPath || settings.catalogPath }, + { name: "IMAGE_TAG_MODE", value: args.imageTagMode || settings.imageTagMode }, + { name: "RUNTIME_PATH", value: gitopsPathForProfile(args, settings.profile) }, + { name: "GIT_URL", value: args.sourceRepo }, + { name: "GIT_READ_URL", value: args.gitReadUrl }, + { name: "SOURCE_BRANCH", value: args.sourceBranch }, + { name: "GITOPS_BRANCH", value: args.gitopsBranch }, + { name: "REGISTRY_PREFIX", value: args.registryPrefix }, + { name: "SERVICES", value: servicesParamForLane(args.lane, deploy) }, + { name: "BASE_IMAGE", value: defaultDevBaseImage } + ], + volumeMounts: [{ name: "git-ssh", mountPath: "/workspace/git-ssh", readOnly: true }], + command: ["/bin/sh", "-c"], + args: [pollerScript()] + }], + volumes: [{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }] + } + } + } + } + } + }; +} + +function tektonControlPlaneReconcilerCronJob(args) { + const settings = ciLaneSettings(args); + if (settings.lane === "v02") throw new Error("v02 CI/CD is manually triggered by UniDesk CLI and must not render a control-plane reconciler CronJob"); + return { + apiVersion: "batch/v1", + kind: "CronJob", + metadata: { + name: settings.controlPlaneReconcilerName, + namespace: "hwlab-ci", + labels: { + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/gitops-target": settings.gitopsTarget + }, + annotations: { + "hwlab.pikastech.local/source-branch": args.sourceBranch, + "hwlab.pikastech.local/gitops-branch": args.gitopsBranch, + "hwlab.pikastech.local/purpose": "render-and-apply-node-ci-control-plane" + } + }, + spec: { + schedule: "*/10 * * * *", + concurrencyPolicy: "Forbid", + successfulJobsHistoryLimit: 3, + failedJobsHistoryLimit: 3, + jobTemplate: { + spec: { + backoffLimit: 1, + template: { + metadata: { + labels: { + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/gitops-target": settings.gitopsTarget + } + }, + spec: { + serviceAccountName: settings.serviceAccountName, + restartPolicy: "Never", + hostNetwork: true, + dnsPolicy: "ClusterFirstWithHostNet", + containers: [{ + name: "reconcile", + image: ciToolsRunnerImage, + imagePullPolicy: "IfNotPresent", + env: [ + ...proxyEnv(), + { name: "POD_NAMESPACE", valueFrom: { fieldRef: { fieldPath: "metadata.namespace" } } }, + { name: "LANE", value: settings.lane }, + { name: "CATALOG_PATH", value: args.catalogPath || settings.catalogPath }, + { name: "IMAGE_TAG_MODE", value: args.imageTagMode || settings.imageTagMode }, + { name: "TEKTON_DIR", value: settings.tektonDir }, + { name: "ARGO_FILES", value: settings.lane === "v02" ? `${gitopsPathFor(args, "argocd/project.yaml")},${gitopsPathFor(args, "argocd/application-v02.yaml")}` : "" }, + { name: "FIELD_MANAGER", value: settings.controlPlaneReconcilerName }, + { name: "RECONCILE_EVENT", value: `${settings.gitopsTarget}-control-plane-reconciled` }, + { name: "GIT_URL", value: args.sourceRepo }, + { name: "GIT_READ_URL", value: args.gitReadUrl }, + { name: "SOURCE_BRANCH", value: args.sourceBranch }, + { name: "GITOPS_BRANCH", value: args.gitopsBranch }, + { name: "REGISTRY_PREFIX", value: args.registryPrefix } + ], + volumeMounts: [{ name: "git-ssh", mountPath: "/workspace/git-ssh", readOnly: true }], + command: ["/bin/sh", "-c"], + args: [controlPlaneReconcileScript()] + }], + volumes: [{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }] + } + } + } + } + } + }; +} + +export { + affectedResultName, + buildResultName, + buildTaskName, + buildkitBinVolume, + buildkitRunVolume, + buildkitSidecar, + ciLaneSettings, + collectArtifactsTask, + imagePublishTaskSet, + perServiceBuildTask, + planArtifactsTask, + prepareBuildkitClientStep, + runtimeReadyScript, + runtimeReadyTask, + serviceResultEnv, + serviceResultEnvName, + serviceResultParamName, + serviceResultParams, + serviceResultParamValues, + serviceWorkVolume, + tektonControlPlaneReconcilerCronJob, + tektonPipeline, + tektonPipelineRunTemplate, + tektonPollerCronJob, + tektonRbac +}; diff --git a/scripts/src/gitops-render/tekton-scripts.mjs b/scripts/src/gitops-render/tekton-scripts.mjs new file mode 100644 index 00000000..1f207e33 --- /dev/null +++ b/scripts/src/gitops-render/tekton-scripts.mjs @@ -0,0 +1,187 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + ciToolsRunnerImage, + defaultServiceIds, + defaultV02RuntimeEndpoint, + primitiveValidationTasks, + proxyEnv +} from "./core.mjs"; + +const templatesDir = path.join(path.dirname(fileURLToPath(import.meta.url)), "templates"); +const templateCache = new Map(); + +function readTemplate(name) { + if (!templateCache.has(name)) { + templateCache.set(name, readFileSync(path.join(templatesDir, name), "utf8")); + } + return templateCache.get(name); +} + +function renderTemplate(name, replacements = {}) { + let rendered = readTemplate(name); + for (const [token, value] of Object.entries(replacements)) { + if (!rendered.includes(token)) throw new Error(`template ${name} is missing token ${token}`); + rendered = rendered.replaceAll(token, String(value)); + } + const unresolved = [...rendered.matchAll(/__HWLAB_[A-Z0-9_]+__/gu)].map((match) => match[0]); + if (unresolved.length > 0) { + throw new Error(`template ${name} has unresolved tokens: ${[...new Set(unresolved)].join(", ")}`); + } + return rendered; +} + +function shellSingleQuote(value) { + return `'${String(value).replace(/'/g, `'"'"'`)}'`; +} + +function ciTimingShellFunction() { + return readTemplate("ci-timing-functions.sh"); +} + +function dependencyProxyProbeScript(phase, targetUrl) { + if (!/^[a-z0-9-]+$/u.test(phase)) throw new Error(`invalid dependency proxy probe phase: ${phase}`); + return renderTemplate("dependency-proxy-probe.sh", { + "__HWLAB_PROXY_PHASE_SHELL__": shellSingleQuote(phase), + "__HWLAB_PROXY_TARGET_SHELL__": shellSingleQuote(targetUrl), + "__HWLAB_PROXY_PHASE__": phase + }); +} + +function curlDownloadProbeFunction() { + return readTemplate("curl-download-probe.sh"); +} + +function gitSshShellFunction() { + return renderTemplate("git-ssh-functions.sh", { + "// __HWLAB_GITHUB_PROXY_CONNECT_MJS__": readTemplate("github-proxy-connect.mjs") + }); +} + +function prepareSourceScript() { + return renderTemplate("prepare-source.sh", { + "# __HWLAB_CI_TIMING_SHELL__\n": ciTimingShellFunction(), + "# __HWLAB_GIT_SSH_SHELL__\n": gitSshShellFunction(), + "__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage, + "__HWLAB_V02_RUNTIME_ENDPOINT__": JSON.stringify(defaultV02RuntimeEndpoint).slice(1, -1), + "__HWLAB_VALIDATION_TASK_COUNT__": primitiveValidationTasks.length + }); +} + +function sourceValidationScript(commands) { + return [ + "#!/bin/sh", + "set -eu", + "git config --global --add safe.directory /workspace/source/repo", + "cd /workspace/source/repo", + "test \"$(git rev-parse HEAD)\" = \"$(params.revision)\"", + ...commands + ].join("\n") + "\n"; +} + +function primitiveValidationTaskNames() { + return primitiveValidationTasks.map((task) => task.name); +} + +function primitiveValidationTask(task) { + return { + name: task.name, + runAfter: ["prepare-source"], + workspaces: [{ name: "source", workspace: "source" }], + taskSpec: { + params: [ + { name: "revision" }, + { name: "lane" }, + { name: "catalog-path" }, + { name: "image-tag-mode" }, + { name: "source-branch" }, + { name: "gitops-branch" } + ], + workspaces: [{ name: "source" }], + steps: [{ name: task.name, image: ciToolsRunnerImage, env: proxyEnv(), script: sourceValidationScript(task.commands) }] + }, + params: [ + { name: "revision", value: "$(params.revision)" }, + { name: "lane", value: "$(params.lane)" }, + { name: "catalog-path", value: "$(params.catalog-path)" }, + { name: "image-tag-mode", value: "$(params.image-tag-mode)" }, + { name: "source-branch", value: "$(params.source-branch)" }, + { name: "gitops-branch", value: "$(params.gitops-branch)" } + ] + }; +} + +function planArtifactsScript() { + return renderTemplate("plan-artifacts.sh", { + "__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage, + '["__HWLAB_DEFAULT_SERVICE_IDS__"]': JSON.stringify(defaultServiceIds) + }); +} + +function perServicePublishScript() { + return renderTemplate("per-service-publish.sh", { + "# __HWLAB_CI_TIMING_SHELL__\n": ciTimingShellFunction(), + "# __HWLAB_DEPENDENCY_PROXY_PROBE__": dependencyProxyProbeScript( + "service-image-publish-proxy-preflight", + "http://deb.debian.org/debian/dists/bookworm/InRelease" + ), + "__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage + }); +} + +function collectArtifactsScript() { + return renderTemplate("collect-artifacts.sh", { + "__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage + }); +} + +function gitopsPromoteScript() { + return renderTemplate("gitops-promote.sh", { + "# __HWLAB_CI_TIMING_SHELL__\n": ciTimingShellFunction(), + "# __HWLAB_GIT_SSH_SHELL__\n": gitSshShellFunction(), + "__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage + }); +} + +function controlPlaneReconcileScript() { + return renderTemplate("control-plane-reconcile.sh", { + "# __HWLAB_DEPENDENCY_PROXY_PROBE__": dependencyProxyProbeScript( + "control-plane-proxy-preflight", + "http://deb.debian.org/debian/dists/bookworm/InRelease" + ), + "# __HWLAB_GIT_SSH_SHELL__\n": gitSshShellFunction(), + "__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage + }); +} + +function pollerScript() { + return renderTemplate("poller.sh", { + "# __HWLAB_DEPENDENCY_PROXY_PROBE__": dependencyProxyProbeScript( + "poller-proxy-preflight", + "http://deb.debian.org/debian/dists/bookworm/InRelease" + ), + "# __HWLAB_GIT_SSH_SHELL__\n": gitSshShellFunction(), + "__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage + }); +} + +export { + ciTimingShellFunction, + collectArtifactsScript, + controlPlaneReconcileScript, + curlDownloadProbeFunction, + dependencyProxyProbeScript, + gitopsPromoteScript, + gitSshShellFunction, + perServicePublishScript, + planArtifactsScript, + pollerScript, + prepareSourceScript, + primitiveValidationTask, + primitiveValidationTaskNames, + renderTemplate, + shellSingleQuote, + sourceValidationScript +}; diff --git a/scripts/src/gitops-render/templates/ci-timing-functions.sh b/scripts/src/gitops-render/templates/ci-timing-functions.sh new file mode 100644 index 00000000..62a0cd2d --- /dev/null +++ b/scripts/src/gitops-render/templates/ci-timing-functions.sh @@ -0,0 +1,13 @@ +ci_now_ms() { + node -e 'console.log(Date.now())' +} + +ci_timing_emit() { + stage="$1" + status="$2" + started_ms="$3" + finished_ms="$(ci_now_ms)" + duration_ms="$((finished_ms - started_ms))" + service_id="${HWLAB_TIMING_SERVICE_ID:-}" + node -e 'const [stage,status,durationMs,serviceId]=process.argv.slice(1); const payload={event:"node-cicd-timing",schemaVersion:"v1",stage,status,durationMs:Number(durationMs),pipelineRun:process.env.HWLAB_TEKTON_PIPELINERUN||null,taskRun:process.env.HWLAB_TEKTON_TASKRUN||null,task:process.env.HWLAB_TEKTON_TASK||null,revision:process.env.HWLAB_SOURCE_REVISION||null,serviceId:serviceId||null,source:"scripts/gitops-render.mjs",at:new Date().toISOString()}; console.log(JSON.stringify(payload));' "$stage" "$status" "$duration_ms" "$service_id" +} diff --git a/scripts/src/gitops-render/templates/collect-artifacts.sh b/scripts/src/gitops-render/templates/collect-artifacts.sh new file mode 100644 index 00000000..748b5280 --- /dev/null +++ b/scripts/src/gitops-render/templates/collect-artifacts.sh @@ -0,0 +1,30 @@ +#!/bin/sh +set -eu +echo '{"event":"ci-base-image","phase":"collect-artifacts","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apk"}' +for tool in node git; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"collect-artifacts","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done +cd /workspace/source/repo +test "$(git rev-parse HEAD)" = "$(params.revision)" +export HWLAB_CI_ARTIFACT_RUN_ID="$(context.pipelineRun.name)-collect" +export HWLAB_CI_ARTIFACT_RUN_OWNER="tekton" +export HWLAB_DEV_REGISTRY_PREFIX="$(params.registry-prefix)" +export HWLAB_DEV_REGISTRY_K8S_NATIVE=1 +if [ -n "$(params.base-image)" ]; then export HWLAB_DEV_BASE_IMAGE="$(params.base-image)"; fi +export HWLAB_ARTIFACT_LANE="$(params.lane)" +export HWLAB_ARTIFACT_CATALOG_PATH="$(params.catalog-path)" +export HWLAB_DEPLOY_MANIFEST_PATH="deploy/deploy.yaml" +export HWLAB_ARTIFACT_IMAGE_TAG_MODE="$(params.image-tag-mode)" +if [ -s /workspace/source/affected-services.json ] && node - /workspace/source/affected-services.json <<'NODE' +const fs = require("node:fs"); +const plan = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); +const buildServices = Array.isArray(plan.buildServices) ? plan.buildServices : []; +const affectedServices = Array.isArray(plan.affectedServices) ? plan.affectedServices : []; +const willRunGitopsPromote = plan.ciCdPlan && plan.ciCdPlan.willRunGitopsPromote === true; +process.exit(!willRunGitopsPromote && buildServices.length === 0 && affectedServices.length === 0 ? 0 : 1); +NODE +then + rm -f /workspace/source/dev-artifacts.json + echo '{"event":"collect-artifacts","status":"skipped","reason":"no-build-no-rollout-plan"}' + exit 0 +fi +HWLAB_CI_PLAN_VERIFY_REUSE_REGISTRY=1 node scripts/artifact-publish.mjs --publish --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --registry-prefix "$(params.registry-prefix)" --report /workspace/source/dev-artifacts.json --quiet-build --concurrency 1 --services "$(params.services)" --external-service-report-dir /workspace/source/service-results --ci-plan-path /workspace/source/affected-services.json +node scripts/refresh-artifact-catalog.mjs --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --target-ref HEAD --publish-report /workspace/source/dev-artifacts.json --no-write diff --git a/scripts/src/gitops-render/templates/control-plane-reconcile.sh b/scripts/src/gitops-render/templates/control-plane-reconcile.sh new file mode 100644 index 00000000..1e404cf8 --- /dev/null +++ b/scripts/src/gitops-render/templates/control-plane-reconcile.sh @@ -0,0 +1,139 @@ +#!/bin/sh +set -eu +# __HWLAB_DEPENDENCY_PROXY_PROBE__ +echo '{"event":"ci-base-image","phase":"control-plane-reconciler","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apk"}' +for tool in node npm git python3 ssh curl timeout; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"control-plane-reconciler","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done +# __HWLAB_GIT_SSH_SHELL__ + +git_ssh_setup +workdir="$(mktemp -d)" +git_timed control-plane-clone 180 git clone --depth 1 --branch "$SOURCE_BRANCH" "$GIT_READ_URL" "$workdir/repo" +cd "$workdir/repo" +git remote set-url origin "$GIT_READ_URL" +revision="$(git rev-parse HEAD)" +: "${LANE:?LANE is required}" +: "${CATALOG_PATH:?CATALOG_PATH is required}" +: "${IMAGE_TAG_MODE:?IMAGE_TAG_MODE is required}" +: "${RUNTIME_PATH:?RUNTIME_PATH is required}" +gitops_root="${RUNTIME_PATH%/*}" +if [ "$gitops_root" = "$RUNTIME_PATH" ]; then gitops_root="."; fi +node scripts/run-bun.mjs scripts/gitops-render.mjs --lane "$LANE" --catalog-path "$CATALOG_PATH" --image-tag-mode "$IMAGE_TAG_MODE" --source-revision "$revision" --source-repo "$GIT_URL" --source-branch "$SOURCE_BRANCH" --gitops-branch "$GITOPS_BRANCH" --gitops-root "$gitops_root" --out "$gitops_root" --registry-prefix "$REGISTRY_PREFIX" +node scripts/run-bun.mjs scripts/gitops-render.mjs --lane "$LANE" --catalog-path "$CATALOG_PATH" --image-tag-mode "$IMAGE_TAG_MODE" --source-revision "$revision" --source-repo "$GIT_URL" --source-branch "$SOURCE_BRANCH" --gitops-branch "$GITOPS_BRANCH" --gitops-root "$gitops_root" --out "$gitops_root" --registry-prefix "$REGISTRY_PREFIX" --check +node <<'NODE' +const fs = require("node:fs"); +const https = require("node:https"); + +function requiredEnv(name) { + const value = process.env[name]; + if (!value) throw new Error(name + " is required"); + return value; +} + +const namespace = requiredEnv("POD_NAMESPACE"); +const host = process.env.KUBERNETES_SERVICE_HOST; +const port = process.env.KUBERNETES_SERVICE_PORT || "443"; +if (!host) throw new Error("KUBERNETES_SERVICE_HOST is not available"); + +const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8"); +const ca = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); +const tektonDir = requiredEnv("TEKTON_DIR"); +const argoFiles = String(process.env.ARGO_FILES || "").split(",").map((item) => item.trim()).filter(Boolean); +const fieldManager = requiredEnv("FIELD_MANAGER"); +const eventName = requiredEnv("RECONCILE_EVENT"); +const manifestFiles = [ + "deploy/gitops/node/" + tektonDir + "/rbac.yaml", + "deploy/gitops/node/" + tektonDir + "/pipeline.yaml", + ...(process.env.LANE === "v02" ? [] : [ + "deploy/gitops/node/" + tektonDir + "/poller.yaml", + "deploy/gitops/node/" + tektonDir + "/control-plane-reconciler.yaml" + ]), + ...argoFiles +]; +const plurals = new Map([ + ["ServiceAccount", "serviceaccounts"], + ["Role", "roles"], + ["RoleBinding", "rolebindings"], + ["Pipeline", "pipelines"], + ["CronJob", "cronjobs"], + ["Application", "applications"], + ["AppProject", "appprojects"] +]); + +function encode(value) { + return encodeURIComponent(String(value)); +} + +function itemsFrom(file) { + const parsed = JSON.parse(fs.readFileSync(file, "utf8")); + return parsed.kind === "List" ? parsed.items || [] : [parsed]; +} + +function apiPath(item) { + if (item.kind === "Namespace") return null; + const name = item.metadata?.name; + if (!name) throw new Error("manifest item is missing metadata.name"); + const ns = item.metadata?.namespace || namespace; + const plural = plurals.get(item.kind); + if (!plural) throw new Error("unsupported reconciler manifest kind " + item.kind); + if (item.apiVersion === "v1") return "/api/v1/namespaces/" + encode(ns) + "/" + plural + "/" + encode(name); + const parts = String(item.apiVersion || "").split("/"); + if (parts.length !== 2) throw new Error("unsupported apiVersion " + item.apiVersion + " for " + item.kind); + return "/apis/" + encode(parts[0]) + "/" + encode(parts[1]) + "/namespaces/" + encode(ns) + "/" + plural + "/" + encode(name); +} + +function reconcilePolicy(item) { + const ns = item.metadata?.namespace || namespace; + const crossNamespaceRbac = (item.kind === "Role" || item.kind === "RoleBinding") && ns !== namespace; + if (crossNamespaceRbac && process.env.HWLAB_RECONCILE_CROSS_NAMESPACE_RBAC !== "1") { + return { action: "skip", reason: "cross-namespace-rbac-bootstrap-managed", namespace: ns }; + } + return { action: "apply", namespace: ns }; +} + +function request(method, path, body) { + const payload = body ? JSON.stringify(body) : ""; + const headers = { Authorization: "Bearer " + token }; + if (payload) { + headers["Content-Type"] = "application/apply-patch+yaml"; + headers["Content-Length"] = Buffer.byteLength(payload); + } + return new Promise((resolve, reject) => { + const req = https.request({ host, port, method, path, ca, headers }, (res) => { + let data = ""; + res.setEncoding("utf8"); + res.on("data", (chunk) => { data += chunk; }); + res.on("end", () => resolve({ statusCode: res.statusCode || 0, body: data })); + }); + req.on("error", reject); + if (payload) req.write(payload); + req.end(); + }); +} + +(async () => { + const results = []; + for (const file of manifestFiles) { + for (const item of itemsFrom(file)) { + const policy = reconcilePolicy(item); + if (policy.action === "skip") { + results.push({ file, kind: item.kind, name: item.metadata?.name || null, namespace: policy.namespace, status: "skipped", reason: policy.reason }); + continue; + } + const path = apiPath(item); + if (!path) { + results.push({ file, kind: item.kind, name: item.metadata?.name || null, status: "skipped-cluster-scoped" }); + continue; + } + const response = await request("PATCH", path + "?fieldManager=" + encode(fieldManager) + "&force=true", item); + if (response.statusCode < 200 || response.statusCode > 299) { + throw new Error("apply failed for " + item.kind + "/" + item.metadata.name + " status=" + response.statusCode + " body=" + response.body.slice(0, 1000)); + } + results.push({ file, kind: item.kind, name: item.metadata.name, status: "applied", statusCode: response.statusCode }); + } + } + console.log(JSON.stringify({ event: eventName, sourceRevision: process.env.RECONCILE_REVISION || null, results }, null, 2)); +})().catch((error) => { + console.error(error.stack || error.message); + process.exitCode = 1; +}); +NODE diff --git a/scripts/src/gitops-render/templates/curl-download-probe.sh b/scripts/src/gitops-render/templates/curl-download-probe.sh new file mode 100644 index 00000000..a3a44cc2 --- /dev/null +++ b/scripts/src/gitops-render/templates/curl-download-probe.sh @@ -0,0 +1,17 @@ +redact_proxy_value() { + printf '%s' "${1:-}" | sed -E 's#(https?://)[^/@]+@#\1***@#; s#(socks5h?://)[^/@]+@#\1***@#' +} + +curl_dependency_probe() { + phase="$1" + url="$2" + proxy="${HTTPS_PROXY:-${https_proxy:-${HTTP_PROXY:-${http_proxy:-}}}}" + if [ -z "$proxy" ]; then + echo '{"event":"dependency-curl-probe","phase":"'"$phase"'","ok":false,"reason":"proxy-env-missing"}' + return 21 + fi + safe_proxy="$(redact_proxy_value "$proxy")" + echo '{"event":"dependency-curl-probe-start","phase":"'"$phase"'","proxy":"'"$safe_proxy"'","target":"'"$url"'"}' + curl -sS -L --range 0-1048575 -o /dev/null --connect-timeout 15 --max-time 60 --proxy "$proxy" -w '{"event":"dependency-curl-probe","phase":"'"$phase"'","http_code":"%{http_code}","time_namelookup":%{time_namelookup},"time_connect":%{time_connect},"time_starttransfer":%{time_starttransfer},"time_total":%{time_total},"speed_download":%{speed_download},"size_download":%{size_download},"remote_ip":"%{remote_ip}"} +' "$url" +} \ No newline at end of file diff --git a/scripts/src/gitops-render/templates/dependency-proxy-probe.sh b/scripts/src/gitops-render/templates/dependency-proxy-probe.sh new file mode 100644 index 00000000..7b2063ee --- /dev/null +++ b/scripts/src/gitops-render/templates/dependency-proxy-probe.sh @@ -0,0 +1,96 @@ +HWLAB_PROXY_PROBE_PHASE=__HWLAB_PROXY_PHASE_SHELL__ HWLAB_PROXY_PROBE_URL=__HWLAB_PROXY_TARGET_SHELL__ node <<'NODE' || echo '{"event":"dependency-proxy-probe-nonblocking","phase":"__HWLAB_PROXY_PHASE__","ok":false,"reason":"probe-failed-but-continuing"}' +const net = require("node:net"); + +const phase = process.env.HWLAB_PROXY_PROBE_PHASE || "unknown"; +const target = new URL(process.env.HWLAB_PROXY_PROBE_URL || "http://deb.debian.org/debian/dists/bookworm/InRelease"); +const proxyRaw = process.env.HTTP_PROXY || process.env.http_proxy || ""; + +function redactProxy(value) { + if (!value) return ""; + try { + const parsed = new URL(value); + if (parsed.username || parsed.password) { + parsed.username = "***"; + parsed.password = ""; + } + return parsed.toString(); + } catch { + return ""; + } +} + +function emit(payload, exitCode = 0) { + console.log(JSON.stringify({ + event: "dependency-proxy-probe", + phase, + target: target.href, + proxy: redactProxy(proxyRaw), + ...payload + })); + if (exitCode) process.exit(exitCode); +} + +if (!proxyRaw) emit({ ok: false, reason: "proxy-env-missing" }, 21); + +let proxy; +try { + proxy = new URL(proxyRaw); +} catch (error) { + emit({ ok: false, reason: "proxy-url-invalid", error: error.message }, 22); +} + +if (proxy.protocol !== "http:" && proxy.protocol !== "https:") { + emit({ ok: false, reason: "http-proxy-required", protocol: proxy.protocol }, 23); +} + +const startedAt = Date.now(); +const socket = net.createConnection({ host: proxy.hostname, port: Number(proxy.port || (proxy.protocol === "https:" ? 443 : 80)) }); +let bytes = 0; +let firstByteMs = null; +let responseHead = ""; +let finished = false; + +const timeout = setTimeout(() => { + if (finished) return; + finished = true; + socket.destroy(); + emit({ ok: false, reason: "timeout", timeoutMs: 15000 }, 24); +}, 15000); + +socket.on("connect", () => { + socket.write("GET " + target.href + " HTTP/1.1\r\nHost: " + target.host + "\r\nUser-Agent: hwlab-node-proxy-probe\r\nConnection: close\r\n\r\n"); +}); + +socket.on("data", (chunk) => { + if (firstByteMs === null) firstByteMs = Date.now() - startedAt; + bytes += chunk.length; + if (responseHead.length < 240) responseHead += chunk.toString("utf8", 0, Math.min(chunk.length, 240 - responseHead.length)); +}); + +socket.on("end", () => { + if (finished) return; + finished = true; + clearTimeout(timeout); + const totalMs = Date.now() - startedAt; + const statusLine = responseHead.split("\r\n", 1)[0] || ""; + const statusCode = Number(statusLine.split(" ")[1] || 0); + const ok = statusLine.startsWith("HTTP/") && statusCode >= 200 && statusCode < 400; + emit({ + ok, + reason: ok ? null : "bad-http-status", + statusLine, + statusCode, + firstByteMs, + totalMs, + bytes, + speedBytesPerSecond: totalMs > 0 ? Math.round(bytes * 1000 / totalMs) : 0 + }, ok ? 0 : 25); +}); + +socket.on("error", (error) => { + if (finished) return; + finished = true; + clearTimeout(timeout); + emit({ ok: false, reason: "proxy-connect-failed", error: error.message, totalMs: Date.now() - startedAt }, 26); +}); +NODE \ No newline at end of file diff --git a/scripts/src/gitops-render/templates/git-mirror-flush.sh b/scripts/src/gitops-render/templates/git-mirror-flush.sh new file mode 100644 index 00000000..cf0931b2 --- /dev/null +++ b/scripts/src/gitops-render/templates/git-mirror-flush.sh @@ -0,0 +1,76 @@ +#!/bin/sh +set -eu +repo_url="ssh://git@ssh.github.com:443/pikasTech/HWLAB.git" +repo_path="/cache/pikasTech/HWLAB.git" +lock_dir="/cache/.hwlab-flush.lock" +started_ms=$(node -e 'console.log(Date.now())') +now_ms() { node -e 'console.log(Date.now())'; } +emit_timing() { + phase="$1" + status="$2" + started="$3" + finished=$(now_ms) + duration=$((finished - started)) + printf '{"event":"git-mirror-flush-timing","phase":"%s","status":"%s","durationMs":%s,"repository":"pikasTech/HWLAB"} +' "$phase" "$status" "$duration" +} +if ! mkdir "$lock_dir" 2>/dev/null; then + echo "git mirror flush already running" + exit 0 +fi +trap 'rmdir "$lock_dir" 2>/dev/null || true' EXIT INT TERM +mkdir -p /root/.ssh +cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa +chmod 0400 /root/.ssh/id_rsa +# __HWLAB_GIT_MIRROR_SSH_PRELUDE__ +/script/install-hooks.sh +for gitops_branch in __HWLAB_GITOPS_BRANCH_ARGS__; do + local_head=$(git -C "$repo_path" show-ref --verify --hash "refs/heads/$gitops_branch" 2>/dev/null || true) + if [ -z "$local_head" ]; then + echo "git mirror flush skips missing local refs/heads/$gitops_branch" + continue + fi + fetch_started=$(now_ms) + timeout 90 git -C "$repo_path" fetch --quiet "$repo_url" "refs/heads/$gitops_branch:refs/mirror-stage/heads/$gitops_branch" + emit_timing "fetch-$gitops_branch" succeeded "$fetch_started" + github_head=$(git -C "$repo_path" show-ref --verify --hash "refs/mirror-stage/heads/$gitops_branch" 2>/dev/null || true) + if [ -n "$github_head" ] && [ "$github_head" != "$local_head" ] && ! git -C "$repo_path" merge-base --is-ancestor "$github_head" "$local_head"; then + echo "git mirror flush rejected divergent $gitops_branch local=$local_head github=$github_head" >&2 + exit 52 + fi + if [ "$local_head" != "$github_head" ]; then + push_started=$(now_ms) + timeout 120 git -C "$repo_path" push "$repo_url" "$local_head:refs/heads/$gitops_branch" + emit_timing "push-$gitops_branch" succeeded "$push_started" + else + echo "git mirror flush $gitops_branch already matches GitHub" + fi + git -C "$repo_path" update-ref "refs/mirror-stage/heads/$gitops_branch" "$local_head" +done +git -C "$repo_path" update-server-info +flushed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +export flushed_at +export gitops_branches_json=__HWLAB_GITOPS_BRANCHES_JSON_SHELL__ +node - <<'NODE' | tee /cache/HWLAB.last-flush.json +const { execFileSync } = require("node:child_process"); +const repo = "/cache/pikasTech/HWLAB.git"; +const gitopsBranches = JSON.parse(process.env.gitops_branches_json || "[]"); +function rev(ref) { + try { return execFileSync("git", ["-C", repo, "rev-parse", ref], { encoding: "utf8" }).trim() || null; } catch { return null; } +} +const refs = {}; +for (const branch of gitopsBranches) { + refs["refs/heads/" + branch] = rev("refs/heads/" + branch); + refs["refs/mirror-stage/heads/" + branch] = rev("refs/mirror-stage/heads/" + branch); +} +const pendingFlush = gitopsBranches.some((branch) => refs["refs/heads/" + branch] && refs["refs/mirror-stage/heads/" + branch] && refs["refs/heads/" + branch] !== refs["refs/mirror-stage/heads/" + branch]); +console.log(JSON.stringify({ + event: "git-mirror-flush", + status: "flushed", + repository: "pikasTech/HWLAB", + flushedAt: process.env.flushed_at, + pendingFlush, + refs +})); +NODE +emit_timing total succeeded "$started_ms" diff --git a/scripts/src/gitops-render/templates/git-mirror-http.mjs b/scripts/src/gitops-render/templates/git-mirror-http.mjs new file mode 100644 index 00000000..a69446d9 --- /dev/null +++ b/scripts/src/gitops-render/templates/git-mirror-http.mjs @@ -0,0 +1,196 @@ +import { createServer } from "node:http"; +import { spawn } from "node:child_process"; +import { createReadStream, statSync } from "node:fs"; +import path from "node:path"; +import { createGunzip, createInflate } from "node:zlib"; + +const cacheRoot = process.env.GIT_MIRROR_CACHE_ROOT || "/cache"; +const port = Number(process.env.PORT || 8080); + +createServer((req, res) => { + handle(req, res).catch((error) => { + if (!res.headersSent) res.writeHead(500, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: false, error: error.message })); + }); +}).listen(port, "0.0.0.0", () => { + console.log(JSON.stringify({ event: "git-mirror-smart-http-listening", port, cacheRoot })); +}); + +async function handle(req, res) { + const url = new URL(req.url || "/", "http://git-mirror-http"); + const writeHost = isWriteHost(req.headers.host || ""); + if (req.method === "GET" && (url.pathname === "/" || url.pathname === "/health")) { + res.writeHead(200, { "content-type": "application/json", "cache-control": "no-cache" }); + res.end(JSON.stringify({ ok: true, service: "git-mirror-smart-http", writeHost })); + return; + } + if (req.method === "GET" && url.pathname.endsWith("/info/refs") && url.searchParams.get("service") === "git-upload-pack") { + await runUploadPackAdvertisement(repoPathFromGitServicePath(url.pathname, "/info/refs"), res); + return; + } + if (req.method === "POST" && url.pathname.endsWith("/git-upload-pack")) { + await runUploadPackRpc(repoPathFromGitServicePath(url.pathname, "/git-upload-pack"), req, res); + return; + } + if (req.method === "GET" && url.pathname.endsWith("/info/refs") && url.searchParams.get("service") === "git-receive-pack") { + if (!writeHost) return rejectReadOnly(res); + await runReceivePackAdvertisement(repoPathFromGitServicePath(url.pathname, "/info/refs"), res); + return; + } + if (req.method === "POST" && url.pathname.endsWith("/git-receive-pack")) { + if (!writeHost) return rejectReadOnly(res); + await runReceivePackRpc(repoPathFromGitServicePath(url.pathname, "/git-receive-pack"), req, res); + return; + } + if (req.method === "GET" || req.method === "HEAD") { + serveStaticFile(url.pathname, req, res); + return; + } + res.writeHead(405, { "content-type": "text/plain" }); + res.end("method not allowed\n"); +} + +function isWriteHost(hostHeader) { + const host = String(hostHeader || "").split(":", 1)[0].toLowerCase(); + return host === "git-mirror-write" || host.startsWith("git-mirror-write."); +} + +function rejectReadOnly(res) { + res.writeHead(403, { "content-type": "application/json", "cache-control": "no-cache" }); + res.end(JSON.stringify({ ok: false, error: "git mirror receive-pack is only available through git-mirror-write" })); +} + +function repoPathFromGitServicePath(urlPath, suffix) { + const repoUrlPath = urlPath.slice(0, -suffix.length); + if (!repoUrlPath.endsWith(".git")) throw new Error(`unsupported git repo path: ${urlPath}`); + return safePath(repoUrlPath); +} + +function safePath(urlPath) { + const decoded = decodeURIComponent(urlPath.replace(/^[/]+/u, "")); + const fullPath = path.resolve(cacheRoot, decoded); + const normalizedRoot = path.resolve(cacheRoot); + if (fullPath !== normalizedRoot && !fullPath.startsWith(`${normalizedRoot}${path.sep}`)) throw new Error("path escapes cache root"); + return fullPath; +} + +function smartHeaders(contentType) { + return { + "content-type": contentType, + "cache-control": "no-cache, max-age=0, must-revalidate", + expires: "Fri, 01 Jan 1980 00:00:00 GMT", + pragma: "no-cache" + }; +} + +function pktLine(text) { + const length = Buffer.byteLength(text) + 4; + return `${length.toString(16).padStart(4, "0")}${text}`; +} + +function requestBodyStream(req) { + const encoding = String(req.headers["content-encoding"] || "identity").toLowerCase(); + if (encoding === "identity" || encoding === "") return req; + if (encoding === "gzip" || encoding === "x-gzip") return req.pipe(createGunzip()); + if (encoding === "deflate") return req.pipe(createInflate()); + throw new Error(`unsupported git upload-pack content-encoding: ${encoding}`); +} + +function runUploadPackAdvertisement(repoPath, res) { + return new Promise((resolve, reject) => { + const child = spawn("git-upload-pack", ["--stateless-rpc", "--advertise-refs", repoPath], { stdio: ["ignore", "pipe", "pipe"] }); + let stderr = ""; + child.stderr.on("data", (chunk) => { stderr += chunk; process.stderr.write(chunk); }); + child.on("error", reject); + child.on("close", (code) => { + if (code !== 0 && !res.headersSent) reject(new Error(`git-upload-pack advertise failed: ${stderr.trim()}`)); + else resolve(); + }); + res.writeHead(200, smartHeaders("application/x-git-upload-pack-advertisement")); + res.write(pktLine("# service=git-upload-pack\n")); + res.write("0000"); + child.stdout.pipe(res, { end: false }); + child.stdout.on("end", () => res.end()); + }); +} + +function runUploadPackRpc(repoPath, req, res) { + return new Promise((resolve, reject) => { + const child = spawn("git-upload-pack", ["--stateless-rpc", repoPath], { stdio: ["pipe", "pipe", "pipe"] }); + let stderr = ""; + child.stderr.on("data", (chunk) => { stderr += chunk; process.stderr.write(chunk); }); + child.on("error", reject); + child.on("close", (code) => { + if (code !== 0 && !res.headersSent) reject(new Error(`git-upload-pack rpc failed: ${stderr.trim()}`)); + else resolve(); + }); + res.writeHead(200, smartHeaders("application/x-git-upload-pack-result")); + const body = requestBodyStream(req); + body.on("error", (error) => child.stdin.destroy(error)); + body.pipe(child.stdin); + child.stdout.pipe(res); + }); +} + +function runReceivePackAdvertisement(repoPath, res) { + return new Promise((resolve, reject) => { + const child = spawn("git-receive-pack", ["--stateless-rpc", "--advertise-refs", repoPath], { stdio: ["ignore", "pipe", "pipe"] }); + let stderr = ""; + child.stderr.on("data", (chunk) => { stderr += chunk; process.stderr.write(chunk); }); + child.on("error", reject); + child.on("close", (code) => { + if (code !== 0 && !res.headersSent) reject(new Error(`git-receive-pack advertise failed: ${stderr.trim()}`)); + else resolve(); + }); + res.writeHead(200, smartHeaders("application/x-git-receive-pack-advertisement")); + res.write(pktLine("# service=git-receive-pack\n")); + res.write("0000"); + child.stdout.pipe(res, { end: false }); + child.stdout.on("end", () => res.end()); + }); +} + +function runReceivePackRpc(repoPath, req, res) { + return new Promise((resolve, reject) => { + const child = spawn("git-receive-pack", ["--stateless-rpc", repoPath], { stdio: ["pipe", "pipe", "pipe"] }); + let stderr = ""; + child.stderr.on("data", (chunk) => { stderr += chunk; process.stderr.write(chunk); }); + child.on("error", reject); + child.on("close", (code) => { + if (code !== 0 && !res.headersSent) reject(new Error(`git-receive-pack rpc failed: ${stderr.trim()}`)); + else resolve(); + }); + res.writeHead(200, smartHeaders("application/x-git-receive-pack-result")); + const body = requestBodyStream(req); + body.on("error", (error) => child.stdin.destroy(error)); + body.pipe(child.stdin); + child.stdout.pipe(res); + }); +} + +function serveStaticFile(urlPath, req, res) { + const fullPath = safePath(urlPath); + let stat; + try { stat = statSync(fullPath); } catch { + res.writeHead(404, { "content-type": "text/plain" }); + res.end("not found\n"); + return; + } + if (!stat.isFile()) { + res.writeHead(404, { "content-type": "text/plain" }); + res.end("not found\n"); + return; + } + res.writeHead(200, { "content-length": stat.size, "content-type": contentType(fullPath), "cache-control": "no-cache" }); + if (req.method === "HEAD") { + res.end(); + return; + } + createReadStream(fullPath).pipe(res); +} + +function contentType(filePath) { + if (filePath.endsWith(".pack")) return "application/x-git-packed-objects"; + if (filePath.endsWith(".idx")) return "application/x-git-packed-objects-toc"; + return "text/plain"; +} diff --git a/scripts/src/gitops-render/templates/git-mirror-install-hooks.sh b/scripts/src/gitops-render/templates/git-mirror-install-hooks.sh new file mode 100644 index 00000000..b8f70d18 --- /dev/null +++ b/scripts/src/gitops-render/templates/git-mirror-install-hooks.sh @@ -0,0 +1,65 @@ +#!/bin/sh +set -eu +repo_path="/cache/pikasTech/HWLAB.git" +mkdir -p "$repo_path/hooks" +cat > "$repo_path/hooks/pre-receive" <<'HOOK' +#!/bin/sh +set -eu +zero="0000000000000000000000000000000000000000" +status=0 +while read -r old new ref; do + if [ "$new" = "$zero" ]; then + echo "git mirror write rejected: deleting $ref is forbidden" >&2 + status=1 + continue + fi + git cat-file -e "$new^{commit}" || status=1 + git cat-file -e "$new^{tree}" || status=1 + if git rev-list --objects --missing=print "$new" | grep -q '^?'; then + echo "git mirror write rejected: missing objects for $new" >&2 + status=1 + continue + fi + if [ "$old" != "$zero" ] && ! git merge-base --is-ancestor "$old" "$new"; then + echo "git mirror write rejected: non-fast-forward $ref" >&2 + status=1 + continue + fi +done +exit "$status" +HOOK +cat > "$repo_path/hooks/post-receive" <<'HOOK' +#!/bin/sh +set -eu +repo_path=$(git rev-parse --git-dir) +git -C "$repo_path" update-server-info +written_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +export repo_path written_at +export gitops_branches_json=__HWLAB_GITOPS_BRANCHES_JSON_SHELL__ +node - <<'NODE' > /cache/HWLAB.last-write.json +const { execFileSync } = require("node:child_process"); +const repo = process.env.repo_path || "/cache/pikasTech/HWLAB.git"; +const gitopsBranches = JSON.parse(process.env.gitops_branches_json || "[]"); +function rev(ref) { + try { return execFileSync("git", ["-C", repo, "rev-parse", ref], { encoding: "utf8" }).trim() || null; } catch { return null; } +} +const refs = {}; +for (const branch of gitopsBranches) { + refs["refs/heads/" + branch] = rev("refs/heads/" + branch); + refs["refs/mirror-stage/heads/" + branch] = rev("refs/mirror-stage/heads/" + branch); +} +const pendingFlush = gitopsBranches.some((branch) => refs["refs/heads/" + branch] && refs["refs/heads/" + branch] !== refs["refs/mirror-stage/heads/" + branch]); +console.log(JSON.stringify({ + event: "git-mirror-write", + status: "received", + repository: "pikasTech/HWLAB", + writtenAt: process.env.written_at, + pendingFlush, + refs +})); +NODE +HOOK +chmod 0755 "$repo_path/hooks/pre-receive" "$repo_path/hooks/post-receive" +git -C "$repo_path" config http.receivepack true +git -C "$repo_path" config receive.denyDeletes true +git -C "$repo_path" config receive.denyNonFastForwards true diff --git a/scripts/src/gitops-render/templates/git-mirror-proxy-connect.mjs b/scripts/src/gitops-render/templates/git-mirror-proxy-connect.mjs new file mode 100644 index 00000000..372bd7a4 --- /dev/null +++ b/scripts/src/gitops-render/templates/git-mirror-proxy-connect.mjs @@ -0,0 +1,48 @@ +#!/usr/bin/env node +import net from "node:net"; +const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2); +const proxyPort = Number.parseInt(proxyPortRaw || "", 10); +const targetPort = Number.parseInt(targetPortRaw || "", 10); +if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) { + console.error("hwlab git-mirror proxy-connect: invalid ProxyCommand arguments"); + process.exit(64); +} +let settled = false; +let tunnelEstablished = false; +function finish(code, message) { + if (settled) return; + settled = true; + if (message) console.error("hwlab git-mirror proxy-connect: " + message); + process.exit(code); +} +const socket = net.createConnection({ host: proxyHost, port: proxyPort }); +let buffer = Buffer.alloc(0); +socket.setTimeout(15000, () => { socket.destroy(); finish(65, "timeout connecting via " + proxyHost + ":" + proxyPort + " to " + targetHost + ":" + targetPort); }); +socket.on("connect", () => socket.write("CONNECT " + targetHost + ":" + targetPort + " HTTP/1.1\r\nHost: " + targetHost + ":" + targetPort + "\r\nProxy-Connection: Keep-Alive\r\n\r\n")); +socket.on("error", (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? "tunnel socket error: " : "tcp error connecting to proxy: ") + (error && error.message ? error.message : String(error)))); +socket.on("close", () => { if (!tunnelEstablished) finish(68, "proxy closed before CONNECT completed via " + proxyHost + ":" + proxyPort + " to " + targetHost + ":" + targetPort); else finish(0); }); +function onData(chunk) { + buffer = Buffer.concat([buffer, chunk]); + const headerEnd = buffer.indexOf("\r\n\r\n"); + if (headerEnd === -1 && buffer.length < 8192) return; + if (headerEnd === -1) { socket.destroy(); finish(68, "proxy response header exceeded 8192 bytes before CONNECT status via " + proxyHost + ":" + proxyPort + " to " + targetHost + ":" + targetPort); return; } + const head = buffer.slice(0, headerEnd + 4).toString("latin1"); + const statusLine = head.split("\r\n", 1)[0] || ""; + const statusCode = Number.parseInt(statusLine.split(" ")[1] || "", 10); + if (!statusLine.startsWith("HTTP/1.") || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) { + const safeStatus = statusLine.replace(/[^\x20-\x7e]/g, "?").slice(0, 160); + socket.destroy(); + finish(67, "proxy CONNECT failed via " + proxyHost + ":" + proxyPort + " to " + targetHost + ":" + targetPort + ": " + safeStatus); + return; + } + socket.off("data", onData); + socket.setTimeout(0); + tunnelEstablished = true; + const rest = buffer.slice(headerEnd + 4); + if (rest.length) process.stdout.write(rest); + process.stdin.on("error", () => {}); + process.stdout.on("error", () => {}); + process.stdin.pipe(socket); + socket.pipe(process.stdout); +} +socket.on("data", onData); \ No newline at end of file diff --git a/scripts/src/gitops-render/templates/git-mirror-ssh-direct.sh b/scripts/src/gitops-render/templates/git-mirror-ssh-direct.sh new file mode 100644 index 00000000..b2ce5451 --- /dev/null +++ b/scripts/src/gitops-render/templates/git-mirror-ssh-direct.sh @@ -0,0 +1,11 @@ +printf '%s\n' __HWLAB_PROXY_SUMMARY_SHELL__ >&2 +cat > /tmp/hwlab-git-ssh-proxy.sh <<'SH_PROXY' +#!/bin/sh +exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 "$@" +SH_PROXY +chmod 0700 /tmp/hwlab-git-ssh-proxy.sh +unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy +export NO_PROXY='*' +export no_proxy='*' +export GIT_SSH=/tmp/hwlab-git-ssh-proxy.sh +unset GIT_SSH_COMMAND diff --git a/scripts/src/gitops-render/templates/git-mirror-ssh-proxied.sh b/scripts/src/gitops-render/templates/git-mirror-ssh-proxied.sh new file mode 100644 index 00000000..354bb9a9 --- /dev/null +++ b/scripts/src/gitops-render/templates/git-mirror-ssh-proxied.sh @@ -0,0 +1,20 @@ +printf '%s\n' __HWLAB_PROXY_SUMMARY_SHELL__ >&2 +cat > /tmp/hwlab-github-proxy-connect.mjs <<'NODE_PROXY' +// __HWLAB_GIT_MIRROR_PROXY_CONNECT_MJS__ +NODE_PROXY +chmod 0700 /tmp/hwlab-github-proxy-connect.mjs +cat > /tmp/hwlab-git-ssh-proxy.sh <<'SH_PROXY' +#!/bin/sh +exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o __HWLAB_PROXY_COMMAND_SHELL__ "$@" +SH_PROXY +chmod 0700 /tmp/hwlab-git-ssh-proxy.sh +export HTTP_PROXY=__HWLAB_PROXY_URL_SHELL__ +export HTTPS_PROXY=__HWLAB_PROXY_URL_SHELL__ +export ALL_PROXY=__HWLAB_PROXY_URL_SHELL__ +export http_proxy=__HWLAB_PROXY_URL_SHELL__ +export https_proxy=__HWLAB_PROXY_URL_SHELL__ +export all_proxy=__HWLAB_PROXY_URL_SHELL__ +export NO_PROXY=__HWLAB_PROXY_NO_PROXY_SHELL__ +export no_proxy=__HWLAB_PROXY_NO_PROXY_SHELL__ +export GIT_SSH=/tmp/hwlab-git-ssh-proxy.sh +unset GIT_SSH_COMMAND \ No newline at end of file diff --git a/scripts/src/gitops-render/templates/git-mirror-sync.sh b/scripts/src/gitops-render/templates/git-mirror-sync.sh new file mode 100644 index 00000000..b007bc27 --- /dev/null +++ b/scripts/src/gitops-render/templates/git-mirror-sync.sh @@ -0,0 +1,181 @@ +#!/bin/sh +set -eu +repo_url="ssh://git@ssh.github.com:443/pikasTech/HWLAB.git" +repo_path="/cache/pikasTech/HWLAB.git" +source_snapshot_ref_prefix=__HWLAB_SOURCE_SNAPSHOT_REF_PREFIX_SHELL__ +lock_dir="/cache/.hwlab-sync.lock" +sync_started_ms=$(node -e 'console.log(Date.now())') +now_ms() { node -e 'console.log(Date.now())'; } +emit_timing() { + phase="$1" + status="$2" + started_ms="$3" + finished_ms=$(now_ms) + duration_ms=$((finished_ms - started_ms)) + printf '{"event":"git-mirror-sync-timing","phase":"%s","status":"%s","durationMs":%s,"repository":"pikasTech/HWLAB"} +' "$phase" "$status" "$duration_ms" +} +mkdir -p /cache/pikasTech /root/.ssh +if ! mkdir "$lock_dir" 2>/dev/null; then + echo "git mirror sync already running" + exit 0 +fi +trap 'rmdir "$lock_dir" 2>/dev/null || true' EXIT INT TERM +cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa +chmod 0400 /root/.ssh/id_rsa +# __HWLAB_GIT_MIRROR_SSH_PRELUDE__ +if [ -d "$repo_path/objects" ] && [ -f "$repo_path/HEAD" ]; then + git -C "$repo_path" remote set-url origin "$repo_url" || git -C "$repo_path" remote add origin "$repo_url" +else + rm -rf "$repo_path" + git init --bare "$repo_path" + git -C "$repo_path" remote add origin "$repo_url" +fi +/script/install-hooks.sh +git -C "$repo_path" config uploadpack.hideRefs refs/mirror-stage +git -C "$repo_path" config uploadpack.allowReachableSHA1InWant true +git -C "$repo_path" config uploadpack.allowAnySHA1InWant true +git -C "$repo_path" config --unset-all remote.origin.fetch 2>/dev/null || true +# __HWLAB_FETCH_CONFIG_COMMANDS__ +fetch_started_ms=$(now_ms) +timeout 180 git -C "$repo_path" fetch --quiet --prune origin +emit_timing fetch succeeded "$fetch_started_ms" +git -C "$repo_path" for-each-ref --format='%(objectname) %(refname)' refs/mirror-stage/heads > /tmp/hwlab-stage-heads +git -C "$repo_path" for-each-ref --format='%(refname)' refs/mirror-stage/heads > /tmp/hwlab-stage-head-refs +git -C "$repo_path" for-each-ref --format='%(objectname) %(refname)' refs/mirror-stage/tags > /tmp/hwlab-stage-tags +git -C "$repo_path" for-each-ref --format='%(refname)' refs/mirror-stage/tags > /tmp/hwlab-stage-tag-refs +if [ ! -s /tmp/hwlab-stage-heads ]; then + echo "git mirror sync fetched no branch refs" >&2 + exit 41 +fi +for required_ref in __HWLAB_REQUIRED_REF_ARGS__; do + git -C "$repo_path" show-ref --verify --quiet "$required_ref" || { echo "git mirror sync missing required ref $required_ref" >&2; exit 43; } +done +validate_started_ms=$(now_ms) +while read -r sha ref; do + [ -n "$sha" ] || continue + git -C "$repo_path" cat-file -e "$sha^{commit}" + git -C "$repo_path" cat-file -e "$sha^{tree}" + if git -C "$repo_path" rev-list --objects --missing=print "$sha" | grep -q '^?'; then + echo "git mirror sync found missing objects for $ref $sha" >&2 + exit 42 + fi +done < /tmp/hwlab-stage-heads +emit_timing validate succeeded "$validate_started_ms" +publish_started_ms=$(now_ms) +while read -r sha ref; do + [ -n "$sha" ] || continue + name=$(printf '%s +' "$ref" | sed 's#^refs/mirror-stage/heads/##') + is_gitops_branch=false + for gitops_branch in __HWLAB_GITOPS_BRANCH_ARGS__; do + if [ "$name" = "$gitops_branch" ]; then is_gitops_branch=true; break; fi + done + if [ "$is_gitops_branch" = true ]; then + local_sha=$(git -C "$repo_path" show-ref --verify --hash "refs/heads/$name" 2>/dev/null || true) + if [ -n "$local_sha" ] && [ "$local_sha" != "$sha" ]; then + if git -C "$repo_path" merge-base --is-ancestor "$sha" "$local_sha"; then + echo "git mirror sync keeps local $name ahead of GitHub" + continue + fi + if ! git -C "$repo_path" merge-base --is-ancestor "$local_sha" "$sha"; then + echo "git mirror sync found divergent $name local=$local_sha github=$sha" >&2 + exit 44 + fi + fi + fi + git -C "$repo_path" update-ref "refs/heads/$name" "$sha" +done < /tmp/hwlab-stage-heads +for source_branch in __HWLAB_SOURCE_BRANCH_ARGS__; do + source_sha=$(git -C "$repo_path" show-ref --verify --hash "refs/heads/$source_branch" 2>/dev/null || true) + if [ -n "$source_sha" ]; then + source_stage_ref="${source_snapshot_ref_prefix%/}/$source_branch/$source_sha" + git -C "$repo_path" update-ref "$source_stage_ref" "$source_sha" + fi +done +while read -r sha ref; do + [ -n "$sha" ] || continue + name=$(printf '%s +' "$ref" | sed 's#^refs/mirror-stage/tags/##') + git -C "$repo_path" update-ref "refs/tags/$name" "$sha" +done < /tmp/hwlab-stage-tags +git -C "$repo_path" for-each-ref --format='%(refname)' refs/heads > /tmp/hwlab-public-heads +while read -r ref; do + [ -n "$ref" ] || continue + name=$(printf '%s +' "$ref" | sed 's#^refs/heads/##') + if ! grep -Fxq "refs/mirror-stage/heads/$name" /tmp/hwlab-stage-head-refs; then + git -C "$repo_path" update-ref -d "$ref" + fi +done < /tmp/hwlab-public-heads +git -C "$repo_path" for-each-ref --format='%(refname)' refs/tags > /tmp/hwlab-public-tags +while read -r ref; do + [ -n "$ref" ] || continue + name=$(printf '%s +' "$ref" | sed 's#^refs/tags/##') + if ! grep -Fxq "refs/mirror-stage/tags/$name" /tmp/hwlab-stage-tag-refs; then + git -C "$repo_path" update-ref -d "$ref" + fi +done < /tmp/hwlab-public-tags +emit_timing publish succeeded "$publish_started_ms" +for head_branch in __HWLAB_SOURCE_BRANCH_ARGS__; do + if git -C "$repo_path" show-ref --verify --quiet "refs/heads/$head_branch"; then + git -C "$repo_path" symbolic-ref HEAD "refs/heads/$head_branch" + break + fi +done +fsck_started_ms=$(now_ms) +git -C "$repo_path" fsck --connectivity-only --no-dangling >/tmp/hwlab-git-fsck.out +emit_timing fsck succeeded "$fsck_started_ms" +git -C "$repo_path" update-server-info +published_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +for source_branch in __HWLAB_SOURCE_BRANCH_ARGS__; do + git -C "$repo_path" rev-parse "refs/heads/$source_branch" 2>/dev/null | tee "/cache/HWLAB-$source_branch.head" >/dev/null || true +done +printf '%s +' "$published_at" > /cache/HWLAB.last-sync +export published_at +export source_snapshot_ref_prefix +export mirror_branches_json=__HWLAB_MIRROR_BRANCHES_JSON_SHELL__ +export source_branches_json=__HWLAB_SOURCE_BRANCHES_JSON_SHELL__ +export gitops_branches_json=__HWLAB_GITOPS_BRANCHES_JSON_SHELL__ +node - <<'NODE' | tee /cache/HWLAB.last-sync.json +const { execFileSync } = require("node:child_process"); +const repo = "/cache/pikasTech/HWLAB.git"; +const mirrorBranches = JSON.parse(process.env.mirror_branches_json || "[]"); +const sourceBranches = JSON.parse(process.env.source_branches_json || "[]"); +const gitopsBranches = JSON.parse(process.env.gitops_branches_json || "[]"); +const sourceSnapshotRefPrefix = (process.env.source_snapshot_ref_prefix || "refs/unidesk/snapshots/hwlab-node-runtime").replace(//+$/u, ""); +function rev(ref) { + try { return execFileSync("git", ["-C", repo, "rev-parse", ref], { encoding: "utf8" }).trim() || null; } catch { return null; } +} +const refs = {}; +for (const branch of mirrorBranches) { + refs["refs/heads/" + branch] = rev("refs/heads/" + branch); + refs["refs/mirror-stage/heads/" + branch] = rev("refs/mirror-stage/heads/" + branch); +} +const sourceSnapshots = {}; +for (const branch of sourceBranches) { + const source = refs["refs/heads/" + branch] || null; + const stageRef = source ? sourceSnapshotRefPrefix + "/" + branch + "/" + source : null; + const sourceSnapshot = stageRef ? rev(stageRef) : null; + if (stageRef) refs[stageRef] = sourceSnapshot; + sourceSnapshots[branch] = { stageRef, sourceSnapshot }; +} +const sourceInSync = sourceBranches.every((branch) => refs["refs/heads/" + branch] && refs["refs/heads/" + branch] === refs["refs/mirror-stage/heads/" + branch] && sourceSnapshots[branch]?.sourceSnapshot === refs["refs/mirror-stage/heads/" + branch]); +const gitopsInSync = gitopsBranches.every((branch) => refs["refs/heads/" + branch] && refs["refs/heads/" + branch] === refs["refs/mirror-stage/heads/" + branch]); +const pendingFlush = gitopsBranches.some((branch) => refs["refs/heads/" + branch] && refs["refs/mirror-stage/heads/" + branch] && refs["refs/heads/" + branch] !== refs["refs/mirror-stage/heads/" + branch]); +const payload = { + event: "git-mirror-sync", + status: "published", + repository: "pikasTech/HWLAB", + publishedAt: process.env.published_at, + pendingFlush, + sourceInSync, + gitopsInSync, + sourceSnapshots, + refs +}; +console.log(JSON.stringify(payload)); +NODE +emit_timing total succeeded "$sync_started_ms" diff --git a/scripts/src/gitops-render/templates/git-ssh-functions.sh b/scripts/src/gitops-render/templates/git-ssh-functions.sh new file mode 100644 index 00000000..d133a16b --- /dev/null +++ b/scripts/src/gitops-render/templates/git-ssh-functions.sh @@ -0,0 +1,46 @@ +git_ssh_setup() { + mkdir -p /root/.ssh + cp /workspace/git-ssh/ssh-privatekey /root/.ssh/id_rsa + chmod 600 /root/.ssh/id_rsa + timeout 10 ssh-keyscan github.com >> /root/.ssh/known_hosts 2>/dev/null || true + timeout 10 ssh-keyscan -p 443 ssh.github.com >> /root/.ssh/known_hosts 2>/dev/null || true + write_github_proxy_command + export GIT_SSH_COMMAND="ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ServerAliveInterval=10 -o ServerAliveCountMax=2 -o ProxyCommand='node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808 %h %p'" + git config --global url."ssh://git@ssh.github.com:443/".insteadOf "git@github.com:" + git config --global url."ssh://git@ssh.github.com:443/".insteadOf "ssh://git@github.com/" +} + +write_github_proxy_command() { + cat > /tmp/hwlab-github-proxy-connect.mjs <<'NODE_PROXY' +// __HWLAB_GITHUB_PROXY_CONNECT_MJS__ +NODE_PROXY + chmod 0700 /tmp/hwlab-github-proxy-connect.mjs +} + +git_now_ms() { + node -e 'console.log(Date.now())' 2>/dev/null || date +%s000 +} + +git_timed() { + phase="$1" + timeout_seconds="$2" + shift 2 + started_ms="$(git_now_ms)" + echo '{"event":"git-operation","phase":"'"$phase"'","status":"started","timeoutSeconds":'"$timeout_seconds"'}' >&2 + set +e + timeout "$timeout_seconds" "$@" + status="$?" + set -e + finished_ms="$(git_now_ms)" + duration_ms="$((finished_ms - started_ms))" + if [ "$status" -eq 0 ]; then + echo '{"event":"git-operation","phase":"'"$phase"'","status":"succeeded","durationMs":'"$duration_ms"'}' >&2 + return 0 + fi + if [ "$status" -eq 124 ] || [ "$status" -eq 137 ]; then + echo '{"event":"git-operation","phase":"'"$phase"'","status":"failed","reason":"timeout","durationMs":'"$duration_ms"',"timeoutSeconds":'"$timeout_seconds"'}' >&2 + else + echo '{"event":"git-operation","phase":"'"$phase"'","status":"failed","exitCode":'"$status"',"durationMs":'"$duration_ms"'}' >&2 + fi + return "$status" +} diff --git a/scripts/src/gitops-render/templates/github-proxy-connect.mjs b/scripts/src/gitops-render/templates/github-proxy-connect.mjs new file mode 100644 index 00000000..06e8cdcb --- /dev/null +++ b/scripts/src/gitops-render/templates/github-proxy-connect.mjs @@ -0,0 +1,51 @@ +#!/usr/bin/env node +import net from "node:net"; + +const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2); +const proxyPort = Number.parseInt(proxyPortRaw || "", 10); +const targetPort = Number.parseInt(targetPortRaw || "", 10); +if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) { + console.error("usage: hwlab-github-proxy-connect "); + process.exit(64); +} + +const socket = net.createConnection({ host: proxyHost, port: proxyPort }); +let buffer = Buffer.alloc(0); + +socket.setTimeout(10000, () => { + console.error("proxy connect timeout " + proxyHost + ":" + proxyPort + " -> " + targetHost + ":" + targetPort); + socket.destroy(); + process.exit(65); +}); + +socket.on("connect", () => { + socket.write("CONNECT " + targetHost + ":" + targetPort + " HTTP/1.1\r\nHost: " + targetHost + ":" + targetPort + "\r\nProxy-Connection: Keep-Alive\r\n\r\n"); +}); + +socket.on("error", (error) => { + console.error("proxy connect failed: " + error.message); + process.exit(66); +}); + +function onData(chunk) { + buffer = Buffer.concat([buffer, chunk]); + const headerEnd = buffer.indexOf("\r\n\r\n"); + if (headerEnd === -1 && buffer.length < 8192) return; + const head = buffer.slice(0, headerEnd + 4).toString("latin1"); + const statusLine = head.split("\r\n", 1)[0] || ""; + const statusCode = Number.parseInt(statusLine.split(" ")[1] || "", 10); + if (!statusLine.startsWith("HTTP/1.") || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) { + console.error("proxy CONNECT rejected: " + (statusLine || "missing-status")); + socket.destroy(); + process.exit(67); + } + socket.off("data", onData); + socket.setTimeout(0); + const rest = buffer.slice(headerEnd + 4); + if (rest.length) process.stdout.write(rest); + process.stdin.pipe(socket); + socket.pipe(process.stdout); +} + +socket.on("data", onData); +socket.on("close", () => process.exit(0)); \ No newline at end of file diff --git a/scripts/src/gitops-render/templates/gitops-promote.sh b/scripts/src/gitops-render/templates/gitops-promote.sh new file mode 100644 index 00000000..1c2431c4 --- /dev/null +++ b/scripts/src/gitops-render/templates/gitops-promote.sh @@ -0,0 +1,331 @@ +#!/bin/sh +set -eu +# __HWLAB_CI_TIMING_SHELL__ + +export HWLAB_TEKTON_PIPELINERUN="$(context.pipelineRun.name)" +export HWLAB_TEKTON_TASKRUN="$(context.taskRun.name)" +export HWLAB_TEKTON_TASK="gitops-promote" +export HWLAB_SOURCE_REVISION="$(params.revision)" +echo '{"event":"ci-base-image","phase":"gitops-promote","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apt"}' +for tool in node git timeout; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"gitops-promote","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done +# __HWLAB_GIT_SSH_SHELL__ + +git_url_requires_ssh() { case "$1" in git@*|ssh://*) return 0 ;; *) return 1 ;; esac; } +lane="$(params.lane)" +case "$lane" in + v[0-9][0-9]*) runtime_lane=true ;; + *) runtime_lane=false ;; +esac +if [ "$runtime_lane" = "true" ]; then + printf 'false' > /tekton/results/runtime-ready-required +else + printf 'true' > /tekton/results/runtime-ready-required +fi +source_head_url_for_setup="$(params.git-url)" +gitops_read_url_for_setup="$(params.git-url)" +if [ "$runtime_lane" = "true" ]; then + source_head_url_for_setup="$(params.git-read-url)" + gitops_read_url_for_setup="$(params.git-read-url)" +fi +gitops_write_url_for_setup="$(params.git-write-url)" +if git_url_requires_ssh "$source_head_url_for_setup" || git_url_requires_ssh "$gitops_read_url_for_setup" || git_url_requires_ssh "$gitops_write_url_for_setup"; then + for tool in ssh ssh-keyscan; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"gitops-promote","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done + git_ssh_setup +else + echo '{"event":"git-ssh-setup","phase":"gitops-promote","status":"skipped","reason":"non-ssh-git-url"}' +fi +cd /workspace/source/repo +test "$(git rev-parse HEAD)" = "$(params.revision)" + +check_source_head() { + phase="$1" + expected="${2:-$(params.revision)}" + source_head_url="$(params.git-url)" + if [ "$runtime_lane" = "true" ]; then + source_head_url="$(params.git-read-url)" + fi + latest_file="$(mktemp)" + if ! git_timed "source-head-$phase" 45 git ls-remote "$source_head_url" "refs/heads/$(params.source-branch)" > "$latest_file"; then + echo '{"status":"failed","reason":"source-head-check-failed","phase":"'"$phase"'","sourceBranch":"'"$(params.source-branch)"'","sourceRevision":"'"$(params.revision)"'"}' + exit 1 + fi + latest="$(cut -f1 "$latest_file" | head -n 1)" + if [ -z "$latest" ]; then + echo '{"status":"failed","reason":"source-head-unresolved","phase":"'"$phase"'","sourceBranch":"'"$(params.source-branch)"'","sourceRevision":"'"$(params.revision)"'"}' + exit 1 + fi + if [ "$latest" != "$expected" ]; then + printf 'false' > /tekton/results/runtime-ready-required || true + echo '{"status":"skipped-stale-source","verdict":"superseded","phase":"'"$phase"'","sourceBranch":"'"$(params.source-branch)"'","sourceRevision":"'"$(params.revision)"'","expectedRevision":"'"$expected"'","latestRevision":"'"$latest"'"}' + exit 0 + fi +} + +check_source_head before-render +if [ -s /workspace/source/affected-services.json ] && node - /workspace/source/affected-services.json <<'NODE' +const fs = require("node:fs"); +const plan = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); +const buildServices = Array.isArray(plan.buildServices) ? plan.buildServices : []; +const affectedServices = Array.isArray(plan.affectedServices) ? plan.affectedServices : []; +const willRunGitopsPromote = plan.ciCdPlan && plan.ciCdPlan.willRunGitopsPromote === true; +process.exit(!willRunGitopsPromote && buildServices.length === 0 && affectedServices.length === 0 ? 0 : 1); +NODE +then + printf 'false' > /tekton/results/runtime-ready-required || true + echo '{"event":"gitops-promote","status":"skipped","reason":"no-build-no-rollout-plan"}' + exit 0 +fi +git config --global user.name "HWLAB node GitOps Bot" +git config --global user.email "hwlab-node-gitops-bot@users.noreply.github.com" +catalog_path="$(params.catalog-path)" +runtime_path="$(params.runtime-path)" +gitops_root_from_runtime_path() { + path="$1" + case "$path" in + */*) printf '%s +' "${path%/*}" ;; + *) printf '. +' ;; + esac +} +gitops_root="$(gitops_root_from_runtime_path "$runtime_path")" + +argo_hard_refresh_runtime_lane() { + if [ "$runtime_lane" != "true" ]; then return 0; fi + ARGO_APPLICATION="hwlab-node-$lane" ARGO_FIELD_MANAGER="hwlab-$lane-gitops-promote" node <<'NODE' +const fs = require("node:fs"); +const https = require("node:https"); + +const host = process.env.KUBERNETES_SERVICE_HOST; +const port = process.env.KUBERNETES_SERVICE_PORT || "443"; +const startedAt = Date.now(); +const application = process.env.ARGO_APPLICATION || "hwlab-node-v02"; +const fieldManager = process.env.ARGO_FIELD_MANAGER || "hwlab-v02-gitops-promote"; +const pipelineRun = process.env.HWLAB_TEKTON_PIPELINERUN || null; +const taskRun = process.env.HWLAB_TEKTON_TASKRUN || null; +const task = process.env.HWLAB_TEKTON_TASK || null; +const revision = process.env.HWLAB_SOURCE_REVISION || null; + +function emit(payload) { + console.log(JSON.stringify({ + event: "node-cicd-timing", + schemaVersion: "v1", + stage: "argo-hard-refresh", + pipelineRun, + taskRun, + task, + revision, + application, + source: "scripts/gitops-render.mjs", + at: new Date().toISOString(), + ...payload + })); +} + +function safeJson(text) { + try { + return JSON.parse(text); + } catch { + return null; + } +} + +function request(method, path, body, contentType = "application/merge-patch+json") { + const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8"); + const ca = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); + const payload = body ? JSON.stringify(body) : ""; + const headers = { Authorization: "Bearer " + token }; + if (payload) { + headers["Content-Type"] = contentType; + headers["Content-Length"] = Buffer.byteLength(payload); + } + return new Promise((resolve, reject) => { + const req = https.request({ host, port, method, path, ca, headers }, (res) => { + let data = ""; + res.setEncoding("utf8"); + res.on("data", (chunk) => { data += chunk; }); + res.on("end", () => resolve({ statusCode: res.statusCode || 0, body: data, json: safeJson(data) })); + }); + req.on("error", reject); + if (payload) req.write(payload); + req.end(); + }); +} + +(async () => { + if (!host) { + emit({ status: "skipped", reason: "kubernetes-service-host-missing", durationMs: Date.now() - startedAt }); + return; + } + const response = await request( + "PATCH", + "/apis/argoproj.io/v1alpha1/namespaces/argocd/applications/" + encodeURIComponent(application) + "?fieldManager=" + encodeURIComponent(fieldManager), + { metadata: { annotations: { "argocd.argoproj.io/refresh": "hard" } } } + ); + if (response.statusCode >= 200 && response.statusCode < 300) { + emit({ status: "succeeded", durationMs: Date.now() - startedAt, statusCode: response.statusCode }); + return; + } + emit({ status: "degraded", reason: "argo-refresh-patch-failed", durationMs: Date.now() - startedAt, statusCode: response.statusCode, body: response.body.slice(0, 1000) }); +})().catch((error) => { + emit({ status: "degraded", reason: "argo-refresh-patch-error", durationMs: Date.now() - startedAt, error: error.message }); +}); +NODE +} + +if [ -s /workspace/source/dev-artifacts.json ]; then + node scripts/refresh-artifact-catalog.mjs --lane "$lane" --catalog-path "$catalog_path" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --target-ref "$(params.revision)" --publish-report /workspace/source/dev-artifacts.json --write +fi +gitops_render_started_ms="$(ci_now_ms)" +node scripts/run-bun.mjs scripts/gitops-render.mjs --lane "$lane" --catalog-path "$catalog_path" --image-tag-mode "$(params.image-tag-mode)" --source-revision "$(params.revision)" --source-repo "$(params.git-url)" --source-branch "$(params.source-branch)" --gitops-branch "$(params.gitops-branch)" --gitops-root "$gitops_root" --out "$gitops_root" --registry-prefix "$(params.registry-prefix)" --use-deploy-images +node scripts/run-bun.mjs scripts/gitops-render.mjs --lane "$lane" --catalog-path "$catalog_path" --image-tag-mode "$(params.image-tag-mode)" --source-revision "$(params.revision)" --source-repo "$(params.git-url)" --source-branch "$(params.source-branch)" --gitops-branch "$(params.gitops-branch)" --gitops-root "$gitops_root" --out "$gitops_root" --registry-prefix "$(params.registry-prefix)" --use-deploy-images --check +ci_timing_emit gitops-render succeeded "$gitops_render_started_ms" +check_source_head before-push +workdir="$(mktemp -d)" +gitops_read_url="$(params.git-url)" +gitops_write_url="$(params.git-write-url)" +if [ "$runtime_lane" = "true" ]; then + gitops_read_url="$(params.git-read-url)" +fi +gitops_clone_started_ms="$(ci_now_ms)" +git_timed gitops-clone 180 git clone --no-checkout "$gitops_read_url" "$workdir/gitops" +cd "$workdir/gitops" +git remote set-url origin "$gitops_write_url" +old_runtime_snapshot="$workdir/old-runtime-snapshot" +if git_timed gitops-branch-ls 45 git ls-remote --exit-code --heads origin "$(params.gitops-branch)" >/dev/null; then + git_timed gitops-fetch 90 git fetch origin "$(params.gitops-branch)" + git checkout -B "$(params.gitops-branch)" "origin/$(params.gitops-branch)" + if [ "$runtime_lane" = "true" ] && [ -d "$runtime_path" ]; then + mkdir -p "$old_runtime_snapshot" + cp -a "$runtime_path"/. "$old_runtime_snapshot"/ + fi + if [ "$runtime_lane" = "true" ]; then rm -rf "$runtime_path" "$catalog_path"; else rm -rf deploy/gitops/node "$catalog_path"; fi +else + git checkout --orphan "$(params.gitops-branch)" + git rm -rf . >/dev/null 2>&1 || true +fi +ci_timing_emit gitops-clone succeeded "$gitops_clone_started_ms" +mkdir -p "$(dirname "$runtime_path")" "$(dirname "$catalog_path")" +if [ "$runtime_lane" = "true" ]; then + cp -a "/workspace/source/repo/$runtime_path" "$runtime_path" + cp "/workspace/source/repo/$catalog_path" "$catalog_path" + git add "$catalog_path" "$runtime_path" +else + mkdir -p deploy/gitops + cp -a /workspace/source/repo/deploy/gitops/node deploy/gitops/node + cp "/workspace/source/repo/$catalog_path" "$catalog_path" + git add "$catalog_path" deploy/gitops/node +fi +if [ "$runtime_lane" = "true" ] && [ -s /workspace/source/affected-services.json ]; then + runtime_noop_decision="$(node - "$runtime_path" "$old_runtime_snapshot" /workspace/source/affected-services.json <<'NODE' +const fs = require("node:fs"); +const path = require("node:path"); + +const [runtimePath, oldRuntimePath, planPath] = process.argv.slice(2); + +function readJson(filePath) { + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +function stable(value) { + if (Array.isArray(value)) return "[" + value.map(stable).join(",") + "]"; + if (value && typeof value === "object") { + return "{" + Object.keys(value).sort().map((key) => JSON.stringify(key) + ":" + stable(value[key])).join(",") + "}"; + } + return JSON.stringify(value); +} + +function scrub(value) { + if (Array.isArray(value)) return value.map(scrub); + if (!value || typeof value !== "object") return value; + const result = {}; + for (const [key, child] of Object.entries(value)) { + if (key === "hwlab.pikastech.local/source-commit" || + key === "hwlab.pikastech.local/artifact-source-commit" || + key === "hwlab.pikastech.local/boot-commit" || + key === "sourceCommitId" || + key === "bootCommit") { + result[key] = ""; + continue; + } + const envName = String(value.name || ""); + if (key === "value" && /^HWLAB_.*COMMIT/.test(envName)) { + result[key] = ""; + continue; + } + result[key] = scrub(child); + } + return result; +} + +function listFiles(rootDir) { + if (!rootDir || !fs.existsSync(rootDir)) return null; + const files = []; + function walk(current) { + for (const entry of fs.readdirSync(current, { withFileTypes: true })) { + const fullPath = path.join(current, entry.name); + if (entry.isDirectory()) walk(fullPath); + else if (entry.isFile()) files.push(path.relative(rootDir, fullPath).replaceAll(path.sep, "/")); + } + } + walk(rootDir); + return files.sort(); +} + +function normalizedTree(rootDir) { + const files = listFiles(rootDir); + if (!files) return null; + const entries = {}; + for (const relativePath of files) { + const filePath = path.join(rootDir, relativePath); + const text = fs.readFileSync(filePath, "utf8"); + try { + entries[relativePath] = stable(scrub(JSON.parse(text))); + } catch { + entries[relativePath] = text.replace(/[a-f0-9]{40}/gu, ""); + } + } + return stable(entries); +} + +const plan = readJson(planPath); +const buildServices = Array.isArray(plan.buildServices) ? plan.buildServices : []; +const rolloutServices = Array.isArray(plan.rolloutServices) ? plan.rolloutServices : []; +if (buildServices.length > 0 || rolloutServices.length > 0) { + process.stdout.write("runtime-required"); + process.exit(0); +} + +const oldTree = normalizedTree(oldRuntimePath); +const newTree = normalizedTree(runtimePath); +process.stdout.write(oldTree && newTree && oldTree === newTree ? "runtime-identity-only" : "runtime-required"); +NODE +)" + if [ "$runtime_noop_decision" = "runtime-identity-only" ]; then + printf 'false' > /tekton/results/runtime-ready-required + echo '{"status":"skipped-runtime-unchanged","reason":"runtime-identity-only","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}' + ci_timing_emit gitops-commit skipped "$(ci_now_ms)" + ci_timing_emit gitops-push skipped "$(ci_now_ms)" + exit 0 + fi +fi +if git diff --cached --quiet; then + printf 'false' > /tekton/results/runtime-ready-required + echo '{"status":"unchanged","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}' + ci_timing_emit gitops-commit unchanged "$(ci_now_ms)" + ci_timing_emit gitops-push unchanged "$(ci_now_ms)" + exit 0 +fi +short="$(printf '%.7s' "$(params.revision)")" +gitops_commit_started_ms="$(ci_now_ms)" +git commit -m "chore: promote node GitOps source $short" +ci_timing_emit gitops-commit succeeded "$gitops_commit_started_ms" +gitops_push_started_ms="$(ci_now_ms)" +git_timed gitops-push 120 git push origin "HEAD:$(params.gitops-branch)" +ci_timing_emit gitops-push succeeded "$gitops_push_started_ms" +if [ "$runtime_lane" = "true" ]; then + printf 'false' > /tekton/results/runtime-ready-required + echo '{"event":"runtime-ready","phase":"gitops-promote","status":"delegated-post-flush-closeout","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}' +fi +argo_hard_refresh_runtime_lane +echo '{"status":"pushed","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'","gitopsWriteUrl":"'"$gitops_write_url"'"}' diff --git a/scripts/src/gitops-render/templates/per-service-publish.sh b/scripts/src/gitops-render/templates/per-service-publish.sh new file mode 100644 index 00000000..70c4e0e5 --- /dev/null +++ b/scripts/src/gitops-render/templates/per-service-publish.sh @@ -0,0 +1,99 @@ +#!/bin/sh +set -eu +# __HWLAB_CI_TIMING_SHELL__ + +export HWLAB_TEKTON_PIPELINERUN="$(context.pipelineRun.name)" +export HWLAB_TEKTON_TASKRUN="$(context.taskRun.name)" +export HWLAB_TEKTON_TASK="build-$(params.service-id)" +export HWLAB_SOURCE_REVISION="$(params.revision)" +export HWLAB_TIMING_SERVICE_ID="$(params.service-id)" +# __HWLAB_DEPENDENCY_PROXY_PROBE__ +echo '{"event":"ci-base-image","phase":"service-image-publish","serviceId":"'"$(params.service-id)"'","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apk"}' +for tool in node npm git python3 ssh curl; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"service-image-publish","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done +mkdir -p /workspace/service-work/home +export HOME=/workspace/service-work/home +git config --global --add safe.directory /workspace/source/repo +cd /workspace/source/repo +test "$(git rev-parse HEAD)" = "$(params.revision)" +export HWLAB_ARTIFACT_LANE="$(params.lane)" +export HWLAB_ARTIFACT_CATALOG_PATH="$(params.catalog-path)" +export HWLAB_DEPLOY_MANIFEST_PATH="deploy/deploy.yaml" +export HWLAB_ARTIFACT_IMAGE_TAG_MODE="$(params.image-tag-mode)" +mkdir -p /workspace/source/service-results +if [ -s /workspace/source/affected-services.json ] && ! node -e 'const fs=require("node:fs"); const p=JSON.parse(fs.readFileSync("/workspace/source/affected-services.json","utf8")); const service=process.argv[1]; const item=(p.entries||[]).find((entry)=>entry.serviceId===service); process.exit(item && item.buildRequired ? 0 : 1);' "$(params.service-id)"; then + node - "$(params.service-id)" <<'NODE' +const fs = require("node:fs"); +const serviceId = process.argv[2]; +const catalog = JSON.parse(fs.readFileSync(process.env.HWLAB_ARTIFACT_CATALOG_PATH, "utf8")); +const plan = fs.existsSync("/workspace/source/affected-services.json") ? JSON.parse(fs.readFileSync("/workspace/source/affected-services.json", "utf8")) : {}; +const service = (catalog.services || []).find((item) => item.serviceId === serviceId) || {}; +const planned = (plan.services || []).find((item) => item.serviceId === serviceId) || {}; +const envReuse = planned.runtimeMode === "env-reuse-git-mirror-checkout" || service.runtimeMode === "env-reuse-git-mirror-checkout" || planned.envReuse === true || service.envReuse === true; +const image = envReuse ? (service.environmentImage || service.image || "") : (service.image || ""); +const digest = envReuse ? (service.environmentDigest || service.digest || "not_published") : (service.digest || "not_published"); +const imageTag = service.imageTag || (image.includes(":") ? image.slice(image.lastIndexOf(":") + 1) : ""); +const repository = image.includes(":") ? image.slice(0, image.lastIndexOf(":")) : image; +const revision = process.env.HWLAB_SOURCE_REVISION || planned.bootCommit || service.bootCommit || service.sourceCommitId || service.commitId || imageTag; +const componentInputHash = envReuse ? (planned.componentInputHash || service.componentInputHash || "") : (service.componentInputHash || ""); +const environmentInputHash = envReuse ? (service.environmentInputHash || planned.environmentInputHash || "") : (service.environmentInputHash || ""); +const codeInputHash = envReuse ? (planned.codeInputHash || service.codeInputHash || "") : (service.codeInputHash || ""); +const values = { + "service-id": serviceId, + status: /^sha256:[a-f0-9]{64}$/.test(digest) ? "reused" : "blocked_reuse_unavailable", + image, + "image-tag": imageTag, + digest, + "repository-digest": /^sha256:[a-f0-9]{64}$/.test(digest) && repository ? repository + "@" + digest : "", + "source-commit-id": envReuse ? revision : (service.sourceCommitId || service.commitId || imageTag), + "component-input-hash": componentInputHash, + "environment-input-hash": environmentInputHash, + "code-input-hash": codeInputHash, + "runtime-mode": envReuse ? "env-reuse-git-mirror-checkout" : "service-image", + "boot-repo": planned.bootRepo || service.bootRepo || "", + "boot-commit": envReuse ? revision : (service.bootCommit || ""), + "boot-sh": planned.bootSh || service.bootSh || "", + "build-created-at": service.buildCreatedAt || "", + "build-backend": envReuse ? "reused-env-catalog" : "reused-catalog", + "reused-from": (envReuse ? service.environmentInputHash : service.componentInputHash) || service.commitId || imageTag || "catalog" +}; +for (const [name, value] of Object.entries(values)) fs.writeFileSync("/tekton/results/" + name, String(value || "")); +NODE + echo '{"event":"service-build-skip","serviceId":"'"$(params.service-id)"'","reason":"component-inputs-unchanged"}' + exit 0 +fi +rm -rf /workspace/service-work/repo +mkdir -p /workspace/service-work/repo +tar -C /workspace/source/repo -cf - . | tar -C /workspace/service-work/repo -xo -f - +cd /workspace/service-work/repo +export HWLAB_CI_ARTIFACT_RUN_ID="$(context.pipelineRun.name)-$(params.service-id)" +export HWLAB_CI_ARTIFACT_RUN_OWNER="tekton" +export HWLAB_DEV_REGISTRY_PREFIX="$(params.registry-prefix)" +export HWLAB_DEV_REGISTRY_K8S_NATIVE=1 +export HWLAB_NODE_CICD_TIMING=1 +export HWLAB_TEKTON_PIPELINERUN +export HWLAB_TEKTON_TASKRUN +export HWLAB_TEKTON_TASK +export HWLAB_SOURCE_REVISION +export PATH="/workspace/buildkit-bin:$PATH" +export HWLAB_BUILDKIT_ADDR="unix:///workspace/buildkit-run/buildkitd.sock" +if [ -n "$(params.base-image)" ]; then export HWLAB_DEV_BASE_IMAGE="$(params.base-image)"; fi +if [ -n "${HWLAB_DEV_BASE_IMAGE:-}" ]; then + echo '{"event":"dependency-local-registry-probe-start","phase":"service-image-publish-pre-base-image","serviceId":"'"$(params.service-id)"'","target":"http://127.0.0.1:5000/v2/"}' + registry_started_at="$(date +%s)" + curl -fsS --max-time 10 http://127.0.0.1:5000/v2/ >/dev/null + registry_finished_at="$(date +%s)" + echo '{"event":"dependency-local-registry-probe","phase":"service-image-publish-pre-base-image","serviceId":"'"$(params.service-id)"'","ok":true,"durationSeconds":'"$((registry_finished_at - registry_started_at))"'}' +fi +buildkit_ready=0 +for attempt in $(seq 1 60); do + if /workspace/buildkit-bin/buildctl --addr "$HWLAB_BUILDKIT_ADDR" debug workers >/dev/null 2>&1; then + buildkit_ready=1 + break + fi + sleep 1 +done +if [ "$buildkit_ready" != "1" ]; then + echo '{"event":"buildkit-sidecar-not-ready","serviceId":"'"$(params.service-id)"'","addr":"'"$HWLAB_BUILDKIT_ADDR"'"}' >&2 + exit 1 +fi +HWLAB_CI_PLAN_VERIFY_REUSE_REGISTRY=1 node scripts/artifact-publish.mjs --publish --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --registry-prefix "$(params.registry-prefix)" --report "/workspace/source/service-results/$(params.service-id).json" --tekton-results-dir /tekton/results --quiet-build --concurrency 1 --services "$(params.service-id)" --buildkit-command /workspace/buildkit-bin/buildctl --buildkit-addr "$HWLAB_BUILDKIT_ADDR" --build-cache-mode "$(params.build-cache-mode)" diff --git a/scripts/src/gitops-render/templates/plan-artifacts.sh b/scripts/src/gitops-render/templates/plan-artifacts.sh new file mode 100644 index 00000000..8de35581 --- /dev/null +++ b/scripts/src/gitops-render/templates/plan-artifacts.sh @@ -0,0 +1,58 @@ +#!/bin/sh +set -eu +echo '{"event":"ci-base-image","phase":"plan-artifacts","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apk"}' +for tool in node git; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"plan-artifacts","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done +cd /workspace/source/repo +ci_node_deps="${HWLAB_CI_NODE_DEPS:-/opt/hwlab-ci-node-deps/node_modules}" +if [ -d "$ci_node_deps/yaml" ] && [ ! -e node_modules/yaml ]; then + mkdir -p node_modules + ln -s "$ci_node_deps/yaml" node_modules/yaml +fi +test "$(git rev-parse HEAD)" = "$(params.revision)" +HWLAB_CATALOG_PATH="$(params.catalog-path)" HWLAB_GIT_URL="$(params.git-url)" HWLAB_GIT_READ_URL="$(params.git-read-url)" HWLAB_GITOPS_BRANCH="$(params.gitops-branch)" node scripts/ci/restore-artifact-catalog.mjs +node scripts/ci-plan.mjs --lane "$(params.lane)" --target-ref HEAD --deploy-config deploy/deploy.yaml --artifact-catalog "$(params.catalog-path)" --registry-prefix "$(params.registry-prefix)" --services "$(params.services)" --verify-reuse-registry > /workspace/source/ci-plan.json +node - <<'NODE' +const fs = require("node:fs"); +const plan = JSON.parse(fs.readFileSync("/workspace/source/ci-plan.json", "utf8")); +const selected = String(process.env.HWLAB_SELECTED_SERVICES || "").split(",").map((item) => item.trim()).filter(Boolean); +const allServices = selected.length > 0 ? selected : ["__HWLAB_DEFAULT_SERVICE_IDS__"]; +const affected = new Set(plan.affectedServices || []); +const plannedBuildServices = Array.isArray(plan.buildServices) ? new Set(plan.buildServices) : null; +const selectedSet = new Set(selected); +const byService = new Map((plan.services || []).map((service) => [service.serviceId, service])); +const entries = allServices.map((serviceId) => { + const service = byService.get(serviceId) || {}; + const serviceSelected = selectedSet.has(serviceId); + const rolloutAffected = serviceSelected && affected.has(serviceId); + const envReuse = service.runtimeMode === "env-reuse-git-mirror-checkout" || service.envReuse === true; + const buildRequired = serviceSelected && (plannedBuildServices ? plannedBuildServices.has(serviceId) : (envReuse ? service.envChanged === true : rolloutAffected)); + return { + serviceId, + selected: serviceSelected, + affected: rolloutAffected, + buildRequired, + rolloutAffected, + runtimeMode: service.runtimeMode || "service-image", + envChanged: service.envChanged ?? null, + codeChanged: service.codeChanged ?? null + }; +}); +for (const entry of entries) { + fs.writeFileSync("/tekton/results/affected-" + entry.serviceId, entry.affected ? "true" : "false"); + fs.writeFileSync("/tekton/results/build-" + entry.serviceId, entry.buildRequired ? "true" : "false"); +} +fs.writeFileSync("/workspace/source/affected-services.json", JSON.stringify({ + sourceCommitId: plan.sourceCommitId, + affectedServices: plan.affectedServices || [], + rolloutServices: plan.rolloutServices || plan.affectedServices || [], + buildServices: plan.buildServices || entries.filter((entry) => entry.buildRequired).map((entry) => entry.serviceId), + reusedServices: plan.reusedServices || [], + buildSkippedCount: plan.buildSkippedCount || 0, + envArtifactGroups: plan.envArtifactGroups || [], + changedPathSummary: plan.changedPathSummary || null, + ciCdPlan: plan.ciCdPlan || null, + services: plan.services || [], + entries +}, null, 2) + String.fromCharCode(10)); +console.log(JSON.stringify({ event: "g14-ci-plan", sourceCommitId: plan.sourceCommitId, affectedServices: plan.affectedServices || [], rolloutServices: plan.rolloutServices || plan.affectedServices || [], buildServices: plan.buildServices || [], reusedServices: plan.reusedServices || [], buildSkippedCount: plan.buildSkippedCount || 0, envArtifactGroups: plan.envArtifactGroups || [] })); +NODE diff --git a/scripts/src/gitops-render/templates/poller.sh b/scripts/src/gitops-render/templates/poller.sh new file mode 100644 index 00000000..1d9b8d5b --- /dev/null +++ b/scripts/src/gitops-render/templates/poller.sh @@ -0,0 +1,142 @@ +#!/bin/sh +set -eu +# __HWLAB_DEPENDENCY_PROXY_PROBE__ +echo '{"event":"ci-base-image","phase":"poller","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apk"}' +for tool in node git ssh curl timeout; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"poller","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done +# __HWLAB_GIT_SSH_SHELL__ + +git_ssh_setup +workdir="$(mktemp -d)" +git_timed poller-clone 180 git clone --depth 1 --branch "$SOURCE_BRANCH" "$GIT_READ_URL" "$workdir/repo" +cd "$workdir/repo" +git remote set-url origin "$GIT_READ_URL" +revision="$(git rev-parse HEAD)" +subject="$(git log -1 --pretty=%s)" +case "$subject" in + "chore: promote node GitOps source "*) + echo "Skipping generated GitOps promotion commit $revision" + exit 0 + ;; +esac +short="$(printf '%.12s' "$revision")" +: "${PIPELINERUN_PREFIX:?PIPELINERUN_PREFIX is required}" +: "${GITOPS_TARGET:?GITOPS_TARGET is required}" +prefix="$PIPELINERUN_PREFIX" +name="$prefix-$short" +export POLLER_REVISION="$revision" +export POLLER_PIPELINERUN="$name" +export POLLER_SOURCE_SUBJECT="$subject" +echo "Checking $GITOPS_TARGET source $revision with PipelineRun $name" +node <<'NODE' +const fs = require("node:fs"); +const https = require("node:https"); + +function requiredEnv(name) { + const value = process.env[name]; + if (!value) throw new Error(name + " is required"); + return value; +} + +function optionalEnv(name) { + return process.env[name] || ""; +} + +const namespace = requiredEnv("POD_NAMESPACE"); +const name = requiredEnv("POLLER_PIPELINERUN"); +const revision = requiredEnv("POLLER_REVISION"); +const host = process.env.KUBERNETES_SERVICE_HOST; +const port = process.env.KUBERNETES_SERVICE_PORT || "443"; +if (!host) throw new Error("KUBERNETES_SERVICE_HOST is not available"); + +const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8"); +const ca = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); + +function request(method, path, body) { + const payload = body ? JSON.stringify(body) : ""; + const headers = { Authorization: "Bearer " + token }; + if (payload) { + headers["Content-Type"] = "application/json"; + headers["Content-Length"] = Buffer.byteLength(payload); + } + return new Promise((resolve, reject) => { + const req = https.request({ host, port, method, path, ca, headers }, (res) => { + let data = ""; + res.setEncoding("utf8"); + res.on("data", (chunk) => { data += chunk; }); + res.on("end", () => resolve({ statusCode: res.statusCode || 0, body: data })); + }); + req.on("error", reject); + if (payload) req.write(payload); + req.end(); + }); +} + +const labels = { + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/gitops-target": requiredEnv("GITOPS_TARGET"), + "hwlab.pikastech.local/source-commit": revision, + "hwlab.pikastech.local/trigger": "polling" +}; + +const pipelineRun = { + apiVersion: "tekton.dev/v1", + kind: "PipelineRun", + metadata: { + name, + namespace, + labels, + annotations: { + "hwlab.pikastech.local/source-subject": optionalEnv("POLLER_SOURCE_SUBJECT"), + "hwlab.pikastech.local/source-branch": requiredEnv("SOURCE_BRANCH"), + "hwlab.pikastech.local/gitops-branch": requiredEnv("GITOPS_BRANCH") + } + }, + spec: { + pipelineRef: { name: requiredEnv("PIPELINE_NAME") }, + taskRunTemplate: { + serviceAccountName: requiredEnv("SERVICE_ACCOUNT_NAME"), + podTemplate: { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet", securityContext: { fsGroup: 1000 } } + }, + params: [ + { name: "git-url", value: requiredEnv("GIT_URL") }, + { name: "git-read-url", value: requiredEnv("GIT_READ_URL") }, + { name: "source-branch", value: requiredEnv("SOURCE_BRANCH") }, + { name: "gitops-branch", value: requiredEnv("GITOPS_BRANCH") }, + { name: "lane", value: requiredEnv("LANE") }, + { name: "catalog-path", value: requiredEnv("CATALOG_PATH") }, + { name: "image-tag-mode", value: requiredEnv("IMAGE_TAG_MODE") }, + { name: "runtime-path", value: requiredEnv("RUNTIME_PATH") }, + { name: "revision", value: revision }, + { name: "registry-prefix", value: requiredEnv("REGISTRY_PREFIX") }, + { name: "services", value: requiredEnv("SERVICES") }, + { name: "base-image", value: requiredEnv("BASE_IMAGE") } + ], + workspaces: [ + { name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } }, + { name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } } + ] + } +}; + +const base = "/apis/tekton.dev/v1/namespaces/" + encodeURIComponent(namespace) + "/pipelineruns"; +const item = base + "/" + encodeURIComponent(name); + +(async () => { + const existing = await request("GET", item); + if (existing.statusCode === 200) { + console.log(JSON.stringify({ status: "exists", pipelineRun: name, revision })); + return; + } + if (existing.statusCode !== 404) { + throw new Error("unexpected PipelineRun lookup status " + existing.statusCode + ": " + existing.body.slice(0, 500)); + } + const created = await request("POST", base, pipelineRun); + if (created.statusCode < 200 || created.statusCode > 299) { + throw new Error("PipelineRun create failed with status " + created.statusCode + ": " + created.body.slice(0, 1000)); + } + console.log(JSON.stringify({ status: "created", pipelineRun: name, revision })); +})().catch((error) => { + console.error(error.stack || error.message); + process.exitCode = 1; +}); +NODE diff --git a/scripts/src/gitops-render/templates/prepare-source.sh b/scripts/src/gitops-render/templates/prepare-source.sh new file mode 100644 index 00000000..40896017 --- /dev/null +++ b/scripts/src/gitops-render/templates/prepare-source.sh @@ -0,0 +1,117 @@ +#!/bin/sh +set -eu +# __HWLAB_CI_TIMING_SHELL__ + +export HWLAB_TEKTON_PIPELINERUN="$(context.pipelineRun.name)" +export HWLAB_TEKTON_TASKRUN="$(context.taskRun.name)" +export HWLAB_TEKTON_TASK="prepare-source" +export HWLAB_SOURCE_REVISION="$(params.revision)" +echo '{"event":"ci-base-image","phase":"prepare-source","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apk"}' +for tool in node git timeout; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done +node -e 'console.log(JSON.stringify({event:"ci-base-image",phase:"prepare-source",ok:true,node:process.version}))' +# __HWLAB_GIT_SSH_SHELL__ + +git_url_requires_ssh() { case "$1" in git@*|ssh://*) return 0 ;; *) return 1 ;; esac; } +git_read_url="$(params.git-read-url)" +if git_url_requires_ssh "$git_read_url"; then + for tool in ssh ssh-keyscan; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done + git_ssh_setup +else + echo '{"event":"git-ssh-setup","phase":"prepare-source","status":"skipped","reason":"non-ssh-git-read-url"}' +fi +rm -rf /workspace/source/repo +source_clone_started_ms="$(ci_now_ms)" +git_timed source-clone 180 git clone --branch "$(params.source-branch)" "$git_read_url" /workspace/source/repo +cd /workspace/source/repo +git config --global --add safe.directory /workspace/source/repo +git remote set-url origin "$git_read_url" +git checkout "$(params.revision)" +test "$(git rev-parse HEAD)" = "$(params.revision)" +git merge-base --is-ancestor "$(params.revision)" "origin/$(params.source-branch)" || { echo '{"event":"source-ancestry","status":"failed","revision":"'"$(params.revision)"'","sourceBranch":"'"$(params.source-branch)"'"}'; exit 32; } +ci_timing_emit source-clone succeeded "$source_clone_started_ms" +prepare_source_dependencies_started_ms="$(ci_now_ms)" +echo '{"event":"prepare-source-dependencies","status":"skipped","reason":"renderer-dependency-install-disabled","dependency":"yaml"}' +ci_timing_emit prepare-source-dependencies succeeded "$prepare_source_dependencies_started_ms" +catalog_path="$(params.catalog-path)" +mkdir -p "$(dirname "$catalog_path")" +catalog_fetch_started_ms="$(ci_now_ms)" +if git_timed catalog-ls-remote 45 git ls-remote --exit-code --heads "$git_read_url" "$(params.gitops-branch)" >/dev/null; then + git remote set-url gitops-catalog "$git_read_url" 2>/dev/null || git remote add gitops-catalog "$git_read_url" + git_timed catalog-fetch 90 git fetch --depth 1 gitops-catalog "$(params.gitops-branch)" >/dev/null || true + if git cat-file -e FETCH_HEAD:"$catalog_path" 2>/dev/null; then + git show FETCH_HEAD:"$catalog_path" > /tmp/hwlab-gitops-artifact-catalog.json + if node --input-type=module - /tmp/hwlab-gitops-artifact-catalog.json deploy/deploy.yaml "$(params.services)" <<'NODE' +import fs from "node:fs"; +const [catalogPath, deployPath, selectedServices] = process.argv.slice(2); +const ids = (doc) => (doc.services || []).map((service) => service.serviceId).filter(Boolean); +const topLevelServiceIds = (text) => { + const values = []; + let inServices = false; + for (const line of text.split(/\r?\n/u)) { + if (/^services:\s*$/u.test(line)) { inServices = true; continue; } + if (!inServices) continue; + if (/^\S/u.test(line)) break; + const match = line.match(/^\s*-\s+serviceId:\s*['"]?([^'"#\s]+)['"]?/u); + if (match) values.push(match[1]); + } + return values; +}; +const selected = (selectedServices || "").split(",").map((item) => item.trim()).filter(Boolean); +const uniqueSorted = (items) => [...new Set(items)].sort(); +const gitops = JSON.parse(fs.readFileSync(catalogPath, "utf8")); +const expected = selected.length ? selected : topLevelServiceIds(fs.readFileSync(deployPath, "utf8")); +const actual = ids(gitops); +const expectedSet = uniqueSorted(expected); +const actualSet = uniqueSorted(actual); +const sameServices = expectedSet.length === actualSet.length && expectedSet.every((item, index) => item === actualSet[index]); +if (!sameServices) { + const missing = expectedSet.filter((item) => !actualSet.includes(item)); + const extra = actualSet.filter((item) => !expectedSet.includes(item)); + console.error(JSON.stringify({ event: "gitops-artifact-catalog", phase: "prepare-source", status: "ignored-stale", reason: "service-ids-mismatch", expected, actual, missing, extra })); + process.exit(42); +} +NODE + then + cp /tmp/hwlab-gitops-artifact-catalog.json "$catalog_path" + echo '{"event":"gitops-artifact-catalog","phase":"prepare-source","status":"loaded","branch":"'"$(params.gitops-branch)"'"}' + else + echo '{"event":"gitops-artifact-catalog","phase":"prepare-source","status":"ignored-stale","reason":"service-ids-mismatch","branch":"'"$(params.gitops-branch)"'"}' + fi + else + echo '{"event":"gitops-artifact-catalog","phase":"prepare-source","status":"seed","reason":"missing-on-gitops-branch"}' + fi +else + echo '{"event":"gitops-artifact-catalog","phase":"prepare-source","status":"seed","reason":"gitops-branch-missing"}' +fi +if [ ! -s "$catalog_path" ] && [ "$(params.lane)" = "v02" ]; then + node --input-type=module - "$catalog_path" "$(params.revision)" "$(params.registry-prefix)" "$(params.image-tag-mode)" "$(params.services)" <<'NODE' +import fs from 'node:fs'; +import path from 'node:path'; +const [catalogPath, revision, registryPrefix, imageTagMode, selectedServices] = process.argv.slice(2); +const topLevelServiceIds = (text) => { + const values = []; + let inServices = false; + for (const line of text.split(/\r?\n/u)) { + if (/^services:\s*$/u.test(line)) { inServices = true; continue; } + if (!inServices) continue; + if (/^\S/u.test(line)) break; + const match = line.match(/^\s*-\s+serviceId:\s*['"]?([^'"#\s]+)['"]?/u); + if (match) values.push(match[1]); + } + return values; +}; +const tag = imageTagMode === 'full' ? revision : revision.slice(0, 7); +const namespace = 'hwlab-v02'; +const selected = (selectedServices || '').split(',').filter(Boolean); +const serviceIds = selected.length ? selected : topLevelServiceIds(fs.readFileSync('deploy/deploy.yaml', 'utf8')); +const services = serviceIds.map((serviceId) => { + const service = { serviceId, profile: 'v02', namespace, commitId: tag, sourceCommitId: revision, image: `${registryPrefix}/${serviceId}:${tag}`, imageTag: tag, digest: null, buildBackend: 'contract-skeleton' }; + return service; +}); +fs.mkdirSync(path.dirname(catalogPath), { recursive: true }); +fs.writeFileSync(catalogPath, JSON.stringify({ catalogVersion: 'v1', kind: 'hwlab-artifact-catalog', environment: 'v02', profile: 'v02', namespace, endpoint: "__HWLAB_V02_RUNTIME_ENDPOINT__", commitId: tag, artifactState: 'contract-skeleton', publish: { registryPrefix, sourceCommitId: revision, imageTag: tag, publishedAt: null }, allowedProfiles: ['v02'], forbiddenProfiles: ['dev', 'prod'], services }, null, 2) + '\n'); +NODE + echo '{"event":"gitops-artifact-catalog","phase":"prepare-source","status":"seeded-v02-skeleton"}' +fi +ci_timing_emit catalog-fetch succeeded "$catalog_fetch_started_ms" +echo 'node prepare-source complete; validation task count=__HWLAB_VALIDATION_TASK_COUNT__' diff --git a/web/hwlab-cloud-web/scripts/check.ts b/web/hwlab-cloud-web/scripts/check.ts index 49da44b8..18568f72 100644 --- a/web/hwlab-cloud-web/scripts/check.ts +++ b/web/hwlab-cloud-web/scripts/check.ts @@ -98,7 +98,7 @@ const workbenchHealthRuntimeSource = readWeb("src/utils/workbench-health.ts"); const traceTimelineSource = readWeb("src/components/agent/TraceTimeline.vue"); const messageTraceDebugSource = readWeb("src/components/workbench/MessageTraceDebugPanel.vue"); const conversationPanelSource = readWeb("src/components/workbench/ConversationPanel.vue"); -const serverWorkbenchHttpSource = fs.readFileSync(path.resolve(rootDir, "..", "..", "internal/cloud/server-workbench-http.ts"), "utf8"); +const serverWorkbenchRealtimeHttpSource = fs.readFileSync(path.resolve(rootDir, "..", "..", "internal/cloud/server-workbench-realtime-http.ts"), "utf8"); assert.match(html, /
<\/div>/u, "index.html must expose Vue mount root"); assert.match(html, /src="\/src\/main\.ts"/u, "index.html must load Vue TS entry"); @@ -208,9 +208,9 @@ assert.doesNotMatch(workbenchStoreSource, /refreshActiveTraceFromRest[\s\S]{0,12 assert.doesNotMatch(workbenchStoreSource, /refreshTerminalTraceFromRest[\s\S]{0,1200}refreshSessions\([^;]+force:\s*true/u, "Terminal trace REST refresh must not force-refresh the full session list"); assert.doesNotMatch(workbenchStoreSource, /message\.runnerTrace(?:\?\.|\.)status/u, "Workbench message lifecycle must not be inferred from runnerTrace.status"); assert.doesNotMatch(conversationPanelSource, /message\.runnerTrace(?:\?\.|\.)status/u, "ConversationPanel must not override message completion from runnerTrace.status"); -assert.doesNotMatch(serverWorkbenchHttpSource, /return\s+turnSnapshot\(context\)|\.\.\.turnSnapshot\(context\)|blockedTraceEventProjection\(context\.projection/u, "Realtime turn snapshots must not fall back to legacy trace/result projection"); -assertIncludes(serverWorkbenchHttpSource, "blockedRealtimeTurnSnapshot", "Realtime read-model gaps must produce blocked diagnostic turn snapshots"); -assertIncludes(serverWorkbenchHttpSource, "workbench_facts_session_missing", "Realtime read-model gaps must expose a facts-missing blocker"); +assert.doesNotMatch(serverWorkbenchRealtimeHttpSource, /return\s+turnSnapshot\(context\)|\.\.\.turnSnapshot\(context\)|blockedTraceEventProjection\(context\.projection/u, "Realtime turn snapshots must not fall back to legacy trace/result projection"); +assertIncludes(serverWorkbenchRealtimeHttpSource, "blockedRealtimeTurnSnapshot", "Realtime read-model gaps must produce blocked diagnostic turn snapshots"); +assertIncludes(serverWorkbenchRealtimeHttpSource, "workbench_facts_session_missing", "Realtime read-model gaps must expose a facts-missing blocker"); assertIncludes(appSource, "/v1/workbench/events", "Workbench realtime client must use the RESTful same-origin events endpoint"); assertIncludes(appSource, "/v1/workbench/traces/", "trace detail read must use Workbench read-model trace API"); assert.doesNotMatch(appSource, /\/v1\/agent\/(?:turns|traces|chat\/result)\//u, "Cloud Web must not call legacy Code Agent read-through turn/trace/result APIs"); diff --git a/web/hwlab-cloud-web/scripts/workbench-realtime-runtime.test.ts b/web/hwlab-cloud-web/scripts/workbench-realtime-runtime.test.ts index 036fe4d0..ff99b6ab 100644 --- a/web/hwlab-cloud-web/scripts/workbench-realtime-runtime.test.ts +++ b/web/hwlab-cloud-web/scripts/workbench-realtime-runtime.test.ts @@ -6,7 +6,9 @@ import test from "node:test"; import type { ChatMessage } from "../src/types/index.ts"; import { workbenchSessionDetailPathForTest, workbenchSessionMessagesPathForTest } from "../src/api/workbench.ts"; +import { workbenchEventStreamPath, workbenchProjectionEventStreamPath } from "../src/api/workbench-events.ts"; import type { WorkbenchStreamTransportRecovery } from "../src/utils/workbench-realtime-runtime.ts"; +import { workbenchRealtimeTraceIdForCapabilities, workbenchRealtimeTransportEnabled } from "../src/utils/workbench-stream-transport.ts"; import { workbenchRuntimePolicy } from "../src/config/workbench-runtime-policy.ts"; import { AsyncQueue, work } from "../src/utils/scheduler/async-queue.ts"; import { createCoalescedEventQueue } from "../src/utils/scheduler/coalesced-event-queue.ts"; @@ -17,7 +19,8 @@ import { createSafeStorageRuntime, isStorageQuotaError, migrateLegacyStorage, no import { checkWorkbenchHealth, createWorkbenchHealthProbeCache } from "../src/utils/workbench-health.ts"; import { messageDiagnosticView } from "../src/utils/workbench-error-runtime.ts"; import { WORKBENCH_TIMELINE_OPENCODE_PARITY, buildWorkbenchTimelineRows, normalizeWorkbenchTimelineMessages, workbenchTimelineSignature } from "../src/stores/workbench-timeline-model.ts"; -import { reduceWorkbenchRealtimeEvent } from "../src/stores/workbench-event-reducer.ts"; +import { reduceWorkbenchRealtimeEvent, workbenchRealtimeEventIsBusinessActivity } from "../src/stores/workbench-event-reducer.ts"; +import { reduceWorkbenchLiveKafkaMessageState, workbenchLiveKafkaMessageId } from "../src/stores/workbench-live-kafka-event.ts"; import { planWorkbenchRealtimeApply, planWorkbenchRealtimeRecovery } from "../src/stores/workbench-realtime-plan.ts"; import { WORKBENCH_REALTIME_AUTHORITY_VERSION, workbenchRealtimePrimaryAuthorityDecision, workbenchSyncReplayEvents } from "../src/stores/workbench-realtime-authority.ts"; import { cleanupWorkbenchServerStateDroppedSessions, cleanupWorkbenchServerStateSessions, createWorkbenchServerState, reduceWorkbenchServerState } from "../src/stores/workbench-server-state.ts"; @@ -79,6 +82,25 @@ test("Workbench API uses metadata-only session detail and bounded messages paths assert.equal(workbenchSessionMessagesPathForTest("ses_metadata", { limit: 9 }), "/v1/workbench/sessions/ses_metadata/messages?limit=9"); }); +test("live SSE transport never sends an afterSeq cursor while projection transport can", () => { + assert.equal(workbenchEventStreamPath({ realtimeCapabilities: { liveKafkaSse: true, projectionRealtime: false }, sessionId: "ses_live", traceId: "trc_live", afterSeq: 42 }), "/v1/workbench/events?sessionId=ses_live&traceId=trc_live"); + assert.equal(workbenchEventStreamPath({ realtimeCapabilities: { liveKafkaSse: false, projectionRealtime: true }, sessionId: "ses_projection", traceId: "trc_projection", afterSeq: 42 }), "/v1/workbench/events?sessionId=ses_projection&traceId=trc_projection&afterSeq=42"); + assert.equal(workbenchEventStreamPath({ realtimeCapabilities: { liveKafkaSse: true, projectionRealtime: true }, sessionId: "ses_both", traceId: "trc_both", afterSeq: 42 }), "/v1/workbench/events?sessionId=ses_both&traceId=trc_both"); + assert.equal(workbenchProjectionEventStreamPath({ sessionId: "ses_both", traceId: "trc_both", afterSeq: 42 }), "/v1/workbench/projection-events?sessionId=ses_both&traceId=trc_both&afterSeq=42"); + assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: false, projectionRealtime: false }), false); + assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: true, projectionRealtime: false }), true); + assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: false, projectionRealtime: true }), true); + assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: true, projectionRealtime: true }), true); +}); + +test("live Kafka keeps the pre-submit session SSE key when a turn trace becomes active", () => { + const capabilities = { liveKafkaSse: true, projectionRealtime: false }; + const beforeSubmit = workbenchRealtimeScopeKey("ses_live", workbenchRealtimeTraceIdForCapabilities(capabilities, null, null)); + const afterSubmit = workbenchRealtimeScopeKey("ses_live", workbenchRealtimeTraceIdForCapabilities(capabilities, "trc_current_request", "trc_message")); + assert.equal(afterSubmit, beforeSubmit); + assert.equal(workbenchRealtimeTraceIdForCapabilities({ liveKafkaSse: false, projectionRealtime: true }, "trc_current_request", "trc_message"), "trc_current_request"); +}); + test("Error runtime owns Workbench message diagnostic view model", () => { const degraded = messageDiagnosticView(agentMessage({ status: "running", @@ -416,6 +438,53 @@ test("realtime event reducer classifies SSE payloads before store side effects", assert.equal(error.diagnostic.traceId, "trc_2"); }); +test("live transport connected and heartbeat frames do not extend business activity timeouts", () => { + assert.equal(workbenchRealtimeEventIsBusinessActivity({ type: "connected" }, "workbench.connected", true), false); + assert.equal(workbenchRealtimeEventIsBusinessActivity({ type: "heartbeat" }, "workbench.heartbeat", true), false); + assert.equal(workbenchRealtimeEventIsBusinessActivity({ schema: "hwlab.event.v1", event: { type: "assistant" } }, "hwlab.event.v1", true), true); + assert.equal(workbenchRealtimeEventIsBusinessActivity({ type: "trace.event" }, "workbench.trace.event", false), true); +}); + +test("live hwlab envelope projects assistant, tool/output, and terminal without replay or finalizer", () => { + const envelope = (event: Record) => ({ + schema: "hwlab.event.v1", + eventType: "hwlab.trace.event.projected", + eventId: `hwlab:${String(event.sourceEventId ?? event.type)}`, + hwlabSessionId: "ses_live_web", + sessionId: "ses_live_web", + traceId: "trc_live_web", + runId: "run_live_web", + commandId: "cmd_live_web", + context: { runId: "run_live_web", commandId: "cmd_live_web", valuesRedacted: true }, + event + }); + const assistantEnvelope = envelope({ type: "assistant", eventType: "assistant", sourceEventId: "evt_assistant", traceId: "trc_live_web", sessionId: "ses_live_web", status: "running", assistantText: "running increment", terminal: false }); + const assistant = reduceWorkbenchRealtimeEvent(assistantEnvelope, "hwlab.event.v1"); + assert.equal(assistant.action.type, "trace.event"); + let visible = reduceWorkbenchLiveKafkaMessageState({ text: "", status: "running", terminal: false }, assistantEnvelope.event); + assert.deepEqual(visible, { text: "running increment", status: "running", terminal: false }); + + const toolEnvelope = envelope({ type: "tool", eventType: "tool", sourceEventId: "evt_tool", traceId: "trc_live_web", sessionId: "ses_live_web", status: "completed", toolName: "commandExecution", outputSummary: "ok", terminal: false }); + assert.equal(reduceWorkbenchRealtimeEvent(toolEnvelope, "hwlab.event.v1").action.type, "trace.event"); + visible = reduceWorkbenchLiveKafkaMessageState(visible, toolEnvelope.event); + assert.deepEqual(visible, { text: "running increment", status: "running", terminal: false }); + + const outputEnvelope = envelope({ type: "output", eventType: "status", sourceEventId: "evt_output", traceId: "trc_live_web", sessionId: "ses_live_web", status: "running", message: "stdout increment", terminal: false }); + visible = reduceWorkbenchLiveKafkaMessageState(visible, outputEnvelope.event); + assert.deepEqual(visible, { text: "running increment", status: "running", terminal: false }); + + const terminalEnvelope = envelope({ type: "result", eventType: "terminal", sourceEventId: "evt_terminal", traceId: "trc_live_web", sessionId: "ses_live_web", status: "completed", terminal: true }); + visible = reduceWorkbenchLiveKafkaMessageState(visible, terminalEnvelope.event); + assert.deepEqual(visible, { text: "running increment", status: "completed", terminal: true }); + assert.equal(workbenchLiveKafkaMessageId("trc_live_web"), "msg_live_trc_live_web"); + + const failed = reduceWorkbenchLiveKafkaMessageState( + { text: "partial response", status: "running", terminal: false }, + { type: "result", eventType: "terminal", status: "failed", message: "provider stream disconnected", terminal: true } + ); + assert.deepEqual(failed, { text: "provider stream disconnected", status: "failed", terminal: true }); +}); + test("realtime apply planner turns reducer actions into store steps", () => { const reduced = reduceWorkbenchRealtimeEvent(realtimeEvent({ type: "trace.event", traceId: "trc_1", event: { traceId: "trc_1", label: "delta" }, snapshot: { traceId: "trc_1", status: "running" }, entity: { family: "traceEvents", id: "trc_1:1", version: 1, projectionRevision: "prj_1" } }), "workbench.trace.event"); const tracePlan = planWorkbenchRealtimeApply(reduced.action); diff --git a/web/hwlab-cloud-web/src/api/index.ts b/web/hwlab-cloud-web/src/api/index.ts index c941e47a..bfabd7a6 100644 --- a/web/hwlab-cloud-web/src/api/index.ts +++ b/web/hwlab-cloud-web/src/api/index.ts @@ -4,7 +4,7 @@ export { fetchJson, fetchText, type ActivityRef, type ActivityRefSource, type ApiRequestOptions } from "./client"; export { authAPI, HWLAB_WEB_SESSION_COOKIE } from "./auth"; export { workbenchAPI } from "./workbench"; -export { workbenchDebugAPI, connectWorkbenchDebugFakeSse, connectWorkbenchKafkaSseDebug, type WorkbenchDebugFakeSseQueue, type WorkbenchDebugFakeSseResponse, type WorkbenchDebugFakeSseSequence, type WorkbenchDebugFakeSseStream, type WorkbenchKafkaSseDebugEvent, type WorkbenchKafkaSseDebugFilters } from "./workbench-debug"; +export { workbenchDebugAPI, connectWorkbenchDebugFakeSse, connectWorkbenchKafkaSseDebug, workbenchKafkaSseDebugCorrelationIds, type WorkbenchDebugFakeSseQueue, type WorkbenchDebugFakeSseResponse, type WorkbenchDebugFakeSseSequence, type WorkbenchDebugFakeSseStream, type WorkbenchKafkaSseDebugEvent, type WorkbenchKafkaSseDebugStream, type WorkbenchKafkaSseDebugStreamName } from "./workbench-debug"; export { connectWorkbenchEvents, type WorkbenchEventStream, type WorkbenchRealtimeEvent } from "./workbench-events"; export { agentAPI } from "./agent"; export { hwpodAPI } from "./hwpod"; diff --git a/web/hwlab-cloud-web/src/api/workbench-debug.test.ts b/web/hwlab-cloud-web/src/api/workbench-debug.test.ts new file mode 100644 index 00000000..20991c39 --- /dev/null +++ b/web/hwlab-cloud-web/src/api/workbench-debug.test.ts @@ -0,0 +1,106 @@ +import assert from "node:assert/strict"; +import { test } from "bun:test"; + +import { connectWorkbenchKafkaSseDebug, workbenchKafkaSseDebugCorrelationIds, workbenchKafkaSseDebugPath } from "./workbench-debug"; + +test("Kafka SSE debug path preserves stream and explicit correlation filters", () => { + assert.equal( + workbenchKafkaSseDebugPath({ + stream: "agentrun", + fromBeginning: true, + traceId: "trc_debug", + sessionId: "ses_debug", + runId: "run_debug", + commandId: "cmd_debug" + }), + "/v1/workbench/debug/kafka-sse/events?stream=agentrun&fromBeginning=true&traceId=trc_debug&sessionId=ses_debug&runId=run_debug&commandId=cmd_debug" + ); +}); + +test("Kafka SSE debug correlation ids normalize stdio snake_case metadata", () => { + assert.deepEqual( + workbenchKafkaSseDebugCorrelationIds({ + value: { + trace_id: "trc_stdio_debug", + metadata: { + session_id: "ses_stdio_debug", + run_id: "run_stdio_debug", + command_id: "cmd_stdio_debug" + } + } + }), + ["trc_stdio_debug", "ses_stdio_debug", "run_stdio_debug", "cmd_stdio_debug"] + ); +}); + +test("Kafka SSE debug client reports open from named consumer-ready and error from named Kafka error", () => { + const originalEventSource = globalThis.EventSource; + try { + globalThis.EventSource = FakeEventSource as unknown as typeof EventSource; + const opened: string[] = []; + const errors: string[] = []; + const events: string[] = []; + const stream = connectWorkbenchKafkaSseDebug({ + stream: "hwlab", + onOpen: () => opened.push("open"), + onError: (event) => errors.push(event.type), + onEvent: (_event, eventName) => events.push(eventName) + }); + assert.ok(stream); + const source = FakeEventSource.latest; + assert.ok(source); + + source.emitTransportOpen(); + assert.deepEqual(opened, []); + + source.emitNamed("hwlab.kafka.connected", { ok: true, consumerReady: false, groupId: "not-ready" }); + assert.deepEqual(opened, []); + source.emitNamed("hwlab.kafka.connected", { ok: true, consumerReady: true, groupId: "hwlab-v03-debug-sse-ready" }); + assert.deepEqual(opened, ["open"]); + + source.emitNamed("hwlab.kafka.error", { ok: false, error: { code: "kafka_sse_error", message: "consumer stopped" } }); + assert.deepEqual(errors, ["hwlab.kafka.error"]); + assert.deepEqual(events, ["hwlab.kafka.connected", "hwlab.kafka.connected", "hwlab.kafka.error"]); + stream.close(); + assert.equal(source.closed, true); + } finally { + globalThis.EventSource = originalEventSource; + FakeEventSource.latest = null; + } +}); + +class FakeEventSource { + static latest: FakeEventSource | null = null; + onopen: ((event: Event) => void) | null = null; + onerror: ((event: Event) => void) | null = null; + onmessage: ((event: MessageEvent) => void) | null = null; + closed = false; + private listeners = new Map void>>(); + + constructor(_url: string | URL, _eventSourceInitDict?: EventSourceInit) { + FakeEventSource.latest = this; + } + + addEventListener(name: string, listener: (event: MessageEvent) => void) { + const listeners = this.listeners.get(name) ?? new Set(); + listeners.add(listener); + this.listeners.set(name, listeners); + } + + removeEventListener(name: string, listener: (event: MessageEvent) => void) { + this.listeners.get(name)?.delete(listener); + } + + close() { + this.closed = true; + } + + emitTransportOpen() { + this.onopen?.(new Event("open")); + } + + emitNamed(name: string, payload: Record) { + const event = new MessageEvent(name, { data: JSON.stringify(payload) }); + for (const listener of this.listeners.get(name) ?? []) listener(event); + } +} diff --git a/web/hwlab-cloud-web/src/api/workbench-debug.ts b/web/hwlab-cloud-web/src/api/workbench-debug.ts index 68134dc8..a3b61806 100644 --- a/web/hwlab-cloud-web/src/api/workbench-debug.ts +++ b/web/hwlab-cloud-web/src/api/workbench-debug.ts @@ -1,5 +1,5 @@ // SPEC: PJ2026-010401080313 Workbench实时权威 draft-2026-07-09-p1-single-step-debug. -// Responsibility: Cloud Web client for Workbench debug-only fake SSE queues. +// Responsibility: Cloud Web clients for Workbench fake SSE and raw Kafka stream diagnostics. import { fetchJson, type ApiRequestOptions } from "@/api/client"; import type { ApiResult } from "@/types"; @@ -50,39 +50,43 @@ export interface WorkbenchDebugFakeSseStream { close: () => void; } -export interface WorkbenchKafkaSseDebugFilters { - traceId?: string; - sessionId?: string; - runId?: string; - commandId?: string; -} - export type WorkbenchKafkaSseDebugStreamName = "stdio" | "agentrun" | "hwlab"; export interface WorkbenchKafkaSseDebugEvent { ok?: boolean; contractVersion?: string; - stream?: string; + consumerReady?: boolean; + groupId?: string; + stream?: WorkbenchKafkaSseDebugStreamName; topic?: string; partition?: number; offset?: string; key?: string | null; timestamp?: string | null; - value?: Record; - filters?: WorkbenchKafkaSseDebugFilters; + value?: unknown; + filters?: Record; + resolvedFilters?: Record; serverSentAt?: string; - error?: { code?: string; message?: string; [key: string]: unknown }; valuesPrinted?: boolean; + error?: { code?: string; message?: string; [key: string]: unknown }; } -export interface WorkbenchKafkaSseDebugStreamOptions extends WorkbenchKafkaSseDebugFilters { - stream?: WorkbenchKafkaSseDebugStreamName; +export interface WorkbenchKafkaSseDebugStreamOptions { + stream: WorkbenchKafkaSseDebugStreamName; fromBeginning?: boolean; + traceId?: string | null; + sessionId?: string | null; + runId?: string | null; + commandId?: string | null; onEvent: (event: WorkbenchKafkaSseDebugEvent, eventName: string) => void; onOpen?: () => void; onError?: (event: Event) => void; } +export interface WorkbenchKafkaSseDebugStream { + close: () => void; +} + const DEBUG_FAKE_SSE_EVENTS = [ "workbench.connected", "workbench.trace.snapshot", @@ -93,6 +97,8 @@ const DEBUG_FAKE_SSE_EVENTS = [ "workbench.error" ]; +const DEBUG_KAFKA_SSE_EVENTS = ["hwlab.kafka.connected", "hwlab.kafka.event", "hwlab.kafka.error"]; + export const workbenchDebugAPI = { describe: (queueId = "trace-card", options: ApiRequestOptions = {}): Promise> => fetchJson(debugPath("", { queueId }), { ...options, timeoutName: "workbench debug fake sse" }), reset: (input: { queueId: string; sequenceId: string }, options: ApiRequestOptions = {}): Promise> => fetchJson(debugPath("/reset"), { ...options, method: "POST", body: JSON.stringify(input), timeoutName: "workbench debug fake sse reset" }), @@ -126,29 +132,23 @@ export function connectWorkbenchDebugFakeSse(options: WorkbenchDebugFakeSseStrea }; } -export function connectWorkbenchKafkaSseDebug(options: WorkbenchKafkaSseDebugStreamOptions): WorkbenchDebugFakeSseStream | null { +export function connectWorkbenchKafkaSseDebug(options: WorkbenchKafkaSseDebugStreamOptions): WorkbenchKafkaSseDebugStream | null { if (typeof EventSource === "undefined") return null; - const source = new EventSource(kafkaDebugPath("/events", { - stream: options.stream || "hwlab", - traceId: options.traceId, - sessionId: options.sessionId, - runId: options.runId, - commandId: options.commandId, - fromBeginning: options.fromBeginning ? "true" : undefined - }), { withCredentials: true }); - source.onopen = () => options.onOpen?.(); + const source = new EventSource(workbenchKafkaSseDebugPath(options), { withCredentials: true }); source.onerror = (event) => options.onError?.(event); - const names = ["hwlab.kafka.connected", "hwlab.kafka.event", "hwlab.kafka.error"]; - const listeners = names.map((name) => { + const listeners = DEBUG_KAFKA_SSE_EVENTS.map((name) => { const listener = (event: MessageEvent) => { - const payload = parseKafkaEvent(event.data); - if (payload) options.onEvent(payload, name); + const payload = parseKafkaSseDebugEvent(event.data); + if (!payload) return; + options.onEvent(payload, name); + if (name === "hwlab.kafka.connected" && payload.consumerReady === true) options.onOpen?.(); + if (name === "hwlab.kafka.error") options.onError?.(event); }; source.addEventListener(name, listener); return { name, listener }; }); source.onmessage = (event) => { - const payload = parseKafkaEvent(event.data); + const payload = parseKafkaSseDebugEvent(event.data); if (payload) options.onEvent(payload, "message"); }; return { @@ -159,6 +159,31 @@ export function connectWorkbenchKafkaSseDebug(options: WorkbenchKafkaSseDebugStr }; } +export function workbenchKafkaSseDebugPath(options: Pick): string { + const query = new URLSearchParams({ stream: options.stream }); + if (options.fromBeginning === true) query.set("fromBeginning", "true"); + appendDebugFilter(query, "traceId", options.traceId); + appendDebugFilter(query, "sessionId", options.sessionId); + appendDebugFilter(query, "runId", options.runId); + appendDebugFilter(query, "commandId", options.commandId); + return `/v1/workbench/debug/kafka-sse/events?${query.toString()}`; +} + +export function workbenchKafkaSseDebugCorrelationIds(event: WorkbenchKafkaSseDebugEvent): string[] { + const value = recordValue(event.value); + const context = recordValue(value.context); + const metadata = recordValue(value.metadata); + const run = recordValue(value.run); + const command = recordValue(value.command); + const payload = recordValue(value.payload); + return uniqueStrings([ + value.traceId, value.trace_id, context.traceId, context.trace_id, metadata.traceId, metadata.trace_id, payload.traceId, payload.trace_id, + value.hwlabSessionId, value.sessionId, value.session_id, context.hwlabSessionId, context.sessionId, context.session_id, metadata.sessionId, metadata.session_id, payload.hwlabSessionId, payload.sessionId, payload.session_id, + value.runId, value.run_id, context.runId, context.run_id, metadata.runId, metadata.run_id, run.runId, run.run_id, payload.runId, payload.run_id, + value.commandId, value.command_id, context.commandId, context.command_id, metadata.commandId, metadata.command_id, command.commandId, command.command_id, payload.commandId, payload.command_id + ]); +} + function debugPath(suffix: string, params: Record = {}): string { const query = new URLSearchParams(); for (const [key, value] of Object.entries(params)) { @@ -169,16 +194,6 @@ function debugPath(suffix: string, params: Record = {}): string { - const query = new URLSearchParams(); - for (const [key, value] of Object.entries(params)) { - const text = typeof value === "string" ? value.trim() : ""; - if (text) query.set(key, text); - } - const qs = query.toString(); - return `/v1/workbench/debug/kafka-sse${suffix}${qs ? `?${qs}` : ""}`; -} - function parseRealtimeEvent(raw: string): WorkbenchRealtimeEvent | null { try { const value = JSON.parse(raw); @@ -188,11 +203,24 @@ function parseRealtimeEvent(raw: string): WorkbenchRealtimeEvent | null { } } -function parseKafkaEvent(raw: string): WorkbenchKafkaSseDebugEvent | null { +function parseKafkaSseDebugEvent(raw: string): WorkbenchKafkaSseDebugEvent | null { try { const value = JSON.parse(raw); - return value && typeof value === "object" ? value as WorkbenchKafkaSseDebugEvent : null; + return value && typeof value === "object" && !Array.isArray(value) ? value as WorkbenchKafkaSseDebugEvent : null; } catch { return null; } } + +function appendDebugFilter(query: URLSearchParams, name: string, value: string | null | undefined): void { + const text = typeof value === "string" ? value.trim() : ""; + if (text) query.set(name, text); +} + +function recordValue(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} + +function uniqueStrings(values: unknown[]): string[] { + return [...new Set(values.map((value) => typeof value === "string" ? value.trim() : "").filter(Boolean))]; +} diff --git a/web/hwlab-cloud-web/src/api/workbench-events.test.ts b/web/hwlab-cloud-web/src/api/workbench-events.test.ts index 33273739..7d5af779 100644 --- a/web/hwlab-cloud-web/src/api/workbench-events.test.ts +++ b/web/hwlab-cloud-web/src/api/workbench-events.test.ts @@ -2,6 +2,7 @@ import assert from "node:assert/strict"; import { test } from "bun:test"; import { realtimeCoalesceKey, type WorkbenchRealtimeEvent } from "./workbench-events"; +import { createCoalescedEventQueue } from "../utils/scheduler/coalesced-event-queue"; test("turn snapshot coalescing keys by trace instead of per sequence", () => { const first: WorkbenchRealtimeEvent = { type: "turn.snapshot", turn: { sessionId: "ses_queue", traceId: "trc_queue", status: "running" }, cursor: { traceSeq: 10 } }; @@ -16,3 +17,25 @@ test("trace events keep sequence-specific coalescing keys", () => { assert.notEqual(realtimeCoalesceKey(first, "workbench.trace.event"), realtimeCoalesceKey(next, "workbench.trace.event")); }); + +test("live Kafka envelopes from one trace retain every assistant, tool, output, and terminal event", () => { + const delivered: WorkbenchRealtimeEvent[] = []; + const queue = createCoalescedEventQueue({ + keyOf: (event) => realtimeCoalesceKey(event, "hwlab.event.v1"), + onFlush: (events) => delivered.push(...events) + }); + for (const sourceEventId of ["evt_assistant", "evt_tool", "evt_output", "evt_terminal"]) { + queue.push({ + schema: "hwlab.event.v1", + eventId: `hwlab:${sourceEventId}`, + sourceEventId, + sessionId: "ses_live_queue", + traceId: "trc_live_queue", + event: { sourceEventId, traceId: "trc_live_queue", sessionId: "ses_live_queue" } + }); + } + + queue.flush(); + + assert.deepEqual(delivered.map((event) => event.sourceEventId), ["evt_assistant", "evt_tool", "evt_output", "evt_terminal"]); +}); diff --git a/web/hwlab-cloud-web/src/api/workbench-events.ts b/web/hwlab-cloud-web/src/api/workbench-events.ts index 0922b813..12ba9def 100644 --- a/web/hwlab-cloud-web/src/api/workbench-events.ts +++ b/web/hwlab-cloud-web/src/api/workbench-events.ts @@ -6,6 +6,7 @@ import type { ApiResult, ChatMessage, ProjectionDiagnostic, TraceEvent } from "@ import { createCoalescedEventQueue } from "@/utils/scheduler/coalesced-event-queue"; import { composeWorkbenchScopedKey, firstScopePart } from "@/utils/workbench-key"; import { recordWorkbenchRuntimeDiagnostic, recordWorkbenchSseLifecycle } from "@/utils/workbench-performance"; +import type { WorkbenchRealtimeCapabilities } from "@/config/runtime"; export interface WorkbenchRealtimeTraceSnapshot { traceId?: string | null; @@ -23,15 +24,26 @@ export interface WorkbenchRealtimeTraceSnapshot { } export interface WorkbenchRealtimeEvent { + schema?: string | null; + eventType?: string | null; + eventId?: string | null; + sourceEventId?: string | null; + capabilities?: WorkbenchRealtimeCapabilities | null; + liveOnly?: boolean | null; + replay?: boolean | null; + lossPossible?: boolean | null; type?: string; contractVersion?: string | null; realtimeAuthority?: string | null; status?: string; serverSentAt?: string | null; eventCreatedAt?: string | null; + hwlabSessionId?: string | null; sessionId?: string | null; threadId?: string | null; traceId?: string | null; + runId?: string | null; + commandId?: string | null; event?: TraceEvent | null; snapshot?: WorkbenchRealtimeTraceSnapshot | null; message?: ChatMessage | null; @@ -88,6 +100,7 @@ export interface WorkbenchSyncReplayResponse { } export interface WorkbenchEventStreamOptions { + realtimeCapabilities: WorkbenchRealtimeCapabilities; sessionId?: string | null; traceId?: string | null; afterSeq?: number | null; @@ -109,6 +122,7 @@ interface QueuedRealtimeEvent { } const WORKBENCH_EVENT_NAMES = [ + "hwlab.event.v1", "workbench.connected", "workbench.trace.snapshot", "workbench.trace.event", @@ -121,11 +135,7 @@ const WORKBENCH_EVENT_NAMES = [ export function connectWorkbenchEvents(options: WorkbenchEventStreamOptions): WorkbenchEventStream | null { if (typeof EventSource === "undefined") return null; - const params = new URLSearchParams(); - appendParam(params, "sessionId", options.sessionId); - appendParam(params, "traceId", options.traceId); - appendNumberParam(params, "afterSeq", options.afterSeq); - const eventRoute = `/v1/workbench/events?${params.toString()}`; + const eventRoute = workbenchEventStreamPath(options); const connectStartedAt = typeof performance === "undefined" ? Date.now() : performance.now(); recordWorkbenchSseLifecycle({ state: "connect", route: eventRoute, sessionId: options.sessionId, traceId: options.traceId }); const source = new EventSource(eventRoute, { withCredentials: true }); @@ -192,6 +202,22 @@ export function connectWorkbenchEvents(options: WorkbenchEventStreamOptions): Wo }; } +export function workbenchEventStreamPath(options: Pick): string { + const params = new URLSearchParams(); + appendParam(params, "sessionId", options.sessionId); + appendParam(params, "traceId", options.traceId); + if (!options.realtimeCapabilities.liveKafkaSse && options.realtimeCapabilities.projectionRealtime) appendNumberParam(params, "afterSeq", options.afterSeq); + return `/v1/workbench/events?${params.toString()}`; +} + +export function workbenchProjectionEventStreamPath(options: Pick): string { + const params = new URLSearchParams(); + appendParam(params, "sessionId", options.sessionId); + appendParam(params, "traceId", options.traceId); + appendNumberParam(params, "afterSeq", options.afterSeq); + return `/v1/workbench/projection-events?${params.toString()}`; +} + export async function fetchWorkbenchSyncReplay(input: WorkbenchSyncReplayRequest, options: ApiRequestOptions = {}): Promise> { return fetchJson(workbenchSyncReplayPath(input), { ...options, @@ -238,6 +264,10 @@ function scheduleRealtimeFlushYield(flush: () => void, yieldMs: number | null | } export function realtimeCoalesceKey(event: WorkbenchRealtimeEvent, eventName: string): string | null { + if (eventName === "hwlab.event.v1" || event.schema === "hwlab.event.v1") { + const eventId = firstScopePart(event.eventId, event.sourceEventId, event.event?.sourceEventId); + return eventId ? composeWorkbenchScopedKey("workbench.sse.live-kafka-event", eventId) : null; + } const entity = event.entity; const entityFamily = firstScopePart(entity?.family); const entityId = firstScopePart(entity?.id); diff --git a/web/hwlab-cloud-web/src/composables/workbench-trace-snapshot.test.ts b/web/hwlab-cloud-web/src/composables/workbench-trace-snapshot.test.ts index 8cfc0ffa..5c1d9765 100644 --- a/web/hwlab-cloud-web/src/composables/workbench-trace-snapshot.test.ts +++ b/web/hwlab-cloud-web/src/composables/workbench-trace-snapshot.test.ts @@ -47,3 +47,22 @@ test("mergeRunnerTrace can seal terminal metadata without requiring trace rows", assert.equal(merged.status, "completed"); assert.equal(merged.events, previousEvents); }); + +test("mergeRunnerTrace accumulates live Kafka deltas with an honest source and event count", () => { + const previous = trace({ + eventSource: "hwlab-kafka-sse", + eventCount: 1, + events: [{ source: "agentrun.kafka", sourceSeq: 1, type: "assistant", label: "assistant" }] + }); + const next = trace({ + eventSource: "hwlab-kafka-sse", + eventCount: 1, + events: [{ source: "agentrun.kafka", sourceSeq: 2, type: "tool", label: "tool" }] + }); + + const merged = mergeRunnerTrace(previous, next); + + assert.equal(merged.eventSource, "hwlab-kafka-sse"); + assert.equal(merged.eventCount, 2); + assert.deepEqual(merged.events?.map((event) => event.sourceSeq), [1, 2]); +}); diff --git a/web/hwlab-cloud-web/src/composables/workbench-trace-snapshot.ts b/web/hwlab-cloud-web/src/composables/workbench-trace-snapshot.ts index 32b5073d..80ee4d03 100644 --- a/web/hwlab-cloud-web/src/composables/workbench-trace-snapshot.ts +++ b/web/hwlab-cloud-web/src/composables/workbench-trace-snapshot.ts @@ -91,7 +91,9 @@ export function mergeRunnerTrace(previous: ChatMessage["runnerTrace"], next: Non ? previousEvents.length > 0 ? mergeTraceEvents(previousEvents, nextEvents, { previousCursor, nextRange: next.range }) : nextEvents : keepPreviousEvents && previousAuthoritative && next.eventsCompacted === true ? previousEvents : keepPreviousEvents ? mergeTraceEvents(previousEvents, nextEvents, { previousCursor, nextRange: next.range }) : nextEvents.length > previousEvents.length ? mergeTraceEvents(previousEvents, nextEvents, { previousCursor, nextRange: next.range }) : nextEvents; - const eventCount = keepPreviousEvents ? previous.eventCount ?? events.length : next.eventCount ?? previous.eventCount ?? events.length; + const eventCount = next.eventSource === "hwlab-kafka-sse" + ? events.length + : keepPreviousEvents ? previous.eventCount ?? events.length : next.eventCount ?? previous.eventCount ?? events.length; const timing = mergeTraceTimingProjection(previous, next); return { ...previous, diff --git a/web/hwlab-cloud-web/src/config/runtime.ts b/web/hwlab-cloud-web/src/config/runtime.ts index adf9018a..673cb10c 100644 --- a/web/hwlab-cloud-web/src/config/runtime.ts +++ b/web/hwlab-cloud-web/src/config/runtime.ts @@ -10,6 +10,11 @@ export interface OpenCodeFrameConfig { url: string; } +export interface WorkbenchRealtimeCapabilities { + liveKafkaSse: boolean; + projectionRealtime: boolean; +} + const DEFAULT_DISPLAY_DATE_TIME_OPTIONS: Intl.DateTimeFormatOptions = { year: "numeric", month: "2-digit", @@ -80,6 +85,14 @@ export function workbenchTraceTimelinePolicy(): WorkbenchTraceTimelinePolicy { }; } +export function workbenchRealtimeCapabilities(): WorkbenchRealtimeCapabilities { + const features = window.HWLAB_CLOUD_WEB_CONFIG?.workbench?.realtimeFeatures; + if (typeof features?.liveKafkaSse !== "boolean" || typeof features?.projectionRealtime !== "boolean") { + throw new Error("HWLAB Cloud Web workbench.realtimeFeatures are required"); + } + return { liveKafkaSse: features.liveKafkaSse, projectionRealtime: features.projectionRealtime }; +} + export function traceExplorerHref(traceId: string | null | undefined): string | null { const safeTraceId = normalizedTraceId(traceId); if (!safeTraceId) return null; diff --git a/web/hwlab-cloud-web/src/config/workbench-runtime-policy.ts b/web/hwlab-cloud-web/src/config/workbench-runtime-policy.ts index 800f16bf..ac94a12c 100644 --- a/web/hwlab-cloud-web/src/config/workbench-runtime-policy.ts +++ b/web/hwlab-cloud-web/src/config/workbench-runtime-policy.ts @@ -29,8 +29,6 @@ export interface WorkbenchRuntimePolicy { workbenchRealtimeFlushMaxItemsPerChunk: number; workbenchRealtimeFlushMaxChunkMs: number; workbenchRealtimeFlushYieldMs: number; - workbenchActiveTraceSyncReplayInitialMs: number; - workbenchActiveTraceSyncReplayRepeatMs: number; } const DEFAULT_WORKBENCH_RUNTIME_POLICY: WorkbenchRuntimePolicy = Object.freeze({ @@ -60,9 +58,7 @@ const DEFAULT_WORKBENCH_RUNTIME_POLICY: WorkbenchRuntimePolicy = Object.freeze({ workbenchRealtimeErrorSyncReplayMinMs: 2_000, workbenchRealtimeFlushMaxItemsPerChunk: 4, workbenchRealtimeFlushMaxChunkMs: 8, - workbenchRealtimeFlushYieldMs: 0, - workbenchActiveTraceSyncReplayInitialMs: 2_500, - workbenchActiveTraceSyncReplayRepeatMs: 5_000 + workbenchRealtimeFlushYieldMs: 0 }); export function workbenchRuntimePolicy(input: unknown = runtimePolicyConfig()): WorkbenchRuntimePolicy { @@ -94,9 +90,7 @@ export function workbenchRuntimePolicy(input: unknown = runtimePolicyConfig()): workbenchRealtimeErrorSyncReplayMinMs: nonNegativeNumber(source.workbenchRealtimeErrorSyncReplayMinMs ?? source.workbenchRealtimeErrorGapFillMinMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchRealtimeErrorSyncReplayMinMs), workbenchRealtimeFlushMaxItemsPerChunk: positiveInteger(source.workbenchRealtimeFlushMaxItemsPerChunk, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchRealtimeFlushMaxItemsPerChunk), workbenchRealtimeFlushMaxChunkMs: nonNegativeNumber(source.workbenchRealtimeFlushMaxChunkMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchRealtimeFlushMaxChunkMs), - workbenchRealtimeFlushYieldMs: nonNegativeNumber(source.workbenchRealtimeFlushYieldMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchRealtimeFlushYieldMs), - workbenchActiveTraceSyncReplayInitialMs: nonNegativeNumber(source.workbenchActiveTraceSyncReplayInitialMs ?? source.workbenchActiveTraceRestGapFillInitialMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchActiveTraceSyncReplayInitialMs), - workbenchActiveTraceSyncReplayRepeatMs: nonNegativeNumber(source.workbenchActiveTraceSyncReplayRepeatMs ?? source.workbenchActiveTraceRestGapFillRepeatMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchActiveTraceSyncReplayRepeatMs) + workbenchRealtimeFlushYieldMs: nonNegativeNumber(source.workbenchRealtimeFlushYieldMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchRealtimeFlushYieldMs) }; } diff --git a/web/hwlab-cloud-web/src/stores/workbench-colada.test.ts b/web/hwlab-cloud-web/src/stores/workbench-colada.test.ts index 128e7b1c..185eb3d2 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-colada.test.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-colada.test.ts @@ -61,7 +61,8 @@ test("workbench colada sync replay is the only live repair query scope", () => { const storeSource = fs.readFileSync(path.join(storeDir, "workbench.ts"), "utf8"); assert.match(querySource, /fetchSyncReplay: \(input, options = \{\}\) => runWorkbenchQuery\(queryCache, workbenchColadaKeys\.syncReplay\(input\),/u); - assert.match(storeSource, /workbenchColadaQueries\.fetchSyncReplay\(\{ sessionId, traceId, since: sinceOutboxSeq \}/u); + assert.match(storeSource, /workbenchColadaQueries\.fetchSyncReplay\(\{ sessionId, traceId, since: cursor \}/u); + assert.match(storeSource, /responseCursor\?\.hasMore !== true/u); assert.doesNotMatch(storeSource, /import \{ fetchWorkbenchSyncReplay \} from "@\/api\/workbench-events"/u); assert.match(mutationSource, /workbenchColadaKeys\.syncRoot\(\)/u); @@ -135,7 +136,7 @@ test("workbench sync replay projects durable message and turn facts into observe if (action.type === "message.snapshot") { state = reduceWorkbenchServerState(state, { type: "message.snapshot", sessionId: action.realtimeEvent.sessionId, message: action.realtimeEvent.message }); } else if (action.type === "turn.snapshot") { - state = reduceWorkbenchServerState(state, { type: "turn.status", turn: action.turn as TurnStatusAuthority }); + state = reduceWorkbenchServerState(state, { type: "turn.status", turn: action.turn as unknown as TurnStatusAuthority }); } } diff --git a/web/hwlab-cloud-web/src/stores/workbench-debug-fake-sse.ts b/web/hwlab-cloud-web/src/stores/workbench-debug-fake-sse.ts index 05046984..3182dd45 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-debug-fake-sse.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-debug-fake-sse.ts @@ -79,7 +79,11 @@ function applyTraceEvent(state: WorkbenchDebugFakeSseState, traceId: string | nu function applyTraceSnapshot(state: WorkbenchDebugFakeSseState, traceId: string | null, snapshot: WorkbenchRealtimeEvent["snapshot"]): WorkbenchDebugFakeSseState { const id = firstNonEmptyString(traceId, snapshot?.traceId); if (!id || !snapshot) return state; - const message = state.message ?? debugMessageForTrace(id, { sessionId: snapshot.sessionId, threadId: snapshot.threadId, traceId: id }); + const message = state.message ?? debugMessageForTrace(id, { + sessionId: firstNonEmptyString(snapshot.sessionId), + threadId: firstNonEmptyString(snapshot.threadId), + traceId: id + }); const trace = traceFromRealtime(id, snapshot, Array.isArray(snapshot.events) ? snapshot.events : [], message.runnerTrace ?? null); return { ...state, diff --git a/web/hwlab-cloud-web/src/stores/workbench-event-reducer.ts b/web/hwlab-cloud-web/src/stores/workbench-event-reducer.ts index 147611cd..2c81c088 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-event-reducer.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-event-reducer.ts @@ -48,7 +48,15 @@ export function reduceWorkbenchRealtimeEvent(event: WorkbenchRealtimeEvent, even }; } +export function workbenchRealtimeEventIsBusinessActivity(event: WorkbenchRealtimeEvent, eventName: string, liveKafkaSse: boolean): boolean { + if (!liveKafkaSse) return true; + return eventName === "hwlab.event.v1" || event.schema === "hwlab.event.v1"; +} + function reduceRealtimeAction(event: WorkbenchRealtimeEvent, eventName: string): WorkbenchRealtimeAction { + if (event.schema === "hwlab.event.v1" && event.event) { + return { type: "trace.event", traceId: realtimeTraceId(event), event: event.event, snapshot: null, realtimeEvent: event }; + } const authority = primaryAuthority(event); if (authority) return authority; switch (event.type) { @@ -81,5 +89,5 @@ function realtimeTraceId(event: WorkbenchRealtimeEvent): string | null { } function realtimeSessionId(event: WorkbenchRealtimeEvent): string | null { - return firstNonEmptyString(event.sessionId, event.message?.sessionId, event.snapshot?.sessionId, event.event?.sessionId) ?? null; + return firstNonEmptyString(event.hwlabSessionId, event.sessionId, event.message?.sessionId, event.snapshot?.sessionId, event.event?.sessionId) ?? null; } diff --git a/web/hwlab-cloud-web/src/stores/workbench-live-kafka-event.ts b/web/hwlab-cloud-web/src/stores/workbench-live-kafka-event.ts new file mode 100644 index 00000000..baf2343c --- /dev/null +++ b/web/hwlab-cloud-web/src/stores/workbench-live-kafka-event.ts @@ -0,0 +1,50 @@ +// SPEC: PJ2026-0104010803 Workbench live Kafka SSE. +// Responsibility: project one transparent hwlab.event.v1 business event into visible turn state without replay/finalizer/polling. + +import type { TraceEvent } from "@/types"; +import { firstNonEmptyString } from "@/utils"; + +export interface WorkbenchLiveMessageState { + text: string; + status: string; + terminal: boolean; +} + +export function reduceWorkbenchLiveKafkaMessageState(previous: WorkbenchLiveMessageState, event: TraceEvent): WorkbenchLiveMessageState { + const terminal = workbenchLiveKafkaEventIsTerminal(event); + const status = terminal ? firstNonEmptyString(event.status, "completed") ?? "completed" : "running"; + const assistantText = workbenchLiveKafkaAssistantText(event); + return { + text: assistantText ?? (terminal ? workbenchLiveKafkaTerminalText(event, previous.text, status) : previous.text), + status, + terminal + }; +} + +export function workbenchLiveKafkaMessageId(traceId: string): string { + return `msg_live_${traceId}`; +} + +export function workbenchLiveKafkaEventIsTerminal(event: TraceEvent | null | undefined): boolean { + return Boolean(event?.terminal === true || firstNonEmptyString(event?.eventType) === "terminal" || firstNonEmptyString(event?.type) === "result"); +} + +export function workbenchLiveKafkaAssistantText(event: TraceEvent | null | undefined): string | null { + if (!event || firstNonEmptyString(event.type, event.eventType) !== "assistant") return null; + const finalResponse = recordValue(event.finalResponse); + return firstNonEmptyString(finalResponse?.text, event.assistantText, event.text, event.message) ?? null; +} + +export function workbenchLiveKafkaTerminalText(event: TraceEvent, previousText: string, status: string): string { + const finalResponse = recordValue(event.finalResponse); + const error = recordValue(event.error); + const failed = ["failed", "blocked", "timeout", "canceled", "cancelled", "interrupted", "expired"].includes(status); + const text = failed + ? firstNonEmptyString(finalResponse?.text, event.message, error?.message, event.failureKind, event.errorCode, previousText) + : firstNonEmptyString(finalResponse?.text, event.assistantText, event.text, previousText, event.message); + return text ?? `AgentRun ${status}`; +} + +function recordValue(value: unknown): Record | null { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : null; +} diff --git a/web/hwlab-cloud-web/src/stores/workbench-message-projection-runtime.test.ts b/web/hwlab-cloud-web/src/stores/workbench-message-projection-runtime.test.ts index e4ef7799..bb1918a8 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-message-projection-runtime.test.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-message-projection-runtime.test.ts @@ -136,15 +136,13 @@ test("workbench active terminal paths seal final response from turn authority", const source = fs.readFileSync(path.join(storeDir, "workbench.ts"), "utf8"); const projectBlock = source.slice(source.indexOf("function projectTurnAuthorityToMessages"), source.indexOf("async function submitMessage")); const realtimeTurnBlock = source.slice(source.indexOf("function applyRealtimeTurnSnapshot"), source.indexOf("function scheduleRealtimeTurnProjection")); - const realtimeTurnProjectionBlock = source.slice(source.indexOf("function flushRealtimeTurnProjection"), source.indexOf("async function refreshTerminalTraceFromSyncReplay")); - const terminalSyncBlock = source.slice(source.indexOf("async function refreshTerminalTraceFromSyncReplay"), source.indexOf("function installRealtimeVisibilityHandler")); + const realtimeTurnProjectionBlock = source.slice(source.indexOf("function flushRealtimeTurnProjection"), source.indexOf("function installRealtimeVisibilityHandler")); const completeBlock = source.slice(source.indexOf("function completeTrace"), source.indexOf("async function hydrateTerminalMessageDiagnostics")); - const realtimeSessionSyncBlock = source.slice(source.indexOf("async function refreshRealtimeSessionFromSyncReplay"), source.indexOf("function completeTrace")); + const crossTabSyncBlock = source.slice(source.indexOf("async function refreshCrossTabSessionFromSyncReplay"), source.indexOf("function completeTrace")); const sessionDetailReadBlock = source.slice(source.indexOf("function fetchSessionDetailPage"), source.indexOf("function sessionMessageProjectionWindowLimit")); const traceDetailReadBlock = source.slice(source.indexOf("async function readTraceEventsForExplicitDetailPages"), source.indexOf("async function fetchTraceDetailEventsPage")); const loadBlock = source.slice(source.indexOf("async function loadWorkbenchSession"), source.indexOf("async function sealRestoredActiveTurnMessages")); const restoreSealBlock = source.slice(source.indexOf("async function sealRestoredActiveTurnMessages"), source.indexOf("function reattachRestoredActiveTrace")); - const activeSyncReplayBlock = source.slice(source.indexOf("async function refreshActiveTraceFromSyncReplay"), source.indexOf("function stopRealtime")); assert.match(projectBlock, /terminalAuthorityMessageFromTurnResult\(result\)/u); assert.match(projectBlock, /type:\s*"message\.upsert"/u); @@ -155,19 +153,14 @@ test("workbench active terminal paths seal final response from turn authority", assert.match(realtimeTurnBlock, /rememberTurnStatus\(traceId, result\)[\s\S]*scheduleRealtimeTurnProjection\(\{ traceId, result, terminalTurn \}\)/u); assert.doesNotMatch(realtimeTurnBlock, new RegExp(`applyTurnStatusSnapshot\\(|${["refresh", "TerminalTraceFromRest"].join("")}\\(`, "u")); assert.match(realtimeTurnProjectionBlock, /syncTurnStatusToMessage\(next\.traceId, next\.result\)/u); - assert.match(realtimeTurnProjectionBlock, /next\.terminalTurn[\s\S]*refreshTerminalTraceFromSyncReplay\(next\.traceId, "realtime-turn-snapshot"\)/u); assert.match(realtimeTurnProjectionBlock, /workbench_realtime_turn_projection_budget/u); - assert.match(terminalSyncBlock, /refreshWorkbenchSyncReplay\(ownerSessionId, id, null, `terminal-sync-replay:\$\{reason\}`\)/u); - assert.doesNotMatch(terminalSyncBlock, /refreshTurnStatusByTraceId|refreshMessageProjectionForTrace|fetchSessionMessagesPage|readTraceEventsForMessage/u); + assert.doesNotMatch(realtimeTurnProjectionBlock, /refreshWorkbenchSyncReplay|refreshTerminalTraceFromSyncReplay/u); assert.match(completeBlock, /projectTurnAuthorityToMessages\(traceId, result, "complete-trace"\)/u); assert.match(completeBlock, /options\.forceRead[\s\S]*refreshMessageProjectionForTrace\(ownerSessionId, traceId, \{ force: true \}\)/u); - assert.match(completeBlock, /else scheduleActiveTraceSyncReplay\(traceId, "complete-trace-sync-replay", 0\)/u); - assert.match(realtimeSessionSyncBlock, /traceIdFromRealtimeRefreshReason\(reason\)/u); - assert.match(realtimeSessionSyncBlock, /refreshWorkbenchSyncReplay\(id, traceId, null, `realtime-session-sync:\$\{reason\}`\)/u); - assert.doesNotMatch(realtimeSessionSyncBlock, /hydrateTurnStatusAuthority|sessionDetailAutoReadDecision|fetchSessionDetailPage|fetchSessionMessagesPage|refreshMessageProjectionForTrace/u); - assert.doesNotMatch(realtimeSessionSyncBlock, /loadWorkbenchSession|applySelectedSessionDetail/u); - assert.match(activeSyncReplayBlock, /refreshWorkbenchSyncReplay\(ownerSessionId, id, null, `active-sync-replay:\$\{reason\}`\)/u); - assert.doesNotMatch(activeSyncReplayBlock, /refreshTurnStatusByTraceId|refreshMessageProjectionForTrace|fetchSessionMessagesPage|readTraceEventsForMessage/u); + assert.doesNotMatch(completeBlock, /scheduleActiveTraceSyncReplay|refreshWorkbenchSyncReplay/u); + assert.match(crossTabSyncBlock, /traceIdFromRealtimeRefreshReason\(reason\)/u); + assert.match(crossTabSyncBlock, /refreshWorkbenchSyncReplay\(id, traceId, null, `cross-tab-sync:\$\{reason\}`\)/u); + assert.doesNotMatch(source, /scheduleActiveTraceSyncReplay|refreshActiveTraceFromSyncReplay|active-sync-replay:repeat|complete-trace-sync-replay/u); assert.match(traceDetailReadBlock, /traceAuthorityById\.value\[traceId\] \?\? message\.runnerTrace/u); assert.match(traceDetailReadBlock, /traceEventsDetailReadDecision\(traceId, afterProjectedSeq, message, options\)/u); assert.match(traceDetailReadBlock, /trace_events_auto_read_skip/u); @@ -183,27 +176,26 @@ test("workbench active terminal paths seal final response from turn authority", assert.doesNotMatch(restoreSealBlock, /force:\s*true/u); }); -test("workbench automatic live repair does not call legacy REST fan-out", () => { +test("workbench active SSE path never starts periodic sync replay", () => { const source = fs.readFileSync(path.join(storeDir, "workbench.ts"), "utf8"); - const automaticBlocks = [ + const activeBlocks = [ source.slice(source.indexOf("async function submitMessage"), source.indexOf("async function cancelAgentMessage")), source.slice(source.indexOf("function reattachTrace"), source.indexOf("function restartRealtime")), - source.slice(source.indexOf("async function refreshActiveTraceFromSyncReplay"), source.indexOf("function stopRealtime")), source.slice(source.indexOf("function flushRealtimeTurnProjection"), source.indexOf("function installRealtimeVisibilityHandler")), - source.slice(source.indexOf("async function refreshRealtimeSessionFromSyncReplay"), source.indexOf("function completeTrace")), + source.slice(source.indexOf("function completeTrace"), source.indexOf("async function hydrateTerminalMessageDiagnostics")), source.slice(source.indexOf("async function sealRestoredActiveTurnMessages"), source.indexOf("function reattachRestoredActiveTrace")) ].join("\n"); - assert.match(automaticBlocks, /refreshWorkbenchSyncReplay/u); - assert.doesNotMatch(automaticBlocks, /fetchWorkbenchTurnStatus|fetchWorkbenchTraceEvents|fetchSessionMessagesPage|fetchSessionDetailPage/u); - assert.doesNotMatch(automaticBlocks, /refreshTurnStatusByTraceId|refreshMessageProjectionForTrace|readTraceEventsForMessage\(/u); + assert.doesNotMatch(activeBlocks, /refreshWorkbenchSyncReplay|scheduleActiveTraceSyncReplay|setTimeout|setInterval/u); + assert.match(source.slice(source.indexOf("function executeRealtimeRecoveryStep"), source.indexOf("function stopRealtime")), /case "sync-replay"[\s\S]*refreshWorkbenchSyncReplay/u); + assert.match(source.slice(source.indexOf("async function refreshCrossTabSessionFromSyncReplay"), source.indexOf("function completeTrace")), /refreshWorkbenchSyncReplay/u); }); test("cross page projection signal always replays durable sync", () => { const source = fs.readFileSync(path.join(storeDir, "workbench.ts"), "utf8"); const signalBlock = source.slice(source.indexOf("function handleWorkbenchProjectionSignal"), source.indexOf("function applyTraceSnapshot")); - assert.match(signalBlock, /refreshRealtimeSessionFromSyncReplay\(sessionId, `cross-tab-session-projection:/u); + assert.match(signalBlock, /refreshCrossTabSessionFromSyncReplay\(sessionId, `cross-tab-session-projection:/u); assert.doesNotMatch(signalBlock, /messages\.value\.some/u); - assert.doesNotMatch(signalBlock, /return;\s*\n\s*void refreshRealtimeSessionFromSyncReplay/u); + assert.doesNotMatch(signalBlock, /return;\s*\n\s*void refreshCrossTabSessionFromSyncReplay/u); }); diff --git a/web/hwlab-cloud-web/src/stores/workbench-server-state.ts b/web/hwlab-cloud-web/src/stores/workbench-server-state.ts index bf77b27f..e4d641ec 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-server-state.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-server-state.ts @@ -200,8 +200,9 @@ function reduceMessageUpsert(state: WorkbenchServerState, sessionId: string | nu const existingSession = state.sessionsById[id]; const existing = state.messagesBySessionId[id] ?? existingSession?.messages ?? []; const index = existing.findIndex((item) => messageMatchesSnapshot(item, message)); - const mergedMessage = index >= 0 ? mergeMessageSnapshot(existing[index], message) : message; - if (index >= 0 && messageEquivalent(existing[index], mergedMessage)) return state; + const current = index >= 0 ? existing[index] : undefined; + const mergedMessage = current ? mergeMessageSnapshot(current, message) : message; + if (current && messageEquivalent(current, mergedMessage)) return state; const mergedMessages = index >= 0 ? existing.map((item, itemIndex) => itemIndex === index ? mergedMessage : item) : [...existing, mergedMessage]; @@ -488,7 +489,7 @@ function valuesEquivalent(left: unknown, right: unknown): boolean { if (leftKeys.length !== rightKeys.length) return false; for (let index = 0; index < leftKeys.length; index += 1) { const key = leftKeys[index]; - if (key !== rightKeys[index]) return false; + if (key === undefined || key !== rightKeys[index]) return false; if (!valuesEquivalent(leftRecord[key], rightRecord[key])) return false; } return true; diff --git a/web/hwlab-cloud-web/src/stores/workbench-session.test.ts b/web/hwlab-cloud-web/src/stores/workbench-session.test.ts index 083a591e..e24615f0 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-session.test.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-session.test.ts @@ -65,6 +65,17 @@ test("composer treats failed message with terminal body as sealed terminal turn" assert.equal(composer.targetTraceId, null); }); +test("composer blocks turn submission until the live session stream is connected", () => { + const sessionId = "ses_realtime_connecting"; + const connecting = resolveComposerState({ messages: [], activeSessionId: sessionId, chatPending: false, realtimeReady: false }); + const connected = resolveComposerState({ messages: [], activeSessionId: sessionId, chatPending: false, realtimeReady: true }); + + assert.equal(connecting.disabled, true); + assert.equal(connecting.disabledReason, "realtime_connecting"); + assert.equal(connecting.sessionId, sessionId); + assert.equal(connected.disabled, false); +}); + test("cancel target remains available for completed message until final response exists", () => { const traceId = "trc_cancel_completed_without_final"; const sessionId = "ses_cancel_completed_without_final"; @@ -95,9 +106,10 @@ test("turn status refresh selector keeps automatic hydrate bounded to the active test("turn status refresh selector skips sealed terminal traces", () => { const sessionId = "ses_turn_detail_sealed"; - const sealed = agentMessage({ id: "msg_sealed", traceId: "trc_sealed", sessionId, status: "completed", text: "final" }); + const traceId = "trc_sealed"; + const sealed = agentMessage({ id: "msg_sealed", traceId, sessionId, status: "completed", text: "final" }); - assert.deepEqual(selectActiveTurnStatusRefreshTraceIds({ messages: [sealed], currentRequestTraceId: sealed.traceId, turnStatusAuthority: { [sealed.traceId]: turnStatus({ traceId: sealed.traceId, sessionId, status: "completed", running: false, terminal: true }) }, limit: 1 }), []); + assert.deepEqual(selectActiveTurnStatusRefreshTraceIds({ messages: [sealed], currentRequestTraceId: traceId, turnStatusAuthority: { [traceId]: turnStatus({ traceId, sessionId, status: "completed", running: false, terminal: true }) }, limit: 1 }), []); }); test("terminal detail read skips sealed terminal trace authority", () => { diff --git a/web/hwlab-cloud-web/src/stores/workbench-session.ts b/web/hwlab-cloud-web/src/stores/workbench-session.ts index 2568f407..f4c3deca 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-session.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-session.ts @@ -75,7 +75,7 @@ const BUILTIN_PROVIDER_PROFILE_LABELS: Readonly> = Object "minimax-m3": "MiniMax-M3" }); -export function resolveComposerState(input: { messages: ChatMessage[]; sessions?: WorkbenchSessionRecord[]; activeSessionId: string | null; chatPending: boolean; currentRequest?: { traceId?: string | null; sessionId?: string | null; threadId?: string | null; status?: string | null } | null; turnStatusAuthority?: TurnStatusAuthorityMap }): ComposerState { +export function resolveComposerState(input: { messages: ChatMessage[]; sessions?: WorkbenchSessionRecord[]; activeSessionId: string | null; chatPending: boolean; realtimeReady?: boolean; currentRequest?: { traceId?: string | null; sessionId?: string | null; threadId?: string | null; status?: string | null } | null; turnStatusAuthority?: TurnStatusAuthorityMap }): ComposerState { const sessionId = firstNonEmptyString(input.activeSessionId); const latestMessage = latestSessionMessage(input.messages, sessionId); const currentRequest = input.currentRequest && input.currentRequest.sessionId === sessionId ? input.currentRequest : null; @@ -94,6 +94,9 @@ export function resolveComposerState(input: { messages: ChatMessage[]; sessions? const effectiveSessionId = firstNonEmptyString(turn?.sessionId, currentRequest?.sessionId, active?.sessionId, sessionId); const threadId = firstNonEmptyString(turn?.threadId, currentRequest?.threadId, active?.threadId); const canSteer = Boolean(effectiveSessionId && activeTraceId && activeByStatus && !terminal); + if (effectiveSessionId && input.realtimeReady === false) { + return { disabled: true, disabledReason: "realtime_connecting", submitMode: canSteer ? "steer" : "turn", route: canSteer ? "/v1/agent/chat/steer" : "/v1/agent/chat", targetTraceId: canSteer ? activeTraceId : null, sessionId: effectiveSessionId, threadId }; + } if (canSteer) return { disabled: false, disabledReason: null, submitMode: "steer", route: "/v1/agent/chat/steer", targetTraceId: activeTraceId, sessionId: effectiveSessionId, threadId }; if (!effectiveSessionId) return { disabled: true, disabledReason: "session_required", submitMode: "turn", route: "/v1/agent/chat", targetTraceId: null, sessionId: null, threadId }; return { disabled: false, disabledReason: null, submitMode: "turn", route: "/v1/agent/chat", targetTraceId: null, sessionId: effectiveSessionId, threadId }; diff --git a/web/hwlab-cloud-web/src/stores/workbench-trace-detail.test.ts b/web/hwlab-cloud-web/src/stores/workbench-trace-detail.test.ts index d5999179..51c62aef 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-trace-detail.test.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-trace-detail.test.ts @@ -45,7 +45,7 @@ test("terminal seal strips heavy trace rows while preserving final authority", ( const sealed = terminalSealResultWithoutTraceEvents(result); assert.equal(sealed.status, "completed"); - assert.equal(sealed.finalResponse?.text, "final answer"); + assert.equal((sealed.finalResponse as { text?: string } | null | undefined)?.text, "final answer"); assert.equal(sealed.events, undefined); assert.equal(sealed.traceEvents, undefined); assert.equal(sealed.runnerTrace?.events, undefined); diff --git a/web/hwlab-cloud-web/src/stores/workbench-trace-detail.ts b/web/hwlab-cloud-web/src/stores/workbench-trace-detail.ts index 2e6f1884..e2a2e48b 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-trace-detail.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-trace-detail.ts @@ -1,4 +1,8 @@ +import type { WorkbenchRealtimeEvent } from "../api/workbench-events"; +import type { TraceSnapshot } from "../composables/workbench-trace-snapshot"; +import { normalizeProjectionDiagnostic } from "../utils/workbench-error-runtime"; import type { AgentChatResultResponse, ChatMessage, TraceEvent } from "../types"; +import { asAgentRun, firstFiniteNumber, messageTimingPatch, optionalString, recordValue, traceSnapshotError } from "./workbench-message-projection-runtime"; export function traceDetailProjectedSeq(trace: ChatMessage["runnerTrace"]): number { const cursor = traceDetailCursor(trace); @@ -21,6 +25,39 @@ export function terminalSealResultWithoutTraceEvents(result: AgentChatResultResp return { ...rest, runnerTrace: traceRest }; } +export function realtimeSnapshotToTraceSnapshot(traceId: string, snapshot: WorkbenchRealtimeEvent["snapshot"], eventPageEvents?: TraceEvent[], options: { eventSource?: string } = {}): TraceSnapshot { + const source = snapshot ?? { traceId }; + const events = Array.isArray(eventPageEvents) ? eventPageEvents : Array.isArray(source.events) ? source.events : []; + const { traceId: _sourceTraceId, error: _sourceError, ...sourceRest } = source; + return { + ...sourceRest, + traceId: optionalString(source.traceId, traceId), + status: text(source.status) ?? undefined, + events, + eventCount: firstFiniteNumber(source.eventCount, events.length), + fullTraceLoaded: source.fullTraceLoaded === true, + hasMore: source.hasMore === true, + truncated: source.truncated === true, + nextProjectedSeq: firstFiniteNumber(source.nextProjectedSeq) ?? null, + range: recordValue(source.range) as TraceSnapshot["range"], + agentRun: asAgentRun(source.agentRun) ?? undefined, + traceStatus: text(source.traceStatus) ?? undefined, + retention: source.retention, + terminalEvidence: source.terminalEvidence, + traceSummary: source.traceSummary, + error: traceSnapshotError(source.error), + projection: normalizeProjectionDiagnostic(source.projection ?? source) ?? null, + projectionStatus: text(source.projectionStatus) ?? text(recordValue(source.projection)?.projectionStatus) ?? undefined, + projectionHealth: text(source.projectionHealth) ?? text(recordValue(source.projection)?.projectionHealth) ?? undefined, + staleMs: firstFiniteNumber(source.staleMs, recordValue(source.projection)?.staleMs), + blocker: recordValue(source.blocker) ?? recordValue(recordValue(source.projection)?.blocker) ?? undefined, + ...messageTimingPatch(source), + lastEventLabel: text(source.lastEventLabel) ?? undefined, + eventSource: options.eventSource ?? "trace-api", + updatedAt: text(source.updatedAt) ?? new Date().toISOString() + } as TraceSnapshot; +} + function traceDetailCursor(source: { nextProjectedSeq?: unknown; range?: { toProjectedSeq?: unknown } | null } | null | undefined): number | null { return finiteProjectedSeq(source?.nextProjectedSeq) ?? finiteProjectedSeq(source?.range?.toProjectedSeq); } @@ -41,3 +78,8 @@ function traceEventProjectedSeq(event: TraceEvent): number { const seq = Number(event.projectedSeq); return Number.isFinite(seq) ? Math.trunc(seq) : Number.NaN; } + +function text(value: unknown): string | null { + const normalized = typeof value === "string" ? value.trim() : ""; + return normalized || null; +} diff --git a/web/hwlab-cloud-web/src/stores/workbench.ts b/web/hwlab-cloud-web/src/stores/workbench.ts index 10412fb8..05ca500a 100644 --- a/web/hwlab-cloud-web/src/stores/workbench.ts +++ b/web/hwlab-cloud-web/src/stores/workbench.ts @@ -4,23 +4,25 @@ import { computed, nextTick, ref } from "vue"; import { defineStore } from "pinia"; import { api } from "@/api"; +import { workbenchRealtimeCapabilities } from "@/config/runtime"; import { workbenchRuntimePolicy } from "@/config/workbench-runtime-policy"; import { createKeyedSingleflight } from "@/utils/scheduler/keyed-singleflight"; import { createWorkbenchHealthProbeCache } from "@/utils/workbench-health"; import { agentErrorFromProjection, normalizeApiErrorRecord, normalizeErrorDiagnostic, normalizeProjectionDiagnostic, projectionDiagnosticFromApiFailure, projectionDiagnosticFromFailure } from "@/utils/workbench-error-runtime"; import { readWorkbenchJson, readWorkbenchNumber, readWorkbenchString, removeWorkbenchStorageKey, writeWorkbenchJson, writeWorkbenchString } from "@/utils/workbench-storage-runtime"; -import { createWorkbenchStreamTransportRuntime, type WorkbenchRealtimeEvent, type WorkbenchStreamTransportRecovery } from "@/utils/workbench-realtime-runtime"; +import { createWorkbenchStreamTransportRuntime, workbenchRealtimeTraceIdForCapabilities, type WorkbenchRealtimeEvent, type WorkbenchStreamTransportRecovery } from "@/utils/workbench-realtime-runtime"; import { mergeRunnerTrace, snapshotToRunnerTrace, type TraceSnapshot } from "@/composables/workbench-trace-snapshot"; import type { WorkbenchMessagePageResponse, WorkbenchSessionDetailResponse } from "@/api/workbench"; import type { AgentChatResponse, AgentChatResultResponse, AgentRunProvenance, ApiError, ApiResult, ChatMessage, ErrorDiagnostic, LiveSurface, ProjectionBlocker, ProjectionDiagnostic, ProviderProfile, TraceEvent, WorkbenchSessionRecord, WorkbenchTurnTimingProjection } from "@/types"; import { firstNonEmptyString, nextProtocolId, normalizeWorkbenchSessionId, normalizeWorkbenchSessionRouteId } from "@/utils"; -import { composeWorkbenchScopedKey } from "@/utils/workbench-key"; +import { composeWorkbenchScopedKey, workbenchRealtimeScopeKey } from "@/utils/workbench-key"; import { failWorkbenchSessionSwitch, failWorkbenchSubmitJourney, finishWorkbenchSessionSwitchFullLoad, markWorkbenchSubmitApiAccepted, markWorkbenchTraceEventsReceived, markWorkbenchTraceProjected, recordWorkbenchLoadingState, recordWorkbenchRuntimeDiagnostic, startWorkbenchSessionSwitch, startWorkbenchSubmitJourney } from "@/utils/workbench-performance"; import { RECENT_DRAFTS_STORAGE_KEY, appendSessionPage, defaultProviderProfileOptions, isArchivedSession, mergeSessionIntoList, normalizeChatMessageStatus, normalizeRecentDrafts, normalizeWorkbenchMessageTitle, providerProfileOptionsFromPayload, recordRecentDraft, resolveCancelableAgentMessage, resolveComposerState, selectActiveTurnStatusRefreshTraceIds, shouldReadTerminalTraceDetail, shouldShowSessionListLoading, sortSessionTabs, stableSessionList, type DraftEntry, type ProviderProfileOption, type TurnStatusAuthority } from "./workbench-session"; import { initialWorkbenchSessionIdFromLocation } from "./workbench-projection"; import { cleanupWorkbenchServerStateSessions, selectActiveMessages, selectActiveSession, selectSessionList, selectSessionStatusAuthority, selectTraceAuthorityById, selectTurnStatusAuthority, type WorkbenchServerAction } from "./workbench-server-state"; import { cleanupDroppedWorkbenchSessionCaches, trimWorkbenchSessionCache } from "./workbench-session-cache"; -import { reduceWorkbenchRealtimeEvent, type WorkbenchRealtimeAction } from "./workbench-event-reducer"; +import { reduceWorkbenchRealtimeEvent, workbenchRealtimeEventIsBusinessActivity, type WorkbenchRealtimeAction } from "./workbench-event-reducer"; +import { reduceWorkbenchLiveKafkaMessageState, workbenchLiveKafkaEventIsTerminal, workbenchLiveKafkaMessageId } from "./workbench-live-kafka-event"; import { messageHasSealedTerminalResult, messageIsSealedTerminal, traceAuthorityIsSealed } from "./workbench-terminal-authority"; import { boundedProjectionMessageLimit, mergeBoundedProjectionMessages, selectProjectionMessageWindow, traceProjectionIsTerminalSealed } from "./workbench-message-projection-budget"; import { @@ -64,7 +66,6 @@ import { traceHasTerminalResponse, traceHasEvents, traceResultHasTerminalEvidence, - traceSnapshotError } from "./workbench-message-projection-runtime"; import { planWorkbenchRealtimeApply, planWorkbenchRealtimeRecovery, type WorkbenchRealtimeApplyStep, type WorkbenchRealtimeRecoveryStep } from "./workbench-realtime-plan"; import { workbenchSyncReplayDiagnostic, workbenchSyncReplayEvents } from "./workbench-realtime-authority"; @@ -72,7 +73,7 @@ import { useWorkbenchColadaMutations } from "./workbench-colada-mutations"; import { useWorkbenchColadaQueries } from "./workbench-colada-queries"; import { useWorkbenchColadaReducer } from "./workbench-colada-reducer"; import { projectionMergeCommitSummary, traceEventsAutoReadDecision, workbenchSessionDetailReadKey, workbenchSessionMessagesReadKey, workbenchTraceEventsReadKey, type TraceEventsReadRangeRecord } from "./workbench-session-messages-read-budget"; -import { terminalSealResultWithoutTraceEvents, traceDetailProjectedSeq, traceNextProjectedSeq } from "./workbench-trace-detail"; +import { realtimeSnapshotToTraceSnapshot, terminalSealResultWithoutTraceEvents, traceDetailProjectedSeq, traceNextProjectedSeq } from "./workbench-trace-detail"; const WORKBENCH_SESSION_PROJECTION_SIGNAL_CHANNEL = "hwlab.workbench.sessionProjection.v1"; const WORKBENCH_SESSION_PROJECTION_SIGNAL_KEY = "hwlab.workbench.sessionProjectionSignal.v1"; @@ -107,6 +108,7 @@ interface RealtimeTurnProjectionItem { export const useWorkbenchStore = defineStore("workbench", () => { const runtimePolicy = workbenchRuntimePolicy(); + const realtimeCapabilities = workbenchRealtimeCapabilities(); const workbenchColadaReducer = useWorkbenchColadaReducer(); const workbenchColadaQueries = useWorkbenchColadaQueries(); const workbenchColadaMutations = useWorkbenchColadaMutations(); @@ -132,6 +134,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { const sessionListLoadingMore = ref(false); const sessionListLoadMoreError = ref(null); const chatPending = ref(false); + const liveRealtimeReadySessionId = ref(null); const error = ref(null); const activityRef = ref({ lastActivityAt: Date.now(), lastActivityIso: new Date().toISOString(), waitingFor: "idle", lastEventLabel: null as string | null }); const currentRequest = ref<{ traceId: string; sessionId: string | null; threadId: string | null; status?: string | null } | null>(null); @@ -161,7 +164,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { const sessionListLoadedCount = computed(() => sessionTabs.value.length); const sessionListLoading = computed(() => shouldShowSessionListLoading({ loading: loading.value, sessionsReady: sessionsReady.value })); const sessionDetailLoading = computed(() => Boolean(sessionDetailLoadingId.value || switchingSessionId.value || (loading.value && messages.value.length === 0))); - const composer = computed(() => resolveComposerState({ messages: messages.value, sessions: sessions.value, activeSessionId: activeSessionId.value, chatPending: chatPending.value, currentRequest: currentRequest.value, turnStatusAuthority: turnStatusAuthority.value })); + const composer = computed(() => resolveComposerState({ messages: messages.value, sessions: sessions.value, activeSessionId: activeSessionId.value, chatPending: chatPending.value, realtimeReady: !realtimeCapabilities.liveKafkaSse || liveRealtimeReadySessionId.value === activeSessionId.value, currentRequest: currentRequest.value, turnStatusAuthority: turnStatusAuthority.value })); function recordActivity(label = "user-activity"): void { const now = Date.now(); @@ -376,6 +379,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { } function scheduleSessionListRefresh(includeSessionId: string | null | undefined = activeSessionId.value, delayMs = runtimePolicy.sessionListRealtimeRefreshDelayMs): void { + if (realtimeCapabilities.liveKafkaSse) return; void includeSessionId; if (typeof window === "undefined") { void workbenchColadaQueries.invalidateSessionList(); @@ -755,6 +759,11 @@ export const useWorkbenchStore = defineStore("workbench", () => { const traceId = steerMode && composer.value.targetTraceId ? composer.value.targetTraceId : nextProtocolId("trc"); const steerTraceId = steerMode ? nextProtocolId("trc_steer") : null; const sessionId = composer.value.sessionId; + if (realtimeCapabilities.liveKafkaSse && liveRealtimeReadySessionId.value !== sessionId) { + error.value = "realtime_connecting"; + restartRealtime("submit-readiness"); + return false; + } const threadId = composer.value.threadId; const providerThreadId = providerThreadIdForRequest(threadId); const submittedAt = new Date().toISOString(); @@ -764,6 +773,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { const pending = makeMessage("agent", "", "running", { traceId, sessionId, threadId, title: "Code Agent", retryInput: value, traceAutoLifecycle: "running", createdAt: submittedAt, updatedAt: submittedAt, ...optimisticRunningTimingPatch(submittedAt) }); appendActiveMessages(user, pending); projectOptimisticRunningTurn({ sessionId, threadId, traceId, userText: value }); + currentRequest.value = { traceId, sessionId, threadId, status: "running" }; } startWorkbenchSubmitJourney({ traceId, sessionId, entry: submitEntry, backend: providerProfile.value, transport: "sse" }); chatPending.value = true; @@ -812,6 +822,11 @@ export const useWorkbenchStore = defineStore("workbench", () => { if (canonicalTraceId !== traceId) { reduceServerState({ type: "turn.forget", traceId }); } + if (traceTerminalBodyIsVisible(canonicalTraceId, sessionId)) { + chatPending.value = false; + currentRequest.value = null; + return true; + } currentRequest.value = { traceId: canonicalTraceId, sessionId: firstNonEmptyString(response.data.sessionId, sessionId), @@ -825,7 +840,6 @@ export const useWorkbenchStore = defineStore("workbench", () => { completeTrace(canonicalTraceId, response.data as AgentChatResultResponse); return true; } - scheduleActiveTraceSyncReplay(canonicalTraceId, "submit-sync-replay"); restartRealtime("submit"); return true; } @@ -842,7 +856,6 @@ export const useWorkbenchStore = defineStore("workbench", () => { recordActivity("cancel:accepted"); if (message.status === "running") chatPending.value = false; currentRequest.value = null; - clearActiveTraceSyncReplay(traceId); void clearActiveTrace(traceId, "cancel-agent-message"); scheduleSessionListRefresh(sessionId, runtimePolicy.sessionListTerminalRefreshDelayMs); } @@ -1050,7 +1063,6 @@ export const useWorkbenchStore = defineStore("workbench", () => { })); markWorkbenchTraceProjected(traceId); if (traceResultHasTerminalEvidence(result) && !turnResultIsTerminalForMerge(result)) { - void refreshTerminalTraceFromSyncReplay(traceId, "trace-detail-read-terminal-evidence"); } if (turnResultIsTerminalForMerge(result)) { rememberTurnStatus(traceId, result); @@ -1100,7 +1112,6 @@ export const useWorkbenchStore = defineStore("workbench", () => { const initial: AgentChatResponse = { status: "running", traceId }; if (!messages.value.some((message) => message.traceId === traceId)) appendActiveMessages(makeMessage("agent", "", "running", { traceId, sessionId: selectedSessionId.value ?? undefined, threadId: selectedThreadId.value ?? undefined, title: "Code Agent", traceAutoLifecycle: "running" })); currentRequest.value = { traceId, sessionId: selectedSessionId.value ?? null, threadId: selectedThreadId.value ?? null, status: initial.status }; - scheduleActiveTraceSyncReplay(traceId, "reattach-sync-replay"); restartRealtime("reattach"); } @@ -1108,7 +1119,10 @@ export const useWorkbenchStore = defineStore("workbench", () => { const traceId = realtimeTraceId(); const sessionId = selectedSessionId.value ?? null; void reason; + const scopeChanged = realtimeTransport.currentKey() !== workbenchRealtimeScopeKey(sessionId, traceId); + if (realtimeCapabilities.liveKafkaSse && scopeChanged) liveRealtimeReadySessionId.value = null; realtimeTransport.restart({ + realtimeCapabilities, sessionId, traceId, errorRecoveryMinMs: runtimePolicy.workbenchRealtimeErrorSyncReplayMinMs, @@ -1116,8 +1130,23 @@ export const useWorkbenchStore = defineStore("workbench", () => { flushMaxChunkMs: runtimePolicy.workbenchRealtimeFlushMaxChunkMs, flushYieldMs: runtimePolicy.workbenchRealtimeFlushYieldMs, onOpen: () => undefined, + onError: () => { + if (realtimeCapabilities.liveKafkaSse) liveRealtimeReadySessionId.value = null; + }, + onState: (state) => { + if (realtimeCapabilities.liveKafkaSse && ["connecting", "error", "closed", "blocked"].includes(state.phase)) liveRealtimeReadySessionId.value = null; + }, onRecovery: (recovery) => handleRealtimeRecovery(recovery), - onEvent: (event, eventName) => applyRealtimeEvent(event, eventName) + onEvent: (event, eventName) => { + if (realtimeCapabilities.liveKafkaSse && eventName === "workbench.connected") { + const filters = recordValue(event.filters); + const connectedSessionId = normalizeWorkbenchSessionId(filters?.sessionId); + const valid = event.liveOnly === true && event.replay === false && event.capabilities?.liveKafkaSse === true && connectedSessionId === sessionId; + liveRealtimeReadySessionId.value = valid ? connectedSessionId : null; + if (!valid) error.value = "workbench_live_realtime_contract_invalid"; + } + applyRealtimeEvent(event, eventName); + } }); } @@ -1144,63 +1173,39 @@ export const useWorkbenchStore = defineStore("workbench", () => { } async function refreshWorkbenchSyncReplay(sessionId: string | null, traceId: string | null, sinceOutboxSeq: number | null, reason: string): Promise { - const result = await workbenchColadaQueries.fetchSyncReplay({ sessionId, traceId, since: sinceOutboxSeq }, { timeoutMs: 8000, activityRef: () => activityRef.value }); - if (!result.ok || !result.data) { - recordWorkbenchRuntimeDiagnostic({ module: "workbench-sync-replay", sessionId, traceId, outcome: "network", diagnostic: { code: "workbench_sync_replay_failed", reason, status: result.status, apiError: result.apiError, valuesRedacted: true } }); - return; + if (realtimeCapabilities.liveKafkaSse || !realtimeCapabilities.projectionRealtime) return; + let cursor = firstFiniteNumber(sinceOutboxSeq) ?? 0; + for (;;) { + const result = await workbenchColadaQueries.fetchSyncReplay({ sessionId, traceId, since: cursor }, { timeoutMs: 8000, activityRef: () => activityRef.value }); + if (!result.ok || !result.data) { + recordWorkbenchRuntimeDiagnostic({ module: "workbench-sync-replay", sessionId, traceId, outcome: "network", diagnostic: { code: "workbench_sync_replay_failed", reason, status: result.status, apiError: result.apiError, valuesRedacted: true } }); + return; + } + const events = workbenchSyncReplayEvents(result.data); + for (const event of events) applyRealtimeEvent(event, realtimeEventName(event)); + recordWorkbenchRuntimeDiagnostic({ module: "workbench-sync-replay", sessionId, traceId, outcome: "ok", diagnostic: workbenchSyncReplayDiagnostic(result.data, events, { reason, sinceOutboxSeq: cursor }) }); + const responseCursor = recordValue(result.data.cursor); + if (responseCursor?.hasMore !== true) return; + const nextCursor = firstFiniteNumber(responseCursor.outboxSeq); + if (nextCursor === undefined || nextCursor <= cursor) { + recordWorkbenchRuntimeDiagnostic({ module: "workbench-sync-replay", sessionId, traceId, outcome: "error", diagnostic: { code: "workbench_sync_replay_cursor_stalled", reason, cursor, nextCursor, valuesRedacted: true } }); + return; + } + cursor = nextCursor; } - const events = workbenchSyncReplayEvents(result.data); - for (const event of events) applyRealtimeEvent(event, realtimeEventName(event)); - recordWorkbenchRuntimeDiagnostic({ module: "workbench-sync-replay", sessionId, traceId, outcome: "ok", diagnostic: workbenchSyncReplayDiagnostic(result.data, events, { reason, sinceOutboxSeq }) }); - } - - function scheduleActiveTraceSyncReplay(traceId: string | null | undefined, reason: string, delayMs = runtimePolicy.workbenchActiveTraceSyncReplayInitialMs): void { - const id = firstNonEmptyString(traceId); - if (!id) return; - if (typeof window === "undefined") { - void refreshActiveTraceFromSyncReplay(id, reason); - return; - } - window.setTimeout(() => { void refreshActiveTraceFromSyncReplay(id, reason); }, delayMs); - } - - function clearActiveTraceSyncReplay(traceId: string | null | undefined): void { - void traceId; - } - - async function refreshActiveTraceFromSyncReplay(traceId: string, reason: string): Promise { - const id = firstNonEmptyString(traceId); - if (!id) return; - if (traceTerminalBodyIsVisible(id)) { - clearActiveTraceSyncReplay(id); - recordWorkbenchRuntimeDiagnostic({ module: "workbench-terminal-priority", traceId: id, outcome: "ok", diagnostic: { code: "terminal_low_priority_sync_replay_skip", reason, source: "active-sync-replay", valuesRedacted: true } }); - return; - } - const ownerSessionId = traceOwnerSessionId(id, currentRequest.value?.sessionId ?? null); - if (ownerSessionId === activeSessionId.value) recordActivity(reason); - await refreshWorkbenchSyncReplay(ownerSessionId, id, null, `active-sync-replay:${reason}`); - const ownerMessages = ownerSessionId ? serverState.value.messagesBySessionId[ownerSessionId] ?? [] : messages.value; - if (traceProjectionIsTerminalSealed(id, ownerMessages)) { - clearActiveTraceSyncReplay(id); - return; - } - const message = latestMessageForTrace(id, ownerMessages); - const turn = turnStatusAuthority.value[id]; - const terminal = messageHasTerminalResponse(message); - if (terminal) { - clearActiveTraceSyncReplay(id); - return; - } - const stillActive = currentRequest.value?.traceId === id || isTraceActiveStatus(turn?.status) || isTraceActiveStatus(message?.status); - if (stillActive) scheduleActiveTraceSyncReplay(id, "active-sync-replay:repeat", runtimePolicy.workbenchActiveTraceSyncReplayRepeatMs); } function stopRealtime(): void { + liveRealtimeReadySessionId.value = null; realtimeTransport.stop(); } function realtimeTraceId(): string | null { - return firstNonEmptyString(currentRequest.value?.traceId, activeTraceIdFromMessages(messages.value, turnStatusAuthority.value)); + return workbenchRealtimeTraceIdForCapabilities( + realtimeCapabilities, + currentRequest.value?.traceId, + activeTraceIdFromMessages(messages.value, turnStatusAuthority.value) + ); } function normalizeTraceAuthoritySessionId(value: unknown): string | null { @@ -1214,7 +1219,8 @@ export const useWorkbenchStore = defineStore("workbench", () => { const turn = recordValue(event.turn); const snapshot = recordValue(event.snapshot); const traceEvent = recordValue(event.event); - return normalizeTraceAuthoritySessionId(firstNonEmptyString(event.sessionId, turn?.sessionId, snapshot?.sessionId, traceEvent?.sessionId)); + const sessionId = firstNonEmptyString(event.hwlabSessionId, event.sessionId, turn?.sessionId, snapshot?.sessionId, traceEvent?.sessionId); + return event.schema === "hwlab.event.v1" ? normalizeWorkbenchSessionId(sessionId) : normalizeTraceAuthoritySessionId(sessionId); } function traceResultSessionId(result: AgentChatResultResponse | TraceSnapshot | Record | null | undefined): string | null { @@ -1228,8 +1234,8 @@ export const useWorkbenchStore = defineStore("workbench", () => { return normalizeTraceAuthoritySessionId(firstNonEmptyString(message?.sessionId, message?.runnerTrace?.sessionId)); } - function traceOwnerSessionId(traceId: string | null | undefined, authoritySessionId: string | null | undefined): string | null { - const sessionId = normalizeTraceAuthoritySessionId(authoritySessionId); + function traceOwnerSessionId(traceId: string | null | undefined, authoritySessionId: string | null | undefined, canonicalSession = false): string | null { + const sessionId = canonicalSession ? normalizeWorkbenchSessionId(authoritySessionId) : normalizeTraceAuthoritySessionId(authoritySessionId); if (sessionId) return sessionId; const id = firstNonEmptyString(traceId); if (!id) return activeSessionId.value; @@ -1239,21 +1245,22 @@ export const useWorkbenchStore = defineStore("workbench", () => { return activeSessionId.value; } - function messageMatchesTraceAuthority(message: ChatMessage, traceId: string, authoritySessionId: string | null | undefined, ownerSessionId: string | null | undefined): boolean { + function messageMatchesTraceAuthority(message: ChatMessage, traceId: string, authoritySessionId: string | null | undefined, ownerSessionId: string | null | undefined, canonicalSession = false): boolean { if (message.role !== "agent" || firstNonEmptyString(message.traceId, message.runnerTrace?.traceId) !== traceId) return false; - const sessionId = normalizeTraceAuthoritySessionId(authoritySessionId); - const ownerId = normalizeTraceAuthoritySessionId(ownerSessionId); - const messageSessionId = messageSessionAuthority(message); + const normalizeSession = canonicalSession ? normalizeWorkbenchSessionId : normalizeTraceAuthoritySessionId; + const sessionId = normalizeSession(authoritySessionId); + const ownerId = normalizeSession(ownerSessionId); + const messageSessionId = canonicalSession ? normalizeWorkbenchSessionId(firstNonEmptyString(message.sessionId, message.runnerTrace?.sessionId)) : messageSessionAuthority(message); if (sessionId && ownerId && sessionId !== ownerId) return false; if (messageSessionId && ownerId && messageSessionId !== ownerId) return false; if (sessionId && messageSessionId && sessionId !== messageSessionId) return false; return true; } - function shouldApplyActiveTraceAuthority(traceId: string | null | undefined, authoritySessionId: string | null | undefined): boolean { + function shouldApplyActiveTraceAuthority(traceId: string | null | undefined, authoritySessionId: string | null | undefined, canonicalSession = false): boolean { const id = firstNonEmptyString(traceId); const activeId = activeSessionId.value; - const sessionId = normalizeTraceAuthoritySessionId(authoritySessionId); + const sessionId = canonicalSession ? normalizeWorkbenchSessionId(authoritySessionId) : normalizeTraceAuthoritySessionId(authoritySessionId); if (!activeId) return false; if (sessionId && sessionId !== activeId) return false; if (!id) return Boolean(sessionId && sessionId === activeId); @@ -1280,7 +1287,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { function applyRealtimeEvent(event: WorkbenchRealtimeEvent, eventName: string): void { const reduced = reduceWorkbenchRealtimeEvent(event, eventName); - recordActivity(reduced.activityLabel); + if (workbenchRealtimeEventIsBusinessActivity(event, eventName, realtimeCapabilities.liveKafkaSse)) recordActivity(reduced.activityLabel); applyWorkbenchRealtimeAction(reduced.action); } @@ -1336,15 +1343,69 @@ export const useWorkbenchStore = defineStore("workbench", () => { function applyRealtimeTraceEvent(traceId: string | null | undefined, event: WorkbenchRealtimeEvent["event"], snapshot: WorkbenchRealtimeEvent["snapshot"], realtimeEvent?: WorkbenchRealtimeEvent | null): void { const id = firstNonEmptyString(traceId, event?.traceId, snapshot?.traceId); if (!id) return; + const liveKafkaEnvelope = realtimeEvent?.schema === "hwlab.event.v1"; const sessionId = realtimeEvent ? realtimeEventSessionId(realtimeEvent) : traceResultSessionId(snapshot ?? event ?? null); - if (!shouldApplyActiveTraceAuthority(id, sessionId)) return; + if (!shouldApplyActiveTraceAuthority(id, sessionId, liveKafkaEnvelope)) return; if (traceTerminalBodyIsVisible(id, sessionId)) { recordWorkbenchRuntimeDiagnostic({ module: "workbench-terminal-priority", sessionId, traceId: id, outcome: "ok", diagnostic: { code: "terminal_low_priority_sse_trace_skip", source: "realtime-trace-event", valuesRedacted: true } }); return; } const events = event ? [event] : Array.isArray(snapshot?.events) ? snapshot.events : []; markWorkbenchTraceEventsReceived({ traceId: id, events, transport: "sse", serverSentAt: realtimeEvent?.serverSentAt, eventCreatedAt: realtimeEvent?.eventCreatedAt, traceSeq: realtimeEvent?.traceSeq ?? realtimeEvent?.cursor?.traceSeq }); - applyTraceSnapshot(id, realtimeSnapshotToTraceSnapshot(id, snapshot ?? { traceId: id, status: event?.status, events }, events)); + const liveTerminal = liveKafkaEnvelope && workbenchLiveKafkaEventIsTerminal(event); + const eventSnapshot = snapshot ?? { traceId: id, status: liveKafkaEnvelope && !liveTerminal ? "running" : event?.status, events }; + if (liveKafkaEnvelope && event) applyLiveKafkaBusinessEvent(id, sessionId, event); + applyTraceSnapshot(id, realtimeSnapshotToTraceSnapshot(id, eventSnapshot, events, liveKafkaEnvelope ? { eventSource: "hwlab-kafka-sse" } : undefined), liveKafkaEnvelope); + } + + function applyLiveKafkaBusinessEvent(traceId: string, authoritySessionId: string | null, event: TraceEvent): void { + const eventSessionId = firstNonEmptyString(event.sessionId); + const ownerSessionId = traceOwnerSessionId(traceId, authoritySessionId ?? eventSessionId, true); + if (!ownerSessionId) return; + const previousMessage = (serverState.value.messagesBySessionId[ownerSessionId] ?? []).find((message) => messageMatchesTraceAuthority(message, traceId, authoritySessionId ?? eventSessionId, ownerSessionId, true)) ?? null; + const liveState = reduceWorkbenchLiveKafkaMessageState({ text: previousMessage?.text ?? "", status: previousMessage?.status ?? "running", terminal: false }, event); + const terminal = liveState.terminal; + const status = liveState.status; + const assistantText = liveState.text; + const updatedAt = firstNonEmptyString(event.createdAt, event.updatedAt) ?? new Date().toISOString(); + const messageId = workbenchLiveKafkaMessageId(traceId); + const message = previousMessage ?? makeMessage("agent", "", "running", { + id: messageId, + messageId, + traceId, + sessionId: ownerSessionId, + title: "Code Agent", + traceAutoLifecycle: "running", + createdAt: updatedAt, + updatedAt + }); + reduceServerState({ + type: "message.upsert", + sessionId: ownerSessionId, + message: { + ...message, + ...(assistantText ? { text: assistantText } : {}), + status: status as ChatMessage["status"], + traceAutoLifecycle: terminal ? "terminal" : "running", + ...(terminal && assistantText ? { finalResponse: { text: assistantText, status, traceId } } : {}), + updatedAt + } + }); + rememberTurnStatus(traceId, { + traceId, + sessionId: ownerSessionId, + status, + running: !terminal, + terminal, + finalResponse: terminal && assistantText ? { text: assistantText, status, traceId } : undefined, + updatedAt + } as AgentChatResultResponse); + const existing = sessions.value.find((session) => session.sessionId === ownerSessionId) ?? null; + if (existing) rememberSessionList(mergeSessionIntoList(sessions.value, { ...existing, status, lastTraceId: traceId, updatedAt: firstNonEmptyString(event.createdAt, event.updatedAt) ?? new Date().toISOString() })); + if (terminal && currentRequest.value?.traceId === traceId) { + chatPending.value = false; + currentRequest.value = null; + } } function applyRealtimeTurnSnapshot(turn: Record): void { @@ -1405,7 +1466,6 @@ export const useWorkbenchStore = defineStore("workbench", () => { const startedAt = performanceNowMs(); if (shouldApplyActiveTraceAuthority(next.traceId, traceResultSessionId(next.result))) { syncTurnStatusToMessage(next.traceId, next.result); - if (next.terminalTurn) void refreshTerminalTraceFromSyncReplay(next.traceId, "realtime-turn-snapshot"); } recordWorkbenchRuntimeDiagnostic({ module: "workbench-turn-status", @@ -1425,25 +1485,6 @@ export const useWorkbenchStore = defineStore("workbench", () => { if (realtimeTurnProjectionQueue.size > 0) scheduleRealtimeTurnProjectionFlush(); } - async function refreshTerminalTraceFromSyncReplay(traceId: string, reason: string): Promise { - const id = firstNonEmptyString(traceId); - if (!id) return; - if (traceTerminalBodyIsVisible(id)) { - clearActiveTraceSyncReplay(id); - recordWorkbenchRuntimeDiagnostic({ module: "workbench-terminal-priority", traceId: id, outcome: "ok", diagnostic: { code: "terminal_low_priority_terminal_sync_skip", reason, source: "terminal-sync-replay", valuesRedacted: true } }); - return; - } - const ownerSessionId = traceOwnerSessionId(id, null); - if (ownerSessionId === activeSessionId.value) recordActivity(reason); - await refreshWorkbenchSyncReplay(ownerSessionId, id, null, `terminal-sync-replay:${reason}`); - const ownerMessages = ownerSessionId ? serverState.value.messagesBySessionId[ownerSessionId] ?? [] : messages.value; - if (traceProjectionIsTerminalSealed(id, ownerMessages)) { - clearActiveTraceSyncReplay(id); - return; - } - if (ownerSessionId) scheduleSessionListRefresh(ownerSessionId, runtimePolicy.sessionListTerminalRefreshDelayMs); - } - function installRealtimeVisibilityHandler(): void { if (typeof document === "undefined") return; document.addEventListener("visibilitychange", () => { @@ -1469,6 +1510,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { } function publishWorkbenchProjectionSignal(sessionId: string | null | undefined, traceId: string | null | undefined, reason: string): void { + if (realtimeCapabilities.liveKafkaSse || !realtimeCapabilities.projectionRealtime) return; if (typeof window === "undefined") return; const id = normalizeWorkbenchSessionId(sessionId); if (!id) return; @@ -1478,27 +1520,28 @@ export const useWorkbenchStore = defineStore("workbench", () => { } function handleWorkbenchProjectionSignal(value: unknown): void { + if (realtimeCapabilities.liveKafkaSse || !realtimeCapabilities.projectionRealtime) return; const record = recordValue(value); if (!record || record.sourceId === workbenchProjectionSignalSourceId) return; if (firstNonEmptyString(record.type) !== "session-projection") return; const sessionId = normalizeWorkbenchSessionId(record.sessionId); if (!sessionId || sessionId !== activeSessionId.value) return; const traceId = firstNonEmptyString(record.traceId); - void refreshRealtimeSessionFromSyncReplay(sessionId, `cross-tab-session-projection:${firstNonEmptyString(record.reason, traceId, "session") ?? "session"}`); + void refreshCrossTabSessionFromSyncReplay(sessionId, `cross-tab-session-projection:${firstNonEmptyString(record.reason, traceId, "session") ?? "session"}`); } - function applyTraceSnapshot(traceId: string, snapshot: TraceSnapshot): void { + function applyTraceSnapshot(traceId: string, snapshot: TraceSnapshot, canonicalSession = false): void { const trace = snapshotToRunnerTrace({ ...snapshot, traceStatus: firstNonEmptyString(snapshot.traceStatus, snapshot.status) ?? undefined }); - const authoritySessionId = traceResultSessionId(trace) ?? traceResultSessionId(snapshot); - const ownerSessionId = traceOwnerSessionId(traceId, authoritySessionId); + const authoritySessionId = canonicalSession + ? normalizeWorkbenchSessionId(firstNonEmptyString(trace.sessionId, snapshot.sessionId)) + : traceResultSessionId(trace) ?? traceResultSessionId(snapshot); + const ownerSessionId = traceOwnerSessionId(traceId, authoritySessionId, canonicalSession); if (!ownerSessionId) return; - let matchedMessage = false; updateSessionMessages(ownerSessionId, (source) => source.map((message) => { - if (!messageMatchesTraceAuthority(message, traceId, authoritySessionId, ownerSessionId)) return message; - matchedMessage = true; + if (!messageMatchesTraceAuthority(message, traceId, authoritySessionId, ownerSessionId, canonicalSession)) return message; const mergedRunnerTrace = mergeRunnerTrace(message.runnerTrace, trace); const traceStatus = normalizedStatusText(trace.status ?? snapshot.status) ?? null; const clearCompletedDiagnostics = shouldClearCompletedTurnDiagnostics(traceStatus, null); @@ -1509,14 +1552,11 @@ export const useWorkbenchStore = defineStore("workbench", () => { const projection = clearCompletedDiagnostics ? nonBlockingProjection(trace.projection ?? null) : trace.projection ?? runnerTrace.projection ?? message.projection ?? null; return { ...message, ...messageTimingPatchForMerge(message, trace), ...messageStatusPatchForTerminalMerge(message, traceStatus, terminal), runnerTrace, error: clearCompletedDiagnostics ? null : error ?? message.error ?? null, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, updatedAt: new Date().toISOString() }; })); - if (!matchedMessage && ownerSessionId === activeSessionId.value) { - void refreshRealtimeSessionFromSyncReplay(ownerSessionId, `realtime-trace-snapshot:${traceId}`); - } markWorkbenchTraceProjected(traceId); if (!traceProjectionIsTerminalSealed(traceId, serverState.value.messagesBySessionId[ownerSessionId] ?? [])) scheduleSessionListRefresh(ownerSessionId, runtimePolicy.sessionListRealtimeRefreshDelayMs); } - async function refreshRealtimeSessionFromSyncReplay(sessionId: string, reason: string): Promise { + async function refreshCrossTabSessionFromSyncReplay(sessionId: string, reason: string): Promise { const id = normalizeWorkbenchSessionId(sessionId); if (!id || id !== activeSessionId.value) return; const traceId = traceIdFromRealtimeRefreshReason(reason); @@ -1525,7 +1565,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { return; } recordActivity(reason); - await refreshWorkbenchSyncReplay(id, traceId, null, `realtime-session-sync:${reason}`); + await refreshWorkbenchSyncReplay(id, traceId, null, `cross-tab-sync:${reason}`); } function completeTrace(traceId: string, result: AgentChatResultResponse, options: { forceRead?: boolean } = {}): void { @@ -1536,12 +1576,10 @@ export const useWorkbenchStore = defineStore("workbench", () => { projectTurnAuthorityToMessages(traceId, result, "complete-trace"); rememberTurnStatus(traceId, result); markWorkbenchTraceProjected(traceId); - clearActiveTraceSyncReplay(traceId); const ownerMessages = serverState.value.messagesBySessionId[ownerSessionId] ?? []; const terminalMessage = latestMessageForTrace(traceId, ownerMessages); if (!traceProjectionIsTerminalSealed(traceId, ownerMessages)) { if (options.forceRead) void refreshMessageProjectionForTrace(ownerSessionId, traceId, { force: true }); - else scheduleActiveTraceSyncReplay(traceId, "complete-trace-sync-replay", 0); } if (options.forceRead && terminalMessage && !traceProjectionIsTerminalSealed(traceId, ownerMessages)) void readTraceEventsForMessage(terminalMessage, { force: true }); if (ownerSessionId === activeSessionId.value) { @@ -1802,7 +1840,6 @@ export const useWorkbenchStore = defineStore("workbench", () => { } async function clearActiveTrace(traceId: string, reason: string): Promise { - clearActiveTraceSyncReplay(traceId); if (currentRequest.value?.traceId === traceId) currentRequest.value = null; } @@ -2010,7 +2047,7 @@ function normalizeChatMessage(message: ChatMessage): ChatMessage { const text = role === "agent" ? isTerminalMessageStatus(status) ? projectedAgentMessageText({ status, finalText, errorText, baseText }) - : "" + : firstNonEmptyString(baseText, finalText) ?? "" : firstNonEmptyString(baseText, finalText, errorText) ?? ""; const messageId = firstNonEmptyString((message as Record).messageId, message.id) ?? nextProtocolId("msg"); const timingPatch = isTerminalMessageStatus(status) ? terminalMessageTimingPatchForNormalize(message) : messageTimingPatch(message); @@ -2030,39 +2067,6 @@ function activeTraceIdFromMessages(messages: ChatMessage[], turnStatusAuthority: return null; } -function realtimeSnapshotToTraceSnapshot(traceId: string, snapshot: WorkbenchRealtimeEvent["snapshot"], eventPageEvents?: TraceEvent[]): TraceSnapshot { - const source = snapshot ?? { traceId }; - const events = Array.isArray(eventPageEvents) ? eventPageEvents : Array.isArray(source.events) ? source.events : []; - const { traceId: _sourceTraceId, error: _sourceError, ...sourceRest } = source; - return { - ...sourceRest, - traceId: optionalString(source.traceId, traceId), - status: firstNonEmptyString(source.status) ?? undefined, - events, - eventCount: firstFiniteNumber(source.eventCount, events.length), - fullTraceLoaded: source.fullTraceLoaded === true, - hasMore: source.hasMore === true, - truncated: source.truncated === true, - nextProjectedSeq: firstFiniteNumber(source.nextProjectedSeq) ?? null, - range: recordValue(source.range) as TraceSnapshot["range"], - agentRun: asAgentRun(source.agentRun) ?? undefined, - traceStatus: firstNonEmptyString(source.traceStatus) ?? undefined, - retention: source.retention, - terminalEvidence: source.terminalEvidence, - traceSummary: source.traceSummary, - error: traceSnapshotError(source.error), - projection: normalizeProjectionDiagnostic(source.projection ?? source) ?? null, - projectionStatus: firstNonEmptyString(source.projectionStatus, recordValue(source.projection)?.projectionStatus) ?? undefined, - projectionHealth: firstNonEmptyString(source.projectionHealth, recordValue(source.projection)?.projectionHealth) ?? undefined, - staleMs: firstFiniteNumber(source.staleMs, recordValue(source.projection)?.staleMs), - blocker: recordValue(source.blocker) ?? recordValue(recordValue(source.projection)?.blocker) ?? undefined, - ...messageTimingPatch(source), - lastEventLabel: firstNonEmptyString(source.lastEventLabel) ?? undefined, - eventSource: "trace-api", - updatedAt: firstNonEmptyString(source.updatedAt) ?? new Date().toISOString() - } as TraceSnapshot; -} - export async function liveCall(label: string, call: () => Promise>): Promise> { try { return await call(); diff --git a/web/hwlab-cloud-web/src/types/global.d.ts b/web/hwlab-cloud-web/src/types/global.d.ts index 17b5c110..8bbded04 100644 --- a/web/hwlab-cloud-web/src/types/global.d.ts +++ b/web/hwlab-cloud-web/src/types/global.d.ts @@ -8,6 +8,10 @@ declare global { label?: string; }; workbench?: { + realtimeFeatures?: { + liveKafkaSse?: boolean; + projectionRealtime?: boolean; + }; traceTimeline?: { autoExpandRunning?: boolean; autoCollapseTerminal?: boolean; @@ -54,14 +58,8 @@ declare global { sessionListTerminalRefreshDelayMs?: number; sessionListMinRefreshIntervalMs?: number; workbenchRealtimeErrorSyncReplayMinMs?: number; - workbenchActiveTraceSyncReplayInitialMs?: number; - workbenchActiveTraceSyncReplayRepeatMs?: number; /** @deprecated Use workbenchRealtimeErrorSyncReplayMinMs. */ workbenchRealtimeErrorGapFillMinMs?: number; - /** @deprecated Use workbenchActiveTraceSyncReplayInitialMs. */ - workbenchActiveTraceRestGapFillInitialMs?: number; - /** @deprecated Use workbenchActiveTraceSyncReplayRepeatMs. */ - workbenchActiveTraceRestGapFillRepeatMs?: number; }; }; opencode?: { diff --git a/web/hwlab-cloud-web/src/utils/workbench-realtime-runtime.ts b/web/hwlab-cloud-web/src/utils/workbench-realtime-runtime.ts index a0ec201c..b1eddb6a 100644 --- a/web/hwlab-cloud-web/src/utils/workbench-realtime-runtime.ts +++ b/web/hwlab-cloud-web/src/utils/workbench-realtime-runtime.ts @@ -1,4 +1,4 @@ // SPEC: PJ2026-0106050514 Workbench实时运行面 draft-2026-06-30-p0-1297-spec-first. // Responsibility: Production Workbench realtime runtime entry point for SSE transport, event coalescing, and scoped cursor ownership. -export { createWorkbenchStreamTransportRuntime, WorkbenchStreamTransportRuntime, type WorkbenchStreamTransportRecovery, type WorkbenchStreamTransportRestartInput, type WorkbenchStreamTransportRestartResult, type WorkbenchStreamTransportState, type WorkbenchRealtimeEvent } from "@/utils/workbench-stream-transport"; +export { createWorkbenchStreamTransportRuntime, WorkbenchStreamTransportRuntime, workbenchRealtimeTraceIdForCapabilities, type WorkbenchStreamTransportRecovery, type WorkbenchStreamTransportRestartInput, type WorkbenchStreamTransportRestartResult, type WorkbenchStreamTransportState, type WorkbenchRealtimeEvent } from "@/utils/workbench-stream-transport"; diff --git a/web/hwlab-cloud-web/src/utils/workbench-stream-transport.ts b/web/hwlab-cloud-web/src/utils/workbench-stream-transport.ts index 8d11d039..7a558a1e 100644 --- a/web/hwlab-cloud-web/src/utils/workbench-stream-transport.ts +++ b/web/hwlab-cloud-web/src/utils/workbench-stream-transport.ts @@ -8,11 +8,13 @@ // - OpenCode stream.transport.ts:504-530 reduce output/trace. import { connectWorkbenchEvents, type WorkbenchEventStream, type WorkbenchRealtimeEvent } from "@/api/workbench-events"; +import type { WorkbenchRealtimeCapabilities } from "@/config/runtime"; import { workbenchRealtimeScopeKey } from "@/utils/workbench-key"; export type { WorkbenchRealtimeEvent }; export interface WorkbenchStreamTransportRestartInput { + realtimeCapabilities: WorkbenchRealtimeCapabilities; sessionId?: string | null; traceId?: string | null; afterSeq?: number | null; @@ -74,6 +76,22 @@ interface WorkbenchTransportCursor { traceSeq: number | null; } +export function workbenchRealtimeTraceIdForCapabilities( + capabilities: WorkbenchRealtimeCapabilities, + ...candidates: Array +): string | null { + if (capabilities.liveKafkaSse) return null; + for (const candidate of candidates) { + const traceId = typeof candidate === "string" ? candidate.trim() : ""; + if (traceId) return traceId; + } + return null; +} + +export function workbenchRealtimeTransportEnabled(capabilities: WorkbenchRealtimeCapabilities): boolean { + return capabilities.liveKafkaSse || capabilities.projectionRealtime; +} + export class WorkbenchStreamTransportRuntime { private stream: WorkbenchEventStream | null = null; private key = ""; @@ -88,12 +106,14 @@ export class WorkbenchStreamTransportRuntime { this.stop(); this.key = key; this.armWait(key); + if (!workbenchRealtimeTransportEnabled(input.realtimeCapabilities)) return { key, changed: true, streamStarted: false }; if (!input.sessionId && !input.traceId) return { key, changed: true, streamStarted: false }; this.emitState(input, "connecting", null); this.stream = connectWorkbenchEvents({ + realtimeCapabilities: input.realtimeCapabilities, sessionId: input.sessionId ?? null, traceId: input.traceId ?? null, - afterSeq: input.afterSeq ?? this.cursorByKey.get(key)?.outboxSeq ?? null, + afterSeq: !input.realtimeCapabilities.liveKafkaSse && input.realtimeCapabilities.projectionRealtime ? input.afterSeq ?? this.cursorByKey.get(key)?.outboxSeq ?? null : null, flushMaxItemsPerChunk: input.flushMaxItemsPerChunk, flushMaxChunkMs: input.flushMaxChunkMs, flushYieldMs: input.flushYieldMs, @@ -109,7 +129,7 @@ export class WorkbenchStreamTransportRuntime { this.requestRecovery(input, event.type || "eventsource-error"); }, onEvent: (event, eventName) => { - this.rememberCursor(key, event); + if (!input.realtimeCapabilities.liveKafkaSse && input.realtimeCapabilities.projectionRealtime) this.rememberCursor(key, event); this.markLive(key); this.emitState(input, "event", eventName); input.onEvent(event, eventName); @@ -180,7 +200,7 @@ export class WorkbenchStreamTransportRuntime { const sessionId = input.sessionId ?? null; const traceId = input.traceId ?? null; const cursor = this.currentCursor(key); - const actions: WorkbenchStreamTransportRecoveryAction[] = sessionId || traceId ? ["sync-replay"] : []; + const actions: WorkbenchStreamTransportRecoveryAction[] = !input.realtimeCapabilities.liveKafkaSse && input.realtimeCapabilities.projectionRealtime && (sessionId || traceId) ? ["sync-replay"] : []; const diagnostic = this.diagnosticEnvelope("workbench_sse_recovery", reason, key, actions); input.onRecovery?.({ key, sessionId, traceId, tick: this.wait?.tick ?? this.tick, reason, actions, outboxSeq: cursor.outboxSeq, traceSeq: cursor.traceSeq, diagnostic }); } diff --git a/web/hwlab-cloud-web/src/views/workbench/WorkbenchDebugView.vue b/web/hwlab-cloud-web/src/views/workbench/WorkbenchDebugView.vue index 11687892..68fa1a13 100644 --- a/web/hwlab-cloud-web/src/views/workbench/WorkbenchDebugView.vue +++ b/web/hwlab-cloud-web/src/views/workbench/WorkbenchDebugView.vue @@ -3,7 +3,7 @@ // Responsibility: Workbench debug route for backend-driven fake SSE single-step Trace card inspection. import { computed, onBeforeUnmount, onMounted, ref } from "vue"; -import { connectWorkbenchDebugFakeSse, connectWorkbenchKafkaSseDebug, workbenchDebugAPI, type WorkbenchDebugFakeSseQueue, type WorkbenchDebugFakeSseSequence, type WorkbenchDebugFakeSseStream, type WorkbenchKafkaSseDebugEvent, type WorkbenchKafkaSseDebugStreamName } from "@/api"; +import { connectWorkbenchDebugFakeSse, connectWorkbenchKafkaSseDebug, workbenchDebugAPI, workbenchKafkaSseDebugCorrelationIds, type WorkbenchDebugFakeSseQueue, type WorkbenchDebugFakeSseSequence, type WorkbenchDebugFakeSseStream, type WorkbenchKafkaSseDebugEvent, type WorkbenchKafkaSseDebugStream, type WorkbenchKafkaSseDebugStreamName } from "@/api"; import StatusBadge from "@/components/common/StatusBadge.vue"; import WorkbenchMessageCard from "@/components/workbench/WorkbenchMessageCard.vue"; import type { WorkbenchRealtimeEvent } from "@/api/workbench-events"; @@ -21,14 +21,16 @@ const kafkaStreamName = ref("hwlab"); const kafkaFromBeginning = ref(false); const kafkaFilters = ref({ traceId: "", sessionId: "", runId: "", commandId: "" }); const kafkaEvents = ref([]); +const kafkaConnection = ref(null); const kafkaTraceStatuses = ref>(createKafkaStatusMap()); const kafkaTraceEvents = ref>(createKafkaEventMap()); +const kafkaTraceConnections = ref>(createKafkaConnectionMap()); const busy = ref(false); const error = ref(null); const appendDraft = ref(defaultAppendDraft()); let stream: WorkbenchDebugFakeSseStream | null = null; -let kafkaStream: WorkbenchDebugFakeSseStream | null = null; -const kafkaTraceStreams: Partial> = {}; +let kafkaStream: WorkbenchKafkaSseDebugStream | null = null; +const kafkaTraceStreams: Partial> = {}; const KAFKA_TRACE_STREAMS: Array<{ name: WorkbenchKafkaSseDebugStreamName; label: string; topic: string }> = [ { name: "stdio", label: "codex stdio", topic: "codex-stdio.raw.v1" }, @@ -100,6 +102,7 @@ function openStream(): void { function openKafkaStream(): void { kafkaStream?.close(); kafkaStatus.value = "connecting"; + kafkaConnection.value = null; kafkaStream = connectWorkbenchKafkaSseDebug({ stream: kafkaStreamName.value, fromBeginning: kafkaFromBeginning.value, @@ -110,7 +113,10 @@ function openKafkaStream(): void { onOpen: () => { kafkaStatus.value = "open"; }, onError: () => { kafkaStatus.value = "error"; }, onEvent: (event, eventName) => { - if (eventName === "hwlab.kafka.connected") return; + if (eventName === "hwlab.kafka.connected") { + kafkaConnection.value = event; + return; + } if (eventName === "hwlab.kafka.error") { kafkaStatus.value = "error"; error.value = event.error?.message || "Kafka SSE 事件流错误"; @@ -134,6 +140,7 @@ function clearKafkaEvents(): void { function openKafkaTraceStreams(): void { closeKafkaTraceStreams(); kafkaTraceEvents.value = createKafkaEventMap(); + kafkaTraceConnections.value = createKafkaConnectionMap(); kafkaTraceStatuses.value = createKafkaStatusMap("connecting"); for (const streamName of KAFKA_TRACE_STREAMS.map((entry) => entry.name)) { const opened = connectWorkbenchKafkaSseDebug({ @@ -146,7 +153,10 @@ function openKafkaTraceStreams(): void { onOpen: () => setKafkaTraceStatus(streamName, "open"), onError: () => setKafkaTraceStatus(streamName, "error"), onEvent: (event, eventName) => { - if (eventName === "hwlab.kafka.connected") return; + if (eventName === "hwlab.kafka.connected") { + kafkaTraceConnections.value = { ...kafkaTraceConnections.value, [streamName]: event }; + return; + } if (eventName === "hwlab.kafka.error") { setKafkaTraceStatus(streamName, "error"); error.value = event.error?.message || "Kafka SSE 三流订阅错误"; @@ -187,6 +197,10 @@ function createKafkaEventMap(): Record { + return { stdio: null, agentrun: null, hwlab: null }; +} + async function resetQueue(): Promise { busy.value = true; error.value = null; @@ -264,9 +278,7 @@ function kafkaEventSummary(event: WorkbenchKafkaSseDebugEvent): string { } function kafkaEventIds(event: WorkbenchKafkaSseDebugEvent): string { - const value = recordValue(event.value); - const context = recordValue(value.context); - return [value.traceId, value.sessionId, context.runId || value.runId, context.commandId || value.commandId].filter(Boolean).join(" · ") || "-"; + return workbenchKafkaSseDebugCorrelationIds(event).join(" · ") || "-"; } function kafkaEventJson(event: WorkbenchKafkaSseDebugEvent): string { @@ -410,6 +422,7 @@ function defaultAppendDraft(): string {
{{ streamEntry.label }} {{ streamEntry.topic }} + consumer ready · {{ kafkaTraceConnections[streamEntry.name]?.groupId || '-' }}
@@ -435,7 +448,8 @@ function defaultAppendDraft(): string {
Raw SSE Events - {{ kafkaEvents.length }} events + consumer ready · {{ kafkaConnection.groupId || '-' }} · {{ kafkaEvents.length }} events + {{ kafkaEvents.length }} events
等待后端从 Kafka 订阅并透传 hwlab event。
    @@ -735,6 +749,15 @@ function defaultAppendDraft(): string { white-space: nowrap; } +.kafka-consumer-ready { + overflow: hidden; + color: #047857; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + font-size: 10px; + text-overflow: ellipsis; + white-space: nowrap; +} + .kafka-debug-panel { display: grid; min-height: 0;