From ce83ffdf49462ad18b80d75679ad8f449cdbee4e Mon Sep 17 00:00:00 2001 From: root Date: Fri, 10 Jul 2026 05:23:03 +0200 Subject: [PATCH] feat: make Kafka projector the Workbench realtime authority --- deploy/deploy.yaml | 17 +- internal/cloud/backend-performance.test.ts | 107 +- internal/cloud/backend-performance.ts | 10 +- internal/cloud/bun-server.ts | 8 +- internal/cloud/code-agent-agentrun-adapter.ts | 488 +- .../cloud/code-agent-agentrun-cancel.test.ts | 43 +- ...e-agent-agentrun-durable-admission.test.ts | 152 +- .../code-agent-agentrun-projection-polling.ts | 350 -- internal/cloud/code-agent-agentrun-result.ts | 29 +- internal/cloud/code-agent-trace-store.test.ts | 11 +- internal/cloud/code-agent-trace-store.ts | 17 +- internal/cloud/health-contract.ts | 23 +- internal/cloud/kafka-event-bridge.test.ts | 274 +- internal/cloud/kafka-event-bridge.ts | 392 +- internal/cloud/kafka-shadow-producer.ts | 185 - ...server-agent-chat-agentrun-backoff.test.ts | 130 - .../cloud/server-agent-chat-lifecycle.test.ts | 2 +- .../cloud/server-agent-chat-session.test.ts | 392 +- internal/cloud/server-agent-chat.test.ts | 232 +- .../cloud/server-code-agent-admission-http.ts | 1414 +++++ .../cloud/server-code-agent-http-support.ts | 1602 ++++++ internal/cloud/server-code-agent-http.ts | 4761 +---------------- .../cloud/server-code-agent-trace-http.ts | 434 ++ internal/cloud/server-code-agent-turn-http.ts | 892 +++ internal/cloud/server-health.test.ts | 12 +- internal/cloud/server-hwpod-http.ts | 528 ++ internal/cloud/server-workbench-facts.ts | 1 - internal/cloud/server-workbench-read-http.ts | 1 - .../server-workbench-realtime-http.test.ts | 347 +- .../cloud/server-workbench-realtime-http.ts | 511 +- internal/cloud/server.ts | 547 +- .../cloud/user-billing-integration.test.ts | 152 - internal/cloud/workbench-facts-store.ts | 55 +- .../cloud/workbench-kafka-sse-debug.test.ts | 172 - internal/cloud/workbench-kafka-sse-debug.ts | 216 - .../cloud/workbench-projection-cursor.test.ts | 153 - internal/cloud/workbench-projection-cursor.ts | 205 - .../cloud/workbench-projection-finalizer.ts | 106 - .../workbench-projection-outbox-events.ts | 99 + .../cloud/workbench-projection-writer.test.ts | 63 +- internal/cloud/workbench-projection-writer.ts | 171 +- internal/cloud/workbench-read-model.ts | 1 - ...kbench-realtime-authority-contract.test.ts | 120 +- .../cloud/workbench-realtime-authority.ts | 98 +- internal/cloud/workbench-runtime-client.ts | 1 + .../0008_workbench_kafka_realtime.sql | 106 + internal/db/runtime-store-core.ts | 9 +- internal/db/runtime-store-memory.ts | 1 - .../db/runtime-store-postgres-kafka.test.ts | 179 + internal/db/runtime-store-postgres-kafka.ts | 410 ++ .../db/runtime-store-postgres-notify.test.ts | 26 + internal/db/runtime-store-postgres-notify.ts | 145 + internal/db/runtime-store-postgres.ts | 104 +- internal/db/runtime-store.test.ts | 45 +- internal/db/schema.test.ts | 41 +- internal/db/schema.ts | 62 +- internal/workbenchruntime/realtime_sync.go | 52 +- .../workbenchruntime/realtime_sync_test.go | 51 + scripts/src/dev-runtime-migration.mjs | 32 +- scripts/src/dev-runtime-migration.test.mjs | 8 +- web/hwlab-cloud-web/scripts/check.ts | 8 +- web/hwlab-cloud-web/src/api/index.ts | 2 +- .../src/api/workbench-debug.ts | 85 - .../src/config/workbench-runtime-policy.ts | 10 +- .../src/stores/workbench-colada.test.ts | 5 +- .../src/stores/workbench-debug-fake-sse.ts | 6 +- ...rkbench-message-projection-runtime.test.ts | 38 +- .../src/stores/workbench-server-state.ts | 7 +- .../src/stores/workbench-session.test.ts | 5 +- .../src/stores/workbench-trace-detail.test.ts | 2 +- .../src/stores/workbench-trace-detail.ts | 42 + web/hwlab-cloud-web/src/stores/workbench.ts | 142 +- web/hwlab-cloud-web/src/types/global.d.ts | 6 - .../views/workbench/WorkbenchDebugView.vue | 456 +- 74 files changed, 7746 insertions(+), 9863 deletions(-) delete mode 100644 internal/cloud/code-agent-agentrun-projection-polling.ts delete mode 100644 internal/cloud/kafka-shadow-producer.ts delete mode 100644 internal/cloud/server-agent-chat-agentrun-backoff.test.ts create mode 100644 internal/cloud/server-code-agent-admission-http.ts create mode 100644 internal/cloud/server-code-agent-http-support.ts create mode 100644 internal/cloud/server-code-agent-trace-http.ts create mode 100644 internal/cloud/server-code-agent-turn-http.ts create mode 100644 internal/cloud/server-hwpod-http.ts delete mode 100644 internal/cloud/workbench-kafka-sse-debug.test.ts delete mode 100644 internal/cloud/workbench-kafka-sse-debug.ts delete mode 100644 internal/cloud/workbench-projection-cursor.test.ts delete mode 100644 internal/cloud/workbench-projection-cursor.ts delete mode 100644 internal/cloud/workbench-projection-finalizer.ts create mode 100644 internal/cloud/workbench-projection-outbox-events.ts create mode 100644 internal/db/migrations/0008_workbench_kafka_realtime.sql create mode 100644 internal/db/runtime-store-postgres-kafka.test.ts create mode 100644 internal/db/runtime-store-postgres-kafka.ts create mode 100644 internal/db/runtime-store-postgres-notify.test.ts create mode 100644 internal/db/runtime-store-postgres-notify.ts diff --git a/deploy/deploy.yaml b/deploy/deploy.yaml index c4b7c753..c4b8a753 100644 --- a/deploy/deploy.yaml +++ b/deploy/deploy.yaml @@ -521,19 +521,9 @@ lanes: TZ: Asia/Shanghai HWLAB_METRICS_NAMESPACE: hwlab-v03 HWLAB_OBSERVABILITY_CONFIG_REVISION: PJ2026-01060505-20260619-long-trace-timeout-mitigation - HWLAB_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS: "30000" - HWLAB_CODE_AGENT_AGENTRUN_NO_RESPONSE_TIMEOUT_MS: "600000" HWLAB_CODE_AGENT_AGENTRUN_DISPATCH_RETRY_MAX: "5" HWLAB_CODE_AGENT_AGENTRUN_DISPATCH_RETRY_BASE_MS: "1000" HWLAB_CODE_AGENT_AGENTRUN_DISPATCH_RETRY_MAX_DELAY_MS: "30000" - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_POLL_INTERVAL_MS: "1000" - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_ENABLED: "1" - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INITIAL_DELAY_MS: "5000" - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INTERVAL_MS: "60000" - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS: "30000" - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE: "100" - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_JITTER_MS: "15000" - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_FAILURE_BACKOFF_MS: "120000" observable: true hwlab-workbench-runtime: runtimeKind: go-service @@ -845,6 +835,13 @@ lanes: HWLAB_KAFKA_EVENT_TOPIC: hwlab.event.v1 HWLAB_KAFKA_CLIENT_ID: hwlab-v03-cloud-api HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID: hwlab-v03-agentrun-event-bridge + HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS: "2000" + HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS: "250" + HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE: "100" + HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS: "30000" + HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS: "10000" + HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS: "1000" + HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" HWLAB_WORKBENCH_EMPTY_SESSION_GC_ENABLED: "1" HWLAB_WORKBENCH_EMPTY_SESSION_TTL_MS: "600000" HWLAB_WORKBENCH_EMPTY_SESSION_GC_INTERVAL_MS: "60000" diff --git a/internal/cloud/backend-performance.test.ts b/internal/cloud/backend-performance.test.ts index 0ee1bf49..fb20b6cc 100644 --- a/internal/cloud/backend-performance.test.ts +++ b/internal/cloud/backend-performance.test.ts @@ -6,18 +6,18 @@ import { createServer as createHttpServer } from "node:http"; import { test } from "bun:test"; import { createBackendPerformanceStore, backendPerformanceRouteTemplate, withBackendPerformanceContext } from "./backend-performance.ts"; -import { submitAgentRunChatTurn, syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts"; +import { submitAgentRunChatTurn } from "./code-agent-agentrun-adapter.ts"; import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; import { createCloudApiServer } from "./server.ts"; -import { createCodeAgentChatResultStore } from "./server-code-agent-http.ts"; test("backend performance store templates routes and appends Cloud API metrics to loopback metrics", async () => { - assert.equal(backendPerformanceRouteTemplate("/v1/agent/traces/trc_secret_0123456789abcdef?full=1"), "/v1/agent/traces/:id"); + assert.equal(backendPerformanceRouteTemplate("/v1/agent/traces/trc_secret_0123456789abcdef?full=1"), "/v1/agent/traces/:traceId"); assert.equal(backendPerformanceRouteTemplate("/api/v1/runs/run_0123456789abcdef/commands/cmd_0123456789abcdef/result"), "/api/v1/runs/:id/commands/:id/result"); assert.equal(backendPerformanceRouteTemplate("/v1/admin/billing/users/12345/credits/adjust"), "/v1/admin/billing/users/:id/credits/adjust"); const backendPerformanceStore = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } }); - const server = createCloudApiServer({ backendPerformanceStore, env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } }); + const accessController = { required: true, async authenticate() { return { ok: true, actor: { id: "usr_backend_metrics", role: "admin" }, session: { id: "web_backend_metrics" } }; } }; + const server = createCloudApiServer({ backendPerformanceStore, accessController, env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); try { const address = server.address(); @@ -37,7 +37,6 @@ test("backend performance store templates routes and appends Cloud API metrics t await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); } }); - test("backend performance store exposes low-cardinality request evidence for RUM alignment", () => { const store = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } }); const context = store.createManualContext({ route: "/v1/workbench/sessions/ses_secret/messages", method: "GET" }); @@ -79,10 +78,9 @@ test("backend performance records Code Agent trace handler phases without high-c const runId = "run_backend_phase"; const commandId = "cmd_backend_phase"; const traceStore = createCodeAgentTraceStore(); - const codeAgentChatResults = createCodeAgentChatResultStore(); traceStore.append(traceId, { type: "message", role: "assistant", label: "assistant:start", sourceSeq: 1, commandId }); traceStore.append(traceId, { type: "result", status: "completed", terminal: true, label: "result:completed", sourceSeq: 2, commandId }); - codeAgentChatResults.set(traceId, { + const projectedResult = { status: "completed", conversationId: "cnv_backend_phase", sessionId: "ses_backend_phase", @@ -107,9 +105,16 @@ test("backend performance records Code Agent trace handler phases without high-c terminalStatus: "completed", eventCount: 2, updatedAt: "2026-06-18T00:00:00.000Z" - } - }); + }, + runnerTrace: traceStore.snapshot(traceId) + }; const backendPerformanceStore = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } }); + const workbenchRuntime = { + async queryAgentTraceEvents({ traceId: requestedTraceId }: { traceId: string }) { + assert.equal(requestedTraceId, traceId); + return { events: traceStore.snapshot(traceId).events }; + } + }; const accessController = { required: true, async authenticate() { @@ -123,11 +128,11 @@ test("backend performance records Code Agent trace handler phases without high-c projectId: "prj_hwpod_workbench", ownerUserId: "usr_backend_phase", ownerRole: "user", - session: {} + session: { traceResults: { [traceId]: projectedResult } } }; } }; - const server = createCloudApiServer({ backendPerformanceStore, traceStore, codeAgentChatResults, accessController, env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } }); + const server = createCloudApiServer({ backendPerformanceStore, traceStore, workbenchRuntime, accessController, env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); try { const address = server.address(); @@ -145,8 +150,8 @@ test("backend performance records Code Agent trace handler phases without high-c const metrics = await fetch(`${baseUrl}/v1/web-performance/metrics`); const text = await metrics.text(); assert.equal(metrics.status, 200); - assert.match(text, /hwlab_cloud_api_request_phase_duration_seconds_count\{[^}]*route="\/v1\/agent\/traces\/:id"[^}]*phase="trace_project_check"[^}]*\} 1/u); - assert.match(text, /hwlab_cloud_api_request_phase_duration_seconds_count\{[^}]*route="\/v1\/agent\/traces\/:id"[^}]*phase="trace_paginate"[^}]*\} 1/u); + assert.match(text, /hwlab_cloud_api_request_phase_duration_seconds_count\{[^}]*route="\/v1\/agent\/traces\/:traceId"[^}]*phase="trace_project_check"[^}]*\} 1/u); + assert.match(text, /hwlab_cloud_api_request_phase_duration_seconds_count\{[^}]*route="\/v1\/agent\/traces\/:traceId"[^}]*phase="trace_paginate"[^}]*\} 1/u); assert.doesNotMatch(text, /trc_backend_phase_trace|run_backend_phase|cmd_backend_phase|ses_backend_phase|cnv_backend_phase/u); } finally { await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); @@ -167,8 +172,7 @@ test("AgentRun adapter records upstream timing through backend performance conte }; if (request.method === "POST" && url.pathname === "/api/v1/sessions") return send({ ok: true }); if (request.method === "POST" && url.pathname === "/api/v1/runs") return send({ id: "run_backend_perf", status: "pending", backendProfile: "codex", sessionRef: body.sessionRef, resourceBundleRef: body.resourceBundleRef }); - if (request.method === "POST" && url.pathname === "/api/v1/runs/run_backend_perf/commands") return send({ id: "cmd_backend_perf", runId: "run_backend_perf", state: "pending", type: "turn", seq: 1 }); - if (request.method === "POST" && url.pathname === "/api/v1/runs/run_backend_perf/runner-jobs") return send({ action: "create-kubernetes-job", runId: "run_backend_perf", commandId: body.commandId, attemptId: "attempt_backend_perf", runnerId: "runner_backend_perf", namespace: "agentrun-v01", jobName: "agentrun-v01-runner-backend-perf" }); + if (request.method === "POST" && url.pathname === "/api/v1/runs/run_backend_perf/commands") return send({ id: "cmd_backend_perf", runId: "run_backend_perf", state: "pending", type: "turn", seq: 1, dispatchIntent: { id: "dispatch_backend_perf", state: "pending", runnerJobId: "rjob_backend_perf", attemptCount: 0, durable: true } }); response.writeHead(404, { "content-type": "application/json" }); response.end(`${JSON.stringify({ ok: false, message: `unexpected ${request.method} ${url.pathname}` })}\n`); }); @@ -215,11 +219,11 @@ test("AgentRun adapter records upstream timing through backend performance conte })); assert.equal(result.status, "running"); - assert.ok(calls.some((call) => call.path === "/api/v1/runs/run_backend_perf/runner-jobs")); + assert.ok(!calls.some((call) => call.path === "/api/v1/runs/run_backend_perf/runner-jobs")); const text = store.metricsText(); assert.match(text, /hwlab_cloud_api_upstream_duration_seconds_count\{[^}]*component="agentrun"[^}]*route="\/api\/v1\/runs"[^}]*\} 1/u); assert.match(text, /hwlab_cloud_api_upstream_duration_seconds_count\{[^}]*component="agentrun"[^}]*route="\/api\/v1\/runs\/:id\/commands"[^}]*\} 1/u); - assert.match(text, /hwlab_cloud_api_upstream_duration_seconds_count\{[^}]*component="agentrun"[^}]*route="\/api\/v1\/runs\/:id\/runner-jobs"[^}]*\} 1/u); + assert.doesNotMatch(text, /runner-jobs/u); assert.doesNotMatch(text, /run_backend_perf|cmd_backend_perf|ses_backend_perf|cnv_backend_perf/u); } finally { await new Promise((resolve, reject) => agentRunServer.close((error) => error ? reject(error) : resolve())); @@ -289,72 +293,3 @@ test("backend performance reports zero projection lag for caught-up terminal tra assert.ok(stuckLine?.endsWith(" 0")); assert.ok(terminalDelayBucketLine?.endsWith(" 1")); }); - -test("AgentRun read-side sync advances event cursor without forcing result aggregation before terminal evidence", async () => { - const calls: string[] = []; - const agentRunServer = createHttpServer(async (request, response) => { - const url = new URL(request.url || "/", "http://127.0.0.1"); - calls.push(`${request.method} ${url.pathname}`); - const send = (data: Record) => { - response.writeHead(200, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: true, data })}\n`); - }; - if (request.method === "GET" && url.pathname === "/api/v1/runs/run_read_side/events") { - return send({ - items: [ - { seq: 11, type: "assistant_message", runId: "run_read_side", payload: { commandId: "cmd_read_side", text: "working" }, createdAt: "2026-06-19T02:00:01.000Z" }, - { seq: 12, type: "tool_call", runId: "run_read_side", payload: { commandId: "cmd_read_side", toolName: "shell" }, createdAt: "2026-06-19T02:00:02.000Z" }, - ], - }); - } - if (request.method === "GET" && url.pathname.endsWith("/result")) return send({ terminalStatus: "completed", text: "should not be fetched" }); - response.writeHead(404, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: false, message: `unexpected ${request.method} ${url.pathname}` })}\n`); - }); - await new Promise((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve)); - try { - const address = agentRunServer.address(); - assert.ok(address && typeof address === "object"); - const store = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03", HWLAB_RUNTIME_LANE: "v03", HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601" } }); - const traceStore = createCodeAgentTraceStore(); - const synced = await syncAgentRunChatResult({ - traceId: "trc_read_side_cursor", - currentResult: { - status: "running", - traceId: "trc_read_side_cursor", - agentRun: { - adapter: "agentrun-v01", - runId: "run_read_side", - commandId: "cmd_read_side", - traceId: "trc_read_side_cursor", - providerTrace: { traceId: "trc_read_side_cursor" }, - lastSeq: 10, - managerUrl: `http://127.0.0.1:${address.port}`, - backendProfile: "codex", - }, - }, - options: { - backendPerformanceStore: store, - env: { - HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1", - HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601", - HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS: "2500", - }, - }, - traceStore, - forceResultSync: false, - }); - - assert.equal(synced.resultSynced, false); - assert.equal(synced.eventsRefreshed, true); - assert.equal(synced.result.agentRun.lastSeq, 12); - assert.ok(calls.some((call) => call === "GET /api/v1/runs/run_read_side/events")); - assert.ok(!calls.some((call) => call.endsWith("/result"))); - const text = store.metricsText(); - assert.match(text, /hwlab_workbench_projection_cursor_advance_total\{[^}]*status="advanced"[^}]*\} 2/u); - assert.match(text, /hwlab_workbench_projector_events_processed_total\{[^}]*event_kind="assistant_message"[^}]*\} 1/u); - assert.match(text, /hwlab_workbench_projector_batch_duration_seconds_count\{[^}]*phase="agentrun_events_fetch"[^}]*status="ok"[^}]*\} 1/u); - } finally { - await new Promise((resolve, reject) => agentRunServer.close((error) => error ? reject(error) : resolve())); - } -}); diff --git a/internal/cloud/backend-performance.ts b/internal/cloud/backend-performance.ts index 07b7703e..17b708cc 100644 --- a/internal/cloud/backend-performance.ts +++ b/internal/cloud/backend-performance.ts @@ -582,19 +582,19 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp "# HELP hwlab_agentrun_result_timeout_total AgentRun result timeout count by refresh budget.", "# TYPE hwlab_agentrun_result_timeout_total counter", ...renderCounters("hwlab_agentrun_result_timeout_total", agentRunResultTimeoutSeries), - "# HELP hwlab_workbench_projector_batch_duration_seconds Workbench projector/finalizer batch phase duration in seconds.", + "# HELP hwlab_workbench_projector_batch_duration_seconds Workbench Kafka projector batch phase duration in seconds.", "# TYPE hwlab_workbench_projector_batch_duration_seconds histogram", ...renderHistogram("hwlab_workbench_projector_batch_duration_seconds", projectorBatchDurationSeries, PROJECTOR_BATCH_BUCKETS_SECONDS), - "# HELP hwlab_workbench_projector_candidates_total Workbench projector/finalizer candidate count.", + "# HELP hwlab_workbench_projector_candidates_total Workbench Kafka projector candidate count.", "# TYPE hwlab_workbench_projector_candidates_total counter", ...renderCounters("hwlab_workbench_projector_candidates_total", projectorCandidatesSeries), - "# HELP hwlab_workbench_projector_events_processed_total Workbench projector/finalizer processed event count.", + "# HELP hwlab_workbench_projector_events_processed_total Workbench Kafka projector processed event count.", "# TYPE hwlab_workbench_projector_events_processed_total counter", ...renderCounters("hwlab_workbench_projector_events_processed_total", projectorEventsProcessedSeries), - "# HELP hwlab_workbench_projector_errors_total Workbench projector/finalizer error count.", + "# HELP hwlab_workbench_projector_errors_total Workbench Kafka projector error count.", "# TYPE hwlab_workbench_projector_errors_total counter", ...renderCounters("hwlab_workbench_projector_errors_total", projectorErrorsSeries), - "# HELP hwlab_workbench_projector_last_success_unixtime Last successful Workbench projector/finalizer sync unix time.", + "# HELP hwlab_workbench_projector_last_success_unixtime Last successful Workbench Kafka projector transaction unix time.", "# TYPE hwlab_workbench_projector_last_success_unixtime gauge", ...renderGauges("hwlab_workbench_projector_last_success_unixtime", projectorLastSuccessSeries), "" diff --git a/internal/cloud/bun-server.ts b/internal/cloud/bun-server.ts index 871e5b18..78430ebc 100644 --- a/internal/cloud/bun-server.ts +++ b/internal/cloud/bun-server.ts @@ -11,6 +11,12 @@ export async function createCloudApiBunServer(options: any = {}) { installPostgresTransientUnhandledRejectionBoundary({ env, logger }); const hwpodNodeWsRegistry = options.hwpodNodeWsRegistry || createHwpodNodeWsRegistry({ ...options, env }); const innerServer = createCloudApiServer({ ...options, env, hwpodNodeWsRegistry }); + try { + await innerServer.hwlabStartupReady; + } catch (error) { + await innerServer.hwlabAbortStartup?.(); + throw error; + } await new Promise((resolve) => innerServer.listen(0, "127.0.0.1", resolve)); const innerAddress = innerServer.address(); const innerPort = typeof innerAddress === "object" && innerAddress ? innerAddress.port : 0; @@ -19,7 +25,7 @@ export async function createCloudApiBunServer(options: any = {}) { const host = options.host ?? "0.0.0.0"; const port = options.port ?? 0; const idleTimeout = parsePositiveInteger(options.idleTimeout ?? env.HWLAB_CLOUD_API_IDLE_TIMEOUT_SECONDS, 120); - const server = Bun.serve({ + const server = (options.bunServe ?? Bun.serve)({ hostname: host, port, idleTimeout, diff --git a/internal/cloud/code-agent-agentrun-adapter.ts b/internal/cloud/code-agent-agentrun-adapter.ts index 0658f7dc..268c495d 100644 --- a/internal/cloud/code-agent-agentrun-adapter.ts +++ b/internal/cloud/code-agent-agentrun-adapter.ts @@ -1,5 +1,5 @@ // SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-17-r0; draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. -// Responsibility: AgentRun v0.1 adapter, incremental Workbench projection cursor sync, and upstream timing projection for Cloud API observability. +// Responsibility: AgentRun command admission and control adapter; Kafka owns event/result projection. import { randomUUID } from "node:crypto"; @@ -13,32 +13,12 @@ import { safeTraceId, text } from "./server-http-utils.ts"; -import { currentBackendPerformanceContext } from "./backend-performance.ts"; import { DEFAULT_AGENTRUN_MGR_URL, agentRunJson, isAgentRunManagerInternalServiceHost, resolveAgentRunManagerUrl } from "./code-agent-agentrun-runtime.ts"; -import { - agentRunErrorWithPolling, - agentRunEventCommandId, - agentRunMetricStatusForError, - agentRunPollingRetryableError, - agentRunPollingRootCause, - agentRunProjectionPollCount, - agentRunProjectionPollingState, - agentRunTerminalConsistencyGap, - agentRunTerminalResultRefreshSatisfied, - agentRunTerminalStatusFromEvents, - agentRunTraceCursorSeq, - emitAgentRunProjectionPollingSpan, - fetchAgentRunEventsForTrace, - measuredAgentRunEventsFetch, - measuredAgentRunResultFetch, - nonNegativeNumber, - recordAgentRunEventsProjectionMetrics -} from "./code-agent-agentrun-projection-polling.ts"; import { agentRunAlreadyTerminalCancelPayload, agentRunCancelBlockedPayload, @@ -65,7 +45,6 @@ import { agentRunPromptMetadataFromText, messageAuthorityTextValue } from "./code-agent-agentrun-prompt.ts"; -import { agentRunResultSyncDeferred, buildAgentRunProjectionEventsFetchPlan, buildWorkbenchProjectionStateUpdate } from "./workbench-projection-cursor.ts"; import { codeAgentOtelTraceFields, codeAgentOtelTraceContext, emitCodeAgentOtelSpan } from "./otel-trace.ts"; import * as agentRunResult from "./code-agent-agentrun-result.ts"; @@ -700,333 +679,11 @@ function sleepAgentRunDispatchRetry(ms) { return new Promise((resolve) => setTimeout(resolve, Math.max(0, Number(ms) || 0))); } -export async function syncAgentRunChatResult({ traceId, currentResult = null, options = {}, traceStore = defaultCodeAgentTraceStore, appendResultEvent = true, refreshEvents = true, forceResultSync = false, retryParams = null }) { - const initial = currentResult ?? await loadPersistedAgentRunResult(traceId, options); - if (isAgentRunUserCancelSealed(initial)) { - const runnerTrace = initial.runnerTrace ?? traceStore.snapshot(traceId); - return { result: initial, runnerTrace, found: true, eventsRefreshed: false, resultSynced: false, terminalRefreshSkipped: true, localCancelSealed: true }; - } - const mapped = initial ? await resolveAgentRunTraceCommandMapping({ traceId, mapped: initial, options }) : initial; - if (isAgentRunUserCancelSealed(mapped)) { - const runnerTrace = mapped.runnerTrace ?? traceStore.snapshot(traceId); - return { result: mapped, runnerTrace, found: true, eventsRefreshed: false, resultSynced: false, terminalRefreshSkipped: true, localCancelSealed: true }; - } - const env = options.env ?? process.env; - const performanceStore = options.backendPerformanceStore ?? currentBackendPerformanceContext(); - if (!forceResultSync && agentRunTerminalResultRefreshSatisfied(mapped, traceId)) { - const runnerTrace = mapped.runnerTrace ?? traceStore.snapshot(traceId); - const polling = agentRunProjectionPollingState({ env, terminalFromEvents: "completed", rootCause: "terminal_result_sealed", terminalConsistencyGap: 0 }); - emitAgentRunProjectionPollingSpan({ traceId, env, agentRun: mapped?.agentRun, polling, terminalRefreshSkipped: true }); - return { result: mapped, runnerTrace, found: true, eventsRefreshed: false, resultSynced: false, terminalRefreshSkipped: true, polling }; - } - if (!mapped?.agentRun?.runId || !mapped?.agentRun?.commandId) return { result: currentResult, runnerTrace: traceStore.snapshot(traceId), found: Boolean(currentResult), eventsRefreshed: false, resultSynced: false }; - const fetchImpl = options.fetchImpl ?? globalThis.fetch; - const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl); - const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000); - const projectionState = await loadWorkbenchProjectionStateForAgentRun(options.runtimeStore, { traceId, agentRun: mapped.agentRun }); - const projectionRetryParams = retryParams ?? projectionState?.retryParams ?? null; - const projectionPollCount = agentRunProjectionPollCount(projectionState); - const fetchPlan = buildAgentRunProjectionEventsFetchPlan({ - projectionState, - agentRun: mapped.agentRun, - pageLimit: env.HWLAB_CODE_AGENT_AGENTRUN_EVENTS_PAGE_LIMIT - }); - const previousLastSeq = Number(fetchPlan.afterSeq ?? mapped.agentRun.lastSeq ?? 0); - let eventsResponse = null; - try { - eventsResponse = refreshEvents ? await measuredAgentRunEventsFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapping: { ...mapped.agentRun, lastSeq: fetchPlan.afterSeq, afterSeqOverride: fetchPlan.afterSeq, eventsPageLimit: fetchPlan.limit, traceSummary: mapped.traceSummary, pollCount: projectionPollCount, backoffMs: nonNegativeNumber(projectionState?.backoffMs), rootCause: projectionState?.rootCause ?? null, terminalConsistencyGap: nonNegativeNumber(projectionState?.terminalConsistencyGap) } }) : null; - } catch (error) { - const polling = agentRunProjectionPollingState({ projectionState, env, error, rootCause: agentRunPollingRootCause(error), terminalConsistencyGap: nonNegativeNumber(projectionState?.terminalConsistencyGap) }); - await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, { - currentState: projectionState, - result: mapped, - agentRun: { ...mapped.agentRun, lastSeq: previousLastSeq }, - eventsResponse: null, - resultSyncState: projectionState?.resultSyncState ?? null, - projectionStatus: "degraded", - projectionHealth: "degraded", - retryParams: projectionRetryParams, - polling, - error, - now: options.now - }); - throw agentRunErrorWithPolling(error, polling); - } - if (eventsResponse) appendAgentRunEventsToTrace(traceStore, traceId, eventsResponse.events, mapped.agentRun); - const nextLastSeq = eventsResponse ? agentRunTraceCursorSeq(eventsResponse, mapped.agentRun.lastSeq) : mapped.agentRun.lastSeq; - const backendProfile = resolveAgentRunBackendProfile(env, { - providerProfile: firstNonEmpty( - projectionRetryParams?.providerProfile, - projectionRetryParams?.codeAgentProviderProfile, - projectionRetryParams?.backendProfile, - mapped.agentRun.backendProfile - ) - }); - recordAgentRunEventsProjectionMetrics(performanceStore, eventsResponse, previousLastSeq, nextLastSeq); - const terminalFromEvents = agentRunTerminalStatusFromEvents(eventsResponse?.events); - const cursorAdvance = Math.max(0, Number(nextLastSeq ?? 0) - Number(previousLastSeq ?? 0)); - const terminalConsistencyGap = agentRunTerminalConsistencyGap({ mapped, eventsResponse }); - const projectionPolling = agentRunProjectionPollingState({ - projectionState, - env, - pollCount: projectionPollCount, - cursorAdvance, - terminalFromEvents, - rootCause: terminalFromEvents ? "terminal_from_events" : cursorAdvance > 0 ? "cursor_advanced" : "projection_no_progress", - terminalConsistencyGap - }); - await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, { - currentState: projectionState, - result: mapped, - agentRun: { ...mapped.agentRun, lastSeq: nextLastSeq }, - eventsResponse, - resultSyncState: terminalFromEvents ? "pending" : null, - projectionStatus: terminalFromEvents ? "terminal" : null, - polling: projectionPolling, - retryParams: projectionRetryParams, - now: options.now - }); - const deferResultSync = terminalFromEvents ? false : agentRunResultSyncDeferred({ forceResultSync, options, env }); - if (!forceResultSync && (deferResultSync || !agentRunCommandResultSyncRequired(mapped, eventsResponse))) { - const nextMapping = withAgentRunRunnerJobCount({ - ...mapped.agentRun, - lastSeq: nextLastSeq, - status: mapped.agentRun.status ?? "running", - runStatus: mapped.agentRun.runStatus ?? null, - commandState: mapped.agentRun.commandState ?? null, - terminalStatus: null, - resultSyncState: terminalFromEvents ? "pending" : mapped.agentRun.resultSyncState ?? null, - updatedAt: nowIso(options.now), - valuesPrinted: false - }); - const payload = decorateAgentRunRunningResult({ base: { ...mapped, status: "running", agentRun: nextMapping, updatedAt: nowIso(options.now) }, mapping: nextMapping, traceStore, traceId }); - options.codeAgentChatResults?.set?.(traceId, payload); - return { result: payload, runnerTrace: payload.runnerTrace ?? traceStore.snapshot(traceId), found: true, eventsRefreshed: Boolean(eventsResponse), resultSynced: false, polling: projectionPolling }; - } - let result; - try { - result = await measuredAgentRunResultFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapped, eventsResponse }); - } catch (error) { - const retryable = agentRunPollingRetryableError(error); - const polling = agentRunProjectionPollingState({ projectionState, env, pollCount: projectionPollCount, cursorAdvance, terminalFromEvents, error, rootCause: agentRunPollingRootCause(error), terminalConsistencyGap }); - await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, { - currentState: projectionState, - result: mapped, - agentRun: { ...mapped.agentRun, lastSeq: nextLastSeq }, - eventsResponse, - resultSyncState: retryable ? "pending" : agentRunMetricStatusForError(error) === "timeout" ? "timed_out" : "failed", - projectionStatus: retryable ? "degraded" : terminalFromEvents ? "terminal" : null, - projectionHealth: "degraded", - retryParams: projectionRetryParams, - polling, - error, - now: options.now - }); - throw agentRunErrorWithPolling(error, polling); - } - const nextMapping = withAgentRunRunnerJobCount({ - ...mapped.agentRun, - ...agentRunResultRefs(result), - lastSeq: nextLastSeq, - status: result?.status ?? mapped.agentRun.status ?? "running", - runStatus: result?.runStatus ?? mapped.agentRun.runStatus ?? null, - commandState: result?.commandState ?? mapped.agentRun.commandState ?? null, - terminalStatus: result?.terminalStatus ?? mapped.agentRun.terminalStatus ?? null, - updatedAt: nowIso(options.now), - valuesPrinted: false - }); - const freshSessionRetry = await retryAgentRunTurnAfterFreshSessionFailure({ - params: projectionRetryParams, - options, - traceId, - traceStore, - mapped, - failedResult: result, - failedMapping: nextMapping, - env, - managerUrl, - fetchImpl, - timeoutMs, - backendProfile - }); - if (freshSessionRetry) { - options.codeAgentChatResults?.set?.(traceId, freshSessionRetry.payload); - await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, { - currentState: projectionState, - result: freshSessionRetry.payload, - agentRun: freshSessionRetry.mapping, - eventsResponse: null, - resultSyncState: null, - projectionStatus: "projecting", - retryParams: projectionRetryParams, - now: options.now - }); - const retrySync = await syncAgentRunChatResult({ - traceId, - currentResult: freshSessionRetry.payload, - options, - traceStore, - appendResultEvent, - refreshEvents: true, - forceResultSync, - retryParams: projectionRetryParams - }); - if (retrySync?.result) { - return { - ...retrySync, - found: true, - eventsRefreshed: Boolean(eventsResponse) || Boolean(retrySync.eventsRefreshed), - freshSessionRetried: true - }; - } - return { result: freshSessionRetry.payload, runnerTrace: freshSessionRetry.payload.runnerTrace ?? traceStore.snapshot(traceId), found: true, eventsRefreshed: Boolean(eventsResponse), resultSynced: false, freshSessionRetried: true }; - } - const base = { ...mapped, agentRun: nextMapping, updatedAt: nowIso(options.now) }; - const payload = agentRunResultToCodeAgentPayload({ base, result, traceStore, traceId, appendResultEvent }); - options.codeAgentChatResults?.set?.(traceId, payload); - const resultPolling = agentRunProjectionPollingState({ - projectionState, - env, - pollCount: projectionPollCount, - cursorAdvance, - terminalFromEvents: isAgentRunTerminalStatus(payload?.status ?? nextMapping.terminalStatus ?? nextMapping.commandState) ? (nextMapping.terminalStatus ?? payload?.status) : terminalFromEvents, - rootCause: isAgentRunTerminalStatus(payload?.status ?? nextMapping.terminalStatus ?? nextMapping.commandState) ? "terminal_result_synced" : "result_synced", - terminalConsistencyGap: agentRunTerminalConsistencyGap({ mapped: payload, eventsResponse, result }) - }); - await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, { - currentState: projectionState, - result: payload, - agentRun: nextMapping, - eventsResponse, - resultSyncState: "synced", - projectionStatus: isAgentRunTerminalStatus(payload?.status) ? "terminal" : null, - polling: resultPolling, - retryParams: projectionRetryParams, - now: options.now - }); - return { result: payload, runnerTrace: payload.runnerTrace ?? traceStore.snapshot(traceId), found: true, eventsRefreshed: Boolean(eventsResponse), resultSynced: true, polling: resultPolling }; -} - -async function loadWorkbenchProjectionStateForAgentRun(runtimeStore, { traceId, agentRun = {} } = {}) { - if (!runtimeStore || typeof runtimeStore.getWorkbenchProjectionState !== "function") return null; - try { - const result = await runtimeStore.getWorkbenchProjectionState({ traceId }); - const state = result?.projectionState ?? null; - if (!state) return null; - if (text(state.runId ?? state.sourceRunId) !== text(agentRun.runId)) return null; - if (text(state.commandId ?? state.sourceCommandId) !== text(agentRun.commandId)) return null; - return state; - } catch { - return null; - } -} - -async function writeWorkbenchProjectionStateForAgentRun(runtimeStore, params = {}) { - if (!runtimeStore || typeof runtimeStore.writeWorkbenchProjectionState !== "function") return null; - const projectionState = buildWorkbenchProjectionStateUpdate(params); - if (!projectionState.traceId || !projectionState.runId || !projectionState.commandId) return null; - return await runtimeStore.writeWorkbenchProjectionState({ projectionState }); -} - -function agentRunCommandResultSyncRequired(mapped = {}, eventsResponse = null) { - const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : {}; - if (agentRunTerminalResultRefreshSatisfied(mapped, mapped?.traceId)) return true; - if (isAgentRunTerminalStatus(agentRun.terminalStatus ?? agentRun.commandState)) return true; - return Boolean(agentRunTerminalStatusFromEvents(eventsResponse?.events)); -} - -function isAgentRunTerminalStatus(value) { - const status = normalizeAgentRunStatus(value); - return status === "timeout" || TERMINAL_RUN_STATUSES.has(status); -} - function normalizeAgentRunStatus(value) { const status = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); return status === "cancelled" ? "canceled" : status; } -function isAgentRunUserCancelSealed(payload = null) { - if (!payload || typeof payload !== "object") return false; - const status = normalizeAgentRunStatus(payload.status); - const cancelStatus = normalizeAgentRunStatus(payload.cancelStatus ?? payload.cancelDisposition?.status); - const terminalStatus = normalizeAgentRunStatus(payload.agentRun?.terminalStatus ?? payload.agentRun?.commandState ?? payload.agentRun?.status); - return payload.canceled === true || status === "canceled" || cancelStatus === "canceled" || terminalStatus === "canceled"; -} - -async function resolveAgentRunTraceCommandMapping({ traceId, mapped, options = {} }) { - const safeId = safeTraceId(traceId); - const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : null; - if (!safeId || !agentRun?.runId) return mapped; - const agentRunTraceId = safeTraceId(agentRun.traceId); - const providerTraceId = safeTraceId(agentRun.providerTrace?.traceId); - const traceSummaryCommandId = text(mapped?.traceSummary?.agentRun?.commandId); - const commandId = text(agentRun.commandId); - const traceFieldsMatch = agentRunTraceId === safeId && providerTraceId === safeId && (!traceSummaryCommandId || traceSummaryCommandId === commandId); - if (traceFieldsMatch && (mapped.status === "running" || agentRunTerminalResultRefreshSatisfied(mapped, safeId))) return mapped; - const env = options.env ?? process.env; - const fetchImpl = options.fetchImpl ?? globalThis.fetch; - const managerUrl = resolveAgentRunManagerUrl(env, agentRun.managerUrl); - const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000); - const command = await findAgentRunCommandForTrace({ fetchImpl, managerUrl, timeoutMs, env, runId: agentRun.runId, traceId: safeId }); - if (!command?.id) { - throw Object.assign(new Error(`AgentRun command registry has no command for ${safeId} in run ${agentRun.runId}`), { - code: "agentrun_trace_command_not_found", - statusCode: 404, - traceId: safeId, - runId: agentRun.runId - }); - } - if (command.id === commandId) { - return { - ...mapped, - finalResponse: null, - traceSummary: null, - agentRun: { - ...agentRun, - traceId: safeId, - lastSeq: 0, - providerTrace: null, - commandState: command.state ?? agentRun.commandState ?? null, - updatedAt: nowIso(options.now), - valuesPrinted: false - } - }; - } - return { - ...mapped, - finalResponse: null, - traceSummary: null, - agentRun: { - ...agentRun, - commandId: command.id, - traceId: safeId, - lastSeq: 0, - providerTrace: null, - commandState: command.state ?? agentRun.commandState ?? null, - updatedAt: nowIso(options.now), - valuesPrinted: false - } - }; -} - -export async function refreshAgentRunTrace({ traceId, result = null, options = {}, traceStore = defaultCodeAgentTraceStore }) { - const initial = result ?? await loadPersistedAgentRunResult(traceId, options); - const mapped = initial ? await resolveAgentRunTraceCommandMapping({ traceId, mapped: initial, options }) : initial; - if (!mapped?.agentRun?.runId) return traceStore.snapshot(traceId); - const env = options.env ?? process.env; - const fetchImpl = options.fetchImpl ?? globalThis.fetch; - const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl); - const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000); - const eventsResponse = await fetchAgentRunEventsForTrace({ fetchImpl, managerUrl, timeoutMs, env, mapping: { ...mapped.agentRun, traceSummary: mapped.traceSummary } }); - const events = eventsResponse.events; - appendAgentRunEventsToTrace(traceStore, traceId, events, mapped.agentRun); - const lastSeq = agentRunTraceCursorSeq(eventsResponse, mapped.agentRun.lastSeq); - if (lastSeq !== Number(mapped.agentRun.lastSeq ?? 0)) { - options.codeAgentChatResults?.set?.(traceId, { ...mapped, agentRun: { ...mapped.agentRun, lastSeq, updatedAt: nowIso(options.now) } }); - } - return traceStore.snapshot(traceId); -} - export async function cancelAgentRunChatTurn({ traceId, currentResult = null, options = {}, traceStore = defaultCodeAgentTraceStore }) { let mapped = currentResult ?? await loadPersistedAgentRunResult(traceId, options); if (!mapped?.agentRun?.commandId) return null; @@ -1043,12 +700,6 @@ export async function cancelAgentRunChatTurn({ traceId, currentResult = null, op }; const fetchImpl = options.fetchImpl ?? globalThis.fetch; const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl); - try { - const synced = await syncAgentRunChatResult({ traceId, currentResult: mapped, options: cancelOptions, traceStore, forceResultSync: true }); - mapped = synced?.result ?? synced ?? mapped; - } catch { - // Cancel must still expose the command cancel disposition; result refresh failures are reported by the normal trace/result paths. - } const terminalStatus = agentRunCancelTerminalStatus(mapped); if (terminalStatus) return agentRunAlreadyTerminalCancelPayload({ traceId, payload: mapped, terminalStatus, options: cancelOptions, traceStore }); const cancelPath = `/api/v1/commands/${encodeURIComponent(mapped.agentRun.commandId)}/cancel`; @@ -1155,7 +806,7 @@ export async function steerAgentRunChatTurn({ traceId, currentResult = null, par accepted: true, status: "running", shortConnection: true, - controlSemantics: "steer-active-turn-and-poll-target-trace", + controlSemantics: "steer-active-turn-and-project-sse", route: "/v1/agent/chat/steer", traceId, targetTraceId: traceId, @@ -1163,8 +814,8 @@ export async function steerAgentRunChatTurn({ traceId, currentResult = null, par conversationId: safeConversationId(mapped.conversationId ?? params.conversationId) || null, sessionId: safeSessionId(mapped.sessionId ?? params.sessionId) || null, threadId: safeOpaqueId(mapped.threadId ?? params.threadId) || null, - resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`, traceUrl: `/v1/agent/traces/${encodeURIComponent(traceId)}`, + workbenchEventsUrl: `/v1/workbench/events?sessionId=${encodeURIComponent(safeSessionId(mapped.sessionId ?? params.sessionId) || "")}&traceId=${encodeURIComponent(traceId)}`, agentRun: { ...agentRun, steerCommandId, @@ -1243,18 +894,6 @@ function agentRunSeedFromSession(session, traceId) { return topLevelAgentRun?.runId ? topLevelAgentRun : null; } -async function findAgentRunCommandForTrace({ fetchImpl, managerUrl, timeoutMs, env, runId, traceId }) { - if (!runId || !safeTraceId(traceId)) return null; - const commands = await agentRunJson(fetchImpl, managerUrl, `/api/v1/runs/${encodeURIComponent(runId)}/commands?afterSeq=0&limit=100`, { method: "GET", timeoutMs, env }); - return (Array.isArray(commands?.items) ? commands.items : []).find((item) => agentRunCommandMatchesTrace(item, traceId)) ?? null; -} - -function agentRunCommandMatchesTrace(command, traceId) { - const payload = command?.payload && typeof command.payload === "object" ? command.payload : {}; - return command?.idempotencyKey === traceId || payload.traceId === traceId; -} - - function withoutUserBillingReservation(value = {}) { if (!value || typeof value !== "object") return value; const { userBillingReservation, ...rest } = value; @@ -1353,126 +992,6 @@ function newSessionIdAfterEviction(baseSessionId, traceId) { return baseSessionId + "-reset-" + profile; } -async function retryAgentRunTurnAfterFreshSessionFailure({ - params = null, - options = {}, - traceId, - traceStore = defaultCodeAgentTraceStore, - mapped = {}, - failedResult = {}, - failedMapping = {}, - env = process.env, - managerUrl, - fetchImpl, - timeoutMs, - backendProfile -} = {}) { - const failure = agentRunFreshSessionFailure(failedResult); - if (!failure.requiresFreshSession) return null; - if (mapped?.agentRun?.freshSessionRetryOf || failedMapping?.freshSessionRetryOf) return null; - const promptText = messageAuthorityTextValue(params?.message ?? params?.prompt ?? params?.text ?? ""); - if (!promptText) { - traceStore.append(traceId, agentRunTraceEvent({ - type: "backend", - status: "failed", - label: "agentrun:session-reset:skipped", - errorCode: "agentrun_retry_prompt_missing", - message: "AgentRun session storage was evicted, but hwlab-cloud-api cannot retry the current turn because the original prompt text is unavailable in this sync path.", - runId: failedMapping?.runId ?? mapped?.agentRun?.runId ?? null, - commandId: failedMapping?.commandId ?? mapped?.agentRun?.commandId ?? null, - terminal: true, - valuesPrinted: false - }, failedMapping || mapped?.agentRun || backendProfile)); - return null; - } - const baseSessionId = scopedAgentRunSessionIdForParams(params, traceId); - const sessionId = newSessionIdAfterEviction(baseSessionId, traceId); - const retryParams = paramsWithAgentRunSessionThread(params, null); - const startedAt = nowIso(options.now); - const previous = { - runId: failedMapping?.runId ?? mapped?.agentRun?.runId ?? null, - commandId: failedMapping?.commandId ?? mapped?.agentRun?.commandId ?? null, - sessionId: failedMapping?.sessionId ?? mapped?.agentRun?.sessionId ?? baseSessionId, - threadId: failedMapping?.threadId ?? mapped?.agentRun?.threadId ?? mapped?.threadId ?? null, - failureKind: failure.code, - failureMessage: failure.message, - valuesPrinted: false - }; - traceStore.append(traceId, agentRunTraceEvent({ - type: "backend", - status: "running", - label: "agentrun:session-reset:retry-current-turn", - message: "AgentRun session storage/thread resume failed; hwlab-cloud-api is retrying the same user turn with a fresh AgentRun session and threadId=null.", - runId: previous.runId, - commandId: previous.commandId, - sessionId, - previousSessionId: previous.sessionId, - previousThreadId: previous.threadId, - errorCode: failure.code, - backendProfile, - waitingFor: "agentrun-run-create", - valuesPrinted: false - }, failedMapping || mapped?.agentRun || backendProfile)); - await ensureAgentRunSessionPersistent({ fetchImpl, managerUrl, sessionId, env, traceId, backendProfile, traceStore }); - const toolCapabilities = await resolveToolCapabilities({ params: retryParams, options }); - const runInput = buildAgentRunCreateRunInput({ params: retryParams, env, traceId, backendProfile, sessionId, toolCapabilities }); - const run = await agentRunDispatchJson({ fetchImpl, managerUrl, path: "/api/v1/runs", method: "POST", body: runInput, timeoutMs, env, traceStore, traceId, backendProfile, stage: "run-create-after-session-reset" }); - const runId = requiredString(run?.id, "run.id"); - traceStore.append(traceId, agentRunTraceEvent({ - type: "backend", - status: "running", - label: "agentrun:run:created-after-session-reset", - message: "AgentRun fresh-session retry run " + runId + " created for the current user turn.", - runId, - backendProfile, - waitingFor: "agentrun-command-create", - valuesPrinted: false - }, backendProfile)); - const dispatchInput = buildAgentRunDurableDispatchInput({ env, traceId, backendProfile }); - const commandInput = buildAgentRunCommandInput({ params: retryParams, traceId, backendProfile, sessionId, dispatchInput }); - const command = await agentRunDispatchJson({ fetchImpl, managerUrl, path: "/api/v1/runs/" + encodeURIComponent(runId) + "/commands", method: "POST", body: commandInput, timeoutMs, env, traceStore, traceId, backendProfile, runId, stage: "command-create-after-session-reset" }); - const commandId = requiredString(command?.id, "command.id"); - const dispatchIntent = requireAgentRunDurableDispatchIntent(command?.dispatchIntent); - traceStore.append(traceId, agentRunTraceEvent({ - type: "backend", - status: "running", - label: "agentrun:command:created-after-session-reset", - message: "AgentRun fresh-session retry command " + commandId + " and durable dispatch intent " + dispatchIntent.id + " were admitted atomically.", - runId, - commandId, - backendProfile, - dispatchIntentId: dispatchIntent.id, - runnerJobId: dispatchIntent.runnerJobId, - waitingFor: "agentrun-result", - valuesPrinted: false - }, backendProfile)); - const mapping = { - ...agentRunMapping({ env, managerUrl, backendProfile, run, command, dispatchIntent, traceId, startedAt, params: retryParams }), - freshSessionRetryAttempt: 1, - freshSessionRetryOf: previous, - status: `dispatch-intent-${dispatchIntent.state}`, - valuesPrinted: false - }; - traceStore.append(traceId, agentRunTraceEvent({ - type: "backend", - status: "running", - label: "agentrun:dispatch-intent:admitted-after-session-reset", - message: "AgentRun owns the durable fresh-session runner dispatch; HWLAB has no process-local runner Job creation window.", - runId: mapping.runId, - commandId: mapping.commandId, - backendProfile, - dispatchIntentId: dispatchIntent.id, - dispatchIntentState: dispatchIntent.state, - runnerJobId: dispatchIntent.runnerJobId, - waitingFor: "agentrun-result", - valuesPrinted: false - }, mapping)); - const base = initialAgentRunChatResult({ params: retryParams, options, traceId }); - const payload = decorateAgentRunRunningResult({ base: { ...base, freshSessionRetryOf: previous }, mapping, traceStore, traceId }); - return { payload, mapping }; -} - - function buildAgentRunCreateRunInput({ params, env, traceId, backendProfile, sessionId, toolCapabilities = null }) { const commitId = requireAgentRunSourceCommit(env); const runtimeAuthority = requireAgentRunRuntimeAuthority(env); @@ -1536,7 +1055,6 @@ function buildAgentRunCreateRunInput({ params, env, traceId, backendProfile, ses kind: "hwlab-cloud-api", traceId, ...codeAgentOtelTraceFields(traceId, env), - resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`, traceUrl: `/v1/agent/traces/${encodeURIComponent(traceId)}`, valuesPrinted: false } diff --git a/internal/cloud/code-agent-agentrun-cancel.test.ts b/internal/cloud/code-agent-agentrun-cancel.test.ts index 9796756c..6a55cb14 100644 --- a/internal/cloud/code-agent-agentrun-cancel.test.ts +++ b/internal/cloud/code-agent-agentrun-cancel.test.ts @@ -4,7 +4,7 @@ import assert from "node:assert/strict"; import { test } from "bun:test"; -import { cancelAgentRunChatTurn, syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts"; +import { cancelAgentRunChatTurn } from "./code-agent-agentrun-adapter.ts"; import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; test("AgentRun cancel downstream failure returns visible blocked disposition instead of throwing", async () => { @@ -54,44 +54,3 @@ test("AgentRun cancel downstream failure returns visible blocked disposition ins assert.equal(codeAgentChatResults.get(traceId)?.cancelDisposition?.code, "command_not_cancelable"); assert.ok(traceStore.snapshot(traceId).events.some((event) => event.label === "agentrun:cancel:blocked" && event.status === "blocked")); }); - -test("AgentRun result sync preserves a user-canceled payload instead of accepting late completed result", async () => { - const traceId = "trc_cancel_sealed_result"; - const traceStore = createCodeAgentTraceStore(); - const currentResult = { - traceId, - status: "canceled", - canceled: true, - cancelStatus: "canceled", - cancelDisposition: { status: "canceled", forwarded: true, terminal: true, traceId, valuesPrinted: false }, - finalResponse: { text: "hwlab-user-cancel", status: "canceled", traceId, valuesPrinted: false }, - agentRun: { - runId: "run_cancel_sealed_result", - commandId: "cmd_cancel_sealed_result", - managerUrl: "http://127.0.0.1:65535", - status: "cancelled", - commandState: "cancelled", - terminalStatus: "cancelled", - traceId, - valuesPrinted: false - }, - valuesPrinted: false - }; - - const synced = await syncAgentRunChatResult({ - traceId, - currentResult, - traceStore, - options: { - env: { HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1" }, - fetchImpl() { - throw new Error("late AgentRun result must not be fetched after HWLAB user cancel is sealed"); - } - } - }); - - assert.equal(synced.localCancelSealed, true); - assert.equal(synced.resultSynced, false); - assert.equal(synced.result.status, "canceled"); - assert.equal(synced.result.finalResponse.text, "hwlab-user-cancel"); -}); diff --git a/internal/cloud/code-agent-agentrun-durable-admission.test.ts b/internal/cloud/code-agent-agentrun-durable-admission.test.ts index 017fbd16..4fb1b7a3 100644 --- a/internal/cloud/code-agent-agentrun-durable-admission.test.ts +++ b/internal/cloud/code-agent-agentrun-durable-admission.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { test } from "bun:test"; -import { submitAgentRunChatTurn, syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts"; +import { submitAgentRunChatTurn } from "./code-agent-agentrun-adapter.ts"; import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; const sourceCommit = "0123456789abcdef0123456789abcdef01234567"; @@ -154,70 +154,6 @@ for (const terminal of [ }); } -test("gpt.pika session-eviction retry uses the same durable dispatch admission", async () => { - const traceId = "trc_gpt_pika_durable_retry"; - const calls = []; - const traceStore = createCodeAgentTraceStore(); - const synced = await syncAgentRunChatResult({ - traceId, - traceStore, - refreshEvents: false, - forceResultSync: true, - retryParams: { - ownerUserId: "usr_gpt_pika_retry", - projectId: "pikasTech/HWLAB", - conversationId: "cnv_gpt_pika_retry", - sessionId: "ses_gpt_pika_retry", - threadId: "thread_evicted", - message: "retry the evicted gpt.pika turn", - providerProfile: "gpt.pika" - }, - currentResult: { - traceId, - status: "running", - conversationId: "cnv_gpt_pika_retry", - sessionId: "ses_gpt_pika_retry", - threadId: "thread_evicted", - providerProfile: "gpt.pika", - agentRun: { - adapter: "agentrun-v01", - runId: "run_gpt_pika_evicted", - commandId: "cmd_gpt_pika_evicted", - sessionId: "ses_agentrun_gpt_pika_retry", - threadId: "thread_evicted", - backendProfile: "gpt.pika", - managerUrl: "http://127.0.0.1:65535", - status: "running", - traceId, - lastSeq: 0, - providerTrace: { traceId, valuesPrinted: false }, - valuesPrinted: false - }, - valuesPrinted: false - }, - options: { - env: agentRunTestEnv(), - fetchImpl: createAgentRunRetryFetch(calls, traceId) - } - }); - - assert.equal(synced.freshSessionRetried, true); - assert.equal(synced.result.status, "completed"); - assert.equal(synced.result.agentRun.runId, "run_gpt_pika_retry"); - assert.equal(synced.result.agentRun.commandId, "cmd_gpt_pika_retry"); - assert.equal(synced.result.agentRun.freshSessionRetryOf.runId, "run_gpt_pika_evicted"); - const retryCommand = calls.find((call) => call.path === "/api/v1/runs/run_gpt_pika_retry/commands"); - assert.ok(retryCommand); - assert.equal(retryCommand.body.idempotencyKey, traceId); - assert.equal(retryCommand.body.payload.threadId, null); - assert.equal(retryCommand.body.dispatch.kind, "kubernetes-job"); - assert.equal(Object.hasOwn(retryCommand.body.dispatch.input, "commandId"), false); - assert.equal(retryCommand.body.dispatch.input.transientEnv.every((entry) => entry.sensitive === false), true); - assert.equal(JSON.stringify(retryCommand.body).includes("HWLAB_API_KEY"), false); - assert.equal(calls.some((call) => call.path.endsWith("/runner-jobs")), false); - assert.ok(traceStore.snapshot(traceId).events.some((event) => event.label === "agentrun:dispatch-intent:admitted-after-session-reset")); -}); - function agentRunTestEnv() { return { AGENTRUN_MGR_URL: "http://127.0.0.1:65535", @@ -273,89 +209,3 @@ function createAgentRunFetch(calls, { dispatchIntent }) { throw new Error(`unexpected AgentRun call ${method} ${parsed.pathname}`); }; } - -function createAgentRunRetryFetch(calls, traceId) { - return async (url, init = {}) => { - const parsed = new URL(url); - const method = init.method ?? "GET"; - const body = init.body ? JSON.parse(String(init.body)) : null; - calls.push({ method, path: parsed.pathname, body }); - const send = (data) => new Response(JSON.stringify({ ok: true, data }), { - status: 200, - headers: { "content-type": "application/json" } - }); - if (method === "GET" && parsed.pathname === "/api/v1/runs/run_gpt_pika_evicted/commands/cmd_gpt_pika_evicted/result") { - return send({ - runId: "run_gpt_pika_evicted", - commandId: "cmd_gpt_pika_evicted", - status: "failed", - runStatus: "failed", - commandState: "failed", - terminalStatus: "failed", - failureKind: "session-store-evicted", - failureMessage: "session storage was evicted while resuming the thread", - completed: false, - lastSeq: 1 - }); - } - if (method === "POST" && parsed.pathname === "/api/v1/sessions") { - return send({ session: { id: body.sessionId, metadata: {} } }); - } - if (method === "POST" && parsed.pathname === "/api/v1/runs") { - return send({ - id: "run_gpt_pika_retry", - status: "pending", - sessionRef: body.sessionRef, - backendProfile: body.backendProfile - }); - } - if (method === "POST" && parsed.pathname === "/api/v1/runs/run_gpt_pika_retry/commands") { - return send({ - id: "cmd_gpt_pika_retry", - runId: "run_gpt_pika_retry", - state: "pending", - type: "turn", - seq: 1, - dispatchIntent: { - id: "dispatch_gpt_pika_retry", - state: "pending", - runnerJobId: "rjob_gpt_pika_retry", - attemptCount: 0, - durable: true - } - }); - } - if (method === "GET" && parsed.pathname === "/api/v1/runs/run_gpt_pika_retry/events") { - return send({ items: [] }); - } - if (method === "GET" && parsed.pathname === "/api/v1/runs/run_gpt_pika_retry/commands") { - return send({ items: [{ - id: "cmd_gpt_pika_retry", - runId: "run_gpt_pika_retry", - state: "completed", - idempotencyKey: traceId, - payload: { traceId } - }] }); - } - if (method === "GET" && parsed.pathname === "/api/v1/runs/run_gpt_pika_retry/commands/cmd_gpt_pika_retry/result") { - return send({ - runId: "run_gpt_pika_retry", - commandId: "cmd_gpt_pika_retry", - status: "completed", - runStatus: "completed", - commandState: "completed", - terminalStatus: "completed", - completed: true, - reply: "GPT_PIKA_RETRY_OK", - lastSeq: 2, - sessionRef: { - sessionId: "ses_agentrun_gpt_pika_retry-reset-trcgptpikadu", - conversationId: "cnv_gpt_pika_retry", - threadId: "thread_gpt_pika_retry" - }, - traceId - }); - } - throw new Error(`unexpected AgentRun retry call ${method} ${parsed.pathname}`); - }; -} diff --git a/internal/cloud/code-agent-agentrun-projection-polling.ts b/internal/cloud/code-agent-agentrun-projection-polling.ts deleted file mode 100644 index 546f82d1..00000000 --- a/internal/cloud/code-agent-agentrun-projection-polling.ts +++ /dev/null @@ -1,350 +0,0 @@ -// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. -// Responsibility: AgentRun projection polling/backoff I/O, durable polling state, and projection_sync observability attributes. - -import { agentRunJson } from "./code-agent-agentrun-runtime.ts"; -import { emitCodeAgentOtelSpan } from "./otel-trace.ts"; -import { parsePositiveInteger, safeTraceId, text } from "./server-http-utils.ts"; - -const DEFAULT_AGENTRUN_PROJECTION_SYNC_BACKOFF_INITIAL_MS = 1_000; -const DEFAULT_AGENTRUN_PROJECTION_SYNC_BACKOFF_MAX_MS = 30_000; -const AGENTRUN_TERMINAL_STATUSES = new Set(["completed", "failed", "blocked", "cancelled", "canceled", "timeout"]); - -export function agentRunProjectionPollCount(projectionState = null) { - return nonNegativeNumber(projectionState?.pollCount) + 1; -} - -export function agentRunProjectionPollingState({ projectionState = null, env = process.env, pollCount = null, cursorAdvance = 0, terminalFromEvents = null, error = null, rootCause = null, terminalConsistencyGap = 0 } = {}) { - const retryable = agentRunPollingRetryableError(error); - const noProgress = !error && !terminalFromEvents && Number(cursorAdvance ?? 0) <= 0; - const previousNoProgress = nonNegativeNumber(projectionState?.noProgressPollCount); - const noProgressPollCount = retryable || noProgress ? previousNoProgress + 1 : 0; - const backoffReason = retryable ? "provider_retry_or_rate_limit" : noProgress ? "projection_no_progress" : null; - const backoffMs = backoffReason ? agentRunProjectionBackoffMs(noProgressPollCount, env) : 0; - return { - pollCount: Number.isInteger(Number(pollCount)) && Number(pollCount) > 0 ? Math.floor(Number(pollCount)) : agentRunProjectionPollCount(projectionState), - noProgressPollCount, - backoffMs, - backoffReason, - rootCause: boundedAgentRunRootCause(rootCause ?? agentRunPollingRootCause(error) ?? backoffReason ?? "projection_sync"), - terminalConsistencyGap: nonNegativeNumber(terminalConsistencyGap), - nextRetryAt: backoffMs > 0 ? new Date(Date.now() + backoffMs).toISOString() : null, - valuesPrinted: false - }; -} - -export function agentRunPollingRetryableError(error = null) { - const statusCode = Number(error?.statusCode ?? error?.agentRunError?.statusCode ?? 0); - if ([408, 425, 429, 500, 502, 503, 504].includes(statusCode)) return true; - const code = String(error?.code ?? error?.agentRunError?.failureKind ?? "").toLowerCase(); - return error?.retryable === true || code === "agentrun_timeout" || /rate.?limit|retry|timeout|temporar|unavailable|overload/u.test(code); -} - -export function agentRunPollingRootCause(error = null) { - if (!error) return null; - const statusCode = Number(error?.statusCode ?? error?.agentRunError?.statusCode ?? 0); - if (statusCode === 429) return "provider_rate_limited"; - if ([408, 425, 500, 502, 503, 504].includes(statusCode)) return `provider_retry_http_${statusCode}`; - return error?.code ?? error?.agentRunError?.failureKind ?? error?.name ?? "agentrun_sync_failed"; -} - -export function agentRunErrorWithPolling(error, polling = null) { - if (!error || !polling) return error; - error.polling = polling; - error.backoffMs = polling.backoffMs; - error.nextRetryAt = polling.nextRetryAt; - error.rootCause = polling.rootCause; - return error; -} - -export function agentRunTerminalConsistencyGap({ mapped = {}, eventsResponse = null, result = null } = {}) { - const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : {}; - const terminal = agentRunPollingTerminalStatus(result?.terminalStatus ?? result?.commandState ?? result?.status ?? mapped?.status ?? agentRun.terminalStatus ?? agentRun.commandState) || Boolean(agentRunTerminalStatusFromEvents(eventsResponse?.events)); - if (!terminal) return 0; - const latest = Math.max( - nonNegativeNumber(eventsResponse?.maxSeq), - nonNegativeNumber(result?.eventCount), - nonNegativeNumber(result?.lastSeq), - nonNegativeNumber(agentRun.lastSeq) - ); - const observed = Math.max( - nonNegativeNumber(eventsResponse?.traceLastSeq), - nonNegativeNumber(result?.lastSeq), - nonNegativeNumber(agentRun.lastSeq) - ); - return Math.max(0, latest - observed); -} - -export function agentRunTerminalResultRefreshSatisfied(mapped = {}, traceId = null) { - const safeId = safeTraceId(traceId ?? mapped?.traceId); - const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : null; - const traceSummary = mapped?.traceSummary && typeof mapped.traceSummary === "object" ? mapped.traceSummary : null; - const finalResponse = mapped?.finalResponse && typeof mapped.finalResponse === "object" ? mapped.finalResponse : null; - const providerTrace = agentRun?.providerTrace && typeof agentRun.providerTrace === "object" - ? agentRun.providerTrace - : mapped?.providerTrace && typeof mapped.providerTrace === "object" - ? mapped.providerTrace - : null; - const commandId = text(agentRun?.commandId); - const runId = text(agentRun?.runId); - if (!safeId || !runId || !commandId) return false; - if (!agentRunPollingTerminalStatus(mapped?.status ?? agentRun?.terminalStatus ?? agentRun?.commandState ?? agentRun?.status)) return false; - if (safeTraceId(mapped?.traceId ?? agentRun?.traceId) !== safeId) return false; - if (safeTraceId(providerTrace?.traceId) !== safeId) return false; - const finalTraceId = safeTraceId(finalResponse?.traceId ?? safeId); - if (finalTraceId && finalTraceId !== safeId) return false; - if (!String(finalResponse?.text ?? "").trim()) return false; - if (traceSummary?.source !== "agentrun-command-result") return false; - if (safeTraceId(traceSummary.traceId ?? safeId) !== safeId) return false; - const summaryAgentRun = traceSummary.agentRun && typeof traceSummary.agentRun === "object" ? traceSummary.agentRun : null; - if (text(summaryAgentRun?.runId) !== runId) return false; - if (text(summaryAgentRun?.commandId) !== commandId) return false; - return true; -} - -export function emitAgentRunProjectionPollingSpan({ traceId, env = process.env, agentRun = {}, polling = {}, terminalRefreshSkipped = false } = {}) { - void emitCodeAgentOtelSpan("projection_sync", traceId, env, { - attributes: { - runId: agentRun?.runId ?? null, - commandId: agentRun?.commandId ?? null, - terminalRefreshSkipped: terminalRefreshSkipped === true, - pollCount: nonNegativeNumber(polling.pollCount), - backoffMs: nonNegativeNumber(polling.backoffMs), - rootCause: boundedAgentRunRootCause(polling.rootCause), - terminalConsistencyGap: nonNegativeNumber(polling.terminalConsistencyGap) - } - }); -} - -export async function fetchAgentRunEventsForTrace({ fetchImpl, managerUrl, timeoutMs, env, mapping = {} }) { - const runId = requiredString(mapping.runId, "runId"); - const currentCommandId = typeof mapping.commandId === "string" ? mapping.commandId : ""; - const replayWindow = agentRunTraceReplayWindow(mapping); - const afterSeqOverride = Number(mapping.afterSeqOverride); - const afterSeq = Number.isFinite(afterSeqOverride) && afterSeqOverride >= 0 ? Math.floor(afterSeqOverride) : replayWindow.afterSeq; - const endSeq = Number.isFinite(afterSeqOverride) && afterSeqOverride >= 0 ? 0 : replayWindow.endSeq; - const limit = parsePositiveInteger(mapping.eventsPageLimit ?? env?.HWLAB_CODE_AGENT_AGENTRUN_EVENTS_PAGE_LIMIT, 500); - const path = `/api/v1/runs/${encodeURIComponent(runId)}/events?afterSeq=${encodeURIComponent(String(afterSeq))}&limit=${encodeURIComponent(String(limit))}`; - const response = await agentRunJson(fetchImpl, managerUrl, path, { method: "GET", timeoutMs, env }); - const rawEvents = Array.isArray(response?.items) ? response.items : []; - const events = rawEvents.filter((event) => agentRunEventBelongsToTrace(event, { currentCommandId, afterSeq, endSeq })); - const maxSeq = Math.max(afterSeq, ...rawEvents.map((event) => Number(event?.seq ?? 0)).filter(Number.isFinite)); - const traceLastSeq = Math.max(afterSeq, ...events.map((event) => Number(event?.seq ?? 0)).filter(Number.isFinite)); - void emitCodeAgentOtelSpan("projection_sync", mapping.traceId ?? mapping.hwlabTraceId ?? mapping.businessTraceId, env, { - attributes: { - runId, - commandId: currentCommandId || null, - afterSeq, - endSeq, - limit, - rawEventCount: rawEvents.length, - eventCount: events.length, - commandFiltered: Boolean(currentCommandId), - maxSeq, - traceLastSeq, - terminalFromRawEvents: agentRunTerminalStatusFromEvents(rawEvents), - terminalFromEvents: agentRunTerminalStatusFromEvents(events), - pollCount: nonNegativeNumber(mapping.pollCount), - backoffMs: nonNegativeNumber(mapping.backoffMs), - rootCause: boundedAgentRunRootCause(mapping.rootCause), - terminalConsistencyGap: nonNegativeNumber(mapping.terminalConsistencyGap) - } - }); - return { - events, - afterSeq, - endSeq, - rawEventCount: rawEvents.length, - commandFiltered: Boolean(currentCommandId), - maxSeq, - traceLastSeq - }; -} - -export async function measuredAgentRunEventsFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapping = {} }) { - const startedAt = nowMs(); - try { - const response = await fetchAgentRunEventsForTrace({ fetchImpl, managerUrl, timeoutMs, env, mapping }); - performanceStore?.recordWorkbenchProjectorBatch?.({ phase: "agentrun_events_fetch", status: "ok", durationMs: nowMs() - startedAt }); - return response; - } catch (error) { - performanceStore?.recordWorkbenchProjectorBatch?.({ phase: "agentrun_events_fetch", status: agentRunMetricStatusForError(error), durationMs: nowMs() - startedAt }); - performanceStore?.recordWorkbenchProjectorError?.({ reason: error?.code ?? "agentrun_events_fetch_failed" }); - throw error; - } -} - -export async function measuredAgentRunResultFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapped = {}, eventsResponse = null }) { - const startedAt = nowMs(); - const path = `/api/v1/runs/${encodeURIComponent(mapped.agentRun.runId)}/commands/${encodeURIComponent(mapped.agentRun.commandId)}/result`; - try { - const result = await agentRunJson(fetchImpl, managerUrl, path, { method: "GET", timeoutMs, env }); - const eventCount = agentRunResultEventCount(result, eventsResponse); - performanceStore?.recordAgentRunResult?.({ - durationMs: nowMs() - startedAt, - eventCount, - eventsScanned: eventCount, - pagesScanned: estimateAgentRunResultPages(eventCount), - status: result?.terminalStatus ?? result?.status ?? "ok", - budget: timeoutMs - }); - return result; - } catch (error) { - const status = agentRunMetricStatusForError(error); - const eventCount = agentRunResultEventCount(null, eventsResponse); - performanceStore?.recordAgentRunResult?.({ - durationMs: nowMs() - startedAt, - eventCount, - eventsScanned: eventCount, - pagesScanned: estimateAgentRunResultPages(eventCount), - status, - budget: timeoutMs, - timedOut: status === "timeout" - }); - performanceStore?.recordWorkbenchProjectorError?.({ reason: error?.code ?? "agentrun_result_failed" }); - throw error; - } -} - -export function recordAgentRunEventsProjectionMetrics(performanceStore, eventsResponse = null, previousLastSeq = 0, nextLastSeq = 0) { - if (!performanceStore || !eventsResponse) return; - const cursorAdvance = Math.max(0, Number(nextLastSeq ?? 0) - Number(previousLastSeq ?? 0)); - if (cursorAdvance > 0) { - performanceStore.recordWorkbenchProjectionCursorAdvance?.({ - status: "advanced", - reason: eventsResponse.commandFiltered ? "command_filtered_events" : "run_events", - projectionStatus: "projecting", - source: "agentrun_v01", - count: cursorAdvance - }); - } - const counts = new Map(); - for (const event of eventsResponse.events ?? []) { - const kind = String(event?.type ?? event?.payload?.phase ?? "unknown").trim() || "unknown"; - counts.set(kind, (counts.get(kind) ?? 0) + 1); - } - if (counts.size === 0 && Number(eventsResponse.rawEventCount ?? 0) > 0) counts.set("filtered", Number(eventsResponse.rawEventCount)); - for (const [eventKind, count] of counts.entries()) { - performanceStore.recordWorkbenchProjectorEvents?.({ eventKind, status: "ok", count }); - } -} - -export function agentRunMetricStatusForError(error) { - return error?.code === "agentrun_timeout" || error?.name === "AbortError" || Number(error?.statusCode) === 504 ? "timeout" : "error"; -} - -export function agentRunTraceCursorSeq(eventsResponse = {}, previousLastSeq = 0) { - const afterSeq = Number(eventsResponse.afterSeq ?? 0); - const endSeq = Number(eventsResponse.endSeq ?? 0); - const traceLastSeq = Number(eventsResponse.traceLastSeq ?? 0); - if (Number.isFinite(traceLastSeq) && traceLastSeq > afterSeq) return Math.floor(traceLastSeq); - if (Number.isFinite(endSeq) && endSeq > 0) return Math.floor(endSeq); - if (eventsResponse.commandFiltered === true) return Math.max(Number(previousLastSeq ?? 0), afterSeq); - return Math.max(Number(previousLastSeq ?? 0), Number(eventsResponse.maxSeq ?? 0)); -} - -export function agentRunTerminalStatusFromEvents(events = []) { - if (!Array.isArray(events)) return null; - for (const event of [...events].reverse()) { - const payload = event?.payload && typeof event.payload === "object" ? event.payload : {}; - const phase = payload.phase ?? event?.phase; - const candidates = [ - event?.type === "terminal_status" ? payload.terminalStatus ?? event?.terminalStatus ?? payload.status ?? event?.status : null, - phase === "command-terminal" ? payload.terminalStatus ?? event?.terminalStatus ?? payload.status ?? event?.status : null, - phase === "turn:completed" || phase === "turn/steer:completed" ? "completed" : null, - event?.terminal === true ? payload.terminalStatus ?? event?.terminalStatus ?? payload.status ?? event?.status ?? "completed" : null, - payload.terminal === true ? payload.terminalStatus ?? event?.terminalStatus ?? payload.status ?? event?.status ?? "completed" : null - ]; - for (const value of candidates) { - const status = normalizeAgentRunStatus(value); - if (agentRunPollingTerminalStatus(status)) return status; - } - } - return null; -} - -export function agentRunEventCommandId(event) { - const payload = event?.payload && typeof event.payload === "object" ? event.payload : {}; - if (typeof event?.commandId === "string") return event.commandId; - return typeof payload.commandId === "string" ? payload.commandId : ""; -} - -export function nonNegativeNumber(value) { - const number = Number(value ?? 0); - return Number.isFinite(number) && number > 0 ? Math.floor(number) : 0; -} - -function agentRunProjectionBackoffMs(attempt, env = process.env) { - const initialMs = parsePositiveInteger(env?.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_BACKOFF_INITIAL_MS, DEFAULT_AGENTRUN_PROJECTION_SYNC_BACKOFF_INITIAL_MS); - const maxMs = parsePositiveInteger(env?.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_BACKOFF_MAX_MS, DEFAULT_AGENTRUN_PROJECTION_SYNC_BACKOFF_MAX_MS); - const exponent = Math.max(0, Math.min(10, nonNegativeNumber(attempt) - 1)); - return Math.min(maxMs, initialMs * 2 ** exponent); -} - -function boundedAgentRunRootCause(value) { - const raw = String(value ?? "").trim().replace(/[^a-z0-9_.:-]+/giu, "_").replace(/^_+|_+$/gu, "").toLowerCase(); - return raw ? raw.slice(0, 120) : null; -} - -function agentRunResultEventCount(result = null, eventsResponse = null) { - const candidates = [ - result?.traceSummary?.sourceEventCount, - result?.traceSummary?.eventCount, - result?.sourceEventCount, - result?.eventCount, - result?.providerTrace?.eventCount, - eventsResponse?.maxSeq, - eventsResponse?.rawEventCount, - eventsResponse?.events?.length - ]; - for (const value of candidates) { - const count = Number(value); - if (Number.isFinite(count) && count >= 0) return Math.floor(count); - } - return 0; -} - -function estimateAgentRunResultPages(eventCount) { - const count = Number(eventCount); - if (!Number.isFinite(count) || count <= 0) return 0; - return Math.max(1, Math.ceil(count / 500)); -} - -function agentRunTraceReplayWindow(mapping = {}) { - const eventStartSeq = Number(mapping.eventStartSeq ?? mapping.commandStartSeq ?? mapping.startSeq ?? 0); - const summary = mapping.traceSummary && typeof mapping.traceSummary === "object" ? mapping.traceSummary : null; - const summaryAgentRun = summary?.agentRun && typeof summary.agentRun === "object" ? summary.agentRun : null; - const summaryLastSeq = Number(summaryAgentRun?.lastSeq ?? summary?.lastSeq ?? 0); - const currentLastSeq = Number(mapping.lastSeq ?? 0); - const endSeq = Number.isFinite(summaryLastSeq) && summaryLastSeq > 0 ? Math.floor(summaryLastSeq) : 0; - if (Number.isFinite(eventStartSeq) && eventStartSeq > 0) return { afterSeq: Math.max(0, Math.floor(eventStartSeq) - 1), endSeq }; - if (endSeq > 0) return { afterSeq: Math.max(0, endSeq - 500), endSeq }; - if (Number.isFinite(currentLastSeq) && currentLastSeq > 0) return { afterSeq: Math.floor(currentLastSeq), endSeq: 0 }; - return { afterSeq: 0, endSeq: 0 }; -} - -function agentRunEventBelongsToTrace(event, { currentCommandId = "", afterSeq = 0, endSeq = 0 } = {}) { - const eventCommandId = agentRunEventCommandId(event); - const seq = Number(event?.seq ?? 0); - if (currentCommandId && eventCommandId && eventCommandId !== currentCommandId) return false; - if (endSeq > 0 && Number.isFinite(seq)) return seq > afterSeq && seq <= endSeq; - return true; -} - -function agentRunPollingTerminalStatus(value) { - return AGENTRUN_TERMINAL_STATUSES.has(normalizeAgentRunStatus(value)); -} - -function normalizeAgentRunStatus(value) { - const status = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); - return status === "cancelled" ? "canceled" : status; -} - -function requiredString(value, fieldName) { - if (typeof value === "string" && value.trim()) return value.trim(); - throw Object.assign(new Error(`AgentRun response missing ${fieldName}`), { code: "agentrun_response_invalid", statusCode: 502 }); -} - -function nowMs() { - if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now(); - return Date.now(); -} diff --git a/internal/cloud/code-agent-agentrun-result.ts b/internal/cloud/code-agent-agentrun-result.ts index b629af2c..f4924d13 100644 --- a/internal/cloud/code-agent-agentrun-result.ts +++ b/internal/cloud/code-agent-agentrun-result.ts @@ -1,5 +1,5 @@ // SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-17-r0; draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. -// Responsibility: AgentRun command-result/event mapping, terminal projection, and trace observability helpers. +// Responsibility: AgentRun command-result/event mapping and trace observability helpers; Kafka owns persistence. import { randomUUID } from "node:crypto"; @@ -20,25 +20,6 @@ import { isAgentRunManagerInternalServiceHost, resolveAgentRunManagerUrl } from "./code-agent-agentrun-runtime.ts"; -import { - agentRunErrorWithPolling, - agentRunEventCommandId, - agentRunMetricStatusForError, - agentRunPollingRetryableError, - agentRunPollingRootCause, - agentRunProjectionPollCount, - agentRunProjectionPollingState, - agentRunTerminalConsistencyGap, - agentRunTerminalResultRefreshSatisfied, - agentRunTerminalStatusFromEvents, - agentRunTraceCursorSeq, - emitAgentRunProjectionPollingSpan, - fetchAgentRunEventsForTrace, - measuredAgentRunEventsFetch, - measuredAgentRunResultFetch, - nonNegativeNumber, - recordAgentRunEventsProjectionMetrics -} from "./code-agent-agentrun-projection-polling.ts"; import { agentRunAlreadyTerminalCancelPayload, agentRunCancelBlockedPayload, @@ -65,7 +46,6 @@ import { agentRunPromptMetadataFromText, messageAuthorityTextValue } from "./code-agent-agentrun-prompt.ts"; -import { agentRunResultSyncDeferred, buildAgentRunProjectionEventsFetchPlan, buildWorkbenchProjectionStateUpdate } from "./workbench-projection-cursor.ts"; import { codeAgentOtelTraceFields, codeAgentOtelTraceContext, emitCodeAgentOtelSpan } from "./otel-trace.ts"; const ADAPTER_ID = "agentrun-v01"; @@ -313,7 +293,7 @@ export function agentRunResultToCodeAgentPayload({ base, result, traceStore, tra status: "completed", label: "agentrun:result:completed", createdAt: terminalEventCreatedAt, - message: "AgentRun result is ready for HWLAB short-connection polling.", + message: "AgentRun result is committed to the Workbench projection event stream.", runId: base.agentRun.runId, commandId: base.agentRun.commandId, terminal: true, @@ -574,6 +554,11 @@ export function appendAgentRunEventsToTrace(traceStore, traceId, events, mapping if (normalized) traceStore.append(traceId, normalized, agentRunTraceMeta({}, {})); } } +function agentRunEventCommandId(event) { + const payload = event?.payload && typeof event.payload === "object" ? event.payload : {}; + if (typeof event?.commandId === "string") return event.commandId; + return typeof payload.commandId === "string" ? payload.commandId : ""; +} export function isForeignAgentRunCommandEvent(event, mapping = {}) { const currentCommandId = typeof mapping.commandId === "string" ? mapping.commandId : ""; diff --git a/internal/cloud/code-agent-trace-store.test.ts b/internal/cloud/code-agent-trace-store.test.ts index 920b75dd..6359f5ce 100644 --- a/internal/cloud/code-agent-trace-store.test.ts +++ b/internal/cloud/code-agent-trace-store.test.ts @@ -45,9 +45,8 @@ test("code agent trace store dedupes repeated AgentRun source events", () => { assert.deepEqual(snapshot.events.map((item) => item.label), ["agentrun:backend:resource-bundle-materialized", "agentrun:assistant:message"]); }); -test("durable code agent trace store projects appended events without counting assistant deltas", async () => { +test("durable code agent trace store persists diagnostics without counting assistant deltas", async () => { const writes = []; - const workbenchWrites = []; const baseStore = createCodeAgentTraceStore({ maxEvents: 20 }); const traceStore = createDurableCodeAgentTraceStore({ traceStore: baseStore, @@ -56,10 +55,6 @@ test("durable code agent trace store projects appended events without counting a writes.push({ params, requestMeta }); return { written: true }; } - }, - workbenchEventWriter(event, requestMeta) { - workbenchWrites.push({ event, requestMeta }); - return { written: true }; } }); const traceId = "trc_trace-store-durable-projection"; @@ -76,10 +71,6 @@ test("durable code agent trace store projects appended events without counting a assert.equal(writes[0].params.event.traceId, traceId); assert.equal(writes[0].params.event.seq, 1); assert.equal(writes[0].requestMeta.agentSessionId, "ses_trace_store_durable"); - assert.equal(workbenchWrites.length, 1); - assert.equal(workbenchWrites[0].event.traceId, traceId); - assert.equal(workbenchWrites[0].requestMeta.sessionId, "ses_trace_store_durable"); - assert.equal(workbenchWrites[0].requestMeta.agentSessionId, "ses_trace_store_durable"); }); test("code agent trace store retains six thousand regular events", () => { diff --git a/internal/cloud/code-agent-trace-store.ts b/internal/cloud/code-agent-trace-store.ts index 59943573..b8c63464 100644 --- a/internal/cloud/code-agent-trace-store.ts +++ b/internal/cloud/code-agent-trace-store.ts @@ -228,12 +228,12 @@ export function createCodeAgentTraceStore(options = {}) { export const defaultCodeAgentTraceStore = createCodeAgentTraceStore(); -export function createDurableCodeAgentTraceStore({ traceStore = defaultCodeAgentTraceStore, traceEventStore = null, workbenchEventWriter = null } = {}) { +export function createDurableCodeAgentTraceStore({ traceStore = defaultCodeAgentTraceStore, traceEventStore = null } = {}) { if (!traceEventStore || typeof traceEventStore.writeAgentTraceEvent !== "function") return traceStore; function append(traceId, event = {}, meta = {}) { const normalized = traceStore.append(traceId, event, meta); - if (normalized?.traceId) persistTraceEvent(traceEventStore, normalized, meta, workbenchEventWriter); + if (normalized?.traceId) persistTraceEvent(traceEventStore, normalized, meta); return normalized; } @@ -247,11 +247,11 @@ export function createDurableCodeAgentTraceStore({ traceStore = defaultCodeAgent }; } -function persistTraceEvent(traceEventStore, event, meta = {}, workbenchEventWriter = null) { - void persistTraceEventAsync(traceEventStore, event, meta, workbenchEventWriter); +function persistTraceEvent(traceEventStore, event, meta = {}) { + void persistTraceEventAsync(traceEventStore, event, meta); } -async function persistTraceEventAsync(traceEventStore, event, meta = {}, workbenchEventWriter = null) { +async function persistTraceEventAsync(traceEventStore, event, meta = {}) { try { const resolvedAgentSessionId = event.agentSessionId ?? event.sessionId ?? meta.agentSessionId ?? meta.sessionId; const resolvedSessionId = event.sessionId ?? meta.sessionId ?? resolvedAgentSessionId; @@ -265,13 +265,6 @@ async function persistTraceEventAsync(traceEventStore, event, meta = {}, workben } catch (error) { console.error(`[workbench-projection] durable trace event write failed: traceId=${event.traceId} sourceEventId=${event.sourceEventId ?? "none"} error=${error?.message ?? error}`); } - if (typeof workbenchEventWriter === "function") { - try { - await workbenchEventWriter(event, projectionContext); - } catch (error) { - console.error(`[workbench-projection] workbench event writer failed: traceId=${event.traceId} error=${error?.message ?? error}`); - } - } } catch (error) { // Trace capture must not fail the user turn, but projection write errors must be visible for diagnostics. console.error(`[workbench-projection] persistTraceEvent sync error: traceId=${event.traceId} error=${error?.message ?? error}`); diff --git a/internal/cloud/health-contract.ts b/internal/cloud/health-contract.ts index d3a0e7b3..d9920550 100644 --- a/internal/cloud/health-contract.ts +++ b/internal/cloud/health-contract.ts @@ -13,10 +13,11 @@ import { export const CLOUD_API_HEALTH_CONTRACT_VERSION = "v3"; -export function buildCloudApiReadiness({ db, codeAgent, runtime } = {}) { +export function buildCloudApiReadiness({ db, codeAgent, runtime, kafkaProjector } = {}) { const dbReady = isDbReady(db); const codeAgentReady = isCodeAgentReady(codeAgent); const runtimeReady = isRuntimeReady(runtime); + const kafkaProjectorReady = !kafkaProjector?.started || kafkaProjector.status === "ready"; const blockers = []; let runtimeReadinessBlocker = null; @@ -32,6 +33,7 @@ export function buildCloudApiReadiness({ db, codeAgent, runtime } = {}) { runtimeReadinessBlocker = runtimeBlocker(runtime); blockers.push(runtimeReadinessBlocker); } + if (!kafkaProjectorReady) blockers.push(kafkaProjectorBlocker(kafkaProjector)); const provider = buildProviderReadiness(codeAgent); const dbDurable = buildRuntimeDurabilityReadiness({ db, @@ -59,6 +61,7 @@ export function buildCloudApiReadiness({ db, codeAgent, runtime } = {}) { db: dbReady ? "ready" : "blocked", codeAgent: codeAgentReady ? "ready" : "blocked", runtime: runtimeReady ? "ready" : dbReady ? "blocked" : "waiting_for_db", + kafkaProjector: kafkaProjectorReady ? (kafkaProjector?.started ? "ready" : "disabled") : "blocked", provider: provider.status, dbDurable: dbDurable.status, sessionRunner: sessionRunner.status, @@ -73,6 +76,24 @@ export function buildCloudApiReadiness({ db, codeAgent, runtime } = {}) { }; } +function kafkaProjectorBlocker(projector) { + return { + code: projector?.errorCode ?? "hwlab_kafka_projector_not_ready", + type: "runtime_blocker", + scope: "agentrun-kafka-projector", + status: "open", + sourceIssue: "pikasTech/HWLAB#2464", + summary: projector?.message ?? `AgentRun Kafka projector is ${projector?.status ?? "not ready"}`, + evidence: { + status: projector?.status ?? "unknown", + groupId: projector?.groupId ?? null, + agentRunTopic: projector?.agentRunTopic ?? null, + hwlabTopic: projector?.hwlabTopic ?? null, + valuesRedacted: true + } + }; +} + function isDbReady(db) { return Boolean(db?.ready && db?.connected && db?.liveConnected !== false && db?.liveDbEvidence); } diff --git a/internal/cloud/kafka-event-bridge.test.ts b/internal/cloud/kafka-event-bridge.test.ts index c8c7a8b3..5dcfc727 100644 --- a/internal/cloud/kafka-event-bridge.test.ts +++ b/internal/cloud/kafka-event-bridge.test.ts @@ -4,7 +4,25 @@ import assert from "node:assert/strict"; import { test } from "bun:test"; -import { projectAgentRunKafkaEventToHwlabEvent } from "./kafka-event-bridge.ts"; +import { createCloudApiBunServer } from "./bun-server.ts"; +import { buildCloudApiReadiness } from "./health-contract.ts"; +import { decodeCanonicalAgentRunKafkaMessage, projectAgentRunKafkaEventToHwlabEvent, relayHwlabKafkaOutboxOnce, startHwlabKafkaEventBridge } from "./kafka-event-bridge.ts"; + +const PROJECTOR_ENV = Object.freeze({ + HWLAB_KAFKA_ENABLED: "true", + HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED: "true", + HWLAB_KAFKA_BOOTSTRAP_SERVERS: "kafka.test:9092", + HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC: "agentrun.event.v1", + HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1", + HWLAB_KAFKA_CLIENT_ID: "hwlab-projector-test", + HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID: "hwlab-projector-test-v1", + HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS: "10", + HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS: "60000", + HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE: "1", + HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS: "30000", + HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS: "10000", + HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS: "1000" +}); test("projects AgentRun assistant_message Kafka event into HWLAB trace event", () => { const projected = projectAgentRunKafkaEventToHwlabEvent({ @@ -48,12 +66,29 @@ test("projects AgentRun assistant_message Kafka event into HWLAB trace event", ( assert.equal(projected.event.label, "agentrun:assistant:message"); assert.equal(projected.event.status, "running"); assert.equal(projected.event.text, "partial answer"); + assert.equal(projected.event.assistantText, "partial answer"); + assert.equal(projected.event.finalResponse, null); assert.equal(projected.event.terminal, false); assert.equal(projected.sourceEvent.topic, "agentrun.event.v1"); assert.equal(projected.sourceEvent.offset, "42"); assert.equal(projected.valuesPrinted, false); }); +test("projects final AgentRun assistant text as structured final response", () => { + const projected = projectAgentRunKafkaEventToHwlabEvent({ + schema: "agentrun.event.v1", + eventType: "agentrun.run.event", + run: { runId: "run_final", sessionId: "ses_final" }, + command: { commandId: "cmd_final" }, + event: { seq: 9, type: "assistant_message", payload: { traceId: "trc_final", text: "durable final answer", final: true, replyAuthority: true } } + }, { source: "hwlab-test" }); + + assert.equal(projected.event.assistantText, "durable final answer"); + assert.equal(projected.event.terminal, true); + assert.equal(projected.event.finalResponse.text, "durable final answer"); + assert.equal(projected.event.finalResponse.traceId, "trc_final"); +}); + test("projects AgentRun terminal_status Kafka event into terminal HWLAB event", () => { const projected = projectAgentRunKafkaEventToHwlabEvent({ schema: "agentrun.event.v1", @@ -99,3 +134,240 @@ test("projects AgentRun run-created envelope even when no source event is presen assert.equal(projected.event.label, "agentrun:event:agentrun.run.created"); assert.equal(projected.event.status, "running"); }); + +test("accepts canonical runner-job-created payload with redacted tool credential references", () => { + const message = canonicalMessage({ offset: "60" }); + const envelope = JSON.parse(message.value.toString("utf8")); + envelope.event.type = "backend_status"; + envelope.event.payload = { + traceId: envelope.traceId, + hwlabSessionId: envelope.hwlabSessionId, + commandId: envelope.commandId, + phase: "runner-job-created", + toolCredentials: [{ + tool: "unidesk-ssh", + purpose: "ssh-passthrough", + name: "agentrun-v02-tool-unidesk-ssh", + namespace: "agentrun-v02", + keys: ["UNIDESK_SSH_CLIENT_TOKEN"], + projection: { kind: "env", envName: "UNIDESK_SSH_CLIENT_TOKEN" }, + valuesPrinted: false + }], + valuesPrinted: false + }; + message.value = Buffer.from(JSON.stringify(envelope)); + + const decoded = decodeCanonicalAgentRunKafkaMessage({ topic: "agentrun.event.v1", partition: 0, message }); + assert.equal(decoded.ok, true); + const projected = projectAgentRunKafkaEventToHwlabEvent(decoded.event, { source: "hwlab-test" }); + assert.equal(projected.event.label, "agentrun:backend:runner-job-created"); +}); + +test("Kafka projector commits next offset only after durable projection", async () => { + const calls = []; + let commitInput = null; + const harness = kafkaProjectorHarness({ + async commitAgentRunKafkaProjection(input) { + commitInput = input; + calls.push("db"); + return { projectedSeq: 1 }; + } + }, calls); + const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null }); + await bridge.ready; + await harness.runBatch(canonicalMessage({ offset: "41" })); + + assert.ok(calls.indexOf("heartbeat") < calls.indexOf("db")); + assert.ok(calls.indexOf("db") < calls.indexOf("resolve:41")); + assert.equal(commitInput.partitionKey, "ses_projector_test"); + assert.deepEqual(calls.slice(-3), ["resolve:41", "commit:42", "heartbeat"]); + await bridge.stop(); +}); + +test("Kafka projector leaves offset uncommitted when durable projection throws", async () => { + const calls = []; + const expected = Object.assign(new Error("database unavailable"), { code: "db_unavailable" }); + const harness = kafkaProjectorHarness({ + async commitAgentRunKafkaProjection() { + calls.push("db"); + throw expected; + } + }, calls); + const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null }); + await bridge.ready; + + await assert.rejects(harness.runBatch(canonicalMessage({ offset: "42" })), expected); + assert.equal(calls.some((entry) => entry.startsWith("commit:")), false); + assert.equal(calls.some((entry) => entry.startsWith("resolve:")), false); + await bridge.stop(); +}); + +test("Kafka projector signals committed projection but does not commit a stale generation offset", async () => { + const calls = []; + let signal = null; + let harness; + harness = kafkaProjectorHarness({ + async commitAgentRunKafkaProjection() { + calls.push("db"); + harness.setStale(true); + return { projectedSeq: 1 }; + } + }, calls); + const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null }); + bridge.subscribeProjectionCommits((value) => { signal = value; }); + await bridge.ready; + + await harness.runBatch(canonicalMessage({ offset: "43" })); + + assert.equal(calls.includes("resolve:43"), false); + assert.equal(calls.includes("commit:44"), false); + assert.equal(signal.sessionId, "ses_projector_test"); + assert.equal(signal.traceId, "trc_projector_test"); + await bridge.stop(); +}); + +test("Kafka projector persists malformed and non-HWLAB events before committing offsets", async () => { + const calls = []; + const harness = kafkaProjectorHarness({}, calls); + const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null }); + await bridge.ready; + await harness.runBatch({ offset: "50", value: Buffer.from("{}") }); + await harness.runBatch(canonicalMessage({ offset: "51", traceId: null, hwlabSessionId: null })); + + assert.deepEqual(calls.filter((entry) => ["dlq", "ignored", "commit:51", "commit:52"].includes(entry)), ["dlq", "commit:51", "ignored", "commit:52"]); + await bridge.stop(); +}); + +test("Kafka projector exposes startup failure as rejected readiness and blocked status", async () => { + const expected = Object.assign(new Error("broker unavailable"), { code: "kafka_unavailable" }); + const harness = kafkaProjectorHarness({}, []); + harness.kafkaFactory = () => ({ + consumer: () => harness.consumer, + producer: () => ({ ...harness.producer, connect: async () => { throw expected; } }) + }); + const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null }); + + await assert.rejects(bridge.ready, expected); + assert.equal((await bridge.status()).status, "blocked"); + assert.equal((await bridge.status()).errorCode, "kafka_unavailable"); + await bridge.stop(); +}); + +test("cloud API Bun entry does not listen when Kafka projector startup fails", async () => { + const expected = Object.assign(new Error("broker unavailable before listen"), { code: "kafka_startup_blocked" }); + const harness = kafkaProjectorHarness({}, []); + let bunServeCalls = 0; + const kafkaFactory = () => ({ + consumer: () => harness.consumer, + producer: () => ({ ...harness.producer, connect: async () => { throw expected; } }) + }); + + await assert.rejects(createCloudApiBunServer({ + env: { ...PROJECTOR_ENV }, + runtimeStore: harness.runtimeStore, + kafkaFactory, + accessController: { async ensureBootstrap() {}, async authenticate() { return { ok: false }; } }, + bunServe() { bunServeCalls += 1; throw new Error("Bun.serve must not run"); }, + installSignalHandlers: false, + logger: { log() {}, info() {}, warn() {}, error() {} } + }), expected); + + assert.equal(bunServeCalls, 0); +}); + +test("cloud readiness is blocked while the Kafka projector is blocked", () => { + const readiness = buildCloudApiReadiness({ + db: {}, + codeAgent: {}, + runtime: {}, + kafkaProjector: { started: true, status: "blocked", errorCode: "kafka_startup_blocked", message: "broker unavailable" } + }); + + assert.equal(readiness.components.kafkaProjector, "blocked"); + assert.equal(readiness.blockerCodes.includes("kafka_startup_blocked"), true); + assert.equal(readiness.ready, false); +}); + +test("HWLAB Kafka relay claims just in time and forwards fencing token", async () => { + const calls = []; + const pending = [ + { outboxSeq: 1, eventId: "evt-relay-1", topic: "hwlab.event.v1", partitionKey: "trc-relay", payload: { eventId: "evt-relay-1" }, headers: {}, attemptCount: 2, leaseOwner: "relay-owner", leaseExpiresAt: "2026-07-10T10:02:30.000Z" }, + { outboxSeq: 2, eventId: "evt-relay-2", topic: "hwlab.event.v1", partitionKey: "trc-relay", payload: { eventId: "evt-relay-2" }, headers: {}, attemptCount: 1, leaseOwner: "relay-owner", leaseExpiresAt: "2026-07-10T10:02:30.000Z" } + ]; + const runtimeStore = { + async claimHwlabKafkaOutbox(params) { calls.push(`claim:${params.limit}`); return pending.length ? [pending.shift()] : []; }, + async completeHwlabKafkaOutbox(item) { calls.push(`complete:${item.outboxSeq}:${item.attemptCount}:${item.leaseOwner}`); } + }; + const producer = { async send(record) { calls.push(`send:${record.messages[0].value.includes("evt-relay-1") ? 1 : 2}`); } }; + const config = { relay: { batchSize: 2, leaseMs: 30000, retryBackoffMs: 1000 } }; + + const result = await relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner: "relay-owner", logger: null }); + + assert.equal(result.deliveredCount, 2); + assert.deepEqual(calls, ["claim:1", "send:1", "complete:1:2:relay-owner", "claim:1", "send:2", "complete:2:1:relay-owner"]); +}); + +function kafkaProjectorHarness(overrides = {}, calls = []) { + let runOptions = null; + let stale = false; + const consumer = { + async connect() {}, + async subscribe() {}, + async run(options) { runOptions = options; }, + async commitOffsets(offsets) { calls.push(`commit:${offsets[0].offset}`); }, + async stop() {}, + async disconnect() {} + }; + const producer = { async connect() {}, async send() {}, async disconnect() {} }; + const runtimeStore = { + async commitAgentRunKafkaProjection() { return { projectedSeq: 1 }; }, + async recordFailedAgentRunKafkaMessage() { calls.push("dlq"); }, + async recordIgnoredAgentRunKafkaMessage() { calls.push("ignored"); return { ignored: true }; }, + async claimHwlabKafkaOutbox() { return []; }, + async completeHwlabKafkaOutbox() {}, + async retryHwlabKafkaOutbox() {}, + async hwlabKafkaProjectorStatus() { return { backlogCount: 0 }; }, + ...overrides + }; + return { + consumer, + producer, + runtimeStore, + kafkaFactory: () => ({ consumer: () => consumer, producer: () => producer }), + setStale(value) { stale = value === true; }, + async runBatch(message) { + assert.ok(runOptions?.eachBatch); + await runOptions.eachBatch({ + batch: { topic: "agentrun.event.v1", partition: 2, messages: [message] }, + resolveOffset(offset) { calls.push(`resolve:${offset}`); }, + async heartbeat() { calls.push("heartbeat"); }, + isRunning: () => true, + isStale: () => stale + }); + } + }; +} + +function canonicalMessage({ offset, traceId = "trc_projector_test", hwlabSessionId = "ses_projector_test" }) { + const eventId = `evt_projector_${offset}`; + const runId = "run_projector_test"; + return { + offset, + key: Buffer.from(runId), + value: Buffer.from(JSON.stringify({ + schema: "agentrun.event.v1", + eventType: "agentrun.event.committed", + source: "agentrun-v02", + eventId, + sourceSeq: Number(offset) + 1, + traceId, + hwlabSessionId, + runId, + commandId: "cmd_projector_test", + run: { runId, status: "running", valuesPrinted: false }, + command: { commandId: "cmd_projector_test", runId, seq: 1, type: "prompt", state: "running", valuesPrinted: false }, + event: { id: eventId, runId, seq: Number(offset) + 1, type: "assistant_message", payload: { traceId, hwlabSessionId, commandId: "cmd_projector_test", text: "projected answer" }, createdAt: "2026-07-10T10:00:00.000Z" }, + valuesPrinted: false + })) + }; +} diff --git a/internal/cloud/kafka-event-bridge.ts b/internal/cloud/kafka-event-bridge.ts index 4e2d7069..c31f411e 100644 --- a/internal/cloud/kafka-event-bridge.ts +++ b/internal/cloud/kafka-event-bridge.ts @@ -4,6 +4,7 @@ import { createHash, randomUUID } from "node:crypto"; import { Kafka, logLevel } from "kafkajs"; +import { buildWorkbenchProjectionEventFacts } from "./workbench-projection-writer.ts"; const TRUE_VALUES = new Set(["1", "true", "yes", "on"]); const DEFAULT_AGENTRUN_EVENT_TOPIC = "agentrun.event.v1"; @@ -13,47 +14,126 @@ const DEFAULT_CLIENT_ID = "hwlab-v03-cloud-api"; const DEFAULT_GROUP_ID = "hwlab-v03-agentrun-event-bridge"; const DEFAULT_QUERY_TIMEOUT_MS = 5000; const DEFAULT_QUERY_LIMIT = 50; +const REQUIRED_PROJECTOR_ENV = Object.freeze([ + "HWLAB_KAFKA_BOOTSTRAP_SERVERS", + "HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC", + "HWLAB_KAFKA_EVENT_TOPIC", + "HWLAB_KAFKA_CLIENT_ID", + "HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID", + "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS", + "HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS", + "HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE", + "HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS", + "HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS", + "HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS" +]); export function kafkaEventBridgeConfig(env = process.env) { if (!truthy(env.HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED ?? env.HWLAB_KAFKA_AGENTRUN_CONSUME_ENABLED ?? env.HWLAB_KAFKA_ENABLED)) return null; + const missing = REQUIRED_PROJECTOR_ENV.filter((name) => !stringValue(env[name])); + if (missing.length > 0) { + const error = new Error(`HWLAB Kafka projector configuration is incomplete: ${missing.join(", ")}`); + error.code = "hwlab_kafka_projector_config_incomplete"; + error.missing = missing; + throw error; + } const brokers = csv(env.HWLAB_KAFKA_BOOTSTRAP_SERVERS); - if (brokers.length === 0) return null; - const agentRunTopic = stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC) || DEFAULT_AGENTRUN_EVENT_TOPIC; - const hwlabTopic = stringValue(env.HWLAB_KAFKA_EVENT_TOPIC) || DEFAULT_HWLAB_EVENT_TOPIC; - const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID) || DEFAULT_CLIENT_ID; - const groupId = stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID ?? env.HWLAB_KAFKA_AGENTRUN_CONSUMER_GROUP_ID) || DEFAULT_GROUP_ID; - return { brokers, agentRunTopic, hwlabTopic, clientId, groupId }; + const agentRunTopic = stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC); + const hwlabTopic = stringValue(env.HWLAB_KAFKA_EVENT_TOPIC); + const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID); + const groupId = stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID); + const relay = { + intervalMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS, "HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS"), + batchSize: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE, "HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE"), + leaseMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS, "HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS"), + sendTimeoutMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS, "HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS"), + retryBackoffMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS, "HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS") + }; + if (relay.sendTimeoutMs + 5000 >= relay.leaseMs) { + const error = new Error("HWLAB Kafka relay lease must exceed producer send timeout by at least 5000ms."); + error.code = "hwlab_kafka_relay_lease_invalid"; + throw error; + } + return { + brokers, + agentRunTopic, + hwlabTopic, + clientId, + groupId, + projectorHeartbeatIntervalMs: strictPositiveInteger(env.HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS, "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS"), + relay + }; } -export function startHwlabKafkaEventBridge({ env = process.env, logger = console, kafkaFactory = defaultKafkaFactory } = {}) { +export function startHwlabKafkaEventBridge({ env = process.env, logger = console, kafkaFactory = defaultKafkaFactory, runtimeStore = null, now = () => new Date().toISOString() } = {}) { const config = kafkaEventBridgeConfig(env); - if (!config) return { started: false, reason: "disabled_or_unconfigured", stop() {}, valuesPrinted: false }; + if (!config) return { started: false, reason: "disabled_or_unconfigured", stop() {}, subscribeProjectionCommits() { return () => {}; }, valuesPrinted: false }; + requireKafkaProjectorStore(runtimeStore); let stopped = false; let consumer = null; let producer = null; + let relayTimer = null; + let relayRunning = false; + let stopProjectionNotifications = null; + const projectionCommitListeners = new Set(); + const publishProjectionCommit = (signal = {}) => { + for (const listener of [...projectionCommitListeners]) { + try { listener({ sessionId: signal.sessionId ?? null, traceId: signal.traceId ?? null, outboxSeq: signal.outboxSeq ?? null, recovery: signal.recovery === true, valuesRedacted: true }); } catch {} + } + }; + const relayOwner = `${config.clientId}:outbox-relay:${randomUUID()}`; + let startupError = null; + let startupReady = false; const ready = (async () => { const kafka = kafkaFactory(config); consumer = kafka.consumer({ groupId: config.groupId, allowAutoTopicCreation: false }); producer = kafka.producer({ allowAutoTopicCreation: false }); await producer.connect(); await consumer.connect(); + if (typeof runtimeStore.subscribeWorkbenchProjectionCommits === "function") { + stopProjectionNotifications = await runtimeStore.subscribeWorkbenchProjectionCommits(publishProjectionCommit); + } await consumer.subscribe({ topic: config.agentRunTopic, fromBeginning: false }); await consumer.run({ - eachMessage: async ({ topic, partition, message }) => { - if (stopped) return; - const projected = projectAgentRunKafkaMessageToHwlabEvent({ topic, partition, message }, { source: config.clientId }); - if (!projected) return; - await producer.send({ - topic: config.hwlabTopic, - messages: [{ - key: kafkaMessageKey(projected), - value: JSON.stringify(projected), - headers: kafkaHeaders(projected) - }] - }); + autoCommit: false, + eachBatchAutoResolve: false, + eachBatch: async ({ batch, resolveOffset, heartbeat, isRunning, isStale }) => { + for (const message of batch.messages) { + if (stopped || !isRunning() || isStale()) break; + await heartbeat(); + const heartbeatLoop = startKafkaHeartbeatLoop(heartbeat, config.projectorHeartbeatIntervalMs); + let projection; + try { + projection = await projectAgentRunKafkaMessage({ topic: batch.topic, partition: batch.partition, message }, { runtimeStore, config, logger }); + } finally { + await heartbeatLoop.stop(); + } + if (projection?.projected) publishProjectionCommit(projection); + if (heartbeatLoop.error) throw heartbeatLoop.error; + if (stopped || !isRunning() || isStale()) { + logWarn(logger, "hwlab-kafka-offset-commit-skipped", { topic: batch.topic, partition: batch.partition, offset: message.offset, reason: isStale() ? "stale_generation" : "consumer_stopped", valuesPrinted: false }); + break; + } + resolveOffset(message.offset); + await consumer.commitOffsets([{ topic: batch.topic, partition: batch.partition, offset: nextKafkaOffset(message.offset) }]); + await heartbeat(); + } } }); + const runRelay = async () => { + if (stopped || relayRunning) return; + relayRunning = true; + try { + await relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner: relayOwner, now, logger }); + } finally { + relayRunning = false; + } + }; + relayTimer = setInterval(() => { void runRelay().catch((error) => logWarn(logger, "hwlab-kafka-outbox-relay-failed", { message: errorMessage(error), valuesPrinted: false })); }, config.relay.intervalMs); + relayTimer.unref?.(); + await runRelay(); + startupReady = true; logInfo(logger, "hwlab-kafka-event-bridge-started", { agentRunTopic: config.agentRunTopic, hwlabTopic: config.hwlabTopic, @@ -61,7 +141,9 @@ export function startHwlabKafkaEventBridge({ env = process.env, logger = console groupId: config.groupId, valuesPrinted: false }); - })().catch((error) => { + })(); + void ready.catch((error) => { + startupError = error; logWarn(logger, "hwlab-kafka-event-bridge-start-failed", { message: errorMessage(error), agentRunTopic: config.agentRunTopic, @@ -76,12 +158,113 @@ export function startHwlabKafkaEventBridge({ env = process.env, logger = console ready, async stop() { stopped = true; + if (relayTimer) clearInterval(relayTimer); + if (typeof stopProjectionNotifications === "function") { + try { await stopProjectionNotifications(); } catch {} + } + if (consumer?.stop) await consumer.stop().catch(() => undefined); await Promise.allSettled([consumer?.disconnect?.(), producer?.disconnect?.()]); }, + async status() { + if (startupError) return { status: "blocked", errorCode: startupError.code ?? "hwlab_kafka_projector_start_failed", message: errorMessage(startupError), valuesRedacted: true }; + if (!startupReady) return { status: "initializing", valuesRedacted: true }; + return { status: "ready", ...(await runtimeStore.hwlabKafkaProjectorStatus()), valuesRedacted: true }; + }, + startupStatus() { + if (startupError) return { status: "blocked", errorCode: startupError.code ?? "hwlab_kafka_projector_start_failed", message: errorMessage(startupError), valuesRedacted: true }; + return { status: startupReady ? "ready" : "initializing", valuesRedacted: true }; + }, + subscribeProjectionCommits(listener) { + if (typeof listener !== "function") return () => {}; + projectionCommitListeners.add(listener); + return () => projectionCommitListeners.delete(listener); + }, valuesPrinted: false }; } +export async function projectAgentRunKafkaMessage(kafkaMessage = {}, { runtimeStore, config, logger = console } = {}) { + const transport = kafkaTransport(kafkaMessage); + const decoded = decodeCanonicalAgentRunKafkaMessage(kafkaMessage); + if (!decoded.ok) { + await runtimeStore.recordFailedAgentRunKafkaMessage({ transport, sourceEventId: decoded.sourceEventId, envelope: decoded.envelope, error: decoded.error }); + logWarn(logger, "hwlab-kafka-message-dlq", { topic: transport.sourceTopic, partition: transport.sourcePartition, offset: transport.sourceOffset, errorCode: decoded.error.code, valuesPrinted: false }); + return { handled: true, invalid: true, valuesPrinted: false }; + } + if (!stringValue(decoded.event.traceId) || !stringValue(decoded.event.hwlabSessionId)) { + const ignored = await runtimeStore.recordIgnoredAgentRunKafkaMessage({ transport, envelope: decoded.event, reason: "hwlab-correlation-absent" }); + return { handled: true, ignored: true, duplicate: ignored?.duplicate === true, valuesPrinted: false }; + } + const projected = projectAgentRunKafkaEventToHwlabEvent(decoded.event, { + source: config.clientId, + sourceTopic: transport.sourceTopic, + sourcePartition: transport.sourcePartition, + sourceOffset: transport.sourceOffset, + sourceKey: transport.sourceKey, + inputSha256: transport.inputSha256 + }); + const result = await runtimeStore.commitAgentRunKafkaProjection({ + transport, + canonicalEvent: decoded.event, + projectedEvent: projected.event, + requestMeta: { traceId: projected.traceId, sessionId: projected.sessionId, agentSessionId: projected.sessionId, valuesPrinted: false }, + factsFactory: ({ projectedSeq, previousCheckpoint, projectedAt }) => buildWorkbenchProjectionEventFacts({ event: projected.event, requestMeta: { traceId: projected.traceId, sessionId: projected.sessionId }, previousCheckpoint, projectedSeq, projectedAt }), + hwlabEvent: projected, + hwlabTopic: config.hwlabTopic, + partitionKey: kafkaMessageKey(projected), + headers: kafkaHeaders(projected) + }); + if (result.conflict) logWarn(logger, "hwlab-kafka-message-conflict", { topic: transport.sourceTopic, partition: transport.sourcePartition, offset: transport.sourceOffset, errorCode: result.diagnostic?.code, valuesPrinted: false }); + return { handled: true, projected: !result.duplicate && !result.conflict, duplicate: result.duplicate === true, conflict: result.conflict === true, projectedSeq: result.projectedSeq, sessionId: projected.sessionId, traceId: projected.traceId, valuesPrinted: false }; +} + +export async function relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner, now = () => new Date().toISOString(), logger = console } = {}) { + let claimedCount = 0; + let deliveredCount = 0; + let retryCount = 0; + for (let index = 0; index < config.relay.batchSize; index += 1) { + const item = (await runtimeStore.claimHwlabKafkaOutbox({ owner, leaseMs: config.relay.leaseMs, limit: 1 }))[0]; + if (!item) break; + claimedCount += 1; + try { + await producer.send({ topic: item.topic, timeout: config.relay.sendTimeoutMs, messages: [{ key: item.partitionKey, value: JSON.stringify(item.payload), headers: item.headers }] }); + await runtimeStore.completeHwlabKafkaOutbox(item); + deliveredCount += 1; + } catch (error) { + const retryAt = new Date(Date.parse(now()) + config.relay.retryBackoffMs).toISOString(); + await runtimeStore.retryHwlabKafkaOutbox(item, error, retryAt); + retryCount += 1; + logWarn(logger, "hwlab-kafka-outbox-publish-retry", { outboxSeq: item.outboxSeq, eventId: item.eventId, retryAt, message: errorMessage(error), valuesPrinted: false }); + break; + } + } + return { claimedCount, deliveredCount, retryCount, valuesPrinted: false }; +} + +export function startKafkaHeartbeatLoop(heartbeat, intervalMs) { + let stopped = false; + let running = null; + let heartbeatError = null; + const pulse = () => { + if (stopped || running || typeof heartbeat !== "function") return; + running = Promise.resolve() + .then(() => heartbeat()) + .catch((error) => { heartbeatError ??= error; }) + .finally(() => { running = null; }); + }; + pulse(); + const timer = setInterval(pulse, intervalMs); + timer.unref?.(); + return { + get error() { return heartbeatError; }, + async stop() { + stopped = true; + clearInterval(timer); + if (running) await running; + } + }; +} + export function projectAgentRunKafkaMessageToHwlabEvent(kafkaMessage = {}, options = {}) { const valueText = kafkaMessage?.message?.value ? Buffer.from(kafkaMessage.message.value).toString("utf8") : ""; const input = parseJson(valueText); @@ -106,7 +289,8 @@ export function projectAgentRunKafkaEventToHwlabEvent(input = {}, options = {}) const context = objectValue(input.context ?? payload.context); const sourceSeq = integerValue(sourceEvent.seq ?? input.seq ?? input.sourceSeq); const traceId = firstText(input.traceId, hwlab.traceId, context.traceId, sourceEvent.traceId, payload.traceId, payload.hwlabTraceId, payload.businessTraceId, payload.metadata?.traceId, command.traceId, run.traceId); - const sessionId = firstText(input.sessionId, hwlab.sessionId, context.sessionId, run.sessionId, payload.sessionId, payload.hwlabSessionId); + const sourceEventId = firstText(input.eventId, sourceEvent.id); + const sessionId = firstText(input.hwlabSessionId, payload.hwlabSessionId, hwlab.sessionId, context.hwlabSessionId, input.sessionId, context.sessionId, run.hwlabSessionId, run.sessionId, payload.sessionId); const runId = firstText(run.runId, sourceEvent.runId, payload.runId, input.runId); const commandId = firstText(command.commandId, sourceEvent.commandId, payload.commandId, input.commandId); const event = mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run, command, traceId, sessionId, sourceSeq, runId, commandId }); @@ -114,6 +298,8 @@ export function projectAgentRunKafkaEventToHwlabEvent(input = {}, options = {}) return { schema: "hwlab.event.v1", eventType: "hwlab.trace.event.projected", + eventId: sourceEventId ? `hwlab:${sourceEventId}` : null, + sourceEventId, source: stringValue(options.source) || DEFAULT_CLIENT_ID, producedAt, traceId, @@ -129,7 +315,7 @@ export function projectAgentRunKafkaEventToHwlabEvent(input = {}, options = {}) agentRunEventType: firstText(sourceEvent.type, payload.type), valuesRedacted: true }, - event, + event: { ...event, sourceEventId }, sourceEvent: { schema: firstText(input.schema), topic: stringValue(options.sourceTopic), @@ -147,6 +333,81 @@ export function projectAgentRunKafkaEventToHwlabEvent(input = {}, options = {}) }; } +export function decodeCanonicalAgentRunKafkaMessage(kafkaMessage = {}) { + const valueText = kafkaMessage?.message?.value ? Buffer.from(kafkaMessage.message.value).toString("utf8") : ""; + const envelope = parseJson(valueText); + const sourceEventId = firstText(envelope?.eventId, envelope?.event?.id); + try { + assertCanonicalAgentRunEvent(envelope); + return { ok: true, event: envelope, sourceEventId, inputSha256: sha256(valueText), valuesPrinted: false }; + } catch (error) { + return { ok: false, envelope, sourceEventId, inputSha256: sha256(valueText), error: { code: error?.code ?? "workbench_kafka_message_invalid", message: errorMessage(error), valuesRedacted: true }, valuesPrinted: false }; + } +} + +function assertCanonicalAgentRunEvent(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) throw contractError("workbench_kafka_json_invalid", "Kafka value must be a JSON object."); + assertClosedObject(value, ["schema", "eventType", "source", "eventId", "outboxSeq", "sourceSeq", "committedAt", "traceId", "sessionId", "hwlabSessionId", "runId", "commandId", "run", "command", "event", "valuesPrinted"], "AgentRun envelope"); + if (value.schema !== "agentrun.event.v1" || value.eventType !== "agentrun.event.committed") throw contractError("workbench_kafka_schema_invalid", "Kafka value is not a canonical AgentRun committed event."); + for (const [name, field] of [["eventId", value.eventId], ["runId", value.runId]]) { + if (!stringValue(field)) throw contractError("workbench_kafka_schema_invalid", `Canonical AgentRun ${name} is required.`); + } + for (const [name, field] of [["traceId", value.traceId], ["hwlabSessionId", value.hwlabSessionId]]) { + if (field !== null && field !== undefined && !stringValue(field)) throw contractError("workbench_kafka_schema_invalid", `Canonical AgentRun ${name} must be a string or null.`); + } + if (!Number.isInteger(Number(value.sourceSeq)) || Number(value.sourceSeq) <= 0) throw contractError("workbench_kafka_schema_invalid", "Canonical AgentRun sourceSeq must be a positive integer."); + if (value.valuesPrinted !== false) throw contractError("workbench_kafka_redaction_invalid", "Canonical AgentRun event must declare valuesPrinted=false."); + assertClosedObject(value.run, ["runId", "status", "terminalStatus", "failureKind", "tenantId", "projectId", "providerId", "backendProfile", "sessionId", "hwlabSessionId", "conversationId", "threadId", "valuesPrinted"], "AgentRun run"); + if (value.command !== null && value.command !== undefined) assertClosedObject(value.command, ["commandId", "runId", "seq", "type", "state", "payloadHash", "valuesPrinted"], "AgentRun command"); + assertClosedObject(value.event, ["id", "runId", "seq", "type", "payload", "createdAt"], "AgentRun event"); + if (value.event.id !== value.eventId || Number(value.event.seq) !== Number(value.sourceSeq)) throw contractError("workbench_kafka_identity_invalid", "Canonical AgentRun event identity does not match the envelope."); + if (!value.event.payload || typeof value.event.payload !== "object" || Array.isArray(value.event.payload)) throw contractError("workbench_kafka_schema_invalid", "Canonical AgentRun event payload must be an object."); + if (containsSecretLikeKey(value)) throw contractError("workbench_kafka_redaction_invalid", "Canonical AgentRun event contains a secret-like field."); + return value; +} + +function assertClosedObject(value, allowed, label) { + if (!value || typeof value !== "object" || Array.isArray(value)) throw contractError("workbench_kafka_schema_invalid", `${label} must be an object.`); + const unknown = Object.keys(value).filter((key) => !allowed.includes(key)); + if (unknown.length > 0) throw contractError("workbench_kafka_schema_invalid", `${label} contains unsupported fields: ${unknown.join(", ")}`); +} + +function containsSecretLikeKey(value, depth = 0) { + if (depth > 12 || value === null || value === undefined) return false; + if (Array.isArray(value)) return value.some((item) => containsSecretLikeKey(item, depth + 1)); + if (typeof value !== "object") return false; + return Object.entries(value).some(([key, entry]) => { + const normalized = key.replace(/[^a-z0-9]/giu, "").toLowerCase(); + if (/^(apikey|accesstoken|refreshtoken|password|authorization|credentialvalue|secretvalue|privatekey|clientsecret|bearertoken)$/u.test(normalized)) return !redactedSecretMarker(entry); + if (typeof entry === "string" && /^(?:https?|postgres(?:ql)?):\/\/[^/@\s]+:[^/@\s]+@/iu.test(entry)) return true; + return containsSecretLikeKey(entry, depth + 1); + }); +} + +function redactedSecretMarker(value) { + if (value === null || value === undefined) return true; + const normalized = stringValue(value)?.toLowerCase(); + return normalized === "redacted" || normalized === "[redacted]" || normalized === "present" || normalized === "missing"; +} + +function kafkaTransport(kafkaMessage = {}) { + const valueText = kafkaMessage?.message?.value ? Buffer.from(kafkaMessage.message.value).toString("utf8") : ""; + return { + sourceTopic: stringValue(kafkaMessage.topic), + sourcePartition: integerValue(kafkaMessage.partition), + sourceOffset: stringValue(kafkaMessage.message?.offset), + sourceKey: kafkaMessage.message?.key ? Buffer.from(kafkaMessage.message.key).toString("utf8") : null, + inputSha256: sha256(valueText), + valuesPrinted: false + }; +} + +function requireKafkaProjectorStore(runtimeStore) { + const required = ["commitAgentRunKafkaProjection", "recordFailedAgentRunKafkaMessage", "recordIgnoredAgentRunKafkaMessage", "claimHwlabKafkaOutbox", "completeHwlabKafkaOutbox", "retryHwlabKafkaOutbox", "hwlabKafkaProjectorStatus"]; + const missing = required.filter((name) => typeof runtimeStore?.[name] !== "function"); + if (missing.length > 0) throw contractError("hwlab_kafka_projector_store_invalid", `Kafka projector requires a durable runtime store: ${missing.join(", ")}`); +} + export async function queryKafkaEventStream({ env = process.env, stream = "hwlab", topic = null, traceId = null, sessionId = null, runId = null, commandId = null, limit = DEFAULT_QUERY_LIMIT, timeoutMs = DEFAULT_QUERY_TIMEOUT_MS, fromBeginning = true, kafkaFactory = defaultKafkaFactory } = {}) { const brokers = csv(env.HWLAB_KAFKA_BOOTSTRAP_SERVERS); if (brokers.length === 0) throw new Error("HWLAB_KAFKA_BOOTSTRAP_SERVERS is required for Kafka event queries."); @@ -207,52 +468,6 @@ export async function queryKafkaEventStream({ env = process.env, stream = "hwlab }; } -export async function openKafkaEventStream({ env = process.env, stream = "hwlab", topic = null, traceId = null, sessionId = null, runId = null, commandId = null, fromBeginning = false, onEvent, onError = null, kafkaFactory = defaultKafkaFactory } = {}) { - if (typeof onEvent !== "function") throw new Error("onEvent callback is required for Kafka event streaming."); - const brokers = csv(env.HWLAB_KAFKA_BOOTSTRAP_SERVERS); - if (brokers.length === 0) throw new Error("HWLAB_KAFKA_BOOTSTRAP_SERVERS is required for Kafka event streams."); - const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID) || DEFAULT_CLIENT_ID; - const resolvedTopic = stringValue(topic) || kafkaTopicForStream(stream, env); - const filters = compactObject({ traceId, sessionId, runId, commandId }); - const kafka = kafkaFactory({ brokers, clientId: `${clientId}-sse` }); - const consumer = kafka.consumer({ groupId: `${clientId}-sse-${Date.now()}-${randomUUID().slice(0, 8)}`, allowAutoTopicCreation: false }); - let running = false; - let stopped = false; - await consumer.connect(); - await consumer.subscribe({ topic: resolvedTopic, fromBeginning: fromBeginning === true }); - running = true; - consumer.run({ - eachMessage: async ({ topic: messageTopic, partition, message }) => { - if (stopped) return; - const valueText = message.value ? Buffer.from(message.value).toString("utf8") : ""; - const value = parseJson(valueText); - if (!value || !eventMatchesFilters(value, filters)) return; - await onEvent({ - topic: messageTopic, - partition, - offset: message.offset, - key: message.key ? Buffer.from(message.key).toString("utf8") : null, - timestamp: message.timestamp ?? null, - value - }); - } - }).catch((error) => { - if (!stopped && typeof onError === "function") onError(error); - }); - return { - ok: true, - stream, - topic: resolvedTopic, - filters, - async stop() { - stopped = true; - if (running) await consumer.stop().catch(() => undefined); - await boundedDisconnect(consumer); - }, - valuesPrinted: false - }; -} - function mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run, command, traceId, sessionId, sourceSeq, runId, commandId }) { const base = { traceId, @@ -270,7 +485,22 @@ function mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run, const type = firstText(sourceEvent.type, payload.type, input.eventType) || "event"; if (type === "assistant_message") { const terminal = payload.replyAuthority === true || payload.final === true; - return { ...base, type: "assistant", eventType: "assistant", status: terminal ? "completed" : "running", label: "agentrun:assistant:message", message: textPayload(payload, "assistant message"), text: textPayload(payload, ""), itemId: firstText(payload.itemId), replyAuthority: payload.replyAuthority === true, final: payload.final === true, terminal }; + const assistantText = textPayload(payload, "assistant message"); + return { + ...base, + type: "assistant", + eventType: "assistant", + status: terminal ? "completed" : "running", + label: "agentrun:assistant:message", + message: assistantText, + text: assistantText, + assistantText, + finalResponse: terminal ? { text: assistantText, status: "completed", traceId, source: "agentrun.kafka", valuesPrinted: false } : null, + itemId: firstText(payload.itemId), + replyAuthority: payload.replyAuthority === true, + final: payload.final === true, + terminal + }; } if (type === "backend_status") { const phase = firstText(payload.phase) || "status"; @@ -349,7 +579,7 @@ async function boundedDisconnect(consumer, timeoutMs = 2000) { } function kafkaMessageKey(projected) { - return firstText(projected.traceId, projected.sessionId, projected.context?.commandId, projected.context?.runId) || "hwlab-event"; + return firstText(projected.sessionId, projected.traceId, projected.context?.commandId, projected.context?.runId) || "hwlab-event"; } function kafkaHeaders(projected) { @@ -408,6 +638,26 @@ function integerValue(value) { return Number.isFinite(parsed) ? Math.floor(parsed) : null; } +function nextKafkaOffset(value) { + try { + return (BigInt(String(value)) + 1n).toString(); + } catch { + throw contractError("workbench_kafka_offset_invalid", "Kafka source offset must be an integer string."); + } +} + +function strictPositiveInteger(value, name) { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) throw contractError("hwlab_kafka_projector_config_invalid", `${name} must be a positive integer.`); + return parsed; +} + +function contractError(code, message) { + const error = new Error(message); + error.code = code; + return error; +} + function timestampValue(value) { const text = stringValue(value); if (!text) return null; diff --git a/internal/cloud/kafka-shadow-producer.ts b/internal/cloud/kafka-shadow-producer.ts deleted file mode 100644 index d5ef390a..00000000 --- a/internal/cloud/kafka-shadow-producer.ts +++ /dev/null @@ -1,185 +0,0 @@ -// SPEC: PJ2026-010205 HWLAB接入 draft-2026-06-25-p0-session-warm-runner-contract. -// Responsibility: best-effort Kafka shadow production for HWLAB -> AgentRun command admission. - -import { createHash } from "node:crypto"; - -import { Kafka, logLevel } from "kafkajs"; - -const TRUE_VALUES = new Set(["1", "true", "yes", "on"]); -const warnedCodes = new Set(); -let producerState = null; - -export function publishHwlabAgentRunCommandShadow({ params = {}, payload = {}, traceId, lifecycle = {}, env = process.env } = {}) { - const config = kafkaShadowConfig(env); - if (!config) return; - const message = buildHwlabCommandShadowMessage({ params, payload, traceId, lifecycle, config }); - void producerForConfig(config) - .then((producer) => producer.send({ topic: config.topic, messages: [message] })) - .catch((error) => warnShadowProducer("hwlab-kafka-shadow-produce-failed", { - message: error instanceof Error ? error.message : String(error), - topic: config.topic, - clientId: config.clientId - })); -} - -function kafkaShadowConfig(env) { - if (!truthy(env.HWLAB_KAFKA_SHADOW_PRODUCE_ENABLED)) return null; - if (truthy(env.HWLAB_KAFKA_SHADOW_CONSUME_ENABLED)) { - warnOnce("hwlab-kafka-shadow-consume-ignored", { - message: "HWLAB Kafka consumer cutover is disabled in this stage; producer continues in shadow mode.", - valuesPrinted: false - }); - } - const brokers = csv(env.HWLAB_KAFKA_BOOTSTRAP_SERVERS); - const topic = stringValue(env.HWLAB_KAFKA_COMMAND_TOPIC); - const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID); - const missing = []; - if (brokers.length === 0) missing.push("HWLAB_KAFKA_BOOTSTRAP_SERVERS"); - if (!topic) missing.push("HWLAB_KAFKA_COMMAND_TOPIC"); - if (!clientId) missing.push("HWLAB_KAFKA_CLIENT_ID"); - if (missing.length > 0) { - warnOnce("hwlab-kafka-shadow-config-missing", { missing, valuesPrinted: false }); - return null; - } - return { brokers, topic, clientId, key: `${clientId}|${topic}|${brokers.join(",")}` }; -} - -function buildHwlabCommandShadowMessage({ params, payload, traceId, lifecycle, config }) { - const agentRun = objectValue(payload.agentRun); - const prompt = promptEvidence(params); - const event = { - schema: "hwlab.agentrun.command.shadow.v1", - eventType: "hwlab.command.admitted", - source: config.clientId, - producedAt: new Date().toISOString(), - mode: "shadow-produce-only", - traceId: safeText(traceId ?? payload.traceId ?? params.traceId), - hwlab: { - sessionId: safeText(params.sessionId ?? payload.sessionId ?? agentRun.hwlabSessionId), - conversationId: safeText(params.conversationId ?? payload.conversationId ?? agentRun.conversationId), - turnId: safeText(lifecycle.turnId ?? payload.turnId), - userMessageId: safeText(lifecycle.userMessageId ?? payload.userMessageId), - assistantMessageId: safeText(lifecycle.assistantMessageId ?? payload.assistantMessageId), - projectId: safeText(params.projectId ?? payload.projectId), - providerProfile: safeText(params.providerProfile ?? agentRun.backendProfile) - }, - agentRun: { - adapter: safeText(agentRun.adapter), - runId: safeText(agentRun.runId), - commandId: safeText(agentRun.commandId), - attemptId: safeText(agentRun.attemptId), - runnerId: safeText(agentRun.runnerId), - jobName: safeText(agentRun.jobName), - namespace: safeText(agentRun.namespace), - backendProfile: safeText(agentRun.backendProfile), - status: safeText(agentRun.status), - commandState: safeText(agentRun.commandState), - terminalStatus: safeText(agentRun.terminalStatus) - }, - prompt, - diagnostics: { - payloadSha256: jsonSha256({ traceId: traceId ?? null, agentRun: eventKeyParts(agentRun), prompt }), - payloadBytes: jsonBytes({ traceId: traceId ?? null, agentRun: eventKeyParts(agentRun), prompt }), - valuesPrinted: false, - shadowConsumeEnabled: false - }, - valuesPrinted: false - }; - return { - key: safeText(agentRun.commandId ?? traceId ?? params.sessionId) || config.clientId, - value: JSON.stringify(event), - headers: { - "x-shadow-mode": "produce-only", - "x-values-printed": "false", - "x-trace-id": event.traceId ?? "" - } - }; -} - -async function producerForConfig(config) { - if (producerState?.key === config.key && producerState.producer) return producerState.producer; - if (producerState?.key === config.key && producerState.connecting) return producerState.connecting; - const kafka = new Kafka({ clientId: config.clientId, brokers: config.brokers, logLevel: logLevel.NOTHING }); - const producer = kafka.producer({ allowAutoTopicCreation: false }); - const state = { - key: config.key, - producer, - connecting: producer.connect().then(() => { - state.connecting = null; - return producer; - }).catch((error) => { - if (producerState === state) producerState = null; - throw error; - }) - }; - producerState = state; - return state.connecting; -} - -function promptEvidence(params) { - const text = firstString(params.prompt, params.message, params.input, objectValue(params.userMessage).text, objectValue(params.userMessage).content); - if (!text) return { present: false, bytes: 0, sha256: null, valuesPrinted: false }; - return { - present: true, - bytes: Buffer.byteLength(text, "utf8"), - sha256: createHash("sha256").update(text).digest("hex"), - valuesPrinted: false - }; -} - -function eventKeyParts(agentRun) { - return { - runId: safeText(agentRun.runId), - commandId: safeText(agentRun.commandId), - attemptId: safeText(agentRun.attemptId), - backendProfile: safeText(agentRun.backendProfile) - }; -} - -function truthy(value) { - return TRUE_VALUES.has(String(value ?? "").trim().toLowerCase()); -} - -function csv(value) { - return String(value ?? "").split(",").map((entry) => entry.trim()).filter(Boolean); -} - -function stringValue(value) { - const text = String(value ?? "").trim(); - return text.length > 0 ? text : null; -} - -function safeText(value, max = 240) { - const text = stringValue(value); - if (!text) return null; - return text.length > max ? `${text.slice(0, max)}...` : text; -} - -function firstString(...values) { - for (const value of values) { - if (typeof value === "string" && value.trim().length > 0) return value; - } - return null; -} - -function objectValue(value) { - return value && typeof value === "object" && !Array.isArray(value) ? value : {}; -} - -function jsonSha256(value) { - return createHash("sha256").update(JSON.stringify(value)).digest("hex"); -} - -function jsonBytes(value) { - return Buffer.byteLength(JSON.stringify(value), "utf8"); -} - -function warnOnce(code, details) { - if (warnedCodes.has(code)) return; - warnedCodes.add(code); - warnShadowProducer(code, details); -} - -function warnShadowProducer(code, details = {}) { - console.warn(JSON.stringify({ code, component: "hwlab-kafka-shadow-producer", ...details, valuesPrinted: false })); -} diff --git a/internal/cloud/server-agent-chat-agentrun-backoff.test.ts b/internal/cloud/server-agent-chat-agentrun-backoff.test.ts deleted file mode 100644 index 06ae85eb..00000000 --- a/internal/cloud/server-agent-chat-agentrun-backoff.test.ts +++ /dev/null @@ -1,130 +0,0 @@ -// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. -// Responsibility: Focused AgentRun projection polling/backoff regressions for Cloud API chat result sync. - -import assert from "node:assert/strict"; -import { createServer as createHttpServer } from "node:http"; -import { test } from "bun:test"; - -import { syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts"; -import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; - -test("AgentRun sync stops upstream polling when terminal command result is already sealed (#1445)", async () => { - const traceId = "trc_1445_sealed_terminal"; - const runId = "run_1445_sealed_terminal"; - const commandId = "cmd_1445_sealed_terminal"; - const traceStore = createCodeAgentTraceStore(); - let fetched = false; - const synced = await syncAgentRunChatResult({ - traceId, - currentResult: { - ok: true, - status: "completed", - traceId, - finalResponse: { text: "sealed final response", traceId }, - traceSummary: { source: "agentrun-command-result", traceId, agentRun: { runId, commandId, lastSeq: 17 } }, - agentRun: { - adapter: "agentrun-v01", - runId, - commandId, - traceId, - status: "completed", - runStatus: "completed", - commandState: "completed", - terminalStatus: "completed", - providerTrace: { traceId, runId, commandId, valuesPrinted: false }, - lastSeq: 17, - managerUrl: "http://127.0.0.1:1", - valuesPrinted: false - }, - valuesPrinted: false - }, - options: { - fetchImpl: async () => { - fetched = true; - throw new Error("sealed terminal result should not fetch AgentRun"); - }, - env: { HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1" } - }, - traceStore - }); - - assert.equal(fetched, false); - assert.equal(synced.terminalRefreshSkipped, true); - assert.equal(synced.resultSynced, false); - assert.equal(synced.polling.rootCause, "terminal_result_sealed"); - assert.equal(synced.polling.backoffMs, 0); - assert.equal(synced.polling.terminalConsistencyGap, 0); -}); - -test("AgentRun events rate limit writes durable projection backoff (#1445)", async () => { - const traceId = "trc_1445_events_rate_limit"; - const runId = "run_1445_events_rate_limit"; - const commandId = "cmd_1445_events_rate_limit"; - const calls = []; - const writes = []; - const runtimeStore = { - async getWorkbenchProjectionState() { - return { projectionState: { traceId, sourceRunId: runId, sourceCommandId: commandId, lastSourceSeq: 40, pollCount: 2, noProgressPollCount: 1, resultSyncState: "not_started", createdAt: "2026-07-02T00:00:00.000Z" } }; - }, - async writeWorkbenchProjectionState(input = {}) { - writes.push(input.projectionState); - return { written: true, projectionState: input.projectionState }; - } - }; - const agentRunServer = createHttpServer(async (request, response) => { - const url = new URL(request.url || "/", "http://127.0.0.1"); - calls.push(`${request.method} ${url.pathname}${url.search}`); - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/events`) { - response.writeHead(429, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: false, failureKind: "provider_rate_limited", message: "provider rate limited; retry later" })}\n`); - return; - } - response.writeHead(404, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: false, message: `unexpected ${request.method} ${url.pathname}` })}\n`); - }); - await new Promise((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve)); - const agentRunPort = agentRunServer.address().port; - const traceStore = createCodeAgentTraceStore(); - try { - await assert.rejects( - syncAgentRunChatResult({ - traceId, - currentResult: { - status: "running", - traceId, - agentRun: { adapter: "agentrun-v01", runId, commandId, traceId, providerTrace: { traceId }, lastSeq: 40, managerUrl: `http://127.0.0.1:${agentRunPort}`, valuesPrinted: false }, - valuesPrinted: false - }, - options: { - runtimeStore, - env: { - HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_BACKOFF_INITIAL_MS: "1000", - HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS: "2500" - } - }, - traceStore - }), - (error) => { - assert.equal(error.statusCode, 429); - assert.equal(error.polling.rootCause, "provider_rate_limited"); - assert.equal(error.polling.backoffReason, "provider_retry_or_rate_limit"); - assert.equal(error.polling.backoffMs, 2000); - assert.ok(error.polling.nextRetryAt); - return true; - } - ); - - assert.equal(calls.length, 1); - assert.match(calls[0], /afterSeq=40/u); - assert.equal(writes.length, 1); - assert.equal(writes[0].projectionStatus, "degraded"); - assert.equal(writes[0].projectionHealth, "degraded"); - assert.equal(writes[0].backoffMs, 2000); - assert.equal(writes[0].backoffReason, "provider_retry_or_rate_limit"); - assert.equal(writes[0].rootCause, "provider_rate_limited"); - assert.ok(writes[0].nextRetryAt); - } finally { - await new Promise((resolve, reject) => agentRunServer.close((error) => (error ? reject(error) : resolve()))); - } -}); diff --git a/internal/cloud/server-agent-chat-lifecycle.test.ts b/internal/cloud/server-agent-chat-lifecycle.test.ts index a59d1828..55120ee5 100644 --- a/internal/cloud/server-agent-chat-lifecycle.test.ts +++ b/internal/cloud/server-agent-chat-lifecycle.test.ts @@ -13,7 +13,7 @@ import { createCloudRuntimeStore } from "../db/runtime-store.ts"; import { validateCodeAgentChatSchema } from "./code-agent-chat.ts"; import { createCodexStdioSessionManager } from "./codex-stdio-session.ts"; import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; -import { submitAgentRunChatTurn, syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts"; +import { submitAgentRunChatTurn } from "./code-agent-agentrun-adapter.ts"; import { codexStdioChatFixture, codexStdioReadyFixture, diff --git a/internal/cloud/server-agent-chat-session.test.ts b/internal/cloud/server-agent-chat-session.test.ts index 965bcb5c..9bfc2cad 100644 --- a/internal/cloud/server-agent-chat-session.test.ts +++ b/internal/cloud/server-agent-chat-session.test.ts @@ -1,5 +1,5 @@ // SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. -// Responsibility: Cloud API AgentRun session/profile continuity and durable projection resume regression tests. +// Responsibility: Cloud API AgentRun session/profile continuity regression tests. import assert from "node:assert/strict"; import { createServer as createHttpServer } from "node:http"; @@ -13,7 +13,7 @@ import { createCloudRuntimeStore } from "../db/runtime-store.ts"; import { validateCodeAgentChatSchema } from "./code-agent-chat.ts"; import { createCodexStdioSessionManager } from "./codex-stdio-session.ts"; import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; -import { submitAgentRunChatTurn, syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts"; +import { submitAgentRunChatTurn } from "./code-agent-agentrun-adapter.ts"; import { codexStdioChatFixture, codexStdioReadyFixture, @@ -1356,391 +1356,3 @@ test("cloud api turn status reuses request session lookup without loading persis }); } }); - -test("cloud api resumes AgentRun projection from durable state after restart", async () => { - const calls = []; - const durableEvents = []; - const projectionStates = new Map(); - const ownerWrites = []; - const traceId = "trc_projection_resume_restart"; - const runId = "run_projection_resume_restart"; - const commandId = "cmd_projection_resume_restart"; - const sessionId = "ses_projection_resume_restart"; - const conversationId = "cnv_projection_resume_restart"; - const finalText = "OK from resumed AgentRun projection."; - - const agentRunServer = createHttpServer(async (request, response) => { - const url = new URL(request.url || "/", "http://127.0.0.1"); - calls.push({ method: request.method, path: url.pathname, search: url.search, afterSeq: url.searchParams.get("afterSeq") }); - const send = (body, status = 200) => { - response.writeHead(status, { "content-type": "application/json" }); - response.end(`${JSON.stringify(body)}\n`); - }; - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/events`) { - assert.equal(url.searchParams.get("afterSeq"), "21"); - return send({ items: [ - { id: "evt_projection_resume_22", runId, seq: 22, type: "assistant_message", payload: { commandId, text: finalText }, createdAt: "2026-06-18T16:31:11.000Z" }, - { id: "evt_projection_resume_23", runId, seq: 23, type: "terminal_status", payload: { commandId, terminalStatus: "completed" }, createdAt: "2026-06-18T16:31:12.000Z" } - ] }); - } - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/commands/${commandId}/result`) { - return send({ - runId, - commandId, - status: "completed", - runStatus: "completed", - commandState: "completed", - terminalStatus: "completed", - completed: true, - reply: finalText, - lastSeq: 23, - eventCount: 23, - sessionRef: { sessionId, conversationId, threadId: "thread-projection-resume-restart" } - }); - } - return send({ ok: false, message: `unexpected ${request.method} ${url.pathname}` }, 404); - }); - await new Promise((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve)); - const agentRunPort = agentRunServer.address().port; - - const session = testAgentSessionRecord({ - sessionId, - conversationId, - projectId: "prj_hwpod_workbench", - status: "running", - traceId, - updatedAt: "2026-06-18T16:31:04.000Z", - session: { - traceResults: { - [traceId]: { - traceId, - status: "running", - sessionId, - conversationId, - updatedAt: "2026-06-18T16:31:04.000Z", - agentRun: { - adapter: "agentrun-v01", - managerUrl: `http://127.0.0.1:${agentRunPort}`, - runId, - commandId, - traceId, - lastSeq: 0, - status: "running", - commandState: "running", - backendProfile: "codex", - valuesPrinted: false - }, - valuesRedacted: true - } - } - } - }); - - durableEvents.push({ - traceId, - seq: 27, - source: "agentrun", - sourceSeq: 21, - type: "backend", - status: "running", - label: "agentrun:backend:mcpServer/startupStatus/updated", - commandId, - runId, - createdAt: "2026-06-18T16:31:04.200Z", - valuesPrinted: false - }); - projectionStates.set(traceId, { - traceId, - sessionId, - conversationId, - threadId: "thread-projection-resume-restart", - ownerUserId: TEST_AGENT_ACTOR.id, - ownerRole: TEST_AGENT_ACTOR.role, - runId, - commandId, - sourceRunId: runId, - sourceCommandId: commandId, - managerUrl: `http://127.0.0.1:${agentRunPort}`, - backendProfile: "codex", - lastSourceSeq: 21, - lastAgentRunSeq: 21, - lastProjectedSeq: 27, - sourceLatestSeq: 21, - upstreamLatestSeq: 21, - projectionStatus: "projecting", - projectionHealth: "healthy", - resultSyncState: "not_started", - createdAt: "2026-06-18T16:31:04.000Z", - updatedAt: "2026-06-18T16:31:04.000Z", - valuesPrinted: false - }); - - const runtimeStore = { - async queryAgentTraceEvents(params = {}) { - return { events: durableEvents.filter((event) => event.traceId === params.traceId), count: durableEvents.length }; - }, - async writeAgentTraceEvent(params = {}) { - const event = params.event ?? params.traceEvent ?? params; - durableEvents.push(event); - return { written: true, traceEvent: event }; - }, - async getWorkbenchProjectionState(params = {}) { - const projectionState = projectionStates.get(params.traceId) ?? null; - return { projectionState, found: Boolean(projectionState) }; - }, - async queryWorkbenchProjectionStates() { - return { states: [...projectionStates.values()], count: projectionStates.size }; - }, - async writeWorkbenchProjectionState(params = {}) { - const projectionState = params.projectionState ?? params.state ?? params; - projectionStates.set(projectionState.traceId, projectionState); - return { written: true, projectionState }; - } - }; - const accessController = { - required: false, - store: { - async listAgentSessionsForUser() { return [session]; }, - async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; } - }, - async authenticate() { return { ok: true, actor: TEST_AGENT_ACTOR, session: TEST_AUTH_SESSION }; }, - async recordAgentSessionOwner(input = {}) { - ownerWrites.push(input); - return { ...session, status: input.status ?? session.status, session: input.session ?? session.session }; - } - }; - - const server = createCloudApiServer({ - accessController, - runtimeStore, - codeAgentChatResults: new Map(), - env: { - HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", - HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_ENABLED: "1", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INITIAL_DELAY_MS: "1", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INTERVAL_MS: "0", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS: "0", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE: "10" - } - }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - for (let index = 0; index < 80 && projectionStates.get(traceId)?.resultSyncState !== "synced"; index += 1) await delay(10); - assert.ok(calls.some((call) => call.path === `/api/v1/runs/${runId}/events` && call.afterSeq === "21")); - assert.equal(calls.some((call) => call.path === `/api/v1/runs/${runId}/commands/${commandId}/result`), true); - assert.equal(projectionStates.get(traceId)?.lastSourceSeq, 23); - assert.equal(projectionStates.get(traceId)?.resultSyncState, "synced"); - assert.equal(ownerWrites.some((write) => write.status === "completed"), true); - assert.ok(durableEvents.some((event) => event.sourceSeq === 22)); - } finally { - await new Promise((resolve, reject) => { - server.close((error) => (error ? reject(error) : resolve())); - }); - await new Promise((resolve, reject) => { - agentRunServer.close((error) => (error ? reject(error) : resolve())); - }); - } -}); - -test("cloud api projection resume retries evicted AgentRun session with stored prompt", async () => { - const calls = []; - const durableEvents = []; - const projectionStates = new Map(); - const traceId = "trc_projection_resume_evicted_retry"; - const runId = "run_projection_resume_evicted"; - const commandId = "cmd_projection_resume_evicted"; - const retryRunId = "run_projection_resume_evicted_retry"; - const retryCommandId = "cmd_projection_resume_evicted_retry"; - const sessionId = "ses_projection_resume_evicted"; - const conversationId = "cnv_projection_resume_evicted"; - const promptText = "resume projection evicted session with original prompt"; - - const agentRunServer = createHttpServer(async (request, response) => { - const url = new URL(request.url || "/", "http://127.0.0.1"); - const chunks = []; - for await (const chunk of request) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); - const body = chunks.length ? JSON.parse(Buffer.concat(chunks).toString("utf8")) : null; - calls.push({ method: request.method, path: url.pathname, search: url.search, afterSeq: url.searchParams.get("afterSeq"), body }); - const send = (data, status = 200) => { - response.writeHead(status, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: status < 400, data })}\n`); - }; - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/events`) { - return send({ items: [ - { id: "evt_projection_evicted_1", runId, seq: 1, type: "error", payload: { commandId, failureKind: "session-store-evicted", message: "codex app-server thread/resume reported no rollout found for PVC-backed session; session storage was likely evicted" }, createdAt: "2026-06-19T12:01:00.000Z" }, - { id: "evt_projection_evicted_2", runId, seq: 2, type: "terminal_status", payload: { commandId, terminalStatus: "failed", failureKind: "session-store-evicted" }, createdAt: "2026-06-19T12:01:01.000Z" } - ] }); - } - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/commands/${commandId}/result`) { - return send({ - runId, - commandId, - status: "failed", - runStatus: "failed", - commandState: "failed", - terminalStatus: "failed", - failureKind: "session-store-evicted", - failureMessage: "codex app-server thread/resume reported no rollout found for PVC-backed session; session storage was likely evicted", - completed: false, - lastSeq: 2, - eventCount: 2, - sessionRef: { sessionId: "ses_agentrun_projection_resume_evicted", conversationId, threadId: "thread_projection_evicted" } - }); - } - if (request.method === "POST" && url.pathname === "/api/v1/sessions") { - assert.match(body.sessionId, /-reset-/u); - return send({ sessionId: body.sessionId, backendProfile: body.backendProfile }); - } - if (request.method === "POST" && url.pathname === "/api/v1/runs") { - assert.match(body.sessionRef?.sessionId, /-reset-/u); - assert.equal(body.sessionRef?.threadId, undefined); - return send({ id: retryRunId, status: "pending", backendProfile: "dsflash-go", sessionRef: body.sessionRef, resourceBundleRef: body.resourceBundleRef }); - } - if (request.method === "POST" && url.pathname === `/api/v1/runs/${retryRunId}/commands`) { - assert.equal(body.type, "turn"); - assert.equal(body.payload.prompt, promptText); - assert.equal(body.payload.message, promptText); - assert.equal(body.payload.threadId, null); - assert.match(body.payload.sessionId, /-reset-/u); - assert.equal(body.payload.providerProfile, "dsflash-go"); - assert.equal(body.dispatch?.kind, "kubernetes-job"); - assert.equal(body.dispatch?.input?.commandId, undefined); - return send({ - id: retryCommandId, - runId: retryRunId, - state: "queued", - dispatchIntent: { - id: "dispatch_projection_resume_evicted_retry", - state: "pending", - runnerJobId: "rjob_projection_resume_evicted_retry", - attemptCount: 0, - durable: true - } - }); - } - return send({ message: `unexpected ${request.method} ${url.pathname}` }, 500); - }); - await new Promise((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve)); - const agentRunPort = agentRunServer.address().port; - - projectionStates.set(traceId, { - traceId, - sessionId, - conversationId, - threadId: "thread_projection_evicted", - ownerUserId: TEST_AGENT_ACTOR.id, - ownerRole: TEST_AGENT_ACTOR.role, - runId, - commandId, - sourceRunId: runId, - sourceCommandId: commandId, - managerUrl: `http://127.0.0.1:${agentRunPort}`, - backendProfile: "dsflash-go", - providerId: "JD01", - lastSourceSeq: 0, - lastAgentRunSeq: 0, - lastProjectedSeq: 0, - sourceLatestSeq: 0, - upstreamLatestSeq: 0, - projectionStatus: "projecting", - projectionHealth: "healthy", - resultSyncState: "not_started", - retryParams: { - message: promptText, - prompt: promptText, - text: promptText, - sessionId, - conversationId, - threadId: "thread_projection_evicted", - ownerUserId: TEST_AGENT_ACTOR.id, - ownerRole: TEST_AGENT_ACTOR.role, - providerProfile: "dsflash-go", - codeAgentProviderProfile: "dsflash-go", - backendProfile: "dsflash-go", - valuesPrinted: false - }, - createdAt: "2026-06-19T12:00:00.000Z", - updatedAt: "2026-06-19T12:00:00.000Z", - valuesPrinted: false - }); - - const runtimeStore = { - async queryAgentTraceEvents(params = {}) { - return { events: durableEvents.filter((event) => event.traceId === params.traceId), count: durableEvents.length }; - }, - async writeAgentTraceEvent(params = {}) { - const event = params.event ?? params.traceEvent ?? params; - durableEvents.push(event); - return { written: true, traceEvent: event }; - }, - async getWorkbenchProjectionState(params = {}) { - const projectionState = projectionStates.get(params.traceId) ?? null; - return { projectionState, found: Boolean(projectionState) }; - }, - async queryWorkbenchProjectionStates() { - return { states: [...projectionStates.values()], count: projectionStates.size }; - }, - async writeWorkbenchProjectionState(params = {}) { - const projectionState = params.projectionState ?? params.state ?? params; - projectionStates.set(projectionState.traceId, projectionState); - return { written: true, projectionState }; - } - }; - const accessController = { - required: false, - store: { - async listAgentSessionsForUser() { return []; }, - async getAgentSessionByTraceId() { return null; } - }, - async authenticate() { return { ok: true, actor: TEST_AGENT_ACTOR, session: TEST_AUTH_SESSION }; } - }; - - const server = createCloudApiServer({ - accessController, - runtimeStore, - codeAgentChatResults: new Map(), - env: { - HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", - HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1", - HWLAB_CODE_AGENT_AGENTRUN_SOURCE_COMMIT: "a".repeat(40), - HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "JD01", - HWLAB_RUNTIME_NAMESPACE: "hwlab-v03", - HWLAB_RUNTIME_LANE: "v03", - HWLAB_RUNTIME_API_URL: "http://hwlab-cloud-api.hwlab-v03.svc.cluster.local:6667", - HWLAB_RUNTIME_WEB_URL: "http://hwlab-cloud-web.hwlab-v03.svc.cluster.local:8080", - HWLAB_RUNTIME_ENDPOINT_LOCKED: "1", - HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME: "1", - HWLAB_CODE_AGENT_AGENTRUN_BACKEND_PROFILE: "dsflash-go", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_ENABLED: "1", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INITIAL_DELAY_MS: "1", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INTERVAL_MS: "0", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS: "0", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE: "10" - } - }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - for (let index = 0; index < 100 && projectionStates.get(traceId)?.runId !== retryRunId; index += 1) await delay(10); - assert.equal(calls.some((call) => call.path === `/api/v1/runs/${runId}/commands/${commandId}/result`), true); - assert.equal(calls.some((call) => call.path === `/api/v1/runs/${retryRunId}/commands`), true); - const state = projectionStates.get(traceId); - assert.equal(state.runId, retryRunId); - assert.equal(state.commandId, retryCommandId); - assert.equal(state.sourceRunId, retryRunId); - assert.equal(state.sourceCommandId, retryCommandId); - assert.equal(state.projectionStatus, "projecting"); - assert.equal(state.retryParams.message, promptText); - assert.equal(state.retryParams.providerProfile, "dsflash-go"); - assert.equal(calls.some((call) => call.path.endsWith("/runner-jobs")), false); - } finally { - await new Promise((resolve, reject) => { - server.close((error) => (error ? reject(error) : resolve())); - }); - await new Promise((resolve, reject) => { - agentRunServer.close((error) => (error ? reject(error) : resolve())); - }); - } -}); diff --git a/internal/cloud/server-agent-chat.test.ts b/internal/cloud/server-agent-chat.test.ts index e19f3aad..42bf5a97 100644 --- a/internal/cloud/server-agent-chat.test.ts +++ b/internal/cloud/server-agent-chat.test.ts @@ -13,7 +13,7 @@ import { createCloudRuntimeStore } from "../db/runtime-store.ts"; import { validateCodeAgentChatSchema } from "./code-agent-chat.ts"; import { createCodexStdioSessionManager } from "./codex-stdio-session.ts"; import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; -import { submitAgentRunChatTurn, syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts"; +import { submitAgentRunChatTurn } from "./code-agent-agentrun-adapter.ts"; import { codexStdioChatFixture, codexStdioReadyFixture, @@ -991,236 +991,6 @@ test("cloud api keeps admitted user message when billing preflight fails before } }); -test("AgentRun sync converts terminal command result even when run remains claimed (#1555)", async () => { - const calls = []; - const traceId = "trc_issue1555_terminal_command"; - const runId = "run_issue1555_claimed"; - const commandId = "cmd_issue1555_completed"; - const finalText = [ - "已新增并实际运行：", - "", - "- Go：`go test ./...` 保留两个空格", - "- Rust：`cargo test --release`", - "", - "```sh", - "go test ./...", - "cargo test --release", - "```", - "", - "| benchmark | Go | Rust |", - "|---|---:|---:|", - "| fib | 1.2ms | 1.0ms |" - ].join("\n"); - const agentRunServer = createHttpServer(async (request, response) => { - const url = new URL(request.url || "/", "http://127.0.0.1"); - calls.push({ method: request.method, path: url.pathname, search: url.search }); - const send = (data) => { - response.writeHead(200, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: true, data, traceId: "trc_fake_issue1555" })}\n`); - }; - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/events`) { - return send({ items: [ - { id: "evt_issue1555_assistant", runId, seq: 138, type: "assistant_message", payload: { commandId, text: finalText, final: true, replyAuthority: true }, createdAt: "2026-06-18T00:00:00.000Z" }, - { id: "evt_issue1555_terminal", runId, seq: 139, type: "terminal_status", payload: { commandId, terminalStatus: "completed" }, createdAt: "2026-06-18T00:00:00.000Z" } - ] }); - } - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/commands`) { - return send({ items: [ - { id: commandId, runId, state: "completed", type: "turn", seq: 1, idempotencyKey: traceId, payload: { traceId, conversationId: "cnv_issue1555", hwlabSessionId: "ses_issue1555", threadId: "thread_issue1555" } } - ] }); - } - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/commands/${commandId}/result`) { - return send({ - runId, - commandId, - attemptId: "attempt_issue1555", - runnerId: "runner_issue1555", - jobName: "agentrun-v01-runner-issue1555", - namespace: "agentrun-v02", - status: "completed", - runStatus: "claimed", - commandState: "completed", - terminalStatus: "completed", - completed: true, - reply: finalText, - lastSeq: 139, - eventCount: 139, - sessionRef: { sessionId: "ses_issue1555", conversationId: "cnv_issue1555", threadId: "thread_issue1555" } - }); - } - response.writeHead(404, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: false, message: `unexpected ${request.method} ${url.pathname}` })}\n`); - }); - await new Promise((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve)); - const agentRunPort = agentRunServer.address().port; - const traceStore = createCodeAgentTraceStore(); - try { - const currentResult = { - ok: true, - accepted: true, - shortConnection: true, - status: "running", - traceId, - conversationId: "cnv_issue1555", - sessionId: "ses_issue1555", - threadId: "thread_issue1555", - agentRun: { - adapter: "agentrun-v01", - managerUrl: `http://127.0.0.1:${agentRunPort}`, - runId, - commandId, - status: "pending", - runStatus: "claimed", - commandState: "pending", - terminalStatus: null, - lastSeq: 0, - valuesPrinted: false - }, - valuesPrinted: false - }; - const synced = await syncAgentRunChatResult({ - traceId, - currentResult, - traceStore, - options: { - env: { - AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, - HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1" - } - } - }); - assert.equal(synced.resultSynced, true); - assert.equal(synced.result.status, "completed"); - assert.equal(synced.result.agentRun.runStatus, "claimed"); - assert.equal(synced.result.agentRun.commandState, "completed"); - assert.equal(synced.result.agentRun.terminalStatus, "completed"); - assert.equal(synced.result.finalResponse.text, finalText); - assert.equal(synced.result.reply.content, finalText); - assert.ok((synced.result.finalResponse.text.match(/\n/gu) ?? []).length > 8); - assert.ok(synced.result.finalResponse.text.includes("`go test ./...` 保留两个空格")); - const assistantEvent = traceStore.snapshot(traceId).events.find((event) => event.label === "agentrun:assistant:message"); - assert.equal(assistantEvent?.message, finalText); - assert.deepEqual(calls.map((call) => call.path), [ - `/api/v1/runs/${runId}/commands`, - `/api/v1/runs/${runId}/events`, - `/api/v1/runs/${runId}/commands/${commandId}/result` - ]); - assert.ok(traceStore.snapshot(traceId).events.some((event) => event.label === "agentrun:result:completed")); - } finally { - await new Promise((resolve, reject) => agentRunServer.close((error) => (error ? reject(error) : resolve()))); - } -}); - -test("AgentRun sync seals completed final response from authoritative terminal assistant trace event (#1629)", async () => { - const calls = []; - const traceId = "trc_issue1629_terminal_assistant_final"; - const runId = "run_issue1629_trace_final"; - const commandId = "cmd_issue1629_trace_final"; - const finalText = [ - "全部六份数据到手！下面是完整的六语言终极性能对比：", - "", - "| language | runtime | status |", - "|---|---|---|", - "| Lua | LuaJIT | pass |" - ].join("\n"); - const agentRunServer = createHttpServer(async (request, response) => { - const url = new URL(request.url || "/", "http://127.0.0.1"); - calls.push({ method: request.method, path: url.pathname, search: url.search }); - const send = (data) => { - response.writeHead(200, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: true, data, traceId: "trc_fake_issue1629" })}\n`); - }; - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/commands`) { - return send({ items: [ - { id: commandId, runId, state: "completed", type: "turn", seq: 1, idempotencyKey: traceId, payload: { traceId, conversationId: "cnv_issue1629", hwlabSessionId: "ses_issue1629", threadId: "thread_issue1629" } } - ] }); - } - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/events`) { - return send({ items: [ - { id: "evt_issue1629_progress", runId, seq: 21, type: "assistant_message", payload: { commandId, text: "正在补 Lua 基准测试。" }, createdAt: "2026-06-19T15:47:13.000Z" }, - { id: "evt_issue1629_final", runId, seq: 28, type: "assistant_message", payload: { commandId, text: finalText, final: true, replyAuthority: true }, createdAt: "2026-06-19T15:47:28.000Z" }, - { id: "evt_issue1629_terminal", runId, seq: 29, type: "terminal_status", payload: { commandId, terminalStatus: "completed" }, createdAt: "2026-06-19T15:47:29.000Z" } - ] }); - } - if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/commands/${commandId}/result`) { - return send({ - runId, - commandId, - attemptId: "attempt_issue1629", - runnerId: "runner_issue1629", - jobName: "agentrun-v01-runner-issue1629", - namespace: "agentrun-v01", - status: "completed", - runStatus: "completed", - commandState: "completed", - terminalStatus: "completed", - completed: true, - reply: null, - finalResponse: null, - lastSeq: 29, - eventCount: 29, - sessionRef: { sessionId: "ses_issue1629", conversationId: "cnv_issue1629", threadId: "thread_issue1629" } - }); - } - response.writeHead(404, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: false, message: `unexpected ${request.method} ${url.pathname}` })}\n`); - }); - await new Promise((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve)); - const agentRunPort = agentRunServer.address().port; - const traceStore = createCodeAgentTraceStore(); - try { - const currentResult = { - ok: true, - accepted: true, - shortConnection: true, - status: "running", - traceId, - conversationId: "cnv_issue1629", - sessionId: "ses_issue1629", - threadId: "thread_issue1629", - agentRun: { - adapter: "agentrun-v01", - managerUrl: `http://127.0.0.1:${agentRunPort}`, - runId, - commandId, - status: "running", - runStatus: "running", - commandState: "running", - terminalStatus: null, - lastSeq: 0, - valuesPrinted: false - }, - valuesPrinted: false - }; - const synced = await syncAgentRunChatResult({ - traceId, - currentResult, - traceStore, - options: { - env: { - AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, - HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1" - } - } - }); - assert.equal(synced.resultSynced, true); - assert.equal(synced.result.status, "completed"); - assert.equal(synced.result.finalResponse.text, finalText); - assert.equal(synced.result.assistantText ?? synced.result.finalResponse.text, finalText); - assert.equal(synced.result.reply.content, finalText); - assert.equal(synced.result.traceSummary.finalAssistantRow.textChars, finalText.length); - const assistantEvent = synced.result.runnerTrace.events.find((event) => event.label === "agentrun:assistant:message" && event.message === finalText); - assert.equal(assistantEvent?.message, finalText); - assert.deepEqual(calls.map((call) => call.path), [ - `/api/v1/runs/${runId}/commands`, - `/api/v1/runs/${runId}/events`, - `/api/v1/runs/${runId}/commands/${commandId}/result` - ]); - } finally { - await new Promise((resolve, reject) => agentRunServer.close((error) => (error ? reject(error) : resolve()))); - } -}); - test("cloud api AgentRun adapter reports persistent thread resume when a completed run needs a new runner", async () => { const calls = []; const hwlabSessionId = "ses_server-test-thread-resume"; diff --git a/internal/cloud/server-code-agent-admission-http.ts b/internal/cloud/server-code-agent-admission-http.ts new file mode 100644 index 00000000..92addb2d --- /dev/null +++ b/internal/cloud/server-code-agent-admission-http.ts @@ -0,0 +1,1414 @@ +// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-0102 Agent编排 draft-2026-06-17-r0; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. +// Responsibility: Code Agent HTTP session and turn admission. + +import { createHash, randomUUID } from "node:crypto"; + +import { M3_IO_CONTROL_ROUTE, M3_STATUS_ROUTE, describeM3StatusLive, handleM3IoControl } from "./m3-io-control.ts"; +import { + agentRunSessionEvidence, + cancelAgentRunChatTurn, + codeAgentAgentRunAdapterEnabled, + initialAgentRunChatResult, + loadPersistedAgentRunResult, + steerAgentRunChatTurn, + submitAgentRunChatTurn +} from "./code-agent-agentrun-adapter.ts"; +import { createCodeAgentErrorPayload, handleCodeAgentChat } from "./code-agent-chat.ts"; +import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; +import { codeAgentSessionLifecycleSummary } from "./code-agent-session-lifecycle.ts"; +import { messageAuthorityTextValue } from "./code-agent-agentrun-prompt.ts"; +import { writeWorkbenchProjectionSession, writeWorkbenchSessionAdmissionFact } from "./workbench-projection-writer.ts"; +import { createWorkbenchReadModel } from "./workbench-read-model.ts"; +import { codeAgentOtelTraceFields, emitCodeAgentOtelSpan } from "./otel-trace.ts"; +import { + firstHeaderValue, + getHeader, + parsePositiveInteger, + positiveInteger, + readBody, + safeConversationId, + safeOpaqueId, + safeSessionId, + safeTraceId, + sendJson, + truthyFlag +} from "./server-http-utils.ts"; + +import { + normalizeCodeAgentProviderProfile, + firstNonEmptyValue, + recordCodeAgentSessionInputFact, + stableCodeAgentInputId, + codeAgentInputDelivery, + codeAgentInputStatus, + codeAgentResultTimingFields, + firstTimestampIso, + firstLatestTimestampIso, + codeAgentChatShortConnectionRequested, + preflightCodeAgentBilling, + recordCodeAgentBillingUsage, + finalizeCodeAgentBillingUsage, + releaseCodeAgentBillingReservation, + withCodeAgentBillingReservation, + codeAgentBillingEnabled, + codeAgentBillingMetadata, + sanitizeBillingReservation, + sanitizeBillingRecord, + sanitizeBillingRelease, + codeAgentUsedTokens, + normalizeTurnStatus, + recordCodeAgentConversationFact, + recordCodeAgentSessionOwner, + codeAgentOwnerStatusForResult, + textValue, + timestampIsoOrNow, + timestampIsoOrNull, + codeAgentSessionOwnerEvidence, + codeAgentContinuationEvidence, + codeAgentFreshContinuationFailure, + normalizeCodeAgentFailureKind, + codeAgentFailureRequiresFreshContinuation, + codeAgentProviderProfileEvidence, + codeAgentTraceResultEvidence, + codeAgentPromptMetadata, + codeAgentPromptMetadataFromText, + codeAgentPromptMetadataFromParts, + codeAgentPromptFields, + codeAgentPromptId, + clippedPromptText, + sha256Text, + compactCodeAgentObject, + codeAgentBillingReservationEvidence, + codeAgentConversationMessagesEvidence, + boundedConversationMessageText, + conversationText, + conversationTextRaw, + codeAgentMessageTraceSuffix, + codeAgentTurnLifecycleFields, + codeAgentLifecycleMessageId, + safeTurnId, + safeMessageId, + codeAgentConversationAgentMessageStatus, + codeAgentPayloadHasSealedFinalResponse, + codeAgentConversationRunnerTraceEvidence, + codeAgentFinalResponseEvidence, + codeAgentFinalResponseText, + codeAgentTraceSummaryEvidence, + annotateOwner, + canAccessOwnedResult, + isCodeAgentResultCanceled, + codeAgentCancelScopeMismatch, + cancelExpectedValues, + cancelScopeFieldMismatch, + cancelBlockedPayload, + isTraceCommandTerminalStatus, + traceSnapshotWithTerminalEvidence, + traceSnapshotWithAttachedTerminalEvidence, + agentRunTerminalTraceEvidence, + agentRunTerminalFinalResponse, + traceRetentionSummary, + numberOrNull, + createCodeAgentChatResultStore, + compactCodeAgentChatResultPayload, + terminalEvidencePayload, + compactRunnerTraceForResult, + traceEventLabel, + resultTraceEventLimit, + createCodeAgentM3HwlabApiRequestJson, + recordCodeAgentProjectionMetrics, + recordCodeAgentTurnReadMetric, + turnReadDegradedReason, + responseBodyBytes, + statusClassLabel, + sourceLatestSeqFromResult, + sourceLatestAtFromResult, + terminalSourceAtFromResult, + projectionReason, + nowMs, + uniqueStrings +} from "./server-code-agent-http-support.ts"; + +const DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT = 120; +const DEFAULT_CODE_AGENT_TRACE_PAGE_LIMIT = 100; +const MAX_CODE_AGENT_TRACE_PAGE_LIMIT = 100; +const DEFAULT_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS = 2500; +const DEFAULT_CODE_AGENT_PROJECT_ID = "prj_hwpod_workbench"; +const DEFAULT_CODE_AGENT_PROVIDER_PROFILE = "deepseek"; +const CODE_AGENT_TERMINAL_STATUSES = new Set(["completed", "failed", "blocked", "timeout", "cancelled", "canceled"]); +const CODE_AGENT_PROVIDER_PROFILE_ALIASES = Object.freeze({ codex: "codex-api" }); +const CODE_AGENT_PROVIDER_PROFILE_ID_PATTERN = /^[a-z0-9][a-z0-9.-]{0,63}$/u; +const CODE_AGENT_PROVIDER_PROFILE_LABELS = Object.freeze({ + "deepseek": "DeepSeek", + "codex-api": "Codex API", + "gpt.pika": "gpt.pika", + "minimax-m3": "MiniMax-M3 via AgentRun", + "runtime-default": "运行默认" +}); +const DEFAULT_CODE_AGENT_CODEX_API_MODEL = "gpt-5.5"; +const DEFAULT_CODE_AGENT_CODEX_API_BASE_URL = "http://127.0.0.1:49280/responses"; +const DEFAULT_CODE_AGENT_DEEPSEEK_MODEL = "deepseek-chat"; +const DEFAULT_CODE_AGENT_MINIMAX_M3_MODEL = "MiniMax-M3"; + +export async function handleCodeAgentChatHttp(request, response, options) { + const perf = options.backendPerformance; + const body = perf ? await perf.measure("body_read", () => readBody(request, options.bodyLimitBytes)) : await readBody(request, options.bodyLimitBytes); + let params = {}; + + try { + params = body ? JSON.parse(body) : {}; + } catch (error) { + sendJson(response, 400, { + ...createCodeAgentErrorPayload({ + code: "parse_error", + message: "Invalid JSON body", + reason: error.message, + traceId: getHeader(request, "x-trace-id") || "trc_unassigned", + model: options.env?.HWLAB_CODE_AGENT_MODEL || options.env?.OPENAI_MODEL || process.env.HWLAB_CODE_AGENT_MODEL || process.env.OPENAI_MODEL || "unknown", + layer: "api", + retryable: true + }) + }); + return; + } + + if (!params || typeof params !== "object" || Array.isArray(params)) { + sendJson(response, 400, { + ...createCodeAgentErrorPayload({ + code: "invalid_params", + message: "Code Agent chat body must be a JSON object", + traceId: getHeader(request, "x-trace-id") || "trc_unassigned", + model: options.env?.HWLAB_CODE_AGENT_MODEL || options.env?.OPENAI_MODEL || process.env.HWLAB_CODE_AGENT_MODEL || process.env.OPENAI_MODEL || "unknown", + layer: "api", + retryable: true + }) + }); + return; + } + params = withMdtodoExecutionPrompt(params); + + const traceId = safeTraceId(getHeader(request, "x-trace-id") || params.traceId) || `trc_${randomUUID()}`; + const lifecycle = codeAgentTurnLifecycleFields(traceId, params); + const chatParams = { + ...params, + traceId, + ...lifecycle, + ownerUserId: options.actor?.id ?? params.ownerUserId, + ownerRole: options.actor?.role ?? params.ownerRole + }; + const manualSession = perf ? await perf.measure("manual_session", () => prepareManualCodeAgentSession({ params: chatParams, options, traceId, response })) : await prepareManualCodeAgentSession({ params: chatParams, options, traceId, response }); + if (manualSession?.blocked) return; + const nativeSessionChatParams = stripSyntheticConversationContext(chatParams, traceId, options); + + if (codeAgentChatShortConnectionRequested(request, params, options)) { + if (perf) perf.recordPhase({ phase: "short_connection_accept", durationMs: 0, outcome: "ok" }); + let submitted = null; + try { + submitted = await submitCodeAgentChatTurn({ + params: nativeSessionChatParams, + options, + traceId + }); + } catch (error) { + if (isCodeAgentAdmissionUnavailableError(error)) { + sendJson(response, error.statusCode ?? 503, error.payload); + return; + } + throw error; + } + const traceUrl = `/v1/agent/traces/${encodeURIComponent(traceId)}`; + const turnUrl = `/v1/agent/turns/${encodeURIComponent(traceId)}`; + const workbenchEventsUrl = `/v1/workbench/events?sessionId=${encodeURIComponent(safeSessionId(nativeSessionChatParams.sessionId) || "")}&traceId=${encodeURIComponent(traceId)}`; + const statusCode = 202; + const admittedTiming = codeAgentAdmissionTimingFields(submitted?.createdAt ?? submitted?.updatedAt); + void emitCodeAgentOtelSpan("POST /v1/agent/chat", traceId, options.env ?? process.env, { + kind: 2, + attributes: { "http.method": "POST", "http.route": "/v1/agent/chat", "http.status_code": statusCode, sessionId: safeSessionId(nativeSessionChatParams.sessionId) || null, turnId: lifecycle.turnId, ...agentChatOtelAttributes(nativeSessionChatParams) } + }); + sendJson(response, 202, { + accepted: true, + status: "running", + shortConnection: true, + controlSemantics: "submit-and-project-sse", + traceId, + ...codeAgentOtelTraceFields(traceId, options.env ?? process.env), + ...lifecycle, + sessionId: safeSessionId(nativeSessionChatParams.sessionId) || null, + createdAt: submitted?.createdAt ?? admittedTiming.createdAt, + updatedAt: submitted?.updatedAt ?? admittedTiming.updatedAt, + timing: submitted?.timing ?? admittedTiming.timing, + startedAt: submitted?.startedAt ?? admittedTiming.startedAt, + lastEventAt: submitted?.lastEventAt ?? admittedTiming.lastEventAt, + finishedAt: submitted?.finishedAt ?? admittedTiming.finishedAt, + durationMs: submitted?.durationMs ?? admittedTiming.durationMs, + traceUrl, + turnUrl, + workbenchEventsUrl, + cancelUrl: "/v1/agent/chat/cancel", + runnerTrace: (options.traceStore ?? defaultCodeAgentTraceStore).snapshot(traceId) + }); + return; + } + + const admittedBase = codeAgentAdmittedFailureBasePayload({ params: nativeSessionChatParams, options, traceId }); + try { + await recordCodeAgentTurnAdmission({ payload: admittedBase, params: nativeSessionChatParams, options, traceId }); + } catch (error) { + if (isCodeAgentAdmissionUnavailableError(error)) { + sendJson(response, error.statusCode ?? 503, error.payload); + return; + } + throw error; + } + const billingPreflight = perf ? await perf.measure("billing_preflight", () => preflightCodeAgentBilling({ params: nativeSessionChatParams, options, traceId, response: null, sendResponse: false })) : await preflightCodeAgentBilling({ params: nativeSessionChatParams, options, traceId, response: null, sendResponse: false }); + if (billingPreflight?.blocked) { + const failed = await settleAdmittedCodeAgentFailure({ + base: admittedBase, + params: nativeSessionChatParams, + options, + traceId, + results: options.codeAgentChatResults, + failure: billingPreflight, + traceLabel: "billing:preflight:failed" + }); + sendJson(response, billingPreflight.statusCode ?? 503, failed); + return; + } + const nativeSessionChatParamsWithBilling = { ...nativeSessionChatParams, userBillingReservation: billingPreflight?.reservation ?? null }; + const payload = perf ? await perf.measure("code_agent_run", () => runCodeAgentChat(nativeSessionChatParamsWithBilling, options)) : await runCodeAgentChat(nativeSessionChatParamsWithBilling, options); + await (perf ? perf.measure("billing_finalize", () => finalizeCodeAgentBillingUsage({ payload, params: nativeSessionChatParamsWithBilling, options })) : finalizeCodeAgentBillingUsage({ payload, params: nativeSessionChatParamsWithBilling, options })); + await (perf ? perf.measure("session_owner_record", () => recordCodeAgentSessionOwner({ payload, params: nativeSessionChatParamsWithBilling, options, status: codeAgentOwnerStatusForResult(payload) })) : recordCodeAgentSessionOwner({ payload, params: nativeSessionChatParamsWithBilling, options, status: codeAgentOwnerStatusForResult(payload) })); + const responsePayload = annotateOwner(payload, nativeSessionChatParamsWithBilling); + + sendJson(response, responsePayload.status === "failed" && responsePayload.error?.code === "invalid_params" ? 400 : 200, responsePayload); +} + +export async function handleCodeAgentSessionsHttp(request, response, url, options) { + const match = url.pathname.match(/^\/v1\/agent\/sessions(?:\/([^/]+)(?:\/(select))?)?$/u); + const sessionId = safeSessionId(match?.[1] ? decodeURIComponent(match[1]) : ""); + const action = match?.[2] ?? null; + if (!match) { + sendJson(response, 404, manualSessionErrorPayload({ code: "session_route_not_found", message: "Code Agent session route is not implemented." })); + return; + } + if (request.method === "GET" && !sessionId) { + await listManualCodeAgentSessions(request, response, url, options); + return; + } + if (request.method === "POST" && !sessionId) { + await createManualCodeAgentSession(request, response, options); + return; + } + if (request.method === "GET" && sessionId && !action) { + await getManualCodeAgentSession(response, options, sessionId); + return; + } + if (request.method === "DELETE" && sessionId && !action) { + await archiveManualCodeAgentSession(response, options, sessionId); + return; + } + if (request.method === "POST" && sessionId && action === "select") { + await selectManualCodeAgentSession(request, response, options, sessionId); + return; + } + sendJson(response, 405, manualSessionErrorPayload({ code: "session_method_not_allowed", message: "Unsupported Code Agent session method." })); +} + +async function listManualCodeAgentSessions(request, response, url, options) { + const projectId = textValue(url.searchParams.get("projectId")) || DEFAULT_CODE_AGENT_PROJECT_ID; + const limit = parsePositiveInteger(url.searchParams.get("limit"), 20); + const sessions = await options.accessController?.store?.listAgentSessionsForUser?.({ + ownerUserId: options.actor?.id, + actorRole: options.actor?.role, + ownerScoped: true, + projectId, + limit + }) ?? []; + sendJson(response, 200, { + ok: true, + status: "succeeded", + contractVersion: "code-agent-manual-session-v1", + projectId, + sessions: sessions.map(publicManualAgentSession), + count: sessions.length, + valuesRedacted: true, + secretMaterialStored: false + }); +} + +async function createManualCodeAgentSession(request, response, options) { + const body = await readJsonObjectBody(request, options.bodyLimitBytes); + if (!body.ok) { + sendJson(response, 400, manualSessionErrorPayload({ code: body.code, message: body.message, reason: body.reason })); + return; + } + const params = body.value; + const projectId = textValue(params.projectId) || DEFAULT_CODE_AGENT_PROJECT_ID; + const conversationId = safeConversationId(params.conversationId) || `cnv_${randomUUID()}`; + const sessionId = safeSessionId(params.sessionId) || `ses_${randomUUID()}`; + const providerProfile = normalizeCodeAgentProviderProfile(params.providerProfile ?? params.codeAgentProviderProfile); + const now = new Date().toISOString(); + const session = await options.accessController.recordAgentSessionOwner({ + ownerUserId: options.actor?.id, + ownerRole: options.actor?.role, + sessionId, + projectId, + agentId: "hwlab-code-agent", + status: "idle", + conversationId, + threadId: null, + traceId: null, + session: { + source: "manual-session-create", + providerProfile, + sessionStatus: "idle", + createdAt: now, + valuesRedacted: true, + secretMaterialStored: false + } + }); + await writeWorkbenchSessionAdmissionFact({ + runtimeStore: options.runtimeStore, + session, + ownerUserId: options.actor?.id, + ownerRole: options.actor?.role, + projectId, + conversationId, + threadId: null, + status: "idle", + now + }); + sendJson(response, 201, { + ok: true, + status: "created", + contractVersion: "code-agent-manual-session-v1", + session: publicManualAgentSession(session), + nextCommands: manualSessionNextCommands(session), + valuesRedacted: true, + secretMaterialStored: false + }); +} + +async function getManualCodeAgentSession(response, options, sessionId) { + const session = await getVisibleManualAgentSession(options, sessionId); + if (!session) { + sendJson(response, 404, manualSessionErrorPayload({ code: "session_not_found", message: "Code Agent session is not visible to the current actor.", sessionId })); + return; + } + sendJson(response, 200, { + ok: true, + status: "found", + contractVersion: "code-agent-manual-session-v1", + session: publicManualAgentSession(session), + valuesRedacted: true, + secretMaterialStored: false + }); +} + +async function archiveManualCodeAgentSession(response, options, sessionId) { + const session = await getVisibleManualAgentSession(options, sessionId); + if (!session) { + sendJson(response, 404, manualSessionErrorPayload({ code: "session_not_found", message: "Code Agent session is not visible to the current actor.", sessionId })); + return; + } + const result = await options.accessController?.store?.archiveAgentConversation?.({ + ownerUserId: options.actor?.id, + actorRole: options.actor?.role, + ownerScoped: true, + sessionId, + conversationId: session.conversationId, + projectId: session.projectId + }) ?? { count: 0 }; + sendJson(response, 200, { + ok: true, + status: "archived", + contractVersion: "code-agent-manual-session-v1", + archivedCount: result.count ?? 0, + session: publicManualAgentSession({ ...session, status: "archived", endedAt: session.endedAt ?? new Date().toISOString(), updatedAt: new Date().toISOString() }), + valuesRedacted: true, + secretMaterialStored: false + }); +} + +async function selectManualCodeAgentSession(request, response, options, sessionId) { + const body = await readJsonObjectBody(request, options.bodyLimitBytes); + if (!body.ok) { + sendJson(response, 400, manualSessionErrorPayload({ code: body.code, message: body.message, reason: body.reason, sessionId })); + return; + } + const session = await getVisibleManualAgentSession(options, sessionId); + if (!session) { + sendJson(response, 404, manualSessionErrorPayload({ code: "session_not_found", message: "Code Agent session is not visible to the current actor.", sessionId })); + return; + } + if (!manualSessionUsable(session.status)) { + sendJson(response, 409, manualSessionErrorPayload({ code: "session_not_usable", message: `Code Agent session ${sessionId} is ${session.status}; create a new session before continuing.`, sessionId, session })); + return; + } + sendJson(response, 200, { + ok: true, + status: "selected", + contractVersion: "code-agent-manual-session-v1", + session: publicManualAgentSession(session), + nextCommands: manualSessionNextCommands(session), + valuesRedacted: true, + secretMaterialStored: false + }); +} + +async function prepareManualCodeAgentSession({ params, options, traceId, response }) { + if (!options.actor?.id) { + sendJson(response, 401, manualSessionErrorPayload({ + code: "auth_required", + message: "Authentication is required before using an explicit HWLAB Code Agent session.", + traceId, + retryable: true + })); + return { blocked: true }; + } + const sessionId = safeSessionId(params.sessionId); + if (!sessionId) { + sendJson(response, 409, manualSessionErrorPayload({ + code: "session_required", + message: "Code Agent session is explicit in HWLAB v0.2; create/select a session before sending a turn.", + traceId, + retryable: true, + nextCommands: ["hwlab-cli client agent session create --provider-profile PROFILE", "hwlab-cli client agent send --session-id --message TEXT"] + })); + return { blocked: true }; + } + const session = await getVisibleManualAgentSession(options, sessionId); + if (!session) { + sendJson(response, 404, manualSessionErrorPayload({ code: "session_not_found", message: "Code Agent session is not visible to the current actor.", traceId, sessionId })); + return { blocked: true }; + } + const freshContinuation = manualSessionFreshContinuationRequired(session); + if (!manualSessionUsable(session.status) && !freshContinuation) { + sendJson(response, 409, manualSessionErrorPayload({ code: "session_not_usable", message: `Code Agent session ${sessionId} is ${session.status}; create a new session before continuing.`, traceId, sessionId, session })); + return { blocked: true }; + } + const requestedProjectId = textValue(params.projectId); + const sessionProjectId = textValue(session.projectId); + if (requestedProjectId && sessionProjectId && requestedProjectId !== sessionProjectId) { + sendJson(response, 409, manualSessionErrorPayload({ code: "session_project_mismatch", message: `sessionId ${sessionId} belongs to ${sessionProjectId}; requested ${requestedProjectId}.`, traceId, sessionId, session })); + return { blocked: true }; + } + const storedConversationId = safeConversationId(session.conversationId); + const requestedConversationId = safeConversationId(params.conversationId); + if (requestedConversationId && storedConversationId && requestedConversationId !== storedConversationId) { + sendJson(response, 409, manualSessionErrorPayload({ code: "session_conversation_mismatch", message: `sessionId ${sessionId} is bound to ${storedConversationId}; requested ${requestedConversationId}.`, traceId, sessionId, session })); + return { blocked: true }; + } + const conversationId = requestedConversationId || storedConversationId; + if (!conversationId) { + sendJson(response, 409, manualSessionErrorPayload({ code: "session_conversation_required", message: "Code Agent session has no bound conversationId; create a new session before continuing.", traceId, sessionId, session })); + return { blocked: true }; + } + params.conversationId = conversationId; + params.sessionId = sessionId; + params.projectId = requestedProjectId || sessionProjectId || DEFAULT_CODE_AGENT_PROJECT_ID; + applyManualSessionProviderProfile(params, session); + if (freshContinuation) { + params.threadId = undefined; + params.forceFreshAgentRunSession = true; + params.forceFreshAgentRunSessionReason = manualSessionFreshContinuationReason(session); + } else { + params.threadId = safeOpaqueId(params.threadId) || manualSessionContinuationThreadId(session) || undefined; + } + return { session }; +} + +function applyManualSessionProviderProfile(params = {}, session = null) { + const requested = firstNonEmptyValue(params.providerProfile, params.codeAgentProviderProfile); + if (requested) { + const normalized = normalizeCodeAgentProviderProfile(requested); + params.providerProfile = normalized; + params.codeAgentProviderProfile = normalized; + params.providerProfileSource = "request"; + return normalized; + } + const stored = manualSessionStoredProviderProfile(session); + if (!stored) return null; + params.providerProfile = stored; + params.codeAgentProviderProfile = stored; + params.providerProfileSource = "session"; + return stored; +} + +function manualSessionStoredProviderProfile(session = null) { + const evidence = session?.session && typeof session.session === "object" ? session.session : {}; + const traceProfile = latestCompletedTraceResultProviderProfile(evidence.traceResults); + const raw = firstNonEmptyValue( + traceProfile, + evidence.agentRun?.backendProfile, + evidence.providerProfile, + evidence.codeAgentProviderProfile, + evidence.backendProfile + ); + if (!raw) return null; + const normalized = normalizeCodeAgentProviderProfile(raw); + return normalized === "runtime-default" ? null : normalized; +} + +function latestCompletedTraceResultProviderProfile(traceResults = null) { + if (!traceResults || typeof traceResults !== "object") return null; + const entries = Object.values(traceResults).filter((value) => value && typeof value === "object"); + for (const result of entries.reverse()) { + const agentRun = result.agentRun && typeof result.agentRun === "object" ? result.agentRun : null; + if (!agentRun) continue; + const status = normalizeTurnStatus(result.status ?? agentRun.terminalStatus ?? agentRun.commandState ?? agentRun.status); + if (status && status !== "completed") continue; + const profile = firstNonEmptyValue(agentRun.backendProfile, result.providerProfile, result.codeAgentProviderProfile); + if (profile) return profile; + } + return null; +} + +function manualSessionFreshContinuationRequired(session = null) { + const status = String(session?.status ?? "").trim().toLowerCase().replace(/_/gu, "-"); + if (status === "thread-resume-failed") return true; + const continuation = session?.session?.continuation && typeof session.session.continuation === "object" ? session.session.continuation : null; + return continuation?.requiresFreshSession === true; +} + +function manualSessionFreshContinuationReason(session = null) { + const continuation = session?.session?.continuation && typeof session.session.continuation === "object" ? session.session.continuation : null; + return firstNonEmptyValue(continuation?.reasonCode, continuation?.reason, session?.status, "stale-continuation") ?? "stale-continuation"; +} + +function manualSessionContinuationThreadId(session = null) { + if (manualSessionFreshContinuationRequired(session)) return null; + const evidence = session?.session && typeof session.session === "object" ? session.session : {}; + return safeOpaqueId(evidence.threadId) || safeOpaqueId(evidence.agentRun?.threadId) || safeOpaqueId(session?.threadId) || null; +} + +async function getVisibleManualAgentSession(options, sessionId) { + if (!safeSessionId(sessionId) || !options.actor?.id || !options.accessController?.getAgentSession) return null; + const session = await options.accessController.getAgentSession(sessionId); + if (!session) return null; + if (session.ownerUserId === options.actor.id) return session; + return null; +} + +function publicManualAgentSession(session) { + if (!session || typeof session !== "object") return null; + return { + sessionId: session.id, + agentId: session.agentId ?? "hwlab-code-agent", + status: session.status ?? null, + usable: manualSessionUsable(session.status), + threadId: safeOpaqueId(session.threadId) || null, + lastTraceId: safeTraceId(session.lastTraceId) || null, + providerProfile: textValue(session.session?.providerProfile) || null, + createdAt: session.startedAt ?? null, + updatedAt: session.updatedAt ?? null, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +function manualSessionUsable(status) { + const value = String(status ?? "").trim().toLowerCase().replace(/_/gu, "-"); + return !["blocked", "expired", "stale"].includes(value); +} + +function manualSessionErrorPayload({ code, message, reason = null, traceId = null, sessionId = null, session = null, retryable = true, nextCommands = undefined } = {}) { + return { + ok: false, + accepted: false, + status: "blocked", + traceId: safeTraceId(traceId) || null, + sessionId: safeSessionId(sessionId) || safeSessionId(session?.id) || null, + session: session ? publicManualAgentSession(session) : null, + error: { + code, + layer: "api", + category: "session_blocked", + retryable, + message, + reason, + userMessage: message, + route: "/v1/agent/chat" + }, + blocker: { + code, + layer: "api", + retryable, + summary: message + }, + nextCommands, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +function manualSessionNextCommands(session) { + const sessionId = session?.id ?? ""; + return [ + `hwlab-cli client agent send --session-id ${sessionId} --message TEXT`, + `hwlab-cli client agent session status ${sessionId}` + ]; +} + +async function readJsonObjectBody(request, bodyLimitBytes) { + const body = await readBody(request, bodyLimitBytes); + try { + const value = body ? JSON.parse(body) : {}; + if (!value || typeof value !== "object" || Array.isArray(value)) return { ok: false, code: "invalid_params", message: "body must be a JSON object" }; + return { ok: true, value }; + } catch (error) { + return { ok: false, code: "parse_error", message: "Invalid JSON body", reason: error.message }; + } +} + +async function runCodeAgentChat(params, options) { + return handleCodeAgentChat(params, await codeAgentChatExecutionOptions(options, params)); +} + +function stripSyntheticConversationContext(params = {}, traceId, options = {}) { + if (Array.isArray(params.conversationContext?.messages) || Array.isArray(params.messages)) { + (options.traceStore ?? defaultCodeAgentTraceStore).append(traceId, { + type: "synthetic-context", + status: "ignored", + label: "synthetic-context:ignored", + message: "Synthetic conversation context was stripped; Code Agent continuation uses only Codex stdio native thread/resume and turn/start.", + valuesPrinted: false + }); + } + const { conversationContext, conversationContextTruncated, contextSource, messages, ...nativeParams } = params; + return nativeParams; +} + +async function codeAgentChatExecutionOptions(options = {}, params = {}) { + const env = codeAgentProviderProfileEnv(options.env ?? process.env, params); + Object.assign(env, await codeAgentAuthEnv(options, env, params)); + return { + callProvider: options.callCodeAgentProvider, + timeoutMs: options.codeAgentTimeoutMs, + hardTimeoutMs: options.codeAgentHardTimeoutMs, + env, + workspace: options.workspace, + skillsDirs: options.skillsDirs, + skillsDirsExact: options.skillsDirsExact, + skillsDiscoveryDelayMs: options.skillsDiscoveryDelayMs, + sessionRegistry: options.sessionRegistry, + m3IoSkillRequestJson: options.m3IoSkillRequestJson ?? createCodeAgentM3HwlabApiRequestJson(options), + codexStdioManager: options.codexStdioManager, + traceStore: options.traceStore, + gatewayRegistry: options.gatewayRegistry + }; +} + +async function codeAgentAuthEnv(options = {}, env = process.env, params = {}) { + const ownerUserId = textValue(params.ownerUserId ?? options.actor?.id ?? ""); + const store = options.accessController?.store; + const key = ownerUserId && store?.findActiveDefaultApiKeyForUser + ? await store.findActiveDefaultApiKeyForUser(ownerUserId) + : null; + const envRuntimeLane = runtimeLaneForEnv(env); + const runtimeNamespace = runtimeNamespaceForEnv(env); + const namespaceRuntimeLane = runtimeLaneForNamespace(runtimeNamespace); + const runtimeLane = resolveRuntimeLaneAuthority({ envRuntimeLane, runtimeNamespace, namespaceRuntimeLane }); + const inClusterApiUrl = internalCodeAgentApiUrl(env, runtimeNamespace); + const inClusterWebUrl = internalCodeAgentWebUrl(env, runtimeNamespace); + const apiUrl = firstNonEmptyValue( + codeAgentAgentRunAdapterEnabled(env) ? inClusterApiUrl : null, + env.HWLAB_RUNTIME_API_URL, + sameRuntimeEndpoint(env.HWLAB_CLOUD_API_URL, runtimeNamespace, runtimeLane), + env.HWLAB_PUBLIC_ENDPOINT, + localCloudApiUrl(env) + ); + return { + HWLAB_CLOUD_API_URL: apiUrl, + HWLAB_RUNTIME_API_URL: apiUrl, + HWLAB_RUNTIME_WEB_URL: firstNonEmptyValue(codeAgentAgentRunAdapterEnabled(env) ? inClusterWebUrl : null, env.HWLAB_RUNTIME_WEB_URL, apiUrl), + HWLAB_RUNTIME_NAMESPACE: runtimeNamespace, + HWLAB_RUNTIME_LANE: runtimeLane, + HWLAB_RUNTIME_ENDPOINT_SOURCE: codeAgentAgentRunAdapterEnabled(env) ? "runtime-namespace" : "runtime-env", + HWLAB_RUNTIME_ENDPOINT_LOCKED: "1", + HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME: "1", + ...(options.actorApiKeySecret ? { HWLAB_API_KEY: options.actorApiKeySecret } : key?.displaySecret ? { HWLAB_API_KEY: key.displaySecret } : {}) + }; +} + +function internalCodeAgentApiUrl(env = process.env, namespace = runtimeNamespaceForEnv(env)) { + const serviceName = firstNonEmptyValue(env.HWLAB_CODE_AGENT_INTERNAL_API_SERVICE_NAME, env.HWLAB_CLOUD_API_SERVICE_NAME, "hwlab-cloud-api"); + const port = parsePositiveInteger(firstNonEmptyValue(env.HWLAB_CODE_AGENT_INTERNAL_API_PORT, env.HWLAB_CLOUD_API_PORT), 6667); + return `http://${serviceName}.${namespace}.svc.cluster.local:${port}`; +} + +function internalCodeAgentWebUrl(env = process.env, namespace = runtimeNamespaceForEnv(env)) { + const serviceName = firstNonEmptyValue(env.HWLAB_CODE_AGENT_INTERNAL_WEB_SERVICE_NAME, env.HWLAB_CLOUD_WEB_SERVICE_NAME, "hwlab-cloud-web"); + const port = parsePositiveInteger(firstNonEmptyValue(env.HWLAB_CODE_AGENT_INTERNAL_WEB_PORT, env.HWLAB_CLOUD_WEB_PORT), 8080); + return `http://${serviceName}.${namespace}.svc.cluster.local:${port}`; +} + +function runtimeNamespaceForEnv(env = process.env) { + return firstNonEmptyValue( + env.HWLAB_RUNTIME_NAMESPACE, + env.POD_NAMESPACE, + env.HWLAB_NAMESPACE, + defaultRuntimeNamespace(env) + ); +} + +function runtimeLaneForEnv(env = process.env) { + const raw = firstNonEmptyValue(env.HWLAB_RUNTIME_LANE, env.HWLAB_GITOPS_LANE, env.HWLAB_GITOPS_PROFILE, env.HWLAB_ENVIRONMENT, env.HWLAB_BOOT_REF); + if (!raw) return null; + const lane = normalizeRuntimeLane(raw); + if (!lane) { + throw runtimeAuthorityError("unsupported_runtime_lane", `unsupported HWLAB runtime lane: ${String(raw).trim()}`); + } + return lane; +} + +function runtimeLaneForNamespace(namespace) { + const value = String(namespace ?? "").trim().toLowerCase(); + if (value === "hwlab-prod") return "prod"; + if (value === "hwlab-dev") return "dev"; + const versionMatch = value.match(/^hwlab-v(\d+)$/u); + if (versionMatch) return versionRuntimeLane(versionMatch[1]); + return null; +} + +function sameRuntimeEndpoint(value, namespace, lane) { + const text = firstNonEmptyValue(value); + if (!text) return null; + const endpointLane = runtimeLaneForEndpoint(text); + const endpointNamespace = runtimeNamespaceForEndpoint(text); + if (namespace && endpointNamespace && endpointNamespace !== namespace) return null; + if (lane && endpointLane && endpointLane !== lane) return null; + return text; +} + +function runtimeNamespaceForEndpoint(value) { + const match = String(value ?? "").match(/\.((?:hwlab-[a-z0-9-]+))\.svc\.cluster\.local(?::|\/|$)/iu); + return match?.[1] ?? null; +} + +function runtimeLaneForEndpoint(value) { + const text = String(value ?? "").toLowerCase(); + const namespaceLane = runtimeLaneForNamespace(runtimeNamespaceForEndpoint(text)); + if (namespaceLane) return namespaceLane; + if (/hwlab-prod|:18666(?:\/|$)|:18667(?:\/|$)/u.test(text)) return "prod"; + if (/hwlab-dev|:17666(?:\/|$)|:17667(?:\/|$)|:16666(?:\/|$)|:16667(?:\/|$)/u.test(text)) return "dev"; + return null; +} + +function normalizeRuntimeLane(value) { + const profile = String(value ?? "").trim().toLowerCase(); + if (!profile) return null; + if (profile === "prod" || profile === "production") return "prod"; + if (profile === "dev" || profile === "development" || profile === "local") return "dev"; + const dotted = profile.match(/^v?0+\.(\d+)$/u); + if (dotted) return versionRuntimeLane(dotted[1]); + const compact = profile.match(/^v?0*(\d+)$/u); + if (compact) return versionRuntimeLane(compact[1]); + return null; +} + +function versionRuntimeLane(value) { + const version = Number.parseInt(value, 10); + if (!Number.isFinite(version) || version <= 0) return null; + return `v${String(version).padStart(2, "0")}`; +} + +function resolveRuntimeLaneAuthority({ envRuntimeLane, runtimeNamespace, namespaceRuntimeLane }) { + if (!namespaceRuntimeLane) { + throw runtimeAuthorityError("unsupported_runtime_namespace", `unsupported HWLAB runtime namespace: ${runtimeNamespace}`); + } + if (envRuntimeLane && envRuntimeLane !== namespaceRuntimeLane) { + throw runtimeAuthorityError( + "runtime_authority_mismatch", + `HWLAB runtime lane ${envRuntimeLane} does not match namespace ${runtimeNamespace} (${namespaceRuntimeLane})` + ); + } + return namespaceRuntimeLane; +} + +function runtimeAuthorityError(code, message) { + return Object.assign(new Error(message), { + code, + statusCode: 500, + valuesPrinted: false + }); +} + +function defaultRuntimeNamespace(env = process.env) { + const profile = runtimeLaneForEnv(env) ?? "dev"; + if (profile === "prod") return "hwlab-prod"; + if (profile === "dev") return "hwlab-dev"; + return `hwlab-${profile}`; +} + +function localCloudApiUrl(env = process.env) { + const port = parsePositiveInteger(env.HWLAB_CLOUD_API_PORT, 6667); + return `http://127.0.0.1:${port}`; +} + +function codeAgentProviderProfileEnv(env = process.env, params = {}) { + const profile = normalizeCodeAgentProviderProfile(params.providerProfile ?? params.codeAgentProviderProfile ?? params.provider ?? env.HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE); + if (profile === "runtime-default") return env; + const overlay = { ...env }; + overlay.HWLAB_CODE_AGENT_SELECTED_PROVIDER_PROFILE = profile; + overlay.HWLAB_CODE_AGENT_PROVIDER_PROFILE_LABEL = CODE_AGENT_PROVIDER_PROFILE_LABELS[profile] ?? profile; + overlay.HWLAB_CODE_AGENT_PROVIDER = "codex-stdio"; + if (profile === "codex-api") { + overlay.HWLAB_CODE_AGENT_MODEL = firstNonEmptyValue(env.HWLAB_CODE_AGENT_CODEX_API_MODEL, env.HWLAB_CODE_AGENT_LEGACY_CODEX_MODEL, DEFAULT_CODE_AGENT_CODEX_API_MODEL); + overlay.HWLAB_CODE_AGENT_OPENAI_BASE_URL = firstNonEmptyValue(env.HWLAB_CODE_AGENT_CODEX_API_BASE_URL, env.HWLAB_CODE_AGENT_LEGACY_OPENAI_BASE_URL, DEFAULT_CODE_AGENT_CODEX_API_BASE_URL); + } else if (profile === "minimax-m3") { + overlay.HWLAB_CODE_AGENT_MODEL = firstNonEmptyValue(env.HWLAB_CODE_AGENT_MINIMAX_M3_MODEL, DEFAULT_CODE_AGENT_MINIMAX_M3_MODEL); + } else { + const dynamicProfile = profile !== "deepseek"; + overlay.HWLAB_CODE_AGENT_MODEL = dynamicProfile + ? firstNonEmptyValue(env[`HWLAB_CODE_AGENT_${providerProfileEnvKey(profile)}_MODEL`], profile) + : firstNonEmptyValue(env.HWLAB_CODE_AGENT_DEEPSEEK_MODEL, DEFAULT_CODE_AGENT_DEEPSEEK_MODEL); + if (!dynamicProfile || !codeAgentAgentRunAdapterEnabled(env)) { + overlay.HWLAB_CODE_AGENT_OPENAI_BASE_URL = firstNonEmptyValue(env.HWLAB_CODE_AGENT_DEEPSEEK_BASE_URL, defaultDeepSeekBaseUrlForEnv(env)); + } + } + overlay.NO_PROXY = mergeNoProxyEntries(env.NO_PROXY, codeAgentClusterNoProxyEntries(env)); + overlay.no_proxy = mergeNoProxyEntries(env.no_proxy ?? env.NO_PROXY, codeAgentClusterNoProxyEntries(env)); + return overlay; +} + +function providerProfileEnvKey(profile) { + return String(profile ?? "") + .trim() + .toUpperCase() + .replace(/[^A-Z0-9]+/gu, "_") + .replace(/^_+|_+$/gu, "") || "PROFILE"; +} + +function defaultDeepSeekBaseUrlForEnv(env = process.env) { + const namespace = env.HWLAB_GITOPS_PROFILE === "prod" || env.HWLAB_ENVIRONMENT === "prod" ? "hwlab-prod" : "hwlab-dev"; + return `http://hwlab-deepseek-proxy.${namespace}.svc.cluster.local:4000/v1/responses`; +} + +function codeAgentClusterNoProxyEntries(env = process.env) { + const namespace = env.HWLAB_GITOPS_PROFILE === "prod" || env.HWLAB_ENVIRONMENT === "prod" ? "hwlab-prod" : "hwlab-dev"; + return [ + `${namespace}.svc.cluster.local`, + ".svc", + ".cluster.local", + "127.0.0.1", + "localhost", + "::1", + "10.0.0.0/8", + "10.42.0.0/16", + "10.43.0.0/16", + "hyueapi.com", + ".hyueapi.com", + "hyui.com", + ".hyui.com", + "api.minimaxi.com", + ".minimaxi.com" + ]; +} + +function mergeNoProxyEntries(current, additions = []) { + return uniqueStrings([...String(current ?? "").split(","), ...additions]).join(","); +} + +function safeOtelConfigText(value) { + const text = firstNonEmptyValue(value); + return text === null ? null : text.slice(0, 120); +} + +function safeOtelUrlHost(value) { + const text = firstNonEmptyValue(value); + if (text === null) return null; + try { + return new URL(text).host.slice(0, 120); + } catch { + return null; + } +} + +function withMdtodoExecutionPrompt(params = {}) { + const execution = mdtodoExecutionContextFromParams(params); + if (!execution?.hwpodId || !execution?.hwpodWorkspaceArgs) return params; + const message = firstNonEmptyValue(params.message, params.prompt, params.text); + if (!message || /hwpodWorkspaceArgs\s*:/u.test(message)) return params; + const taskRef = firstNonEmptyValue(params.taskRef, execution.taskRef) ?? "-"; + const mdtodoRootRef = execution.mdtodoRootRef ?? "docs/MDTODO"; + const prefix = [ + "# MDTODO HWPOD 执行上下文", + `TaskRef: ${taskRef}`, + `sourceId: ${execution.sourceId ?? "-"}`, + `hwpodId: ${execution.hwpodId}`, + `nodeId: ${execution.nodeId ?? "-"}`, + `mdtodoRootRef: ${mdtodoRootRef}`, + `hwpodWorkspaceArgs: ${execution.hwpodWorkspaceArgs}`, + "所有 hwpod/hwpod-ctl 命令都必须携带上述 hwpodWorkspaceArgs。读取 MDTODO、报告和工程文件必须通过 HWPOD workspace/node-ops 入口，不得猜测或访问普通容器本地路径，也不得创建本地 .hwlab/hwpod-spec.yaml fallback。" + ].join("\n"); + const nextMessage = `${prefix}\n\n${message}`; + return { + ...params, + message: nextMessage, + prompt: nextMessage, + launchContext: { + ...(params.launchContext && typeof params.launchContext === "object" ? params.launchContext : {}), + hwpodId: execution.hwpodId, + nodeId: execution.nodeId, + mdtodoRootRef: execution.mdtodoRootRef, + hwpodWorkspaceArgs: execution.hwpodWorkspaceArgs, + contextFingerprint: execution.contextFingerprint, + executionContext: execution, + valuesRedacted: true + } + }; +} + +function mdtodoExecutionContextFromParams(params = {}) { + const launchContext = params.launchContext && typeof params.launchContext === "object" && !Array.isArray(params.launchContext) ? params.launchContext : null; + const execution = launchContext?.executionContext && typeof launchContext.executionContext === "object" && !Array.isArray(launchContext.executionContext) ? launchContext.executionContext : launchContext; + if (!execution || typeof execution !== "object") return null; + return { + sourceId: safeOtelConfigText(execution.sourceId ?? launchContext?.sourceId), + sourceKind: safeOtelConfigText(execution.sourceKind ?? launchContext?.sourceKind), + projectId: safeOtelConfigText(execution.projectId ?? launchContext?.projectId ?? params.projectId), + taskRef: safeOtelText(execution.taskRef ?? launchContext?.taskRef ?? params.taskRef), + fileRef: safeOtelConfigText(execution.fileRef ?? launchContext?.fileRef), + relativePath: safeOtelText(execution.relativePath ?? launchContext?.relativePath), + taskId: safeOtelConfigText(execution.taskId ?? launchContext?.taskId), + hwpodId: safeOtelText(execution.hwpodId ?? launchContext?.hwpodId), + nodeId: safeOtelText(execution.nodeId ?? launchContext?.nodeId), + mdtodoRootRef: safeOtelText(execution.mdtodoRootRef ?? launchContext?.mdtodoRootRef), + workspaceRootHash: safeOtelText(execution.workspaceRootHash ?? launchContext?.workspaceRootHash), + hwpodWorkspaceArgs: safeOtelText(execution.hwpodWorkspaceArgs ?? launchContext?.hwpodWorkspaceArgs, 500), + contextFingerprint: safeOtelText(execution.contextFingerprint ?? launchContext?.contextFingerprint), + valuesRedacted: true + }; +} + +function agentChatOtelAttributes(params = {}) { + const execution = mdtodoExecutionContextFromParams(params); + return { + "agent.chat.session_id": safeSessionId(params.sessionId) || null, + "agent.chat.provider_profile": safeOtelConfigText(params.providerProfile ?? params.codeAgentProviderProfile), + "agent.chat.provider_profile_source": safeOtelConfigText(params.providerProfileSource), + "agent.chat.task_ref": safeOtelText(params.taskRef ?? execution?.taskRef), + "agent.chat.source_id": safeOtelConfigText(execution?.sourceId ?? params.sourceId), + "agent.chat.hwpod_id": safeOtelText(execution?.hwpodId), + "agent.chat.workspace_root_hash": safeOtelText(execution?.workspaceRootHash), + "agent.chat.context_fingerprint": safeOtelText(execution?.contextFingerprint) + }; +} + +function safeOtelText(value, max = 360) { + const text = firstNonEmptyValue(value); + return text === null ? null : text.slice(0, max); +} + +async function submitCodeAgentChatTurn({ params, options, traceId }) { + const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; + const results = options.codeAgentChatResults ?? createCodeAgentChatResultStore(); + const timestamp = new Date().toISOString(); + const runtimeEnv = options.env ?? process.env; + const adapterEnabled = codeAgentAgentRunAdapterEnabled(runtimeEnv); + const acceptedPayload = { + accepted: true, + status: "running", + traceId, + ...codeAgentOtelTraceFields(traceId, process.env), + ...codeAgentTurnLifecycleFields(traceId, params), + conversationId: safeConversationId(params.conversationId) || null, + sessionId: safeSessionId(params.sessionId) || null, + threadId: safeOpaqueId(params.threadId) || null, + projectId: params.projectId ?? null, + ...codeAgentAdmissionTimingFields(timestamp) + }; + await recordCodeAgentTurnAdmission({ payload: acceptedPayload, params, options, traceId }); + results.set(traceId, annotateOwner(acceptedPayload, params)); + void emitCodeAgentOtelSpan("provider_decision", traceId, runtimeEnv, { + attributes: { + ...agentChatOtelAttributes(params), + adapterEnabled, + adapter: safeOtelConfigText(runtimeEnv.HWLAB_CODE_AGENT_ADAPTER ?? runtimeEnv.HWLAB_CODE_AGENT_PROVIDER), + provider: safeOtelConfigText(runtimeEnv.HWLAB_CODE_AGENT_PROVIDER), + defaultProviderProfile: safeOtelConfigText(runtimeEnv.HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE), + agentRunManagerHost: safeOtelUrlHost(runtimeEnv.AGENTRUN_MGR_URL ?? runtimeEnv.HWLAB_CODE_AGENT_AGENTRUN_MGR_URL), + agentRunRunnerNamespace: safeOtelConfigText(runtimeEnv.HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE ?? runtimeEnv.AGENTRUN_RUNTIME_NAMESPACE), + agentRunSourceCommitPresent: Boolean(firstNonEmptyValue(runtimeEnv.HWLAB_CODE_AGENT_AGENTRUN_SOURCE_COMMIT)) + } + }); + if (adapterEnabled) { + const run = async () => { + let initial = null; + let executionOptions = options; + try { + initial = initialAgentRunChatResult({ params, options, traceId }); + results.set(traceId, annotateOwner(initial, params)); + const billingStartedAt = Date.now(); + const billingPreflight = await preflightCodeAgentBilling({ params, options, traceId, sendResponse: false }); + void emitCodeAgentOtelSpan("billing_preflight", traceId, options.env ?? process.env, { + startTimeMs: billingStartedAt, + attributes: { sessionId: safeSessionId(params.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId, blocked: Boolean(billingPreflight?.blocked) }, + status: billingPreflight?.blocked ? "error" : "ok", + error: billingPreflight?.blocked ? billingPreflight?.payload?.error?.message ?? billingPreflight?.blocker?.summary ?? "billing preflight blocked" : null + }); + if (billingPreflight?.blocked) { + await settleAdmittedCodeAgentFailure({ + base: initial, + params, + options, + traceId, + results, + failure: billingPreflight, + traceLabel: "billing:preflight:failed" + }); + return; + } + const dispatchParams = { ...params, userBillingReservation: billingPreflight?.reservation ?? null }; + initial = withCodeAgentBillingReservation(initial, dispatchParams); + results.set(traceId, annotateOwner(initial, dispatchParams)); + executionOptions = { ...options, ...(await codeAgentChatExecutionOptions(options, dispatchParams)) }; + const dispatchStartedAt = Date.now(); + const payload = await submitAgentRunChatTurn({ params: dispatchParams, options: executionOptions, traceId, traceStore, results }); + void emitCodeAgentOtelSpan("agentrun_dispatch", traceId, executionOptions.env ?? process.env, { + startTimeMs: dispatchStartedAt, + attributes: { sessionId: safeSessionId(dispatchParams.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, dispatchParams).turnId, runId: payload?.agentRun?.runId ?? null, commandId: payload?.agentRun?.commandId ?? null, providerProfile: dispatchParams.providerProfile ?? null } + }); + if (isCodeAgentResultCanceled(results.get(traceId))) return; + const owned = annotateOwner(withCodeAgentBillingReservation(payload, dispatchParams), dispatchParams); + await recordCodeAgentSessionOwner({ payload: owned, params: dispatchParams, options: executionOptions, status: "running" }); + results.set(traceId, owned); + } catch (error) { + if (isCodeAgentResultCanceled(results.get(traceId))) return; + const dispatchErrorAttributes = codeAgentDispatchErrorOtelAttributes(error); + void emitCodeAgentOtelSpan("agentrun_dispatch", traceId, executionOptions.env ?? process.env, { + status: "error", + error, + attributes: { sessionId: safeSessionId(params.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId, ...dispatchErrorAttributes } + }); + await settleAdmittedCodeAgentFailure({ + base: initial ?? codeAgentAdmittedFailureBasePayload({ params, options, traceId }), + params, + options: executionOptions, + traceId, + results, + failure: codeAgentDispatchFailure(error), + traceLabel: "agentrun:dispatch:failed" + }); + } + }; + setImmediate(() => { run(); }); + return annotateOwner(acceptedPayload, params); + } + traceStore.ensure(traceId, { + runnerKind: "codex-app-server-stdio-runner", + workspace: options.workspace ?? options.env?.HWLAB_CODE_AGENT_CODEX_WORKSPACE ?? options.env?.HWLAB_CODE_AGENT_WORKSPACE ?? null, + sandbox: options.env?.HWLAB_CODE_AGENT_CODEX_SANDBOX ?? null, + sessionMode: "codex-app-server-stdio-long-lived", + implementationType: "repo-owned-codex-app-server-stdio-session" + }); + traceStore.append(traceId, { + type: "request", + status: "accepted", + label: "request:accepted-short-connection", + message: "Code Agent request accepted; committed progress is delivered through the Workbench projection event stream.", + waitingFor: "codex-stdio" + }); + + const run = async () => { + try { + const billingStartedAt = Date.now(); + const billingPreflight = await preflightCodeAgentBilling({ params, options, traceId, sendResponse: false }); + void emitCodeAgentOtelSpan("billing_preflight", traceId, options.env ?? process.env, { + startTimeMs: billingStartedAt, + attributes: { sessionId: safeSessionId(params.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId, blocked: Boolean(billingPreflight?.blocked) }, + status: billingPreflight?.blocked ? "error" : "ok", + error: billingPreflight?.blocked ? billingPreflight?.payload?.error?.message ?? billingPreflight?.blocker?.summary ?? "billing preflight blocked" : null + }); + if (billingPreflight?.blocked) { + await settleAdmittedCodeAgentFailure({ + base: codeAgentAdmittedFailureBasePayload({ params, options, traceId }), + params, + options, + traceId, + results, + failure: billingPreflight, + traceLabel: "billing:preflight:failed" + }); + return; + } + const dispatchParams = { ...params, userBillingReservation: billingPreflight?.reservation ?? null }; + const payload = await runCodeAgentChat(dispatchParams, options); + if (isCodeAgentResultCanceled(results.get(traceId))) return; + await finalizeCodeAgentBillingUsage({ payload, params: dispatchParams, options }); + await recordCodeAgentSessionOwner({ payload, params: dispatchParams, options, status: codeAgentOwnerStatusForResult(payload) }); + results.set(traceId, annotateOwner(payload, dispatchParams)); + traceStore.append(traceId, { + type: "result", + status: payload.status === "completed" ? "completed" : "failed", + label: `result:${payload.status}`, + message: payload.status === "completed" + ? "Code Agent result is committed to the Workbench projection event stream." + : payload.error?.message ?? "Code Agent completed without a successful reply.", + sessionId: payload.sessionId ?? payload.session?.sessionId, + sessionStatus: payload.session?.status, + terminal: true + }); + } catch (error) { + if (isCodeAgentResultCanceled(results.get(traceId))) return; + const payload = { + status: "failed", + traceId, + conversationId: safeConversationId(params.conversationId) || null, + sessionId: safeSessionId(params.sessionId) || null, + error: { + code: "code_agent_background_failed", + layer: "api", + retryable: true, + message: error?.message ?? "Code Agent background turn failed", + userMessage: "Code Agent 后台任务失败；错误已提交到 Workbench 实时投影。" + }, + updatedAt: new Date().toISOString() + }; + await finalizeCodeAgentBillingUsage({ payload, params, options }); + results.set(traceId, payload); + traceStore.append(traceId, { + type: "result", + status: "failed", + label: "result:failed", + errorCode: payload.error.code, + message: payload.error.message, + terminal: true + }); + } + }; + setImmediate(() => { + run(); + }); + return annotateOwner(acceptedPayload, params); +} + +async function recordCodeAgentTurnAdmission({ payload = {}, params = {}, options = {}, traceId } = {}) { + const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; + let ownerRecord = null; + try { + ownerRecord = await recordCodeAgentSessionOwner({ payload, params, options, status: "running" }); + if (!ownerRecord) { + const error = new Error("Code Agent turn admission did not durably persist Workbench session ownership."); + error.code = "workbench_admission_owner_not_persisted"; + error.valuesRedacted = true; + throw error; + } + await recordCodeAgentSessionInputFact({ + params, + options, + traceId, + delivery: "queue", + status: "admitted", + messageId: codeAgentTurnLifecycleFields(traceId, params).userMessageId, + commandId: payload.agentRun?.commandId ?? null + }); + } catch (error) { + throw codeAgentAdmissionUnavailableError(error, { params, options, traceId }); + } + traceStore.append(traceId, { + type: "request", + status: "admitted", + label: "turn:admitted", + message: "Code Agent turn was durably admitted before billing or dispatch preflight.", + sessionId: safeSessionId(params.sessionId) || null, + threadId: safeOpaqueId(params.threadId) || null, + ...codeAgentOtelTraceFields(traceId, options.env ?? process.env), + waitingFor: "billing-or-dispatch", + valuesPrinted: false + }); + void emitCodeAgentOtelSpan("durable_admission", traceId, options.env ?? process.env, { + attributes: { sessionId: safeSessionId(params.sessionId) || null, threadId: safeOpaqueId(params.threadId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId } + }); + return ownerRecord; +} + +function codeAgentAdmissionUnavailableError(error, { params = {}, options = {}, traceId } = {}) { + const payload = codeAgentAdmissionUnavailablePayload({ error, params, options, traceId }); + return Object.assign(new Error(payload.error.message), { + code: "admission_unavailable", + statusCode: 503, + payload, + alreadyClassified: true + }); +} + +function isCodeAgentAdmissionUnavailableError(error) { + return error?.code === "admission_unavailable" && error?.payload; +} + +function codeAgentAdmissionUnavailablePayload({ error, params = {}, options = {}, traceId } = {}) { + const message = error?.message ?? "Code Agent turn admission is unavailable; the user message was not persisted."; + return { + ok: false, + accepted: false, + status: "failed", + traceId: safeTraceId(traceId) || null, + ...codeAgentOtelTraceFields(traceId, options.env ?? process.env), + ...codeAgentTurnLifecycleFields(traceId, params), + conversationId: safeConversationId(params.conversationId) || null, + sessionId: safeSessionId(params.sessionId) || null, + threadId: safeOpaqueId(params.threadId) || null, + error: { + code: "admission_unavailable", + layer: "admission", + retryable: true, + message, + userMessage: "本次输入尚未持久化，请保留草稿后重试。", + route: "/v1/agent/chat" + }, + blocker: { + code: "admission_unavailable", + layer: "admission", + retryable: true, + summary: message + }, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +function codeAgentDispatchFailure(error) { + const message = error?.message ?? "AgentRun dispatch failed"; + return { + statusCode: 503, + payload: { + error: { + code: error?.code ?? "agentrun_dispatch_failed", + layer: "agentrun", + retryable: true, + message, + userMessage: "AgentRun 调度失败；错误已提交到 Workbench 实时投影。", + route: "/v1/agent/chat" + }, + blocker: { + code: error?.code ?? "agentrun_dispatch_failed", + layer: "agentrun", + retryable: true, + summary: message + } + } + }; +} + +function codeAgentDispatchErrorOtelAttributes(error) { + const agentRunError = error?.agentRunError && typeof error.agentRunError === "object" ? error.agentRunError : null; + const details = agentRunError?.error?.details && typeof agentRunError.error.details === "object" ? agentRunError.error.details : null; + return { + failureKind: safeOtelText(error?.code ?? agentRunError?.failureKind, 120), + upstreamStatusCode: Number.isInteger(error?.statusCode) ? error.statusCode : null, + upstreamTraceId: safeTraceId(agentRunError?.traceId) || null, + upstreamMessage: safeOtelText(agentRunError?.message ?? error?.message, 300), + upstreamStderrTail: safeOtelText(details?.stderr, 600) + }; +} + +function safeOtelText(value, limit = 300) { + const text = String(value ?? "").replace(/[\u0000-\u001f\u007f]+/gu, " ").trim(); + return text ? text.slice(0, limit) : null; +} + +function codeAgentAdmittedFailureBasePayload({ params = {}, options = {}, traceId } = {}) { + const now = new Date().toISOString(); + return { + accepted: true, + status: "running", + shortConnection: true, + traceId, + ...codeAgentTurnLifecycleFields(traceId, params), + messageId: codeAgentTurnLifecycleFields(traceId, params).assistantMessageId, + projectId: params.projectId ?? null, + conversationId: safeConversationId(params.conversationId) || null, + sessionId: safeSessionId(params.sessionId) || null, + threadId: safeOpaqueId(params.threadId) || null, + ...codeAgentAdmissionTimingFields(now), + provider: codeAgentAgentRunAdapterEnabled(options.env ?? process.env) ? "agentrun" : "codex-stdio", + model: options.env?.HWLAB_CODE_AGENT_MODEL ?? "unknown", + backend: codeAgentAgentRunAdapterEnabled(options.env ?? process.env) ? "agentrun-v01" : "hwlab-cloud-api/code-agent-chat", + valuesPrinted: false + }; +} + +function codeAgentAdmissionTimingFields(value = null) { + const admittedAt = timestampIsoOrNow(value); + const timing = { + startedAt: admittedAt, + lastEventAt: admittedAt, + finishedAt: null, + durationMs: null, + observedAt: admittedAt, + lastEventAgeMs: 0, + valuesRedacted: true + }; + return { + createdAt: admittedAt, + updatedAt: admittedAt, + timing, + startedAt: admittedAt, + lastEventAt: admittedAt, + finishedAt: null, + durationMs: null + }; +} + +async function settleAdmittedCodeAgentFailure({ base = {}, params = {}, options = {}, traceId, results, failure = {}, traceLabel = "code-agent:failed" } = {}) { + const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; + const error = failure.payload?.error ?? failure.error ?? {}; + const blocker = failure.payload?.blocker ?? failure.blocker ?? null; + const message = error.message ?? blocker?.summary ?? "Code Agent request failed after admission."; + traceStore.append(traceId, { + type: "result", + status: "failed", + label: traceLabel, + errorCode: error.code ?? "code_agent_failed_after_admission", + message, + terminal: true, + valuesPrinted: false + }); + const now = new Date().toISOString(); + const payload = annotateOwner({ + ...base, + accepted: true, + status: "failed", + traceId, + ...codeAgentTurnLifecycleFields(traceId, base), + messageId: base.messageId ?? codeAgentTurnLifecycleFields(traceId, base).assistantMessageId, + conversationId: safeConversationId(base.conversationId ?? params.conversationId) || null, + sessionId: safeSessionId(base.sessionId ?? params.sessionId) || null, + threadId: safeOpaqueId(base.threadId ?? params.threadId) || null, + error: { + code: error.code ?? "code_agent_failed_after_admission", + layer: error.layer ?? blocker?.layer ?? "api", + retryable: error.retryable !== false, + message, + userMessage: error.userMessage ?? message, + route: error.route ?? "/v1/agent/chat" + }, + blocker: blocker ? { + code: blocker.code ?? error.code ?? "code_agent_failed_after_admission", + layer: blocker.layer ?? error.layer ?? "api", + retryable: blocker.retryable !== false, + summary: blocker.summary ?? message + } : null, + finalResponse: { + text: error.userMessage ?? message, + textChars: String(error.userMessage ?? message).length, + role: "assistant", + status: "failed", + traceId, + updatedAt: now, + source: "code-agent-admitted-failure", + valuesPrinted: false + }, + runnerTrace: traceStore.snapshot(traceId), + updatedAt: now, + valuesPrinted: false, + secretMaterialStored: false + }, params); + await finalizeCodeAgentBillingUsage({ payload, params, options }); + payload.runnerTrace = traceStore.snapshot(traceId); + recordCodeAgentConversationFact(payload, options); + results?.set?.(traceId, payload); + await recordCodeAgentSessionOwner({ payload, params, options, status: codeAgentOwnerStatusForResult(payload) }); + return payload; +} diff --git a/internal/cloud/server-code-agent-http-support.ts b/internal/cloud/server-code-agent-http-support.ts new file mode 100644 index 00000000..f37d50af --- /dev/null +++ b/internal/cloud/server-code-agent-http-support.ts @@ -0,0 +1,1602 @@ +// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-0102 Agent编排 draft-2026-06-17-r0; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. +// Responsibility: Shared Code Agent HTTP lifecycle, billing, ownership, projection-read, and response helpers. + +import { createHash, randomUUID } from "node:crypto"; + +import { M3_IO_CONTROL_ROUTE, M3_STATUS_ROUTE, describeM3StatusLive, handleM3IoControl } from "./m3-io-control.ts"; +import { + agentRunSessionEvidence, + cancelAgentRunChatTurn, + codeAgentAgentRunAdapterEnabled, + initialAgentRunChatResult, + loadPersistedAgentRunResult, + steerAgentRunChatTurn, + submitAgentRunChatTurn +} from "./code-agent-agentrun-adapter.ts"; +import { createCodeAgentErrorPayload, handleCodeAgentChat } from "./code-agent-chat.ts"; +import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; +import { codeAgentSessionLifecycleSummary } from "./code-agent-session-lifecycle.ts"; +import { messageAuthorityTextValue } from "./code-agent-agentrun-prompt.ts"; +import { writeWorkbenchProjectionSession, writeWorkbenchSessionAdmissionFact } from "./workbench-projection-writer.ts"; +import { createWorkbenchReadModel } from "./workbench-read-model.ts"; +import { codeAgentOtelTraceFields, emitCodeAgentOtelSpan } from "./otel-trace.ts"; +import { + firstHeaderValue, + getHeader, + parsePositiveInteger, + positiveInteger, + readBody, + safeConversationId, + safeOpaqueId, + safeSessionId, + safeTraceId, + sendJson, + truthyFlag +} from "./server-http-utils.ts"; + +const DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT = 120; +const DEFAULT_CODE_AGENT_TRACE_PAGE_LIMIT = 100; +const MAX_CODE_AGENT_TRACE_PAGE_LIMIT = 100; +const DEFAULT_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS = 2500; +const DEFAULT_CODE_AGENT_PROJECT_ID = "prj_hwpod_workbench"; +const DEFAULT_CODE_AGENT_PROVIDER_PROFILE = "deepseek"; +const CODE_AGENT_TERMINAL_STATUSES = new Set(["completed", "failed", "blocked", "timeout", "cancelled", "canceled"]); +const CODE_AGENT_PROVIDER_PROFILE_ALIASES = Object.freeze({ codex: "codex-api" }); +const CODE_AGENT_PROVIDER_PROFILE_ID_PATTERN = /^[a-z0-9][a-z0-9.-]{0,63}$/u; +const CODE_AGENT_PROVIDER_PROFILE_LABELS = Object.freeze({ + "deepseek": "DeepSeek", + "codex-api": "Codex API", + "gpt.pika": "gpt.pika", + "minimax-m3": "MiniMax-M3 via AgentRun", + "runtime-default": "运行默认" +}); +const DEFAULT_CODE_AGENT_CODEX_API_MODEL = "gpt-5.5"; +const DEFAULT_CODE_AGENT_CODEX_API_BASE_URL = "http://127.0.0.1:49280/responses"; +const DEFAULT_CODE_AGENT_DEEPSEEK_MODEL = "deepseek-chat"; +const DEFAULT_CODE_AGENT_MINIMAX_M3_MODEL = "MiniMax-M3"; + + +function normalizeCodeAgentProviderProfile(value) { + const text = String(value ?? DEFAULT_CODE_AGENT_PROVIDER_PROFILE).trim().toLowerCase(); + if (!text) return DEFAULT_CODE_AGENT_PROVIDER_PROFILE; + if (text === "runtime-default") return text; + const normalized = CODE_AGENT_PROVIDER_PROFILE_ALIASES[text] ?? text; + return CODE_AGENT_PROVIDER_PROFILE_ID_PATTERN.test(normalized) ? normalized : DEFAULT_CODE_AGENT_PROVIDER_PROFILE; +} + +function firstNonEmptyValue(...values) { + for (const value of values) { + const text = String(value ?? "").trim(); + if (text) return text; + } + return null; +} + +async function recordCodeAgentSessionInputFact({ params = {}, options = {}, traceId, delivery = "queue", status = "admitted", messageId = null, commandId = null, error = null } = {}) { + const runtimeStore = options.runtimeStore ?? null; + if (typeof runtimeStore?.writeWorkbenchFacts !== "function") return null; + const resolvedTraceId = safeTraceId(traceId ?? params.traceId) || null; + const lifecycle = codeAgentTurnLifecycleFields(resolvedTraceId, params); + const sessionId = safeSessionId(params.sessionId ?? params.session?.sessionId) || null; + if (!sessionId) return null; + const resolvedMessageId = safeMessageId(messageId ?? params.messageId ?? lifecycle.userMessageId) || lifecycle.userMessageId; + const resolvedCommandId = textValue(commandId ?? params.commandId ?? params.agentRun?.commandId) || null; + const normalizedDelivery = codeAgentInputDelivery(delivery); + const normalizedStatus = codeAgentInputStatus(status); + const now = new Date().toISOString(); + const sourceEventId = `${sessionId}:${lifecycle.turnId}:${normalizedDelivery}:${resolvedTraceId ?? resolvedCommandId ?? resolvedMessageId}`; + const inputId = stableCodeAgentInputId({ sessionId, turnId: lifecycle.turnId, traceId: resolvedTraceId, messageId: resolvedMessageId, commandId: resolvedCommandId, delivery: normalizedDelivery, sourceEventId }); + const fact = { + inputId, + sessionId, + turnId: lifecycle.turnId, + traceId: resolvedTraceId, + messageId: resolvedMessageId, + commandId: resolvedCommandId, + delivery: normalizedDelivery, + status: normalizedStatus, + errorCode: textValue(error?.code ?? error?.error?.code) || null, + sourceEventId, + promptHash: params.message || params.prompt || params.text ? sha256Text(params.message ?? params.prompt ?? params.text).slice(0, 16) : null, + textBytes: params.message || params.prompt || params.text ? Buffer.byteLength(String(params.message ?? params.prompt ?? params.text), "utf8") : 0, + valuesRedacted: true, + secretMaterialStored: false, + createdAt: now, + updatedAt: now + }; + return runtimeStore.writeWorkbenchFacts({ facts: { inputs: [fact] } }, { + sessionId, + traceId: resolvedTraceId, + turnId: lifecycle.turnId, + messageId: resolvedMessageId, + ownerUserId: options.actor?.id ?? params.ownerUserId, + projectId: params.projectId ?? DEFAULT_CODE_AGENT_PROJECT_ID, + valuesPrinted: false + }); +} + +function stableCodeAgentInputId(value) { + const payload = Object.fromEntries(Object.entries(value ?? {}).filter(([, item]) => item !== undefined && item !== null && item !== "").sort(([left], [right]) => left.localeCompare(right))); + return `wsi_${sha256Text(JSON.stringify(payload)).slice(0, 32)}`; +} + +function codeAgentInputDelivery(value) { + const text = textValue(value).toLowerCase(); + return ["queue", "steer", "cancel"].includes(text) ? text : "queue"; +} + +function codeAgentInputStatus(value) { + const text = textValue(value).toLowerCase().replace(/-/gu, "_"); + return ["admitted", "promoted", "blocked", "failed", "canceled", "completed"].includes(text) ? text : "admitted"; +} + +function codeAgentResultTimingFields(...sources) { + const records = sources.filter((source) => source && typeof source === "object"); + const timingSource = records.map((source) => source.timing).find((timing) => timing && typeof timing === "object") ?? {}; + const startedAt = firstTimestampIso(timingSource.startedAt, ...records.map((source) => source.startedAt), ...records.map((source) => source.createdAt)); + const lastEventAt = firstLatestTimestampIso(timingSource.lastEventAt, ...records.map((source) => source.lastEventAt), ...records.map((source) => source.updatedAt), ...records.map((source) => source.createdAt)); + const finishedAt = firstLatestTimestampIso(timingSource.finishedAt, ...records.map((source) => source.finishedAt), ...records.map((source) => source.completedAt)); + const durationMs = numberOrNull(timingSource.durationMs ?? records.map((source) => source.durationMs).find((value) => numberOrNull(value) !== null)); + const observedAt = timestampIsoOrNull(timingSource.observedAt); + const lastEventAgeMs = numberOrNull(timingSource.lastEventAgeMs); + if (!startedAt && !lastEventAt && !finishedAt && durationMs === null && lastEventAgeMs === null) return {}; + const timing = { + ...(timingSource && typeof timingSource === "object" ? timingSource : {}), + startedAt: startedAt ?? null, + lastEventAt: lastEventAt ?? null, + finishedAt: finishedAt ?? null, + durationMs, + observedAt: observedAt ?? null, + lastEventAgeMs, + valuesRedacted: timingSource.valuesRedacted !== false + }; + return { + timing, + startedAt: timing.startedAt, + lastEventAt: timing.lastEventAt, + finishedAt: timing.finishedAt, + durationMs: timing.durationMs + }; +} + +function firstTimestampIso(...values) { + for (const value of values) { + const timestamp = timestampIsoOrNull(value); + if (timestamp) return timestamp; + } + return null; +} + +function firstLatestTimestampIso(...values) { + let latest = null; + let latestMs = Number.NEGATIVE_INFINITY; + for (const value of values) { + const timestamp = timestampIsoOrNull(value); + if (!timestamp) continue; + const ms = Date.parse(timestamp); + if (!Number.isFinite(ms) || ms < latestMs) continue; + latest = timestamp; + latestMs = ms; + } + return latest; +} + +function codeAgentChatShortConnectionRequested(request, params, options = {}) { + const header = firstHeaderValue(request, "prefer"); + const explicit = firstHeaderValue(request, "x-hwlab-short-connection"); + const envValue = options.env?.HWLAB_CODE_AGENT_CHAT_SHORT_CONNECTION; + return codeAgentAgentRunAdapterEnabled(options.env ?? process.env) || + /(?:^|,)\s*respond-async\s*(?:,|$)/iu.test(String(header ?? "")) || + truthyFlag(explicit) || + params?.shortConnection === true || + truthyFlag(envValue); +} + +async function preflightCodeAgentBilling({ params = {}, options = {}, traceId, response, sendResponse = true } = {}) { + const client = options.userBillingClient; + if (!codeAgentBillingEnabled(options) || !client?.configured || !options.userBillingAuth?.active) return null; + const estimatedCredits = parsePositiveInteger(options.env?.HWLAB_USER_BILLING_CODE_AGENT_ESTIMATED_CREDITS, 1); + const body = { + ...(options.actorApiKeySecret ? { apiKey: options.actorApiKeySecret } : { userId: options.actor?.id }), + serviceId: "hwlab-code-agent", + estimatedCredits, + idempotencyKey: `code-agent:${traceId}:preflight`, + metadata: codeAgentBillingMetadata(params, traceId, { stage: "preflight" }) + }; + const result = await client.billingPreflight(body); + if (result.ok && result.body?.allowed !== false) return { reservation: sanitizeBillingReservation(result.body) }; + const status = [402, 403, 429].includes(Number(result.status)) ? Number(result.status) : 503; + const payload = { + ok: false, + accepted: false, + status: status === 402 ? "payment_required" : status === 403 ? "not_entitled" : status === 429 ? "rate_limited" : "billing_unavailable", + traceId, + error: { + code: result.error?.code ?? (status === 402 ? "insufficient_credits" : "user_billing_preflight_failed"), + layer: "billing", + retryable: status === 429 || status === 503, + message: result.error?.message ?? "Code Agent billing preflight failed", + route: "/v1/agent/chat" + }, + blocker: { + code: result.error?.code ?? (status === 402 ? "insufficient_credits" : "user_billing_preflight_failed"), + layer: "billing", + retryable: status === 429 || status === 503, + summary: result.error?.message ?? "Code Agent billing preflight failed" + }, + valuesRedacted: true + }; + if (sendResponse && response) sendJson(response, status, payload); + return { blocked: true, statusCode: status, payload, error: payload.error, blocker: payload.blocker }; +} + +async function recordCodeAgentBillingUsage({ payload = {}, params = {}, options = {} } = {}) { + const reservation = params.userBillingReservation ?? payload.userBillingReservation; + const reservationId = typeof reservation?.reservationId === "string" ? reservation.reservationId : ""; + const client = options.userBillingClient; + if (payload.billing?.recorded === true || payload.billing?.released === true || payload.status === "running") return payload.billing ?? null; + if (!reservationId || !client?.configured) return null; + const traceId = safeTraceId(payload.traceId ?? params.traceId); + const usedTokens = codeAgentUsedTokens(payload); + const usedCredits = parsePositiveInteger(options.env?.HWLAB_USER_BILLING_CODE_AGENT_USED_CREDITS, 1); + const result = await client.billingRecord({ + reservationId, + serviceId: "hwlab-code-agent", + usedCredits, + usedTokens, + idempotencyKey: `code-agent:${traceId}:record`, + metadata: codeAgentBillingMetadata(params, traceId, { stage: "record", status: payload.status ?? "unknown" }) + }); + const billing = result.ok ? sanitizeBillingRecord(result.body, reservation) : { + recorded: false, + reservationId, + errorCode: result.error?.code ?? "user_billing_record_failed", + valuesRedacted: true + }; + payload.billing = billing; + const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; + if (traceId) { + traceStore.append(traceId, { + type: "billing", + status: result.ok ? "recorded" : "degraded", + label: result.ok ? "billing:recorded" : "billing:record_failed", + reservationId, + recordId: result.ok ? result.body?.recordId ?? null : null, + errorCode: result.ok ? null : billing.errorCode, + valuesPrinted: false + }); + } + return billing; +} + +async function finalizeCodeAgentBillingUsage({ payload = {}, params = {}, options = {} } = {}) { + if (payload.billing?.recorded === true || payload.billing?.released === true) return payload.billing; + if (payload.status === "running") return null; + if (payload.status === "completed") { + return recordCodeAgentBillingUsage({ payload, params, options }); + } + return releaseCodeAgentBillingReservation({ payload, params, options }); +} + +async function releaseCodeAgentBillingReservation({ payload = {}, params = {}, options = {} } = {}) { + const reservation = params.userBillingReservation ?? payload.userBillingReservation; + const reservationId = typeof reservation?.reservationId === "string" ? reservation.reservationId : ""; + const client = options.userBillingClient; + if (payload.billing?.recorded === true || payload.billing?.released === true || !reservationId || !client?.configured) return payload.billing ?? null; + const traceId = safeTraceId(payload.traceId ?? params.traceId); + const releaseReason = String(payload.status ?? "not_completed").trim() || "not_completed"; + const releaseBody = { + reservationId, + serviceId: "hwlab-code-agent", + idempotencyKey: `code-agent:${traceId}:release`, + reason: releaseReason, + metadata: codeAgentBillingMetadata(params, traceId, { + stage: "release", + status: payload.status ?? "unknown", + errorCode: payload.error?.code ?? payload.blocker?.code ?? null + }) + }; + const result = typeof client.billingRelease === "function" + ? await client.billingRelease(releaseBody) + : { ok: false, error: { code: "user_billing_release_not_supported" } }; + const billing = result.ok ? sanitizeBillingRelease(result.body, reservation) : { + released: false, + reservationId, + errorCode: result.error?.code ?? "user_billing_release_failed", + valuesRedacted: true + }; + payload.billing = billing; + const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; + if (traceId) { + traceStore.append(traceId, { + type: "billing", + status: result.ok ? "released" : "degraded", + label: result.ok ? "billing:released" : "billing:release_failed", + reservationId, + reason: releaseReason, + errorCode: result.ok ? null : billing.errorCode, + valuesPrinted: false + }); + } + return billing; +} + +function withCodeAgentBillingReservation(payload = {}, params = {}) { + if (!payload || typeof payload !== "object" || payload.userBillingReservation) return payload; + const reservation = params.userBillingReservation; + return reservation ? { ...payload, userBillingReservation: reservation } : payload; +} + +function codeAgentBillingEnabled(options = {}) { + const env = options.env ?? process.env; + if (String(env.HWLAB_USER_BILLING_CODE_AGENT_ENABLED ?? "").trim() === "0") return false; + return Boolean(options.userBillingClient?.configured); +} + +function codeAgentBillingMetadata(params = {}, traceId, extra = {}) { + return { + traceId, + conversationId: safeConversationId(params.conversationId) || null, + sessionId: safeSessionId(params.sessionId) || null, + projectId: textValue(params.projectId) || null, + route: "/v1/agent/chat", + resource: "code-agent-turn", + ...extra, + valuesRedacted: true + }; +} + +function sanitizeBillingReservation(value = {}) { + return { + allowed: value.allowed === true, + reservationId: textValue(value.reservationId) || null, + resourceType: textValue(value.resourceType) || null, + planId: textValue(value.planId) || null, + estimatedCredits: Number.isFinite(Number(value.estimatedCredits)) ? Number(value.estimatedCredits) : null, + estimatedTokens: Number.isFinite(Number(value.estimatedTokens)) ? Number(value.estimatedTokens) : null, + expiresAt: textValue(value.expiresAt) || null, + valuesRedacted: true + }; +} + +function sanitizeBillingRecord(value = {}, reservation = {}) { + return { + recorded: true, + reservationId: textValue(reservation.reservationId) || null, + recordId: textValue(value.recordId) || null, + credits: Number.isFinite(Number(value.credits)) ? Number(value.credits) : null, + balance: Number.isFinite(Number(value.balance)) ? Number(value.balance) : null, + valuesRedacted: true + }; +} + +function sanitizeBillingRelease(value = {}, reservation = {}) { + return { + released: true, + reservationId: textValue(value.reservationId) || textValue(reservation.reservationId) || null, + releasedCredits: Number.isFinite(Number(value.releasedCredits)) ? Number(value.releasedCredits) : null, + status: textValue(value.status) || "cancelled", + valuesRedacted: true + }; +} + +function codeAgentUsedTokens(payload = {}) { + const usage = payload.usage && typeof payload.usage === "object" ? payload.usage : null; + const traceSummary = payload.traceSummary && typeof payload.traceSummary === "object" ? payload.traceSummary : null; + const candidates = [ + usage?.totalTokens, + usage?.total_tokens, + usage?.tokens, + Number(usage?.inputTokens ?? usage?.input_tokens ?? 0) + Number(usage?.outputTokens ?? usage?.output_tokens ?? 0), + traceSummary?.tokenCount, + traceSummary?.tokens + ]; + for (const value of candidates) { + const parsed = Number(value); + if (Number.isFinite(parsed) && parsed > 0) return Math.ceil(parsed); + } + return 0; +} + +function normalizeTurnStatus(...values) { + for (const value of values) { + const text = textValue(value).toLowerCase().replace(/_/gu, "-"); + if (!text) continue; + if (["accepted", "pending", "processing", "running", "busy", "creating", "queued", "in-flight"].includes(text)) return "running"; + if (["completed", "done", "succeeded", "success"].includes(text)) return "completed"; + if (["failed", "failure", "error", "stale", "thread-resume-failed", "aborted", "interrupted", "expired"].includes(text)) return "failed"; + if (["cancelled", "canceled"].includes(text)) return "canceled"; + if (["blocked", "timeout"].includes(text)) return text; + } + return null; +} + +function recordCodeAgentConversationFact(payload = {}, options = {}) { + const registry = options.sessionRegistry; + const conversationId = safeConversationId(payload.conversationId); + if (!conversationId || typeof registry?.recordFact !== "function") return null; + try { + return registry.recordFact(conversationId, { + kind: payload.status === "completed" ? "runner_turn" : "runner_turn_terminal", + conversationId, + sessionId: payload.sessionId, + traceId: payload.traceId, + provider: payload.provider, + backend: payload.backend, + workspace: payload.workspace, + sandbox: payload.sandbox, + session: payload.session, + sessionMode: payload.sessionMode, + sessionReuse: payload.sessionReuse, + implementationType: payload.implementationType, + runner: payload.runner, + runnerTrace: payload.runnerTrace, + partialContext: payload.partialContext ?? payload.runnerTrace?.partialContext, + capabilityLevel: payload.capabilityLevel, + toolCalls: payload.toolCalls, + skills: payload.skills + }, { now: options.now }); + } catch { + return null; + } +} + +async function recordCodeAgentSessionOwner({ payload = {}, params = {}, options = {}, status = "active", preserveLastTraceId = false } = {}) { + const ownerUserId = options.actor?.id ?? params.ownerUserId; + if (!ownerUserId || !options.accessController?.recordAgentSessionOwner) return null; + const traceId = safeTraceId(payload.traceId ?? params.traceId); + const session = payload.session && typeof payload.session === "object" ? payload.session : null; + const sessionReuse = payload.sessionReuse && typeof payload.sessionReuse === "object" ? payload.sessionReuse : null; + const sessionId = safeSessionId(payload.sessionId ?? session?.sessionId ?? sessionReuse?.sessionId ?? params.sessionId) || null; + const conversationId = safeConversationId(payload.conversationId ?? session?.conversationId ?? sessionReuse?.conversationId ?? params.conversationId) || null; + const threadId = safeOpaqueId(session?.threadId ?? sessionReuse?.threadId ?? payload.threadId ?? params.threadId) || null; + return writeWorkbenchProjectionSession({ + accessController: options.accessController, + runtimeStore: options.runtimeStore, + traceStore: options.traceStore ?? defaultCodeAgentTraceStore, + traceId, + ownerUserId, + ownerRole: options.actor?.role ?? params.ownerRole ?? null, + sessionId, + projectId: params.projectId ?? payload.projectId ?? null, + conversationId, + threadId, + status, + preserveLastTraceId, + session: codeAgentSessionOwnerEvidence(payload, params), + payload, + params + }); +} + +function codeAgentOwnerStatusForResult(result = {}) { + const sessionStatus = textValue(result?.session?.status ?? result?.sessionSummary?.status ?? result?.sessionLifecycleStatus); + if (sessionStatus === "thread-resume-failed") return "thread-resume-failed"; + if (codeAgentFreshContinuationFailure(result)) return "thread-resume-failed"; + if (codeAgentPayloadHasSealedFinalResponse(result)) return "completed"; + if (result?.status === "completed") return "completed"; + if (result?.status === "canceled" || result?.status === "cancelled") return "canceled"; + return result?.status ?? "active"; +} + +function textValue(value) { + return String(value ?? "").trim(); +} + +function timestampIsoOrNow(value = null) { + const parsed = Date.parse(String(value ?? "")); + return Number.isFinite(parsed) ? new Date(parsed).toISOString() : new Date().toISOString(); +} + +function timestampIsoOrNull(value = null) { + const parsed = Date.parse(String(value ?? "")); + return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null; +} + +function codeAgentSessionOwnerEvidence(payload = {}, params = {}) { + const traceId = payload.traceId ?? params.traceId ?? null; + const finalResponse = codeAgentFinalResponseEvidence(payload, traceId); + const traceSummary = codeAgentTraceSummaryEvidence(payload, traceId, finalResponse); + const messages = codeAgentConversationMessagesEvidence(payload, params, traceId, finalResponse); + const firstUserMessagePreview = messages.find((message) => message.role === "user")?.text?.slice(0, 240) ?? null; + const promptEvidence = codeAgentPromptMetadata(payload, params, traceId); + const traceResult = codeAgentTraceResultEvidence(payload, params, traceId, finalResponse, traceSummary); + const providerProfile = codeAgentProviderProfileEvidence(payload, params); + const continuation = codeAgentContinuationEvidence(payload, params); + const sessionLifecycleStatus = continuation.requiresFreshSession ? "thread-resume-failed" : payload.sessionLifecycleStatus ?? null; + return { + ...(providerProfile ? { providerProfile, codeAgentProviderProfile: providerProfile } : {}), + provider: payload.provider ?? null, + model: payload.model ?? null, + backend: payload.backend ?? null, + status: payload.status ?? null, + sessionMode: payload.sessionMode ?? payload.session?.sessionMode ?? null, + sessionLifecycleStatus, + continuation, + runnerKind: payload.runner?.kind ?? payload.runnerTrace?.runnerKind ?? null, + conversationId: payload.conversationId ?? params.conversationId ?? null, + sessionId: payload.sessionId ?? payload.session?.sessionId ?? payload.sessionReuse?.sessionId ?? params.sessionId ?? null, + threadId: payload.session?.threadId ?? payload.sessionReuse?.threadId ?? payload.threadId ?? params.threadId ?? null, + traceId, + projectId: payload.projectId ?? params.projectId ?? null, + ...codeAgentPromptFields(promptEvidence), + finalResponse, + traceSummary, + ...(traceResult ? { traceResults: { [traceResult.traceId]: traceResult } } : {}), + ...(messages.length ? { messages, messageCount: messages.length } : {}), + ...(firstUserMessagePreview ? { firstUserMessagePreview } : {}), + ...agentRunSessionEvidence(payload), + secretMaterialStored: false, + valuesRedacted: true + }; +} + +function codeAgentContinuationEvidence(payload = {}, params = {}) { + const failure = codeAgentFreshContinuationFailure(payload); + const providerProfile = codeAgentProviderProfileEvidence(payload, params); + const traceId = safeTraceId(payload.traceId ?? params.traceId) || null; + if (!failure) { + return { + requiresFreshSession: false, + providerProfile, + traceId, + valuesRedacted: true + }; + } + return { + requiresFreshSession: true, + reasonCode: failure.code, + reason: failure.message, + staleThreadId: safeOpaqueId(payload.session?.threadId ?? payload.sessionReuse?.threadId ?? payload.threadId ?? params.threadId ?? payload.agentRun?.threadId) || null, + providerProfile, + traceId, + valuesRedacted: true + }; +} + +function codeAgentFreshContinuationFailure(payload = {}) { + if (!payload || typeof payload !== "object") return null; + const sessionStatus = String(payload.session?.status ?? payload.sessionLifecycleStatus ?? payload.sessionReuse?.status ?? "").trim().toLowerCase().replace(/_/gu, "-"); + if (sessionStatus === "thread-resume-failed") return { code: "thread-resume-failed", message: "previous AgentRun thread/session resume failed", valuesRedacted: true }; + const status = normalizeTurnStatus(payload.status, payload.agentRun?.terminalStatus, payload.agentRun?.commandState, payload.agentRun?.status, payload.session?.status, payload.sessionReuse?.status); + if (!status || !["failed", "blocked", "timeout"].includes(status)) return null; + const code = normalizeCodeAgentFailureKind(firstNonEmptyValue( + payload.error?.code, + payload.blocker?.code, + payload.providerTrace?.failureKind, + payload.providerTrace?.errorCode, + payload.agentRun?.failureKind, + payload.agentRun?.errorCode, + payload.transient?.failureKind, + payload.transient?.errorCode + )); + const message = firstNonEmptyValue( + payload.error?.message, + payload.error?.userMessage, + payload.blocker?.summary, + payload.blocker?.message, + payload.providerTrace?.failureMessage, + payload.providerTrace?.summary, + payload.agentRun?.failureMessage, + payload.agentRun?.message + ) ?? ""; + if (!codeAgentFailureRequiresFreshContinuation(code, message)) return null; + return { code: code ?? "stale-continuation", message: message || code || "stale continuation", valuesRedacted: true }; +} + +function normalizeCodeAgentFailureKind(value) { + const normalized = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); + return normalized || null; +} + +function codeAgentFailureRequiresFreshContinuation(code, message = "") { + const failureKind = normalizeCodeAgentFailureKind(code); + if (["thread-resume-failed", "session-store-evicted", "provider-stream-disconnected"].includes(failureKind)) return true; + return /session stor(?:e|age).*evicted|pvc-backed session|no rollout found for thread id|no rollout found|thread\/resume failed|provider stream disconnected/iu.test(String(message ?? "")); +} + +function codeAgentProviderProfileEvidence(payload = {}, params = {}) { + const raw = firstNonEmptyValue( + params.providerProfile, + params.codeAgentProviderProfile, + payload.providerProfile, + payload.codeAgentProviderProfile, + payload.agentRun?.backendProfile, + payload.providerTrace?.backendProfile, + payload.backendProfile + ); + if (!raw) return null; + const normalized = normalizeCodeAgentProviderProfile(raw); + return normalized === "runtime-default" ? null : normalized; +} + +function codeAgentTraceResultEvidence(payload = {}, params = {}, traceId = null, finalResponse = null, traceSummary = null) { + const resolvedTraceId = safeTraceId(traceId); + if (!resolvedTraceId) return null; + const agentRun = agentRunSessionEvidence(payload).agentRun ?? null; + const userBillingReservation = codeAgentBillingReservationEvidence(payload.userBillingReservation ?? params.userBillingReservation); + const resultSession = payload.session && typeof payload.session === "object" ? payload.session : null; + const sessionReuse = payload.sessionReuse && typeof payload.sessionReuse === "object" ? payload.sessionReuse : null; + const conversationId = safeConversationId(payload.conversationId ?? agentRun?.conversationId ?? resultSession?.conversationId ?? sessionReuse?.conversationId ?? params.conversationId) || null; + const sessionId = safeSessionId(payload.sessionId ?? resultSession?.sessionId ?? sessionReuse?.sessionId ?? params.sessionId) || null; + const threadId = safeOpaqueId(payload.threadId ?? resultSession?.threadId ?? sessionReuse?.threadId ?? params.threadId ?? agentRun?.threadId) || null; + const promptEvidence = codeAgentPromptMetadata(payload, params, resolvedTraceId); + if (!agentRun && !finalResponse && !traceSummary) return null; + return { + traceId: resolvedTraceId, + status: codeAgentPayloadHasSealedFinalResponse(payload) ? "completed" : payload.status ?? agentRun?.terminalStatus ?? agentRun?.commandState ?? null, + conversationId, + sessionId, + threadId, + ...codeAgentPromptFields(promptEvidence), + messageId: finalResponse?.messageId ?? payload.messageId ?? null, + createdAt: payload.createdAt ?? finalResponse?.createdAt ?? null, + updatedAt: payload.updatedAt ?? finalResponse?.updatedAt ?? null, + finalResponse, + traceSummary, + agentRun, + ...(userBillingReservation ? { userBillingReservation } : {}), + valuesRedacted: true, + secretMaterialStored: false + }; +} + +function codeAgentPromptMetadata(payload = {}, params = {}, traceId = null) { + const prompt = payload?.prompt && typeof payload.prompt === "object" ? payload.prompt : null; + const rawPrompt = firstNonEmptyValue( + params.message, + params.prompt, + params.text, + payload.userMessage, + typeof payload.prompt === "string" ? payload.prompt : null + ); + const fromText = codeAgentPromptMetadataFromText(rawPrompt, traceId, "code-agent-request", { fullText: Boolean(rawPrompt) }); + return codeAgentPromptMetadataFromParts({ + traceId, + source: prompt?.source ?? payload.promptSource ?? fromText?.prompt?.source ?? "code-agent-request", + promptId: firstNonEmptyValue(payload.promptId, prompt?.id, fromText?.promptId), + promptSha256: firstNonEmptyValue(payload.promptSha256, prompt?.sha256, fromText?.promptSha256), + promptSummary: firstNonEmptyValue(payload.promptSummary, prompt?.summary, fromText?.promptSummary), + promptPreview: firstNonEmptyValue(payload.promptPreview, prompt?.preview, fromText?.promptPreview), + textChars: Number.isInteger(prompt?.textChars) ? prompt.textChars : fromText?.prompt?.textChars + }); +} + +function codeAgentPromptMetadataFromText(value, traceId = null, source = "code-agent-request", options = {}) { + const promptText = String(value ?? "").trim(); + if (!promptText) return null; + return codeAgentPromptMetadataFromParts({ + traceId, + source, + promptSha256: options.fullText === true ? sha256Text(promptText) : undefined, + promptSummary: promptText, + promptPreview: promptText, + textChars: options.fullText === true ? promptText.length : undefined + }); +} + +function codeAgentPromptMetadataFromParts({ traceId = null, source = "code-agent-request", promptId = null, promptSha256 = null, promptSummary = null, promptPreview = null, textChars = undefined } = {}) { + const digest = firstNonEmptyValue(promptSha256); + const summary = clippedPromptText(firstNonEmptyValue(promptSummary, promptPreview), 240); + const preview = clippedPromptText(firstNonEmptyValue(promptPreview, promptSummary), 500); + const explicitId = firstNonEmptyValue(promptId); + if (!explicitId && !digest && !summary && !preview) return null; + const id = explicitId || codeAgentPromptId(traceId, digest); + const prompt = compactCodeAgentObject({ + id, + sha256: digest || undefined, + summary, + preview, + textChars: Number.isInteger(textChars) ? textChars : undefined, + source, + valuesPrinted: false + }); + return compactCodeAgentObject({ + promptId: id, + promptSha256: digest || undefined, + promptSummary: summary, + promptPreview: preview, + prompt, + valuesPrinted: false + }) ?? null; +} + +function codeAgentPromptFields(metadata = null) { + if (!metadata) return {}; + return compactCodeAgentObject({ + promptId: metadata.promptId, + promptSha256: metadata.promptSha256, + promptSummary: metadata.promptSummary, + promptPreview: metadata.promptPreview, + prompt: metadata.prompt + }) ?? {}; +} + +function codeAgentPromptId(traceId = null, digest = null) { + const traceSuffix = safeTraceId(traceId)?.slice(4).replace(/[^A-Za-z0-9_.:-]/gu, "_"); + const digestSuffix = firstNonEmptyValue(digest)?.slice(0, 16); + const suffix = firstNonEmptyValue(traceSuffix, digestSuffix); + return suffix ? `prm_${suffix}` : undefined; +} + +function clippedPromptText(value, limit) { + const promptText = String(value ?? "").trim(); + if (!promptText) return undefined; + return promptText.length > limit ? `${promptText.slice(0, limit)}...` : promptText; +} + +function sha256Text(value) { + return createHash("sha256").update(String(value)).digest("hex"); +} + +function compactCodeAgentObject(value) { + const result = {}; + for (const [key, item] of Object.entries(value ?? {})) { + if (item === undefined || item === null) continue; + if (Array.isArray(item) && item.length === 0) continue; + if (typeof item === "object" && !Array.isArray(item) && Object.keys(item).length === 0) continue; + result[key] = item; + } + return Object.keys(result).length > 0 ? result : undefined; +} + +function codeAgentBillingReservationEvidence(value = null) { + if (!value || typeof value !== "object") return null; + const reservationId = textValue(value.reservationId); + if (!reservationId) return null; + return { + reservationId, + resourceType: textValue(value.resourceType) || null, + planId: textValue(value.planId) || null, + estimatedCredits: numberOrNull(value.estimatedCredits), + estimatedTokens: numberOrNull(value.estimatedTokens), + expiresAt: textValue(value.expiresAt) || null, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +function codeAgentConversationMessagesEvidence(payload = {}, params = {}, traceId = null, finalResponse = null) { + const userText = boundedConversationMessageText(params.message ?? params.prompt ?? payload.userMessage ?? payload.prompt); + if (!userText) return []; + const resolvedTraceId = safeTraceId(traceId) || null; + const lifecycle = codeAgentTurnLifecycleFields(resolvedTraceId, payload); + const session = payload.session && typeof payload.session === "object" ? payload.session : null; + const sessionReuse = payload.sessionReuse && typeof payload.sessionReuse === "object" ? payload.sessionReuse : null; + const conversationId = safeConversationId(payload.conversationId ?? session?.conversationId ?? sessionReuse?.conversationId ?? params.conversationId) || null; + const sessionId = safeSessionId(payload.sessionId ?? session?.sessionId ?? sessionReuse?.sessionId ?? params.sessionId) || null; + const threadId = safeOpaqueId(session?.threadId ?? sessionReuse?.threadId ?? payload.threadId ?? params.threadId) || null; + const createdAt = payload.createdAt ?? payload.reply?.createdAt ?? new Date().toISOString(); + const updatedAt = payload.updatedAt ?? createdAt; + const userMessage = { + id: lifecycle.userMessageId, + messageId: lifecycle.userMessageId, + role: "user", + title: "用户", + text: userText, + status: "sent", + traceId: resolvedTraceId, + turnId: lifecycle.turnId, + conversationId, + sessionId, + threadId, + createdAt, + updatedAt, + source: "code-agent-submit", + secretMaterialStored: false, + valuesRedacted: true + }; + const agentText = boundedConversationMessageText( + finalResponse?.text + ?? payload.reply?.content + ?? payload.message?.content + ?? payload.assistantText + ?? payload.error?.userMessage + ?? payload.error?.message + ?? payload.blocker?.userMessage + ?? payload.blocker?.message + ?? payload.blocker?.summary + ); + const agentStatus = codeAgentConversationAgentMessageStatus(payload); + const agentMessage = { + id: lifecycle.assistantMessageId, + messageId: lifecycle.assistantMessageId, + role: "agent", + title: agentStatus === "failed" ? "Code Agent 返回阻塞" : agentStatus === "running" ? "Code Agent 处理中" : "Code Agent 回复", + text: agentText || (agentStatus === "running" ? "" : "Code Agent 请求已结束，请查看 Trace 详情。"), + status: agentStatus, + traceId: resolvedTraceId, + turnId: lifecycle.turnId, + conversationId, + sessionId, + threadId, + createdAt: payload.reply?.createdAt ?? updatedAt, + updatedAt, + source: "code-agent-result", + ...(payload.runnerTrace && typeof payload.runnerTrace === "object" ? { runnerTrace: codeAgentConversationRunnerTraceEvidence(payload.runnerTrace, resolvedTraceId) } : {}), + secretMaterialStored: false, + valuesRedacted: true + }; + return [userMessage, agentMessage]; +} + +function boundedConversationMessageText(value, limit = 12000) { + const text = conversationText(value); + if (!text) return ""; + return text.length > limit ? text.slice(0, limit) : text; +} + +function conversationText(value) { + const extracted = conversationTextRaw(value).replace(/\r\n?/gu, "\n"); + if (!extracted.trim() || extracted.trim() === "[object Object]") return ""; + return extracted; +} + +function conversationTextRaw(value) { + if (value === undefined || value === null) return ""; + if (typeof value === "string") return value; + if (typeof value === "number" || typeof value === "boolean") return String(value); + if (Array.isArray(value)) return value.map(conversationTextRaw).filter(Boolean).join("\n"); + if (typeof value === "object") { + for (const key of ["text", "content", "message", "prompt", "value", "summary", "preview", "title"]) { + const text = conversationTextRaw(value[key]); + if (text) return text; + } + } + return ""; +} + +function codeAgentMessageTraceSuffix(traceId) { + const text = safeTraceId(traceId) || `local_${Date.now().toString(36)}`; + return text.replace(/^trc_/u, "").replace(/[^A-Za-z0-9_.:-]/gu, "_").slice(0, 48) || "trace"; +} + +function codeAgentTurnLifecycleFields(traceId, source = {}) { + const resolvedTraceId = safeTraceId(source?.traceId ?? traceId) || safeTraceId(traceId) || null; + const turnId = safeTurnId(source?.turnId) || resolvedTraceId; + const traceSuffix = codeAgentMessageTraceSuffix(resolvedTraceId); + return { + turnId, + userMessageId: safeMessageId(source?.userMessageId ?? source?.userMessage?.messageId) || codeAgentLifecycleMessageId(traceSuffix, "user"), + assistantMessageId: safeMessageId(source?.assistantMessageId ?? source?.assistantMessage?.messageId) || codeAgentLifecycleMessageId(traceSuffix, "agent") + }; +} + +function codeAgentLifecycleMessageId(traceSuffix, role) { + return `msg_${traceSuffix}_${role}`; +} + +function safeTurnId(value) { + const text = textValue(value); + return /^turn_[A-Za-z0-9_.:-]+$/u.test(text) ? text : null; +} + +function safeMessageId(value) { + const text = textValue(value); + return /^msg_[A-Za-z0-9_.:-]+$/u.test(text) ? text : null; +} + +function codeAgentConversationAgentMessageStatus(payload = {}) { + const status = textValue(payload.status ?? payload.agentRun?.terminalStatus ?? payload.commandState ?? payload.runStatus).toLowerCase(); + if (codeAgentPayloadHasSealedFinalResponse(payload)) return "completed"; + if (["completed", "done", "success"].includes(status)) return "completed"; + if (["failed", "blocked", "error", "timeout", "canceled", "cancelled"].includes(status)) return "failed"; + if (payload.error || payload.blocker) return "failed"; + return "running"; +} + +function codeAgentPayloadHasSealedFinalResponse(payload = {}) { + if (!payload || typeof payload !== "object") return false; + if (payload.error || payload.blocker) return false; + if (!codeAgentFinalResponseText(payload)) return false; + const agentRun = payload.agentRun && typeof payload.agentRun === "object" ? payload.agentRun : {}; + return isTraceCommandTerminalStatus(payload.status) || + isTraceCommandTerminalStatus(agentRun.status) || + isTraceCommandTerminalStatus(agentRun.runStatus) || + isTraceCommandTerminalStatus(agentRun.commandState) || + isTraceCommandTerminalStatus(agentRun.terminalStatus); +} + +function codeAgentConversationRunnerTraceEvidence(runnerTrace = {}, traceId = null) { + const events = Array.isArray(runnerTrace.events) ? runnerTrace.events : []; + const lastEvent = runnerTrace.lastEvent && typeof runnerTrace.lastEvent === "object" ? runnerTrace.lastEvent : events.at(-1) ?? null; + const eventCount = Number.isFinite(Number(runnerTrace.eventCount)) ? Number(runnerTrace.eventCount) : events.length; + return { + traceId: safeTraceId(runnerTrace.traceId ?? traceId) || traceId || null, + status: runnerTrace.status ?? lastEvent?.status ?? null, + sessionId: runnerTrace.sessionId ?? null, + threadId: runnerTrace.threadId ?? null, + runId: runnerTrace.runId ?? lastEvent?.runId ?? null, + commandId: runnerTrace.commandId ?? lastEvent?.commandId ?? null, + eventCount, + lastEvent: lastEvent ? { + label: lastEvent.label ?? null, + status: lastEvent.status ?? null, + terminal: lastEvent.terminal === true, + createdAt: lastEvent.createdAt ?? null, + message: boundedConversationMessageText(lastEvent.message ?? lastEvent.text, 1000), + valuesPrinted: false + } : null, + eventsCompacted: true, + fullTraceLoaded: false, + valuesPrinted: false + }; +} + +function codeAgentFinalResponseEvidence(payload = {}, traceId = null) { + const text = codeAgentFinalResponseText(payload); + if (!text) return null; + return { + text, + textChars: text.length, + messageId: payload.finalResponse?.messageId ?? payload.reply?.messageId ?? payload.messageId ?? null, + role: payload.reply?.role ?? payload.message?.role ?? "assistant", + status: codeAgentPayloadHasSealedFinalResponse(payload) ? "completed" : payload.finalResponse?.status ?? payload.status ?? null, + traceId, + createdAt: payload.finalResponse?.createdAt ?? payload.reply?.createdAt ?? payload.createdAt ?? null, + updatedAt: payload.finalResponse?.updatedAt ?? payload.updatedAt ?? null, + source: payload.finalResponse?.source ?? "code-agent-result", + valuesPrinted: false + }; +} + +function codeAgentFinalResponseText(payload = {}) { + for (const value of [ + payload?.finalResponse?.text, + payload?.finalResponse?.content, + payload?.finalResponse, + payload?.reply?.content, + payload?.reply, + payload?.message?.content, + payload?.message, + payload?.assistantText, + payload?.finalText, + payload?.text, + payload?.content + ]) { + const text = conversationText(value); + if (text) return text; + } + return ""; +} + +function codeAgentTraceSummaryEvidence(payload = {}, traceId = null, finalResponse = null) { + const runnerTrace = payload.runnerTrace && typeof payload.runnerTrace === "object" ? payload.runnerTrace : null; + const events = Array.isArray(runnerTrace?.events) ? runnerTrace.events : []; + const terminalEvent = [...events].reverse().find((event) => event?.terminal === true) ?? runnerTrace?.lastEvent ?? null; + return { + traceId, + source: "code-agent-derived-session-evidence", + sourceEventCount: Number.isFinite(Number(runnerTrace?.eventCount)) ? Number(runnerTrace.eventCount) : events.length, + terminalStatus: payload.agentRun?.terminalStatus ?? payload.status ?? terminalEvent?.status ?? null, + lastEventLabel: terminalEvent?.label ?? null, + finalAssistantRow: finalResponse ? { + role: finalResponse.role, + status: finalResponse.status, + textChars: finalResponse.textChars, + textPreview: finalResponse.text.slice(0, 240), + messageId: finalResponse.messageId, + valuesPrinted: false + } : null, + agentRun: payload.agentRun ? { + runId: payload.agentRun.runId ?? null, + commandId: payload.agentRun.commandId ?? null, + attemptId: payload.agentRun.attemptId ?? null, + runnerId: payload.agentRun.runnerId ?? null, + jobName: payload.agentRun.jobName ?? null, + namespace: payload.agentRun.namespace ?? null, + lastSeq: payload.agentRun.lastSeq ?? null, + valuesPrinted: false + } : null, + updatedAt: payload.updatedAt ?? null, + valuesPrinted: false + }; +} + +function annotateOwner(payload, params = {}) { + if (!params.ownerUserId) return payload; + return { + ...payload, + ownerUserId: params.ownerUserId, + ownerRole: params.ownerRole ?? null + }; +} + +function canAccessOwnedResult(result, actor) { + if (!result?.ownerUserId || !actor) return true; + return actor.role === "admin" || actor.id === result.ownerUserId; +} + +function isCodeAgentResultCanceled(result) { + return result?.status === "canceled" || result?.canceled === true || result?.error?.code === "codex_stdio_canceled"; +} + +function codeAgentCancelScopeMismatch(params = {}, result = {}) { + return cancelScopeFieldMismatch("conversationId", safeConversationId(params.conversationId), cancelExpectedValues(safeConversationId, + result.conversationId, + result.agentRun?.conversationId, + result.session?.conversationId, + result.sessionReuse?.conversationId, + result.runnerTrace?.conversationId + )) ?? cancelScopeFieldMismatch("sessionId", safeSessionId(params.sessionId), cancelExpectedValues(safeSessionId, + result.sessionId, + result.session?.sessionId, + result.sessionReuse?.sessionId, + result.agentRun?.hwlabSessionId + )) ?? cancelScopeFieldMismatch("threadId", safeOpaqueId(params.threadId), cancelExpectedValues(safeOpaqueId, + result.threadId, + result.agentRun?.threadId, + result.session?.threadId, + result.sessionReuse?.threadId, + result.runnerTrace?.threadId + )); +} + +function cancelExpectedValues(normalize, ...values) { + return [...new Set(values.map((value) => normalize(value)).filter(Boolean))]; +} + +function cancelScopeFieldMismatch(field, requested, expectedValues) { + if (!requested || expectedValues.length === 0 || expectedValues.includes(requested)) return null; + return { field, requested, expected: expectedValues[0] }; +} + +function cancelBlockedPayload({ + code, + message, + traceId, + conversationId = null, + sessionId = null, + session = null, + runnerTrace = null, + status = "failed", + unsupported = false, + degraded = false +}) { + const sessionSummary = codeAgentSessionLifecycleSummary({ + session, + runnerTrace, + status, + unsupported, + degraded, + error: { code, userMessage: message, message } + }); + return { + accepted: false, + canceled: false, + status, + conversationId, + sessionId, + traceId, + session, + sessionLifecycleStatus: sessionSummary.status, + sessionLifecycle: sessionSummary, + sessionSummary, + unsupported, + degraded, + runnerTrace, + lastTraceEvent: runnerTrace?.lastEvent ?? null, + error: { + code, + layer: "session", + category: "cancel_blocked", + retryable: true, + userMessage: message, + message, + traceId, + route: "/v1/agent/chat/cancel", + toolName: "codex-stdio.cancel" + }, + blocker: { + code, + layer: "session", + category: "cancel_blocked", + retryable: true, + summary: message, + userMessage: message, + traceId, + route: "/v1/agent/chat/cancel", + toolName: "codex-stdio.cancel" + } + }; +} + +function isTraceCommandTerminalStatus(status) { + return CODE_AGENT_TERMINAL_STATUSES.has(String(status ?? "").trim().toLowerCase().replace(/_/gu, "-")); +} + +function traceSnapshotWithTerminalEvidence(snapshot, result, traceId, refreshError = null) { + const evidence = agentRunTerminalTraceEvidence(result, traceId); + if (snapshot && snapshot.status !== "missing") return traceSnapshotWithAttachedTerminalEvidence(snapshot, evidence, refreshError); + if (!evidence) return { + ...snapshot, + ok: false, + traceStatus: "missing", + retention: traceRetentionSummary("missing") + }; + return { + ...snapshot, + ok: true, + status: "expired", + traceStatus: "expired", + projectionStatus: "projecting", + projectionHealth: "degraded", + persisted: true, + retention: traceRetentionSummary("expired"), + conversationId: evidence.conversationId, + sessionId: evidence.sessionId, + threadId: evidence.threadId, + ...codeAgentPromptFields(evidence), + agentRun: evidence.agentRun, + finalResponse: evidence.finalResponse, + traceSummary: evidence.traceSummary, + terminalEvidence: { + available: true, + source: evidence.source, + refresh: refreshError ? { + attempted: true, + ok: false, + code: refreshError?.code ?? "agentrun_trace_refresh_failed", + message: refreshError?.message ?? "AgentRun trace events could not be refreshed; command result remains authoritative for final response.", + valuesPrinted: false + } : { attempted: false, ok: null }, + conversationId: evidence.conversationId, + sessionId: evidence.sessionId, + threadId: evidence.threadId, + ...codeAgentPromptFields(evidence), + agentRun: evidence.agentRun, + traceSummary: evidence.traceSummary, + finalResponse: evidence.finalResponse, + valuesPrinted: false + } + }; +} + +function traceSnapshotWithAttachedTerminalEvidence(snapshot, evidence, refreshError = null) { + if (!evidence) return snapshot; + return { + ...snapshot, + conversationId: snapshot.conversationId ?? evidence.conversationId, + sessionId: snapshot.sessionId ?? evidence.sessionId, + threadId: snapshot.threadId ?? evidence.threadId, + ...codeAgentPromptFields(evidence), + agentRun: snapshot.agentRun ?? evidence.agentRun, + finalResponse: snapshot.finalResponse ?? evidence.finalResponse, + traceSummary: snapshot.traceSummary ?? evidence.traceSummary, + terminalEvidence: snapshot.terminalEvidence ?? (refreshError ? { + available: true, + source: evidence.source, + refresh: { + attempted: true, + ok: false, + code: refreshError?.code ?? "agentrun_trace_refresh_failed", + message: refreshError?.message ?? "AgentRun trace events could not be refreshed; live events may be partial.", + valuesPrinted: false + }, + valuesPrinted: false + } : undefined) + }; +} + +function agentRunTerminalTraceEvidence(result, traceId) { + if (!result || typeof result !== "object") return null; + const storedSummary = result.traceSummary && typeof result.traceSummary === "object" && result.traceSummary.source === "agentrun-command-result" ? result.traceSummary : null; + const finalResponse = agentRunTerminalFinalResponse(result, traceId); + const promptEvidence = codeAgentPromptMetadata(result, {}, traceId); + const agentRun = result.agentRun && typeof result.agentRun === "object" ? { + adapter: result.agentRun.adapter ?? null, + runId: result.agentRun.runId ?? null, + commandId: result.agentRun.commandId ?? null, + attemptId: result.agentRun.attemptId ?? null, + runnerId: result.agentRun.runnerId ?? null, + jobName: result.agentRun.jobName ?? null, + namespace: result.agentRun.namespace ?? null, + terminalStatus: result.agentRun.terminalStatus ?? null, + lastSeq: result.agentRun.lastSeq ?? null, + valuesPrinted: false + } : null; + const terminalStatus = normalizeTurnStatus(result.agentRun?.terminalStatus, result.agentRun?.commandState, result.agentRun?.status, result.agentRun?.runStatus); + if (!isTraceCommandTerminalStatus(terminalStatus)) return null; + if (!agentRun?.runId || !agentRun?.commandId || (!storedSummary && !finalResponse)) return null; + const traceSummary = { + traceId, + source: "agentrun-command-result", + sourceEventCount: numberOrNull(storedSummary?.sourceEventCount ?? storedSummary?.eventCount ?? result.runnerTrace?.eventCount), + renderedRowSummary: storedSummary?.renderedRowSummary ?? null, + noiseEventCount: numberOrNull(storedSummary?.noiseEventCount), + omittedNoiseCount: numberOrNull(storedSummary?.omittedNoiseCount), + terminalStatus: storedSummary?.terminalStatus ?? terminalStatus, + updatedAt: storedSummary?.updatedAt ?? result.updatedAt ?? null, + valuesPrinted: false + }; + return { + source: "agentrun-command-result", + conversationId: safeConversationId(result.conversationId) || null, + sessionId: safeSessionId(result.sessionId) || null, + threadId: safeOpaqueId(result.threadId) || null, + ...codeAgentPromptFields(promptEvidence), + agentRun, + finalResponse, + traceSummary + }; +} + +function agentRunTerminalFinalResponse(result, traceId) { + const stored = result.finalResponse && typeof result.finalResponse === "object" ? result.finalResponse : null; + const text = textValue(stored?.text ?? result.reply?.content ?? result.message?.content ?? result.assistantText); + if (!text) return null; + const storedTraceId = safeTraceId(stored?.traceId ?? result.traceId) || null; + if (storedTraceId && storedTraceId !== traceId) return null; + return { + text, + textChars: text.length, + messageId: stored?.messageId ?? result.reply?.messageId ?? result.messageId ?? null, + role: stored?.role ?? result.reply?.role ?? "assistant", + status: stored?.status ?? result.status ?? null, + traceId: traceId, + createdAt: stored?.createdAt ?? result.reply?.createdAt ?? result.createdAt ?? null, + updatedAt: stored?.updatedAt ?? result.updatedAt ?? null, + source: "agentrun-command-result", + valuesPrinted: false + }; +} + +function traceRetentionSummary(status) { + return { + traceStatus: status, + liveTraceStore: status === "missing" ? "missing" : "expired-or-evicted", + policy: "live trace events are short-term storage; terminal final response and summary are resolved from AgentRun command result by traceId and commandId", + replacementEvidence: status === "missing" ? "resolve the trace through AgentRun command registry/result or report the trace as missing" : "use terminalEvidence.finalResponse, terminalEvidence.traceSummary, conversationId, sessionId, threadId, and agentRun ids", + valuesPrinted: false + }; +} + +function numberOrNull(value) { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function createCodeAgentChatResultStore({ maxResults = 256 } = {}) { + const limit = positiveInteger(maxResults, 256); + const results = new Map(); + function set(traceId, payload) { + const id = safeTraceId(traceId); + if (!id) return; + results.set(id, { + ...payload, + traceId: id, + cachedAt: new Date().toISOString() + }); + while (results.size > limit) { + const oldest = results.keys().next().value; + if (!oldest) break; + results.delete(oldest); + } + } + return { + set, + get: (traceId) => results.get(safeTraceId(traceId)) ?? null, + clear: () => results.clear() + }; +} + +function compactCodeAgentChatResultPayload(payload, options = {}) { + if (!payload || typeof payload !== "object") return payload; + const limit = resultTraceEventLimit(options); + const terminalEvidence = agentRunTerminalTraceEvidence(payload, payload.traceId); + const { userBillingReservation, ...publicPayload } = payload; + return { + ...publicPayload, + ...codeAgentTurnLifecycleFields(publicPayload.traceId, publicPayload), + ...(terminalEvidence ? { terminalEvidence: terminalEvidencePayload(terminalEvidence) } : {}), + ...(publicPayload.runnerTrace && typeof publicPayload.runnerTrace === "object" + ? { runnerTrace: compactRunnerTraceForResult(publicPayload.runnerTrace, limit) } + : {}) + }; +} + +function terminalEvidencePayload(evidence) { + return { + available: true, + source: evidence.source, + conversationId: evidence.conversationId, + sessionId: evidence.sessionId, + threadId: evidence.threadId, + ...codeAgentPromptFields(evidence), + agentRun: evidence.agentRun, + traceSummary: evidence.traceSummary, + finalResponse: evidence.finalResponse, + valuesPrinted: false + }; +} + +function compactRunnerTraceForResult(runnerTrace, limit = DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT) { + if (!runnerTrace || typeof runnerTrace !== "object") return runnerTrace; + const events = Array.isArray(runnerTrace.events) ? runnerTrace.events : []; + const total = Number.isInteger(runnerTrace.eventCount) ? runnerTrace.eventCount : events.length; + const effectiveLimit = Math.max(8, positiveInteger(limit, DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT)); + if (events.length <= effectiveLimit) { + return { + ...runnerTrace, + eventCount: total, + eventsCompacted: false + }; + } + const headCount = Math.min(24, Math.max(1, Math.floor(effectiveLimit / 4))); + const tailCount = Math.max(1, effectiveLimit - headCount - 1); + const omitted = Math.max(0, events.length - headCount - tailCount); + const head = events.slice(0, headCount); + const tail = events.slice(events.length - tailCount); + const marker = { + traceId: runnerTrace.traceId ?? head[0]?.traceId ?? tail[0]?.traceId ?? null, + type: "trace", + stage: "trace", + status: "truncated", + label: "trace:compacted", + createdAt: tail[0]?.createdAt ?? runnerTrace.updatedAt ?? new Date().toISOString(), + message: `Compatibility result response compacted ${omitted} trace events; fetch /v1/agent/traces/${encodeURIComponent(runnerTrace.traceId ?? "")} for the full trace.`, + omittedEventCount: omitted, + retainedEventCount: head.length + tail.length, + totalEventCount: total, + valuesPrinted: false + }; + const compactEvents = [...head, marker, ...tail]; + return { + ...runnerTrace, + eventCount: total, + events: compactEvents, + eventLabels: compactEvents.map(traceEventLabel).filter(Boolean), + eventsCompacted: true, + eventWindow: { + mode: "head-tail", + retained: compactEvents.length, + omitted, + total + } + }; +} + +function traceEventLabel(event) { + return typeof event === "string" ? event : event?.label; +} + +function resultTraceEventLimit(options = {}) { + return positiveInteger( + options.env?.HWLAB_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT, + DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT + ); +} + +function createCodeAgentM3HwlabApiRequestJson(options = {}) { + return async (targetUrl, request = {}) => { + const url = new URL(targetUrl); + const route = url.pathname; + if (route === M3_IO_CONTROL_ROUTE && request.method === "POST") { + return { + ok: true, + status: 200, + body: await handleM3IoControl(request.body ?? {}, { + runtimeStore: options.runtimeStore, + now: options.now, + env: options.env, + requestJson: options.m3IoRequestJson + }) + }; + } + if (route === M3_STATUS_ROUTE && request.method === "GET") { + return { + ok: true, + status: 200, + body: await describeM3StatusLive(options) + }; + } + return { + ok: false, + status: 404, + body: { + error: { + code: "not_found", + message: `Unsupported Code Agent M3 HWLAB API route ${route}` + } + }, + error: `Unsupported Code Agent M3 HWLAB API route ${route}` + }; + }; +} +function recordCodeAgentProjectionMetrics(options = {}, { result = null, trace = null, projection = null, refreshError = null } = {}) { + const performanceStore = options.backendPerformanceStore ?? options.backendPerformance; + if (!performanceStore?.recordWorkbenchProjection || !projection) return; + const sourceLatestSeq = sourceLatestSeqFromResult(result); + const lastProjectedSeq = Number(projection.lastProjectedSeq ?? 0); + if (!Number.isFinite(sourceLatestSeq) && !Number.isFinite(lastProjectedSeq)) return; + performanceStore.recordWorkbenchProjection({ + sourceLatestSeq: Number.isFinite(sourceLatestSeq) ? sourceLatestSeq : lastProjectedSeq, + lastProjectedSeq: Number.isFinite(lastProjectedSeq) ? lastProjectedSeq : 0, + sourceLatestEventAt: sourceLatestAtFromResult(result, trace), + projectedLatestEventAt: projection.updatedAt ?? trace?.updatedAt ?? result?.updatedAt ?? null, + terminalSourceAt: terminalSourceAtFromResult(result), + terminalProjectedAt: projection.projectionStatus === "caught-up" ? projection.updatedAt ?? trace?.updatedAt ?? result?.updatedAt ?? null : null, + status: projection?.status ?? "unknown", + reason: projectionReason(projection, refreshError), + projectionStatus: projection.projectionStatus ?? "unknown", + source: "workbench_read_model" + }); +} + +function recordCodeAgentTurnReadMetric(options = {}, url, statusCode, body = {}, startedAt = nowMs()) { + const performanceStore = options.backendPerformanceStore ?? options.backendPerformance; + if (!performanceStore?.recordWorkbenchTurnRead) return; + performanceStore.recordWorkbenchTurnRead({ + route: url?.pathname ?? "/v1/agent/turns/:id", + status: body?.status ?? statusClassLabel(statusCode), + degradedReason: turnReadDegradedReason(body), + durationMs: nowMs() - startedAt, + responseBytes: responseBodyBytes(body), + timedOut: statusCode === 504 || body?.error?.code === "timeout" || body?.refreshError?.code === "agentrun_timeout" + }); +} + +function turnReadDegradedReason(body = {}) { + if (body?.refreshError?.code) return body.refreshError.code; + if (body?.projection?.blocker?.code) return body.projection.blocker.code; + if (body?.blocker?.code) return body.blocker.code; + if (body?.error?.code) return body.error.code; + if (body?.projectionStatus === "projecting") return "projecting"; + if (body?.projectionStatus === "blocked") return "projection_blocked"; + return "none"; +} + +function responseBodyBytes(body = {}) { + try { return Buffer.byteLength(JSON.stringify(body)); } catch { return 0; } +} + +function statusClassLabel(value) { + const status = Number(value); + if (!Number.isFinite(status) || status <= 0) return "unknown"; + return `${Math.floor(status / 100)}xx`; +} + +function sourceLatestSeqFromResult(result = null) { + const candidates = [ + result?.agentRun?.lastSeq, + result?.traceSummary?.agentRun?.lastSeq, + result?.traceSummary?.lastSeq, + result?.providerTrace?.lastSeq, + result?.runnerTrace?.lastEvent?.sourceSeq, + result?.runnerTrace?.lastEvent?.seq, + result?.runnerTrace?.eventCount + ]; + for (const value of candidates) { + const seq = Number(value); + if (Number.isFinite(seq) && seq >= 0) return Math.floor(seq); + } + return NaN; +} + +function sourceLatestAtFromResult(result = null, trace = null) { + return result?.traceSummary?.updatedAt + ?? result?.agentRun?.updatedAt + ?? result?.providerTrace?.updatedAt + ?? result?.runnerTrace?.updatedAt + ?? trace?.updatedAt + ?? result?.updatedAt + ?? null; +} + +function terminalSourceAtFromResult(result = null) { + if (!result) return null; + const terminalStatus = result?.agentRun?.terminalStatus ?? result?.traceSummary?.terminalStatus ?? result?.status; + if (!isTraceCommandTerminalStatus(terminalStatus)) return null; + return result?.traceSummary?.updatedAt ?? result?.agentRun?.updatedAt ?? result?.updatedAt ?? null; +} + +function projectionReason(projection = null, refreshError = null) { + if (refreshError) return refreshError.code ?? "refresh_error"; + if (projection?.blocker) return projection.blocker.code ?? "projection_blocked"; + if (projection?.projectionStatus === "projecting") return "projecting"; + if (projection?.projectionStatus === "caught-up") return "none"; + return projection?.projectionStatus ?? "unknown"; +} + +function nowMs() { + if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now(); + return Date.now(); +} + +function uniqueStrings(values) { + return [...new Set(values.map((value) => String(value ?? "").trim()).filter(Boolean))]; +} + +export { + normalizeCodeAgentProviderProfile, + firstNonEmptyValue, + recordCodeAgentSessionInputFact, + stableCodeAgentInputId, + codeAgentInputDelivery, + codeAgentInputStatus, + codeAgentResultTimingFields, + firstTimestampIso, + firstLatestTimestampIso, + codeAgentChatShortConnectionRequested, + preflightCodeAgentBilling, + recordCodeAgentBillingUsage, + finalizeCodeAgentBillingUsage, + releaseCodeAgentBillingReservation, + withCodeAgentBillingReservation, + codeAgentBillingEnabled, + codeAgentBillingMetadata, + sanitizeBillingReservation, + sanitizeBillingRecord, + sanitizeBillingRelease, + codeAgentUsedTokens, + normalizeTurnStatus, + recordCodeAgentConversationFact, + recordCodeAgentSessionOwner, + codeAgentOwnerStatusForResult, + textValue, + timestampIsoOrNow, + timestampIsoOrNull, + codeAgentSessionOwnerEvidence, + codeAgentContinuationEvidence, + codeAgentFreshContinuationFailure, + normalizeCodeAgentFailureKind, + codeAgentFailureRequiresFreshContinuation, + codeAgentProviderProfileEvidence, + codeAgentTraceResultEvidence, + codeAgentPromptMetadata, + codeAgentPromptMetadataFromText, + codeAgentPromptMetadataFromParts, + codeAgentPromptFields, + codeAgentPromptId, + clippedPromptText, + sha256Text, + compactCodeAgentObject, + codeAgentBillingReservationEvidence, + codeAgentConversationMessagesEvidence, + boundedConversationMessageText, + conversationText, + conversationTextRaw, + codeAgentMessageTraceSuffix, + codeAgentTurnLifecycleFields, + codeAgentLifecycleMessageId, + safeTurnId, + safeMessageId, + codeAgentConversationAgentMessageStatus, + codeAgentPayloadHasSealedFinalResponse, + codeAgentConversationRunnerTraceEvidence, + codeAgentFinalResponseEvidence, + codeAgentFinalResponseText, + codeAgentTraceSummaryEvidence, + annotateOwner, + canAccessOwnedResult, + isCodeAgentResultCanceled, + codeAgentCancelScopeMismatch, + cancelExpectedValues, + cancelScopeFieldMismatch, + cancelBlockedPayload, + isTraceCommandTerminalStatus, + traceSnapshotWithTerminalEvidence, + traceSnapshotWithAttachedTerminalEvidence, + agentRunTerminalTraceEvidence, + agentRunTerminalFinalResponse, + traceRetentionSummary, + numberOrNull, + createCodeAgentChatResultStore, + compactCodeAgentChatResultPayload, + terminalEvidencePayload, + compactRunnerTraceForResult, + traceEventLabel, + resultTraceEventLimit, + createCodeAgentM3HwlabApiRequestJson, + recordCodeAgentProjectionMetrics, + recordCodeAgentTurnReadMetric, + turnReadDegradedReason, + responseBodyBytes, + statusClassLabel, + sourceLatestSeqFromResult, + sourceLatestAtFromResult, + terminalSourceAtFromResult, + projectionReason, + nowMs, + uniqueStrings +}; diff --git a/internal/cloud/server-code-agent-http.ts b/internal/cloud/server-code-agent-http.ts index 56b80939..a41b4444 100644 --- a/internal/cloud/server-code-agent-http.ts +++ b/internal/cloud/server-code-agent-http.ts @@ -1,4756 +1,7 @@ -// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-0102 Agent编排 draft-2026-06-17-r0; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. -// Responsibility: Code Agent HTTP turn admission, lifecycle compatibility wrappers, and Workbench projection writer/finalizer integration. +// SPEC: PJ2026-0104010803 Workbench唯一投影; PJ2026-010403 API契约; PJ2026-010205 HWLAB接入; PJ2026-0102 Agent编排. +// Responsibility: Stable Code Agent HTTP export facade. -import { createHash, randomUUID } from "node:crypto"; - -import { M3_IO_CONTROL_ROUTE, M3_STATUS_ROUTE, describeM3StatusLive, handleM3IoControl } from "./m3-io-control.ts"; -import { - agentRunSessionEvidence, - cancelAgentRunChatTurn, - codeAgentAgentRunAdapterEnabled, - initialAgentRunChatResult, - loadPersistedAgentRunResult, - steerAgentRunChatTurn, - submitAgentRunChatTurn, - syncAgentRunChatResult -} from "./code-agent-agentrun-adapter.ts"; -import { createCodeAgentErrorPayload, handleCodeAgentChat } from "./code-agent-chat.ts"; -import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; -import { codeAgentSessionLifecycleSummary } from "./code-agent-session-lifecycle.ts"; -import { messageAuthorityTextValue } from "./code-agent-agentrun-prompt.ts"; -import { scheduleWorkbenchProjectionFinalizer } from "./workbench-projection-finalizer.ts"; -import { writeWorkbenchProjectionSession, writeWorkbenchSessionAdmissionFact } from "./workbench-projection-writer.ts"; -import { buildAgentRunProjectionEventsFetchPlan } from "./workbench-projection-cursor.ts"; -import { createWorkbenchReadModel } from "./workbench-read-model.ts"; -import { publishHwlabAgentRunCommandShadow } from "./kafka-shadow-producer.ts"; -import { codeAgentOtelTraceFields, emitCodeAgentOtelSpan } from "./otel-trace.ts"; -import { - firstHeaderValue, - getHeader, - parsePositiveInteger, - positiveInteger, - readBody, - safeConversationId, - safeOpaqueId, - safeSessionId, - safeTraceId, - sendJson, - truthyFlag -} from "./server-http-utils.ts"; - -const DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT = 120; -const DEFAULT_CODE_AGENT_TRACE_PAGE_LIMIT = 100; -const MAX_CODE_AGENT_TRACE_PAGE_LIMIT = 100; -const DEFAULT_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS = 2500; -const DEFAULT_CODE_AGENT_PROJECT_ID = "prj_hwpod_workbench"; -const DEFAULT_CODE_AGENT_PROVIDER_PROFILE = "deepseek"; -const DEFAULT_AGENTRUN_PROJECTION_RESUME_INITIAL_DELAY_MS = 5000; -const DEFAULT_AGENTRUN_PROJECTION_RESUME_INTERVAL_MS = 60000; -const DEFAULT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS = 30000; -const DEFAULT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE = 100; -const DEFAULT_AGENTRUN_PROJECTION_RESUME_JITTER_MS = 15000; -const DEFAULT_AGENTRUN_PROJECTION_RESUME_FAILURE_BACKOFF_MS = 120000; -const CODE_AGENT_TERMINAL_STATUSES = new Set(["completed", "failed", "blocked", "timeout", "cancelled", "canceled"]); -const CODE_AGENT_PROVIDER_PROFILE_ALIASES = Object.freeze({ codex: "codex-api" }); -const CODE_AGENT_PROVIDER_PROFILE_ID_PATTERN = /^[a-z0-9][a-z0-9.-]{0,63}$/u; -const CODE_AGENT_PROVIDER_PROFILE_LABELS = Object.freeze({ - "deepseek": "DeepSeek", - "codex-api": "Codex API", - "gpt.pika": "gpt.pika", - "minimax-m3": "MiniMax-M3 via AgentRun", - "runtime-default": "运行默认" -}); -const DEFAULT_CODE_AGENT_CODEX_API_MODEL = "gpt-5.5"; -const DEFAULT_CODE_AGENT_CODEX_API_BASE_URL = "http://127.0.0.1:49280/responses"; -const DEFAULT_CODE_AGENT_DEEPSEEK_MODEL = "deepseek-chat"; -const DEFAULT_CODE_AGENT_MINIMAX_M3_MODEL = "MiniMax-M3"; - -export async function handleCodeAgentChatHttp(request, response, options) { - const perf = options.backendPerformance; - const body = perf ? await perf.measure("body_read", () => readBody(request, options.bodyLimitBytes)) : await readBody(request, options.bodyLimitBytes); - let params = {}; - - try { - params = body ? JSON.parse(body) : {}; - } catch (error) { - sendJson(response, 400, { - ...createCodeAgentErrorPayload({ - code: "parse_error", - message: "Invalid JSON body", - reason: error.message, - traceId: getHeader(request, "x-trace-id") || "trc_unassigned", - model: options.env?.HWLAB_CODE_AGENT_MODEL || options.env?.OPENAI_MODEL || process.env.HWLAB_CODE_AGENT_MODEL || process.env.OPENAI_MODEL || "unknown", - layer: "api", - retryable: true - }) - }); - return; - } - - if (!params || typeof params !== "object" || Array.isArray(params)) { - sendJson(response, 400, { - ...createCodeAgentErrorPayload({ - code: "invalid_params", - message: "Code Agent chat body must be a JSON object", - traceId: getHeader(request, "x-trace-id") || "trc_unassigned", - model: options.env?.HWLAB_CODE_AGENT_MODEL || options.env?.OPENAI_MODEL || process.env.HWLAB_CODE_AGENT_MODEL || process.env.OPENAI_MODEL || "unknown", - layer: "api", - retryable: true - }) - }); - return; - } - params = withMdtodoExecutionPrompt(params); - - const traceId = safeTraceId(getHeader(request, "x-trace-id") || params.traceId) || `trc_${randomUUID()}`; - const lifecycle = codeAgentTurnLifecycleFields(traceId, params); - const chatParams = { - ...params, - traceId, - ...lifecycle, - ownerUserId: options.actor?.id ?? params.ownerUserId, - ownerRole: options.actor?.role ?? params.ownerRole - }; - const manualSession = perf ? await perf.measure("manual_session", () => prepareManualCodeAgentSession({ params: chatParams, options, traceId, response })) : await prepareManualCodeAgentSession({ params: chatParams, options, traceId, response }); - if (manualSession?.blocked) return; - const nativeSessionChatParams = stripSyntheticConversationContext(chatParams, traceId, options); - - if (codeAgentChatShortConnectionRequested(request, params, options)) { - if (perf) perf.recordPhase({ phase: "short_connection_accept", durationMs: 0, outcome: "ok" }); - let submitted = null; - try { - submitted = await submitCodeAgentChatTurn({ - params: nativeSessionChatParams, - options, - traceId - }); - } catch (error) { - if (isCodeAgentAdmissionUnavailableError(error)) { - sendJson(response, error.statusCode ?? 503, error.payload); - return; - } - throw error; - } - const traceUrl = `/v1/agent/traces/${encodeURIComponent(traceId)}`; - const turnUrl = `/v1/agent/turns/${encodeURIComponent(traceId)}`; - const statusCode = 202; - const admittedTiming = codeAgentAdmissionTimingFields(submitted?.createdAt ?? submitted?.updatedAt); - void emitCodeAgentOtelSpan("POST /v1/agent/chat", traceId, options.env ?? process.env, { - kind: 2, - attributes: { "http.method": "POST", "http.route": "/v1/agent/chat", "http.status_code": statusCode, sessionId: safeSessionId(nativeSessionChatParams.sessionId) || null, turnId: lifecycle.turnId, ...agentChatOtelAttributes(nativeSessionChatParams) } - }); - sendJson(response, 202, { - accepted: true, - status: "running", - shortConnection: true, - controlSemantics: "submit-and-poll", - traceId, - ...codeAgentOtelTraceFields(traceId, options.env ?? process.env), - ...lifecycle, - sessionId: safeSessionId(nativeSessionChatParams.sessionId) || null, - createdAt: submitted?.createdAt ?? admittedTiming.createdAt, - updatedAt: submitted?.updatedAt ?? admittedTiming.updatedAt, - timing: submitted?.timing ?? admittedTiming.timing, - startedAt: submitted?.startedAt ?? admittedTiming.startedAt, - lastEventAt: submitted?.lastEventAt ?? admittedTiming.lastEventAt, - finishedAt: submitted?.finishedAt ?? admittedTiming.finishedAt, - durationMs: submitted?.durationMs ?? admittedTiming.durationMs, - traceUrl, - resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`, - turnUrl, - streamUrl: `${traceUrl}/stream`, - cancelUrl: "/v1/agent/chat/cancel", - polling: { - resultIntervalMs: parsePositiveInteger(options.env?.HWLAB_CODE_AGENT_RESULT_POLL_INTERVAL_MS, 1000), - traceIntervalMs: parsePositiveInteger(options.env?.HWLAB_CODE_AGENT_TRACE_POLL_INTERVAL_MS, 1000) - }, - runnerTrace: (options.traceStore ?? defaultCodeAgentTraceStore).snapshot(traceId) - }); - return; - } - - const admittedBase = codeAgentAdmittedFailureBasePayload({ params: nativeSessionChatParams, options, traceId }); - try { - await recordCodeAgentTurnAdmission({ payload: admittedBase, params: nativeSessionChatParams, options, traceId }); - } catch (error) { - if (isCodeAgentAdmissionUnavailableError(error)) { - sendJson(response, error.statusCode ?? 503, error.payload); - return; - } - throw error; - } - const billingPreflight = perf ? await perf.measure("billing_preflight", () => preflightCodeAgentBilling({ params: nativeSessionChatParams, options, traceId, response: null, sendResponse: false })) : await preflightCodeAgentBilling({ params: nativeSessionChatParams, options, traceId, response: null, sendResponse: false }); - if (billingPreflight?.blocked) { - const failed = await settleAdmittedCodeAgentFailure({ - base: admittedBase, - params: nativeSessionChatParams, - options, - traceId, - results: options.codeAgentChatResults, - failure: billingPreflight, - traceLabel: "billing:preflight:failed" - }); - sendJson(response, billingPreflight.statusCode ?? 503, failed); - return; - } - const nativeSessionChatParamsWithBilling = { ...nativeSessionChatParams, userBillingReservation: billingPreflight?.reservation ?? null }; - const payload = perf ? await perf.measure("code_agent_run", () => runCodeAgentChat(nativeSessionChatParamsWithBilling, options)) : await runCodeAgentChat(nativeSessionChatParamsWithBilling, options); - await (perf ? perf.measure("billing_finalize", () => finalizeCodeAgentBillingUsage({ payload, params: nativeSessionChatParamsWithBilling, options })) : finalizeCodeAgentBillingUsage({ payload, params: nativeSessionChatParamsWithBilling, options })); - await (perf ? perf.measure("session_owner_record", () => recordCodeAgentSessionOwner({ payload, params: nativeSessionChatParamsWithBilling, options, status: codeAgentOwnerStatusForResult(payload) })) : recordCodeAgentSessionOwner({ payload, params: nativeSessionChatParamsWithBilling, options, status: codeAgentOwnerStatusForResult(payload) })); - const responsePayload = annotateOwner(payload, nativeSessionChatParamsWithBilling); - - sendJson(response, responsePayload.status === "failed" && responsePayload.error?.code === "invalid_params" ? 400 : 200, responsePayload); -} - -export async function handleCodeAgentSessionsHttp(request, response, url, options) { - const match = url.pathname.match(/^\/v1\/agent\/sessions(?:\/([^/]+)(?:\/(select))?)?$/u); - const sessionId = safeSessionId(match?.[1] ? decodeURIComponent(match[1]) : ""); - const action = match?.[2] ?? null; - if (!match) { - sendJson(response, 404, manualSessionErrorPayload({ code: "session_route_not_found", message: "Code Agent session route is not implemented." })); - return; - } - if (request.method === "GET" && !sessionId) { - await listManualCodeAgentSessions(request, response, url, options); - return; - } - if (request.method === "POST" && !sessionId) { - await createManualCodeAgentSession(request, response, options); - return; - } - if (request.method === "GET" && sessionId && !action) { - await getManualCodeAgentSession(response, options, sessionId); - return; - } - if (request.method === "DELETE" && sessionId && !action) { - await archiveManualCodeAgentSession(response, options, sessionId); - return; - } - if (request.method === "POST" && sessionId && action === "select") { - await selectManualCodeAgentSession(request, response, options, sessionId); - return; - } - sendJson(response, 405, manualSessionErrorPayload({ code: "session_method_not_allowed", message: "Unsupported Code Agent session method." })); -} - -async function listManualCodeAgentSessions(request, response, url, options) { - const projectId = textValue(url.searchParams.get("projectId")) || DEFAULT_CODE_AGENT_PROJECT_ID; - const limit = parsePositiveInteger(url.searchParams.get("limit"), 20); - const sessions = await options.accessController?.store?.listAgentSessionsForUser?.({ - ownerUserId: options.actor?.id, - actorRole: options.actor?.role, - ownerScoped: true, - projectId, - limit - }) ?? []; - sendJson(response, 200, { - ok: true, - status: "succeeded", - contractVersion: "code-agent-manual-session-v1", - projectId, - sessions: sessions.map(publicManualAgentSession), - count: sessions.length, - valuesRedacted: true, - secretMaterialStored: false - }); -} - -async function createManualCodeAgentSession(request, response, options) { - const body = await readJsonObjectBody(request, options.bodyLimitBytes); - if (!body.ok) { - sendJson(response, 400, manualSessionErrorPayload({ code: body.code, message: body.message, reason: body.reason })); - return; - } - const params = body.value; - const projectId = textValue(params.projectId) || DEFAULT_CODE_AGENT_PROJECT_ID; - const conversationId = safeConversationId(params.conversationId) || `cnv_${randomUUID()}`; - const sessionId = safeSessionId(params.sessionId) || `ses_${randomUUID()}`; - const providerProfile = normalizeCodeAgentProviderProfile(params.providerProfile ?? params.codeAgentProviderProfile); - const now = new Date().toISOString(); - const session = await options.accessController.recordAgentSessionOwner({ - ownerUserId: options.actor?.id, - ownerRole: options.actor?.role, - sessionId, - projectId, - agentId: "hwlab-code-agent", - status: "idle", - conversationId, - threadId: null, - traceId: null, - session: { - source: "manual-session-create", - providerProfile, - sessionStatus: "idle", - createdAt: now, - valuesRedacted: true, - secretMaterialStored: false - } - }); - await writeWorkbenchSessionAdmissionFact({ - runtimeStore: options.runtimeStore, - session, - ownerUserId: options.actor?.id, - ownerRole: options.actor?.role, - projectId, - conversationId, - threadId: null, - status: "idle", - now - }); - sendJson(response, 201, { - ok: true, - status: "created", - contractVersion: "code-agent-manual-session-v1", - session: publicManualAgentSession(session), - nextCommands: manualSessionNextCommands(session), - valuesRedacted: true, - secretMaterialStored: false - }); -} - -async function getManualCodeAgentSession(response, options, sessionId) { - const session = await getVisibleManualAgentSession(options, sessionId); - if (!session) { - sendJson(response, 404, manualSessionErrorPayload({ code: "session_not_found", message: "Code Agent session is not visible to the current actor.", sessionId })); - return; - } - sendJson(response, 200, { - ok: true, - status: "found", - contractVersion: "code-agent-manual-session-v1", - session: publicManualAgentSession(session), - valuesRedacted: true, - secretMaterialStored: false - }); -} - -async function archiveManualCodeAgentSession(response, options, sessionId) { - const session = await getVisibleManualAgentSession(options, sessionId); - if (!session) { - sendJson(response, 404, manualSessionErrorPayload({ code: "session_not_found", message: "Code Agent session is not visible to the current actor.", sessionId })); - return; - } - const result = await options.accessController?.store?.archiveAgentConversation?.({ - ownerUserId: options.actor?.id, - actorRole: options.actor?.role, - ownerScoped: true, - sessionId, - conversationId: session.conversationId, - projectId: session.projectId - }) ?? { count: 0 }; - sendJson(response, 200, { - ok: true, - status: "archived", - contractVersion: "code-agent-manual-session-v1", - archivedCount: result.count ?? 0, - session: publicManualAgentSession({ ...session, status: "archived", endedAt: session.endedAt ?? new Date().toISOString(), updatedAt: new Date().toISOString() }), - valuesRedacted: true, - secretMaterialStored: false - }); -} - -async function selectManualCodeAgentSession(request, response, options, sessionId) { - const body = await readJsonObjectBody(request, options.bodyLimitBytes); - if (!body.ok) { - sendJson(response, 400, manualSessionErrorPayload({ code: body.code, message: body.message, reason: body.reason, sessionId })); - return; - } - const session = await getVisibleManualAgentSession(options, sessionId); - if (!session) { - sendJson(response, 404, manualSessionErrorPayload({ code: "session_not_found", message: "Code Agent session is not visible to the current actor.", sessionId })); - return; - } - if (!manualSessionUsable(session.status)) { - sendJson(response, 409, manualSessionErrorPayload({ code: "session_not_usable", message: `Code Agent session ${sessionId} is ${session.status}; create a new session before continuing.`, sessionId, session })); - return; - } - sendJson(response, 200, { - ok: true, - status: "selected", - contractVersion: "code-agent-manual-session-v1", - session: publicManualAgentSession(session), - nextCommands: manualSessionNextCommands(session), - valuesRedacted: true, - secretMaterialStored: false - }); -} - -async function prepareManualCodeAgentSession({ params, options, traceId, response }) { - if (!options.actor?.id) { - sendJson(response, 401, manualSessionErrorPayload({ - code: "auth_required", - message: "Authentication is required before using an explicit HWLAB Code Agent session.", - traceId, - retryable: true - })); - return { blocked: true }; - } - const sessionId = safeSessionId(params.sessionId); - if (!sessionId) { - sendJson(response, 409, manualSessionErrorPayload({ - code: "session_required", - message: "Code Agent session is explicit in HWLAB v0.2; create/select a session before sending a turn.", - traceId, - retryable: true, - nextCommands: ["hwlab-cli client agent session create --provider-profile PROFILE", "hwlab-cli client agent send --session-id --message TEXT"] - })); - return { blocked: true }; - } - const session = await getVisibleManualAgentSession(options, sessionId); - if (!session) { - sendJson(response, 404, manualSessionErrorPayload({ code: "session_not_found", message: "Code Agent session is not visible to the current actor.", traceId, sessionId })); - return { blocked: true }; - } - const freshContinuation = manualSessionFreshContinuationRequired(session); - if (!manualSessionUsable(session.status) && !freshContinuation) { - sendJson(response, 409, manualSessionErrorPayload({ code: "session_not_usable", message: `Code Agent session ${sessionId} is ${session.status}; create a new session before continuing.`, traceId, sessionId, session })); - return { blocked: true }; - } - const requestedProjectId = textValue(params.projectId); - const sessionProjectId = textValue(session.projectId); - if (requestedProjectId && sessionProjectId && requestedProjectId !== sessionProjectId) { - sendJson(response, 409, manualSessionErrorPayload({ code: "session_project_mismatch", message: `sessionId ${sessionId} belongs to ${sessionProjectId}; requested ${requestedProjectId}.`, traceId, sessionId, session })); - return { blocked: true }; - } - const storedConversationId = safeConversationId(session.conversationId); - const requestedConversationId = safeConversationId(params.conversationId); - if (requestedConversationId && storedConversationId && requestedConversationId !== storedConversationId) { - sendJson(response, 409, manualSessionErrorPayload({ code: "session_conversation_mismatch", message: `sessionId ${sessionId} is bound to ${storedConversationId}; requested ${requestedConversationId}.`, traceId, sessionId, session })); - return { blocked: true }; - } - const conversationId = requestedConversationId || storedConversationId; - if (!conversationId) { - sendJson(response, 409, manualSessionErrorPayload({ code: "session_conversation_required", message: "Code Agent session has no bound conversationId; create a new session before continuing.", traceId, sessionId, session })); - return { blocked: true }; - } - params.conversationId = conversationId; - params.sessionId = sessionId; - params.projectId = requestedProjectId || sessionProjectId || DEFAULT_CODE_AGENT_PROJECT_ID; - applyManualSessionProviderProfile(params, session); - if (freshContinuation) { - params.threadId = undefined; - params.forceFreshAgentRunSession = true; - params.forceFreshAgentRunSessionReason = manualSessionFreshContinuationReason(session); - } else { - params.threadId = safeOpaqueId(params.threadId) || manualSessionContinuationThreadId(session) || undefined; - } - return { session }; -} - -function applyManualSessionProviderProfile(params = {}, session = null) { - const requested = firstNonEmptyValue(params.providerProfile, params.codeAgentProviderProfile); - if (requested) { - const normalized = normalizeCodeAgentProviderProfile(requested); - params.providerProfile = normalized; - params.codeAgentProviderProfile = normalized; - params.providerProfileSource = "request"; - return normalized; - } - const stored = manualSessionStoredProviderProfile(session); - if (!stored) return null; - params.providerProfile = stored; - params.codeAgentProviderProfile = stored; - params.providerProfileSource = "session"; - return stored; -} - -function manualSessionStoredProviderProfile(session = null) { - const evidence = session?.session && typeof session.session === "object" ? session.session : {}; - const traceProfile = latestCompletedTraceResultProviderProfile(evidence.traceResults); - const raw = firstNonEmptyValue( - traceProfile, - evidence.agentRun?.backendProfile, - evidence.providerProfile, - evidence.codeAgentProviderProfile, - evidence.backendProfile - ); - if (!raw) return null; - const normalized = normalizeCodeAgentProviderProfile(raw); - return normalized === "runtime-default" ? null : normalized; -} - -function latestCompletedTraceResultProviderProfile(traceResults = null) { - if (!traceResults || typeof traceResults !== "object") return null; - const entries = Object.values(traceResults).filter((value) => value && typeof value === "object"); - for (const result of entries.reverse()) { - const agentRun = result.agentRun && typeof result.agentRun === "object" ? result.agentRun : null; - if (!agentRun) continue; - const status = normalizeTurnStatus(result.status ?? agentRun.terminalStatus ?? agentRun.commandState ?? agentRun.status); - if (status && status !== "completed") continue; - const profile = firstNonEmptyValue(agentRun.backendProfile, result.providerProfile, result.codeAgentProviderProfile); - if (profile) return profile; - } - return null; -} - -function manualSessionFreshContinuationRequired(session = null) { - const status = String(session?.status ?? "").trim().toLowerCase().replace(/_/gu, "-"); - if (status === "thread-resume-failed") return true; - const continuation = session?.session?.continuation && typeof session.session.continuation === "object" ? session.session.continuation : null; - return continuation?.requiresFreshSession === true; -} - -function manualSessionFreshContinuationReason(session = null) { - const continuation = session?.session?.continuation && typeof session.session.continuation === "object" ? session.session.continuation : null; - return firstNonEmptyValue(continuation?.reasonCode, continuation?.reason, session?.status, "stale-continuation") ?? "stale-continuation"; -} - -function manualSessionContinuationThreadId(session = null) { - if (manualSessionFreshContinuationRequired(session)) return null; - const evidence = session?.session && typeof session.session === "object" ? session.session : {}; - return safeOpaqueId(evidence.threadId) || safeOpaqueId(evidence.agentRun?.threadId) || safeOpaqueId(session?.threadId) || null; -} - -async function getVisibleManualAgentSession(options, sessionId) { - if (!safeSessionId(sessionId) || !options.actor?.id || !options.accessController?.getAgentSession) return null; - const session = await options.accessController.getAgentSession(sessionId); - if (!session) return null; - if (session.ownerUserId === options.actor.id) return session; - return null; -} - -function publicManualAgentSession(session) { - if (!session || typeof session !== "object") return null; - return { - sessionId: session.id, - agentId: session.agentId ?? "hwlab-code-agent", - status: session.status ?? null, - usable: manualSessionUsable(session.status), - threadId: safeOpaqueId(session.threadId) || null, - lastTraceId: safeTraceId(session.lastTraceId) || null, - providerProfile: textValue(session.session?.providerProfile) || null, - createdAt: session.startedAt ?? null, - updatedAt: session.updatedAt ?? null, - valuesRedacted: true, - secretMaterialStored: false - }; -} - -function manualSessionUsable(status) { - const value = String(status ?? "").trim().toLowerCase().replace(/_/gu, "-"); - return !["blocked", "expired", "stale"].includes(value); -} - -function manualSessionErrorPayload({ code, message, reason = null, traceId = null, sessionId = null, session = null, retryable = true, nextCommands = undefined } = {}) { - return { - ok: false, - accepted: false, - status: "blocked", - traceId: safeTraceId(traceId) || null, - sessionId: safeSessionId(sessionId) || safeSessionId(session?.id) || null, - session: session ? publicManualAgentSession(session) : null, - error: { - code, - layer: "api", - category: "session_blocked", - retryable, - message, - reason, - userMessage: message, - route: "/v1/agent/chat" - }, - blocker: { - code, - layer: "api", - retryable, - summary: message - }, - nextCommands, - valuesRedacted: true, - secretMaterialStored: false - }; -} - -function manualSessionNextCommands(session) { - const sessionId = session?.id ?? ""; - return [ - `hwlab-cli client agent send --session-id ${sessionId} --message TEXT`, - `hwlab-cli client agent session status ${sessionId}` - ]; -} - -async function readJsonObjectBody(request, bodyLimitBytes) { - const body = await readBody(request, bodyLimitBytes); - try { - const value = body ? JSON.parse(body) : {}; - if (!value || typeof value !== "object" || Array.isArray(value)) return { ok: false, code: "invalid_params", message: "body must be a JSON object" }; - return { ok: true, value }; - } catch (error) { - return { ok: false, code: "parse_error", message: "Invalid JSON body", reason: error.message }; - } -} - -async function runCodeAgentChat(params, options) { - return handleCodeAgentChat(params, await codeAgentChatExecutionOptions(options, params)); -} - -function stripSyntheticConversationContext(params = {}, traceId, options = {}) { - if (Array.isArray(params.conversationContext?.messages) || Array.isArray(params.messages)) { - (options.traceStore ?? defaultCodeAgentTraceStore).append(traceId, { - type: "synthetic-context", - status: "ignored", - label: "synthetic-context:ignored", - message: "Synthetic conversation context was stripped; Code Agent continuation uses only Codex stdio native thread/resume and turn/start.", - valuesPrinted: false - }); - } - const { conversationContext, conversationContextTruncated, contextSource, messages, ...nativeParams } = params; - return nativeParams; -} - -async function codeAgentChatExecutionOptions(options = {}, params = {}) { - const env = codeAgentProviderProfileEnv(options.env ?? process.env, params); - Object.assign(env, await codeAgentAuthEnv(options, env, params)); - return { - callProvider: options.callCodeAgentProvider, - timeoutMs: options.codeAgentTimeoutMs, - hardTimeoutMs: options.codeAgentHardTimeoutMs, - env, - workspace: options.workspace, - skillsDirs: options.skillsDirs, - skillsDirsExact: options.skillsDirsExact, - skillsDiscoveryDelayMs: options.skillsDiscoveryDelayMs, - sessionRegistry: options.sessionRegistry, - m3IoSkillRequestJson: options.m3IoSkillRequestJson ?? createCodeAgentM3HwlabApiRequestJson(options), - codexStdioManager: options.codexStdioManager, - traceStore: options.traceStore, - gatewayRegistry: options.gatewayRegistry - }; -} - -async function codeAgentAuthEnv(options = {}, env = process.env, params = {}) { - const ownerUserId = textValue(params.ownerUserId ?? options.actor?.id ?? ""); - const store = options.accessController?.store; - const key = ownerUserId && store?.findActiveDefaultApiKeyForUser - ? await store.findActiveDefaultApiKeyForUser(ownerUserId) - : null; - const envRuntimeLane = runtimeLaneForEnv(env); - const runtimeNamespace = runtimeNamespaceForEnv(env); - const namespaceRuntimeLane = runtimeLaneForNamespace(runtimeNamespace); - const runtimeLane = resolveRuntimeLaneAuthority({ envRuntimeLane, runtimeNamespace, namespaceRuntimeLane }); - const inClusterApiUrl = internalCodeAgentApiUrl(env, runtimeNamespace); - const inClusterWebUrl = internalCodeAgentWebUrl(env, runtimeNamespace); - const apiUrl = firstNonEmptyValue( - codeAgentAgentRunAdapterEnabled(env) ? inClusterApiUrl : null, - env.HWLAB_RUNTIME_API_URL, - sameRuntimeEndpoint(env.HWLAB_CLOUD_API_URL, runtimeNamespace, runtimeLane), - env.HWLAB_PUBLIC_ENDPOINT, - localCloudApiUrl(env) - ); - return { - HWLAB_CLOUD_API_URL: apiUrl, - HWLAB_RUNTIME_API_URL: apiUrl, - HWLAB_RUNTIME_WEB_URL: firstNonEmptyValue(codeAgentAgentRunAdapterEnabled(env) ? inClusterWebUrl : null, env.HWLAB_RUNTIME_WEB_URL, apiUrl), - HWLAB_RUNTIME_NAMESPACE: runtimeNamespace, - HWLAB_RUNTIME_LANE: runtimeLane, - HWLAB_RUNTIME_ENDPOINT_SOURCE: codeAgentAgentRunAdapterEnabled(env) ? "runtime-namespace" : "runtime-env", - HWLAB_RUNTIME_ENDPOINT_LOCKED: "1", - HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME: "1", - ...(options.actorApiKeySecret ? { HWLAB_API_KEY: options.actorApiKeySecret } : key?.displaySecret ? { HWLAB_API_KEY: key.displaySecret } : {}) - }; -} - -function internalCodeAgentApiUrl(env = process.env, namespace = runtimeNamespaceForEnv(env)) { - const serviceName = firstNonEmptyValue(env.HWLAB_CODE_AGENT_INTERNAL_API_SERVICE_NAME, env.HWLAB_CLOUD_API_SERVICE_NAME, "hwlab-cloud-api"); - const port = parsePositiveInteger(firstNonEmptyValue(env.HWLAB_CODE_AGENT_INTERNAL_API_PORT, env.HWLAB_CLOUD_API_PORT), 6667); - return `http://${serviceName}.${namespace}.svc.cluster.local:${port}`; -} - -function internalCodeAgentWebUrl(env = process.env, namespace = runtimeNamespaceForEnv(env)) { - const serviceName = firstNonEmptyValue(env.HWLAB_CODE_AGENT_INTERNAL_WEB_SERVICE_NAME, env.HWLAB_CLOUD_WEB_SERVICE_NAME, "hwlab-cloud-web"); - const port = parsePositiveInteger(firstNonEmptyValue(env.HWLAB_CODE_AGENT_INTERNAL_WEB_PORT, env.HWLAB_CLOUD_WEB_PORT), 8080); - return `http://${serviceName}.${namespace}.svc.cluster.local:${port}`; -} - -function runtimeNamespaceForEnv(env = process.env) { - return firstNonEmptyValue( - env.HWLAB_RUNTIME_NAMESPACE, - env.POD_NAMESPACE, - env.HWLAB_NAMESPACE, - defaultRuntimeNamespace(env) - ); -} - -function runtimeLaneForEnv(env = process.env) { - const raw = firstNonEmptyValue(env.HWLAB_RUNTIME_LANE, env.HWLAB_GITOPS_LANE, env.HWLAB_GITOPS_PROFILE, env.HWLAB_ENVIRONMENT, env.HWLAB_BOOT_REF); - if (!raw) return null; - const lane = normalizeRuntimeLane(raw); - if (!lane) { - throw runtimeAuthorityError("unsupported_runtime_lane", `unsupported HWLAB runtime lane: ${String(raw).trim()}`); - } - return lane; -} - -function runtimeLaneForNamespace(namespace) { - const value = String(namespace ?? "").trim().toLowerCase(); - if (value === "hwlab-prod") return "prod"; - if (value === "hwlab-dev") return "dev"; - const versionMatch = value.match(/^hwlab-v(\d+)$/u); - if (versionMatch) return versionRuntimeLane(versionMatch[1]); - return null; -} - -function sameRuntimeEndpoint(value, namespace, lane) { - const text = firstNonEmptyValue(value); - if (!text) return null; - const endpointLane = runtimeLaneForEndpoint(text); - const endpointNamespace = runtimeNamespaceForEndpoint(text); - if (namespace && endpointNamespace && endpointNamespace !== namespace) return null; - if (lane && endpointLane && endpointLane !== lane) return null; - return text; -} - -function runtimeNamespaceForEndpoint(value) { - const match = String(value ?? "").match(/\.((?:hwlab-[a-z0-9-]+))\.svc\.cluster\.local(?::|\/|$)/iu); - return match?.[1] ?? null; -} - -function runtimeLaneForEndpoint(value) { - const text = String(value ?? "").toLowerCase(); - const namespaceLane = runtimeLaneForNamespace(runtimeNamespaceForEndpoint(text)); - if (namespaceLane) return namespaceLane; - if (/hwlab-prod|:18666(?:\/|$)|:18667(?:\/|$)/u.test(text)) return "prod"; - if (/hwlab-dev|:17666(?:\/|$)|:17667(?:\/|$)|:16666(?:\/|$)|:16667(?:\/|$)/u.test(text)) return "dev"; - return null; -} - -function normalizeRuntimeLane(value) { - const profile = String(value ?? "").trim().toLowerCase(); - if (!profile) return null; - if (profile === "prod" || profile === "production") return "prod"; - if (profile === "dev" || profile === "development" || profile === "local") return "dev"; - const dotted = profile.match(/^v?0+\.(\d+)$/u); - if (dotted) return versionRuntimeLane(dotted[1]); - const compact = profile.match(/^v?0*(\d+)$/u); - if (compact) return versionRuntimeLane(compact[1]); - return null; -} - -function versionRuntimeLane(value) { - const version = Number.parseInt(value, 10); - if (!Number.isFinite(version) || version <= 0) return null; - return `v${String(version).padStart(2, "0")}`; -} - -function resolveRuntimeLaneAuthority({ envRuntimeLane, runtimeNamespace, namespaceRuntimeLane }) { - if (!namespaceRuntimeLane) { - throw runtimeAuthorityError("unsupported_runtime_namespace", `unsupported HWLAB runtime namespace: ${runtimeNamespace}`); - } - if (envRuntimeLane && envRuntimeLane !== namespaceRuntimeLane) { - throw runtimeAuthorityError( - "runtime_authority_mismatch", - `HWLAB runtime lane ${envRuntimeLane} does not match namespace ${runtimeNamespace} (${namespaceRuntimeLane})` - ); - } - return namespaceRuntimeLane; -} - -function runtimeAuthorityError(code, message) { - return Object.assign(new Error(message), { - code, - statusCode: 500, - valuesPrinted: false - }); -} - -function defaultRuntimeNamespace(env = process.env) { - const profile = runtimeLaneForEnv(env) ?? "dev"; - if (profile === "prod") return "hwlab-prod"; - if (profile === "dev") return "hwlab-dev"; - return `hwlab-${profile}`; -} - -function localCloudApiUrl(env = process.env) { - const port = parsePositiveInteger(env.HWLAB_CLOUD_API_PORT, 6667); - return `http://127.0.0.1:${port}`; -} - -function codeAgentProviderProfileEnv(env = process.env, params = {}) { - const profile = normalizeCodeAgentProviderProfile(params.providerProfile ?? params.codeAgentProviderProfile ?? params.provider ?? env.HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE); - if (profile === "runtime-default") return env; - const overlay = { ...env }; - overlay.HWLAB_CODE_AGENT_SELECTED_PROVIDER_PROFILE = profile; - overlay.HWLAB_CODE_AGENT_PROVIDER_PROFILE_LABEL = CODE_AGENT_PROVIDER_PROFILE_LABELS[profile] ?? profile; - overlay.HWLAB_CODE_AGENT_PROVIDER = "codex-stdio"; - if (profile === "codex-api") { - overlay.HWLAB_CODE_AGENT_MODEL = firstNonEmptyValue(env.HWLAB_CODE_AGENT_CODEX_API_MODEL, env.HWLAB_CODE_AGENT_LEGACY_CODEX_MODEL, DEFAULT_CODE_AGENT_CODEX_API_MODEL); - overlay.HWLAB_CODE_AGENT_OPENAI_BASE_URL = firstNonEmptyValue(env.HWLAB_CODE_AGENT_CODEX_API_BASE_URL, env.HWLAB_CODE_AGENT_LEGACY_OPENAI_BASE_URL, DEFAULT_CODE_AGENT_CODEX_API_BASE_URL); - } else if (profile === "minimax-m3") { - overlay.HWLAB_CODE_AGENT_MODEL = firstNonEmptyValue(env.HWLAB_CODE_AGENT_MINIMAX_M3_MODEL, DEFAULT_CODE_AGENT_MINIMAX_M3_MODEL); - } else { - const dynamicProfile = profile !== "deepseek"; - overlay.HWLAB_CODE_AGENT_MODEL = dynamicProfile - ? firstNonEmptyValue(env[`HWLAB_CODE_AGENT_${providerProfileEnvKey(profile)}_MODEL`], profile) - : firstNonEmptyValue(env.HWLAB_CODE_AGENT_DEEPSEEK_MODEL, DEFAULT_CODE_AGENT_DEEPSEEK_MODEL); - if (!dynamicProfile || !codeAgentAgentRunAdapterEnabled(env)) { - overlay.HWLAB_CODE_AGENT_OPENAI_BASE_URL = firstNonEmptyValue(env.HWLAB_CODE_AGENT_DEEPSEEK_BASE_URL, defaultDeepSeekBaseUrlForEnv(env)); - } - } - overlay.NO_PROXY = mergeNoProxyEntries(env.NO_PROXY, codeAgentClusterNoProxyEntries(env)); - overlay.no_proxy = mergeNoProxyEntries(env.no_proxy ?? env.NO_PROXY, codeAgentClusterNoProxyEntries(env)); - return overlay; -} - -function normalizeCodeAgentProviderProfile(value) { - const text = String(value ?? DEFAULT_CODE_AGENT_PROVIDER_PROFILE).trim().toLowerCase(); - if (!text) return DEFAULT_CODE_AGENT_PROVIDER_PROFILE; - if (text === "runtime-default") return text; - const normalized = CODE_AGENT_PROVIDER_PROFILE_ALIASES[text] ?? text; - return CODE_AGENT_PROVIDER_PROFILE_ID_PATTERN.test(normalized) ? normalized : DEFAULT_CODE_AGENT_PROVIDER_PROFILE; -} - -function providerProfileEnvKey(profile) { - return String(profile ?? "") - .trim() - .toUpperCase() - .replace(/[^A-Z0-9]+/gu, "_") - .replace(/^_+|_+$/gu, "") || "PROFILE"; -} - -function defaultDeepSeekBaseUrlForEnv(env = process.env) { - const namespace = env.HWLAB_GITOPS_PROFILE === "prod" || env.HWLAB_ENVIRONMENT === "prod" ? "hwlab-prod" : "hwlab-dev"; - return `http://hwlab-deepseek-proxy.${namespace}.svc.cluster.local:4000/v1/responses`; -} - -function codeAgentClusterNoProxyEntries(env = process.env) { - const namespace = env.HWLAB_GITOPS_PROFILE === "prod" || env.HWLAB_ENVIRONMENT === "prod" ? "hwlab-prod" : "hwlab-dev"; - return [ - `${namespace}.svc.cluster.local`, - ".svc", - ".cluster.local", - "127.0.0.1", - "localhost", - "::1", - "10.0.0.0/8", - "10.42.0.0/16", - "10.43.0.0/16", - "hyueapi.com", - ".hyueapi.com", - "hyui.com", - ".hyui.com", - "api.minimaxi.com", - ".minimaxi.com" - ]; -} - -function mergeNoProxyEntries(current, additions = []) { - return uniqueStrings([...String(current ?? "").split(","), ...additions]).join(","); -} - -function firstNonEmptyValue(...values) { - for (const value of values) { - const text = String(value ?? "").trim(); - if (text) return text; - } - return null; -} - -function safeOtelConfigText(value) { - const text = firstNonEmptyValue(value); - return text === null ? null : text.slice(0, 120); -} - -function safeOtelUrlHost(value) { - const text = firstNonEmptyValue(value); - if (text === null) return null; - try { - return new URL(text).host.slice(0, 120); - } catch { - return null; - } -} - -function withMdtodoExecutionPrompt(params = {}) { - const execution = mdtodoExecutionContextFromParams(params); - if (!execution?.hwpodId || !execution?.hwpodWorkspaceArgs) return params; - const message = firstNonEmptyValue(params.message, params.prompt, params.text); - if (!message || /hwpodWorkspaceArgs\s*:/u.test(message)) return params; - const taskRef = firstNonEmptyValue(params.taskRef, execution.taskRef) ?? "-"; - const mdtodoRootRef = execution.mdtodoRootRef ?? "docs/MDTODO"; - const prefix = [ - "# MDTODO HWPOD 执行上下文", - `TaskRef: ${taskRef}`, - `sourceId: ${execution.sourceId ?? "-"}`, - `hwpodId: ${execution.hwpodId}`, - `nodeId: ${execution.nodeId ?? "-"}`, - `mdtodoRootRef: ${mdtodoRootRef}`, - `hwpodWorkspaceArgs: ${execution.hwpodWorkspaceArgs}`, - "所有 hwpod/hwpod-ctl 命令都必须携带上述 hwpodWorkspaceArgs。读取 MDTODO、报告和工程文件必须通过 HWPOD workspace/node-ops 入口，不得猜测或访问普通容器本地路径，也不得创建本地 .hwlab/hwpod-spec.yaml fallback。" - ].join("\n"); - const nextMessage = `${prefix}\n\n${message}`; - return { - ...params, - message: nextMessage, - prompt: nextMessage, - launchContext: { - ...(params.launchContext && typeof params.launchContext === "object" ? params.launchContext : {}), - hwpodId: execution.hwpodId, - nodeId: execution.nodeId, - mdtodoRootRef: execution.mdtodoRootRef, - hwpodWorkspaceArgs: execution.hwpodWorkspaceArgs, - contextFingerprint: execution.contextFingerprint, - executionContext: execution, - valuesRedacted: true - } - }; -} - -function mdtodoExecutionContextFromParams(params = {}) { - const launchContext = params.launchContext && typeof params.launchContext === "object" && !Array.isArray(params.launchContext) ? params.launchContext : null; - const execution = launchContext?.executionContext && typeof launchContext.executionContext === "object" && !Array.isArray(launchContext.executionContext) ? launchContext.executionContext : launchContext; - if (!execution || typeof execution !== "object") return null; - return { - sourceId: safeOtelConfigText(execution.sourceId ?? launchContext?.sourceId), - sourceKind: safeOtelConfigText(execution.sourceKind ?? launchContext?.sourceKind), - projectId: safeOtelConfigText(execution.projectId ?? launchContext?.projectId ?? params.projectId), - taskRef: safeOtelText(execution.taskRef ?? launchContext?.taskRef ?? params.taskRef), - fileRef: safeOtelConfigText(execution.fileRef ?? launchContext?.fileRef), - relativePath: safeOtelText(execution.relativePath ?? launchContext?.relativePath), - taskId: safeOtelConfigText(execution.taskId ?? launchContext?.taskId), - hwpodId: safeOtelText(execution.hwpodId ?? launchContext?.hwpodId), - nodeId: safeOtelText(execution.nodeId ?? launchContext?.nodeId), - mdtodoRootRef: safeOtelText(execution.mdtodoRootRef ?? launchContext?.mdtodoRootRef), - workspaceRootHash: safeOtelText(execution.workspaceRootHash ?? launchContext?.workspaceRootHash), - hwpodWorkspaceArgs: safeOtelText(execution.hwpodWorkspaceArgs ?? launchContext?.hwpodWorkspaceArgs, 500), - contextFingerprint: safeOtelText(execution.contextFingerprint ?? launchContext?.contextFingerprint), - valuesRedacted: true - }; -} - -function agentChatOtelAttributes(params = {}) { - const execution = mdtodoExecutionContextFromParams(params); - return { - "agent.chat.session_id": safeSessionId(params.sessionId) || null, - "agent.chat.provider_profile": safeOtelConfigText(params.providerProfile ?? params.codeAgentProviderProfile), - "agent.chat.provider_profile_source": safeOtelConfigText(params.providerProfileSource), - "agent.chat.task_ref": safeOtelText(params.taskRef ?? execution?.taskRef), - "agent.chat.source_id": safeOtelConfigText(execution?.sourceId ?? params.sourceId), - "agent.chat.hwpod_id": safeOtelText(execution?.hwpodId), - "agent.chat.workspace_root_hash": safeOtelText(execution?.workspaceRootHash), - "agent.chat.context_fingerprint": safeOtelText(execution?.contextFingerprint) - }; -} - -function safeOtelText(value, max = 360) { - const text = firstNonEmptyValue(value); - return text === null ? null : text.slice(0, max); -} - -async function submitCodeAgentChatTurn({ params, options, traceId }) { - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - const results = options.codeAgentChatResults ?? createCodeAgentChatResultStore(); - const timestamp = new Date().toISOString(); - const runtimeEnv = options.env ?? process.env; - const adapterEnabled = codeAgentAgentRunAdapterEnabled(runtimeEnv); - const acceptedPayload = { - accepted: true, - status: "running", - traceId, - ...codeAgentOtelTraceFields(traceId, process.env), - ...codeAgentTurnLifecycleFields(traceId, params), - conversationId: safeConversationId(params.conversationId) || null, - sessionId: safeSessionId(params.sessionId) || null, - threadId: safeOpaqueId(params.threadId) || null, - projectId: params.projectId ?? null, - ...codeAgentAdmissionTimingFields(timestamp) - }; - await recordCodeAgentTurnAdmission({ payload: acceptedPayload, params, options, traceId }); - results.set(traceId, annotateOwner(acceptedPayload, params)); - void emitCodeAgentOtelSpan("provider_decision", traceId, runtimeEnv, { - attributes: { - ...agentChatOtelAttributes(params), - adapterEnabled, - adapter: safeOtelConfigText(runtimeEnv.HWLAB_CODE_AGENT_ADAPTER ?? runtimeEnv.HWLAB_CODE_AGENT_PROVIDER), - provider: safeOtelConfigText(runtimeEnv.HWLAB_CODE_AGENT_PROVIDER), - defaultProviderProfile: safeOtelConfigText(runtimeEnv.HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE), - agentRunManagerHost: safeOtelUrlHost(runtimeEnv.AGENTRUN_MGR_URL ?? runtimeEnv.HWLAB_CODE_AGENT_AGENTRUN_MGR_URL), - agentRunRunnerNamespace: safeOtelConfigText(runtimeEnv.HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE ?? runtimeEnv.AGENTRUN_RUNTIME_NAMESPACE), - agentRunSourceCommitPresent: Boolean(firstNonEmptyValue(runtimeEnv.HWLAB_CODE_AGENT_AGENTRUN_SOURCE_COMMIT)) - } - }); - if (adapterEnabled) { - const run = async () => { - let initial = null; - let executionOptions = options; - try { - initial = initialAgentRunChatResult({ params, options, traceId }); - results.set(traceId, annotateOwner(initial, params)); - const billingStartedAt = Date.now(); - const billingPreflight = await preflightCodeAgentBilling({ params, options, traceId, sendResponse: false }); - void emitCodeAgentOtelSpan("billing_preflight", traceId, options.env ?? process.env, { - startTimeMs: billingStartedAt, - attributes: { sessionId: safeSessionId(params.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId, blocked: Boolean(billingPreflight?.blocked) }, - status: billingPreflight?.blocked ? "error" : "ok", - error: billingPreflight?.blocked ? billingPreflight?.payload?.error?.message ?? billingPreflight?.blocker?.summary ?? "billing preflight blocked" : null - }); - if (billingPreflight?.blocked) { - await settleAdmittedCodeAgentFailure({ - base: initial, - params, - options, - traceId, - results, - failure: billingPreflight, - traceLabel: "billing:preflight:failed" - }); - return; - } - const dispatchParams = { ...params, userBillingReservation: billingPreflight?.reservation ?? null }; - initial = withCodeAgentBillingReservation(initial, dispatchParams); - results.set(traceId, annotateOwner(initial, dispatchParams)); - executionOptions = { ...options, ...(await codeAgentChatExecutionOptions(options, dispatchParams)) }; - const dispatchStartedAt = Date.now(); - const payload = await submitAgentRunChatTurn({ params: dispatchParams, options: executionOptions, traceId, traceStore, results }); - void emitCodeAgentOtelSpan("agentrun_dispatch", traceId, executionOptions.env ?? process.env, { - startTimeMs: dispatchStartedAt, - attributes: { sessionId: safeSessionId(dispatchParams.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, dispatchParams).turnId, runId: payload?.agentRun?.runId ?? null, commandId: payload?.agentRun?.commandId ?? null, providerProfile: dispatchParams.providerProfile ?? null } - }); - if (isCodeAgentResultCanceled(results.get(traceId))) return; - const owned = annotateOwner(withCodeAgentBillingReservation(payload, dispatchParams), dispatchParams); - publishHwlabAgentRunCommandShadow({ - params: dispatchParams, - payload: owned, - traceId, - lifecycle: codeAgentTurnLifecycleFields(traceId, dispatchParams), - env: executionOptions.env ?? process.env - }); - await recordCodeAgentSessionOwner({ payload: owned, params: dispatchParams, options: executionOptions, status: "running" }); - results.set(traceId, owned); - scheduleAgentRunProjectionSync({ traceId, params: dispatchParams, options: executionOptions, traceStore, currentResult: owned }); - void emitCodeAgentOtelSpan("projection_write", traceId, executionOptions.env ?? process.env, { - attributes: { sessionId: safeSessionId(dispatchParams.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, dispatchParams).turnId, runId: owned?.agentRun?.runId ?? null, commandId: owned?.agentRun?.commandId ?? null, projection: "scheduled" } - }); - } catch (error) { - if (isCodeAgentResultCanceled(results.get(traceId))) return; - const dispatchErrorAttributes = codeAgentDispatchErrorOtelAttributes(error); - void emitCodeAgentOtelSpan("agentrun_dispatch", traceId, executionOptions.env ?? process.env, { - status: "error", - error, - attributes: { sessionId: safeSessionId(params.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId, ...dispatchErrorAttributes } - }); - await settleAdmittedCodeAgentFailure({ - base: initial ?? codeAgentAdmittedFailureBasePayload({ params, options, traceId }), - params, - options: executionOptions, - traceId, - results, - failure: codeAgentDispatchFailure(error), - traceLabel: "agentrun:dispatch:failed" - }); - } - }; - setImmediate(() => { run(); }); - return annotateOwner(acceptedPayload, params); - } - traceStore.ensure(traceId, { - runnerKind: "codex-app-server-stdio-runner", - workspace: options.workspace ?? options.env?.HWLAB_CODE_AGENT_CODEX_WORKSPACE ?? options.env?.HWLAB_CODE_AGENT_WORKSPACE ?? null, - sandbox: options.env?.HWLAB_CODE_AGENT_CODEX_SANDBOX ?? null, - sessionMode: "codex-app-server-stdio-long-lived", - implementationType: "repo-owned-codex-app-server-stdio-session" - }); - traceStore.append(traceId, { - type: "request", - status: "accepted", - label: "request:accepted-short-connection", - message: "Code Agent request accepted; client must poll trace/result with short HTTP requests.", - waitingFor: "codex-stdio" - }); - - const run = async () => { - try { - const billingStartedAt = Date.now(); - const billingPreflight = await preflightCodeAgentBilling({ params, options, traceId, sendResponse: false }); - void emitCodeAgentOtelSpan("billing_preflight", traceId, options.env ?? process.env, { - startTimeMs: billingStartedAt, - attributes: { sessionId: safeSessionId(params.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId, blocked: Boolean(billingPreflight?.blocked) }, - status: billingPreflight?.blocked ? "error" : "ok", - error: billingPreflight?.blocked ? billingPreflight?.payload?.error?.message ?? billingPreflight?.blocker?.summary ?? "billing preflight blocked" : null - }); - if (billingPreflight?.blocked) { - await settleAdmittedCodeAgentFailure({ - base: codeAgentAdmittedFailureBasePayload({ params, options, traceId }), - params, - options, - traceId, - results, - failure: billingPreflight, - traceLabel: "billing:preflight:failed" - }); - return; - } - const dispatchParams = { ...params, userBillingReservation: billingPreflight?.reservation ?? null }; - const payload = await runCodeAgentChat(dispatchParams, options); - if (isCodeAgentResultCanceled(results.get(traceId))) return; - await finalizeCodeAgentBillingUsage({ payload, params: dispatchParams, options }); - await recordCodeAgentSessionOwner({ payload, params: dispatchParams, options, status: codeAgentOwnerStatusForResult(payload) }); - results.set(traceId, annotateOwner(payload, dispatchParams)); - traceStore.append(traceId, { - type: "result", - status: payload.status === "completed" ? "completed" : "failed", - label: `result:${payload.status}`, - message: payload.status === "completed" - ? "Code Agent result is ready for short-connection polling." - : payload.error?.message ?? "Code Agent completed without a successful reply.", - sessionId: payload.sessionId ?? payload.session?.sessionId, - sessionStatus: payload.session?.status, - terminal: true - }); - } catch (error) { - if (isCodeAgentResultCanceled(results.get(traceId))) return; - const payload = { - status: "failed", - traceId, - conversationId: safeConversationId(params.conversationId) || null, - sessionId: safeSessionId(params.sessionId) || null, - error: { - code: "code_agent_background_failed", - layer: "api", - retryable: true, - message: error?.message ?? "Code Agent background turn failed", - userMessage: "Code Agent 后台任务失败；trace/result 轮询已保留错误。" - }, - updatedAt: new Date().toISOString() - }; - await finalizeCodeAgentBillingUsage({ payload, params, options }); - results.set(traceId, payload); - traceStore.append(traceId, { - type: "result", - status: "failed", - label: "result:failed", - errorCode: payload.error.code, - message: payload.error.message, - terminal: true - }); - } - }; - setImmediate(() => { - run(); - }); - return annotateOwner(acceptedPayload, params); -} - -async function recordCodeAgentTurnAdmission({ payload = {}, params = {}, options = {}, traceId } = {}) { - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - let ownerRecord = null; - try { - ownerRecord = await recordCodeAgentSessionOwner({ payload, params, options, status: "running" }); - if (!ownerRecord) { - const error = new Error("Code Agent turn admission did not durably persist Workbench session ownership."); - error.code = "workbench_admission_owner_not_persisted"; - error.valuesRedacted = true; - throw error; - } - await recordCodeAgentSessionInputFact({ - params, - options, - traceId, - delivery: "queue", - status: "admitted", - messageId: codeAgentTurnLifecycleFields(traceId, params).userMessageId, - commandId: payload.agentRun?.commandId ?? null - }); - } catch (error) { - throw codeAgentAdmissionUnavailableError(error, { params, traceId }); - } - traceStore.append(traceId, { - type: "request", - status: "admitted", - label: "turn:admitted", - message: "Code Agent turn was durably admitted before billing or dispatch preflight.", - sessionId: safeSessionId(params.sessionId) || null, - threadId: safeOpaqueId(params.threadId) || null, - ...codeAgentOtelTraceFields(traceId, options.env ?? process.env), - waitingFor: "billing-or-dispatch", - valuesPrinted: false - }); - void emitCodeAgentOtelSpan("durable_admission", traceId, options.env ?? process.env, { - attributes: { sessionId: safeSessionId(params.sessionId) || null, threadId: safeOpaqueId(params.threadId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId } - }); - return ownerRecord; -} - -async function recordCodeAgentSessionInputFact({ params = {}, options = {}, traceId, delivery = "queue", status = "admitted", messageId = null, commandId = null, error = null } = {}) { - const runtimeStore = options.runtimeStore ?? null; - if (typeof runtimeStore?.writeWorkbenchFacts !== "function") return null; - const resolvedTraceId = safeTraceId(traceId ?? params.traceId) || null; - const lifecycle = codeAgentTurnLifecycleFields(resolvedTraceId, params); - const sessionId = safeSessionId(params.sessionId ?? params.session?.sessionId) || null; - if (!sessionId) return null; - const resolvedMessageId = safeMessageId(messageId ?? params.messageId ?? lifecycle.userMessageId) || lifecycle.userMessageId; - const resolvedCommandId = textValue(commandId ?? params.commandId ?? params.agentRun?.commandId) || null; - const normalizedDelivery = codeAgentInputDelivery(delivery); - const normalizedStatus = codeAgentInputStatus(status); - const now = new Date().toISOString(); - const sourceEventId = `${sessionId}:${lifecycle.turnId}:${normalizedDelivery}:${resolvedTraceId ?? resolvedCommandId ?? resolvedMessageId}`; - const inputId = stableCodeAgentInputId({ sessionId, turnId: lifecycle.turnId, traceId: resolvedTraceId, messageId: resolvedMessageId, commandId: resolvedCommandId, delivery: normalizedDelivery, sourceEventId }); - const fact = { - inputId, - sessionId, - turnId: lifecycle.turnId, - traceId: resolvedTraceId, - messageId: resolvedMessageId, - commandId: resolvedCommandId, - delivery: normalizedDelivery, - status: normalizedStatus, - errorCode: textValue(error?.code ?? error?.error?.code) || null, - sourceEventId, - promptHash: params.message || params.prompt || params.text ? sha256Text(params.message ?? params.prompt ?? params.text).slice(0, 16) : null, - textBytes: params.message || params.prompt || params.text ? Buffer.byteLength(String(params.message ?? params.prompt ?? params.text), "utf8") : 0, - valuesRedacted: true, - secretMaterialStored: false, - createdAt: now, - updatedAt: now - }; - return runtimeStore.writeWorkbenchFacts({ facts: { inputs: [fact] } }, { - sessionId, - traceId: resolvedTraceId, - turnId: lifecycle.turnId, - messageId: resolvedMessageId, - ownerUserId: options.actor?.id ?? params.ownerUserId, - projectId: params.projectId ?? DEFAULT_CODE_AGENT_PROJECT_ID, - valuesPrinted: false - }); -} - -function stableCodeAgentInputId(value) { - const payload = Object.fromEntries(Object.entries(value ?? {}).filter(([, item]) => item !== undefined && item !== null && item !== "").sort(([left], [right]) => left.localeCompare(right))); - return `wsi_${sha256Text(JSON.stringify(payload)).slice(0, 32)}`; -} - -function codeAgentInputDelivery(value) { - const text = textValue(value).toLowerCase(); - return ["queue", "steer", "cancel"].includes(text) ? text : "queue"; -} - -function codeAgentInputStatus(value) { - const text = textValue(value).toLowerCase().replace(/-/gu, "_"); - return ["admitted", "promoted", "blocked", "failed", "canceled", "completed"].includes(text) ? text : "admitted"; -} - -function codeAgentAdmissionUnavailableError(error, { params = {}, traceId } = {}) { - const payload = codeAgentAdmissionUnavailablePayload({ error, params, traceId }); - return Object.assign(new Error(payload.error.message), { - code: "admission_unavailable", - statusCode: 503, - payload, - alreadyClassified: true - }); -} - -function isCodeAgentAdmissionUnavailableError(error) { - return error?.code === "admission_unavailable" && error?.payload; -} - -function codeAgentAdmissionUnavailablePayload({ error, params = {}, traceId } = {}) { - const message = error?.message ?? "Code Agent turn admission is unavailable; the user message was not persisted."; - return { - ok: false, - accepted: false, - status: "failed", - traceId: safeTraceId(traceId) || null, - ...codeAgentOtelTraceFields(traceId, options.env ?? process.env), - ...codeAgentTurnLifecycleFields(traceId, params), - conversationId: safeConversationId(params.conversationId) || null, - sessionId: safeSessionId(params.sessionId) || null, - threadId: safeOpaqueId(params.threadId) || null, - error: { - code: "admission_unavailable", - layer: "admission", - retryable: true, - message, - userMessage: "本次输入尚未持久化，请保留草稿后重试。", - route: "/v1/agent/chat" - }, - blocker: { - code: "admission_unavailable", - layer: "admission", - retryable: true, - summary: message - }, - valuesRedacted: true, - secretMaterialStored: false - }; -} - -function codeAgentDispatchFailure(error) { - const message = error?.message ?? "AgentRun dispatch failed"; - return { - statusCode: 503, - payload: { - error: { - code: error?.code ?? "agentrun_dispatch_failed", - layer: "agentrun", - retryable: true, - message, - userMessage: "AgentRun 调度失败；trace/result 轮询已保留错误。", - route: "/v1/agent/chat" - }, - blocker: { - code: error?.code ?? "agentrun_dispatch_failed", - layer: "agentrun", - retryable: true, - summary: message - } - } - }; -} - -function codeAgentDispatchErrorOtelAttributes(error) { - const agentRunError = error?.agentRunError && typeof error.agentRunError === "object" ? error.agentRunError : null; - const details = agentRunError?.error?.details && typeof agentRunError.error.details === "object" ? agentRunError.error.details : null; - return { - failureKind: safeOtelText(error?.code ?? agentRunError?.failureKind, 120), - upstreamStatusCode: Number.isInteger(error?.statusCode) ? error.statusCode : null, - upstreamTraceId: safeTraceId(agentRunError?.traceId) || null, - upstreamMessage: safeOtelText(agentRunError?.message ?? error?.message, 300), - upstreamStderrTail: safeOtelText(details?.stderr, 600) - }; -} - -function safeOtelText(value, limit = 300) { - const text = String(value ?? "").replace(/[\u0000-\u001f\u007f]+/gu, " ").trim(); - return text ? text.slice(0, limit) : null; -} - -function codeAgentAdmittedFailureBasePayload({ params = {}, options = {}, traceId } = {}) { - const now = new Date().toISOString(); - return { - accepted: true, - status: "running", - shortConnection: true, - traceId, - ...codeAgentTurnLifecycleFields(traceId, params), - messageId: codeAgentTurnLifecycleFields(traceId, params).assistantMessageId, - projectId: params.projectId ?? null, - conversationId: safeConversationId(params.conversationId) || null, - sessionId: safeSessionId(params.sessionId) || null, - threadId: safeOpaqueId(params.threadId) || null, - ...codeAgentAdmissionTimingFields(now), - provider: codeAgentAgentRunAdapterEnabled(options.env ?? process.env) ? "agentrun" : "codex-stdio", - model: options.env?.HWLAB_CODE_AGENT_MODEL ?? "unknown", - backend: codeAgentAgentRunAdapterEnabled(options.env ?? process.env) ? "agentrun-v01" : "hwlab-cloud-api/code-agent-chat", - valuesPrinted: false - }; -} - -function codeAgentAdmissionTimingFields(value = null) { - const admittedAt = timestampIsoOrNow(value); - const timing = { - startedAt: admittedAt, - lastEventAt: admittedAt, - finishedAt: null, - durationMs: null, - observedAt: admittedAt, - lastEventAgeMs: 0, - valuesRedacted: true - }; - return { - createdAt: admittedAt, - updatedAt: admittedAt, - timing, - startedAt: admittedAt, - lastEventAt: admittedAt, - finishedAt: null, - durationMs: null - }; -} - -function codeAgentResultTimingFields(...sources) { - const records = sources.filter((source) => source && typeof source === "object"); - const timingSource = records.map((source) => source.timing).find((timing) => timing && typeof timing === "object") ?? {}; - const startedAt = firstTimestampIso(timingSource.startedAt, ...records.map((source) => source.startedAt), ...records.map((source) => source.createdAt)); - const lastEventAt = firstLatestTimestampIso(timingSource.lastEventAt, ...records.map((source) => source.lastEventAt), ...records.map((source) => source.updatedAt), ...records.map((source) => source.createdAt)); - const finishedAt = firstLatestTimestampIso(timingSource.finishedAt, ...records.map((source) => source.finishedAt), ...records.map((source) => source.completedAt)); - const durationMs = numberOrNull(timingSource.durationMs ?? records.map((source) => source.durationMs).find((value) => numberOrNull(value) !== null)); - const observedAt = timestampIsoOrNull(timingSource.observedAt); - const lastEventAgeMs = numberOrNull(timingSource.lastEventAgeMs); - if (!startedAt && !lastEventAt && !finishedAt && durationMs === null && lastEventAgeMs === null) return {}; - const timing = { - ...(timingSource && typeof timingSource === "object" ? timingSource : {}), - startedAt: startedAt ?? null, - lastEventAt: lastEventAt ?? null, - finishedAt: finishedAt ?? null, - durationMs, - observedAt: observedAt ?? null, - lastEventAgeMs, - valuesRedacted: timingSource.valuesRedacted !== false - }; - return { - timing, - startedAt: timing.startedAt, - lastEventAt: timing.lastEventAt, - finishedAt: timing.finishedAt, - durationMs: timing.durationMs - }; -} - -function firstTimestampIso(...values) { - for (const value of values) { - const timestamp = timestampIsoOrNull(value); - if (timestamp) return timestamp; - } - return null; -} - -function firstLatestTimestampIso(...values) { - let latest = null; - let latestMs = Number.NEGATIVE_INFINITY; - for (const value of values) { - const timestamp = timestampIsoOrNull(value); - if (!timestamp) continue; - const ms = Date.parse(timestamp); - if (!Number.isFinite(ms) || ms < latestMs) continue; - latest = timestamp; - latestMs = ms; - } - return latest; -} - -async function settleAdmittedCodeAgentFailure({ base = {}, params = {}, options = {}, traceId, results, failure = {}, traceLabel = "code-agent:failed" } = {}) { - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - const error = failure.payload?.error ?? failure.error ?? {}; - const blocker = failure.payload?.blocker ?? failure.blocker ?? null; - const message = error.message ?? blocker?.summary ?? "Code Agent request failed after admission."; - traceStore.append(traceId, { - type: "result", - status: "failed", - label: traceLabel, - errorCode: error.code ?? "code_agent_failed_after_admission", - message, - terminal: true, - valuesPrinted: false - }); - const now = new Date().toISOString(); - const payload = annotateOwner({ - ...base, - accepted: true, - status: "failed", - traceId, - ...codeAgentTurnLifecycleFields(traceId, base), - messageId: base.messageId ?? codeAgentTurnLifecycleFields(traceId, base).assistantMessageId, - conversationId: safeConversationId(base.conversationId ?? params.conversationId) || null, - sessionId: safeSessionId(base.sessionId ?? params.sessionId) || null, - threadId: safeOpaqueId(base.threadId ?? params.threadId) || null, - error: { - code: error.code ?? "code_agent_failed_after_admission", - layer: error.layer ?? blocker?.layer ?? "api", - retryable: error.retryable !== false, - message, - userMessage: error.userMessage ?? message, - route: error.route ?? "/v1/agent/chat" - }, - blocker: blocker ? { - code: blocker.code ?? error.code ?? "code_agent_failed_after_admission", - layer: blocker.layer ?? error.layer ?? "api", - retryable: blocker.retryable !== false, - summary: blocker.summary ?? message - } : null, - finalResponse: { - text: error.userMessage ?? message, - textChars: String(error.userMessage ?? message).length, - role: "assistant", - status: "failed", - traceId, - updatedAt: now, - source: "code-agent-admitted-failure", - valuesPrinted: false - }, - runnerTrace: traceStore.snapshot(traceId), - updatedAt: now, - valuesPrinted: false, - secretMaterialStored: false - }, params); - await finalizeCodeAgentBillingUsage({ payload, params, options }); - payload.runnerTrace = traceStore.snapshot(traceId); - recordCodeAgentConversationFact(payload, options); - results?.set?.(traceId, payload); - await recordCodeAgentSessionOwner({ payload, params, options, status: codeAgentOwnerStatusForResult(payload) }); - return payload; -} - -function scheduleAgentRunProjectionSync({ traceId, params = {}, options = {}, traceStore = defaultCodeAgentTraceStore, currentResult = null } = {}) { - if (!safeTraceId(traceId) || !currentResult?.agentRun?.runId || !currentResult?.agentRun?.commandId) return null; - const env = options.env ?? process.env; - const noResponseTimeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_NO_RESPONSE_TIMEOUT_MS, null); - const pollIntervalMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_POLL_INTERVAL_MS, 1_000); - const refreshOptions = codeAgentTurnStatusRefreshOptions(options); - return scheduleWorkbenchProjectionFinalizer({ - traceId, - currentResult, - traceStore, - noResponseTimeoutMs, - pollIntervalMs, - getCachedResult: () => options.codeAgentChatResults?.get?.(traceId), - isCanceled: isCodeAgentResultCanceled, - isTerminal: agentRunProjectionObservedTerminal, - activitySignature: agentRunProjectionActivitySignature, - sleep: sleepAgentRunProjectionSync, - performanceStore: options.backendPerformanceStore, - syncResult: ({ result }) => syncAgentRunChatResult({ - traceId, - currentResult: result, - options: refreshOptions, - traceStore, - forceResultSync: agentRunProjectionObservedTerminal(result, traceStore?.snapshot?.(traceId)) || traceSnapshotIsTerminal(traceStore?.snapshot?.(traceId)), - retryParams: params - }), - onTerminal: (payload, finalizerOptions = {}) => scheduleCodeAgentTerminalTurnStatusEffects({ payload: codeAgentPayloadWithObservedTerminalStatus(payload, traceStore?.snapshot?.(traceId)) ?? payload, params, options, preserveLastTraceId: finalizerOptions.preserveLastTraceId === true }) - }); -} - -export function startAgentRunProjectionResume({ options = {}, traceStore = defaultCodeAgentTraceStore, logger = console } = {}) { - const env = options.env ?? process.env; - if (!truthyFlag(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_ENABLED)) { - return { started: false, reason: "disabled", stop() {}, valuesPrinted: false }; - } - const initialDelayMs = parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INITIAL_DELAY_MS, DEFAULT_AGENTRUN_PROJECTION_RESUME_INITIAL_DELAY_MS); - const intervalMs = parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INTERVAL_MS, DEFAULT_AGENTRUN_PROJECTION_RESUME_INTERVAL_MS); - const minAgeMs = parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS, DEFAULT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS); - const batchSize = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE, DEFAULT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE); - const jitterMs = intervalMs > 0 ? Math.min(parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_JITTER_MS, DEFAULT_AGENTRUN_PROJECTION_RESUME_JITTER_MS), intervalMs) : 0; - const failureBackoffMs = Math.max(intervalMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_FAILURE_BACKOFF_MS, DEFAULT_AGENTRUN_PROJECTION_RESUME_FAILURE_BACKOFF_MS)); - const retryInitialDelayFloorMs = Math.max(1000, intervalMs * 2); - const candidateRetryMaxAttempts = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_CANDIDATE_RETRY_MAX_ATTEMPTS, 5); - const candidateRetryInitialDelayMs = Math.max(retryInitialDelayFloorMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_CANDIDATE_RETRY_INITIAL_DELAY_MS, retryInitialDelayFloorMs)); - const candidateRetryMaxDelayMs = Math.max(candidateRetryInitialDelayMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_CANDIDATE_RETRY_MAX_DELAY_MS, 600000)); - const passRetryMaxAttempts = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_PASS_RETRY_MAX_ATTEMPTS, 5); - const passRetryInitialDelayMs = Math.max(failureBackoffMs, retryInitialDelayFloorMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_PASS_RETRY_INITIAL_DELAY_MS, retryInitialDelayFloorMs)); - const passRetryMaxDelayMs = Math.max(passRetryInitialDelayMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_PASS_RETRY_MAX_DELAY_MS, 600000)); - const candidateRetryState = new Map(); - let passFailureAttempt = 0; - let stopped = false; - let active = false; - let timer = null; - let nextOffset = 0; - - const schedule = (delayMs, reason) => { - if (stopped) return; - const scheduledDelayMs = Math.max(0, Number(delayMs) || 0) + randomJitterMs(jitterMs); - timer = setTimeout(() => { - timer = null; - const offset = nextOffset; - active = true; - let nextDelayMs = intervalMs; - let nextReason = "interval"; - void runAgentRunProjectionResumePass({ options, traceStore, logger, batchSize, minAgeMs, offset, reason, candidateRetryState, candidateRetryMaxAttempts, candidateRetryInitialDelayMs, candidateRetryMaxDelayMs }) - .then((result) => { - passFailureAttempt = 0; - const candidateFailed = result?.failed > 0; - nextOffset = candidateFailed ? offset : (result?.scannedSessions >= batchSize ? offset + batchSize : 0); - nextDelayMs = candidateFailed ? failureBackoffMs : intervalMs; - nextReason = candidateFailed ? "failure_backoff" : "interval"; - }) - .catch((error) => { - passFailureAttempt += 1; - const retryDelayMs = projectionResumeCandidateRetryDelayMs(passFailureAttempt, passRetryInitialDelayMs, passRetryMaxDelayMs); - const retryExhausted = passFailureAttempt >= passRetryMaxAttempts; - nextDelayMs = retryDelayMs; - nextReason = retryExhausted ? "pass_retry_exhausted" : "pass_failure_backoff"; - logAgentRunProjectionResume(logger, { - event: "workbench_projection_resume", - ok: false, - status: retryExhausted ? "stopped" : "failed", - reason, - offset, - errorCode: error?.code ?? "projection_resume_pass_failed", - message: error?.message ?? "AgentRun projection resume pass failed.", - passRetryAttempt: passFailureAttempt, - passRetryMaxAttempts, - passRetryLabel: passFailureAttempt + "/" + passRetryMaxAttempts, - passRetryDelayMs: retryDelayMs, - passRetryExhausted: retryExhausted, - valuesRedacted: true - }); - if (retryExhausted) stopped = true; - }) - .finally(() => { - active = false; - if (!stopped && intervalMs > 0) schedule(nextDelayMs, nextReason); - }); - }, scheduledDelayMs); - timer.unref?.(); - }; - - schedule(initialDelayMs, "startup"); - return { - started: true, - initialDelayMs, - intervalMs, - minAgeMs, - batchSize, - jitterMs, - failureBackoffMs, - candidateRetryMaxAttempts, - candidateRetryInitialDelayMs, - candidateRetryMaxDelayMs, - passRetryMaxAttempts, - passRetryInitialDelayMs, - passRetryMaxDelayMs, - valuesPrinted: false, - stop() { - stopped = true; - if (timer) clearTimeout(timer); - timer = null; - }, - active: () => active - }; -} - -async function runAgentRunProjectionResumePass({ options = {}, traceStore = defaultCodeAgentTraceStore, logger = console, batchSize, minAgeMs, offset = 0, reason = "manual", candidateRetryState = new Map(), candidateRetryMaxAttempts = 5, candidateRetryInitialDelayMs = 30000, candidateRetryMaxDelayMs = 600000 } = {}) { - const stateCandidates = await listAgentRunProjectionResumeStateCandidates(options, batchSize, offset, { minAgeMs }); - const sessions = stateCandidates.length > 0 ? [] : await listAgentRunProjectionResumeSessions(options, batchSize, offset); - const candidates = stateCandidates.length > 0 ? stateCandidates : sessions.flatMap((session) => agentRunProjectionResumeCandidates(session, { minAgeMs })); - let resumed = 0; - let terminal = 0; - let nonTerminal = 0; - let failed = 0; - let skippedCooldown = 0; - let skippedExhausted = 0; - const nowMs = Date.now(); - for (const candidate of candidates) { - const retryKey = projectionResumeCandidateRetryKey(candidate); - const retryState = candidateRetryState.get(retryKey); - if (retryState?.exhausted === true) { - skippedExhausted += 1; - continue; - } - if (Number.isFinite(retryState?.nextRetryAtMs) && retryState.nextRetryAtMs > nowMs) { - skippedCooldown += 1; - continue; - } - try { - const result = await resumeAgentRunProjectionCandidate({ candidate, options, traceStore }); - resumed += 1; - const currentTraceSnapshot = safeTraceStoreSnapshot(traceStore, candidate.traceId); - if (agentRunProjectionObservedTerminal(result, result?.runnerTrace ?? currentTraceSnapshot) || traceSnapshotIsTerminal(currentTraceSnapshot)) { - terminal += 1; - candidateRetryState.delete(retryKey); - } else { - nonTerminal += 1; - const previousAttempts = Number.isFinite(retryState?.attempts) ? retryState.attempts : 0; - const retryAttempt = previousAttempts + 1; - const retryExhausted = retryAttempt >= candidateRetryMaxAttempts; - const retryDelayMs = retryExhausted ? 0 : projectionResumeCandidateRetryDelayMs(retryAttempt, candidateRetryInitialDelayMs, candidateRetryMaxDelayMs); - const nextRetryAtMs = retryExhausted ? null : Date.now() + retryDelayMs; - const nextRetryAt = Number.isFinite(nextRetryAtMs) ? new Date(nextRetryAtMs).toISOString() : null; - candidateRetryState.set(retryKey, { - attempts: retryAttempt, - nextRetryAtMs, - exhausted: retryExhausted, - lastErrorCode: "projection_resume_non_terminal" - }); - appendProjectionResumeRetry(traceStore, candidate, { retryAttempt, retryMaxAttempts: candidateRetryMaxAttempts, retryLabel: retryAttempt + "/" + candidateRetryMaxAttempts, retryDelayMs, nextRetryAt, retryExhausted, status: result?.status ?? null }); - } - } catch (error) { - failed += 1; - const previousAttempts = Number.isFinite(retryState?.attempts) ? retryState.attempts : 0; - const retryAttempt = previousAttempts + 1; - const retryExhausted = retryAttempt >= candidateRetryMaxAttempts; - const retryDelayMs = retryExhausted ? 0 : projectionResumeCandidateRetryDelayMs(retryAttempt, candidateRetryInitialDelayMs, candidateRetryMaxDelayMs); - const nextRetryAtMs = retryExhausted ? null : Date.now() + retryDelayMs; - const nextRetryAt = Number.isFinite(nextRetryAtMs) ? new Date(nextRetryAtMs).toISOString() : null; - candidateRetryState.set(retryKey, { - attempts: retryAttempt, - nextRetryAtMs, - exhausted: retryExhausted, - lastErrorCode: error?.code ?? "projection_resume_sync_failed", - lastErrorMessage: safeOtelText(error?.message, 300) - }); - appendProjectionResumeError(traceStore, candidate, error, { retryAttempt, retryMaxAttempts: candidateRetryMaxAttempts, retryLabel: retryAttempt + "/" + candidateRetryMaxAttempts, retryDelayMs, nextRetryAt, retryExhausted }); - } - } - logAgentRunProjectionResume(logger, { - event: "workbench_projection_resume", - ok: failed === 0, - status: failed === 0 ? "completed" : "degraded", - reason, - offset, - scannedProjectionStates: stateCandidates.length, - scannedSessions: sessions.length, - candidates: candidates.length, - resumed, - terminal, - nonTerminal, - failed, - skippedCooldown, - skippedExhausted, - candidateRetryTracked: candidateRetryState.size, - candidateRetryLastErrorCode: latestProjectionResumeRetryField(candidateRetryState, "lastErrorCode"), - candidateRetryLastErrorMessage: latestProjectionResumeRetryField(candidateRetryState, "lastErrorMessage"), - candidateRetryMaxAttempts, - valuesRedacted: true - }); - return { offset, scannedSessions: sessions.length, candidates: candidates.length, resumed, terminal, nonTerminal, failed, skippedCooldown, skippedExhausted }; -} - -function latestProjectionResumeRetryField(candidateRetryState, field) { - const values = [...candidateRetryState.values()]; - for (let index = values.length - 1; index >= 0; index -= 1) { - const text = textValue(values[index]?.[field]); - if (text) return text; - } - return null; -} - -async function listAgentRunProjectionResumeStateCandidates(options = {}, batchSize = DEFAULT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE, offset = 0, { minAgeMs = DEFAULT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS } = {}) { - const runtimeStore = options.runtimeStore; - if (typeof runtimeStore?.queryWorkbenchProjectionStates !== "function") return []; - let result; - try { - result = await runtimeStore.queryWorkbenchProjectionStates({ - projectionStatuses: ["projecting", "degraded"], - dueAt: new Date().toISOString(), - limit: batchSize, - offset - }); - } catch { - return []; - } - return (Array.isArray(result?.states) ? result.states : []) - .map((state) => agentRunProjectionResumeCandidateFromState(state, { minAgeMs })) - .filter(Boolean); -} - -async function listAgentRunProjectionResumeSessions(options = {}, batchSize = DEFAULT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE, offset = 0) { - const accessStore = options.accessController?.store ?? options.accessController ?? null; - if (typeof accessStore?.listAgentSessionsForUser !== "function") return []; - return await accessStore.listAgentSessionsForUser({ - actorRole: "admin", - ownerScoped: false, - includeArchived: false, - limit: batchSize, - offset - }) ?? []; -} - -function agentRunProjectionResumeCandidates(session = {}, { minAgeMs = DEFAULT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS, nowMs = Date.now() } = {}) { - const snapshot = session?.session && typeof session.session === "object" ? session.session : {}; - const records = new Map(); - const traceResults = snapshot.traceResults && typeof snapshot.traceResults === "object" ? snapshot.traceResults : null; - if (traceResults) { - for (const [traceId, record] of Object.entries(traceResults)) { - const safeId = safeTraceId(traceId); - if (safeId && record && typeof record === "object") records.set(safeId, { ...record, traceId: safeId }); - } - } - const directTraceId = safeTraceId(snapshot.traceId ?? session.lastTraceId); - if (directTraceId && snapshot.agentRun && typeof snapshot.agentRun === "object") { - records.set(directTraceId, { ...snapshot, traceId: directTraceId }); - } - return [...records.values()] - .map((record) => agentRunProjectionResumeCandidate(session, record, { minAgeMs, nowMs })) - .filter(Boolean); -} - -function agentRunProjectionResumeCandidate(session = {}, record = {}, { minAgeMs = DEFAULT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS, nowMs = Date.now() } = {}) { - const traceId = safeTraceId(record.traceId ?? session.lastTraceId); - const agentRun = record.agentRun && typeof record.agentRun === "object" ? record.agentRun : null; - if (!traceId || !agentRun?.runId || !agentRun?.commandId) return null; - if (isTraceCommandTerminalStatus(record.status ?? agentRun.terminalStatus ?? agentRun.commandState ?? agentRun.status)) return null; - const updatedAtMs = Date.parse(record.updatedAt ?? session.updatedAt ?? ""); - if (minAgeMs > 0 && Number.isFinite(updatedAtMs) && nowMs - updatedAtMs < minAgeMs) return null; - const sessionId = safeSessionId(record.sessionId ?? session.id) || null; - const conversationId = safeConversationId(record.conversationId ?? session.conversationId) || null; - const threadId = safeOpaqueId(record.threadId ?? session.threadId) || null; - const ownerUserId = textValue(record.ownerUserId ?? session.ownerUserId); - const ownerRole = textValue(record.ownerRole ?? session.ownerRole) || "user"; - const retryParams = agentRunProjectionRetryParams(record.retryParams ?? record.projectionRetryParams ?? session.retryParams); - return { - traceId, - params: { - ...retryParams, - traceId, - sessionId, - conversationId, - threadId, - projectId: record.projectId ?? session.projectId ?? null, - ownerUserId, - ownerRole - }, - result: { - ...record, - traceId, - sessionId, - conversationId, - threadId, - ownerUserId, - ownerRole, - status: record.status || session.status || "running", - finalResponse: null, - traceSummary: null, - agentRun: { - ...agentRun, - traceId, - providerTrace: agentRun.providerTrace ?? { traceId, runId: agentRun.runId, commandId: agentRun.commandId, valuesPrinted: false }, - lastSeq: Number.isFinite(Number(agentRun.lastSeq)) ? Number(agentRun.lastSeq) : 0, - valuesPrinted: false - }, - valuesRedacted: true - } - }; -} - -function agentRunProjectionResumeCandidateFromState(state = {}, { minAgeMs = DEFAULT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS, nowMs = Date.now() } = {}) { - const traceId = safeTraceId(state.traceId); - const runId = textValue(state.runId ?? state.sourceRunId); - const commandId = textValue(state.commandId ?? state.sourceCommandId); - if (!traceId || !runId || !commandId) return null; - if (isTraceCommandTerminalStatus(state.projectionStatus)) return null; - const updatedAtMs = Date.parse(state.updatedAt ?? ""); - if (minAgeMs > 0 && Number.isFinite(updatedAtMs) && nowMs - updatedAtMs < minAgeMs) return null; - const sessionId = safeSessionId(state.sessionId) || null; - const conversationId = safeConversationId(state.conversationId) || null; - const threadId = safeOpaqueId(state.threadId) || null; - const ownerUserId = textValue(state.ownerUserId); - const ownerRole = textValue(state.ownerRole) || "user"; - const lastSeq = Number.isFinite(Number(state.lastSourceSeq ?? state.lastAgentRunSeq)) ? Number(state.lastSourceSeq ?? state.lastAgentRunSeq) : 0; - const retryParams = agentRunProjectionRetryParams(state.retryParams ?? state.projectionRetryParams); - return { - traceId, - params: { - ...retryParams, - traceId, - sessionId, - conversationId, - threadId, - ownerUserId, - ownerRole - }, - result: { - accepted: true, - status: "running", - shortConnection: true, - traceId, - sessionId, - conversationId, - threadId, - ownerUserId, - ownerRole, - finalResponse: null, - traceSummary: null, - agentRun: { - traceId, - runId, - commandId, - lastSeq, - managerUrl: state.managerUrl ?? null, - backendProfile: state.backendProfile ?? null, - providerId: state.providerId ?? null, - status: "running", - commandState: "running", - resultSyncState: state.resultSyncState ?? null, - providerTrace: { traceId, runId, commandId, valuesPrinted: false }, - valuesPrinted: false - }, - valuesRedacted: true - } - }; -} - -function agentRunProjectionRetryParams(value = null) { - if (!value || typeof value !== "object" || Array.isArray(value)) return {}; - const message = textValue(value.message ?? value.prompt ?? value.text); - if (!message) return {}; - const result = { - message, - prompt: message, - text: message, - projectId: textValue(value.projectId) || undefined, - ownerUserId: textValue(value.ownerUserId) || undefined, - ownerRole: textValue(value.ownerRole) || undefined, - providerProfile: textValue(value.providerProfile ?? value.codeAgentProviderProfile ?? value.backendProfile) || undefined, - codeAgentProviderProfile: textValue(value.codeAgentProviderProfile ?? value.providerProfile ?? value.backendProfile) || undefined, - backendProfile: textValue(value.backendProfile ?? value.providerProfile ?? value.codeAgentProviderProfile) || undefined, - providerId: textValue(value.providerId) || undefined, - valuesPrinted: false - }; - for (const [key, item] of Object.entries(result)) { - if (item === undefined || item === "") delete result[key]; - } - return result; -} - -async function resumeAgentRunProjectionCandidate({ candidate, options = {}, traceStore = defaultCodeAgentTraceStore } = {}) { - const { currentResult } = await agentRunProjectionResumeResultWithCursor(candidate, options); - const resumeOptions = { ...options, deferAgentRunResultSync: false }; - const synced = await syncAgentRunChatResult({ - traceId: candidate.traceId, - currentResult, - options: resumeOptions, - traceStore, - forceResultSync: true, - refreshEvents: true, - retryParams: candidate.params - }); - const payload = codeAgentPayloadWithObservedTerminalStatus(synced?.result ?? currentResult, synced?.runnerTrace ?? traceStore?.snapshot?.(candidate.traceId)) ?? (synced?.result ?? currentResult); - if (isTraceCommandTerminalStatus(payload?.status)) { - await recordCodeAgentTerminalTurnStatusEffects({ payload, params: candidate.params, options, preserveLastTraceId: true }); - } - return payload; -} - -async function agentRunProjectionResumeResultWithCursor(candidate, options = {}) { - const projectionState = await loadAgentRunProjectionState(options.runtimeStore, candidate); - const plan = buildAgentRunProjectionEventsFetchPlan({ - projectionState, - agentRun: candidate.result.agentRun, - pageLimit: options.env?.HWLAB_CODE_AGENT_AGENTRUN_EVENTS_PAGE_LIMIT - }); - return { - projectionState, - currentResult: { - ...candidate.result, - traceSummary: null, - agentRun: { - ...candidate.result.agentRun, - lastSeq: plan.afterSeq, - valuesPrinted: false - } - } - }; -} - -async function loadAgentRunProjectionState(runtimeStore, candidate) { - if (!runtimeStore || typeof runtimeStore.getWorkbenchProjectionState !== "function") return null; - try { - const result = await runtimeStore.getWorkbenchProjectionState({ traceId: candidate.traceId }); - const state = result?.projectionState ?? null; - if (!state) return null; - if (String(state.runId ?? state.sourceRunId ?? "") !== String(candidate.result.agentRun.runId ?? "")) return null; - if (String(state.commandId ?? state.sourceCommandId ?? "") !== String(candidate.result.agentRun.commandId ?? "")) return null; - return state; - } catch { - return null; - } -} - -function projectionResumeCandidateRetryKey(candidate) { - const traceId = safeTraceId(candidate?.traceId) || "trace-unknown"; - const runId = String(candidate?.result?.agentRun?.runId ?? "run-unknown"); - const commandId = String(candidate?.result?.agentRun?.commandId ?? "command-unknown"); - return [traceId, runId, commandId].join("|"); -} - -function projectionResumeCandidateRetryDelayMs(retryAttempt, initialDelayMs, maxDelayMs) { - const attempt = Math.max(1, Number(retryAttempt) || 1); - return Math.min(maxDelayMs, initialDelayMs * (2 ** Math.max(0, attempt - 1))); -} - -function appendProjectionResumeError(traceStore, candidate, error, retry = {}) { - const traceId = safeTraceId(candidate?.traceId); - if (!traceId) return; - if (traceSnapshotIsTerminal(safeTraceStoreSnapshot(traceStore, traceId))) return; - traceStore.append(traceId, { - type: "turn-status", - status: "degraded", - label: "projection-resume:sync-failed", - errorCode: error?.code ?? "projection_resume_sync_failed", - message: error?.message ?? "AgentRun projection resume failed.", - runId: candidate?.result?.agentRun?.runId ?? null, - commandId: candidate?.result?.agentRun?.commandId ?? null, - waitingFor: "agentrun-result-resume", - retryAttempt: retry.retryAttempt ?? null, - retryMaxAttempts: retry.retryMaxAttempts ?? null, - retryLabel: retry.retryLabel ?? null, - retryDelayMs: retry.retryDelayMs ?? null, - nextRetryAt: retry.nextRetryAt ?? null, - retryExhausted: retry.retryExhausted === true, - terminal: false, - valuesPrinted: false - }); -} - -function appendProjectionResumeRetry(traceStore, candidate, retry = {}) { - const traceId = safeTraceId(candidate?.traceId); - if (!traceId) return; - if (traceSnapshotIsTerminal(safeTraceStoreSnapshot(traceStore, traceId))) return; - traceStore.append(traceId, { - type: "turn-status", - status: "degraded", - label: "projection-resume:non-terminal-backoff", - errorCode: "projection_resume_non_terminal", - message: "AgentRun projection resume did not reach a terminal status; next resume is delayed.", - runId: candidate?.result?.agentRun?.runId ?? null, - commandId: candidate?.result?.agentRun?.commandId ?? null, - waitingFor: "agentrun-result-resume", - observedStatus: retry.status ?? null, - retryAttempt: retry.retryAttempt ?? null, - retryMaxAttempts: retry.retryMaxAttempts ?? null, - retryLabel: retry.retryLabel ?? null, - retryDelayMs: retry.retryDelayMs ?? null, - nextRetryAt: retry.nextRetryAt ?? null, - retryExhausted: retry.retryExhausted === true, - terminal: false, - valuesPrinted: false - }); -} - -function safeTraceStoreSnapshot(traceStore, traceId) { - try { - return typeof traceStore?.snapshot === "function" ? traceStore.snapshot(traceId) : null; - } catch { - return null; - } -} - -function traceSnapshotIsTerminal(snapshot) { - if (!snapshot || typeof snapshot !== "object") return false; - if (snapshot.finishedAt) return true; - if (isTraceCommandTerminalStatus(snapshot.status)) return true; - if (isTraceCommandTerminalStatus(snapshot.lastEvent?.status) || snapshot.lastEvent?.terminal === true) return true; - const events = Array.isArray(snapshot.events) ? snapshot.events : []; - return events.some((event) => event?.terminal === true || isTraceCommandTerminalStatus(event?.status)); -} - -function logAgentRunProjectionResume(logger, payload) { - try { - const line = JSON.stringify(payload); - if (payload.ok === false && typeof logger?.warn === "function") logger.warn(line); - else if (typeof logger?.log === "function") logger.log(line); - } catch { - // Projection resume logging must not affect the projector. - } -} - -function parseNonNegativeInteger(value, fallback) { - const parsed = Number.parseInt(value ?? "", 10); - return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback; -} - -function randomJitterMs(maxJitterMs) { - return maxJitterMs > 0 ? Math.floor(Math.random() * (maxJitterMs + 1)) : 0; -} - -function agentRunProjectionActivitySignature(result, runnerTrace) { - const agentRun = result?.agentRun && typeof result.agentRun === "object" ? result.agentRun : {}; - const trace = runnerTrace && typeof runnerTrace === "object" ? runnerTrace : result?.runnerTrace; - const events = Array.isArray(trace?.events) ? trace.events : []; - const lastEvent = trace?.lastEvent ?? events.at(-1) ?? null; - return [ - result?.status, - result?.updatedAt, - agentRun.status, - agentRun.runStatus, - agentRun.commandState, - agentRun.terminalStatus, - agentRun.lastSeq, - agentRun.updatedAt, - trace?.status, - trace?.eventCount ?? events.length, - trace?.updatedAt, - trace?.lastEventLabel, - lastEvent?.seq, - lastEvent?.sourceSeq, - lastEvent?.label, - lastEvent?.type, - lastEvent?.status, - lastEvent?.createdAt - ].map((value) => value ?? "").join("|"); -} - -function agentRunProjectionObservedTerminal(result, runnerTrace) { - return Boolean(codeAgentObservedTerminalStatus(result, runnerTrace)); -} - -function codeAgentPayloadWithObservedTerminalStatus(payload, runnerTrace) { - const terminalStatus = codeAgentObservedTerminalStatus(payload, runnerTrace); - if (!terminalStatus || !payload || typeof payload !== "object") return null; - const agentRun = payload.agentRun && typeof payload.agentRun === "object" ? payload.agentRun : null; - const agentRunTerminalStatus = terminalStatus === "canceled" ? "cancelled" : terminalStatus; - return { - ...payload, - status: terminalStatus, - runnerTrace: runnerTrace && typeof runnerTrace === "object" ? runnerTrace : payload.runnerTrace, - ...(agentRun ? { agentRun: { ...agentRun, terminalStatus: agentRun.terminalStatus ?? agentRunTerminalStatus, valuesPrinted: false } } : {}) - }; -} - -function codeAgentObservedTerminalStatus(result, runnerTrace) { - const agentRun = result?.agentRun && typeof result.agentRun === "object" ? result.agentRun : {}; - if (codeAgentPayloadHasSealedFinalResponse(result)) return "completed"; - for (const value of [agentRun.terminalStatus, agentRun.commandState]) { - const terminalStatus = normalizeCodeAgentTerminalStatus(value); - if (terminalStatus) return terminalStatus; - } - const commandId = textValue(agentRun.commandId); - const trace = runnerTrace && typeof runnerTrace === "object" ? runnerTrace : result?.runnerTrace; - const events = Array.isArray(trace?.events) ? trace.events : []; - for (const event of [...events].reverse()) { - const payload = event?.payload && typeof event.payload === "object" ? event.payload : {}; - const eventCommandId = textValue(event?.commandId ?? payload.commandId); - if (commandId && eventCommandId && eventCommandId !== commandId) continue; - for (const value of [ - event?.type === "terminal_status" ? payload.terminalStatus : null, - payload.phase === "command-terminal" ? payload.terminalStatus : null, - payload.phase === "turn:completed" || payload.phase === "turn/steer:completed" ? "completed" : null, - event?.terminal === true ? payload.terminalStatus ?? payload.status ?? event.status ?? "completed" : null, - payload.terminal === true ? payload.terminalStatus ?? payload.status ?? "completed" : null - ]) { - const terminalStatus = normalizeCodeAgentTerminalStatus(value); - if (terminalStatus) return terminalStatus; - } - } - return null; -} - -function normalizeCodeAgentTerminalStatus(value) { - const status = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); - if (!CODE_AGENT_TERMINAL_STATUSES.has(status)) return null; - return status === "cancelled" ? "canceled" : status; -} - -function sleepAgentRunProjectionSync(ms) { - return new Promise((resolve) => setTimeout(resolve, Math.max(0, Number(ms) || 0))); -} - -function codeAgentChatShortConnectionRequested(request, params, options = {}) { - const header = firstHeaderValue(request, "prefer"); - const explicit = firstHeaderValue(request, "x-hwlab-short-connection"); - const envValue = options.env?.HWLAB_CODE_AGENT_CHAT_SHORT_CONNECTION; - return codeAgentAgentRunAdapterEnabled(options.env ?? process.env) || - /(?:^|,)\s*respond-async\s*(?:,|$)/iu.test(String(header ?? "")) || - truthyFlag(explicit) || - params?.shortConnection === true || - truthyFlag(envValue); -} - -async function preflightCodeAgentBilling({ params = {}, options = {}, traceId, response, sendResponse = true } = {}) { - const client = options.userBillingClient; - if (!codeAgentBillingEnabled(options) || !client?.configured || !options.userBillingAuth?.active) return null; - const estimatedCredits = parsePositiveInteger(options.env?.HWLAB_USER_BILLING_CODE_AGENT_ESTIMATED_CREDITS, 1); - const body = { - ...(options.actorApiKeySecret ? { apiKey: options.actorApiKeySecret } : { userId: options.actor?.id }), - serviceId: "hwlab-code-agent", - estimatedCredits, - idempotencyKey: `code-agent:${traceId}:preflight`, - metadata: codeAgentBillingMetadata(params, traceId, { stage: "preflight" }) - }; - const result = await client.billingPreflight(body); - if (result.ok && result.body?.allowed !== false) return { reservation: sanitizeBillingReservation(result.body) }; - const status = [402, 403, 429].includes(Number(result.status)) ? Number(result.status) : 503; - const payload = { - ok: false, - accepted: false, - status: status === 402 ? "payment_required" : status === 403 ? "not_entitled" : status === 429 ? "rate_limited" : "billing_unavailable", - traceId, - error: { - code: result.error?.code ?? (status === 402 ? "insufficient_credits" : "user_billing_preflight_failed"), - layer: "billing", - retryable: status === 429 || status === 503, - message: result.error?.message ?? "Code Agent billing preflight failed", - route: "/v1/agent/chat" - }, - blocker: { - code: result.error?.code ?? (status === 402 ? "insufficient_credits" : "user_billing_preflight_failed"), - layer: "billing", - retryable: status === 429 || status === 503, - summary: result.error?.message ?? "Code Agent billing preflight failed" - }, - valuesRedacted: true - }; - if (sendResponse && response) sendJson(response, status, payload); - return { blocked: true, statusCode: status, payload, error: payload.error, blocker: payload.blocker }; -} - -async function recordCodeAgentBillingUsage({ payload = {}, params = {}, options = {} } = {}) { - const reservation = params.userBillingReservation ?? payload.userBillingReservation; - const reservationId = typeof reservation?.reservationId === "string" ? reservation.reservationId : ""; - const client = options.userBillingClient; - if (payload.billing?.recorded === true || payload.billing?.released === true || payload.status === "running") return payload.billing ?? null; - if (!reservationId || !client?.configured) return null; - const traceId = safeTraceId(payload.traceId ?? params.traceId); - const usedTokens = codeAgentUsedTokens(payload); - const usedCredits = parsePositiveInteger(options.env?.HWLAB_USER_BILLING_CODE_AGENT_USED_CREDITS, 1); - const result = await client.billingRecord({ - reservationId, - serviceId: "hwlab-code-agent", - usedCredits, - usedTokens, - idempotencyKey: `code-agent:${traceId}:record`, - metadata: codeAgentBillingMetadata(params, traceId, { stage: "record", status: payload.status ?? "unknown" }) - }); - const billing = result.ok ? sanitizeBillingRecord(result.body, reservation) : { - recorded: false, - reservationId, - errorCode: result.error?.code ?? "user_billing_record_failed", - valuesRedacted: true - }; - payload.billing = billing; - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - if (traceId) { - traceStore.append(traceId, { - type: "billing", - status: result.ok ? "recorded" : "degraded", - label: result.ok ? "billing:recorded" : "billing:record_failed", - reservationId, - recordId: result.ok ? result.body?.recordId ?? null : null, - errorCode: result.ok ? null : billing.errorCode, - valuesPrinted: false - }); - } - return billing; -} - -async function finalizeCodeAgentBillingUsage({ payload = {}, params = {}, options = {} } = {}) { - if (payload.billing?.recorded === true || payload.billing?.released === true) return payload.billing; - if (payload.status === "running") return null; - if (payload.status === "completed") { - return recordCodeAgentBillingUsage({ payload, params, options }); - } - return releaseCodeAgentBillingReservation({ payload, params, options }); -} - -async function releaseCodeAgentBillingReservation({ payload = {}, params = {}, options = {} } = {}) { - const reservation = params.userBillingReservation ?? payload.userBillingReservation; - const reservationId = typeof reservation?.reservationId === "string" ? reservation.reservationId : ""; - const client = options.userBillingClient; - if (payload.billing?.recorded === true || payload.billing?.released === true || !reservationId || !client?.configured) return payload.billing ?? null; - const traceId = safeTraceId(payload.traceId ?? params.traceId); - const releaseReason = String(payload.status ?? "not_completed").trim() || "not_completed"; - const releaseBody = { - reservationId, - serviceId: "hwlab-code-agent", - idempotencyKey: `code-agent:${traceId}:release`, - reason: releaseReason, - metadata: codeAgentBillingMetadata(params, traceId, { - stage: "release", - status: payload.status ?? "unknown", - errorCode: payload.error?.code ?? payload.blocker?.code ?? null - }) - }; - const result = typeof client.billingRelease === "function" - ? await client.billingRelease(releaseBody) - : { ok: false, error: { code: "user_billing_release_not_supported" } }; - const billing = result.ok ? sanitizeBillingRelease(result.body, reservation) : { - released: false, - reservationId, - errorCode: result.error?.code ?? "user_billing_release_failed", - valuesRedacted: true - }; - payload.billing = billing; - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - if (traceId) { - traceStore.append(traceId, { - type: "billing", - status: result.ok ? "released" : "degraded", - label: result.ok ? "billing:released" : "billing:release_failed", - reservationId, - reason: releaseReason, - errorCode: result.ok ? null : billing.errorCode, - valuesPrinted: false - }); - } - return billing; -} - -function withCodeAgentBillingReservation(payload = {}, params = {}) { - if (!payload || typeof payload !== "object" || payload.userBillingReservation) return payload; - const reservation = params.userBillingReservation; - return reservation ? { ...payload, userBillingReservation: reservation } : payload; -} - -function codeAgentBillingEnabled(options = {}) { - const env = options.env ?? process.env; - if (String(env.HWLAB_USER_BILLING_CODE_AGENT_ENABLED ?? "").trim() === "0") return false; - return Boolean(options.userBillingClient?.configured); -} - -function codeAgentBillingMetadata(params = {}, traceId, extra = {}) { - return { - traceId, - conversationId: safeConversationId(params.conversationId) || null, - sessionId: safeSessionId(params.sessionId) || null, - projectId: textValue(params.projectId) || null, - route: "/v1/agent/chat", - resource: "code-agent-turn", - ...extra, - valuesRedacted: true - }; -} - -function sanitizeBillingReservation(value = {}) { - return { - allowed: value.allowed === true, - reservationId: textValue(value.reservationId) || null, - resourceType: textValue(value.resourceType) || null, - planId: textValue(value.planId) || null, - estimatedCredits: Number.isFinite(Number(value.estimatedCredits)) ? Number(value.estimatedCredits) : null, - estimatedTokens: Number.isFinite(Number(value.estimatedTokens)) ? Number(value.estimatedTokens) : null, - expiresAt: textValue(value.expiresAt) || null, - valuesRedacted: true - }; -} - -function sanitizeBillingRecord(value = {}, reservation = {}) { - return { - recorded: true, - reservationId: textValue(reservation.reservationId) || null, - recordId: textValue(value.recordId) || null, - credits: Number.isFinite(Number(value.credits)) ? Number(value.credits) : null, - balance: Number.isFinite(Number(value.balance)) ? Number(value.balance) : null, - valuesRedacted: true - }; -} - -function sanitizeBillingRelease(value = {}, reservation = {}) { - return { - released: true, - reservationId: textValue(value.reservationId) || textValue(reservation.reservationId) || null, - releasedCredits: Number.isFinite(Number(value.releasedCredits)) ? Number(value.releasedCredits) : null, - status: textValue(value.status) || "cancelled", - valuesRedacted: true - }; -} - -function codeAgentUsedTokens(payload = {}) { - const usage = payload.usage && typeof payload.usage === "object" ? payload.usage : null; - const traceSummary = payload.traceSummary && typeof payload.traceSummary === "object" ? payload.traceSummary : null; - const candidates = [ - usage?.totalTokens, - usage?.total_tokens, - usage?.tokens, - Number(usage?.inputTokens ?? usage?.input_tokens ?? 0) + Number(usage?.outputTokens ?? usage?.output_tokens ?? 0), - traceSummary?.tokenCount, - traceSummary?.tokens - ]; - for (const value of candidates) { - const parsed = Number(value); - if (Number.isFinite(parsed) && parsed > 0) return Math.ceil(parsed); - } - return 0; -} - -export async function handleCodeAgentChatResultHttp(request, response, url, options) { - const parts = url.pathname.split("/").filter(Boolean); - const traceId = decodeURIComponent(parts[4] ?? ""); - if (!safeTraceId(traceId)) { - sendJson(response, 400, { - error: { - code: "invalid_trace_id", - message: "traceId must start with trc_ and contain only safe identifier characters" - } - }); - return; - } - const context = await measureCodeAgentHttpPhase(options, "result_projection_read", () => readCodeAgentCompatProjection(traceId, options)); - if (context.statusCode) { - sendJson(response, context.statusCode, context.body); - return; - } - const result = context.result; - if (result && result.status !== "running") { - sendJson(response, 200, codeAgentCompatProjectionPayload(compactCodeAgentChatResultPayload(result, options), context)); - return; - } - if (context.found) { - const body = codeAgentTurnStatusPayload({ traceId, result, snapshot: context.trace, resultPollError: null, refreshError: context.refreshError ?? null, options }); - const resultPollTerminalReady = codeAgentResultPollTerminalReady(body); - sendJson(response, resultPollTerminalReady ? 200 : 202, codeAgentCompatProjectionPayload({ - accepted: body.ok, - shortConnection: true, - ...body, - waitingFor: resultPollTerminalReady ? body.waitingFor ?? context.trace?.waitingFor ?? "workbench-projection" : "agentrun-result", - ...(body.terminal === true && !resultPollTerminalReady ? { status: "running", running: true, terminal: false } : {}) - }, context)); - return; - } - sendJson(response, 404, { - ...codeAgentCompatProjectionPayload({ traceId, status: "unknown" }, context), - error: { - code: "code_agent_result_not_found", - message: `No Code Agent result is registered for ${traceId}` - } - }); -} - -export async function handleCodeAgentTurnHttp(request, response, url, options) { - const startedAt = nowMs(); - const parts = url.pathname.split("/").filter(Boolean); - const traceId = decodeURIComponent(parts[3] ?? ""); - if (!safeTraceId(traceId)) { - const body = { - ok: false, - status: "unknown", - running: false, - terminal: false, - error: { - code: "invalid_trace_id", - message: "traceId must start with trc_ and contain only safe identifier characters" - } - }; - recordCodeAgentTurnReadMetric(options, url, 400, body, startedAt); - sendJson(response, 400, body); - return; - } - const requestOptions = codeAgentRequestScopedOptions(options); - let completed = false; - const emitTurnStatusReadSpan = (name, attributes = {}, error = null) => { - void emitCodeAgentOtelSpan(name, traceId, requestOptions.env ?? process.env, { - startTimeMs: startedAt, - status: error ? "error" : undefined, - error, - attributes: { - "http.method": "GET", - "http.route": "/v1/agent/turns/:traceId", - durationMs: nowMs() - startedAt, - ...attributes - } - }); - }; - response.once?.("close", () => { - if (completed) return; - emitTurnStatusReadSpan("turn_status_read.client_closed", { - status: "client_closed", - phase: "response_close", - terminal: false - }, new Error("Code Agent turn status client connection closed before response completed.")); - }); - try { - const projectMismatch = await measureCodeAgentHttpPhase(requestOptions, "turn_project_check", () => traceProjectMismatchSnapshot(traceId, url, requestOptions, "turn")); - if (projectMismatch) { - completed = true; - recordCodeAgentTurnReadMetric(options, url, projectMismatch.statusCode, projectMismatch.body, startedAt); - emitTurnStatusReadSpan("turn_status_read", { "http.status_code": projectMismatch.statusCode, status: projectMismatch.body?.status ?? null, terminal: projectMismatch.body?.terminal ?? null, phase: "project_check" }); - sendJson(response, projectMismatch.statusCode, projectMismatch.body); - return; - } - const resolved = await measureCodeAgentHttpPhase(requestOptions, "turn_resolve_status", () => resolveCodeAgentTurnStatusSnapshot(traceId, requestOptions)); - completed = true; - recordCodeAgentTurnReadMetric(options, url, resolved.statusCode, resolved.body, startedAt); - emitTurnStatusReadSpan("turn_status_read", { "http.status_code": resolved.statusCode, status: resolved.body?.status ?? null, terminal: resolved.body?.terminal ?? null, phase: "resolved" }); - sendJson(response, resolved.statusCode, resolved.body); - } catch (error) { - completed = true; - const body = { - ok: false, - status: "unknown", - running: false, - terminal: false, - traceId, - error: { - code: error?.code ?? "turn_status_read_failed", - message: error?.message ?? "Code Agent turn status read failed." - }, - valuesRedacted: true - }; - recordCodeAgentTurnReadMetric(options, url, 500, body, startedAt); - emitTurnStatusReadSpan("turn_status_read", { "http.status_code": 500, status: body.status, terminal: false, phase: "failed", errorCode: body.error.code }, error); - sendJson(response, 500, body); - } -} - -async function resolveCodeAgentTurnStatusSnapshot(traceId, options) { - const context = await readCodeAgentCompatProjection(traceId, options); - if (context.statusCode) return context; - const body = await measureCodeAgentHttpPhase(options, "turn_payload", () => codeAgentTurnStatusPayload({ - traceId, - result: context.result, - snapshot: context.trace, - resultPollError: null, - refreshError: context.refreshError ?? null, - options - })); - return { statusCode: body.ok ? 200 : 404, body: codeAgentCompatProjectionPayload(body, context) }; -} - -function codeAgentTurnStatusRefreshOptions(options = {}) { - const env = options.env ?? process.env; - const configuredBudgetMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS, DEFAULT_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS); - const upstreamTimeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, configuredBudgetMs); - const timeoutMs = Math.max(1, Math.min(configuredBudgetMs, upstreamTimeoutMs)); - return { - ...options, - skipAgentRunTerminalRefreshIfComplete: true, - env: { - ...env, - HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS: String(timeoutMs), - HWLAB_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS: String(timeoutMs) - } - }; -} - -function codeAgentRequestScopedOptions(options = {}) { - return { - ...options, - codeAgentTraceSessionCache: options.codeAgentTraceSessionCache instanceof Map ? options.codeAgentTraceSessionCache : new Map() - }; -} - -async function measureCodeAgentHttpPhase(options, phase, callback) { - const perf = options?.backendPerformance; - return perf && typeof perf.measure === "function" ? await perf.measure(phase, callback) : await callback(); -} - -function forbiddenTurnSnapshot(traceId) { - return { - statusCode: 403, - body: { - ok: false, - status: "unknown", - running: false, - terminal: false, - traceId, - error: { - code: "agent_session_owner_required", - message: "Only the session owner or admin can read this Code Agent turn status" - } - } - }; -} - -async function readCodeAgentCompatProjection(traceId, options = {}) { - const readModel = createWorkbenchReadModel(options, options.actor ?? null); - const session = await readModel.getSessionByTraceId(traceId); - let result = readModel.resultForTrace(traceId) ?? projectedSessionTraceResult(session, traceId); - if (result && !canAccessOwnedResult(result, options.actor)) return forbiddenTurnSnapshot(traceId); - if (result?.ownerUserId && !readModel.canReadOwner(result.ownerUserId)) return forbiddenTurnSnapshot(traceId); - const projectionTrace = await readModel.traceSnapshot(traceId); - let trace = traceSnapshotWithTerminalEvidence(projectionTrace, result, traceId, null); - const projection = readModel.projectionDiagnostics({ traceId, result, trace, refreshError: null }); - recordCodeAgentProjectionMetrics(options, { result, trace, projection, refreshError: null }); - const found = Boolean(result || session || (trace && trace.status !== "missing") || trace?.persisted === true); - return { result, session, trace, projection, found, refreshError: null }; -} - -function projectedSessionTraceResult(session, traceId) { - const safeId = safeTraceId(traceId); - if (!session || !safeId) return null; - const snapshot = session.session && typeof session.session === "object" ? session.session : null; - const traceResults = snapshot?.traceResults && typeof snapshot.traceResults === "object" ? snapshot.traceResults : null; - const stored = traceResults?.[safeId] && typeof traceResults[safeId] === "object" ? traceResults[safeId] : null; - if (!stored) return null; - return { - ...stored, - traceId: safeId, - conversationId: safeConversationId(stored.conversationId ?? session.conversationId) || null, - sessionId: safeSessionId(stored.sessionId ?? session.id) || null, - threadId: safeOpaqueId(stored.threadId ?? session.threadId) || null, - ownerUserId: session.ownerUserId ?? stored.ownerUserId ?? null, - ownerRole: session.ownerRole ?? stored.ownerRole ?? null, - status: stored.status ?? session.status ?? null, - valuesRedacted: true - }; -} - -function codeAgentCompatProjectionPayload(payload = {}, context = {}) { - const traceId = safeTraceId(payload.traceId ?? context.trace?.traceId) ?? null; - const projection = context.projection && typeof context.projection === "object" ? context.projection : {}; - return { - ...payload, - projection, - projectionStatus: projection.projectionStatus ?? null, - projectionHealth: projection.projectionHealth ?? null, - lastProjectedSeq: projection.lastProjectedSeq ?? null, - sourceRunId: projection.sourceRunId ?? null, - sourceCommandId: projection.sourceCommandId ?? null, - staleMs: projection.staleMs ?? null, - blocker: projection.blocker ?? null, - workbench: traceId ? { - detailOnly: true, - turnDetailUrl: `/v1/workbench/turns/${encodeURIComponent(traceId)}`, - traceEventsDetailUrl: `/v1/workbench/traces/${encodeURIComponent(traceId)}/events` - } : null, - valuesRedacted: true, - secretMaterialStored: false - }; -} - -function codeAgentResultPollTerminalReady(body = {}) { - if (body?.terminal !== true) return false; - return Boolean(messageAuthorityTextValue(body.finalResponse ?? body.reply ?? body.terminalEvidence?.finalResponse)); -} - -async function traceProjectMismatchSnapshot(traceId, url, options, resourceKind) { - const requestedProjectId = textValue(url.searchParams.get("projectId")); - if (!requestedProjectId || !options.accessController?.getAgentSessionByTraceId) return null; - const session = await getCodeAgentSessionByTraceId(traceId, options); - if (!session) return null; - if (session.ownerUserId && options.actor?.role !== "admin" && session.ownerUserId !== options.actor?.id) return forbiddenTurnSnapshot(traceId); - const actualProjectId = textValue(session.projectId); - if (!actualProjectId || actualProjectId === requestedProjectId) return null; - return { - statusCode: 404, - body: { - ok: false, - status: "not_found", - running: false, - terminal: false, - traceId, - error: { - code: "trace_project_mismatch", - message: `${resourceKind === "turn" ? "Agent turn" : "Agent trace"} is not visible in the requested project` - } - } - }; -} - -async function getCodeAgentSessionByTraceId(traceId, options = {}) { - const safeId = safeTraceId(traceId); - if (!safeId || typeof options.accessController?.getAgentSessionByTraceId !== "function") return null; - const cache = options.codeAgentTraceSessionCache; - if (cache instanceof Map) { - if (cache.has(safeId)) return cache.get(safeId); - const session = await options.accessController.getAgentSessionByTraceId(safeId); - cache.set(safeId, session ?? null); - return session ?? null; - } - return await options.accessController.getAgentSessionByTraceId(safeId); -} - -export function codeAgentTurnStatusPayload({ traceId, result, snapshot, resultPollError, refreshError, options }) { - const resultObject = result && typeof result === "object" ? result : null; - const snapshotObject = snapshot && typeof snapshot === "object" ? snapshot : null; - const lifecycle = codeAgentTurnLifecycleFields(traceId, resultObject ?? snapshotObject ?? {}); - const events = Array.isArray(snapshotObject?.events) ? snapshotObject.events : Array.isArray(resultObject?.runnerTrace?.events) ? resultObject.runnerTrace.events : []; - const lastEvent = events.at(-1) ?? null; - const finalResponse = resultObject?.finalResponse ?? snapshotObject?.finalResponse ?? snapshotObject?.terminalEvidence?.finalResponse ?? codeAgentFinalResponseEvidence(resultObject ?? snapshotObject ?? {}, traceId); - const terminalStatus = codeAgentAuthoritativeTerminalStatus(resultObject, snapshotObject, traceId); - const terminalSealBlocked = Boolean(terminalStatus && isTurnTerminalStatus(terminalStatus) && !codeAgentTerminalFinalText(finalResponse, resultObject, snapshotObject)); - const rawStatus = normalizeTurnStatus( - terminalStatus, - resultObject?.agentRun?.commandState, - resultObject?.agentRun?.status, - resultObject?.agentRun?.runStatus, - codeAgentRunningStatus(snapshotObject?.status), - codeAgentRunningStatus(snapshotObject?.traceStatus), - codeAgentRunningStatus(snapshotObject?.runnerTrace?.status), - codeAgentRunningStatus(resultObject?.status), - resultObject?.agentRun?.terminalStatus, - snapshotObject?.terminalEvidence?.traceSummary?.terminalStatus - ); - const status = terminalSealBlocked ? "running" : rawStatus; - const found = Boolean(resultObject || (snapshotObject && snapshotObject.status !== "missing") || snapshotObject?.persisted === true); - const running = isTurnRunningStatus(status); - const terminal = isTurnTerminalStatus(status); - const runnerTrace = snapshotObject && snapshotObject.status !== "missing" ? snapshotObject : resultObject?.runnerTrace ?? null; - const timingFields = codeAgentResultTimingFields(resultObject, snapshotObject, runnerTrace); - return { - ok: found, - action: "code-agent.turn.status", - contractVersion: "code-agent-turn-status-v1", - status: found ? status ?? "unknown" : "unknown", - running: found ? running : false, - terminal: found ? terminal : false, - traceId, - ...lifecycle, - conversationId: safeConversationId(resultObject?.conversationId ?? snapshotObject?.conversationId) || null, - sessionId: safeSessionId(resultObject?.sessionId ?? resultObject?.session?.sessionId ?? snapshotObject?.sessionId) || null, - threadId: safeOpaqueId(resultObject?.threadId ?? resultObject?.session?.threadId ?? snapshotObject?.threadId) || null, - updatedAt: textValue(resultObject?.updatedAt ?? resultObject?.agentRun?.updatedAt ?? snapshotObject?.updatedAt ?? timingFields.lastEventAt ?? timingFields.startedAt) || null, - ...timingFields, - lastEventLabel: textValue(snapshotObject?.lastEventLabel ?? runnerTrace?.lastEventLabel ?? lastEvent?.label ?? lastEvent?.type) || null, - waitingFor: terminalSealBlocked ? "final_response" : textValue(snapshotObject?.waitingFor ?? runnerTrace?.waitingFor) || null, - terminalObserved: Boolean(terminalStatus), - terminalObservedStatus: terminalStatus ?? null, - terminalSealBlocked, - resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`, - traceUrl: `/v1/agent/traces/${encodeURIComponent(traceId)}`, - turnUrl: `/v1/agent/turns/${encodeURIComponent(traceId)}`, - runnerTrace: runnerTrace ? compactRunnerTraceForResult(runnerTrace, resultTraceEventLimit(options)) : null, - agentRun: resultObject?.agentRun ?? snapshotObject?.agentRun ?? null, - terminalEvidence: snapshotObject?.terminalEvidence ?? null, - finalResponse, - traceSummary: resultObject?.traceSummary ?? snapshotObject?.traceSummary ?? snapshotObject?.terminalEvidence?.traceSummary ?? null, - retention: snapshotObject?.retention ?? null, - eventCount: numberOrNull(snapshotObject?.eventCount ?? runnerTrace?.eventCount ?? events.length), - error: resultPollError || refreshError - ? codeAgentRefreshErrorPayload(resultPollError ?? refreshError, traceId, resultObject?.agentRun ?? snapshotObject?.agentRun, "turn_status_degraded") - : resultObject?.error ?? snapshotObject?.error ?? null, - valuesRedacted: true, - secretMaterialStored: false - }; -} - -function codeAgentTerminalFinalText(finalResponse = null, resultObject = null, snapshotObject = null) { - const candidates = [ - finalResponse, - resultObject?.finalResponse, - snapshotObject?.finalResponse, - snapshotObject?.terminalEvidence?.finalResponse, - resultObject?.terminalEvidence?.finalResponse - ]; - for (const value of candidates) { - const text = conversationText(value); - if (text) return text; - } - return codeAgentFinalResponseText(resultObject ?? {}) || codeAgentFinalResponseText(snapshotObject ?? {}); -} - -function codeAgentAuthoritativeTerminalStatus(resultObject, snapshotObject, traceId) { - const sealedPayload = codeAgentPayloadHasSealedFinalResponse(resultObject ?? {}) ? resultObject : codeAgentPayloadHasSealedFinalResponse(snapshotObject ?? {}) ? snapshotObject : null; - if (sealedPayload) { - const sealedStatus = normalizeTurnStatus(sealedPayload?.finalResponse?.status, sealedPayload?.terminalEvidence?.finalResponse?.status, sealedPayload?.agentRun?.terminalStatus, sealedPayload?.agentRun?.status, sealedPayload?.status); - return sealedStatus && isTurnTerminalStatus(sealedStatus) ? sealedStatus : "completed"; - } - const snapshotStatus = normalizeTurnStatus(snapshotObject?.terminalEvidence?.traceSummary?.terminalStatus, snapshotObject?.terminalEvidence?.agentRun?.terminalStatus, snapshotObject?.terminalEvidence?.status, snapshotObject?.status); - if (snapshotStatus && isTurnTerminalStatus(snapshotStatus) && codeAgentSnapshotHasTerminalAuthority(snapshotObject)) return snapshotStatus; - const agentRunStatus = normalizeTurnStatus(resultObject?.agentRun?.terminalStatus, resultObject?.agentRun?.commandState, resultObject?.agentRun?.status, resultObject?.agentRun?.runStatus); - if (agentRunStatus && isTurnTerminalStatus(agentRunStatus)) return agentRunStatus; - const resultStatus = normalizeTurnStatus(resultObject?.status); - if (resultStatus && isTurnTerminalStatus(resultStatus) && codeAgentResultHasTerminalAuthority(resultObject, traceId)) return resultStatus; - return null; -} - -function codeAgentSnapshotHasTerminalAuthority(snapshot = null) { - if (!snapshot || typeof snapshot !== "object") return false; - if (snapshot.terminal === true || snapshot.sealed === true) return true; - if (snapshot.terminalEvidence?.available === true || snapshot.terminalEvidence?.source) return true; - const events = Array.isArray(snapshot.events) ? snapshot.events : []; - return events.some((event) => event?.terminal === true); -} - -function codeAgentResultHasTerminalAuthority(result = null, traceId = null) { - if (!result || typeof result !== "object") return false; - if (result.terminal === true || result.sealed === true) return true; - if (result.error || result.blocker) return true; - if (codeAgentPayloadHasSealedFinalResponse(result)) return true; - if (agentRunTerminalTraceEvidence(result, traceId)) return true; - return Boolean(textValue(result.finishedAt ?? result.completedAt ?? result.endedAt)); -} - -function codeAgentRefreshErrorPayload(error, traceId, agentRun, fallbackCode) { - return { - code: error?.code ?? fallbackCode, - layer: error?.layer ?? "agentrun", - category: error?.category ?? (error?.code === "agentrun_timeout" ? "upstream-timeout" : "upstream-refresh-failed"), - retryable: error?.retryable !== false, - message: error?.message ?? "Code Agent turn status refresh degraded", - traceId, - runId: agentRun?.runId ?? error?.runId ?? null, - commandId: agentRun?.commandId ?? error?.commandId ?? null, - method: error?.method ?? null, - route: error?.route ?? error?.path ?? null, - timeoutMs: numberOrNull(error?.timeoutMs), - timeoutStage: error?.timeoutStage ?? null, - upstreamStatus: numberOrNull(error?.statusCode), - managerHost: error?.managerHost ?? null, - valuesPrinted: false - }; -} - -function normalizeTurnStatus(...values) { - for (const value of values) { - const text = textValue(value).toLowerCase().replace(/_/gu, "-"); - if (!text) continue; - if (["accepted", "pending", "processing", "running", "busy", "creating", "queued", "in-flight"].includes(text)) return "running"; - if (["completed", "done", "succeeded", "success"].includes(text)) return "completed"; - if (["failed", "failure", "error", "stale", "thread-resume-failed", "aborted", "interrupted", "expired"].includes(text)) return "failed"; - if (["cancelled", "canceled"].includes(text)) return "canceled"; - if (["blocked", "timeout"].includes(text)) return text; - } - return null; -} - -function isTurnRunningStatus(status) { - return status === "running"; -} - -function codeAgentRunningStatus(value) { - const status = normalizeTurnStatus(value); - return isTurnRunningStatus(status) ? status : null; -} - -function isTurnTerminalStatus(status) { - return CODE_AGENT_TERMINAL_STATUSES.has(String(status ?? "").trim().toLowerCase().replace(/_/gu, "-")); -} - -export async function handleCodeAgentInspectHttp(request, response, url, options) { - const query = { - conversationId: safeConversationId(url.searchParams.get("conversationId")), - sessionId: safeSessionId(url.searchParams.get("sessionId")), - threadId: safeOpaqueId(url.searchParams.get("threadId")), - traceId: safeTraceId(url.searchParams.get("traceId")) - }; - const sessionRegistry = options.sessionRegistry; - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - const manager = options.codexStdioManager; - const managerDescription = manager && typeof manager.describe === "function" ? manager.describe() : null; - const managerRecentSessions = Array.isArray(managerDescription?.recentSessions) ? managerDescription.recentSessions : []; - const directManagerSession = query.sessionId && manager && typeof manager.get === "function" - ? manager.get(query.sessionId, { conversationId: query.conversationId }) - : null; - const matchedManagerSession = directManagerSession ?? managerRecentSessions.find((session) => sessionMatchesInspectQuery(session, query)) ?? null; - const registryInspect = sessionRegistry && typeof sessionRegistry.inspect === "function" - ? sessionRegistry.inspect(query) - : { ok: false, status: "unavailable", traceIds: [], session: null, conversationFacts: null }; - const resultEvidence = await codeAgentInspectResultEvidence(query.traceId, options); - const session = matchedManagerSession ?? registryInspect.session ?? resultEvidence.session ?? null; - const conversationFacts = registryInspect.conversationFacts ?? resultEvidence.conversationFacts ?? null; - const requestedRunnerTrace = query.traceId ? traceStore.snapshot(query.traceId) : null; - const requestedTraceFound = Boolean(requestedRunnerTrace && requestedRunnerTrace.status !== "missing"); - const traceIds = uniqueStrings([ - requestedTraceFound ? query.traceId : null, - session?.currentTraceId, - session?.lastTraceId, - conversationFacts?.latestTraceId, - resultEvidence.latestTraceId, - ...(Array.isArray(conversationFacts?.traceIds) ? conversationFacts.traceIds : []), - ...(Array.isArray(registryInspect.traceIds) ? registryInspect.traceIds : []) - ]); - const latestTraceId = traceIds[0] ?? null; - const runnerTrace = latestTraceId - ? latestTraceId === query.traceId && requestedRunnerTrace - ? requestedRunnerTrace - : traceStore.snapshot(latestTraceId) - : null; - const found = Boolean(registryInspect.ok || session || latestTraceId); - sendJson(response, found ? 200 : 404, { - ok: found, - action: "code-agent.chat.inspect", - status: found ? "found" : "not_found", - query, - session, - conversationFacts, - traceIds, - latestTraceId, - traceUrl: latestTraceId ? `/v1/agent/traces/${encodeURIComponent(latestTraceId)}` : null, - resultUrl: latestTraceId ? `/v1/agent/chat/result/${encodeURIComponent(latestTraceId)}` : null, - runnerTrace, - valuesRedacted: true, - secretMaterialStored: false - }); -} - -async function codeAgentInspectResultEvidence(traceId, options = {}) { - if (!safeTraceId(traceId)) return { session: null, conversationFacts: null, latestTraceId: null }; - const cached = options.codeAgentChatResults?.get?.(traceId) ?? null; - const persisted = cached ? null : await loadPersistedAgentRunResult(traceId, options); - const result = cached ?? persisted ?? null; - if (!result || typeof result !== "object") return { session: null, conversationFacts: null, latestTraceId: null }; - const agentRun = result.agentRun && typeof result.agentRun === "object" ? result.agentRun : null; - const resultSession = result.session && typeof result.session === "object" ? result.session : null; - const sessionReuse = result.sessionReuse && typeof result.sessionReuse === "object" ? result.sessionReuse : null; - const providerTrace = result.providerTrace && typeof result.providerTrace === "object" ? result.providerTrace : null; - const sessionId = safeSessionId(result.sessionId ?? resultSession?.sessionId ?? sessionReuse?.sessionId ?? agentRun?.sessionId) || null; - const conversationId = safeConversationId(result.conversationId ?? resultSession?.conversationId ?? sessionReuse?.conversationId ?? agentRun?.conversationId) || null; - const threadId = safeOpaqueId(result.threadId ?? resultSession?.threadId ?? sessionReuse?.threadId ?? providerTrace?.threadId ?? agentRun?.threadId) || null; - const status = textValue(resultSession?.status ?? result.status ?? agentRun?.status) || null; - const updatedAt = textValue(result.updatedAt ?? agentRun?.updatedAt ?? resultSession?.updatedAt) || null; - const session = sessionId || conversationId || threadId ? { - sessionId, - conversationId, - threadId, - status, - lastTraceId: traceId, - source: "code-agent-result", - agentRun: agentRun ? agentRunSessionEvidence(result).agentRun : null, - valuesRedacted: true, - secretMaterialStored: false - } : null; - const conversationFacts = conversationId ? { - conversationId, - sessionId, - threadId, - latestTraceId: traceId, - traceIds: [traceId], - turnCount: 1, - source: "code-agent-result", - latestStatus: status, - updatedAt, - valuesRedacted: true, - secretMaterialStored: false - } : null; - return { session, conversationFacts, latestTraceId: traceId }; -} - -function sessionMatchesInspectQuery(session, query) { - if (!session || typeof session !== "object") return false; - if (query.sessionId && session.sessionId === query.sessionId) return true; - if (query.threadId && session.threadId === query.threadId) return true; - if (query.conversationId && session.conversationId === query.conversationId) return true; - if (query.traceId && (session.currentTraceId === query.traceId || session.lastTraceId === query.traceId)) return true; - return false; -} - -export async function handleCodeAgentCancelHttp(request, response, options) { - const body = await readBody(request, options.bodyLimitBytes); - let params = {}; - - try { - params = body ? JSON.parse(body) : {}; - } catch (error) { - sendJson(response, 400, { - ...createCodeAgentErrorPayload({ - code: "parse_error", - message: "Invalid JSON body", - reason: error.message, - traceId: getHeader(request, "x-trace-id") || "trc_unassigned", - layer: "api", - retryable: true - }) - }); - return; - } - - if (!params || typeof params !== "object" || Array.isArray(params)) { - sendJson(response, 400, { - ...createCodeAgentErrorPayload({ - code: "invalid_params", - message: "Code Agent cancel body must be a JSON object", - traceId: getHeader(request, "x-trace-id") || "trc_unassigned", - layer: "api", - retryable: true - }) - }); - return; - } - - const traceId = safeTraceId(getHeader(request, "x-trace-id") || params.traceId); - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - const snapshot = traceId ? traceStore.snapshot(traceId) : null; - const currentResult = traceId ? options.codeAgentChatResults?.get(traceId) ?? await loadPersistedAgentRunResult(traceId, options) : null; - if (currentResult?.agentRun?.commandId) { - const mismatch = codeAgentCancelScopeMismatch(params, currentResult); - if (mismatch) { - traceStore.append(traceId, { - type: "cancel", - status: "blocked", - label: "agentrun:cancel:scope_mismatch", - errorCode: "cancel_scope_mismatch", - message: `Cancel request ${mismatch.field} does not match the trace owner; AgentRun cancel was not forwarded.`, - requested: mismatch.requested, - expected: mismatch.expected, - waitingFor: "cancel-scope", - valuesPrinted: false - }); - sendJson(response, 409, cancelBlockedPayload({ - code: "cancel_scope_mismatch", - message: `取消请求的 ${mismatch.field} 与 trace 归属不一致，已拒绝转发 AgentRun cancel，避免误取消其他 session。`, - traceId, - conversationId: safeConversationId(params.conversationId) || safeConversationId(currentResult.conversationId) || null, - sessionId: safeSessionId(params.sessionId) || safeSessionId(currentResult.sessionId) || null, - runnerTrace: traceStore.snapshot(traceId), - status: "blocked" - })); - return; - } - await recordCodeAgentSessionInputFact({ - params: { - ...params, - sessionId: currentResult.sessionId ?? currentResult.session?.sessionId ?? params.sessionId, - messageId: codeAgentTurnLifecycleFields(traceId, currentResult).userMessageId, - ownerUserId: options.actor?.id, - ownerRole: options.actor?.role - }, - options, - traceId, - delivery: "cancel", - status: "promoted", - commandId: currentResult.agentRun.commandId - }); - const payload = await cancelAgentRunChatTurn({ traceId, currentResult, options, traceStore }); - if (payload) { - if (payload.alreadyTerminal === true || isTraceCommandTerminalStatus(payload.status)) { - await recordCodeAgentTerminalTurnStatusEffects({ payload, params: { ...params, traceId, ownerUserId: options.actor?.id, ownerRole: options.actor?.role }, options, preserveLastTraceId: true }); - } else { - await recordCodeAgentSessionOwner({ payload, params: { ...params, traceId, ownerUserId: options.actor?.id, ownerRole: options.actor?.role }, options, status: "canceled" }); - } - sendJson(response, 200, payload); - return; - } - } - const sessionId = safeSessionId(params.sessionId) || safeSessionId(snapshot?.sessionId); - const conversationId = safeConversationId(params.conversationId); - const manager = options.codexStdioManager; - - if (!traceId) { - sendJson(response, 400, cancelBlockedPayload({ - code: "cancel_trace_missing", - message: "traceId is required to cancel the current Code Agent request.", - traceId: "trc_unassigned", - conversationId, - sessionId - })); - return; - } - - if (!manager || typeof manager.get !== "function" || typeof manager.cancel !== "function") { - traceStore.append(traceId, { - type: "cancel", - status: "unsupported", - label: "cancel:unsupported", - errorCode: "cancel_unsupported", - message: "Codex stdio cancel/interrupt backend is not available on this runtime.", - waitingFor: "session-control" - }); - const runnerTrace = traceStore.snapshot(traceId); - sendJson(response, 501, cancelBlockedPayload({ - code: "cancel_unsupported", - message: "当前后端不支持 Code Agent interrupt/cancel；本次按 unsupported/degraded 返回，不会静默假装已中断。", - traceId, - conversationId, - sessionId, - runnerTrace, - status: "degraded", - unsupported: true, - degraded: true - })); - return; - } - - if (!sessionId) { - traceStore.append(traceId, { - type: "cancel", - status: "blocked", - label: "cancel:not_cancelable", - errorCode: "cancel_session_missing", - message: "Cancel request did not include a bound Codex stdio sessionId.", - waitingFor: "session-binding" - }); - sendJson(response, 409, cancelBlockedPayload({ - code: "cancel_session_missing", - message: "当前请求尚未暴露可取消的 Codex stdio sessionId；页面不能只隐藏 UI，已保留输入和 trace。", - traceId, - conversationId, - sessionId: null, - runnerTrace: traceStore.snapshot(traceId) - })); - return; - } - - const currentSession = manager.get(sessionId, { conversationId }) ?? null; - if (!currentSession || !["busy", "creating"].includes(currentSession.status)) { - traceStore.append(traceId, { - type: "cancel", - status: "blocked", - label: "cancel:not_in_flight", - errorCode: "cancel_not_in_flight", - message: `Session ${sessionId} is not an in-flight Codex stdio request.`, - sessionId, - sessionStatus: currentSession?.status ?? "missing" - }); - sendJson(response, 409, cancelBlockedPayload({ - code: currentSession ? "cancel_not_in_flight" : "cancel_session_not_found", - message: currentSession - ? `当前 session 状态为 ${currentSession.status}，没有可取消的 in-flight Codex stdio 请求。` - : `没有找到 sessionId=${sessionId} 的 Codex stdio session。`, - traceId, - conversationId, - sessionId, - session: currentSession, - runnerTrace: traceStore.snapshot(traceId) - })); - return; - } - - const canceledSession = manager.cancel(sessionId, { - traceId, - conversationId, - reason: "user_cancel" - }); - traceStore.append(traceId, { - type: "cancel", - status: "canceled", - label: "cancel:canceled", - message: "User canceled the current Codex stdio request.", - sessionId, - sessionStatus: canceledSession?.status ?? "canceled", - sessionLifecycleStatus: codeAgentSessionLifecycleSummary({ session: canceledSession, status: "canceled" }).status, - waitingFor: "user-retry", - terminal: true - }); - const runnerTraceSnapshot = traceStore.snapshot(traceId); - const sessionSummary = codeAgentSessionLifecycleSummary({ - session: canceledSession, - runnerTrace: runnerTraceSnapshot, - status: "canceled" - }); - const runnerTrace = { - ...runnerTraceSnapshot, - sessionLifecycleStatus: sessionSummary.status - }; - const payload = { - accepted: true, - canceled: true, - status: "canceled", - conversationId: conversationId ?? canceledSession?.conversationId ?? null, - sessionId, - traceId, - session: canceledSession, - sessionLifecycleStatus: sessionSummary.status, - sessionLifecycle: sessionSummary, - sessionSummary, - runnerTrace, - lastTraceEvent: runnerTrace.lastEvent, - retryable: true, - error: { - code: "codex_stdio_canceled", - layer: "session", - category: "canceled", - retryable: true, - message: "user canceled current Code Agent request", - userMessage: "当前 Codex stdio 请求已取消；输入、sessionId、traceId 和最后 trace event 已保留，可重试上一条消息。", - traceId, - route: "/v1/agent/chat/cancel", - toolName: "codex-stdio.cancel" - }, - userMessage: "当前 Codex stdio 请求已取消；输入、sessionId、traceId 和最后 trace event 已保留，可重试上一条消息。", - updatedAt: new Date().toISOString() - }; - await recordCodeAgentSessionOwner({ payload, params: { ...params, traceId, ownerUserId: options.actor?.id, ownerRole: options.actor?.role }, options, status: "canceled" }); - recordCodeAgentConversationFact(payload, options); - options.codeAgentChatResults?.set(traceId, annotateOwner(payload, { ownerUserId: options.actor?.id, ownerRole: options.actor?.role })); - sendJson(response, 200, payload); -} - -function recordCodeAgentConversationFact(payload = {}, options = {}) { - const registry = options.sessionRegistry; - const conversationId = safeConversationId(payload.conversationId); - if (!conversationId || typeof registry?.recordFact !== "function") return null; - try { - return registry.recordFact(conversationId, { - kind: payload.status === "completed" ? "runner_turn" : "runner_turn_terminal", - conversationId, - sessionId: payload.sessionId, - traceId: payload.traceId, - provider: payload.provider, - backend: payload.backend, - workspace: payload.workspace, - sandbox: payload.sandbox, - session: payload.session, - sessionMode: payload.sessionMode, - sessionReuse: payload.sessionReuse, - implementationType: payload.implementationType, - runner: payload.runner, - runnerTrace: payload.runnerTrace, - partialContext: payload.partialContext ?? payload.runnerTrace?.partialContext, - capabilityLevel: payload.capabilityLevel, - toolCalls: payload.toolCalls, - skills: payload.skills - }, { now: options.now }); - } catch { - return null; - } -} - -async function recordCodeAgentTerminalTurnStatusEffects({ payload = {}, params = {}, options = {}, preserveLastTraceId = false } = {}) { - payload = codeAgentPayloadWithObservedTerminalStatus(payload, payload?.runnerTrace) ?? payload; - if (!payload || typeof payload !== "object" || !isTraceCommandTerminalStatus(payload.status)) return null; - if (payload.turnStatusTerminalEffects?.recorded === true) return payload.turnStatusTerminalEffects; - const billing = await finalizeCodeAgentBillingUsage({ payload, params, options }); - recordCodeAgentConversationFact(payload, options); - const owner = await recordCodeAgentSessionOwner({ payload, params, options, status: codeAgentOwnerStatusForResult(payload), preserveLastTraceId }); - const billingSettled = codeAgentTerminalBillingSettled({ payload, params, options, billing }); - const ownerRequired = Boolean(options.actor?.id && options.accessController?.recordAgentSessionOwner); - const ownerSettled = !ownerRequired || Boolean(owner); - const effects = { - recorded: billingSettled && ownerSettled, - pending: false, - billingSettled, - ownerSettled, - preserveLastTraceId: Boolean(preserveLastTraceId), - recordedAt: new Date().toISOString(), - valuesPrinted: false - }; - payload.turnStatusTerminalEffects = effects; - const traceId = safeTraceId(payload.traceId ?? params.traceId); - if (traceId) void emitCodeAgentOtelSpan("projection_write", traceId, options.env ?? process.env, { - attributes: { - projection: "terminal-effects", - status: payload.status, - terminal: true, - billingSettled, - ownerSettled, - sessionId: safeSessionId(payload.sessionId ?? params.sessionId) || null, - turnId: codeAgentTurnLifecycleFields(traceId, payload).turnId, - runId: payload.agentRun?.runId ?? null, - commandId: payload.agentRun?.commandId ?? null - } - }); - if (traceId) options.codeAgentChatResults?.set?.(traceId, payload); - return effects; -} - -function scheduleCodeAgentTerminalTurnStatusEffects({ payload = {}, params = {}, options = {}, preserveLastTraceId = false } = {}) { - payload = codeAgentPayloadWithObservedTerminalStatus(payload, payload?.runnerTrace) ?? payload; - if (!payload || typeof payload !== "object" || !isTraceCommandTerminalStatus(payload.status)) return null; - const existing = payload.turnStatusTerminalEffects; - if (existing?.recorded === true || existing?.pending === true) return existing; - const traceId = safeTraceId(payload.traceId ?? params.traceId); - const pending = { - recorded: false, - pending: true, - billingSettled: false, - ownerSettled: false, - preserveLastTraceId: Boolean(preserveLastTraceId), - scheduledAt: new Date().toISOString(), - valuesPrinted: false - }; - payload.turnStatusTerminalEffects = pending; - if (traceId) options.codeAgentChatResults?.set?.(traceId, payload); - setImmediate(() => { - void recordCodeAgentTerminalTurnStatusEffects({ payload, params, options, preserveLastTraceId }).catch((error) => { - const failed = { - recorded: false, - pending: false, - billingSettled: false, - ownerSettled: false, - preserveLastTraceId: Boolean(preserveLastTraceId), - errorCode: error?.code ?? "terminal_turn_status_effects_failed", - recordedAt: new Date().toISOString(), - valuesPrinted: false - }; - payload.turnStatusTerminalEffects = failed; - if (traceId) { - options.codeAgentChatResults?.set?.(traceId, payload); - (options.traceStore ?? defaultCodeAgentTraceStore).append(traceId, { - type: "turn-status", - status: "degraded", - label: "turn-status:terminal-effects-failed", - errorCode: failed.errorCode, - message: error?.message ?? "Terminal turn status side effects failed and will retry on the next poll.", - valuesPrinted: false - }); - } - }); - }); - return pending; -} - -function codeAgentTerminalBillingSettled({ payload = {}, params = {}, options = {}, billing = null } = {}) { - const reservation = params.userBillingReservation ?? payload.userBillingReservation; - const reservationId = typeof reservation?.reservationId === "string" ? reservation.reservationId : ""; - if (!reservationId || !options.userBillingClient?.configured) return true; - return billing?.recorded === true || billing?.released === true || payload.billing?.recorded === true || payload.billing?.released === true; -} - -export async function handleCodeAgentSteerHttp(request, response, options) { - const body = await readBody(request, options.bodyLimitBytes); - let params = {}; - - try { - params = body ? JSON.parse(body) : {}; - } catch (error) { - sendJson(response, 400, { - ...createCodeAgentErrorPayload({ - code: "parse_error", - message: "Invalid JSON body", - reason: error.message, - traceId: getHeader(request, "x-trace-id") || "trc_unassigned", - layer: "api", - retryable: true, - route: "/v1/agent/chat/steer" - }) - }); - return; - } - - if (!params || typeof params !== "object" || Array.isArray(params)) { - sendJson(response, 400, { - ...createCodeAgentErrorPayload({ - code: "invalid_params", - message: "Code Agent steer body must be a JSON object", - traceId: getHeader(request, "x-trace-id") || "trc_unassigned", - layer: "api", - retryable: true, - route: "/v1/agent/chat/steer" - }) - }); - return; - } - - const traceId = safeTraceId(getHeader(request, "x-trace-id") || params.traceId || params.targetTraceId); - const message = firstNonEmptyValue(params.message, params.prompt, params.text); - if (!traceId || !message) { - sendJson(response, 400, { - ...createCodeAgentErrorPayload({ - code: !traceId ? "steer_trace_missing" : "steer_message_missing", - message: !traceId ? "traceId is required to steer the current Code Agent turn." : "message is required to steer the current Code Agent turn.", - traceId: traceId || "trc_unassigned", - layer: "api", - retryable: true, - route: "/v1/agent/chat/steer" - }) - }); - return; - } - - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - const currentResult = options.codeAgentChatResults?.get?.(traceId) ?? await loadPersistedAgentRunResult(traceId, options); - if (!canAccessOwnedResult(currentResult, options.actor)) { - sendJson(response, 403, { - ok: false, - status: "forbidden", - traceId, - error: { code: "code_agent_trace_forbidden", message: "Code Agent trace is not visible to the current actor" } - }); - return; - } - if (!currentResult?.agentRun?.runId || !currentResult?.agentRun?.commandId) { - sendJson(response, 404, { - ok: false, - status: "not_found", - traceId, - error: { code: "steer_trace_not_found", message: "No AgentRun-backed Code Agent turn was found for the requested traceId." }, - route: "/v1/agent/chat/steer", - runnerTrace: traceStore.snapshot(traceId) - }); - return; - } - const unsealedTerminalError = traceSnapshotTerminalFailureDiagnostic(traceStore.snapshot(traceId)); - if (unsealedTerminalError) { - const terminalSync = await syncAgentRunTerminalBeforeSteer({ traceId, currentResult, params, options, traceStore }); - const syncedResult = terminalSync?.payload ?? currentResult; - if (terminalSync?.terminal === true) { - sendJson(response, 409, { - ok: false, - accepted: false, - status: "blocked", - traceId, - route: "/v1/agent/chat/steer", - sessionId: safeSessionId(syncedResult.sessionId ?? params.sessionId) || null, - threadId: safeOpaqueId(syncedResult.threadId ?? params.threadId) || null, - agentRun: syncedResult.agentRun ?? currentResult.agentRun, - runnerTrace: terminalSync.runnerTrace ?? traceStore.snapshot(traceId), - terminalDiagnostic: unsealedTerminalError, - error: { - code: "steer_trace_terminal", - layer: "api", - category: "not_in_flight", - retryable: false, - message: `Trace ${traceId} is already terminal and cannot accept a steer command.`, - traceId, - route: "/v1/agent/chat/steer", - toolName: "agentrun.command.steer" - }, - valuesPrinted: false - }); - return; - } - traceStore.append(traceId, { - type: "backend", - status: "blocked", - label: "agentrun:steer:blocked-unsealed-terminal-error", - errorCode: terminalSync?.error?.code ?? unsealedTerminalError.failureKind ?? "steer_trace_unsealed_terminal_error", - message: terminalSync?.error?.message ?? "Trace has an unsealed terminal error diagnostic; HWLAB refused to steer a new prompt into the old trace.", - waitingFor: "terminal-projection-reconciliation", - terminalDiagnostic: unsealedTerminalError, - terminal: false, - valuesPrinted: false - }); - sendJson(response, terminalSync?.error ? 503 : 409, { - ok: false, - accepted: false, - status: "blocked", - traceId, - route: "/v1/agent/chat/steer", - sessionId: safeSessionId(currentResult.sessionId ?? params.sessionId) || null, - threadId: safeOpaqueId(currentResult.threadId ?? params.threadId) || null, - agentRun: currentResult.agentRun, - runnerTrace: traceStore.snapshot(traceId), - terminalDiagnostic: unsealedTerminalError, - error: { - code: terminalSync?.error?.code ?? "steer_trace_unsealed_terminal_error", - layer: terminalSync?.error?.layer ?? "api", - category: "not_in_flight", - retryable: false, - message: terminalSync?.error?.message ?? "Trace has an unsealed terminal error diagnostic; start a new turn instead of steering into the old trace.", - traceId, - route: "/v1/agent/chat/steer", - toolName: "agentrun.command.steer" - }, - valuesPrinted: false - }); - return; - } - if (isTraceCommandTerminalStatus(currentResult.status)) { - sendJson(response, 409, { - ok: false, - accepted: false, - status: "blocked", - traceId, - route: "/v1/agent/chat/steer", - sessionId: safeSessionId(currentResult.sessionId ?? params.sessionId) || null, - threadId: safeOpaqueId(currentResult.threadId ?? params.threadId) || null, - agentRun: currentResult.agentRun, - runnerTrace: traceStore.snapshot(traceId), - error: { - code: "steer_trace_terminal", - layer: "api", - category: "not_in_flight", - retryable: false, - message: `Trace ${traceId} is already terminal and cannot accept a steer command.`, - traceId, - route: "/v1/agent/chat/steer", - toolName: "agentrun.command.steer" - }, - valuesPrinted: false - }); - return; - } - - try { - await recordCodeAgentSessionInputFact({ - params: { - ...params, - sessionId: currentResult.sessionId ?? currentResult.session?.sessionId ?? params.sessionId, - ownerUserId: options.actor?.id, - ownerRole: options.actor?.role - }, - options, - traceId, - delivery: "steer", - status: "promoted", - commandId: currentResult.agentRun.commandId - }); - const payload = await steerAgentRunChatTurn({ - traceId, - currentResult, - params: { - ...params, - message, - ownerUserId: options.actor?.id ?? params.ownerUserId, - ownerRole: options.actor?.role ?? params.ownerRole - }, - options, - traceStore - }); - sendJson(response, 202, payload); - setImmediate(() => { - void recordCodeAgentSessionOwner({ payload: currentResult, params: { ...params, traceId, ownerUserId: options.actor?.id, ownerRole: options.actor?.role }, options, status: "running" }); - }); - } catch (error) { - traceStore.append(traceId, { - type: "backend", - status: "failed", - label: "agentrun:steer:failed", - errorCode: error?.code ?? "agentrun_steer_failed", - message: error?.message ?? "AgentRun steer command failed.", - terminal: false, - valuesPrinted: false - }); - sendJson(response, error?.statusCode === 404 ? 404 : 409, { - ok: false, - accepted: false, - status: "failed", - traceId, - route: "/v1/agent/chat/steer", - sessionId: safeSessionId(currentResult.sessionId ?? params.sessionId) || null, - threadId: safeOpaqueId(currentResult.threadId ?? params.threadId) || null, - agentRun: currentResult.agentRun, - runnerTrace: traceStore.snapshot(traceId), - error: { - code: error?.code ?? "agentrun_steer_failed", - layer: "agentrun", - category: "steer_failed", - retryable: true, - message: error?.message ?? "AgentRun steer command failed.", - traceId, - route: "/v1/agent/chat/steer", - toolName: "agentrun.command.steer" - } - }); - } -} - -async function syncAgentRunTerminalBeforeSteer({ traceId, currentResult = null, params = {}, options = {}, traceStore = defaultCodeAgentTraceStore } = {}) { - try { - const synced = await syncAgentRunChatResult({ - traceId, - currentResult, - options: { ...options, deferAgentRunResultSync: false }, - traceStore, - forceResultSync: true, - refreshEvents: true - }); - const payload = codeAgentPayloadWithObservedTerminalStatus(synced?.result ?? null, synced?.runnerTrace ?? traceStore.snapshot(traceId)) ?? synced?.result ?? currentResult; - if (isTraceCommandTerminalStatus(payload?.status)) { - await recordCodeAgentTerminalTurnStatusEffects({ - payload, - params: { ...params, traceId, ownerUserId: options.actor?.id ?? params.ownerUserId, ownerRole: options.actor?.role ?? params.ownerRole }, - options, - preserveLastTraceId: true - }); - return { terminal: true, payload, runnerTrace: synced?.runnerTrace ?? payload?.runnerTrace ?? traceStore.snapshot(traceId), valuesPrinted: false }; - } - return { terminal: false, payload, runnerTrace: synced?.runnerTrace ?? payload?.runnerTrace ?? traceStore.snapshot(traceId), valuesPrinted: false }; - } catch (error) { - return { terminal: false, error, runnerTrace: traceStore.snapshot(traceId), valuesPrinted: false }; - } -} - -function traceSnapshotTerminalFailureDiagnostic(snapshot = null) { - const events = Array.isArray(snapshot?.events) ? snapshot.events : []; - for (const event of [...events].reverse()) { - const payload = event?.payload && typeof event.payload === "object" ? event.payload : {}; - const label = textValue(event?.label ?? event?.type); - const status = normalizeTurnStatus(event?.status ?? payload.status); - const failureKind = textValue(event?.failureKind ?? event?.errorCode ?? payload.failureKind ?? payload.errorCode); - const retrying = event?.willRetry === true || payload.willRetry === true; - const retryExhausted = event?.retryExhausted === true || payload.retryExhausted === true; - const explicitlyNonRetryable = event?.retryable === false || payload.retryable === false; - const hardFailure = retryExhausted || explicitlyNonRetryable || failureKind === "backend-timeout" || /^agentrun:error:/u.test(label); - if (retrying || !hardFailure || (status !== "failed" && !failureKind)) continue; - return { - label: label || null, - status: status || null, - failureKind: failureKind || null, - retryExhausted, - message: textValue(event?.message ?? payload.message) || null, - valuesPrinted: false - }; - } - return null; -} - -async function recordCodeAgentSessionOwner({ payload = {}, params = {}, options = {}, status = "active", preserveLastTraceId = false } = {}) { - const ownerUserId = options.actor?.id ?? params.ownerUserId; - if (!ownerUserId || !options.accessController?.recordAgentSessionOwner) return null; - const traceId = safeTraceId(payload.traceId ?? params.traceId); - const session = payload.session && typeof payload.session === "object" ? payload.session : null; - const sessionReuse = payload.sessionReuse && typeof payload.sessionReuse === "object" ? payload.sessionReuse : null; - const sessionId = safeSessionId(payload.sessionId ?? session?.sessionId ?? sessionReuse?.sessionId ?? params.sessionId) || null; - const conversationId = safeConversationId(payload.conversationId ?? session?.conversationId ?? sessionReuse?.conversationId ?? params.conversationId) || null; - const threadId = safeOpaqueId(session?.threadId ?? sessionReuse?.threadId ?? payload.threadId ?? params.threadId) || null; - return writeWorkbenchProjectionSession({ - accessController: options.accessController, - runtimeStore: options.runtimeStore, - traceStore: options.traceStore ?? defaultCodeAgentTraceStore, - traceId, - ownerUserId, - ownerRole: options.actor?.role ?? params.ownerRole ?? null, - sessionId, - projectId: params.projectId ?? payload.projectId ?? null, - conversationId, - threadId, - status, - preserveLastTraceId, - session: codeAgentSessionOwnerEvidence(payload, params), - payload, - params - }); -} - -function codeAgentOwnerStatusForResult(result = {}) { - const sessionStatus = textValue(result?.session?.status ?? result?.sessionSummary?.status ?? result?.sessionLifecycleStatus); - if (sessionStatus === "thread-resume-failed") return "thread-resume-failed"; - if (codeAgentFreshContinuationFailure(result)) return "thread-resume-failed"; - if (codeAgentPayloadHasSealedFinalResponse(result)) return "completed"; - if (result?.status === "completed") return "completed"; - if (result?.status === "canceled" || result?.status === "cancelled") return "canceled"; - return result?.status ?? "active"; -} - -function textValue(value) { - return String(value ?? "").trim(); -} - -function timestampIsoOrNow(value = null) { - const parsed = Date.parse(String(value ?? "")); - return Number.isFinite(parsed) ? new Date(parsed).toISOString() : new Date().toISOString(); -} - -function timestampIsoOrNull(value = null) { - const parsed = Date.parse(String(value ?? "")); - return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null; -} - -function codeAgentSessionOwnerEvidence(payload = {}, params = {}) { - const traceId = payload.traceId ?? params.traceId ?? null; - const finalResponse = codeAgentFinalResponseEvidence(payload, traceId); - const traceSummary = codeAgentTraceSummaryEvidence(payload, traceId, finalResponse); - const messages = codeAgentConversationMessagesEvidence(payload, params, traceId, finalResponse); - const firstUserMessagePreview = messages.find((message) => message.role === "user")?.text?.slice(0, 240) ?? null; - const promptEvidence = codeAgentPromptMetadata(payload, params, traceId); - const traceResult = codeAgentTraceResultEvidence(payload, params, traceId, finalResponse, traceSummary); - const providerProfile = codeAgentProviderProfileEvidence(payload, params); - const continuation = codeAgentContinuationEvidence(payload, params); - const sessionLifecycleStatus = continuation.requiresFreshSession ? "thread-resume-failed" : payload.sessionLifecycleStatus ?? null; - return { - ...(providerProfile ? { providerProfile, codeAgentProviderProfile: providerProfile } : {}), - provider: payload.provider ?? null, - model: payload.model ?? null, - backend: payload.backend ?? null, - status: payload.status ?? null, - sessionMode: payload.sessionMode ?? payload.session?.sessionMode ?? null, - sessionLifecycleStatus, - continuation, - runnerKind: payload.runner?.kind ?? payload.runnerTrace?.runnerKind ?? null, - conversationId: payload.conversationId ?? params.conversationId ?? null, - sessionId: payload.sessionId ?? payload.session?.sessionId ?? payload.sessionReuse?.sessionId ?? params.sessionId ?? null, - threadId: payload.session?.threadId ?? payload.sessionReuse?.threadId ?? payload.threadId ?? params.threadId ?? null, - traceId, - projectId: payload.projectId ?? params.projectId ?? null, - ...codeAgentPromptFields(promptEvidence), - finalResponse, - traceSummary, - ...(traceResult ? { traceResults: { [traceResult.traceId]: traceResult } } : {}), - ...(messages.length ? { messages, messageCount: messages.length } : {}), - ...(firstUserMessagePreview ? { firstUserMessagePreview } : {}), - ...agentRunSessionEvidence(payload), - secretMaterialStored: false, - valuesRedacted: true - }; -} - -function codeAgentContinuationEvidence(payload = {}, params = {}) { - const failure = codeAgentFreshContinuationFailure(payload); - const providerProfile = codeAgentProviderProfileEvidence(payload, params); - const traceId = safeTraceId(payload.traceId ?? params.traceId) || null; - if (!failure) { - return { - requiresFreshSession: false, - providerProfile, - traceId, - valuesRedacted: true - }; - } - return { - requiresFreshSession: true, - reasonCode: failure.code, - reason: failure.message, - staleThreadId: safeOpaqueId(payload.session?.threadId ?? payload.sessionReuse?.threadId ?? payload.threadId ?? params.threadId ?? payload.agentRun?.threadId) || null, - providerProfile, - traceId, - valuesRedacted: true - }; -} - -function codeAgentFreshContinuationFailure(payload = {}) { - if (!payload || typeof payload !== "object") return null; - const sessionStatus = String(payload.session?.status ?? payload.sessionLifecycleStatus ?? payload.sessionReuse?.status ?? "").trim().toLowerCase().replace(/_/gu, "-"); - if (sessionStatus === "thread-resume-failed") return { code: "thread-resume-failed", message: "previous AgentRun thread/session resume failed", valuesRedacted: true }; - const status = normalizeTurnStatus(payload.status, payload.agentRun?.terminalStatus, payload.agentRun?.commandState, payload.agentRun?.status, payload.session?.status, payload.sessionReuse?.status); - if (!status || !["failed", "blocked", "timeout"].includes(status)) return null; - const code = normalizeCodeAgentFailureKind(firstNonEmptyValue( - payload.error?.code, - payload.blocker?.code, - payload.providerTrace?.failureKind, - payload.providerTrace?.errorCode, - payload.agentRun?.failureKind, - payload.agentRun?.errorCode, - payload.transient?.failureKind, - payload.transient?.errorCode - )); - const message = firstNonEmptyValue( - payload.error?.message, - payload.error?.userMessage, - payload.blocker?.summary, - payload.blocker?.message, - payload.providerTrace?.failureMessage, - payload.providerTrace?.summary, - payload.agentRun?.failureMessage, - payload.agentRun?.message - ) ?? ""; - if (!codeAgentFailureRequiresFreshContinuation(code, message)) return null; - return { code: code ?? "stale-continuation", message: message || code || "stale continuation", valuesRedacted: true }; -} - -function normalizeCodeAgentFailureKind(value) { - const normalized = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); - return normalized || null; -} - -function codeAgentFailureRequiresFreshContinuation(code, message = "") { - const failureKind = normalizeCodeAgentFailureKind(code); - if (["thread-resume-failed", "session-store-evicted", "provider-stream-disconnected"].includes(failureKind)) return true; - return /session stor(?:e|age).*evicted|pvc-backed session|no rollout found for thread id|no rollout found|thread\/resume failed|provider stream disconnected/iu.test(String(message ?? "")); -} - -function codeAgentProviderProfileEvidence(payload = {}, params = {}) { - const raw = firstNonEmptyValue( - params.providerProfile, - params.codeAgentProviderProfile, - payload.providerProfile, - payload.codeAgentProviderProfile, - payload.agentRun?.backendProfile, - payload.providerTrace?.backendProfile, - payload.backendProfile - ); - if (!raw) return null; - const normalized = normalizeCodeAgentProviderProfile(raw); - return normalized === "runtime-default" ? null : normalized; -} - -function codeAgentTraceResultEvidence(payload = {}, params = {}, traceId = null, finalResponse = null, traceSummary = null) { - const resolvedTraceId = safeTraceId(traceId); - if (!resolvedTraceId) return null; - const agentRun = agentRunSessionEvidence(payload).agentRun ?? null; - const userBillingReservation = codeAgentBillingReservationEvidence(payload.userBillingReservation ?? params.userBillingReservation); - const resultSession = payload.session && typeof payload.session === "object" ? payload.session : null; - const sessionReuse = payload.sessionReuse && typeof payload.sessionReuse === "object" ? payload.sessionReuse : null; - const conversationId = safeConversationId(payload.conversationId ?? agentRun?.conversationId ?? resultSession?.conversationId ?? sessionReuse?.conversationId ?? params.conversationId) || null; - const sessionId = safeSessionId(payload.sessionId ?? resultSession?.sessionId ?? sessionReuse?.sessionId ?? params.sessionId) || null; - const threadId = safeOpaqueId(payload.threadId ?? resultSession?.threadId ?? sessionReuse?.threadId ?? params.threadId ?? agentRun?.threadId) || null; - const promptEvidence = codeAgentPromptMetadata(payload, params, resolvedTraceId); - if (!agentRun && !finalResponse && !traceSummary) return null; - return { - traceId: resolvedTraceId, - status: codeAgentPayloadHasSealedFinalResponse(payload) ? "completed" : payload.status ?? agentRun?.terminalStatus ?? agentRun?.commandState ?? null, - conversationId, - sessionId, - threadId, - ...codeAgentPromptFields(promptEvidence), - messageId: finalResponse?.messageId ?? payload.messageId ?? null, - createdAt: payload.createdAt ?? finalResponse?.createdAt ?? null, - updatedAt: payload.updatedAt ?? finalResponse?.updatedAt ?? null, - finalResponse, - traceSummary, - agentRun, - ...(userBillingReservation ? { userBillingReservation } : {}), - valuesRedacted: true, - secretMaterialStored: false - }; -} - -function codeAgentPromptMetadata(payload = {}, params = {}, traceId = null) { - const prompt = payload?.prompt && typeof payload.prompt === "object" ? payload.prompt : null; - const rawPrompt = firstNonEmptyValue( - params.message, - params.prompt, - params.text, - payload.userMessage, - typeof payload.prompt === "string" ? payload.prompt : null - ); - const fromText = codeAgentPromptMetadataFromText(rawPrompt, traceId, "code-agent-request", { fullText: Boolean(rawPrompt) }); - return codeAgentPromptMetadataFromParts({ - traceId, - source: prompt?.source ?? payload.promptSource ?? fromText?.prompt?.source ?? "code-agent-request", - promptId: firstNonEmptyValue(payload.promptId, prompt?.id, fromText?.promptId), - promptSha256: firstNonEmptyValue(payload.promptSha256, prompt?.sha256, fromText?.promptSha256), - promptSummary: firstNonEmptyValue(payload.promptSummary, prompt?.summary, fromText?.promptSummary), - promptPreview: firstNonEmptyValue(payload.promptPreview, prompt?.preview, fromText?.promptPreview), - textChars: Number.isInteger(prompt?.textChars) ? prompt.textChars : fromText?.prompt?.textChars - }); -} - -function codeAgentPromptMetadataFromText(value, traceId = null, source = "code-agent-request", options = {}) { - const promptText = String(value ?? "").trim(); - if (!promptText) return null; - return codeAgentPromptMetadataFromParts({ - traceId, - source, - promptSha256: options.fullText === true ? sha256Text(promptText) : undefined, - promptSummary: promptText, - promptPreview: promptText, - textChars: options.fullText === true ? promptText.length : undefined - }); -} - -function codeAgentPromptMetadataFromParts({ traceId = null, source = "code-agent-request", promptId = null, promptSha256 = null, promptSummary = null, promptPreview = null, textChars = undefined } = {}) { - const digest = firstNonEmptyValue(promptSha256); - const summary = clippedPromptText(firstNonEmptyValue(promptSummary, promptPreview), 240); - const preview = clippedPromptText(firstNonEmptyValue(promptPreview, promptSummary), 500); - const explicitId = firstNonEmptyValue(promptId); - if (!explicitId && !digest && !summary && !preview) return null; - const id = explicitId || codeAgentPromptId(traceId, digest); - const prompt = compactCodeAgentObject({ - id, - sha256: digest || undefined, - summary, - preview, - textChars: Number.isInteger(textChars) ? textChars : undefined, - source, - valuesPrinted: false - }); - return compactCodeAgentObject({ - promptId: id, - promptSha256: digest || undefined, - promptSummary: summary, - promptPreview: preview, - prompt, - valuesPrinted: false - }) ?? null; -} - -function codeAgentPromptFields(metadata = null) { - if (!metadata) return {}; - return compactCodeAgentObject({ - promptId: metadata.promptId, - promptSha256: metadata.promptSha256, - promptSummary: metadata.promptSummary, - promptPreview: metadata.promptPreview, - prompt: metadata.prompt - }) ?? {}; -} - -function codeAgentPromptId(traceId = null, digest = null) { - const traceSuffix = safeTraceId(traceId)?.slice(4).replace(/[^A-Za-z0-9_.:-]/gu, "_"); - const digestSuffix = firstNonEmptyValue(digest)?.slice(0, 16); - const suffix = firstNonEmptyValue(traceSuffix, digestSuffix); - return suffix ? `prm_${suffix}` : undefined; -} - -function clippedPromptText(value, limit) { - const promptText = String(value ?? "").trim(); - if (!promptText) return undefined; - return promptText.length > limit ? `${promptText.slice(0, limit)}...` : promptText; -} - -function sha256Text(value) { - return createHash("sha256").update(String(value)).digest("hex"); -} - -function compactCodeAgentObject(value) { - const result = {}; - for (const [key, item] of Object.entries(value ?? {})) { - if (item === undefined || item === null) continue; - if (Array.isArray(item) && item.length === 0) continue; - if (typeof item === "object" && !Array.isArray(item) && Object.keys(item).length === 0) continue; - result[key] = item; - } - return Object.keys(result).length > 0 ? result : undefined; -} - -function codeAgentBillingReservationEvidence(value = null) { - if (!value || typeof value !== "object") return null; - const reservationId = textValue(value.reservationId); - if (!reservationId) return null; - return { - reservationId, - resourceType: textValue(value.resourceType) || null, - planId: textValue(value.planId) || null, - estimatedCredits: numberOrNull(value.estimatedCredits), - estimatedTokens: numberOrNull(value.estimatedTokens), - expiresAt: textValue(value.expiresAt) || null, - valuesRedacted: true, - secretMaterialStored: false - }; -} - -function codeAgentConversationMessagesEvidence(payload = {}, params = {}, traceId = null, finalResponse = null) { - const userText = boundedConversationMessageText(params.message ?? params.prompt ?? payload.userMessage ?? payload.prompt); - if (!userText) return []; - const resolvedTraceId = safeTraceId(traceId) || null; - const lifecycle = codeAgentTurnLifecycleFields(resolvedTraceId, payload); - const session = payload.session && typeof payload.session === "object" ? payload.session : null; - const sessionReuse = payload.sessionReuse && typeof payload.sessionReuse === "object" ? payload.sessionReuse : null; - const conversationId = safeConversationId(payload.conversationId ?? session?.conversationId ?? sessionReuse?.conversationId ?? params.conversationId) || null; - const sessionId = safeSessionId(payload.sessionId ?? session?.sessionId ?? sessionReuse?.sessionId ?? params.sessionId) || null; - const threadId = safeOpaqueId(session?.threadId ?? sessionReuse?.threadId ?? payload.threadId ?? params.threadId) || null; - const createdAt = payload.createdAt ?? payload.reply?.createdAt ?? new Date().toISOString(); - const updatedAt = payload.updatedAt ?? createdAt; - const userMessage = { - id: lifecycle.userMessageId, - messageId: lifecycle.userMessageId, - role: "user", - title: "用户", - text: userText, - status: "sent", - traceId: resolvedTraceId, - turnId: lifecycle.turnId, - conversationId, - sessionId, - threadId, - createdAt, - updatedAt, - source: "code-agent-submit", - secretMaterialStored: false, - valuesRedacted: true - }; - const agentText = boundedConversationMessageText( - finalResponse?.text - ?? payload.reply?.content - ?? payload.message?.content - ?? payload.assistantText - ?? payload.error?.userMessage - ?? payload.error?.message - ?? payload.blocker?.userMessage - ?? payload.blocker?.message - ?? payload.blocker?.summary - ); - const agentStatus = codeAgentConversationAgentMessageStatus(payload); - const agentMessage = { - id: lifecycle.assistantMessageId, - messageId: lifecycle.assistantMessageId, - role: "agent", - title: agentStatus === "failed" ? "Code Agent 返回阻塞" : agentStatus === "running" ? "Code Agent 处理中" : "Code Agent 回复", - text: agentText || (agentStatus === "running" ? "" : "Code Agent 请求已结束，请查看 Trace 详情。"), - status: agentStatus, - traceId: resolvedTraceId, - turnId: lifecycle.turnId, - conversationId, - sessionId, - threadId, - createdAt: payload.reply?.createdAt ?? updatedAt, - updatedAt, - source: "code-agent-result", - ...(payload.runnerTrace && typeof payload.runnerTrace === "object" ? { runnerTrace: codeAgentConversationRunnerTraceEvidence(payload.runnerTrace, resolvedTraceId) } : {}), - secretMaterialStored: false, - valuesRedacted: true - }; - return [userMessage, agentMessage]; -} - -function boundedConversationMessageText(value, limit = 12000) { - const text = conversationText(value); - if (!text) return ""; - return text.length > limit ? text.slice(0, limit) : text; -} - -function conversationText(value) { - const extracted = conversationTextRaw(value).replace(/\r\n?/gu, "\n"); - if (!extracted.trim() || extracted.trim() === "[object Object]") return ""; - return extracted; -} - -function conversationTextRaw(value) { - if (value === undefined || value === null) return ""; - if (typeof value === "string") return value; - if (typeof value === "number" || typeof value === "boolean") return String(value); - if (Array.isArray(value)) return value.map(conversationTextRaw).filter(Boolean).join("\n"); - if (typeof value === "object") { - for (const key of ["text", "content", "message", "prompt", "value", "summary", "preview", "title"]) { - const text = conversationTextRaw(value[key]); - if (text) return text; - } - } - return ""; -} - -function codeAgentMessageTraceSuffix(traceId) { - const text = safeTraceId(traceId) || `local_${Date.now().toString(36)}`; - return text.replace(/^trc_/u, "").replace(/[^A-Za-z0-9_.:-]/gu, "_").slice(0, 48) || "trace"; -} - -function codeAgentTurnLifecycleFields(traceId, source = {}) { - const resolvedTraceId = safeTraceId(source?.traceId ?? traceId) || safeTraceId(traceId) || null; - const turnId = safeTurnId(source?.turnId) || resolvedTraceId; - const traceSuffix = codeAgentMessageTraceSuffix(resolvedTraceId); - return { - turnId, - userMessageId: safeMessageId(source?.userMessageId ?? source?.userMessage?.messageId) || codeAgentLifecycleMessageId(traceSuffix, "user"), - assistantMessageId: safeMessageId(source?.assistantMessageId ?? source?.assistantMessage?.messageId) || codeAgentLifecycleMessageId(traceSuffix, "agent") - }; -} - -function codeAgentLifecycleMessageId(traceSuffix, role) { - return `msg_${traceSuffix}_${role}`; -} - -function safeTurnId(value) { - const text = textValue(value); - return /^turn_[A-Za-z0-9_.:-]+$/u.test(text) ? text : null; -} - -function safeMessageId(value) { - const text = textValue(value); - return /^msg_[A-Za-z0-9_.:-]+$/u.test(text) ? text : null; -} - -function codeAgentConversationAgentMessageStatus(payload = {}) { - const status = textValue(payload.status ?? payload.agentRun?.terminalStatus ?? payload.commandState ?? payload.runStatus).toLowerCase(); - if (codeAgentPayloadHasSealedFinalResponse(payload)) return "completed"; - if (["completed", "done", "success"].includes(status)) return "completed"; - if (["failed", "blocked", "error", "timeout", "canceled", "cancelled"].includes(status)) return "failed"; - if (payload.error || payload.blocker) return "failed"; - return "running"; -} - -function codeAgentPayloadHasSealedFinalResponse(payload = {}) { - if (!payload || typeof payload !== "object") return false; - if (payload.error || payload.blocker) return false; - if (!codeAgentFinalResponseText(payload)) return false; - const agentRun = payload.agentRun && typeof payload.agentRun === "object" ? payload.agentRun : {}; - return isTraceCommandTerminalStatus(payload.status) || - isTraceCommandTerminalStatus(agentRun.status) || - isTraceCommandTerminalStatus(agentRun.runStatus) || - isTraceCommandTerminalStatus(agentRun.commandState) || - isTraceCommandTerminalStatus(agentRun.terminalStatus); -} - -function codeAgentConversationRunnerTraceEvidence(runnerTrace = {}, traceId = null) { - const events = Array.isArray(runnerTrace.events) ? runnerTrace.events : []; - const lastEvent = runnerTrace.lastEvent && typeof runnerTrace.lastEvent === "object" ? runnerTrace.lastEvent : events.at(-1) ?? null; - const eventCount = Number.isFinite(Number(runnerTrace.eventCount)) ? Number(runnerTrace.eventCount) : events.length; - return { - traceId: safeTraceId(runnerTrace.traceId ?? traceId) || traceId || null, - status: runnerTrace.status ?? lastEvent?.status ?? null, - sessionId: runnerTrace.sessionId ?? null, - threadId: runnerTrace.threadId ?? null, - runId: runnerTrace.runId ?? lastEvent?.runId ?? null, - commandId: runnerTrace.commandId ?? lastEvent?.commandId ?? null, - eventCount, - lastEvent: lastEvent ? { - label: lastEvent.label ?? null, - status: lastEvent.status ?? null, - terminal: lastEvent.terminal === true, - createdAt: lastEvent.createdAt ?? null, - message: boundedConversationMessageText(lastEvent.message ?? lastEvent.text, 1000), - valuesPrinted: false - } : null, - eventsCompacted: true, - fullTraceLoaded: false, - valuesPrinted: false - }; -} - -function codeAgentFinalResponseEvidence(payload = {}, traceId = null) { - const text = codeAgentFinalResponseText(payload); - if (!text) return null; - return { - text, - textChars: text.length, - messageId: payload.finalResponse?.messageId ?? payload.reply?.messageId ?? payload.messageId ?? null, - role: payload.reply?.role ?? payload.message?.role ?? "assistant", - status: codeAgentPayloadHasSealedFinalResponse(payload) ? "completed" : payload.finalResponse?.status ?? payload.status ?? null, - traceId, - createdAt: payload.finalResponse?.createdAt ?? payload.reply?.createdAt ?? payload.createdAt ?? null, - updatedAt: payload.finalResponse?.updatedAt ?? payload.updatedAt ?? null, - source: payload.finalResponse?.source ?? "code-agent-result", - valuesPrinted: false - }; -} - -function codeAgentFinalResponseText(payload = {}) { - for (const value of [ - payload?.finalResponse?.text, - payload?.finalResponse?.content, - payload?.finalResponse, - payload?.reply?.content, - payload?.reply, - payload?.message?.content, - payload?.message, - payload?.assistantText, - payload?.finalText, - payload?.text, - payload?.content - ]) { - const text = conversationText(value); - if (text) return text; - } - return ""; -} - -function codeAgentTraceSummaryEvidence(payload = {}, traceId = null, finalResponse = null) { - const runnerTrace = payload.runnerTrace && typeof payload.runnerTrace === "object" ? payload.runnerTrace : null; - const events = Array.isArray(runnerTrace?.events) ? runnerTrace.events : []; - const terminalEvent = [...events].reverse().find((event) => event?.terminal === true) ?? runnerTrace?.lastEvent ?? null; - return { - traceId, - source: "code-agent-derived-session-evidence", - sourceEventCount: Number.isFinite(Number(runnerTrace?.eventCount)) ? Number(runnerTrace.eventCount) : events.length, - terminalStatus: payload.agentRun?.terminalStatus ?? payload.status ?? terminalEvent?.status ?? null, - lastEventLabel: terminalEvent?.label ?? null, - finalAssistantRow: finalResponse ? { - role: finalResponse.role, - status: finalResponse.status, - textChars: finalResponse.textChars, - textPreview: finalResponse.text.slice(0, 240), - messageId: finalResponse.messageId, - valuesPrinted: false - } : null, - agentRun: payload.agentRun ? { - runId: payload.agentRun.runId ?? null, - commandId: payload.agentRun.commandId ?? null, - attemptId: payload.agentRun.attemptId ?? null, - runnerId: payload.agentRun.runnerId ?? null, - jobName: payload.agentRun.jobName ?? null, - namespace: payload.agentRun.namespace ?? null, - lastSeq: payload.agentRun.lastSeq ?? null, - valuesPrinted: false - } : null, - updatedAt: payload.updatedAt ?? null, - valuesPrinted: false - }; -} - -function annotateOwner(payload, params = {}) { - if (!params.ownerUserId) return payload; - return { - ...payload, - ownerUserId: params.ownerUserId, - ownerRole: params.ownerRole ?? null - }; -} - -function canAccessOwnedResult(result, actor) { - if (!result?.ownerUserId || !actor) return true; - return actor.role === "admin" || actor.id === result.ownerUserId; -} - -function isCodeAgentResultCanceled(result) { - return result?.status === "canceled" || result?.canceled === true || result?.error?.code === "codex_stdio_canceled"; -} - -function codeAgentCancelScopeMismatch(params = {}, result = {}) { - return cancelScopeFieldMismatch("conversationId", safeConversationId(params.conversationId), cancelExpectedValues(safeConversationId, - result.conversationId, - result.agentRun?.conversationId, - result.session?.conversationId, - result.sessionReuse?.conversationId, - result.runnerTrace?.conversationId - )) ?? cancelScopeFieldMismatch("sessionId", safeSessionId(params.sessionId), cancelExpectedValues(safeSessionId, - result.sessionId, - result.session?.sessionId, - result.sessionReuse?.sessionId, - result.agentRun?.hwlabSessionId - )) ?? cancelScopeFieldMismatch("threadId", safeOpaqueId(params.threadId), cancelExpectedValues(safeOpaqueId, - result.threadId, - result.agentRun?.threadId, - result.session?.threadId, - result.sessionReuse?.threadId, - result.runnerTrace?.threadId - )); -} - -function cancelExpectedValues(normalize, ...values) { - return [...new Set(values.map((value) => normalize(value)).filter(Boolean))]; -} - -function cancelScopeFieldMismatch(field, requested, expectedValues) { - if (!requested || expectedValues.length === 0 || expectedValues.includes(requested)) return null; - return { field, requested, expected: expectedValues[0] }; -} - -async function syncAlreadyTerminalAgentRunBeforeCancel({ traceId, currentResult = null, params = {}, options = {}, traceStore = defaultCodeAgentTraceStore } = {}) { - if (!safeTraceId(traceId) || !currentResult?.agentRun?.runId || !currentResult?.agentRun?.commandId) return null; - const currentTerminalStatus = currentResult.agentRun?.terminalStatus ?? currentResult.agentRun?.commandState ?? currentResult.agentRun?.commandStatus; - if (isTraceCommandTerminalStatus(currentTerminalStatus)) { - return alreadyTerminalCancelPayload({ traceId, payload: currentResult, params, options, traceStore }); - } - try { - const synced = await syncAgentRunChatResult({ - traceId, - currentResult, - options: { ...options, deferAgentRunResultSync: false }, - traceStore, - forceResultSync: true, - refreshEvents: true - }); - const payload = synced?.result ?? null; - const payloadTerminalStatus = payload?.agentRun?.terminalStatus ?? payload?.agentRun?.commandState ?? payload?.agentRun?.commandStatus; - if (!isTraceCommandTerminalStatus(payloadTerminalStatus)) return null; - return alreadyTerminalCancelPayload({ traceId, payload, params, options, traceStore }); - } catch (error) { - traceStore.append(traceId, { - type: "cancel", - status: "degraded", - label: "agentrun:cancel:terminal-check-degraded", - errorCode: error?.code ?? "terminal_check_failed", - message: error?.message ?? "Failed to check AgentRun terminal state before forwarding cancel.", - waitingFor: "cancel-terminal-check", - valuesPrinted: false - }); - return null; - } -} - -async function alreadyTerminalCancelPayload({ traceId, payload = {}, params = {}, options = {}, traceStore = defaultCodeAgentTraceStore } = {}) { - traceStore.append(traceId, { - type: "cancel", - status: "blocked", - label: "agentrun:cancel:already-terminal", - message: "AgentRun command was already terminal; HWLAB did not forward cancel and kept the sealed terminal projection.", - runId: payload.agentRun?.runId ?? null, - commandId: payload.agentRun?.commandId ?? null, - terminalStatus: payload.agentRun?.terminalStatus ?? payload.status, - waitingFor: "already-terminal-projection", - valuesPrinted: false - }); - await recordCodeAgentTerminalTurnStatusEffects({ - payload, - params: { ...params, traceId, ownerUserId: options.actor?.id, ownerRole: options.actor?.role }, - options, - preserveLastTraceId: true - }); - return { - ...payload, - cancelDisposition: { - status: "already_terminal", - forwarded: false, - route: "/v1/agent/chat/cancel", - message: "AgentRun command was already terminal; cancel was not forwarded.", - valuesPrinted: false - }, - runnerTrace: traceStore.snapshot(traceId) - }; -} - -function cancelBlockedPayload({ - code, - message, - traceId, - conversationId = null, - sessionId = null, - session = null, - runnerTrace = null, - status = "failed", - unsupported = false, - degraded = false -}) { - const sessionSummary = codeAgentSessionLifecycleSummary({ - session, - runnerTrace, - status, - unsupported, - degraded, - error: { code, userMessage: message, message } - }); - return { - accepted: false, - canceled: false, - status, - conversationId, - sessionId, - traceId, - session, - sessionLifecycleStatus: sessionSummary.status, - sessionLifecycle: sessionSummary, - sessionSummary, - unsupported, - degraded, - runnerTrace, - lastTraceEvent: runnerTrace?.lastEvent ?? null, - error: { - code, - layer: "session", - category: "cancel_blocked", - retryable: true, - userMessage: message, - message, - traceId, - route: "/v1/agent/chat/cancel", - toolName: "codex-stdio.cancel" - }, - blocker: { - code, - layer: "session", - category: "cancel_blocked", - retryable: true, - summary: message, - userMessage: message, - traceId, - route: "/v1/agent/chat/cancel", - toolName: "codex-stdio.cancel" - } - }; -} - -export async function handleCodeAgentTraceHttp(request, response, url, options) { - const startedAt = nowMs(); - const parts = url.pathname.split("/").filter(Boolean); - const traceId = decodeURIComponent(parts[3] ?? ""); - const streamSegment = parts[4]; - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - const pageOptions = codeAgentTracePageOptions(url); - if (!/^trc_[A-Za-z0-9_.:-]+$/u.test(traceId)) { - sendJson(response, 400, { - error: { - code: "invalid_trace_id", - message: "traceId must start with trc_ and contain only safe identifier characters" - } - }); - return; - } - const requestOptions = codeAgentRequestScopedOptions(options); - const projectMismatch = await measureCodeAgentHttpPhase(requestOptions, "trace_project_check", () => traceProjectMismatchSnapshot(traceId, url, requestOptions, "trace")); - if (projectMismatch) { - sendJson(response, projectMismatch.statusCode, projectMismatch.body); - return; - } - const context = await measureCodeAgentHttpPhase(requestOptions, "trace_projection_read", () => readCodeAgentCompatProjection(traceId, requestOptions)); - if (context.statusCode) { - sendJson(response, context.statusCode, context.body); - return; - } - - if (streamSegment === "stream") { - sendTraceSse(response, traceStore, traceId, requestOptions); - return; - } - - const body = await measureCodeAgentHttpPhase(requestOptions, "trace_paginate", () => { - return paginateTraceSnapshot(context.trace, pageOptions); - }); - const statusCode = context.found ? 200 : 404; - void emitCodeAgentOtelSpan("trace_events_read", traceId, requestOptions.env ?? process.env, { - startTimeMs: startedAt, - attributes: { - "http.method": "GET", - "http.route": "/v1/agent/traces/:traceId", - "http.status_code": statusCode, - found: Boolean(context.found), - returnedEvents: Array.isArray(body?.events) ? body.events.length : 0, - sinceSeq: pageOptions.sinceSeq, - limit: pageOptions.limit, - fromSeq: body?.range?.fromSeq ?? null, - toSeq: body?.range?.toSeq ?? null, - totalEvents: body?.range?.total ?? body?.eventCount ?? null, - hasMore: Boolean(body?.hasMore), - fullTraceLoaded: Boolean(body?.fullTraceLoaded) - } - }); - sendJson(response, statusCode, codeAgentCompatProjectionPayload(body, context)); -} - -function codeAgentTracePageOptions(url) { - const sinceSeq = nonNegativeInteger(url.searchParams.get("sinceSeq") ?? url.searchParams.get("since"), 0); - const requestedLimit = parsePositiveInteger(url.searchParams.get("limit"), DEFAULT_CODE_AGENT_TRACE_PAGE_LIMIT); - return { - sinceSeq, - limit: Math.min(requestedLimit, MAX_CODE_AGENT_TRACE_PAGE_LIMIT) - }; -} - -function paginateTraceSnapshot(snapshot, { sinceSeq, limit }) { - const sourceEvents = Array.isArray(snapshot?.events) ? snapshot.events : []; - const indexedEvents = sourceEvents.map((event, index) => ({ event, seq: traceEventSequence(event, index) })); - const firstIndex = indexedEvents.findIndex((item) => item.seq > sinceSeq); - const startIndex = firstIndex >= 0 ? firstIndex : indexedEvents.length; - const pageItems = indexedEvents.slice(startIndex, startIndex + limit); - const events = pageItems.map((item) => item.event); - const fromSeq = pageItems.length > 0 ? pageItems[0].seq : null; - const toSeq = pageItems.length > 0 ? pageItems[pageItems.length - 1].seq : null; - const hasMore = startIndex + pageItems.length < indexedEvents.length; - const totalEvents = snapshot?.eventCount ?? indexedEvents.length; - return { - ...snapshot, - events, - eventCount: totalEvents, - range: { - sinceSeq, - fromSeq, - toSeq, - limit, - returned: events.length, - total: totalEvents - }, - truncated: hasMore, - hasMore, - nextSinceSeq: pageItems.length > 0 ? toSeq : sinceSeq, - fullTraceLoaded: !hasMore - }; -} - -function traceEventSequence(event, index) { - const seq = Number(event?.seq); - return Number.isFinite(seq) && seq > 0 ? Math.trunc(seq) : index + 1; -} - -function nonNegativeInteger(value, fallback) { - const parsed = Number.parseInt(value ?? "", 10); - return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback; -} - -function isTraceCommandTerminalStatus(status) { - return CODE_AGENT_TERMINAL_STATUSES.has(String(status ?? "").trim().toLowerCase().replace(/_/gu, "-")); -} - -function traceSnapshotWithTerminalEvidence(snapshot, result, traceId, refreshError = null) { - const evidence = agentRunTerminalTraceEvidence(result, traceId); - if (snapshot?.status !== "missing") return traceSnapshotWithAttachedTerminalEvidence(snapshot, evidence, refreshError); - if (!evidence) return { - ...snapshot, - ok: false, - traceStatus: "missing", - retention: traceRetentionSummary("missing") - }; - return { - ...snapshot, - ok: true, - status: "expired", - traceStatus: "expired", - projectionStatus: "projecting", - projectionHealth: "degraded", - persisted: true, - retention: traceRetentionSummary("expired"), - conversationId: evidence.conversationId, - sessionId: evidence.sessionId, - threadId: evidence.threadId, - ...codeAgentPromptFields(evidence), - agentRun: evidence.agentRun, - finalResponse: evidence.finalResponse, - traceSummary: evidence.traceSummary, - terminalEvidence: { - available: true, - source: evidence.source, - refresh: refreshError ? { - attempted: true, - ok: false, - code: refreshError?.code ?? "agentrun_trace_refresh_failed", - message: refreshError?.message ?? "AgentRun trace events could not be refreshed; command result remains authoritative for final response.", - valuesPrinted: false - } : { attempted: false, ok: null }, - conversationId: evidence.conversationId, - sessionId: evidence.sessionId, - threadId: evidence.threadId, - ...codeAgentPromptFields(evidence), - agentRun: evidence.agentRun, - traceSummary: evidence.traceSummary, - finalResponse: evidence.finalResponse, - valuesPrinted: false - } - }; -} - -function traceSnapshotWithAttachedTerminalEvidence(snapshot, evidence, refreshError = null) { - if (!evidence) return snapshot; - return { - ...snapshot, - conversationId: snapshot.conversationId ?? evidence.conversationId, - sessionId: snapshot.sessionId ?? evidence.sessionId, - threadId: snapshot.threadId ?? evidence.threadId, - ...codeAgentPromptFields(evidence), - agentRun: snapshot.agentRun ?? evidence.agentRun, - finalResponse: snapshot.finalResponse ?? evidence.finalResponse, - traceSummary: snapshot.traceSummary ?? evidence.traceSummary, - terminalEvidence: snapshot.terminalEvidence ?? (refreshError ? { - available: true, - source: evidence.source, - refresh: { - attempted: true, - ok: false, - code: refreshError?.code ?? "agentrun_trace_refresh_failed", - message: refreshError?.message ?? "AgentRun trace events could not be refreshed; live events may be partial.", - valuesPrinted: false - }, - valuesPrinted: false - } : undefined) - }; -} - -function agentRunTerminalTraceEvidence(result, traceId) { - if (!result || typeof result !== "object") return null; - const storedSummary = result.traceSummary && typeof result.traceSummary === "object" && result.traceSummary.source === "agentrun-command-result" ? result.traceSummary : null; - const finalResponse = agentRunTerminalFinalResponse(result, traceId); - const promptEvidence = codeAgentPromptMetadata(result, {}, traceId); - const agentRun = result.agentRun && typeof result.agentRun === "object" ? { - adapter: result.agentRun.adapter ?? null, - runId: result.agentRun.runId ?? null, - commandId: result.agentRun.commandId ?? null, - attemptId: result.agentRun.attemptId ?? null, - runnerId: result.agentRun.runnerId ?? null, - jobName: result.agentRun.jobName ?? null, - namespace: result.agentRun.namespace ?? null, - terminalStatus: result.agentRun.terminalStatus ?? null, - lastSeq: result.agentRun.lastSeq ?? null, - valuesPrinted: false - } : null; - const terminalStatus = normalizeTurnStatus(result.agentRun?.terminalStatus, result.agentRun?.commandState, result.agentRun?.status, result.agentRun?.runStatus); - if (!isTraceCommandTerminalStatus(terminalStatus)) return null; - if (!agentRun?.runId || !agentRun?.commandId || (!storedSummary && !finalResponse)) return null; - const traceSummary = { - traceId, - source: "agentrun-command-result", - sourceEventCount: numberOrNull(storedSummary?.sourceEventCount ?? storedSummary?.eventCount ?? result.runnerTrace?.eventCount), - renderedRowSummary: storedSummary?.renderedRowSummary ?? null, - noiseEventCount: numberOrNull(storedSummary?.noiseEventCount), - omittedNoiseCount: numberOrNull(storedSummary?.omittedNoiseCount), - terminalStatus: storedSummary?.terminalStatus ?? terminalStatus, - updatedAt: storedSummary?.updatedAt ?? result.updatedAt ?? null, - valuesPrinted: false - }; - return { - source: "agentrun-command-result", - conversationId: safeConversationId(result.conversationId) || null, - sessionId: safeSessionId(result.sessionId) || null, - threadId: safeOpaqueId(result.threadId) || null, - ...codeAgentPromptFields(promptEvidence), - agentRun, - finalResponse, - traceSummary - }; -} - -function agentRunTerminalFinalResponse(result, traceId) { - const stored = result.finalResponse && typeof result.finalResponse === "object" ? result.finalResponse : null; - const text = textValue(stored?.text ?? result.reply?.content ?? result.message?.content ?? result.assistantText); - if (!text) return null; - const storedTraceId = safeTraceId(stored?.traceId ?? result.traceId) || null; - if (storedTraceId && storedTraceId !== traceId) return null; - return { - text, - textChars: text.length, - messageId: stored?.messageId ?? result.reply?.messageId ?? result.messageId ?? null, - role: stored?.role ?? result.reply?.role ?? "assistant", - status: stored?.status ?? result.status ?? null, - traceId: traceId, - createdAt: stored?.createdAt ?? result.reply?.createdAt ?? result.createdAt ?? null, - updatedAt: stored?.updatedAt ?? result.updatedAt ?? null, - source: "agentrun-command-result", - valuesPrinted: false - }; -} - -function traceRetentionSummary(status) { - return { - traceStatus: status, - liveTraceStore: status === "missing" ? "missing" : "expired-or-evicted", - policy: "live trace events are short-term storage; terminal final response and summary are resolved from AgentRun command result by traceId and commandId", - replacementEvidence: status === "missing" ? "resolve the trace through AgentRun command registry/result or report the trace as missing" : "use terminalEvidence.finalResponse, terminalEvidence.traceSummary, conversationId, sessionId, threadId, and agentRun ids", - valuesPrinted: false - }; -} - -function numberOrNull(value) { - const parsed = Number(value); - return Number.isFinite(parsed) ? parsed : null; -} - -function sendTraceSse(response, traceStore, traceId, options = {}) { - response.writeHead(200, { - "content-type": "text/event-stream; charset=utf-8", - "cache-control": "no-store", - connection: "keep-alive", - "x-accel-buffering": "no" - }); - const writeEvent = (eventName, payload) => { - response.write(`event: ${eventName}\n`); - response.write(`data: ${JSON.stringify(payload)}\n\n`); - }; - writeEvent("snapshot", traceStore.snapshot(traceId)); - const unsubscribe = traceStore.subscribe(traceId, (event, snapshot) => { - writeEvent("runnerTrace", { event, snapshot }); - }); - const heartbeatMs = parsePositiveInteger(options.env?.HWLAB_CODE_AGENT_TRACE_HEARTBEAT_MS, 15000); - const heartbeat = setInterval(() => { - writeEvent("heartbeat", traceStore.snapshot(traceId)); - }, heartbeatMs); - response.on("close", () => { - clearInterval(heartbeat); - unsubscribe(); - }); -} - -export function createCodeAgentChatResultStore({ maxResults = 256 } = {}) { - const limit = positiveInteger(maxResults, 256); - const results = new Map(); - function set(traceId, payload) { - const id = safeTraceId(traceId); - if (!id) return; - results.set(id, { - ...payload, - traceId: id, - cachedAt: new Date().toISOString() - }); - while (results.size > limit) { - const oldest = results.keys().next().value; - if (!oldest) break; - results.delete(oldest); - } - } - return { - set, - get: (traceId) => results.get(safeTraceId(traceId)) ?? null, - clear: () => results.clear() - }; -} - -function compactCodeAgentChatResultPayload(payload, options = {}) { - if (!payload || typeof payload !== "object") return payload; - const limit = resultTraceEventLimit(options); - const terminalEvidence = agentRunTerminalTraceEvidence(payload, payload.traceId); - const { userBillingReservation, ...publicPayload } = payload; - return { - ...publicPayload, - ...codeAgentTurnLifecycleFields(publicPayload.traceId, publicPayload), - ...(terminalEvidence ? { terminalEvidence: terminalEvidencePayload(terminalEvidence) } : {}), - ...(publicPayload.runnerTrace && typeof publicPayload.runnerTrace === "object" - ? { runnerTrace: compactRunnerTraceForResult(publicPayload.runnerTrace, limit) } - : {}) - }; -} - -function terminalEvidencePayload(evidence) { - return { - available: true, - source: evidence.source, - conversationId: evidence.conversationId, - sessionId: evidence.sessionId, - threadId: evidence.threadId, - ...codeAgentPromptFields(evidence), - agentRun: evidence.agentRun, - traceSummary: evidence.traceSummary, - finalResponse: evidence.finalResponse, - valuesPrinted: false - }; -} - -function compactRunnerTraceForResult(runnerTrace, limit = DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT) { - if (!runnerTrace || typeof runnerTrace !== "object") return runnerTrace; - const events = Array.isArray(runnerTrace.events) ? runnerTrace.events : []; - const total = Number.isInteger(runnerTrace.eventCount) ? runnerTrace.eventCount : events.length; - const effectiveLimit = Math.max(8, positiveInteger(limit, DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT)); - if (events.length <= effectiveLimit) { - return { - ...runnerTrace, - eventCount: total, - eventsCompacted: false - }; - } - const headCount = Math.min(24, Math.max(1, Math.floor(effectiveLimit / 4))); - const tailCount = Math.max(1, effectiveLimit - headCount - 1); - const omitted = Math.max(0, events.length - headCount - tailCount); - const head = events.slice(0, headCount); - const tail = events.slice(events.length - tailCount); - const marker = { - traceId: runnerTrace.traceId ?? head[0]?.traceId ?? tail[0]?.traceId ?? null, - type: "trace", - stage: "trace", - status: "truncated", - label: "trace:compacted", - createdAt: tail[0]?.createdAt ?? runnerTrace.updatedAt ?? new Date().toISOString(), - message: `Result polling response compacted ${omitted} trace events; fetch /v1/agent/traces/${encodeURIComponent(runnerTrace.traceId ?? "")} for the full trace.`, - omittedEventCount: omitted, - retainedEventCount: head.length + tail.length, - totalEventCount: total, - valuesPrinted: false - }; - const compactEvents = [...head, marker, ...tail]; - return { - ...runnerTrace, - eventCount: total, - events: compactEvents, - eventLabels: compactEvents.map(traceEventLabel).filter(Boolean), - eventsCompacted: true, - eventWindow: { - mode: "head-tail", - retained: compactEvents.length, - omitted, - total - } - }; -} - -function traceEventLabel(event) { - return typeof event === "string" ? event : event?.label; -} - -function resultTraceEventLimit(options = {}) { - return positiveInteger( - options.env?.HWLAB_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT, - DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT - ); -} - -function createCodeAgentM3HwlabApiRequestJson(options = {}) { - return async (targetUrl, request = {}) => { - const url = new URL(targetUrl); - const route = url.pathname; - if (route === M3_IO_CONTROL_ROUTE && request.method === "POST") { - return { - ok: true, - status: 200, - body: await handleM3IoControl(request.body ?? {}, { - runtimeStore: options.runtimeStore, - now: options.now, - env: options.env, - requestJson: options.m3IoRequestJson - }) - }; - } - if (route === M3_STATUS_ROUTE && request.method === "GET") { - return { - ok: true, - status: 200, - body: await describeM3StatusLive(options) - }; - } - return { - ok: false, - status: 404, - body: { - error: { - code: "not_found", - message: `Unsupported Code Agent M3 HWLAB API route ${route}` - } - }, - error: `Unsupported Code Agent M3 HWLAB API route ${route}` - }; - }; -} -function recordCodeAgentProjectionMetrics(options = {}, { result = null, trace = null, projection = null, refreshError = null } = {}) { - const performanceStore = options.backendPerformanceStore ?? options.backendPerformance; - if (!performanceStore?.recordWorkbenchProjection || !projection) return; - const sourceLatestSeq = sourceLatestSeqFromResult(result); - const lastProjectedSeq = Number(projection.lastProjectedSeq ?? 0); - if (!Number.isFinite(sourceLatestSeq) && !Number.isFinite(lastProjectedSeq)) return; - performanceStore.recordWorkbenchProjection({ - sourceLatestSeq: Number.isFinite(sourceLatestSeq) ? sourceLatestSeq : lastProjectedSeq, - lastProjectedSeq: Number.isFinite(lastProjectedSeq) ? lastProjectedSeq : 0, - sourceLatestEventAt: sourceLatestAtFromResult(result, trace), - projectedLatestEventAt: projection.updatedAt ?? trace?.updatedAt ?? result?.updatedAt ?? null, - terminalSourceAt: terminalSourceAtFromResult(result), - terminalProjectedAt: projection.projectionStatus === "caught-up" ? projection.updatedAt ?? trace?.updatedAt ?? result?.updatedAt ?? null : null, - status: projection?.status ?? "unknown", - reason: projectionReason(projection, refreshError), - projectionStatus: projection.projectionStatus ?? "unknown", - source: "workbench_read_model" - }); -} - -function recordCodeAgentTurnReadMetric(options = {}, url, statusCode, body = {}, startedAt = nowMs()) { - const performanceStore = options.backendPerformanceStore ?? options.backendPerformance; - if (!performanceStore?.recordWorkbenchTurnRead) return; - performanceStore.recordWorkbenchTurnRead({ - route: url?.pathname ?? "/v1/agent/turns/:id", - status: body?.status ?? statusClassLabel(statusCode), - degradedReason: turnReadDegradedReason(body), - durationMs: nowMs() - startedAt, - responseBytes: responseBodyBytes(body), - timedOut: statusCode === 504 || body?.error?.code === "timeout" || body?.refreshError?.code === "agentrun_timeout" - }); -} - -function turnReadDegradedReason(body = {}) { - if (body?.refreshError?.code) return body.refreshError.code; - if (body?.projection?.blocker?.code) return body.projection.blocker.code; - if (body?.blocker?.code) return body.blocker.code; - if (body?.error?.code) return body.error.code; - if (body?.projectionStatus === "projecting") return "projecting"; - if (body?.projectionStatus === "blocked") return "projection_blocked"; - return "none"; -} - -function responseBodyBytes(body = {}) { - try { return Buffer.byteLength(JSON.stringify(body)); } catch { return 0; } -} - -function statusClassLabel(value) { - const status = Number(value); - if (!Number.isFinite(status) || status <= 0) return "unknown"; - return `${Math.floor(status / 100)}xx`; -} - -function sourceLatestSeqFromResult(result = null) { - const candidates = [ - result?.agentRun?.lastSeq, - result?.traceSummary?.agentRun?.lastSeq, - result?.traceSummary?.lastSeq, - result?.providerTrace?.lastSeq, - result?.runnerTrace?.lastEvent?.sourceSeq, - result?.runnerTrace?.lastEvent?.seq, - result?.runnerTrace?.eventCount - ]; - for (const value of candidates) { - const seq = Number(value); - if (Number.isFinite(seq) && seq >= 0) return Math.floor(seq); - } - return NaN; -} - -function sourceLatestAtFromResult(result = null, trace = null) { - return result?.traceSummary?.updatedAt - ?? result?.agentRun?.updatedAt - ?? result?.providerTrace?.updatedAt - ?? result?.runnerTrace?.updatedAt - ?? trace?.updatedAt - ?? result?.updatedAt - ?? null; -} - -function terminalSourceAtFromResult(result = null) { - if (!result) return null; - const terminalStatus = result?.agentRun?.terminalStatus ?? result?.traceSummary?.terminalStatus ?? result?.status; - if (!isTraceCommandTerminalStatus(terminalStatus)) return null; - return result?.traceSummary?.updatedAt ?? result?.agentRun?.updatedAt ?? result?.updatedAt ?? null; -} - -function projectionReason(projection = null, refreshError = null) { - if (refreshError) return refreshError.code ?? "refresh_error"; - if (projection?.blocker) return projection.blocker.code ?? "projection_blocked"; - if (projection?.projectionStatus === "projecting") return "projecting"; - if (projection?.projectionStatus === "caught-up") return "none"; - return projection?.projectionStatus ?? "unknown"; -} - -function nowMs() { - if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now(); - return Date.now(); -} - -function uniqueStrings(values) { - return [...new Set(values.map((value) => String(value ?? "").trim()).filter(Boolean))]; -} +export { handleCodeAgentChatHttp, handleCodeAgentSessionsHttp } from "./server-code-agent-admission-http.ts"; +export { codeAgentTurnStatusPayload, handleCodeAgentCancelHttp, handleCodeAgentChatResultHttp, handleCodeAgentInspectHttp, handleCodeAgentTurnHttp } from "./server-code-agent-turn-http.ts"; +export { handleCodeAgentSteerHttp, handleCodeAgentTraceHttp } from "./server-code-agent-trace-http.ts"; +export { createCodeAgentChatResultStore } from "./server-code-agent-http-support.ts"; diff --git a/internal/cloud/server-code-agent-trace-http.ts b/internal/cloud/server-code-agent-trace-http.ts new file mode 100644 index 00000000..1d472bd9 --- /dev/null +++ b/internal/cloud/server-code-agent-trace-http.ts @@ -0,0 +1,434 @@ +// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-0102 Agent编排 draft-2026-06-17-r0; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. +// Responsibility: Code Agent HTTP steer and trace resources. + +import { createHash, randomUUID } from "node:crypto"; + +import { M3_IO_CONTROL_ROUTE, M3_STATUS_ROUTE, describeM3StatusLive, handleM3IoControl } from "./m3-io-control.ts"; +import { + agentRunSessionEvidence, + cancelAgentRunChatTurn, + codeAgentAgentRunAdapterEnabled, + initialAgentRunChatResult, + loadPersistedAgentRunResult, + steerAgentRunChatTurn, + submitAgentRunChatTurn +} from "./code-agent-agentrun-adapter.ts"; +import { createCodeAgentErrorPayload, handleCodeAgentChat } from "./code-agent-chat.ts"; +import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; +import { codeAgentSessionLifecycleSummary } from "./code-agent-session-lifecycle.ts"; +import { messageAuthorityTextValue } from "./code-agent-agentrun-prompt.ts"; +import { writeWorkbenchProjectionSession, writeWorkbenchSessionAdmissionFact } from "./workbench-projection-writer.ts"; +import { createWorkbenchReadModel } from "./workbench-read-model.ts"; +import { codeAgentOtelTraceFields, emitCodeAgentOtelSpan } from "./otel-trace.ts"; +import { + firstHeaderValue, + getHeader, + parsePositiveInteger, + positiveInteger, + readBody, + safeConversationId, + safeOpaqueId, + safeSessionId, + safeTraceId, + sendJson, + truthyFlag +} from "./server-http-utils.ts"; + +import { + normalizeCodeAgentProviderProfile, + firstNonEmptyValue, + recordCodeAgentSessionInputFact, + stableCodeAgentInputId, + codeAgentInputDelivery, + codeAgentInputStatus, + codeAgentResultTimingFields, + firstTimestampIso, + firstLatestTimestampIso, + codeAgentChatShortConnectionRequested, + preflightCodeAgentBilling, + recordCodeAgentBillingUsage, + finalizeCodeAgentBillingUsage, + releaseCodeAgentBillingReservation, + withCodeAgentBillingReservation, + codeAgentBillingEnabled, + codeAgentBillingMetadata, + sanitizeBillingReservation, + sanitizeBillingRecord, + sanitizeBillingRelease, + codeAgentUsedTokens, + normalizeTurnStatus, + recordCodeAgentConversationFact, + recordCodeAgentSessionOwner, + codeAgentOwnerStatusForResult, + textValue, + timestampIsoOrNow, + timestampIsoOrNull, + codeAgentSessionOwnerEvidence, + codeAgentContinuationEvidence, + codeAgentFreshContinuationFailure, + normalizeCodeAgentFailureKind, + codeAgentFailureRequiresFreshContinuation, + codeAgentProviderProfileEvidence, + codeAgentTraceResultEvidence, + codeAgentPromptMetadata, + codeAgentPromptMetadataFromText, + codeAgentPromptMetadataFromParts, + codeAgentPromptFields, + codeAgentPromptId, + clippedPromptText, + sha256Text, + compactCodeAgentObject, + codeAgentBillingReservationEvidence, + codeAgentConversationMessagesEvidence, + boundedConversationMessageText, + conversationText, + conversationTextRaw, + codeAgentMessageTraceSuffix, + codeAgentTurnLifecycleFields, + codeAgentLifecycleMessageId, + safeTurnId, + safeMessageId, + codeAgentConversationAgentMessageStatus, + codeAgentPayloadHasSealedFinalResponse, + codeAgentConversationRunnerTraceEvidence, + codeAgentFinalResponseEvidence, + codeAgentFinalResponseText, + codeAgentTraceSummaryEvidence, + annotateOwner, + canAccessOwnedResult, + isCodeAgentResultCanceled, + codeAgentCancelScopeMismatch, + cancelExpectedValues, + cancelScopeFieldMismatch, + cancelBlockedPayload, + isTraceCommandTerminalStatus, + traceSnapshotWithTerminalEvidence, + traceSnapshotWithAttachedTerminalEvidence, + agentRunTerminalTraceEvidence, + agentRunTerminalFinalResponse, + traceRetentionSummary, + numberOrNull, + createCodeAgentChatResultStore, + compactCodeAgentChatResultPayload, + terminalEvidencePayload, + compactRunnerTraceForResult, + traceEventLabel, + resultTraceEventLimit, + createCodeAgentM3HwlabApiRequestJson, + recordCodeAgentProjectionMetrics, + recordCodeAgentTurnReadMetric, + turnReadDegradedReason, + responseBodyBytes, + statusClassLabel, + sourceLatestSeqFromResult, + sourceLatestAtFromResult, + terminalSourceAtFromResult, + projectionReason, + nowMs, + uniqueStrings +} from "./server-code-agent-http-support.ts"; + +import { + codeAgentCompatProjectionPayload, + codeAgentRequestScopedOptions, + measureCodeAgentHttpPhase, + readCodeAgentCompatProjection, + traceProjectMismatchSnapshot +} from "./server-code-agent-turn-http.ts"; + +const DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT = 120; +const DEFAULT_CODE_AGENT_TRACE_PAGE_LIMIT = 100; +const MAX_CODE_AGENT_TRACE_PAGE_LIMIT = 100; +const DEFAULT_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS = 2500; +const DEFAULT_CODE_AGENT_PROJECT_ID = "prj_hwpod_workbench"; +const DEFAULT_CODE_AGENT_PROVIDER_PROFILE = "deepseek"; +const CODE_AGENT_TERMINAL_STATUSES = new Set(["completed", "failed", "blocked", "timeout", "cancelled", "canceled"]); +const CODE_AGENT_PROVIDER_PROFILE_ALIASES = Object.freeze({ codex: "codex-api" }); +const CODE_AGENT_PROVIDER_PROFILE_ID_PATTERN = /^[a-z0-9][a-z0-9.-]{0,63}$/u; +const CODE_AGENT_PROVIDER_PROFILE_LABELS = Object.freeze({ + "deepseek": "DeepSeek", + "codex-api": "Codex API", + "gpt.pika": "gpt.pika", + "minimax-m3": "MiniMax-M3 via AgentRun", + "runtime-default": "运行默认" +}); +const DEFAULT_CODE_AGENT_CODEX_API_MODEL = "gpt-5.5"; +const DEFAULT_CODE_AGENT_CODEX_API_BASE_URL = "http://127.0.0.1:49280/responses"; +const DEFAULT_CODE_AGENT_DEEPSEEK_MODEL = "deepseek-chat"; +const DEFAULT_CODE_AGENT_MINIMAX_M3_MODEL = "MiniMax-M3"; + +export async function handleCodeAgentSteerHttp(request, response, options) { + const body = await readBody(request, options.bodyLimitBytes); + let params = {}; + + try { + params = body ? JSON.parse(body) : {}; + } catch (error) { + sendJson(response, 400, { + ...createCodeAgentErrorPayload({ + code: "parse_error", + message: "Invalid JSON body", + reason: error.message, + traceId: getHeader(request, "x-trace-id") || "trc_unassigned", + layer: "api", + retryable: true, + route: "/v1/agent/chat/steer" + }) + }); + return; + } + + if (!params || typeof params !== "object" || Array.isArray(params)) { + sendJson(response, 400, { + ...createCodeAgentErrorPayload({ + code: "invalid_params", + message: "Code Agent steer body must be a JSON object", + traceId: getHeader(request, "x-trace-id") || "trc_unassigned", + layer: "api", + retryable: true, + route: "/v1/agent/chat/steer" + }) + }); + return; + } + + const traceId = safeTraceId(getHeader(request, "x-trace-id") || params.traceId || params.targetTraceId); + const message = firstNonEmptyValue(params.message, params.prompt, params.text); + if (!traceId || !message) { + sendJson(response, 400, { + ...createCodeAgentErrorPayload({ + code: !traceId ? "steer_trace_missing" : "steer_message_missing", + message: !traceId ? "traceId is required to steer the current Code Agent turn." : "message is required to steer the current Code Agent turn.", + traceId: traceId || "trc_unassigned", + layer: "api", + retryable: true, + route: "/v1/agent/chat/steer" + }) + }); + return; + } + + const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; + const durableResult = await loadPersistedAgentRunResult(traceId, options); + const currentResult = durableResult ?? options.codeAgentChatResults?.get?.(traceId); + if (!canAccessOwnedResult(currentResult, options.actor)) { + sendJson(response, 403, { + ok: false, + status: "forbidden", + traceId, + error: { code: "code_agent_trace_forbidden", message: "Code Agent trace is not visible to the current actor" } + }); + return; + } + if (!currentResult?.agentRun?.runId || !currentResult?.agentRun?.commandId) { + sendJson(response, 404, { + ok: false, + status: "not_found", + traceId, + error: { code: "steer_trace_not_found", message: "No AgentRun-backed Code Agent turn was found for the requested traceId." }, + route: "/v1/agent/chat/steer", + runnerTrace: traceStore.snapshot(traceId) + }); + return; + } + if (isTraceCommandTerminalStatus(currentResult.status)) { + sendJson(response, 409, { + ok: false, + accepted: false, + status: "blocked", + traceId, + route: "/v1/agent/chat/steer", + sessionId: safeSessionId(currentResult.sessionId ?? params.sessionId) || null, + threadId: safeOpaqueId(currentResult.threadId ?? params.threadId) || null, + agentRun: currentResult.agentRun, + runnerTrace: traceStore.snapshot(traceId), + error: { + code: "steer_trace_terminal", + layer: "api", + category: "not_in_flight", + retryable: false, + message: `Trace ${traceId} is already terminal and cannot accept a steer command.`, + traceId, + route: "/v1/agent/chat/steer", + toolName: "agentrun.command.steer" + }, + valuesPrinted: false + }); + return; + } + + try { + await recordCodeAgentSessionInputFact({ + params: { + ...params, + sessionId: currentResult.sessionId ?? currentResult.session?.sessionId ?? params.sessionId, + ownerUserId: options.actor?.id, + ownerRole: options.actor?.role + }, + options, + traceId, + delivery: "steer", + status: "promoted", + commandId: currentResult.agentRun.commandId + }); + const payload = await steerAgentRunChatTurn({ + traceId, + currentResult, + params: { + ...params, + message, + ownerUserId: options.actor?.id ?? params.ownerUserId, + ownerRole: options.actor?.role ?? params.ownerRole + }, + options, + traceStore + }); + sendJson(response, 202, payload); + } catch (error) { + traceStore.append(traceId, { + type: "backend", + status: "failed", + label: "agentrun:steer:failed", + errorCode: error?.code ?? "agentrun_steer_failed", + message: error?.message ?? "AgentRun steer command failed.", + terminal: false, + valuesPrinted: false + }); + sendJson(response, error?.statusCode === 404 ? 404 : 409, { + ok: false, + accepted: false, + status: "failed", + traceId, + route: "/v1/agent/chat/steer", + sessionId: safeSessionId(currentResult.sessionId ?? params.sessionId) || null, + threadId: safeOpaqueId(currentResult.threadId ?? params.threadId) || null, + agentRun: currentResult.agentRun, + runnerTrace: traceStore.snapshot(traceId), + error: { + code: error?.code ?? "agentrun_steer_failed", + layer: "agentrun", + category: "steer_failed", + retryable: true, + message: error?.message ?? "AgentRun steer command failed.", + traceId, + route: "/v1/agent/chat/steer", + toolName: "agentrun.command.steer" + } + }); + } +} + +export async function handleCodeAgentTraceHttp(request, response, url, options) { + const startedAt = nowMs(); + const parts = url.pathname.split("/").filter(Boolean); + const traceId = decodeURIComponent(parts[3] ?? ""); + const streamSegment = parts[4]; + const pageOptions = codeAgentTracePageOptions(url); + if (!/^trc_[A-Za-z0-9_.:-]+$/u.test(traceId)) { + sendJson(response, 400, { + error: { + code: "invalid_trace_id", + message: "traceId must start with trc_ and contain only safe identifier characters" + } + }); + return; + } + const requestOptions = codeAgentRequestScopedOptions(options); + const projectMismatch = await measureCodeAgentHttpPhase(requestOptions, "trace_project_check", () => traceProjectMismatchSnapshot(traceId, url, requestOptions, "trace")); + if (projectMismatch) { + sendJson(response, projectMismatch.statusCode, projectMismatch.body); + return; + } + const context = await measureCodeAgentHttpPhase(requestOptions, "trace_projection_read", () => readCodeAgentCompatProjection(traceId, requestOptions)); + if (context.statusCode) { + sendJson(response, context.statusCode, context.body); + return; + } + + if (streamSegment === "stream") { + sendJson(response, 410, { + ok: false, + status: "removed", + traceId, + error: { + code: "trace_store_sse_removed", + message: "Trace-store SSE was removed; use the committed Workbench projection event stream.", + workbenchEventsUrl: `/v1/workbench/events?traceId=${encodeURIComponent(traceId)}` + }, + valuesRedacted: true + }); + return; + } + + const body = await measureCodeAgentHttpPhase(requestOptions, "trace_paginate", () => { + return paginateTraceSnapshot(context.trace, pageOptions); + }); + const statusCode = context.found ? 200 : 404; + void emitCodeAgentOtelSpan("trace_events_read", traceId, requestOptions.env ?? process.env, { + startTimeMs: startedAt, + attributes: { + "http.method": "GET", + "http.route": "/v1/agent/traces/:traceId", + "http.status_code": statusCode, + found: Boolean(context.found), + returnedEvents: Array.isArray(body?.events) ? body.events.length : 0, + sinceSeq: pageOptions.sinceSeq, + limit: pageOptions.limit, + fromSeq: body?.range?.fromSeq ?? null, + toSeq: body?.range?.toSeq ?? null, + totalEvents: body?.range?.total ?? body?.eventCount ?? null, + hasMore: Boolean(body?.hasMore), + fullTraceLoaded: Boolean(body?.fullTraceLoaded) + } + }); + sendJson(response, statusCode, codeAgentCompatProjectionPayload(body, context)); +} + +function codeAgentTracePageOptions(url) { + const sinceSeq = nonNegativeInteger(url.searchParams.get("sinceSeq") ?? url.searchParams.get("since"), 0); + const requestedLimit = parsePositiveInteger(url.searchParams.get("limit"), DEFAULT_CODE_AGENT_TRACE_PAGE_LIMIT); + return { + sinceSeq, + limit: Math.min(requestedLimit, MAX_CODE_AGENT_TRACE_PAGE_LIMIT) + }; +} + +function paginateTraceSnapshot(snapshot, { sinceSeq, limit }) { + const sourceEvents = Array.isArray(snapshot?.events) ? snapshot.events : []; + const indexedEvents = sourceEvents.map((event, index) => ({ event, seq: traceEventSequence(event, index) })); + const firstIndex = indexedEvents.findIndex((item) => item.seq > sinceSeq); + const startIndex = firstIndex >= 0 ? firstIndex : indexedEvents.length; + const pageItems = indexedEvents.slice(startIndex, startIndex + limit); + const events = pageItems.map((item) => item.event); + const fromSeq = pageItems.length > 0 ? pageItems[0].seq : null; + const toSeq = pageItems.length > 0 ? pageItems[pageItems.length - 1].seq : null; + const hasMore = startIndex + pageItems.length < indexedEvents.length; + const totalEvents = snapshot?.eventCount ?? indexedEvents.length; + return { + ...snapshot, + events, + eventCount: totalEvents, + range: { + sinceSeq, + fromSeq, + toSeq, + limit, + returned: events.length, + total: totalEvents + }, + truncated: hasMore, + hasMore, + nextSinceSeq: pageItems.length > 0 ? toSeq : sinceSeq, + fullTraceLoaded: !hasMore + }; +} + +function traceEventSequence(event, index) { + const seq = Number(event?.seq); + return Number.isFinite(seq) && seq > 0 ? Math.trunc(seq) : index + 1; +} + +function nonNegativeInteger(value, fallback) { + const parsed = Number.parseInt(value ?? "", 10); + return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback; +} diff --git a/internal/cloud/server-code-agent-turn-http.ts b/internal/cloud/server-code-agent-turn-http.ts new file mode 100644 index 00000000..797cedeb --- /dev/null +++ b/internal/cloud/server-code-agent-turn-http.ts @@ -0,0 +1,892 @@ +// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-0102 Agent编排 draft-2026-06-17-r0; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. +// Responsibility: Code Agent HTTP result, turn, inspect, and cancel resources. + +import { createHash, randomUUID } from "node:crypto"; + +import { M3_IO_CONTROL_ROUTE, M3_STATUS_ROUTE, describeM3StatusLive, handleM3IoControl } from "./m3-io-control.ts"; +import { + agentRunSessionEvidence, + cancelAgentRunChatTurn, + codeAgentAgentRunAdapterEnabled, + initialAgentRunChatResult, + loadPersistedAgentRunResult, + steerAgentRunChatTurn, + submitAgentRunChatTurn +} from "./code-agent-agentrun-adapter.ts"; +import { createCodeAgentErrorPayload, handleCodeAgentChat } from "./code-agent-chat.ts"; +import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; +import { codeAgentSessionLifecycleSummary } from "./code-agent-session-lifecycle.ts"; +import { messageAuthorityTextValue } from "./code-agent-agentrun-prompt.ts"; +import { writeWorkbenchProjectionSession, writeWorkbenchSessionAdmissionFact } from "./workbench-projection-writer.ts"; +import { createWorkbenchReadModel } from "./workbench-read-model.ts"; +import { codeAgentOtelTraceFields, emitCodeAgentOtelSpan } from "./otel-trace.ts"; +import { + firstHeaderValue, + getHeader, + parsePositiveInteger, + positiveInteger, + readBody, + safeConversationId, + safeOpaqueId, + safeSessionId, + safeTraceId, + sendJson, + truthyFlag +} from "./server-http-utils.ts"; + +import { + normalizeCodeAgentProviderProfile, + firstNonEmptyValue, + recordCodeAgentSessionInputFact, + stableCodeAgentInputId, + codeAgentInputDelivery, + codeAgentInputStatus, + codeAgentResultTimingFields, + firstTimestampIso, + firstLatestTimestampIso, + codeAgentChatShortConnectionRequested, + preflightCodeAgentBilling, + recordCodeAgentBillingUsage, + finalizeCodeAgentBillingUsage, + releaseCodeAgentBillingReservation, + withCodeAgentBillingReservation, + codeAgentBillingEnabled, + codeAgentBillingMetadata, + sanitizeBillingReservation, + sanitizeBillingRecord, + sanitizeBillingRelease, + codeAgentUsedTokens, + normalizeTurnStatus, + recordCodeAgentConversationFact, + recordCodeAgentSessionOwner, + codeAgentOwnerStatusForResult, + textValue, + timestampIsoOrNow, + timestampIsoOrNull, + codeAgentSessionOwnerEvidence, + codeAgentContinuationEvidence, + codeAgentFreshContinuationFailure, + normalizeCodeAgentFailureKind, + codeAgentFailureRequiresFreshContinuation, + codeAgentProviderProfileEvidence, + codeAgentTraceResultEvidence, + codeAgentPromptMetadata, + codeAgentPromptMetadataFromText, + codeAgentPromptMetadataFromParts, + codeAgentPromptFields, + codeAgentPromptId, + clippedPromptText, + sha256Text, + compactCodeAgentObject, + codeAgentBillingReservationEvidence, + codeAgentConversationMessagesEvidence, + boundedConversationMessageText, + conversationText, + conversationTextRaw, + codeAgentMessageTraceSuffix, + codeAgentTurnLifecycleFields, + codeAgentLifecycleMessageId, + safeTurnId, + safeMessageId, + codeAgentConversationAgentMessageStatus, + codeAgentPayloadHasSealedFinalResponse, + codeAgentConversationRunnerTraceEvidence, + codeAgentFinalResponseEvidence, + codeAgentFinalResponseText, + codeAgentTraceSummaryEvidence, + annotateOwner, + canAccessOwnedResult, + isCodeAgentResultCanceled, + codeAgentCancelScopeMismatch, + cancelExpectedValues, + cancelScopeFieldMismatch, + cancelBlockedPayload, + isTraceCommandTerminalStatus, + traceSnapshotWithTerminalEvidence, + traceSnapshotWithAttachedTerminalEvidence, + agentRunTerminalTraceEvidence, + agentRunTerminalFinalResponse, + traceRetentionSummary, + numberOrNull, + createCodeAgentChatResultStore, + compactCodeAgentChatResultPayload, + terminalEvidencePayload, + compactRunnerTraceForResult, + traceEventLabel, + resultTraceEventLimit, + createCodeAgentM3HwlabApiRequestJson, + recordCodeAgentProjectionMetrics, + recordCodeAgentTurnReadMetric, + turnReadDegradedReason, + responseBodyBytes, + statusClassLabel, + sourceLatestSeqFromResult, + sourceLatestAtFromResult, + terminalSourceAtFromResult, + projectionReason, + nowMs, + uniqueStrings +} from "./server-code-agent-http-support.ts"; + +const DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT = 120; +const DEFAULT_CODE_AGENT_TRACE_PAGE_LIMIT = 100; +const MAX_CODE_AGENT_TRACE_PAGE_LIMIT = 100; +const DEFAULT_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS = 2500; +const DEFAULT_CODE_AGENT_PROJECT_ID = "prj_hwpod_workbench"; +const DEFAULT_CODE_AGENT_PROVIDER_PROFILE = "deepseek"; +const CODE_AGENT_TERMINAL_STATUSES = new Set(["completed", "failed", "blocked", "timeout", "cancelled", "canceled"]); +const CODE_AGENT_PROVIDER_PROFILE_ALIASES = Object.freeze({ codex: "codex-api" }); +const CODE_AGENT_PROVIDER_PROFILE_ID_PATTERN = /^[a-z0-9][a-z0-9.-]{0,63}$/u; +const CODE_AGENT_PROVIDER_PROFILE_LABELS = Object.freeze({ + "deepseek": "DeepSeek", + "codex-api": "Codex API", + "gpt.pika": "gpt.pika", + "minimax-m3": "MiniMax-M3 via AgentRun", + "runtime-default": "运行默认" +}); +const DEFAULT_CODE_AGENT_CODEX_API_MODEL = "gpt-5.5"; +const DEFAULT_CODE_AGENT_CODEX_API_BASE_URL = "http://127.0.0.1:49280/responses"; +const DEFAULT_CODE_AGENT_DEEPSEEK_MODEL = "deepseek-chat"; +const DEFAULT_CODE_AGENT_MINIMAX_M3_MODEL = "MiniMax-M3"; + +export async function handleCodeAgentChatResultHttp(request, response, url, options) { + const parts = url.pathname.split("/").filter(Boolean); + const traceId = decodeURIComponent(parts[4] ?? ""); + if (!safeTraceId(traceId)) { + sendJson(response, 400, { + error: { + code: "invalid_trace_id", + message: "traceId must start with trc_ and contain only safe identifier characters" + } + }); + return; + } + const context = await measureCodeAgentHttpPhase(options, "result_projection_read", () => readCodeAgentCompatProjection(traceId, options)); + if (context.statusCode) { + sendJson(response, context.statusCode, context.body); + return; + } + const result = context.result; + if (result && result.status !== "running") { + sendJson(response, 200, codeAgentCompatProjectionPayload(compactCodeAgentChatResultPayload(result, options), context)); + return; + } + if (context.found) { + const body = codeAgentTurnStatusPayload({ traceId, result, snapshot: context.trace, resultPollError: null, refreshError: context.refreshError ?? null, options }); + const resultPollTerminalReady = codeAgentResultPollTerminalReady(body); + sendJson(response, resultPollTerminalReady ? 200 : 202, codeAgentCompatProjectionPayload({ + accepted: body.ok, + shortConnection: true, + ...body, + waitingFor: resultPollTerminalReady ? body.waitingFor ?? context.trace?.waitingFor ?? "workbench-projection" : "agentrun-result", + ...(body.terminal === true && !resultPollTerminalReady ? { status: "running", running: true, terminal: false } : {}) + }, context)); + return; + } + sendJson(response, 404, { + ...codeAgentCompatProjectionPayload({ traceId, status: "unknown" }, context), + error: { + code: "code_agent_result_not_found", + message: `No Code Agent result is registered for ${traceId}` + } + }); +} + +export async function handleCodeAgentTurnHttp(request, response, url, options) { + const startedAt = nowMs(); + const parts = url.pathname.split("/").filter(Boolean); + const traceId = decodeURIComponent(parts[3] ?? ""); + if (!safeTraceId(traceId)) { + const body = { + ok: false, + status: "unknown", + running: false, + terminal: false, + error: { + code: "invalid_trace_id", + message: "traceId must start with trc_ and contain only safe identifier characters" + } + }; + recordCodeAgentTurnReadMetric(options, url, 400, body, startedAt); + sendJson(response, 400, body); + return; + } + const requestOptions = codeAgentRequestScopedOptions(options); + let completed = false; + const emitTurnStatusReadSpan = (name, attributes = {}, error = null) => { + void emitCodeAgentOtelSpan(name, traceId, requestOptions.env ?? process.env, { + startTimeMs: startedAt, + status: error ? "error" : undefined, + error, + attributes: { + "http.method": "GET", + "http.route": "/v1/agent/turns/:traceId", + durationMs: nowMs() - startedAt, + ...attributes + } + }); + }; + response.once?.("close", () => { + if (completed) return; + emitTurnStatusReadSpan("turn_status_read.client_closed", { + status: "client_closed", + phase: "response_close", + terminal: false + }, new Error("Code Agent turn status client connection closed before response completed.")); + }); + try { + const projectMismatch = await measureCodeAgentHttpPhase(requestOptions, "turn_project_check", () => traceProjectMismatchSnapshot(traceId, url, requestOptions, "turn")); + if (projectMismatch) { + completed = true; + recordCodeAgentTurnReadMetric(options, url, projectMismatch.statusCode, projectMismatch.body, startedAt); + emitTurnStatusReadSpan("turn_status_read", { "http.status_code": projectMismatch.statusCode, status: projectMismatch.body?.status ?? null, terminal: projectMismatch.body?.terminal ?? null, phase: "project_check" }); + sendJson(response, projectMismatch.statusCode, projectMismatch.body); + return; + } + const resolved = await measureCodeAgentHttpPhase(requestOptions, "turn_resolve_status", () => resolveCodeAgentTurnStatusSnapshot(traceId, requestOptions)); + completed = true; + recordCodeAgentTurnReadMetric(options, url, resolved.statusCode, resolved.body, startedAt); + emitTurnStatusReadSpan("turn_status_read", { "http.status_code": resolved.statusCode, status: resolved.body?.status ?? null, terminal: resolved.body?.terminal ?? null, phase: "resolved" }); + sendJson(response, resolved.statusCode, resolved.body); + } catch (error) { + completed = true; + const body = { + ok: false, + status: "unknown", + running: false, + terminal: false, + traceId, + error: { + code: error?.code ?? "turn_status_read_failed", + message: error?.message ?? "Code Agent turn status read failed." + }, + valuesRedacted: true + }; + recordCodeAgentTurnReadMetric(options, url, 500, body, startedAt); + emitTurnStatusReadSpan("turn_status_read", { "http.status_code": 500, status: body.status, terminal: false, phase: "failed", errorCode: body.error.code }, error); + sendJson(response, 500, body); + } +} + +async function resolveCodeAgentTurnStatusSnapshot(traceId, options) { + const context = await readCodeAgentCompatProjection(traceId, options); + if (context.statusCode) return context; + const body = await measureCodeAgentHttpPhase(options, "turn_payload", () => codeAgentTurnStatusPayload({ + traceId, + result: context.result, + snapshot: context.trace, + resultPollError: null, + refreshError: context.refreshError ?? null, + options + })); + return { statusCode: body.ok ? 200 : 404, body: codeAgentCompatProjectionPayload(body, context) }; +} + +function codeAgentRequestScopedOptions(options = {}) { + return { + ...options, + codeAgentTraceSessionCache: options.codeAgentTraceSessionCache instanceof Map ? options.codeAgentTraceSessionCache : new Map() + }; +} + +async function measureCodeAgentHttpPhase(options, phase, callback) { + const perf = options?.backendPerformance; + return perf && typeof perf.measure === "function" ? await perf.measure(phase, callback) : await callback(); +} + +function forbiddenTurnSnapshot(traceId) { + return { + statusCode: 403, + body: { + ok: false, + status: "unknown", + running: false, + terminal: false, + traceId, + error: { + code: "agent_session_owner_required", + message: "Only the session owner or admin can read this Code Agent turn status" + } + } + }; +} + +async function readCodeAgentCompatProjection(traceId, options = {}) { + const readModel = createWorkbenchReadModel(options, options.actor ?? null); + const session = await readModel.getSessionByTraceId(traceId); + let result = readModel.resultForTrace(traceId) ?? projectedSessionTraceResult(session, traceId); + if (result && !canAccessOwnedResult(result, options.actor)) return forbiddenTurnSnapshot(traceId); + if (result?.ownerUserId && !readModel.canReadOwner(result.ownerUserId)) return forbiddenTurnSnapshot(traceId); + const projectionTrace = await readModel.traceSnapshot(traceId); + let trace = traceSnapshotWithTerminalEvidence(projectionTrace, result, traceId, null); + const projection = readModel.projectionDiagnostics({ traceId, result, trace, refreshError: null }); + recordCodeAgentProjectionMetrics(options, { result, trace, projection, refreshError: null }); + const found = Boolean(result || session || (trace && trace.status !== "missing") || trace?.persisted === true); + return { result, session, trace, projection, found, refreshError: null }; +} + +function projectedSessionTraceResult(session, traceId) { + const safeId = safeTraceId(traceId); + if (!session || !safeId) return null; + const snapshot = session.session && typeof session.session === "object" ? session.session : null; + const traceResults = snapshot?.traceResults && typeof snapshot.traceResults === "object" ? snapshot.traceResults : null; + const stored = traceResults?.[safeId] && typeof traceResults[safeId] === "object" ? traceResults[safeId] : null; + if (!stored) return null; + return { + ...stored, + traceId: safeId, + conversationId: safeConversationId(stored.conversationId ?? session.conversationId) || null, + sessionId: safeSessionId(stored.sessionId ?? session.id) || null, + threadId: safeOpaqueId(stored.threadId ?? session.threadId) || null, + ownerUserId: session.ownerUserId ?? stored.ownerUserId ?? null, + ownerRole: session.ownerRole ?? stored.ownerRole ?? null, + status: stored.status ?? session.status ?? null, + valuesRedacted: true + }; +} + +function codeAgentCompatProjectionPayload(payload = {}, context = {}) { + const traceId = safeTraceId(payload.traceId ?? context.trace?.traceId) ?? null; + const projection = context.projection && typeof context.projection === "object" ? context.projection : {}; + return { + ...payload, + projection, + projectionStatus: projection.projectionStatus ?? null, + projectionHealth: projection.projectionHealth ?? null, + lastProjectedSeq: projection.lastProjectedSeq ?? null, + sourceRunId: projection.sourceRunId ?? null, + sourceCommandId: projection.sourceCommandId ?? null, + staleMs: projection.staleMs ?? null, + blocker: projection.blocker ?? null, + workbench: traceId ? { + detailOnly: true, + turnDetailUrl: `/v1/workbench/turns/${encodeURIComponent(traceId)}`, + traceEventsDetailUrl: `/v1/workbench/traces/${encodeURIComponent(traceId)}/events` + } : null, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +function codeAgentResultPollTerminalReady(body = {}) { + if (body?.terminal !== true) return false; + return Boolean(messageAuthorityTextValue(body.finalResponse ?? body.reply ?? body.terminalEvidence?.finalResponse)); +} + +async function traceProjectMismatchSnapshot(traceId, url, options, resourceKind) { + const requestedProjectId = textValue(url.searchParams.get("projectId")); + if (!requestedProjectId || !options.accessController?.getAgentSessionByTraceId) return null; + const session = await getCodeAgentSessionByTraceId(traceId, options); + if (!session) return null; + if (session.ownerUserId && options.actor?.role !== "admin" && session.ownerUserId !== options.actor?.id) return forbiddenTurnSnapshot(traceId); + const actualProjectId = textValue(session.projectId); + if (!actualProjectId || actualProjectId === requestedProjectId) return null; + return { + statusCode: 404, + body: { + ok: false, + status: "not_found", + running: false, + terminal: false, + traceId, + error: { + code: "trace_project_mismatch", + message: `${resourceKind === "turn" ? "Agent turn" : "Agent trace"} is not visible in the requested project` + } + } + }; +} + +async function getCodeAgentSessionByTraceId(traceId, options = {}) { + const safeId = safeTraceId(traceId); + if (!safeId || typeof options.accessController?.getAgentSessionByTraceId !== "function") return null; + const cache = options.codeAgentTraceSessionCache; + if (cache instanceof Map) { + if (cache.has(safeId)) return cache.get(safeId); + const session = await options.accessController.getAgentSessionByTraceId(safeId); + cache.set(safeId, session ?? null); + return session ?? null; + } + return await options.accessController.getAgentSessionByTraceId(safeId); +} + +export function codeAgentTurnStatusPayload({ traceId, result, snapshot, resultPollError, refreshError, options }) { + const resultObject = result && typeof result === "object" ? result : null; + const snapshotObject = snapshot && typeof snapshot === "object" ? snapshot : null; + const lifecycle = codeAgentTurnLifecycleFields(traceId, resultObject ?? snapshotObject ?? {}); + const events = Array.isArray(snapshotObject?.events) ? snapshotObject.events : Array.isArray(resultObject?.runnerTrace?.events) ? resultObject.runnerTrace.events : []; + const lastEvent = events.at(-1) ?? null; + const finalResponse = resultObject?.finalResponse ?? snapshotObject?.finalResponse ?? snapshotObject?.terminalEvidence?.finalResponse ?? codeAgentFinalResponseEvidence(resultObject ?? snapshotObject ?? {}, traceId); + const terminalStatus = codeAgentAuthoritativeTerminalStatus(resultObject, snapshotObject, traceId); + const terminalSealBlocked = Boolean(terminalStatus && isTurnTerminalStatus(terminalStatus) && !codeAgentTerminalFinalText(finalResponse, resultObject, snapshotObject)); + const rawStatus = normalizeTurnStatus( + terminalStatus, + resultObject?.agentRun?.commandState, + resultObject?.agentRun?.status, + resultObject?.agentRun?.runStatus, + codeAgentRunningStatus(snapshotObject?.status), + codeAgentRunningStatus(snapshotObject?.traceStatus), + codeAgentRunningStatus(snapshotObject?.runnerTrace?.status), + codeAgentRunningStatus(resultObject?.status), + resultObject?.agentRun?.terminalStatus, + snapshotObject?.terminalEvidence?.traceSummary?.terminalStatus + ); + const status = terminalSealBlocked ? "running" : rawStatus; + const found = Boolean(resultObject || (snapshotObject && snapshotObject.status !== "missing") || snapshotObject?.persisted === true); + const running = isTurnRunningStatus(status); + const terminal = isTurnTerminalStatus(status); + const runnerTrace = snapshotObject && snapshotObject.status !== "missing" ? snapshotObject : resultObject?.runnerTrace ?? null; + const timingFields = codeAgentResultTimingFields(resultObject, snapshotObject, runnerTrace); + return { + ok: found, + action: "code-agent.turn.status", + contractVersion: "code-agent-turn-status-v1", + status: found ? status ?? "unknown" : "unknown", + running: found ? running : false, + terminal: found ? terminal : false, + traceId, + ...lifecycle, + conversationId: safeConversationId(resultObject?.conversationId ?? snapshotObject?.conversationId) || null, + sessionId: safeSessionId(resultObject?.sessionId ?? resultObject?.session?.sessionId ?? snapshotObject?.sessionId) || null, + threadId: safeOpaqueId(resultObject?.threadId ?? resultObject?.session?.threadId ?? snapshotObject?.threadId) || null, + updatedAt: textValue(resultObject?.updatedAt ?? resultObject?.agentRun?.updatedAt ?? snapshotObject?.updatedAt ?? timingFields.lastEventAt ?? timingFields.startedAt) || null, + ...timingFields, + lastEventLabel: textValue(snapshotObject?.lastEventLabel ?? runnerTrace?.lastEventLabel ?? lastEvent?.label ?? lastEvent?.type) || null, + waitingFor: terminalSealBlocked ? "final_response" : textValue(snapshotObject?.waitingFor ?? runnerTrace?.waitingFor) || null, + terminalObserved: Boolean(terminalStatus), + terminalObservedStatus: terminalStatus ?? null, + terminalSealBlocked, + resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`, + traceUrl: `/v1/agent/traces/${encodeURIComponent(traceId)}`, + turnUrl: `/v1/agent/turns/${encodeURIComponent(traceId)}`, + runnerTrace: runnerTrace ? compactRunnerTraceForResult(runnerTrace, resultTraceEventLimit(options)) : null, + agentRun: resultObject?.agentRun ?? snapshotObject?.agentRun ?? null, + terminalEvidence: snapshotObject?.terminalEvidence ?? null, + finalResponse, + traceSummary: resultObject?.traceSummary ?? snapshotObject?.traceSummary ?? snapshotObject?.terminalEvidence?.traceSummary ?? null, + retention: snapshotObject?.retention ?? null, + eventCount: numberOrNull(snapshotObject?.eventCount ?? runnerTrace?.eventCount ?? events.length), + error: resultPollError || refreshError + ? codeAgentRefreshErrorPayload(resultPollError ?? refreshError, traceId, resultObject?.agentRun ?? snapshotObject?.agentRun, "turn_status_degraded") + : resultObject?.error ?? snapshotObject?.error ?? null, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +function codeAgentTerminalFinalText(finalResponse = null, resultObject = null, snapshotObject = null) { + const candidates = [ + finalResponse, + resultObject?.finalResponse, + snapshotObject?.finalResponse, + snapshotObject?.terminalEvidence?.finalResponse, + resultObject?.terminalEvidence?.finalResponse + ]; + for (const value of candidates) { + const text = conversationText(value); + if (text) return text; + } + return codeAgentFinalResponseText(resultObject ?? {}) || codeAgentFinalResponseText(snapshotObject ?? {}); +} + +function codeAgentAuthoritativeTerminalStatus(resultObject, snapshotObject, traceId) { + const sealedPayload = codeAgentPayloadHasSealedFinalResponse(resultObject ?? {}) ? resultObject : codeAgentPayloadHasSealedFinalResponse(snapshotObject ?? {}) ? snapshotObject : null; + if (sealedPayload) { + const sealedStatus = normalizeTurnStatus(sealedPayload?.finalResponse?.status, sealedPayload?.terminalEvidence?.finalResponse?.status, sealedPayload?.agentRun?.terminalStatus, sealedPayload?.agentRun?.status, sealedPayload?.status); + return sealedStatus && isTurnTerminalStatus(sealedStatus) ? sealedStatus : "completed"; + } + const snapshotStatus = normalizeTurnStatus(snapshotObject?.terminalEvidence?.traceSummary?.terminalStatus, snapshotObject?.terminalEvidence?.agentRun?.terminalStatus, snapshotObject?.terminalEvidence?.status, snapshotObject?.status); + if (snapshotStatus && isTurnTerminalStatus(snapshotStatus) && codeAgentSnapshotHasTerminalAuthority(snapshotObject)) return snapshotStatus; + const agentRunStatus = normalizeTurnStatus(resultObject?.agentRun?.terminalStatus, resultObject?.agentRun?.commandState, resultObject?.agentRun?.status, resultObject?.agentRun?.runStatus); + if (agentRunStatus && isTurnTerminalStatus(agentRunStatus)) return agentRunStatus; + const resultStatus = normalizeTurnStatus(resultObject?.status); + if (resultStatus && isTurnTerminalStatus(resultStatus) && codeAgentResultHasTerminalAuthority(resultObject, traceId)) return resultStatus; + return null; +} + +function codeAgentSnapshotHasTerminalAuthority(snapshot = null) { + if (!snapshot || typeof snapshot !== "object") return false; + if (snapshot.terminal === true || snapshot.sealed === true) return true; + if (snapshot.terminalEvidence?.available === true || snapshot.terminalEvidence?.source) return true; + const events = Array.isArray(snapshot.events) ? snapshot.events : []; + return events.some((event) => event?.terminal === true); +} + +function codeAgentResultHasTerminalAuthority(result = null, traceId = null) { + if (!result || typeof result !== "object") return false; + if (result.terminal === true || result.sealed === true) return true; + if (result.error || result.blocker) return true; + if (codeAgentPayloadHasSealedFinalResponse(result)) return true; + if (agentRunTerminalTraceEvidence(result, traceId)) return true; + return Boolean(textValue(result.finishedAt ?? result.completedAt ?? result.endedAt)); +} + +function codeAgentRefreshErrorPayload(error, traceId, agentRun, fallbackCode) { + return { + code: error?.code ?? fallbackCode, + layer: error?.layer ?? "agentrun", + category: error?.category ?? (error?.code === "agentrun_timeout" ? "upstream-timeout" : "upstream-refresh-failed"), + retryable: error?.retryable !== false, + message: error?.message ?? "Code Agent turn status refresh degraded", + traceId, + runId: agentRun?.runId ?? error?.runId ?? null, + commandId: agentRun?.commandId ?? error?.commandId ?? null, + method: error?.method ?? null, + route: error?.route ?? error?.path ?? null, + timeoutMs: numberOrNull(error?.timeoutMs), + timeoutStage: error?.timeoutStage ?? null, + upstreamStatus: numberOrNull(error?.statusCode), + managerHost: error?.managerHost ?? null, + valuesPrinted: false + }; +} + +function isTurnRunningStatus(status) { + return status === "running"; +} + +function codeAgentRunningStatus(value) { + const status = normalizeTurnStatus(value); + return isTurnRunningStatus(status) ? status : null; +} + +function isTurnTerminalStatus(status) { + return CODE_AGENT_TERMINAL_STATUSES.has(String(status ?? "").trim().toLowerCase().replace(/_/gu, "-")); +} + +export async function handleCodeAgentInspectHttp(request, response, url, options) { + const query = { + conversationId: safeConversationId(url.searchParams.get("conversationId")), + sessionId: safeSessionId(url.searchParams.get("sessionId")), + threadId: safeOpaqueId(url.searchParams.get("threadId")), + traceId: safeTraceId(url.searchParams.get("traceId")) + }; + const sessionRegistry = options.sessionRegistry; + const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; + const manager = options.codexStdioManager; + const managerDescription = manager && typeof manager.describe === "function" ? manager.describe() : null; + const managerRecentSessions = Array.isArray(managerDescription?.recentSessions) ? managerDescription.recentSessions : []; + const directManagerSession = query.sessionId && manager && typeof manager.get === "function" + ? manager.get(query.sessionId, { conversationId: query.conversationId }) + : null; + const matchedManagerSession = directManagerSession ?? managerRecentSessions.find((session) => sessionMatchesInspectQuery(session, query)) ?? null; + const registryInspect = sessionRegistry && typeof sessionRegistry.inspect === "function" + ? sessionRegistry.inspect(query) + : { ok: false, status: "unavailable", traceIds: [], session: null, conversationFacts: null }; + const resultEvidence = await codeAgentInspectResultEvidence(query.traceId, options); + const session = matchedManagerSession ?? registryInspect.session ?? resultEvidence.session ?? null; + const conversationFacts = registryInspect.conversationFacts ?? resultEvidence.conversationFacts ?? null; + const requestedRunnerTrace = query.traceId ? traceStore.snapshot(query.traceId) : null; + const requestedTraceFound = Boolean(requestedRunnerTrace && requestedRunnerTrace.status !== "missing"); + const traceIds = uniqueStrings([ + requestedTraceFound ? query.traceId : null, + session?.currentTraceId, + session?.lastTraceId, + conversationFacts?.latestTraceId, + resultEvidence.latestTraceId, + ...(Array.isArray(conversationFacts?.traceIds) ? conversationFacts.traceIds : []), + ...(Array.isArray(registryInspect.traceIds) ? registryInspect.traceIds : []) + ]); + const latestTraceId = traceIds[0] ?? null; + const runnerTrace = latestTraceId + ? latestTraceId === query.traceId && requestedRunnerTrace + ? requestedRunnerTrace + : traceStore.snapshot(latestTraceId) + : null; + const found = Boolean(registryInspect.ok || session || latestTraceId); + sendJson(response, found ? 200 : 404, { + ok: found, + action: "code-agent.chat.inspect", + status: found ? "found" : "not_found", + query, + session, + conversationFacts, + traceIds, + latestTraceId, + traceUrl: latestTraceId ? `/v1/agent/traces/${encodeURIComponent(latestTraceId)}` : null, + resultUrl: latestTraceId ? `/v1/agent/chat/result/${encodeURIComponent(latestTraceId)}` : null, + runnerTrace, + valuesRedacted: true, + secretMaterialStored: false + }); +} + +async function codeAgentInspectResultEvidence(traceId, options = {}) { + if (!safeTraceId(traceId)) return { session: null, conversationFacts: null, latestTraceId: null }; + const cached = options.codeAgentChatResults?.get?.(traceId) ?? null; + const persisted = cached ? null : await loadPersistedAgentRunResult(traceId, options); + const result = cached ?? persisted ?? null; + if (!result || typeof result !== "object") return { session: null, conversationFacts: null, latestTraceId: null }; + const agentRun = result.agentRun && typeof result.agentRun === "object" ? result.agentRun : null; + const resultSession = result.session && typeof result.session === "object" ? result.session : null; + const sessionReuse = result.sessionReuse && typeof result.sessionReuse === "object" ? result.sessionReuse : null; + const providerTrace = result.providerTrace && typeof result.providerTrace === "object" ? result.providerTrace : null; + const sessionId = safeSessionId(result.sessionId ?? resultSession?.sessionId ?? sessionReuse?.sessionId ?? agentRun?.sessionId) || null; + const conversationId = safeConversationId(result.conversationId ?? resultSession?.conversationId ?? sessionReuse?.conversationId ?? agentRun?.conversationId) || null; + const threadId = safeOpaqueId(result.threadId ?? resultSession?.threadId ?? sessionReuse?.threadId ?? providerTrace?.threadId ?? agentRun?.threadId) || null; + const status = textValue(resultSession?.status ?? result.status ?? agentRun?.status) || null; + const updatedAt = textValue(result.updatedAt ?? agentRun?.updatedAt ?? resultSession?.updatedAt) || null; + const session = sessionId || conversationId || threadId ? { + sessionId, + conversationId, + threadId, + status, + lastTraceId: traceId, + source: "code-agent-result", + agentRun: agentRun ? agentRunSessionEvidence(result).agentRun : null, + valuesRedacted: true, + secretMaterialStored: false + } : null; + const conversationFacts = conversationId ? { + conversationId, + sessionId, + threadId, + latestTraceId: traceId, + traceIds: [traceId], + turnCount: 1, + source: "code-agent-result", + latestStatus: status, + updatedAt, + valuesRedacted: true, + secretMaterialStored: false + } : null; + return { session, conversationFacts, latestTraceId: traceId }; +} + +function sessionMatchesInspectQuery(session, query) { + if (!session || typeof session !== "object") return false; + if (query.sessionId && session.sessionId === query.sessionId) return true; + if (query.threadId && session.threadId === query.threadId) return true; + if (query.conversationId && session.conversationId === query.conversationId) return true; + if (query.traceId && (session.currentTraceId === query.traceId || session.lastTraceId === query.traceId)) return true; + return false; +} + +export async function handleCodeAgentCancelHttp(request, response, options) { + const body = await readBody(request, options.bodyLimitBytes); + let params = {}; + + try { + params = body ? JSON.parse(body) : {}; + } catch (error) { + sendJson(response, 400, { + ...createCodeAgentErrorPayload({ + code: "parse_error", + message: "Invalid JSON body", + reason: error.message, + traceId: getHeader(request, "x-trace-id") || "trc_unassigned", + layer: "api", + retryable: true + }) + }); + return; + } + + if (!params || typeof params !== "object" || Array.isArray(params)) { + sendJson(response, 400, { + ...createCodeAgentErrorPayload({ + code: "invalid_params", + message: "Code Agent cancel body must be a JSON object", + traceId: getHeader(request, "x-trace-id") || "trc_unassigned", + layer: "api", + retryable: true + }) + }); + return; + } + + const traceId = safeTraceId(getHeader(request, "x-trace-id") || params.traceId); + const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; + const snapshot = traceId ? traceStore.snapshot(traceId) : null; + const durableResult = traceId ? await loadPersistedAgentRunResult(traceId, options) : null; + const currentResult = durableResult ?? (traceId ? options.codeAgentChatResults?.get(traceId) : null); + if (currentResult?.agentRun?.commandId) { + const mismatch = codeAgentCancelScopeMismatch(params, currentResult); + if (mismatch) { + traceStore.append(traceId, { + type: "cancel", + status: "blocked", + label: "agentrun:cancel:scope_mismatch", + errorCode: "cancel_scope_mismatch", + message: `Cancel request ${mismatch.field} does not match the trace owner; AgentRun cancel was not forwarded.`, + requested: mismatch.requested, + expected: mismatch.expected, + waitingFor: "cancel-scope", + valuesPrinted: false + }); + sendJson(response, 409, cancelBlockedPayload({ + code: "cancel_scope_mismatch", + message: `取消请求的 ${mismatch.field} 与 trace 归属不一致，已拒绝转发 AgentRun cancel，避免误取消其他 session。`, + traceId, + conversationId: safeConversationId(params.conversationId) || safeConversationId(currentResult.conversationId) || null, + sessionId: safeSessionId(params.sessionId) || safeSessionId(currentResult.sessionId) || null, + runnerTrace: traceStore.snapshot(traceId), + status: "blocked" + })); + return; + } + await recordCodeAgentSessionInputFact({ + params: { + ...params, + sessionId: currentResult.sessionId ?? currentResult.session?.sessionId ?? params.sessionId, + messageId: codeAgentTurnLifecycleFields(traceId, currentResult).userMessageId, + ownerUserId: options.actor?.id, + ownerRole: options.actor?.role + }, + options, + traceId, + delivery: "cancel", + status: "promoted", + commandId: currentResult.agentRun.commandId + }); + const payload = await cancelAgentRunChatTurn({ traceId, currentResult, options, traceStore }); + if (payload) { + sendJson(response, 200, payload); + return; + } + } + const sessionId = safeSessionId(params.sessionId) || safeSessionId(snapshot?.sessionId); + const conversationId = safeConversationId(params.conversationId); + const manager = options.codexStdioManager; + + if (!traceId) { + sendJson(response, 400, cancelBlockedPayload({ + code: "cancel_trace_missing", + message: "traceId is required to cancel the current Code Agent request.", + traceId: "trc_unassigned", + conversationId, + sessionId + })); + return; + } + + if (!manager || typeof manager.get !== "function" || typeof manager.cancel !== "function") { + traceStore.append(traceId, { + type: "cancel", + status: "unsupported", + label: "cancel:unsupported", + errorCode: "cancel_unsupported", + message: "Codex stdio cancel/interrupt backend is not available on this runtime.", + waitingFor: "session-control" + }); + const runnerTrace = traceStore.snapshot(traceId); + sendJson(response, 501, cancelBlockedPayload({ + code: "cancel_unsupported", + message: "当前后端不支持 Code Agent interrupt/cancel；本次按 unsupported/degraded 返回，不会静默假装已中断。", + traceId, + conversationId, + sessionId, + runnerTrace, + status: "degraded", + unsupported: true, + degraded: true + })); + return; + } + + if (!sessionId) { + traceStore.append(traceId, { + type: "cancel", + status: "blocked", + label: "cancel:not_cancelable", + errorCode: "cancel_session_missing", + message: "Cancel request did not include a bound Codex stdio sessionId.", + waitingFor: "session-binding" + }); + sendJson(response, 409, cancelBlockedPayload({ + code: "cancel_session_missing", + message: "当前请求尚未暴露可取消的 Codex stdio sessionId；页面不能只隐藏 UI，已保留输入和 trace。", + traceId, + conversationId, + sessionId: null, + runnerTrace: traceStore.snapshot(traceId) + })); + return; + } + + const currentSession = manager.get(sessionId, { conversationId }) ?? null; + if (!currentSession || !["busy", "creating"].includes(currentSession.status)) { + traceStore.append(traceId, { + type: "cancel", + status: "blocked", + label: "cancel:not_in_flight", + errorCode: "cancel_not_in_flight", + message: `Session ${sessionId} is not an in-flight Codex stdio request.`, + sessionId, + sessionStatus: currentSession?.status ?? "missing" + }); + sendJson(response, 409, cancelBlockedPayload({ + code: currentSession ? "cancel_not_in_flight" : "cancel_session_not_found", + message: currentSession + ? `当前 session 状态为 ${currentSession.status}，没有可取消的 in-flight Codex stdio 请求。` + : `没有找到 sessionId=${sessionId} 的 Codex stdio session。`, + traceId, + conversationId, + sessionId, + session: currentSession, + runnerTrace: traceStore.snapshot(traceId) + })); + return; + } + + const canceledSession = manager.cancel(sessionId, { + traceId, + conversationId, + reason: "user_cancel" + }); + traceStore.append(traceId, { + type: "cancel", + status: "canceled", + label: "cancel:canceled", + message: "User canceled the current Codex stdio request.", + sessionId, + sessionStatus: canceledSession?.status ?? "canceled", + sessionLifecycleStatus: codeAgentSessionLifecycleSummary({ session: canceledSession, status: "canceled" }).status, + waitingFor: "user-retry", + terminal: true + }); + const runnerTraceSnapshot = traceStore.snapshot(traceId); + const sessionSummary = codeAgentSessionLifecycleSummary({ + session: canceledSession, + runnerTrace: runnerTraceSnapshot, + status: "canceled" + }); + const runnerTrace = { + ...runnerTraceSnapshot, + sessionLifecycleStatus: sessionSummary.status + }; + const payload = { + accepted: true, + canceled: true, + status: "canceled", + conversationId: conversationId ?? canceledSession?.conversationId ?? null, + sessionId, + traceId, + session: canceledSession, + sessionLifecycleStatus: sessionSummary.status, + sessionLifecycle: sessionSummary, + sessionSummary, + runnerTrace, + lastTraceEvent: runnerTrace.lastEvent, + retryable: true, + error: { + code: "codex_stdio_canceled", + layer: "session", + category: "canceled", + retryable: true, + message: "user canceled current Code Agent request", + userMessage: "当前 Codex stdio 请求已取消；输入、sessionId、traceId 和最后 trace event 已保留，可重试上一条消息。", + traceId, + route: "/v1/agent/chat/cancel", + toolName: "codex-stdio.cancel" + }, + userMessage: "当前 Codex stdio 请求已取消；输入、sessionId、traceId 和最后 trace event 已保留，可重试上一条消息。", + updatedAt: new Date().toISOString() + }; + await recordCodeAgentSessionOwner({ payload, params: { ...params, traceId, ownerUserId: options.actor?.id, ownerRole: options.actor?.role }, options, status: "canceled" }); + recordCodeAgentConversationFact(payload, options); + options.codeAgentChatResults?.set(traceId, annotateOwner(payload, { ownerUserId: options.actor?.id, ownerRole: options.actor?.role })); + sendJson(response, 200, payload); +} + +export { codeAgentCompatProjectionPayload, codeAgentRequestScopedOptions, measureCodeAgentHttpPhase, readCodeAgentCompatProjection, traceProjectMismatchSnapshot }; diff --git a/internal/cloud/server-health.test.ts b/internal/cloud/server-health.test.ts index 016567e9..4c74c5a5 100644 --- a/internal/cloud/server-health.test.ts +++ b/internal/cloud/server-health.test.ts @@ -348,7 +348,7 @@ test("cloud api health separates DB connected from durable runtime schema readin try { const { port } = server.address(); - const response = await fetch(`http://127.0.0.1:${port}/health/live`); + const response = await fetch(`http://127.0.0.1:${port}/health`); const payload = await response.json(); assert.equal(response.status, 200); assert.equal(payload.status, "degraded"); @@ -449,7 +449,7 @@ test("cloud api health keeps DB live evidence separate when durable adapter quer try { const { port } = server.address(); - const response = await fetch(`http://127.0.0.1:${port}/health/live`); + const response = await fetch(`http://127.0.0.1:${port}/health`); const payload = await response.json(); assert.equal(response.status, 200); assert.equal(payload.status, "degraded"); @@ -593,7 +593,7 @@ test("cloud api health contract distinguishes durable SSL, auth, schema, migrati try { const { port } = server.address(); - const response = await fetch(`http://127.0.0.1:${port}/health/live`); + const response = await fetch(`http://127.0.0.1:${port}/health`); const payload = await response.json(); assert.equal(response.status, 200); assert.equal(payload.status, "degraded"); @@ -653,7 +653,7 @@ test("cloud api health does not treat DB env presence-only as live readiness", a try { const { port } = server.address(); - const response = await fetch(`http://127.0.0.1:${port}/health/live`); + const response = await fetch(`http://127.0.0.1:${port}/health`); const payload = await response.json(); assert.equal(response.status, 200); assert.equal(payload.db.configReady, true); @@ -782,7 +782,7 @@ test("cloud api health reports provider, durable DB, and codex stdio ready when try { const { port } = server.address(); - const response = await fetch(`http://127.0.0.1:${port}/health/live`); + const response = await fetch(`http://127.0.0.1:${port}/health`); assert.equal(response.status, 200); const payload = await response.json(); assert.equal(payload.status, "ok"); @@ -868,7 +868,7 @@ test("cloud api health reports AgentRun adapter readiness without repo-owned cod try { const { port } = server.address(); - const response = await fetch(`http://127.0.0.1:${port}/health/live`); + const response = await fetch(`http://127.0.0.1:${port}/health`); assert.equal(response.status, 200); const payload = await response.json(); assert.equal(payload.status, "ok"); diff --git a/internal/cloud/server-hwpod-http.ts b/internal/cloud/server-hwpod-http.ts new file mode 100644 index 00000000..8d4d7617 --- /dev/null +++ b/internal/cloud/server-hwpod-http.ts @@ -0,0 +1,528 @@ +/* + * Responsibility: Cloud API HTTP handlers for HWLAB node updates, HWPOD discovery, and node-ops dispatch. + */ +import { createHash, randomUUID } from "node:crypto"; +import { readFileSync } from "node:fs"; +import path from "node:path"; + +import { ENVIRONMENT_DEV } from "../protocol/index.mjs"; +import { CLOUD_API_SERVICE_ID } from "../audit/index.mjs"; +import { + discoverHwpodSpecs, + hwpodSpecDiscoveryPayload, + hwpodSpecWorkspaceProbePlan +} from "./hwpod-spec-discovery.ts"; +import { + getHeader, + parsePositiveInteger, + readBody, + safeOpaqueId, + sendJson, + truthyFlag +} from "./server-http-utils.ts"; +import { HWPOD_NODE_OPS, HWPOD_NODE_OPS_CONTRACT_VERSION } from "../../tools/src/hwpod-node-ops-contract.ts"; + +const DEFAULT_BODY_LIMIT_BYTES = 1024 * 1024; + +export async function handleHwpodNodeOpsHttp(request, response, options) { + if (request.method === "GET") { + sendJson(response, 200, { + ok: true, + status: "ready", + contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, + route: "/v1/hwpod-node-ops", + specAuthority: "code-agent-workspace", + compiler: "hwpod-compiler-cli", + apiRole: "node-ops-forwarder", + nodeRole: "thin-hwpod-node-executor", + supportedOps: Array.from(HWPOD_NODE_OPS), + websocket: options.hwpodNodeWsRegistry.describe() + }); + return; + } + if (request.method !== "POST") { + sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "hwpod-node-ops only supports GET and POST" } }); + return; + } + + const body = await readJsonObject(request, options.bodyLimitBytes); + if (!body.ok) { + sendJson(response, 400, body.error); + return; + } + const validation = validateHwpodNodeOpsPlan(body.value); + if (!validation.ok) { + sendJson(response, 400, { + ok: false, + status: "rejected", + contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, + error: validation.error + }); + return; + } + + const plan = validation.plan; + const requestMeta = hwpodNodeOpsRequestMeta(request, options, "hwpod"); + try { + const handled = await dispatchHwpodNodeOpsPlan(plan, requestMeta, options, { request }); + sendJson(response, handled.httpStatus, handled.payload); + } catch (error) { + const summary = error?.message ?? "hwpod-node-ops handler failed"; + recordHwpodNodeOpsBlocked(request, options, plan, requestMeta, "hwpod_node_handler_failed", summary, { dispatchMode: "handler-exception" }); + sendJson(response, 200, hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, { dispatchMode: "handler-exception" })); + } +} + +export function handleHwlabNodeUpdateHttp(request, response, url, options) { + if (request.method !== "GET") { + sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "GET required." } }); + return; + } + const env = options.env ?? process.env; + const currentVersion = cleanText(url.searchParams.get("current") || url.searchParams.get("version")); + const channel = cleanText(url.searchParams.get("channel")) || "stable"; + const platform = cleanText(url.searchParams.get("platform")) || "unknown"; + const bundled = hwlabNodeBundledMetadata(env); + const latestVersion = cleanText(env.HWLAB_NODE_PY_LATEST_VERSION) || bundled.version || "0.1.0"; + const releaseNotesUrl = cleanText(env.HWLAB_NODE_PY_RELEASE_NOTES_URL) || null; + const downloadUrl = hwlabNodeDownloadUrl(env); + const sha256 = cleanText(env.HWLAB_NODE_PY_SHA256) || bundled.sha256 || null; + const newer = currentVersion ? compareVersionStrings(latestVersion, currentVersion) > 0 : Boolean(latestVersion); + const updateAvailable = Boolean(downloadUrl && newer); + sendJson(response, 200, { + ok: true, + contractVersion: "hwlab-node-update-v1", + serviceId: "hwlab-node", + route: "/v1/hwlab-node/update", + channel, + platform, + currentVersion: currentVersion || null, + latestVersion, + updateAvailable, + downloadUrl: updateAvailable ? downloadUrl : null, + sha256: updateAvailable ? sha256 : null, + releaseNotesUrl, + manualDefault: true, + autoApplyDefault: false, + checkIntervalSeconds: parsePositiveInteger(env.HWLAB_NODE_PY_UPDATE_INTERVAL_SECONDS, 300), + observedAt: new Date().toISOString() + }); +} + +export function handleHwlabNodeDownloadHttp(request, response, options) { + if (request.method !== "GET") { + sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "GET required." } }); + return; + } + const bundled = hwlabNodeBundledMetadata(options.env ?? process.env); + if (!bundled.content) { + sendJson(response, 404, { ok: false, error: { code: "hwlab_node_bundle_missing", message: "bundled hwlab-node.py is not available" } }); + return; + } + const bytes = Buffer.from(bundled.content, "utf8"); + response.writeHead(200, { + "content-type": "text/x-python; charset=utf-8", + "cache-control": "no-store", + "x-hwlab-node-version": bundled.version || "unknown", + "x-hwlab-node-sha256": bundled.sha256 || "", + "content-length": String(bytes.length) + }); + response.end(bytes); +} + +export async function handleHwpodSpecDiscoveryHttp(request, response, url, options) { + const probe = truthyFlag(url.searchParams.get("probe")); + const observedAt = new Date().toISOString(); + let specs = await discoverHwpodSpecs(options); + if (probe) { + specs = await Promise.all(specs.map((spec) => probeDiscoveredHwpodSpec(spec, { request, options, observedAt }))); + } + sendJson(response, 200, hwpodSpecDiscoveryPayload(specs, { observedAt })); +} + +function runtimeEnvironment(env = process.env) { + const value = env.HWLAB_GITOPS_PROFILE || env.HWLAB_ENVIRONMENT || ENVIRONMENT_DEV; + return typeof value === "string" && value.trim() ? value.trim() : ENVIRONMENT_DEV; +} + +function hwpodNodeOpsRequestMeta(request, options, label) { + const httpContext = request?.hwlabHttpRequestContext ?? {}; + return { + requestId: getHeader(request, "x-request-id") || httpContext.requestId || `req_${label}_${randomUUID()}`, + traceId: getHeader(request, "x-trace-id") || `trc_${label}_${randomUUID()}`, + serviceId: getHeader(request, "x-source-service-id") || CLOUD_API_SERVICE_ID, + environment: runtimeEnvironment(options.env ?? process.env), + otelTraceId: httpContext.traceId ?? null, + traceparent: httpContext.traceparent ?? null, + valuesPrinted: false + }; +} + +function hwlabNodeBundledMetadata(env) { + const bundlePath = cleanText(env.HWLAB_NODE_PY_BUNDLE_PATH) || path.resolve(process.cwd(), "tools/hwlab-node.py"); + try { + const content = readFileSync(bundlePath, "utf8"); + const version = cleanText(content.match(/APP_VERSION\s*=\s*["']([^"']+)["']/u)?.[1]); + const sha256 = createHash("sha256").update(content, "utf8").digest("hex"); + return { path: bundlePath, content, version, sha256 }; + } catch { + return { path: bundlePath, content: "", version: "", sha256: "" }; + } +} + +function hwlabNodeDownloadUrl(env) { + const explicit = cleanText(env.HWLAB_NODE_PY_DOWNLOAD_URL); + if (explicit) return explicit; + const downloadPath = cleanText(env.HWLAB_NODE_PY_DOWNLOAD_PATH) || "/v1/hwlab-node/download/hwlab-node.py"; + if (/^https?:\/\//iu.test(downloadPath)) return downloadPath; + const publicEndpoint = cleanText(env.HWLAB_PUBLIC_ENDPOINT) || "https://hwlab.pikapython.com"; + try { + return new URL(downloadPath, publicEndpoint.endsWith("/") ? publicEndpoint : `${publicEndpoint}/`).toString(); + } catch { + return null; + } +} + +function compareVersionStrings(left, right) { + const a = versionParts(left); + const b = versionParts(right); + for (let index = 0; index < Math.max(a.length, b.length, 3); index += 1) { + const delta = (a[index] ?? 0) - (b[index] ?? 0); + if (delta !== 0) return delta > 0 ? 1 : -1; + } + return 0; +} + +function versionParts(value) { + return String(value ?? "") + .trim() + .replace(/^v/iu, "") + .split(/[.+-]/u) + .slice(0, 4) + .map((part) => parseInt(part, 10)) + .map((part) => (Number.isFinite(part) && part >= 0 ? part : 0)); +} + +function cleanText(value) { + const text = String(value ?? "").trim(); + return text || ""; +} + +async function probeDiscoveredHwpodSpec(spec, { request, options, observedAt }) { + if (spec.ok === false) return spec; + const plan = hwpodSpecWorkspaceProbePlan(spec); + const validation = validateHwpodNodeOpsPlan(plan); + if (!validation.ok) { + return { + ...spec, + availability: { + ok: false, + status: "invalid_probe_plan", + checkedAt: observedAt, + blocker: validation.error + } + }; + } + const requestMeta = hwpodNodeOpsRequestMeta(request, options, "hwpod_spec"); + const handled = await dispatchHwpodNodeOpsPlan(validation.plan, requestMeta, options, { request }); + const payload = handled.payload; + return { + ...spec, + availability: { + ok: payload.ok === true, + status: payload.ok === true ? "available" : "blocked", + checkedAt: observedAt, + probePlanId: payload.planId, + nodeOpsRoute: "/v1/hwpod-node-ops", + results: payload.results ?? [], + blocker: payload.blocker ?? null, + httpStatus: handled.httpStatus + } + }; +} + +async function dispatchHwpodNodeOpsPlan(plan, requestMeta, options, context = {}) { + const hwpodNodeOpsUrl = normalizedHwpodNodeOpsUrl(options.env ?? process.env); + const hwpodNodeWsRegistry = options.hwpodNodeWsRegistry; + const hasWsNode = hwpodNodeWsRegistry?.hasNode?.(plan.nodeId); + if (typeof options.hwpodNodeOpsHandler !== "function" && !hasWsNode && !hwpodNodeOpsUrl) { + const summary = "no outbound WebSocket hwpod-node is connected and HWLAB_HWPOD_NODE_OPS_URL is not configured; cloud-api is only validating the hwpod-node-ops contract"; + const details = { + dispatchMode: "none", + websocketRegistryConfigured: Boolean(hwpodNodeWsRegistry), + websocketConnected: false, + directUrlConfigured: false + }; + recordHwpodNodeOpsBlocked(context.request, options, plan, requestMeta, "hwpod_node_unavailable", summary, details); + return { + httpStatus: 200, + payload: hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, details) + }; + } + const handled = typeof options.hwpodNodeOpsHandler === "function" + ? await options.hwpodNodeOpsHandler(plan, { request: context.request, requestMeta, env: options.env ?? process.env }) + : hasWsNode + ? await hwpodNodeWsRegistry.dispatch(plan, requestMeta, { timeoutMs: parsePositiveInteger(options.env?.HWLAB_HWPOD_NODE_OPS_TIMEOUT_MS, 30000) }) + : await forwardHwpodNodeOpsPlan(hwpodNodeOpsUrl, plan, requestMeta, options); + const results = Array.isArray(handled?.results) ? handled.results : []; + const failed = results.some((item) => item?.ok === false); + const payload = attachHwpodNodeOpsDiagnostics({ + ok: handled?.ok ?? !failed, + status: handled?.status ?? (failed ? "failed" : "completed"), + contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, + planId: plan.planId, + hwpodId: plan.hwpodId, + nodeId: plan.nodeId, + acceptedOps: plan.ops.length, + results, + blocker: handled?.blocker ?? null, + requestMeta + }, requestMeta, { dispatchMode: hasWsNode ? "websocket" : hwpodNodeOpsUrl ? "direct-url" : "handler" }); + if (payload.ok === false && payload.status === "blocked" && payload.blocker?.code) { + recordHwpodNodeOpsBlocked(context.request, options, plan, requestMeta, payload.blocker.code, payload.blocker.summary ?? "hwpod-node-ops blocked", payload.blocker.details ?? {}); + } + return { httpStatus: handled?.httpStatus ?? (failed ? 409 : 200), payload }; +} + +async function forwardHwpodNodeOpsPlan(targetUrl, plan, requestMeta, options) { + const target = await describeDirectHwpodNode(targetUrl, options); + if (!target.ok || !target.nodeId) { + return directHwpodNodeBlocked(plan, "hwpod_node_identity_unverified", "configured hwpod-node URL did not expose a verifiable nodeId", { + requestedNodeId: plan.nodeId, + targetUrl: redactNodeOpsUrl(targetUrl), + dispatchMode: "direct-url", + targetStatus: target.status ?? null, + error: target.error ?? null + }, requestMeta); + } + if (target.ok && target.nodeId && target.nodeId !== plan.nodeId) { + return directHwpodNodeBlocked(plan, "hwpod_node_id_mismatch", `HWLAB_HWPOD_NODE_OPS_URL points to ${target.nodeId}, not requested node ${plan.nodeId}`, { + requestedNodeId: plan.nodeId, + targetNodeId: target.nodeId, + targetUrl: redactNodeOpsUrl(targetUrl), + dispatchMode: "direct-url" + }, requestMeta); + } + const response = await fetch(targetUrl, { + method: "POST", + headers: { + "content-type": "application/json", + "x-request-id": requestMeta.requestId, + "x-trace-id": requestMeta.traceId, + "x-source-service-id": CLOUD_API_SERVICE_ID, + ...(requestMeta.otelTraceId ? { "x-hwlab-otel-trace-id": requestMeta.otelTraceId } : {}), + ...(requestMeta.traceparent ? { traceparent: requestMeta.traceparent } : {}) + }, + body: JSON.stringify(plan), + signal: AbortSignal.timeout(parsePositiveInteger(options.env?.HWLAB_HWPOD_NODE_OPS_TIMEOUT_MS, 30000)) + }); + const body = await response.json().catch(() => null); + if (!body || typeof body !== "object") { + return directHwpodNodeBlocked(plan, "hwpod_node_response_invalid", `hwpod-node returned HTTP ${response.status} without a JSON object payload`, { + targetUrl: redactNodeOpsUrl(targetUrl), + dispatchMode: "direct-url", + targetStatus: response.status + }, requestMeta); + } + return { + ok: body.ok, + status: body.status, + httpStatus: response.status, + results: body.results, + blocker: body.blocker ?? null + }; +} + +function directHwpodNodeBlocked(plan, code, summary, details, requestMeta = {}) { + const blocker = hwpodNodeOpsBlocker({ code, layer: "hwpod-node", retryable: true, summary, details }, requestMeta, details); + return { + ok: false, + status: "blocked", + httpStatus: 200, + results: plan.ops.map((op) => ({ opId: op.opId, op: op.op, ok: false, status: "blocked", blocker })), + blocker, + otelTraceId: blocker.otelTraceId ?? null, + diagnostic: blocker.diagnostic ?? null + }; +} + +async function describeDirectHwpodNode(targetUrl, options) { + try { + const response = await fetch(targetUrl, { + method: "GET", + signal: AbortSignal.timeout(parsePositiveInteger(options.env?.HWLAB_HWPOD_NODE_OPS_TIMEOUT_MS, 30000)) + }); + const body = await response.json().catch(() => null); + return { ok: response.ok && body && typeof body === "object", status: response.status, nodeId: safeOpaqueId(body?.nodeId), body }; + } catch (error) { + return { ok: false, error: error instanceof Error ? error.message : String(error), nodeId: "" }; + } +} + +function redactNodeOpsUrl(value) { + try { + const parsed = new URL(value); + parsed.username = ""; + parsed.password = ""; + parsed.search = parsed.search ? "?redacted=1" : ""; + return parsed.toString(); + } catch { + return ""; + } +} + +function normalizedHwpodNodeOpsUrl(env = process.env) { + const direct = String(env.HWLAB_HWPOD_NODE_OPS_URL ?? "").trim(); + if (direct) return direct; + const base = String(env.HWLAB_HWPOD_NODE_URL ?? "").trim().replace(/\/+$/u, ""); + return base ? `${base}/v1/hwpod-node-ops` : ""; +} + +function validateHwpodNodeOpsPlan(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return { ok: false, error: { code: "invalid_hwpod_node_ops_plan", message: "hwpod-node-ops body must be a JSON object" } }; + } + if (value.contractVersion !== HWPOD_NODE_OPS_CONTRACT_VERSION) { + return { ok: false, error: { code: "invalid_hwpod_node_ops_contract", message: `contractVersion must be ${HWPOD_NODE_OPS_CONTRACT_VERSION}`, actual: value.contractVersion ?? null } }; + } + const planId = safeOpaqueId(value.planId) || `hwpod_plan_${randomUUID()}`; + const nodeId = safeOpaqueId(value.nodeId); + const hwpodId = safeOpaqueId(value.hwpodId); + if (!nodeId) return { ok: false, error: { code: "invalid_hwpod_node_id", message: "nodeId is required" } }; + if (!hwpodId) return { ok: false, error: { code: "invalid_hwpod_id", message: "hwpodId is required" } }; + if (!Array.isArray(value.ops) || value.ops.length === 0) { + return { ok: false, error: { code: "invalid_hwpod_node_ops", message: "ops must be a non-empty array" } }; + } + const ops = []; + for (const [index, item] of value.ops.entries()) { + if (!item || typeof item !== "object" || Array.isArray(item)) { + return { ok: false, error: { code: "invalid_hwpod_node_op", message: `ops[${index}] must be a JSON object` } }; + } + const op = typeof item.op === "string" ? item.op.trim() : ""; + if (!HWPOD_NODE_OPS.has(op)) { + return { ok: false, error: { code: "unsupported_hwpod_node_op", message: `unsupported hwpod-node op: ${op || ""}`, supportedOps: Array.from(HWPOD_NODE_OPS) } }; + } + ops.push({ + opId: safeOpaqueId(item.opId) || `op_${index + 1}`, + op, + args: item.args && typeof item.args === "object" && !Array.isArray(item.args) ? item.args : {} + }); + } + return { + ok: true, + plan: { + ...value, + planId, + hwpodId, + nodeId, + ops + } + }; +} + +function hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, details = {}) { + const blocker = hwpodNodeOpsBlocker({ + code: "hwpod_node_unavailable", + layer: "hwpod-node", + retryable: true, + summary, + details, + userMessage: "hwpod-node 尚未接入执行面；cloud-api 已完成 hwpod-node-ops 合同校验。" + }, requestMeta, details); + return { + ok: false, + status: "blocked", + contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, + planId: plan.planId, + hwpodId: plan.hwpodId, + nodeId: plan.nodeId, + acceptedOps: plan.ops.length, + results: plan.ops.map((op) => ({ + opId: op.opId, + op: op.op, + ok: false, + status: "blocked", + blocker + })), + blocker, + otelTraceId: blocker.otelTraceId ?? null, + diagnostic: blocker.diagnostic ?? null, + requestMeta + }; +} + +function hwpodNodeOpsBlocker(blocker, requestMeta = {}, details = {}) { + const code = cleanText(blocker?.code) || "hwpod_node_ops_blocked"; + const summary = cleanText(blocker?.summary ?? blocker?.message) || "hwpod-node-ops blocked"; + const otelTraceId = cleanText(blocker?.otelTraceId ?? requestMeta?.otelTraceId) || null; + const diagnostic = { + code, + rootCauseCode: code, + summary, + otelTraceId, + traceLine: otelTraceId ? `OTel traceId: ${otelTraceId}` : null, + details: details && typeof details === "object" ? details : {}, + valuesPrinted: false + }; + const userMessage = cleanText(blocker?.userMessage); + return { + ...blocker, + code, + layer: blocker?.layer ?? "hwpod-node", + retryable: blocker?.retryable ?? true, + summary, + ...(otelTraceId ? { otelTraceId } : {}), + diagnostic, + ...(userMessage ? { userMessage: `${userMessage}\n${diagnostic.traceLine ?? "OTel traceId: unavailable"}` } : {}) + }; +} + +function attachHwpodNodeOpsDiagnostics(payload, requestMeta, details = {}) { + if (!payload || typeof payload !== "object" || payload.ok !== false) return payload; + const topBlocker = payload.blocker ? hwpodNodeOpsBlocker(payload.blocker, requestMeta, payload.blocker.details ?? details) : null; + const results = Array.isArray(payload.results) + ? payload.results.map((result) => result?.blocker ? { ...result, blocker: hwpodNodeOpsBlocker(result.blocker, requestMeta, result.blocker.details ?? details) } : result) + : payload.results; + const diagnostic = topBlocker?.diagnostic ?? results?.find?.((result) => result?.blocker?.diagnostic)?.blocker?.diagnostic ?? null; + return { + ...payload, + results, + blocker: topBlocker, + ...(diagnostic?.otelTraceId ? { otelTraceId: diagnostic.otelTraceId } : {}), + ...(diagnostic ? { diagnostic } : {}) + }; +} + +function recordHwpodNodeOpsBlocked(request, options, plan, requestMeta, code, summary, details = {}) { + const httpContext = request?.hwlabHttpRequestContext; + const attributes = { + "hwlab.hwpod.plan_id": cleanText(plan?.planId) || null, + "hwlab.hwpod.hwpod_id": cleanText(plan?.hwpodId) || null, + "hwlab.hwpod.node_id": cleanText(plan?.nodeId) || null, + "hwlab.hwpod.accepted_ops": Array.isArray(plan?.ops) ? plan.ops.length : 0, + "hwlab.hwpod.ops": Array.isArray(plan?.ops) ? plan.ops.map((op) => cleanText(op?.op)).filter(Boolean).join(",").slice(0, 240) : null, + "hwlab.hwpod.blocker.code": code, + "hwlab.hwpod.blocker.summary": String(summary ?? "").slice(0, 500), + "hwlab.hwpod.dispatch_mode": cleanText(details?.dispatchMode) || "unknown", + "hwlab.hwpod.websocket_connected": Boolean(details?.websocketConnected), + "hwlab.hwpod.direct_url_configured": Boolean(details?.directUrlConfigured), + "hwlab.hwpod.otel_trace_id": cleanText(requestMeta?.otelTraceId) || null, + traceId: cleanText(requestMeta?.traceId) || null + }; + if (httpContext) httpContext.otelAttributes = { ...(httpContext.otelAttributes ?? {}), ...attributes }; + const error = Object.assign(new Error(summary), { code }); + options.emitHttpRoutePhaseSpan?.(request, options, "hwpod-node-ops.blocked", Date.now(), Date.now(), "error", error, attributes); +} + +async function readJsonObject(request, limitBytes) { + const body = await readBody(request, limitBytes ?? DEFAULT_BODY_LIMIT_BYTES); + try { + const value = body ? JSON.parse(body) : {}; + if (!value || typeof value !== "object" || Array.isArray(value)) { + return { ok: false, error: { accepted: false, error: { code: "invalid_params", message: "body must be a JSON object" } } }; + } + return { ok: true, value }; + } catch (error) { + return { ok: false, error: { accepted: false, error: { code: "parse_error", message: "Invalid JSON body", reason: error.message } } }; + } +} diff --git a/internal/cloud/server-workbench-facts.ts b/internal/cloud/server-workbench-facts.ts index 97ce0952..6b5805e4 100644 --- a/internal/cloud/server-workbench-facts.ts +++ b/internal/cloud/server-workbench-facts.ts @@ -17,7 +17,6 @@ import { createWorkbenchReadModel } from "./workbench-read-model.ts"; import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts"; import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts"; import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts"; -import { openKafkaEventStream } from "./kafka-event-bridge.ts"; import { durableTraceStatus, RUNNING_STATUSES, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts"; import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts"; diff --git a/internal/cloud/server-workbench-read-http.ts b/internal/cloud/server-workbench-read-http.ts index 5a4a1f4c..01012dd7 100644 --- a/internal/cloud/server-workbench-read-http.ts +++ b/internal/cloud/server-workbench-read-http.ts @@ -17,7 +17,6 @@ import { createWorkbenchReadModel } from "./workbench-read-model.ts"; import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts"; import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts"; import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts"; -import { openKafkaEventStream } from "./kafka-event-bridge.ts"; import { durableTraceStatus, RUNNING_STATUSES, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts"; import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts"; import * as workbenchFacts from "./server-workbench-facts.ts"; diff --git a/internal/cloud/server-workbench-realtime-http.test.ts b/internal/cloud/server-workbench-realtime-http.test.ts index ad278165..c22d0040 100644 --- a/internal/cloud/server-workbench-realtime-http.test.ts +++ b/internal/cloud/server-workbench-realtime-http.test.ts @@ -2,12 +2,14 @@ // Responsibility: Workbench realtime and projection blocker regression tests. import assert from "node:assert/strict"; +import { get as httpGet } from "node:http"; import { test } from "bun:test"; import { createCloudApiServer } from "./server.ts"; import { createBackendPerformanceStore } from "./backend-performance.ts"; import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; import { classifyWorkbenchReadModelFailure } from "./server-workbench-http.ts"; +import { projectionOutboxRealtimeEvents, workbenchRealtimeAfterSeq } from "./server-workbench-realtime-http.ts"; import { codeAgentTurnStatusPayload, createCodeAgentChatResultStore } from "./server-code-agent-http.ts"; import { createWorkbenchTurnProjection, durableTraceStatus, projectionDiagnostics, traceTerminalEvidence } from "./workbench-turn-projection.ts"; @@ -32,11 +34,177 @@ import { emptyFacts, normalizeTestStatus, getSseEvents, - parseSseBlock, - createFakeKafkaFactory, - waitFor + parseSseBlock } from "./server-workbench-http-test-helpers.ts"; +test("projection outbox emits immutable assistant versions in row order", () => { + const traceId = "trc_outbox_versions"; + const sessionId = "ses_outbox_versions"; + const messageId = "msg_outbox_versions_agent"; + const events = projectionOutboxRealtimeEvents({ + facts: { messages: [{ messageId, traceId, sessionId, text: "current state must not replace history", projectedSeq: 99 }] }, + events: [ + { outboxSeq: 10, outboxEventId: "outbox-progress-1", entityFamily: "messages", entityId: messageId, projectedSeq: 1, projectionRevision: 1, traceId, sessionId, commitType: "message", payload: { family: "messages", fact: { messageId, traceId, sessionId, text: "first progress", projectedSeq: 1, status: "running" } } }, + { outboxSeq: 11, outboxEventId: "outbox-progress-2", entityFamily: "messages", entityId: messageId, projectedSeq: 2, projectionRevision: 2, traceId, sessionId, commitType: "message", payload: { family: "messages", fact: { messageId, traceId, sessionId, text: "second progress", projectedSeq: 2, status: "running" } } } + ] + }); + + assert.deepEqual(events.map((item) => item.payload.message.text), ["first progress", "second progress"]); + assert.deepEqual(events.map((item) => item.payload.cursor.outboxSeq), [10, 11]); +}); + +test("workbench realtime initial connection emits current snapshot without replaying historical outbox", async () => { + const sessionId = "ses_realtime_snapshot_only"; + const traceId = "trc_realtime_snapshot_only"; + const calls = []; + const runtime = { + async readAtomicWorkbenchProjectionSync(params = {}) { + calls.push({ ...params }); + return { + facts: { + sessions: [{ sessionId, lastTraceId: traceId }], + messages: [{ messageId: "msg_snapshot_only", sessionId, traceId, role: "agent", text: "current", projectedSeq: 9 }], + turns: [{ turnId: traceId, sessionId, traceId, status: "running", projectedSeq: 9 }] + }, + events: [{ outboxSeq: 1, entityFamily: "messages", entityId: "msg_old", payload: { family: "messages", fact: { messageId: "msg_old", sessionId, traceId, text: "historical" } } }], + cutoffOutboxSeq: 90, + cursorOutboxSeq: 90, + hasMore: true + }; + } + }; + const server = createCloudApiServer({ + accessController: realtimeAccessController(), + workbenchRuntime: runtime, + env: { HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const events = await getSseEvents(server.address().port, `/v1/workbench/events?sessionId=${sessionId}`, 3); + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.message.snapshot", "workbench.turn.snapshot"]); + assert.equal(JSON.stringify(events).includes("historical"), false); + assert.equal(events[1].id, "90"); + assert.equal(calls.length, 1); + assert.equal(calls[0].snapshotOnly, true); + assert.equal(calls[0].deltaOnly, false); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("projector commit during initial snapshot wakes one coalesced delta scan without idle polling", async () => { + const sessionId = "ses_realtime_commit_wakeup"; + const traceId = "trc_realtime_commit_wakeup"; + const calls = []; + let releaseInitial; + let notify = null; + let unsubscribeCount = 0; + const initialGate = new Promise((resolve) => { releaseInitial = resolve; }); + const kafkaEventBridge = { + subscribeProjectionCommits(listener) { notify = listener; return () => { unsubscribeCount += 1; }; }, + async stop() {} + }; + const runtime = { + async readAtomicWorkbenchProjectionSync(params = {}) { + calls.push({ ...params }); + if (calls.length === 1) { + await initialGate; + return { facts: { sessions: [{ sessionId, lastTraceId: traceId }], messages: [], turns: [] }, events: [], cutoffOutboxSeq: 10, cursorOutboxSeq: 10, hasMore: false }; + } + const event = { id: "wte_commit_wakeup", sessionId, traceId, projectedSeq: 11, message: "commit wakeup" }; + return { facts: {}, events: [{ outboxSeq: 11, entityFamily: "traceEvents", entityId: event.id, projectedSeq: 11, projectionRevision: 11, sessionId, traceId, commitType: "event", payload: { family: "traceEvents", fact: event } }], cutoffOutboxSeq: 11, cursorOutboxSeq: 11, hasMore: false }; + } + }; + const server = createCloudApiServer({ + accessController: realtimeAccessController(), + workbenchRuntime: runtime, + kafkaEventBridge, + env: { HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const eventsPromise = getSseEvents(server.address().port, `/v1/workbench/events?sessionId=${sessionId}`, 2); + await waitForCondition(() => calls.length === 1 && typeof notify === "function"); + notify({ sessionId, traceId }); + notify({ sessionId, traceId }); + releaseInitial(); + const events = await eventsPromise; + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.trace.event"]); + assert.equal(events[1].data.event.message, "commit wakeup"); + assert.equal(calls.length, 2); + assert.equal(calls[0].snapshotOnly, true); + assert.equal(calls[1].deltaOnly, true); + await new Promise((resolve) => setTimeout(resolve, 30)); + assert.equal(calls.length, 2); + } finally { + releaseInitial?.(); + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } + assert.equal(unsubscribeCount, 1); +}); + +test("workbench realtime early disconnect releases commit subscription during initial scan", async () => { + const sessionId = "ses_realtime_early_disconnect"; + let releaseInitial; + let subscribed = 0; + let unsubscribed = 0; + const gate = new Promise((resolve) => { releaseInitial = resolve; }); + const server = createCloudApiServer({ + accessController: realtimeAccessController(), + workbenchRuntime: { async readAtomicWorkbenchProjectionSync() { await gate; return { facts: {}, events: [], cutoffOutboxSeq: 0, cursorOutboxSeq: 0, hasMore: false }; } }, + kafkaEventBridge: { subscribeProjectionCommits() { subscribed += 1; return () => { unsubscribed += 1; }; }, async stop() {} }, + env: { HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + let clientRequest = null; + try { + const response = await new Promise((resolve, reject) => { + clientRequest = httpGet(`http://127.0.0.1:${server.address().port}/v1/workbench/events?sessionId=${sessionId}`, resolve); + clientRequest.once("error", reject); + }); + assert.equal(response.statusCode, 200); + await waitForCondition(() => subscribed === 1); + response.destroy(); + clientRequest.destroy(); + await waitForCondition(() => unsubscribed === 1); + releaseInitial(); + } finally { + clientRequest?.destroy(); + releaseInitial?.(); + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("workbench realtime prefers automatic Last-Event-ID over stale URL cursor", () => { + const url = new URL("http://localhost/v1/workbench/events?afterSeq=10&afterOutboxSeq=11"); + assert.equal(workbenchRealtimeAfterSeq({ headers: { "last-event-id": "42" } }, url), 42); +}); + +test("workbench realtime resets a cursor ahead of scoped cutoff with one fresh snapshot", async () => { + const sessionId = "ses_realtime_cursor_reset"; + const traceId = "trc_realtime_cursor_reset"; + const calls = []; + const runtime = { + async readAtomicWorkbenchProjectionSync(params = {}) { + calls.push({ ...params }); + if (params.snapshotOnly === true) { + return { facts: { sessions: [{ sessionId, lastTraceId: traceId }], messages: [{ messageId: "msg_cursor_reset", sessionId, traceId, text: "fresh", projectedSeq: 5 }], turns: [] }, events: [], cutoffOutboxSeq: 5, cursorOutboxSeq: 5, hasMore: false }; + } + return { facts: {}, events: [], cutoffOutboxSeq: 5, cursorOutboxSeq: 5, hasMore: false }; + } + }; + const server = createCloudApiServer({ accessController: realtimeAccessController(), workbenchRuntime: runtime, env: { HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const events = await getSseEvents(server.address().port, `/v1/workbench/events?sessionId=${sessionId}&afterSeq=42`, 2); + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.message.snapshot"]); + assert.equal(events[1].id, "5"); + assert.deepEqual(calls.map((call) => [call.deltaOnly, call.snapshotOnly]), [[true, false], [false, true]]); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + test("workbench realtime stream surfaces facts blocker instead of legacy trace fallback", async () => { const traceStore = createCodeAgentTraceStore(); const results = createCodeAgentChatResultStore(); @@ -63,36 +231,43 @@ test("workbench realtime stream surfaces facts blocker instead of legacy trace f async ensureBootstrap() {}, async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } }; - const server = createCloudApiServer({ accessController, traceStore, codeAgentChatResults: results, env: { HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000" } }); + const workbenchRuntime = { + async readAtomicWorkbenchProjectionSync() { + const error = new Error("atomic projection unavailable"); + error.code = "TEST_ATOMIC_PROJECTION_UNAVAILABLE"; + throw error; + } + }; + const server = createCloudApiServer({ + accessController, + traceStore, + codeAgentChatResults: results, + workbenchRuntime, + env: { + HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", + HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" + } + }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); try { const { port } = server.address(); - const eventsPromise = getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}&traceId=${encodeURIComponent(traceId)}`, 4); + const eventsPromise = getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}&traceId=${encodeURIComponent(traceId)}`, 2); const events = await eventsPromise; assert.deepEqual(events.map((event) => event.event), [ "workbench.connected", - "workbench.error", - "workbench.trace.snapshot", - "workbench.turn.snapshot" + "workbench.error" ]); assert.equal(events[0].data.filters.sessionId, session.id); - assert.equal(events[1].data.error.code, "workbench_facts_trace_missing"); - assert.equal(events[2].data.traceId, traceId); - assert.equal(events[2].data.snapshot.status, "unknown"); - assert.equal(events[2].data.snapshot.eventCount, 0); - assert.equal(events[3].data.turn.traceId, traceId); - assert.equal(events[3].data.turn.status, "unknown"); - assert.equal(events[3].data.turn.terminal, false); - assert.equal(events[3].data.turn.finalResponse, null); + assert.equal(events[1].data.error.code, "TEST_ATOMIC_PROJECTION_UNAVAILABLE"); assert.equal(JSON.stringify(events).includes("legacy result answer"), false); assert.equal(JSON.stringify(events).includes("legacy trace answer"), false); } finally { await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); } }); -test("workbench realtime stream forwards HWLAB Kafka events after initial connection", async () => { - const fakeKafka = createFakeKafkaFactory(); +test("workbench session realtime follows durable outbox beyond stale lastTraceId", async () => { + const staleTraceId = "trc_workbench_realtime_stale"; const traceId = "trc_workbench_realtime_after_seq"; const session = { id: "ses_workbench_realtime_after_seq", @@ -102,17 +277,25 @@ test("workbench realtime stream forwards HWLAB Kafka events after initial connec ownerUserId: ACTOR.id, conversationId: "cnv_workbench_realtime_after_seq", threadId: "thread-workbench-realtime-after-seq", - lastTraceId: traceId, + lastTraceId: staleTraceId, updatedAt: "2026-06-24T14:00:00.000Z", - session: { sessionStatus: "running", lastTraceId: traceId } + session: { sessionStatus: "running", lastTraceId: staleTraceId } }; const outboxQueries = []; const workbenchRuntime = { - async readWorkbenchProjectionOutbox(params = {}) { + async readAtomicWorkbenchProjectionSync(params = {}) { outboxQueries.push({ ...params }); - return [ - { outboxSeq: 11, projectedSeq: 7, traceId, sessionId: session.id, turnId: traceId, commitType: "event", terminal: false, sealed: false, createdAt: "2026-06-24T14:00:01.000Z" } - ]; + const after = Number(params.afterOutboxSeq ?? 0); + const outboxSeq = after + 1; + const projectedSeq = outboxSeq === 11 ? 7 : 8; + const event = { id: `wte_workbench_realtime_after_seq_${outboxSeq}`, sourceEventId: `src_workbench_realtime_after_seq_${outboxSeq}`, projectedSeq, sourceSeq: projectedSeq, traceId, sessionId: session.id, turnId: traceId, eventType: "backend", status: "running", message: outboxSeq === 11 ? "durable outbox realtime event" : "second outbox page", terminal: false, sealed: false, updatedAt: "2026-06-24T14:00:01.000Z" }; + return { + facts: { sessions: [{ sessionId: session.id, ownerUserId: ACTOR.id, lastTraceId: staleTraceId, status: "running" }], messages: [], parts: [], turns: [], traceEvents: [event], checkpoints: [] }, + events: [{ outboxSeq, outboxEventId: `outbox-workbench-realtime-after-seq-${outboxSeq}`, entityFamily: "traceEvents", entityId: event.id, projectionRevision: projectedSeq, projectedSeq, traceId, sessionId: session.id, turnId: traceId, commitType: "event", terminal: false, sealed: false, payload: { family: "traceEvents", fact: event }, createdAt: event.updatedAt }], + cutoffOutboxSeq: 12, + cursorOutboxSeq: outboxSeq, + hasMore: outboxSeq < 12 + }; } }; const accessController = { @@ -126,110 +309,33 @@ test("workbench realtime stream forwards HWLAB Kafka events after initial connec const serverWithKafka = createCloudApiServer({ accessController, workbenchRuntime, - kafkaFactory: fakeKafka.factory, env: { HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", - HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", - HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", - HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1" + HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } }); await new Promise((resolve) => serverWithKafka.listen(0, "127.0.0.1", resolve)); try { const { port } = serverWithKafka.address(); - setTimeout(() => { void fakeKafka.emit({ - eventType: "hwlab.trace.event.projected", - sessionId: "ses_agentrun_workbench_realtime_after_seq", - traceId, - context: { sourceSeq: 7, runId: "run_workbench_realtime_after_seq", commandId: "cmd_workbench_realtime_after_seq" }, - event: { type: "backend", eventType: "backend", status: "running", label: "agentrun:event:test", message: "Kafka realtime event", sourceSeq: 7 } - }); }, 25); - const events = await getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}&traceId=${encodeURIComponent(traceId)}&afterSeq=10`, 2); - assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.trace.event"]); + const events = await getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}&afterSeq=10`, 3); + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.trace.event", "workbench.trace.event"]); assert.equal(events[0].id, "10"); - assert.equal(events[1].id, "hwlab.event.v1:0:0"); - assert.equal(events[1].data.realtimeSource, "kafka"); + assert.equal(events[1].id, "11"); + assert.equal(events[1].data.realtimeSource, "projection-outbox"); assert.equal(events[1].data.realtimeAuthority, "workbench-realtime-authority-v2"); assert.equal(events[1].data.entity.family, "traceEvents"); assert.equal(events[1].data.entity.version, 7); - assert.equal(events[1].data.entity.projectionRevision, "kafka:hwlab.event.v1:0:0"); - assert.equal(events[1].data.event.message, "Kafka realtime event"); - assert.equal(events[1].data.kafka.topic, "hwlab.event.v1"); + assert.equal(events[1].data.entity.projectionRevision, "7"); + assert.equal(events[1].data.event.message, "durable outbox realtime event"); + assert.equal(events[1].data.traceId, traceId); assert.equal(events[1].data.cursor.traceSeq, 7); - assert.deepEqual(outboxQueries, []); - } finally { - await new Promise((resolve, reject) => serverWithKafka.close((error) => error ? reject(error) : resolve())); - } -}); - -test("workbench session realtime Kafka stream does not pin subscription to stale lastTraceId", async () => { - const fakeKafka = createFakeKafkaFactory(); - const staleTraceId = "trc_workbench_realtime_stale_trace"; - const liveTraceId = "trc_workbench_realtime_live_trace"; - const session = { - id: "ses_workbench_realtime_session_wide", - projectId: "prj_hwpod_workbench", - agentId: "hwlab-code-agent", - status: "running", - ownerUserId: ACTOR.id, - conversationId: "cnv_workbench_realtime_session_wide", - threadId: "thread-workbench-realtime-session-wide", - lastTraceId: staleTraceId, - updatedAt: "2026-06-24T14:00:00.000Z", - session: { sessionStatus: "running", lastTraceId: staleTraceId } - }; - const accessController = { - store: { - async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, - async getAgentSessionByTraceId() { return null; } - }, - async ensureBootstrap() {}, - async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } - }; - const serverWithKafka = createCloudApiServer({ - accessController, - workbenchRuntime: { - async queryWorkbenchFacts(params = {}) { - return { - facts: { - sessions: [{ sessionId: session.id, ownerUserId: ACTOR.id, threadId: session.threadId, lastTraceId: staleTraceId, status: "running", valuesRedacted: true }], - messages: [], - parts: [], - turns: [], - checkpoints: [] - }, - count: 1, - persistence: { adapter: "test-session-wide-kafka", durable: true }, - params - }; - } - }, - kafkaFactory: fakeKafka.factory, - env: { - HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", - HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", - HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", - HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1" - } - }); - await new Promise((resolve) => serverWithKafka.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = serverWithKafka.address(); - setTimeout(() => { void fakeKafka.emit({ - eventType: "hwlab.trace.event.projected", - sessionId: "ses_agentrun_workbench_realtime_session_wide", - traceId: liveTraceId, - context: { runId: "run_workbench_realtime_session_wide", commandId: "cmd_workbench_realtime_session_wide" }, - event: { type: "backend", eventType: "backend", status: "running", label: "agentrun:event:live", message: "Kafka live trace event" } - }); }, 100); - const events = await getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}`, 4); - assert.equal(events[0].event, "workbench.connected"); - const liveEvent = events.find((event) => event.event === "workbench.trace.event" && event.data?.traceId === liveTraceId); - assert.ok(liveEvent, JSON.stringify(events.map((event) => ({ event: event.event, traceId: event.data?.traceId, message: event.data?.event?.message })))); - assert.equal(liveEvent.data.event.message, "Kafka live trace event"); - assert.equal(liveEvent.data.realtimeAuthority, "workbench-realtime-authority-v2"); + assert.equal(events[2].id, "12"); + assert.equal(events[2].data.event.message, "second outbox page"); + assert.equal(outboxQueries.length, 2); + assert.deepEqual(outboxQueries.map((query) => query.afterOutboxSeq), [10, 11]); + assert.equal(outboxQueries[0].sessionId, session.id); + assert.deepEqual(outboxQueries[0].actor, { id: ACTOR.id, role: ACTOR.role }); } finally { await new Promise((resolve, reject) => serverWithKafka.close((error) => error ? reject(error) : resolve())); } @@ -277,7 +383,7 @@ test("workbench read model exposes runtime trace projection query failures as pr throw error; } }; - const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: results }); + const server = createCloudApiServer({ accessController, traceStore, workbenchRuntime: runtimeStore, codeAgentChatResults: results }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); try { @@ -355,7 +461,7 @@ test("workbench trace events remain visible after session lastTraceId moves", as async ensureBootstrap() {}, async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } }; - const server = createCloudApiServer({ accessController, runtimeStore }); + const server = createCloudApiServer({ accessController, workbenchRuntime: runtimeStore }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); try { @@ -446,3 +552,10 @@ test("workbench trace event page keeps per-trace terminal status after later ses await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); } }); + +function realtimeAccessController() { + return { + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_realtime_test" } }; } + }; +} diff --git a/internal/cloud/server-workbench-realtime-http.ts b/internal/cloud/server-workbench-realtime-http.ts index 548b8ec8..fce8e3d8 100644 --- a/internal/cloud/server-workbench-realtime-http.ts +++ b/internal/cloud/server-workbench-realtime-http.ts @@ -4,7 +4,6 @@ */ import { createHash } from "node:crypto"; -import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; import { parsePositiveInteger, safeConversationId, @@ -17,8 +16,8 @@ import { createWorkbenchReadModel } from "./workbench-read-model.ts"; import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts"; import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts"; import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts"; -import { openKafkaEventStream } from "./kafka-event-bridge.ts"; -import { durableTraceStatus, RUNNING_STATUSES, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts"; +import { projectionOutboxRealtimeEvents } from "./workbench-projection-outbox-events.ts"; +import { durableTraceStatus, RUNNING_STATUSES, TERMINAL_STATUSES } from "./workbench-turn-projection.ts"; import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts"; import * as workbenchFacts from "./server-workbench-facts.ts"; @@ -27,7 +26,6 @@ const DEFAULT_SESSION_LIST_LIMIT = 20; const MAX_PAGE_LIMIT = 100; const DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS = 15000; const WORKBENCH_REALTIME_DRAIN_TIMEOUT_MS = 2500; -const WORKBENCH_REALTIME_AUTHORITY_VERSION = "workbench-realtime-authority-v2"; const WORKBENCH_SESSION_LIST_PAGE_FAMILIES = Object.freeze(["sessions"]); const WORKBENCH_SESSION_DETAIL_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]); const WORKBENCH_SESSION_MESSAGE_PAGE_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]); @@ -187,7 +185,7 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option sessionId: requestedSessionId, traceId: requestedTraceId, heartbeatMs, - realtimeSource: "kafka" + realtimeSource: "projection-outbox" }); const perf = options.backendPerformance; const auth = perf ? await perf.measure("workbench_auth", () => authenticateWorkbenchRead(request, response, options)) : await authenticateWorkbenchRead(request, response, options); @@ -198,9 +196,17 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option return; } - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - const readModel = createWorkbenchReadModel(options, auth.actor); + if (!requestedSessionId && !requestedTraceId) { + sendJson(response, 400, workbenchError("workbench_realtime_scope_required", "Workbench realtime requires sessionId or traceId.")); + return; + } + const runtime = options.workbenchRuntime ?? createWorkbenchRuntimeClient({ env: options.env ?? process.env, fetch: options.fetch, traceparent: request?.hwlabHttpRequestContext?.traceparent }); + if (typeof runtime?.readAtomicWorkbenchProjectionSync !== "function") { + sendJson(response, 503, workbenchError("workbench_realtime_runtime_unconfigured", "Workbench realtime requires an atomic projection snapshot reader.")); + return; + } const requestedAfterSeq = workbenchRealtimeAfterSeq(request, url); + const outboxTailBatchSize = requiredPositiveRuntimeSetting(options.env ?? process.env, "HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE"); let closed = false; let realtimeCloseReason = "client_close"; let realtimeCloseSignal = null; @@ -210,35 +216,64 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option const realtimeStartedAtMs = Date.now(); const cleanup = []; - response.writeHead(200, { - "content-type": "text/event-stream; charset=utf-8", - "cache-control": "no-store, no-transform", - connection: "keep-alive", - "x-accel-buffering": "no", - "x-content-type-options": "nosniff" - }); - if (typeof response.flushHeaders === "function") response.flushHeaders(); + let streamSessionId = requestedSessionId ?? null; + let streamThreadId = null; + let activeTraceId = requestedTraceId ?? null; + let outboxCursor = requestedAfterSeq; + let scanRunning = false; + let scanRequested = false; + let realtimeConnection = null; - const writeEvent = (name, payload = {}) => { - if (closed || response.destroyed) return; - const startedAt = nowMs(); - const eventCreatedAt = realtimeEventCreatedAt(payload); - const traceSeq = realtimeTraceSeq(payload); - const eventId = realtimeEventId(payload); - response.write(`event: ${name}\n`); - if (eventId) response.write(`id: ${eventId}\n`); - response.write(`data: ${JSON.stringify({ contractVersion: "workbench-events-v1", serverSentAt: new Date().toISOString(), eventCreatedAt, traceSeq, ...payload })}\n\n`); - perf?.recordPhase({ phase: "sse_write", durationMs: nowMs() - startedAt, outcome: "ok" }); - }; + const isActive = () => !closed && !response.destroyed && !response.writableEnded; + const closeConnection = () => { + if (closed) return; + closed = true; + emitWorkbenchRealtimeClosedOtelSpan(request, options.env, { + reason: realtimeCloseReason, + signal: realtimeCloseSignal, + startedAtMs: realtimeStartedAtMs, + sessionId: realtimeSessionId, + threadId: realtimeThreadId, + traceId: realtimeTraceId, + activeConnectionCount: activeWorkbenchRealtimeConnections.size + }); + for (const item of cleanup.splice(0)) item(); + }; - const realtimeConnection = { - close(fields = {}) { - if (closed || response.destroyed || response.writableEnded) return false; - const reason = textValue(fields.reason) || "server_shutdown"; - realtimeCloseReason = reason; - realtimeCloseSignal = textValue(fields.signal) || null; - try { - writeEvent("workbench.server_draining", { + response.writeHead(200, { + "content-type": "text/event-stream; charset=utf-8", + "cache-control": "no-store, no-transform", + connection: "keep-alive", + "x-accel-buffering": "no", + "x-content-type-options": "nosniff" + }); + response.once("close", closeConnection); + request.once?.("aborted", closeConnection); + request.socket?.once?.("close", closeConnection); + cleanup.push(() => request.off?.("aborted", closeConnection)); + cleanup.push(() => request.socket?.off?.("close", closeConnection)); + if (typeof response.flushHeaders === "function") response.flushHeaders(); + + const writeEvent = async (name, payload = {}) => { + if (!isActive()) return false; + const startedAt = nowMs(); + const eventCreatedAt = realtimeEventCreatedAt(payload); + const traceSeq = realtimeTraceSeq(payload); + const eventId = realtimeEventId(payload); + const block = `event: ${name}\n${eventId ? `id: ${eventId}\n` : ""}data: ${JSON.stringify({ contractVersion: "workbench-events-v1", serverSentAt: new Date().toISOString(), eventCreatedAt, traceSeq, ...payload })}\n\n`; + const writable = response.write(block); + if (!writable) await waitForResponseDrain(response, isActive); + perf?.recordPhase({ phase: "sse_write", durationMs: nowMs() - startedAt, outcome: isActive() ? "ok" : "closed" }); + return isActive(); + }; + + realtimeConnection = { + close(fields = {}) { + if (!isActive()) return false; + const reason = textValue(fields.reason) || "server_shutdown"; + realtimeCloseReason = reason; + realtimeCloseSignal = textValue(fields.signal) || null; + void writeEvent("workbench.server_draining", { type: "server.draining", status: "closing", reason, @@ -247,137 +282,131 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option threadId: realtimeThreadId, traceId: realtimeTraceId, observedAt: new Date().toISOString() + }).finally(() => { + if (!response.writableEnded) response.end(); }); - response.end(); return true; - } catch { - try { response.destroy(); } catch {} - return false; - } - }, - isActive() { - return !closed && !response.destroyed && !response.writableEnded; - } - }; - activeWorkbenchRealtimeConnections.add(realtimeConnection); - cleanup.push(() => activeWorkbenchRealtimeConnections.delete(realtimeConnection)); + }, + isActive + }; + activeWorkbenchRealtimeConnections.add(realtimeConnection); + cleanup.push(() => activeWorkbenchRealtimeConnections.delete(realtimeConnection)); - writeEvent("workbench.connected", { - type: "connected", - status: "connected", - realtimeSource: "kafka", - snapshotSource: "read-model", - heartbeatMs, - cursor: { outboxSeq: requestedAfterSeq }, - filters: { - sessionId: requestedSessionId, - traceId: requestedTraceId - } - }); + await writeEvent("workbench.connected", { + type: "connected", + status: "connected", + realtimeSource: "projection-outbox", + snapshotSource: "atomic-projection-sync", + heartbeatMs, + cursor: { outboxSeq: requestedAfterSeq }, + filters: { sessionId: requestedSessionId, traceId: requestedTraceId } + }); - const resolveInitialSession = requestedSessionId && requestedAfterSeq <= 0; - const initialContext = resolveInitialSession - ? await realtimeInitialSessionContext(readModel, auth.actor, requestedSessionId) - : { facts: emptyFactSet(), session: null, blocker: null }; - if (initialContext.blocker) { - writeEvent("workbench.error", { - type: "error", - reason: "initial-session", + realtimeSessionId = streamSessionId ?? requestedSessionId ?? null; + realtimeTraceId = activeTraceId ?? requestedTraceId ?? null; + realtimeThreadId = streamThreadId ?? null; + attachWorkbenchRealtimeOtelContext(request, { + sessionId: streamSessionId ?? requestedSessionId, + traceId: activeTraceId, + threadId: streamThreadId, + heartbeatMs, + realtimeSource: "projection-outbox" + }); + emitWorkbenchRealtimeAcceptedOtelSpan(request, options.env); + + const syncParams = (extra = {}) => ({ sessionId: requestedSessionId, traceId: requestedTraceId, - error: initialContext.blocker + afterOutboxSeq: outboxCursor, + afterSeq: outboxCursor, + limit: outboxTailBatchSize, + actor: { id: auth.actor?.id, role: auth.actor?.role ?? "user" }, + ...extra }); - } - const initialFactSession = initialContext.session; - const streamSessionId = factSessionId(initialFactSession) ?? requestedSessionId ?? null; - const streamThreadId = safeOpaqueId(initialFactSession?.threadId) ?? (textValue(initialFactSession?.threadId) || null); - const activeTraceId = requestedTraceId - ?? factLastTraceId(initialFactSession) - ?? factLatestTraceIdForSession(initialContext.facts, streamSessionId) - ?? null; - realtimeSessionId = streamSessionId ?? requestedSessionId ?? null; - realtimeTraceId = activeTraceId ?? requestedTraceId ?? null; - realtimeThreadId = streamThreadId ?? null; - attachWorkbenchRealtimeOtelContext(request, { - sessionId: streamSessionId ?? requestedSessionId, - traceId: activeTraceId, - threadId: streamThreadId, - heartbeatMs, - realtimeSource: "kafka" - }); - emitWorkbenchRealtimeAcceptedOtelSpan(request, options.env); - if (activeTraceId && requestedAfterSeq <= 0) { - await (perf ? perf.measure("workbench_initial_trace", () => writeTraceRealtimeSnapshotSafe({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" })) : writeTraceRealtimeSnapshotSafe({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" })); - } - const kafkaFilters = resolveWorkbenchRealtimeKafkaFilters({ traceId: requestedTraceId, sessionId: streamSessionId, traceStore }); - try { - const kafkaStream = await openKafkaEventStream({ - env: options.env ?? process.env, - stream: "hwlab", - fromBeginning: false, - ...kafkaFilters, - kafkaFactory: options.kafkaFactory, - onEvent: async (record) => { - if (closed || response.destroyed) return; - const realtime = workbenchRealtimeEventFromKafka(record, { sessionId: streamSessionId, threadId: streamThreadId, traceId: activeTraceId }); - if (!realtime) return; - writeEvent("workbench.trace.event", realtime.traceEvent); - if (realtime.turnSnapshot) writeEvent("workbench.turn.snapshot", realtime.turnSnapshot); - }, - onError: (error) => { - writeEvent("workbench.error", { - type: "error", - realtimeSource: "kafka", - sessionId: streamSessionId, - threadId: streamThreadId, - traceId: activeTraceId, - error: { code: "workbench_kafka_stream_failed", message: error?.message ?? "Workbench Kafka realtime stream failed." } - }); + const updateStreamScope = (snapshot) => { + const session = factArray(snapshot?.facts?.sessions)[0] ?? null; + streamSessionId = textValue(session?.sessionId) || requestedSessionId || streamSessionId; + streamThreadId = safeOpaqueId(session?.threadId) || textValue(session?.threadId) || streamThreadId; + activeTraceId = requestedTraceId || textValue(session?.lastTraceId) || activeTraceId; + realtimeSessionId = streamSessionId; + realtimeThreadId = streamThreadId; + realtimeTraceId = activeTraceId; + }; + const emitProjectionSnapshot = async (snapshot) => { + updateStreamScope(snapshot); + for (const item of projectionOutboxRealtimeEvents(snapshot, { includeSnapshot: true, includeEvents: false })) { + if (!await writeEvent(item.name, item.payload)) break; } + outboxCursor = nonNegativeInteger(snapshot?.cutoffOutboxSeq ?? snapshot?.cursorOutboxSeq); + }; + const recoverCursor = async () => { + const snapshot = await runtime.readAtomicWorkbenchProjectionSync(syncParams({ afterOutboxSeq: 0, afterSeq: 0, snapshotOnly: true, deltaOnly: false })); + await emitProjectionSnapshot(snapshot); + }; + const runProjectionScan = async ({ initial = false } = {}) => { + const snapshotOnly = initial && requestedAfterSeq <= 0; + if (snapshotOnly) { + await recoverCursor(); + return; + } + let hasMore = true; + while (hasMore && isActive()) { + const snapshot = await runtime.readAtomicWorkbenchProjectionSync(syncParams({ snapshotOnly: false, deltaOnly: true })); + const nextCursor = nonNegativeInteger(snapshot?.cursorOutboxSeq); + if (nextCursor < outboxCursor) { + await recoverCursor(); + return; + } + for (const item of projectionOutboxRealtimeEvents(snapshot)) { + if (!await writeEvent(item.name, item.payload)) return; + } + outboxCursor = nextCursor; + hasMore = snapshot?.hasMore === true; + } + }; + const scanProjectionOutbox = async ({ initial = false } = {}) => { + if (!isActive()) return; + if (scanRunning) { + scanRequested = true; + return; + } + scanRunning = true; + let initialPass = initial; + try { + do { + scanRequested = false; + await runProjectionScan({ initial: initialPass }); + initialPass = false; + } while (scanRequested && isActive()); + } catch (error) { + await writeEvent("workbench.error", { type: "error", realtimeSource: "projection-outbox", sessionId: streamSessionId, traceId: activeTraceId, error: { code: error?.code ?? "workbench_outbox_scan_failed", message: error?.message ?? "Workbench projection outbox scan failed.", valuesRedacted: true } }); + } finally { + scanRunning = false; + } + }; + const requestProjectionScan = () => { + if (!isActive()) return; + scanRequested = true; + void scanProjectionOutbox(); + }; + const unsubscribeProjectionCommits = options.kafkaEventBridge?.subscribeProjectionCommits?.((signal = {}) => { + if (workbenchProjectionSignalMatches(signal, requestedSessionId, requestedTraceId)) requestProjectionScan(); }); - cleanup.push(() => { void kafkaStream.stop?.(); }); - } catch (kafkaError) { - writeEvent("workbench.error", { - type: "error", - realtimeSource: "kafka", - sessionId: streamSessionId, - threadId: streamThreadId, - traceId: activeTraceId, - error: { code: "workbench_kafka_stream_unavailable", message: kafkaError?.message ?? "Workbench Kafka realtime stream is unavailable." } - }); - } + if (typeof unsubscribeProjectionCommits === "function") cleanup.push(unsubscribeProjectionCommits); - const heartbeat = setInterval(async () => { - try { - writeEvent("workbench.heartbeat", { + await scanProjectionOutbox({ initial: true }); + if (!isActive()) return; + + const heartbeat = setInterval(() => { + void writeEvent("workbench.heartbeat", { type: "heartbeat", sessionId: requestedSessionId ?? null, traceId: activeTraceId ?? null, observedAt: new Date().toISOString() }); - } catch (error) { - writeEvent("workbench.error", { - type: "error", - error: { code: "workbench_realtime_heartbeat_failed", message: error?.message ?? "Workbench realtime heartbeat failed." } - }); - } - }, Math.max(1000, heartbeatMs)); - cleanup.push(() => clearInterval(heartbeat)); - - response.on("close", () => { - if (closed) return; - closed = true; - emitWorkbenchRealtimeClosedOtelSpan(request, options.env, { - reason: realtimeCloseReason, - signal: realtimeCloseSignal, - startedAtMs: realtimeStartedAtMs, - sessionId: realtimeSessionId, - threadId: realtimeThreadId, - traceId: realtimeTraceId, - activeConnectionCount: activeWorkbenchRealtimeConnections.size - }); - for (const item of cleanup.splice(0)) item(); - }); + }, Math.max(1000, heartbeatMs)); + heartbeat.unref?.(); + cleanup.push(() => clearInterval(heartbeat)); } catch (error) { handleWorkbenchRealtimeFailure(request, response, url, options, error); } @@ -396,151 +425,11 @@ export function attachWorkbenchRealtimeOtelContext(request, fields = {}) { "workbench.thread_id": fields.threadId ?? null, "workbench.sse.heartbeat_ms": fields.heartbeatMs ?? null, "workbench.sse.realtime_source": fields.realtimeSource ?? null, - "workbench.sse.outbox_mode": false + "workbench.sse.outbox_mode": true }; } -export function resolveWorkbenchRealtimeKafkaFilters({ traceId = null, sessionId = null, traceStore = null } = {}) { - const resolved = traceId && typeof traceStore?.snapshot === "function" ? collectTraceLinkedIds(traceStore.snapshot(traceId)) : {}; - const hasAgentRunKey = Boolean(resolved.runId || resolved.commandId); - const fallbackSessionId = resolved.sessionId || agentRunScopedSessionIdFromWorkbenchSession(sessionId) || sessionId; - return compactObject({ - traceId: hasAgentRunKey ? null : traceId, - sessionId: hasAgentRunKey ? null : fallbackSessionId, - runId: resolved.runId, - commandId: resolved.commandId - }); -} - -export function agentRunScopedSessionIdFromWorkbenchSession(sessionId) { - const safeId = safeSessionId(sessionId); - if (!safeId || isAgentRunAliasSessionId(safeId)) return safeId; - const base = safeId.replace(/^ses_/u, "").replace(/[^A-Za-z0-9_]+/gu, "_").replace(/^_+|_+$/gu, "") || "session"; - return safeSessionId(`ses_agentrun_${base}`); -} - -export function collectTraceLinkedIds(snapshot) { - const out = { sessionId: null, runId: null, commandId: null }; - const events = Array.isArray(snapshot?.events) ? snapshot.events : []; - for (const event of events) collectTraceLinkedIdsFromRecord(event, out); - collectTraceLinkedIdsFromRecord(snapshot?.lastEvent, out); - return out; -} - -export function collectTraceLinkedIdsFromRecord(record, out) { - const value = record && typeof record === "object" && !Array.isArray(record) ? record : {}; - const agentRun = value.agentRun && typeof value.agentRun === "object" && !Array.isArray(value.agentRun) ? value.agentRun : {}; - const payload = value.payload && typeof value.payload === "object" && !Array.isArray(value.payload) ? value.payload : {}; - out.sessionId ||= safeSessionId(value.sessionId ?? value.sourceSessionId ?? agentRun.sessionId ?? payload.sessionId); - out.runId ||= textValue(value.runId ?? value.sourceRunId ?? agentRun.runId ?? payload.runId); - out.commandId ||= textValue(value.commandId ?? value.sourceCommandId ?? agentRun.commandId ?? payload.commandId); -} - -export function workbenchRealtimeEventFromKafka(record, context = {}) { - const value = record?.value && typeof record.value === "object" && !Array.isArray(record.value) ? record.value : null; - if (!value) return null; - const hwlabEvent = value.event && typeof value.event === "object" && !Array.isArray(value.event) ? value.event : {}; - const sourceContext = value.context && typeof value.context === "object" && !Array.isArray(value.context) ? value.context : {}; - const traceId = safeTraceId(value.traceId ?? hwlabEvent.traceId ?? context.traceId) ?? textValue(value.traceId ?? hwlabEvent.traceId ?? context.traceId); - const sessionId = safeSessionId(context.sessionId ?? value.sessionId ?? hwlabEvent.sessionId) ?? textValue(context.sessionId ?? value.sessionId ?? hwlabEvent.sessionId); - const threadId = safeOpaqueId(context.threadId ?? sourceContext.threadId) ?? textValue(context.threadId ?? sourceContext.threadId); - if (!traceId && !sessionId) return null; - const sourceSeq = integerValue(hwlabEvent.sourceSeq ?? sourceContext.sourceSeq); - const kafka = { - topic: textValue(record.topic), - partition: integerValue(record.partition), - offset: textValue(record.offset), - key: textValue(record.key), - timestamp: textValue(record.timestamp), - valuesRedacted: true - }; - const event = { - ...hwlabEvent, - traceId: traceId ?? hwlabEvent.traceId ?? null, - sessionId: sessionId ?? hwlabEvent.sessionId ?? null, - threadId: threadId ?? sourceContext.threadId ?? null, - runId: textValue(hwlabEvent.runId ?? sourceContext.runId), - commandId: textValue(hwlabEvent.commandId ?? sourceContext.commandId), - source: textValue(hwlabEvent.source) || "hwlab.kafka", - sourceSeq, - kafka, - valuesPrinted: false - }; - const status = normalizeStatus(event.status); - const terminal = event.terminal === true || TERMINAL_STATUSES.has(status); - const cursor = { - traceSeq: sourceSeq ?? integerValue(record.offset), - kafkaOffset: textValue(record.offset), - kafkaPartition: integerValue(record.partition) - }; - const entityVersion = sourceSeq ?? integerValue(record.offset) ?? 0; - const projectionRevision = ["kafka", kafka.topic, kafka.partition, kafka.offset].filter((part) => textValue(part)).join(":"); - const traceEntity = { - family: "traceEvents", - id: [traceId ?? "trace", event.runId, event.commandId, kafka.partition, kafka.offset].filter((part) => textValue(part)).join(":"), - version: entityVersion, - traceSeq: cursor.traceSeq, - projectionRevision, - authority: WORKBENCH_REALTIME_AUTHORITY_VERSION - }; - const traceEvent = { - id: kafka.topic && kafka.partition !== null && kafka.offset ? `${kafka.topic}:${kafka.partition}:${kafka.offset}` : null, - type: "trace.event", - contractVersion: "workbench-sync-v1", - realtimeAuthority: WORKBENCH_REALTIME_AUTHORITY_VERSION, - realtimeSource: "kafka", - sessionId, - threadId, - traceId, - event, - snapshot: { traceId, sessionId, threadId, status: status || event.status || null, events: [event], eventCount: 1 }, - cursor, - entity: traceEntity, - projectionRevision, - kafka, - valuesPrinted: false - }; - const turnSnapshot = terminal ? { - type: "turn.snapshot", - contractVersion: "workbench-sync-v1", - realtimeAuthority: WORKBENCH_REALTIME_AUTHORITY_VERSION, - realtimeSource: "kafka", - sessionId, - threadId, - traceId, - reason: "kafka-terminal", - cursor, - turn: { - traceId, - sessionId, - threadId, - status: status || event.status || "completed", - running: false, - terminal: true, - finalResponse: terminalFinalResponse(status || event.status || "completed", { - traceId, - status: status || event.status || "completed", - finalResponse: event.text ? { text: event.text, status: status || event.status || "completed", traceId, source: "kafka-terminal-event", valuesPrinted: false } : null, - finalText: event.text ?? event.message ?? null, - error: event.errorCode ? { message: event.message ?? event.errorCode } : null - }), - agentRun: compactObject({ runId: event.runId, commandId: event.commandId }), - valuesPrinted: false - }, - entity: { - family: "turns", - id: traceId ?? event.runId ?? event.commandId ?? traceEntity.id, - version: entityVersion, - traceSeq: cursor.traceSeq, - projectionRevision, - authority: WORKBENCH_REALTIME_AUTHORITY_VERSION - }, - projectionRevision, - kafka, - valuesPrinted: false - } : null; - return { traceEvent, turnSnapshot }; -} +export { projectionOutboxRealtimeEvents } from "./workbench-projection-outbox-events.ts"; export function compactObject(value) { return Object.fromEntries(Object.entries(value || {}).filter(([, entry]) => textValue(entry))); @@ -713,13 +602,49 @@ export function realtimeEventId(payload) { } export function workbenchRealtimeAfterSeq(request, url) { - for (const value of [url.searchParams.get("afterSeq"), url.searchParams.get("afterOutboxSeq"), request?.headers?.["last-event-id"]]) { + for (const value of [request?.headers?.["last-event-id"], url.searchParams.get("afterSeq"), url.searchParams.get("afterOutboxSeq")]) { const parsed = nonNegativeInteger(value, 0); if (parsed > 0) return parsed; } return 0; } +function requiredPositiveRuntimeSetting(env, name) { + const value = Number(env?.[name]); + if (!Number.isInteger(value) || value <= 0) { + const error = new Error(`${name} must be explicitly configured as a positive integer.`); + error.code = "workbench_outbox_tail_config_invalid"; + throw error; + } + return value; +} + +export function workbenchProjectionSignalMatches(signal = {}, sessionId = null, traceId = null) { + if (signal.recovery === true) return true; + const signalSessionId = textValue(signal.sessionId); + const signalTraceId = textValue(signal.traceId); + if (traceId && signalTraceId !== traceId) return false; + if (sessionId && signalSessionId !== sessionId) return false; + return Boolean(signalSessionId || signalTraceId); +} + +export function waitForResponseDrain(response, isActive = () => true) { + if (!isActive()) return Promise.resolve(false); + return new Promise((resolve) => { + const settle = (result) => { + response.off?.("drain", onDrain); + response.off?.("close", onClose); + response.off?.("error", onClose); + resolve(result); + }; + const onDrain = () => settle(isActive()); + const onClose = () => settle(false); + response.once("drain", onDrain); + response.once("close", onClose); + response.once("error", onClose); + }); +} + export function nowMs() { if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now(); return Date.now(); diff --git a/internal/cloud/server.ts b/internal/cloud/server.ts index a2ffde74..f2af88db 100644 --- a/internal/cloud/server.ts +++ b/internal/cloud/server.ts @@ -3,8 +3,8 @@ * 职责: Cloud API REST/SSE route dispatcher。Workbench 新资源路由应委托 read model / compat wrapper，不在 dispatcher 内写业务事实。 */ import { createServer } from "node:http"; -import { createHash, randomUUID } from "node:crypto"; -import { copyFileSync, existsSync, lstatSync, mkdirSync, readFileSync, symlinkSync } from "node:fs"; +import { randomUUID } from "node:crypto"; +import { copyFileSync, existsSync, lstatSync, mkdirSync, symlinkSync } from "node:fs"; import path from "node:path"; import { @@ -28,7 +28,6 @@ import { handleCodeAgentChat } from "./code-agent-chat.ts"; import { createDurableCodeAgentTraceStore, defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; -import { writeWorkbenchProjectionEvent } from "./workbench-projection-writer.ts"; import { createCodeAgentSessionRegistry } from "./code-agent-session-registry.ts"; import { codeAgentSessionLifecycleSummary } from "./code-agent-session-lifecycle.ts"; import { createCodexStdioSessionManager } from "./codex-stdio-session.ts"; @@ -66,12 +65,10 @@ import { handleCodeAgentSessionsHttp, handleCodeAgentSteerHttp, handleCodeAgentTurnHttp, - handleCodeAgentTraceHttp, - startAgentRunProjectionResume + handleCodeAgentTraceHttp } from "./server-code-agent-http.ts"; import { drainWorkbenchRealtimeConnections, handleWorkbenchReadModelHttp, handleWorkbenchRealtimeHttp } from "./server-workbench-http.ts"; import { handleWorkbenchDebugFakeSseHttp } from "./workbench-debug-fake-sse.ts"; -import { handleWorkbenchKafkaSseDebugHttp } from "./workbench-kafka-sse-debug.ts"; import { handleWorkbenchLaunchHttp } from "./server-workbench-launch-http.ts"; import { startWorkbenchEmptySessionGc } from "./workbench-empty-session-gc.ts"; import { startHwlabKafkaEventBridge } from "./kafka-event-bridge.ts"; @@ -87,10 +84,11 @@ import { } from "./skills-store.ts"; import { createHwpodNodeWsRegistry } from "./hwpod-node-ws-registry.ts"; import { - discoverHwpodSpecs, - hwpodSpecDiscoveryPayload, - hwpodSpecWorkspaceProbePlan -} from "./hwpod-spec-discovery.ts"; + handleHwlabNodeDownloadHttp, + handleHwlabNodeUpdateHttp, + handleHwpodNodeOpsHttp, + handleHwpodSpecDiscoveryHttp +} from "./server-hwpod-http.ts"; import { handleCaseRunHttp } from "./server-caserun-http.ts"; import { handleProjectManagementProxyHttp } from "./project-management-proxy.ts"; import { HWPOD_NODE_OPS, HWPOD_NODE_OPS_CONTRACT_VERSION } from "../../tools/src/hwpod-node-ops-contract.ts"; @@ -155,8 +153,7 @@ export function createCloudApiServer(options = {}) { const runtimeStore = options.runtimeStore || createConfiguredCloudRuntimeStore({ ...options, env }); const traceStore = createDurableCodeAgentTraceStore({ traceStore: options.traceStore || defaultCodeAgentTraceStore, - traceEventStore: runtimeStore, - workbenchEventWriter: (event, meta) => writeWorkbenchProjectionEvent({ runtimeStore, event, requestMeta: meta }) + traceEventStore: runtimeStore }); const codexStdioManager = options.codexStdioManager || createCodexStdioSessionManager({ traceStore }); const codeAgentChatResults = options.codeAgentChatResults || createCodeAgentChatResultStore({ @@ -173,13 +170,8 @@ export function createCloudApiServer(options = {}) { if (typeof accessController.configureCodeAgentWorkspaceContext === "function") { accessController.configureCodeAgentWorkspaceContext({ env, fetchImpl: options.fetchImpl, traceStore, codeAgentChatResults }); } - const workbenchEmptySessionGc = startWorkbenchEmptySessionGc({ env, accessController, logger: console }); - const agentRunProjectionResume = startAgentRunProjectionResume({ - options: { ...options, env, runtimeStore, accessController, traceStore, codeAgentChatResults }, - traceStore, - logger: console - }); - const kafkaEventBridge = startHwlabKafkaEventBridge({ env, logger: console }); + const workbenchEmptySessionGc = startWorkbenchEmptySessionGc({ env, accessController, logger: options.logger ?? console }); + const kafkaEventBridge = options.kafkaEventBridge ?? startHwlabKafkaEventBridge({ env, logger: options.logger ?? console, runtimeStore, kafkaFactory: options.kafkaFactory }); const server = createServer(async (request, response) => { const requestContext = buildHttpRequestContext(request); attachHttpRequestContext(request, response, requestContext); @@ -214,7 +206,7 @@ export function createCloudApiServer(options = {}) { response.once("close", () => emitHttpSpan("close")); await withBackendPerformanceContext(backendPerformance, async () => { try { - await routeRequest(request, response, { ...options, env, runtimeStore, gatewayRegistry, hwpodNodeWsRegistry, accessController, sessionRegistry, traceStore, codexStdioManager, codeAgentChatResults, webPerformanceStore, backendPerformanceStore, backendPerformance }); + await routeRequest(request, response, { ...options, env, runtimeStore, kafkaEventBridge, gatewayRegistry, hwpodNodeWsRegistry, accessController, sessionRegistry, traceStore, codexStdioManager, codeAgentChatResults, webPerformanceStore, backendPerformanceStore, backendPerformance, emitHttpRoutePhaseSpan }); } catch (error) { if (error?.alreadySent || response.headersSent || response.writableEnded) return; backendPerformance.recordPhase({ phase: "route_exception", durationMs: 0, outcome: "error" }); @@ -230,9 +222,10 @@ export function createCloudApiServer(options = {}) { }); server.on("close", () => { workbenchEmptySessionGc.stop(); - agentRunProjectionResume.stop(); void kafkaEventBridge.stop(); }); + server.hwlabStartupReady = kafkaEventBridge?.ready ?? Promise.resolve(); + server.hwlabAbortStartup = async () => { workbenchEmptySessionGc.stop(); await kafkaEventBridge.stop(); }; return server; } @@ -339,8 +332,9 @@ export async function buildHealthPayload(options = {}) { const dbProbe = await buildDbRuntimeReadiness(env, liveProbe ? { ...(options.dbProbe ?? {}), probe: false } : options.dbProbe); const codeAgent = await describeCodeAgentAvailability(env, options); const runtime = liveProbe ? runtimeReadinessSnapshot(runtimeStore) : await runtimeReadiness(runtimeStore); + const kafkaProjector = await kafkaProjectorHealth(options.kafkaEventBridge, { liveProbe }); const db = applyRuntimeDbReadinessLayers(dbProbe, runtime); - const readiness = buildCloudApiReadiness({ db, codeAgent, runtime }); + const readiness = buildCloudApiReadiness({ db, codeAgent, runtime, kafkaProjector }); return { serviceId, @@ -366,10 +360,21 @@ export async function buildHealthPayload(options = {}) { observedAt: new Date().toISOString(), db, codeAgent, - runtime + runtime, + kafkaProjector }; } +async function kafkaProjectorHealth(projector, { liveProbe = false } = {}) { + if (!projector?.started) return { started: false, reason: projector?.reason ?? "disabled", valuesRedacted: true }; + if (liveProbe) return { started: true, groupId: projector.groupId, agentRunTopic: projector.agentRunTopic, hwlabTopic: projector.hwlabTopic, ...(projector.startupStatus?.() ?? { status: "initializing" }), statusSource: "startup-state", valuesRedacted: true }; + try { + return { started: true, groupId: projector.groupId, agentRunTopic: projector.agentRunTopic, hwlabTopic: projector.hwlabTopic, ...(await projector.status()), valuesRedacted: true }; + } catch (error) { + return { started: true, status: "blocked", errorCode: error?.code ?? "hwlab_kafka_projector_status_failed", message: error?.message ?? "Kafka projector status query failed.", valuesRedacted: true }; + } +} + function runtimeReadinessSnapshot(runtimeStore) { if (runtimeStore && typeof runtimeStore.summary === "function") { try { @@ -610,7 +615,8 @@ async function handleRestAdapter(request, response, url, options) { observeHttpRoutePhase(request, options, "v1.runtime_readiness", () => runtimeReadinessSnapshot(options.runtimeStore)) ]); const db = applyRuntimeDbReadinessLayers(dbProbe, runtime); - const readiness = buildCloudApiReadiness({ db, codeAgent, runtime }); + const kafkaProjector = await kafkaProjectorHealth(options.kafkaEventBridge, { liveProbe: false }); + const readiness = buildCloudApiReadiness({ db, codeAgent, runtime, kafkaProjector }); sendJson(response, 200, { serviceId: CLOUD_API_SERVICE_ID, adapter: "rest", @@ -743,11 +749,6 @@ async function handleRestAdapter(request, response, url, options) { return; } - if (url.pathname === "/v1/workbench/debug/kafka-sse" || url.pathname.startsWith("/v1/workbench/debug/kafka-sse/")) { - await handleWorkbenchKafkaSseDebugHttp(request, response, url, options); - return; - } - if (url.pathname === "/v1/workbench/sync" || url.pathname === "/v1/workbench/sessions" || url.pathname.startsWith("/v1/workbench/sessions/") || url.pathname.startsWith("/v1/workbench/turns/") || url.pathname.startsWith("/v1/workbench/traces/")) { await handleWorkbenchReadModelHttp(request, response, url, options); return; @@ -979,7 +980,7 @@ function navIdForRestPath(pathname, method = "GET") { if (pathname === "/v1/project-management" || pathname.startsWith("/v1/project-management/")) return "project.mdtodo"; if (pathname === "/v1/api-keys" || pathname === "/v1/api-keys/default" || pathname.startsWith("/v1/api-keys/")) return "user.apiKeys"; if (pathname === "/v1/users/me/profile" || pathname === "/v1/users/me/password") return "system.settings"; - if (pathname === "/v1/workbench/debug/fake-sse" || pathname.startsWith("/v1/workbench/debug/fake-sse/") || pathname === "/v1/workbench/debug/kafka-sse" || pathname.startsWith("/v1/workbench/debug/kafka-sse/")) return "workbench.debug"; + if (pathname === "/v1/workbench/debug/fake-sse" || pathname.startsWith("/v1/workbench/debug/fake-sse/")) return "workbench.debug"; if (pathname === "/v1/workbench/events" || pathname === "/v1/workbench/sync" || pathname === "/v1/workbench/launches" || pathname === "/v1/workbench/sessions" || pathname.startsWith("/v1/workbench/sessions/") || pathname.startsWith("/v1/workbench/turns/") || pathname.startsWith("/v1/workbench/traces/")) return "workbench.code"; if (pathname === "/v1/agent/chat" || pathname === "/v1/agent/sessions" || pathname.startsWith("/v1/agent/sessions/") || pathname === "/v1/agent/chat/inspect" || pathname.startsWith("/v1/agent/chat/result/") || pathname.startsWith("/v1/agent/turns/") || pathname.startsWith("/v1/agent/traces/") || pathname === "/v1/agent/chat/cancel" || pathname === "/v1/agent/chat/steer") return "workbench.code"; if (pathname === "/v1/admin/provider-profiles" || pathname.startsWith("/v1/admin/provider-profiles/")) return "admin.providerProfiles"; @@ -994,492 +995,6 @@ function navIdForRestPath(pathname, method = "GET") { return ""; } -async function handleHwpodNodeOpsHttp(request, response, options) { - if (request.method === "GET") { - sendJson(response, 200, { - ok: true, - status: "ready", - contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, - route: "/v1/hwpod-node-ops", - specAuthority: "code-agent-workspace", - compiler: "hwpod-compiler-cli", - apiRole: "node-ops-forwarder", - nodeRole: "thin-hwpod-node-executor", - supportedOps: Array.from(HWPOD_NODE_OPS), - websocket: options.hwpodNodeWsRegistry.describe() - }); - return; - } - if (request.method !== "POST") { - sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "hwpod-node-ops only supports GET and POST" } }); - return; - } - - const body = await readJsonObject(request, options.bodyLimitBytes); - if (!body.ok) { - sendJson(response, 400, body.error); - return; - } - const validation = validateHwpodNodeOpsPlan(body.value); - if (!validation.ok) { - sendJson(response, 400, { - ok: false, - status: "rejected", - contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, - error: validation.error - }); - return; - } - - const plan = validation.plan; - const requestMeta = hwpodNodeOpsRequestMeta(request, options, "hwpod"); - try { - const handled = await dispatchHwpodNodeOpsPlan(plan, requestMeta, options, { request }); - sendJson(response, handled.httpStatus, handled.payload); - } catch (error) { - const summary = error?.message ?? "hwpod-node-ops handler failed"; - recordHwpodNodeOpsBlocked(request, options, plan, requestMeta, "hwpod_node_handler_failed", summary, { dispatchMode: "handler-exception" }); - sendJson(response, 200, hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, { dispatchMode: "handler-exception" })); - } -} - -function hwpodNodeOpsRequestMeta(request, options, label) { - const httpContext = request?.hwlabHttpRequestContext ?? {}; - return { - requestId: getHeader(request, "x-request-id") || httpContext.requestId || `req_${label}_${randomUUID()}`, - traceId: getHeader(request, "x-trace-id") || `trc_${label}_${randomUUID()}`, - serviceId: getHeader(request, "x-source-service-id") || CLOUD_API_SERVICE_ID, - environment: runtimeEnvironment(options.env ?? process.env), - otelTraceId: httpContext.traceId ?? null, - traceparent: httpContext.traceparent ?? null, - valuesPrinted: false - }; -} - -function handleHwlabNodeUpdateHttp(request, response, url, options) { - if (request.method !== "GET") { - sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "GET required." } }); - return; - } - const env = options.env ?? process.env; - const currentVersion = cleanText(url.searchParams.get("current") || url.searchParams.get("version")); - const channel = cleanText(url.searchParams.get("channel")) || "stable"; - const platform = cleanText(url.searchParams.get("platform")) || "unknown"; - const bundled = hwlabNodeBundledMetadata(env); - const latestVersion = cleanText(env.HWLAB_NODE_PY_LATEST_VERSION) || bundled.version || "0.1.0"; - const releaseNotesUrl = cleanText(env.HWLAB_NODE_PY_RELEASE_NOTES_URL) || null; - const downloadUrl = hwlabNodeDownloadUrl(env); - const sha256 = cleanText(env.HWLAB_NODE_PY_SHA256) || bundled.sha256 || null; - const newer = currentVersion ? compareVersionStrings(latestVersion, currentVersion) > 0 : Boolean(latestVersion); - const updateAvailable = Boolean(downloadUrl && newer); - sendJson(response, 200, { - ok: true, - contractVersion: "hwlab-node-update-v1", - serviceId: "hwlab-node", - route: "/v1/hwlab-node/update", - channel, - platform, - currentVersion: currentVersion || null, - latestVersion, - updateAvailable, - downloadUrl: updateAvailable ? downloadUrl : null, - sha256: updateAvailable ? sha256 : null, - releaseNotesUrl, - manualDefault: true, - autoApplyDefault: false, - checkIntervalSeconds: parsePositiveInteger(env.HWLAB_NODE_PY_UPDATE_INTERVAL_SECONDS, 300), - observedAt: new Date().toISOString() - }); -} - -function handleHwlabNodeDownloadHttp(request, response, options) { - if (request.method !== "GET") { - sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "GET required." } }); - return; - } - const bundled = hwlabNodeBundledMetadata(options.env ?? process.env); - if (!bundled.content) { - sendJson(response, 404, { ok: false, error: { code: "hwlab_node_bundle_missing", message: "bundled hwlab-node.py is not available" } }); - return; - } - const bytes = Buffer.from(bundled.content, "utf8"); - response.writeHead(200, { - "content-type": "text/x-python; charset=utf-8", - "cache-control": "no-store", - "x-hwlab-node-version": bundled.version || "unknown", - "x-hwlab-node-sha256": bundled.sha256 || "", - "content-length": String(bytes.length) - }); - response.end(bytes); -} - -function hwlabNodeBundledMetadata(env) { - const bundlePath = cleanText(env.HWLAB_NODE_PY_BUNDLE_PATH) || path.resolve(process.cwd(), "tools/hwlab-node.py"); - try { - const content = readFileSync(bundlePath, "utf8"); - const version = cleanText(content.match(/APP_VERSION\s*=\s*["']([^"']+)["']/u)?.[1]); - const sha256 = createHash("sha256").update(content, "utf8").digest("hex"); - return { path: bundlePath, content, version, sha256 }; - } catch { - return { path: bundlePath, content: "", version: "", sha256: "" }; - } -} - -function hwlabNodeDownloadUrl(env) { - const explicit = cleanText(env.HWLAB_NODE_PY_DOWNLOAD_URL); - if (explicit) return explicit; - const downloadPath = cleanText(env.HWLAB_NODE_PY_DOWNLOAD_PATH) || "/v1/hwlab-node/download/hwlab-node.py"; - if (/^https?:\/\//iu.test(downloadPath)) return downloadPath; - const publicEndpoint = cleanText(env.HWLAB_PUBLIC_ENDPOINT) || "https://hwlab.pikapython.com"; - try { - return new URL(downloadPath, publicEndpoint.endsWith("/") ? publicEndpoint : `${publicEndpoint}/`).toString(); - } catch { - return null; - } -} - -function compareVersionStrings(left, right) { - const a = versionParts(left); - const b = versionParts(right); - for (let index = 0; index < Math.max(a.length, b.length, 3); index += 1) { - const delta = (a[index] ?? 0) - (b[index] ?? 0); - if (delta !== 0) return delta > 0 ? 1 : -1; - } - return 0; -} - -function versionParts(value) { - return String(value ?? "") - .trim() - .replace(/^v/iu, "") - .split(/[.+-]/u) - .slice(0, 4) - .map((part) => parseInt(part, 10)) - .map((part) => (Number.isFinite(part) && part >= 0 ? part : 0)); -} - -function cleanText(value) { - const text = String(value ?? "").trim(); - return text || ""; -} - -async function handleHwpodSpecDiscoveryHttp(request, response, url, options) { - const probe = truthyFlag(url.searchParams.get("probe")); - const observedAt = new Date().toISOString(); - let specs = await discoverHwpodSpecs(options); - if (probe) { - specs = await Promise.all(specs.map((spec) => probeDiscoveredHwpodSpec(spec, { request, options, observedAt }))); - } - sendJson(response, 200, hwpodSpecDiscoveryPayload(specs, { observedAt })); -} - -async function probeDiscoveredHwpodSpec(spec, { request, options, observedAt }) { - if (spec.ok === false) return spec; - const plan = hwpodSpecWorkspaceProbePlan(spec); - const validation = validateHwpodNodeOpsPlan(plan); - if (!validation.ok) { - return { - ...spec, - availability: { - ok: false, - status: "invalid_probe_plan", - checkedAt: observedAt, - blocker: validation.error - } - }; - } - const requestMeta = hwpodNodeOpsRequestMeta(request, options, "hwpod_spec"); - const handled = await dispatchHwpodNodeOpsPlan(validation.plan, requestMeta, options, { request }); - const payload = handled.payload; - return { - ...spec, - availability: { - ok: payload.ok === true, - status: payload.ok === true ? "available" : "blocked", - checkedAt: observedAt, - probePlanId: payload.planId, - nodeOpsRoute: "/v1/hwpod-node-ops", - results: payload.results ?? [], - blocker: payload.blocker ?? null, - httpStatus: handled.httpStatus - } - }; -} - -async function dispatchHwpodNodeOpsPlan(plan, requestMeta, options, context = {}) { - const hwpodNodeOpsUrl = normalizedHwpodNodeOpsUrl(options.env ?? process.env); - const hwpodNodeWsRegistry = options.hwpodNodeWsRegistry; - const hasWsNode = hwpodNodeWsRegistry?.hasNode?.(plan.nodeId); - if (typeof options.hwpodNodeOpsHandler !== "function" && !hasWsNode && !hwpodNodeOpsUrl) { - const summary = "no outbound WebSocket hwpod-node is connected and HWLAB_HWPOD_NODE_OPS_URL is not configured; cloud-api is only validating the hwpod-node-ops contract"; - const details = { - dispatchMode: "none", - websocketRegistryConfigured: Boolean(hwpodNodeWsRegistry), - websocketConnected: false, - directUrlConfigured: false - }; - recordHwpodNodeOpsBlocked(context.request, options, plan, requestMeta, "hwpod_node_unavailable", summary, details); - return { - httpStatus: 200, - payload: hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, details) - }; - } - const handled = typeof options.hwpodNodeOpsHandler === "function" - ? await options.hwpodNodeOpsHandler(plan, { request: context.request, requestMeta, env: options.env ?? process.env }) - : hasWsNode - ? await hwpodNodeWsRegistry.dispatch(plan, requestMeta, { timeoutMs: parsePositiveInteger(options.env?.HWLAB_HWPOD_NODE_OPS_TIMEOUT_MS, 30000) }) - : await forwardHwpodNodeOpsPlan(hwpodNodeOpsUrl, plan, requestMeta, options); - const results = Array.isArray(handled?.results) ? handled.results : []; - const failed = results.some((item) => item?.ok === false); - const payload = attachHwpodNodeOpsDiagnostics({ - ok: handled?.ok ?? !failed, - status: handled?.status ?? (failed ? "failed" : "completed"), - contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, - planId: plan.planId, - hwpodId: plan.hwpodId, - nodeId: plan.nodeId, - acceptedOps: plan.ops.length, - results, - blocker: handled?.blocker ?? null, - requestMeta - }, requestMeta, { dispatchMode: hasWsNode ? "websocket" : hwpodNodeOpsUrl ? "direct-url" : "handler" }); - if (payload.ok === false && payload.status === "blocked" && payload.blocker?.code) { - recordHwpodNodeOpsBlocked(context.request, options, plan, requestMeta, payload.blocker.code, payload.blocker.summary ?? "hwpod-node-ops blocked", payload.blocker.details ?? {}); - } - return { httpStatus: handled?.httpStatus ?? (failed ? 409 : 200), payload }; -} - -async function forwardHwpodNodeOpsPlan(targetUrl, plan, requestMeta, options) { - const target = await describeDirectHwpodNode(targetUrl, options); - if (!target.ok || !target.nodeId) { - return directHwpodNodeBlocked(plan, "hwpod_node_identity_unverified", "configured hwpod-node URL did not expose a verifiable nodeId", { - requestedNodeId: plan.nodeId, - targetUrl: redactNodeOpsUrl(targetUrl), - dispatchMode: "direct-url", - targetStatus: target.status ?? null, - error: target.error ?? null - }, requestMeta); - } - if (target.ok && target.nodeId && target.nodeId !== plan.nodeId) { - return directHwpodNodeBlocked(plan, "hwpod_node_id_mismatch", `HWLAB_HWPOD_NODE_OPS_URL points to ${target.nodeId}, not requested node ${plan.nodeId}`, { - requestedNodeId: plan.nodeId, - targetNodeId: target.nodeId, - targetUrl: redactNodeOpsUrl(targetUrl), - dispatchMode: "direct-url" - }, requestMeta); - } - const response = await fetch(targetUrl, { - method: "POST", - headers: { - "content-type": "application/json", - "x-request-id": requestMeta.requestId, - "x-trace-id": requestMeta.traceId, - "x-source-service-id": CLOUD_API_SERVICE_ID, - ...(requestMeta.otelTraceId ? { "x-hwlab-otel-trace-id": requestMeta.otelTraceId } : {}), - ...(requestMeta.traceparent ? { traceparent: requestMeta.traceparent } : {}) - }, - body: JSON.stringify(plan), - signal: AbortSignal.timeout(parsePositiveInteger(options.env?.HWLAB_HWPOD_NODE_OPS_TIMEOUT_MS, 30000)) - }); - const body = await response.json().catch(() => null); - if (!body || typeof body !== "object") { - return directHwpodNodeBlocked(plan, "hwpod_node_response_invalid", `hwpod-node returned HTTP ${response.status} without a JSON object payload`, { - targetUrl: redactNodeOpsUrl(targetUrl), - dispatchMode: "direct-url", - targetStatus: response.status - }, requestMeta); - } - return { - ok: body.ok, - status: body.status, - httpStatus: response.status, - results: body.results, - blocker: body.blocker ?? null - }; -} - -function directHwpodNodeBlocked(plan, code, summary, details, requestMeta = {}) { - const blocker = hwpodNodeOpsBlocker({ code, layer: "hwpod-node", retryable: true, summary, details }, requestMeta, details); - return { - ok: false, - status: "blocked", - httpStatus: 200, - results: plan.ops.map((op) => ({ opId: op.opId, op: op.op, ok: false, status: "blocked", blocker })), - blocker, - otelTraceId: blocker.otelTraceId ?? null, - diagnostic: blocker.diagnostic ?? null - }; -} - -async function describeDirectHwpodNode(targetUrl, options) { - try { - const response = await fetch(targetUrl, { - method: "GET", - signal: AbortSignal.timeout(parsePositiveInteger(options.env?.HWLAB_HWPOD_NODE_OPS_TIMEOUT_MS, 30000)) - }); - const body = await response.json().catch(() => null); - return { ok: response.ok && body && typeof body === "object", status: response.status, nodeId: safeOpaqueId(body?.nodeId), body }; - } catch (error) { - return { ok: false, error: error instanceof Error ? error.message : String(error), nodeId: "" }; - } -} - -function redactNodeOpsUrl(value) { - try { - const parsed = new URL(value); - parsed.username = ""; - parsed.password = ""; - parsed.search = parsed.search ? "?redacted=1" : ""; - return parsed.toString(); - } catch { - return ""; - } -} - -function normalizedHwpodNodeOpsUrl(env = process.env) { - const direct = String(env.HWLAB_HWPOD_NODE_OPS_URL ?? "").trim(); - if (direct) return direct; - const base = String(env.HWLAB_HWPOD_NODE_URL ?? "").trim().replace(/\/+$/u, ""); - return base ? `${base}/v1/hwpod-node-ops` : ""; -} - - -function validateHwpodNodeOpsPlan(value) { - if (!value || typeof value !== "object" || Array.isArray(value)) { - return { ok: false, error: { code: "invalid_hwpod_node_ops_plan", message: "hwpod-node-ops body must be a JSON object" } }; - } - if (value.contractVersion !== HWPOD_NODE_OPS_CONTRACT_VERSION) { - return { ok: false, error: { code: "invalid_hwpod_node_ops_contract", message: `contractVersion must be ${HWPOD_NODE_OPS_CONTRACT_VERSION}`, actual: value.contractVersion ?? null } }; - } - const planId = safeOpaqueId(value.planId) || `hwpod_plan_${randomUUID()}`; - const nodeId = safeOpaqueId(value.nodeId); - const hwpodId = safeOpaqueId(value.hwpodId); - if (!nodeId) return { ok: false, error: { code: "invalid_hwpod_node_id", message: "nodeId is required" } }; - if (!hwpodId) return { ok: false, error: { code: "invalid_hwpod_id", message: "hwpodId is required" } }; - if (!Array.isArray(value.ops) || value.ops.length === 0) { - return { ok: false, error: { code: "invalid_hwpod_node_ops", message: "ops must be a non-empty array" } }; - } - const ops = []; - for (const [index, item] of value.ops.entries()) { - if (!item || typeof item !== "object" || Array.isArray(item)) { - return { ok: false, error: { code: "invalid_hwpod_node_op", message: `ops[${index}] must be a JSON object` } }; - } - const op = typeof item.op === "string" ? item.op.trim() : ""; - if (!HWPOD_NODE_OPS.has(op)) { - return { ok: false, error: { code: "unsupported_hwpod_node_op", message: `unsupported hwpod-node op: ${op || ""}`, supportedOps: Array.from(HWPOD_NODE_OPS) } }; - } - ops.push({ - opId: safeOpaqueId(item.opId) || `op_${index + 1}`, - op, - args: item.args && typeof item.args === "object" && !Array.isArray(item.args) ? item.args : {} - }); - } - return { - ok: true, - plan: { - ...value, - planId, - hwpodId, - nodeId, - ops - } - }; -} - -function hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, details = {}) { - const blocker = hwpodNodeOpsBlocker({ - code: "hwpod_node_unavailable", - layer: "hwpod-node", - retryable: true, - summary, - details, - userMessage: "hwpod-node 尚未接入执行面；cloud-api 已完成 hwpod-node-ops 合同校验。" - }, requestMeta, details); - return { - ok: false, - status: "blocked", - contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION, - planId: plan.planId, - hwpodId: plan.hwpodId, - nodeId: plan.nodeId, - acceptedOps: plan.ops.length, - results: plan.ops.map((op) => ({ - opId: op.opId, - op: op.op, - ok: false, - status: "blocked", - blocker - })), - blocker, - otelTraceId: blocker.otelTraceId ?? null, - diagnostic: blocker.diagnostic ?? null, - requestMeta - }; -} - -function hwpodNodeOpsBlocker(blocker, requestMeta = {}, details = {}) { - const code = cleanText(blocker?.code) || "hwpod_node_ops_blocked"; - const summary = cleanText(blocker?.summary ?? blocker?.message) || "hwpod-node-ops blocked"; - const otelTraceId = cleanText(blocker?.otelTraceId ?? requestMeta?.otelTraceId) || null; - const diagnostic = { - code, - rootCauseCode: code, - summary, - otelTraceId, - traceLine: otelTraceId ? `OTel traceId: ${otelTraceId}` : null, - details: details && typeof details === "object" ? details : {}, - valuesPrinted: false - }; - const userMessage = cleanText(blocker?.userMessage); - return { - ...blocker, - code, - layer: blocker?.layer ?? "hwpod-node", - retryable: blocker?.retryable ?? true, - summary, - ...(otelTraceId ? { otelTraceId } : {}), - diagnostic, - ...(userMessage ? { userMessage: `${userMessage}\n${diagnostic.traceLine ?? "OTel traceId: unavailable"}` } : {}) - }; -} - -function attachHwpodNodeOpsDiagnostics(payload, requestMeta, details = {}) { - if (!payload || typeof payload !== "object" || payload.ok !== false) return payload; - const topBlocker = payload.blocker ? hwpodNodeOpsBlocker(payload.blocker, requestMeta, payload.blocker.details ?? details) : null; - const results = Array.isArray(payload.results) - ? payload.results.map((result) => result?.blocker ? { ...result, blocker: hwpodNodeOpsBlocker(result.blocker, requestMeta, result.blocker.details ?? details) } : result) - : payload.results; - const diagnostic = topBlocker?.diagnostic ?? results?.find?.((result) => result?.blocker?.diagnostic)?.blocker?.diagnostic ?? null; - return { - ...payload, - results, - blocker: topBlocker, - ...(diagnostic?.otelTraceId ? { otelTraceId: diagnostic.otelTraceId } : {}), - ...(diagnostic ? { diagnostic } : {}) - }; -} - -function recordHwpodNodeOpsBlocked(request, options, plan, requestMeta, code, summary, details = {}) { - const httpContext = request?.hwlabHttpRequestContext; - const attributes = { - "hwlab.hwpod.plan_id": cleanText(plan?.planId) || null, - "hwlab.hwpod.hwpod_id": cleanText(plan?.hwpodId) || null, - "hwlab.hwpod.node_id": cleanText(plan?.nodeId) || null, - "hwlab.hwpod.accepted_ops": Array.isArray(plan?.ops) ? plan.ops.length : 0, - "hwlab.hwpod.ops": Array.isArray(plan?.ops) ? plan.ops.map((op) => cleanText(op?.op)).filter(Boolean).join(",").slice(0, 240) : null, - "hwlab.hwpod.blocker.code": code, - "hwlab.hwpod.blocker.summary": String(summary ?? "").slice(0, 500), - "hwlab.hwpod.dispatch_mode": cleanText(details?.dispatchMode) || "unknown", - "hwlab.hwpod.websocket_connected": Boolean(details?.websocketConnected), - "hwlab.hwpod.direct_url_configured": Boolean(details?.directUrlConfigured), - "hwlab.hwpod.otel_trace_id": cleanText(requestMeta?.otelTraceId) || null, - traceId: cleanText(requestMeta?.traceId) || null - }; - if (httpContext) httpContext.otelAttributes = { ...(httpContext.otelAttributes ?? {}), ...attributes }; - const error = Object.assign(new Error(summary), { code }); - emitHttpRoutePhaseSpan(request, options, "hwpod-node-ops.blocked", Date.now(), Date.now(), "error", error, attributes); -} - async function codeAgentOptions(request, response, options, authOptions = {}) { const auth = await options.accessController.authenticate(request, { required: authOptions.required ?? options.accessController.required }); if (!auth.ok) { diff --git a/internal/cloud/user-billing-integration.test.ts b/internal/cloud/user-billing-integration.test.ts index 41dbc4c1..bf2124d1 100644 --- a/internal/cloud/user-billing-integration.test.ts +++ b/internal/cloud/user-billing-integration.test.ts @@ -204,158 +204,6 @@ test("cloud api accepts user-billing API keys and records Code Agent billing usa } }); -test("cloud api defers AgentRun Code Agent billing record until terminal result", async () => { - const calls = []; - const agentRunCalls = []; - let eventPolls = 0; - let resultPolls = 0; - const agentRunServer = createHttpServer(async (request, response) => { - const url = new URL(request.url || "/", "http://127.0.0.1"); - const chunks = []; - for await (const chunk of request) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); - const body = chunks.length ? JSON.parse(Buffer.concat(chunks).toString("utf8")) : null; - agentRunCalls.push({ method: request.method, path: url.pathname, search: url.search, body }); - const send = (data) => { - response.writeHead(200, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: true, data, traceId: "trc_fake_agentrun_billing" })}\n`); - }; - if (request.method === "POST" && url.pathname === "/api/v1/runs") { - assert.equal(body.backendProfile, "deepseek"); - return send({ id: "run_billing_deferred", status: "pending", backendProfile: "deepseek", sessionRef: body.sessionRef, resourceBundleRef: body.resourceBundleRef }); - } - if (request.method === "POST" && url.pathname === "/api/v1/runs/run_billing_deferred/commands") { - assert.equal(body.type, "turn"); - assert.equal(body.idempotencyKey, "trc_user_billing_agentrun_terminal"); - return send({ id: "cmd_billing_deferred", runId: "run_billing_deferred", state: "pending", type: "turn", seq: 1 }); - } - if (request.method === "GET" && url.pathname === "/api/v1/runs/run_billing_deferred/commands") { - return send({ items: [{ id: "cmd_billing_deferred", runId: "run_billing_deferred", state: "running", type: "turn", seq: 1, idempotencyKey: "trc_user_billing_agentrun_terminal", payload: { traceId: "trc_user_billing_agentrun_terminal", conversationId: "cnv_user_billing_agentrun", hwlabSessionId: "ses_user_billing_agentrun", providerProfile: "deepseek" } }] }); - } - if (request.method === "POST" && url.pathname === "/api/v1/runs/run_billing_deferred/runner-jobs") { - assert.equal(body.commandId, "cmd_billing_deferred"); - return send({ - action: "create-kubernetes-job", - runId: "run_billing_deferred", - commandId: "cmd_billing_deferred", - attemptId: "attempt_billing_deferred", - runnerId: "runner_billing_deferred", - namespace: "agentrun-v02", - jobName: "agentrun-v01-runner-billing-deferred" - }); - } - if (request.method === "GET" && url.pathname === "/api/v1/runs/run_billing_deferred/events") { - eventPolls += 1; - return send({ items: eventPolls > 1 ? [{ id: "evt_done", runId: "run_billing_deferred", seq: 1, type: "terminal_status", payload: { commandId: "cmd_billing_deferred", terminalStatus: "completed" }, createdAt: "2026-06-14T04:20:00.000Z" }] : [] }); - } - if (request.method === "GET" && url.pathname === "/api/v1/runs/run_billing_deferred/commands/cmd_billing_deferred/result") { - resultPolls += 1; - if (resultPolls === 1) { - return send({ runId: "run_billing_deferred", commandId: "cmd_billing_deferred", status: "running", runStatus: "claimed", commandState: "running", terminalStatus: null }); - } - return send({ - runId: "run_billing_deferred", - commandId: "cmd_billing_deferred", - attemptId: "attempt_billing_deferred", - runnerId: "runner_billing_deferred", - jobName: "agentrun-v01-runner-billing-deferred", - namespace: "agentrun-v02", - status: "completed", - runStatus: "completed", - commandState: "completed", - terminalStatus: "completed", - completed: true, - reply: "AgentRun billing completed.", - lastSeq: 1, - eventCount: 1, - sessionRef: { sessionId: "ses_agentrun_deepseek_billing", conversationId: "cnv_user_billing_agentrun", threadId: "thr_billing" } - }); - } - response.writeHead(404, { "content-type": "application/json" }); - response.end(`${JSON.stringify({ ok: false, failureKind: "unexpected", message: `${request.method} ${url.pathname}` })}\n`); - }); - await new Promise((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve)); - - const userBillingClient = { - configured: true, - async introspect(token) { - calls.push({ op: "introspect", tokenPrefix: token.slice(0, 8) }); - assert.equal(token, "hwl_user_billing_agentrun_secret"); - return { ok: true, status: 200, body: { active: true, principal: { userId: "usr_user_billing_agentrun", email: "agentrun@hwlab.local", username: "billing-agentrun", role: "user", scopes: ["api"], authType: "api-key", keyId: "key_user_billing_agentrun" } } }; - }, - async billingPreflight(body) { - calls.push({ op: "preflight", body }); - assert.equal(body.apiKey, "hwl_user_billing_agentrun_secret"); - assert.equal(body.idempotencyKey, "code-agent:trc_user_billing_agentrun_terminal:preflight"); - return { ok: true, status: 200, body: { allowed: true, reservationId: "res_user_billing_agentrun", estimatedCredits: 1, expiresAt: "2026-06-14T05:00:00.000Z" } }; - }, - async billingRecord(body) { - calls.push({ op: "record", body }); - assert.equal(body.reservationId, "res_user_billing_agentrun"); - assert.equal(body.idempotencyKey, "code-agent:trc_user_billing_agentrun_terminal:record"); - assert.equal(body.metadata.status, "completed"); - return { ok: true, status: 200, body: { recordId: "use_user_billing_agentrun", credits: 1, balance: 9 } }; - }, - async billingRelease() { - throw new Error("completed AgentRun billing should record usage instead of releasing reservation"); - } - }; - const { port: agentRunPort } = agentRunServer.address(); - const server = createCloudApiServer({ - traceStore: createCodeAgentTraceStore(), - env: { - HWLAB_ACCESS_CONTROL_REQUIRED: "1", - HWLAB_USER_BILLING_CODE_AGENT_ENABLED: "1", - HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", - AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, - AGENTRUN_API_KEY: "test-agentrun-key", - HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1", - HWLAB_CODE_AGENT_AGENTRUN_SOURCE_COMMIT: "0123456789abcdef0123456789abcdef01234567", - HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601", - HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_POLL_INTERVAL_MS: "10", - HWLAB_CODE_AGENT_AGENTRUN_SESSION_STORAGE: "metadata-only", - HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE: "deepseek" - }, - userBillingClient - }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - - try { - const { port } = server.address(); - const authHeader = { authorization: "Bearer hwl_user_billing_agentrun_secret" }; - const sessionResponse = await fetch(`http://127.0.0.1:${port}/v1/agent/sessions`, { - method: "POST", - headers: { "content-type": "application/json", ...authHeader }, - body: JSON.stringify({ conversationId: "cnv_user_billing_agentrun", sessionId: "ses_user_billing_agentrun", providerProfile: "deepseek" }) - }); - assert.equal(sessionResponse.status, 201); - - const submit = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, { - method: "POST", - headers: { "content-type": "application/json", authorization: "Bearer hwl_user_billing_agentrun_secret", "x-trace-id": "trc_user_billing_agentrun_terminal", prefer: "respond-async" }, - body: JSON.stringify({ conversationId: "cnv_user_billing_agentrun", sessionId: "ses_user_billing_agentrun", shortConnection: true, message: "billing AgentRun terminal smoke" }) - }); - assert.equal(submit.status, 202); - await waitForUserBillingCondition(() => agentRunCalls.some((call) => call.path === "/api/v1/runs/run_billing_deferred/runner-jobs")); - assert.deepEqual(calls.map((call) => call.op), ["introspect", "introspect", "preflight"]); - - const running = await fetch(`http://127.0.0.1:${port}/v1/agent/chat/result/trc_user_billing_agentrun_terminal`, { headers: authHeader }); - assert.equal(running.status === 202 || running.status === 200, true); - assert.equal(calls.some((call) => call.op === "record"), false); - - const payload = running.status === 200 ? await running.json() : await pollUserBillingAgentResult(port, "trc_user_billing_agentrun_terminal", authHeader); - assert.equal(payload.status, "completed"); - assert.equal(payload.billing.recorded, true); - assert.equal(payload.billing.reservationId, "res_user_billing_agentrun"); - assert.equal(JSON.stringify(payload).includes("userBillingReservation"), false); - assert.equal(calls.filter((call) => call.op === "preflight").length, 1); - assert.equal(calls.filter((call) => call.op === "record").length, 1); - assert.equal(calls.some((call) => call.op === "release"), false); - } finally { - await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); - await new Promise((resolve, reject) => agentRunServer.close((error) => (error ? reject(error) : resolve()))); - } -}); - test("AgentRun persisted trace evidence restores billing reservation without exposing it on agentRun", async () => { const restored = await loadPersistedAgentRunResult("trc_user_billing_agentrun_restored", { env: { HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01" }, diff --git a/internal/cloud/workbench-facts-store.ts b/internal/cloud/workbench-facts-store.ts index e3cd815b..d5b0b99f 100644 --- a/internal/cloud/workbench-facts-store.ts +++ b/internal/cloud/workbench-facts-store.ts @@ -2,11 +2,10 @@ * SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-18-p0-unique-projection; draft-2026-06-20-p1-zero-split-durable-realtime; PJ2026-010403 API契约 draft-2026-06-18-r1. * 职责: Workbench durable facts store adapter。统一读取 session/message/turn/trace projection facts，不在 GET 中推进 AgentRun facts。 */ -import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; import { emitCodeAgentOtelSpan } from "./otel-trace.ts"; import { safeConversationId, safeSessionId, safeTraceId } from "./server-http-utils.ts"; import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts"; -import { durableTraceStatus, normalizeWorkbenchStatus, TERMINAL_STATUSES, traceTerminalEvidence } from "./workbench-turn-projection.ts"; +import { durableTraceStatus, TERMINAL_STATUSES, traceTerminalEvidence } from "./workbench-turn-projection.ts"; const TERMINAL_DURABLE_TRACE_CACHE_MAX = 256; const DEFAULT_WORKBENCH_FACTS_QUERY_MAX_ATTEMPTS = 5; @@ -17,9 +16,7 @@ const terminalDurableTraceCache = new Map(); export function createWorkbenchFactsStore(options = {}, actor = null) { const accessStore = options.accessController?.store ?? options.accessController ?? null; const traceSessionCache = options.codeAgentTraceSessionCache instanceof Map ? options.codeAgentTraceSessionCache : null; - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - const runtimeStore = options.runtimeStore ?? null; - const runtimeReader = options.workbenchRuntime ?? createWorkbenchRuntimeClient({ env: options.env ?? process.env, fetch: options.fetch, logger: options.logger }) ?? runtimeStore; + const runtimeReader = options.workbenchRuntime ?? createWorkbenchRuntimeClient({ env: options.env ?? process.env, fetch: options.fetch, logger: options.logger }); const results = options.codeAgentChatResults ?? null; async function queryFacts(params = {}) { @@ -135,18 +132,8 @@ export function createWorkbenchFactsStore(options = {}, actor = null) { return safeId ? results?.get?.(safeId) ?? null : null; } - function traceSnapshotSync(traceId) { - return traceStore.snapshot(traceId); - } - async function traceSnapshot(traceId) { - const memory = traceSnapshotSync(traceId); - if (hasTraceProjection(memory) && TERMINAL_STATUSES.has(normalizeStatus(memory.status))) return memory; - const durable = await durableTraceSnapshot(runtimeReader, traceId); - if (isProjectionDiagnosticTrace(durable) && hasTraceProjection(memory)) return mergeTraceProjectionDiagnostic(memory, durable); - if (durable && shouldPreferDurableTrace(memory, durable)) return durable; - if (hasTraceProjection(memory)) return memory; - return durable ?? memory; + return durableTraceSnapshot(runtimeReader, traceId); } return { @@ -157,7 +144,6 @@ export function createWorkbenchFactsStore(options = {}, actor = null) { listSessions, resultForTrace, traceSnapshot, - traceSnapshotSync, canReadOwner: (ownerUserId) => canActorReadOwner(ownerUserId, actor) }; } @@ -391,41 +377,6 @@ function finiteNonNegativeNumber(value) { return Number.isFinite(parsed) && parsed >= 0 ? Math.trunc(parsed) : null; } -function isProjectionDiagnosticTrace(snapshot) { - return Boolean(snapshot?.projection?.blocker || snapshot?.projectionHealth || snapshot?.blocker); -} - -function mergeTraceProjectionDiagnostic(memory, diagnostic) { - return { - ...memory, - projection: diagnostic.projection ?? memory?.projection ?? null, - projectionStatus: diagnostic.projectionStatus ?? diagnostic.projection?.projectionStatus ?? memory?.projectionStatus ?? null, - projectionHealth: diagnostic.projectionHealth ?? diagnostic.projection?.projectionHealth ?? memory?.projectionHealth ?? null, - blocker: diagnostic.blocker ?? diagnostic.projection?.blocker ?? memory?.blocker ?? null, - valuesPrinted: false - }; -} - -function hasTraceProjection(snapshot) { - return Boolean(snapshot?.status !== "missing" && (Number(snapshot?.eventCount ?? 0) > 0 || snapshot?.lastEvent)); -} - -function shouldPreferDurableTrace(memory, durable) { - if (!durable) return false; - if (!hasTraceProjection(memory)) return true; - const memoryStatus = normalizeWorkbenchStatus(memory?.status); - const durableStatus = normalizeWorkbenchStatus(durable?.status); - if (TERMINAL_STATUSES.has(durableStatus) && !TERMINAL_STATUSES.has(memoryStatus)) return true; - return traceLastSeq(durable) > traceLastSeq(memory); -} - -function traceLastSeq(snapshot) { - const events = Array.isArray(snapshot?.events) ? snapshot.events : []; - const lastEvent = snapshot?.lastEvent ?? events.at(-1) ?? null; - const indexedMax = events.reduce((max, event, index) => Math.max(max, eventSeq(event, index)), 0); - return Math.max(indexedMax, lastEvent ? eventSeq(lastEvent, Math.max(0, events.length - 1)) : 0); -} - function canActorReadSession(session, actor) { if (!session || !actor) return false; if (normalizeStatus(session.status) === "archived") return false; diff --git a/internal/cloud/workbench-kafka-sse-debug.test.ts b/internal/cloud/workbench-kafka-sse-debug.test.ts deleted file mode 100644 index 62f54c46..00000000 --- a/internal/cloud/workbench-kafka-sse-debug.test.ts +++ /dev/null @@ -1,172 +0,0 @@ -import assert from "node:assert/strict"; -import { createServer } from "node:http"; -import { test } from "bun:test"; - -import { handleWorkbenchKafkaSseDebugHttp } from "./workbench-kafka-sse-debug.ts"; - -test("workbench Kafka SSE debug endpoint streams filtered raw HWLAB Kafka events", async () => { - const fakeKafka = createFakeKafkaFactory(); - const env = { - HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", - HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", - HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1" - }; - const server = createServer((request, response) => { - const url = new URL(request.url ?? "/", "http://127.0.0.1"); - void handleWorkbenchKafkaSseDebugHttp(request, response, url, { env, kafkaFactory: fakeKafka.factory, logger: null }); - }); - await listen(server); - const abort = new AbortController(); - try { - const response = await fetch(`${serverUrl(server)}/v1/workbench/debug/kafka-sse/events?stream=hwlab&sessionId=ses_kafka_sse_debug`, { signal: abort.signal }); - assert.equal(response.status, 200); - assert.match(response.headers.get("content-type") ?? "", /text\/event-stream/u); - const reader = response.body?.getReader(); - assert.ok(reader, "SSE response must expose a readable stream"); - const connected = await readUntil(reader, "hwlab.kafka.connected"); - assert.match(connected, /hwlab\.event\.v1/u); - - await fakeKafka.emit({ eventType: "hwlab.trace.event.projected", sessionId: "ses_other", event: { label: "ignored" } }); - await fakeKafka.emit({ eventType: "hwlab.trace.event.projected", sessionId: "ses_kafka_sse_debug", traceId: "trc_kafka_sse_debug", context: { runId: "run_kafka_sse_debug", commandId: "cmd_kafka_sse_debug" }, event: { label: "agentrun:event:debug", status: "running" } }); - const streamed = await readUntil(reader, "trc_kafka_sse_debug"); - assert.match(streamed, /hwlab\.kafka\.event/u); - assert.match(streamed, /ses_kafka_sse_debug/u); - assert.doesNotMatch(streamed, /ses_other/u); - } finally { - abort.abort(); - await close(server); - } -}); - -test("workbench Kafka SSE debug endpoint streams filtered codex stdio Kafka events", async () => { - const fakeKafka = createFakeKafkaFactory(); - const env = { - HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", - HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", - HWLAB_KAFKA_STDIO_TOPIC: "codex-stdio.raw.v1" - }; - const server = createServer((request, response) => { - const url = new URL(request.url ?? "/", "http://127.0.0.1"); - void handleWorkbenchKafkaSseDebugHttp(request, response, url, { env, kafkaFactory: fakeKafka.factory, logger: null }); - }); - await listen(server); - const abort = new AbortController(); - try { - const response = await fetch(`${serverUrl(server)}/v1/workbench/debug/kafka-sse/events?stream=stdio&traceId=trc_stdio_sse_debug&fromBeginning=true`, { signal: abort.signal }); - assert.equal(response.status, 200); - const reader = response.body?.getReader(); - assert.ok(reader, "SSE response must expose a readable stream"); - const connected = await readUntil(reader, "hwlab.kafka.connected"); - assert.match(connected, /codex-stdio\.raw\.v1/u); - - await fakeKafka.emit({ traceId: "trc_other", stream: "stdout", line: "ignored" }); - await fakeKafka.emit({ trace_id: "trc_stdio_sse_debug", metadata: { session_id: "ses_stdio_sse_debug", run_id: "run_stdio_sse_debug" }, stream: "stdout", line: "visible stdio frame" }); - const streamed = await readUntil(reader, "visible stdio frame"); - assert.match(streamed, /hwlab\.kafka\.event/u); - assert.match(streamed, /codex-stdio\.raw\.v1/u); - assert.match(streamed, /ses_stdio_sse_debug/u); - assert.doesNotMatch(streamed, /trc_other/u); - } finally { - abort.abort(); - await close(server); - } -}); - -test("workbench Kafka SSE debug endpoint resolves traceId to AgentRun Kafka ids", async () => { - const fakeKafka = createFakeKafkaFactory(); - const traceStore = { - snapshot: () => ({ - traceId: "trc_resolved_kafka_debug", - events: [ - { type: "backend", runId: "run_resolved_kafka_debug", commandId: "cmd_resolved_kafka_debug", sessionId: "ses_agentrun_resolved_kafka_debug" } - ], - lastEvent: null - }) - }; - const env = { - HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", - HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", - HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1" - }; - const server = createServer((request, response) => { - const url = new URL(request.url ?? "/", "http://127.0.0.1"); - void handleWorkbenchKafkaSseDebugHttp(request, response, url, { env, kafkaFactory: fakeKafka.factory, traceStore, logger: null }); - }); - await listen(server); - const abort = new AbortController(); - try { - const response = await fetch(`${serverUrl(server)}/v1/workbench/debug/kafka-sse/events?stream=hwlab&traceId=trc_resolved_kafka_debug`, { signal: abort.signal }); - assert.equal(response.status, 200); - const reader = response.body?.getReader(); - assert.ok(reader, "SSE response must expose a readable stream"); - const connected = await readUntil(reader, "resolvedFilters"); - assert.match(connected, /trc_resolved_kafka_debug/u); - assert.match(connected, /run_resolved_kafka_debug/u); - - await fakeKafka.emit({ eventType: "hwlab.trace.event.projected", traceId: null, sessionId: null, context: { runId: "run_other", commandId: "cmd_other" }, event: { label: "ignored" } }); - await fakeKafka.emit({ eventType: "hwlab.trace.event.projected", traceId: null, sessionId: null, context: { runId: "run_resolved_kafka_debug", commandId: "cmd_resolved_kafka_debug" }, event: { label: "resolved", status: "running" } }); - const streamed = await readUntil(reader, "run_resolved_kafka_debug"); - assert.match(streamed, /hwlab\.kafka\.event/u); - assert.match(streamed, /resolved/u); - assert.doesNotMatch(streamed, /run_other/u); - } finally { - abort.abort(); - await close(server); - } -}); - -function createFakeKafkaFactory() { - let eachMessage: ((input: any) => Promise) | null = null; - let subscribedTopic = "hwlab.event.v1"; - const consumer = { - connect: async () => undefined, - subscribe: async (input: { topic?: string }) => { subscribedTopic = String(input.topic ?? subscribedTopic); }, - run: async (input: any) => { eachMessage = input.eachMessage; }, - stop: async () => undefined, - disconnect: async () => undefined - }; - return { - factory: () => ({ consumer: () => consumer }), - emit: async (value: Record) => { - assert.ok(eachMessage, "consumer.run must be called before emitting fake Kafka events"); - await eachMessage({ - topic: subscribedTopic, - partition: 0, - message: { - offset: String(value.sessionId === "ses_other" ? 1 : 2), - key: Buffer.from(String(value.sessionId ?? "unknown")), - timestamp: "2026-07-09T18:30:00.000Z", - value: Buffer.from(JSON.stringify(value)) - } - }); - } - }; -} - -async function readUntil(reader: ReadableStreamDefaultReader, pattern: string): Promise { - const decoder = new TextDecoder(); - let text = ""; - const deadline = Date.now() + 3000; - while (!text.includes(pattern)) { - if (Date.now() > deadline) throw new Error(`SSE stream did not include ${pattern}: ${text}`); - const next = await reader.read(); - if (next.done) break; - text += decoder.decode(next.value, { stream: true }); - } - return text; -} - -async function listen(server: ReturnType) { - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); -} - -async function close(server: ReturnType) { - await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); -} - -function serverUrl(server: ReturnType): string { - const address = server.address(); - assert.equal(typeof address, "object"); - assert.ok(address && typeof address.port === "number"); - return `http://127.0.0.1:${address.port}`; -} diff --git a/internal/cloud/workbench-kafka-sse-debug.ts b/internal/cloud/workbench-kafka-sse-debug.ts deleted file mode 100644 index ca7df2b7..00000000 --- a/internal/cloud/workbench-kafka-sse-debug.ts +++ /dev/null @@ -1,216 +0,0 @@ -/* - * SPEC: PJ2026-0104010803 Workbench事件流可见性 draft-2026-07-09-p0-kafka-authority. - * Responsibility: debug-only SSE passthrough for HWLAB Kafka event envelopes. - */ -import { sendJson } from "./server-http-utils.ts"; -import { openKafkaEventStream } from "./kafka-event-bridge.ts"; -import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; - -const CONTRACT_VERSION = "workbench-debug-kafka-sse-v1"; -const DEFAULT_STREAM = "hwlab"; -const ALLOWED_STREAMS = new Set(["stdio", "agentrun", "hwlab"]); - -export async function handleWorkbenchKafkaSseDebugHttp(request, response, url, options = {}) { - const route = routeSuffix(url.pathname); - try { - if (route === "") { - if (request.method !== "GET") return methodNotAllowed(response, "GET"); - return sendJson(response, 200, describeKafkaSseDebug(url, options.env ?? process.env)); - } - if (route === "/events") { - if (request.method !== "GET") return methodNotAllowed(response, "GET"); - return openKafkaDebugSse(request, response, url, options); - } - return sendJson(response, 404, debugError("workbench_debug_kafka_sse_route_not_found", "Workbench debug Kafka SSE route is not implemented.", { route })); - } catch (error) { - options.logger?.warn?.({ - event: "workbench_debug_kafka_sse_failed", - route, - errorName: error?.name ?? "Error", - message: error instanceof Error ? error.message : String(error ?? "unknown"), - valuesRedacted: true - }); - if (response.headersSent) return writeSse(response, "hwlab.kafka.error", { ok: false, error: errorMessagePayload(error) }); - return sendJson(response, 500, debugError("workbench_debug_kafka_sse_failed", "Workbench debug Kafka SSE request failed.")); - } -} - -function describeKafkaSseDebug(url, env) { - const stream = streamFromUrl(url); - const requestedFilters = filtersFromUrl(url); - const resolvedFilters = resolveKafkaDebugFilters(requestedFilters, { traceStore: defaultCodeAgentTraceStore }); - return { - ok: true, - contractVersion: CONTRACT_VERSION, - stream, - topic: topicForStream(stream, env), - filters: requestedFilters, - resolvedFilters, - eventsRoute: `/v1/workbench/debug/kafka-sse/events${url.search || ""}`, - valuesPrinted: false - }; -} - -async function openKafkaDebugSse(request, response, url, options) { - const env = options.env ?? process.env; - const stream = streamFromUrl(url); - const filters = filtersFromUrl(url); - const resolvedFilters = resolveKafkaDebugFilters(filters, options); - response.writeHead(200, { - "content-type": "text/event-stream; charset=utf-8", - "cache-control": "no-store, no-transform", - connection: "keep-alive", - "x-accel-buffering": "no", - "x-content-type-options": "nosniff" - }); - if (typeof response.flushHeaders === "function") response.flushHeaders(); - writeSse(response, "hwlab.kafka.connected", { - ok: true, - contractVersion: CONTRACT_VERSION, - stream, - topic: topicForStream(stream, env), - filters, - resolvedFilters, - serverSentAt: new Date().toISOString(), - valuesPrinted: false - }); - let kafkaStream = null; - let closed = false; - const close = () => { - if (closed) return; - closed = true; - void kafkaStream?.stop?.(); - }; - request.on("close", close); - try { - kafkaStream = await openKafkaEventStream({ - env, - stream, - fromBeginning: url.searchParams.get("fromBeginning") === "1" || url.searchParams.get("fromBeginning") === "true", - ...resolvedFilters, - kafkaFactory: options.kafkaFactory, - onEvent: async (record) => { - writeSse(response, "hwlab.kafka.event", { - ok: true, - contractVersion: CONTRACT_VERSION, - stream, - topic: record.topic, - partition: record.partition, - offset: record.offset, - key: record.key, - timestamp: record.timestamp, - value: record.value, - serverSentAt: new Date().toISOString(), - valuesPrinted: false - }); - }, - onError: (error) => writeSse(response, "hwlab.kafka.error", { ok: false, error: errorMessagePayload(error), valuesPrinted: false }) - }); - } catch (error) { - writeSse(response, "hwlab.kafka.error", { ok: false, error: errorMessagePayload(error), valuesPrinted: false }); - } -} - -function resolveKafkaDebugFilters(filters = {}, options = {}) { - const traceId = textValue(filters.traceId); - if (!traceId) return filters; - const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; - const snapshot = typeof traceStore?.snapshot === "function" ? traceStore.snapshot(traceId) : null; - const resolved = collectTraceLinkedIds(snapshot); - const hasResolvedKafkaKey = Boolean(resolved.runId || resolved.commandId || resolved.sessionId); - if (!hasResolvedKafkaKey) return filters; - const resolvedSessionId = resolved.runId || resolved.commandId ? null : resolved.sessionId; - return compactObject({ - sessionId: filters.sessionId || resolvedSessionId, - runId: filters.runId || resolved.runId, - commandId: filters.commandId || resolved.commandId - }); -} - -function collectTraceLinkedIds(snapshot) { - const out = { sessionId: null, runId: null, commandId: null }; - const events = Array.isArray(snapshot?.events) ? snapshot.events : []; - for (const event of events) collectIdsFromRecord(event, out); - collectIdsFromRecord(snapshot?.lastEvent, out); - return out; -} - -function collectIdsFromRecord(record, out) { - const value = record && typeof record === "object" && !Array.isArray(record) ? record : {}; - const agentRun = value.agentRun && typeof value.agentRun === "object" && !Array.isArray(value.agentRun) ? value.agentRun : {}; - const payload = value.payload && typeof value.payload === "object" && !Array.isArray(value.payload) ? value.payload : {}; - out.sessionId ||= textValue(value.sessionId ?? value.sourceSessionId ?? agentRun.sessionId ?? payload.sessionId); - out.runId ||= textValue(value.runId ?? value.sourceRunId ?? agentRun.runId ?? payload.runId); - out.commandId ||= textValue(value.commandId ?? value.sourceCommandId ?? agentRun.commandId ?? payload.commandId); -} - -function writeSse(response, eventName, payload) { - if (response.destroyed || response.writableEnded) return false; - try { - response.write(`event: ${eventName}\n`); - const id = sseEventId(payload); - if (id) response.write(`id: ${id}\n`); - response.write(`data: ${JSON.stringify(payload)}\n\n`); - return true; - } catch { - return false; - } -} - -function routeSuffix(pathname) { - return String(pathname || "").replace(/^\/v1\/workbench\/debug\/kafka-sse/u, ""); -} - -function streamFromUrl(url) { - const stream = String(url.searchParams.get("stream") || DEFAULT_STREAM).trim(); - return ALLOWED_STREAMS.has(stream) ? stream : DEFAULT_STREAM; -} - -function filtersFromUrl(url) { - return compactObject({ - traceId: safeId(url.searchParams.get("traceId") || url.searchParams.get("trace-id")), - sessionId: safeId(url.searchParams.get("sessionId") || url.searchParams.get("session-id")), - runId: safeId(url.searchParams.get("runId") || url.searchParams.get("run-id")), - commandId: safeId(url.searchParams.get("commandId") || url.searchParams.get("command-id")) - }); -} - -function topicForStream(stream, env) { - if (stream === "stdio") return textValue(env.HWLAB_KAFKA_STDIO_TOPIC ?? env.AGENTRUN_KAFKA_STDIO_TOPIC) || "codex-stdio.raw.v1"; - if (stream === "agentrun") return textValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC) || "agentrun.event.v1"; - return textValue(env.HWLAB_KAFKA_EVENT_TOPIC) || "hwlab.event.v1"; -} - -function methodNotAllowed(response, allow) { - response.setHeader("allow", allow); - return sendJson(response, 405, debugError("method_not_allowed", `Method not allowed; expected ${allow}.`)); -} - -function debugError(code, message, extra = {}) { - return { ok: false, error: { code, message, ...extra }, valuesPrinted: false }; -} - -function errorMessagePayload(error) { - return { code: "kafka_sse_error", message: error instanceof Error ? error.message : String(error ?? "unknown") }; -} - -function safeId(value) { - const text = textValue(value); - return text && /^[A-Za-z0-9_.:-]{3,220}$/u.test(text) ? text : null; -} - -function textValue(value) { - const text = typeof value === "string" ? value.trim() : value === null || value === undefined ? "" : String(value).trim(); - return text.length > 0 ? text : null; -} - -function compactObject(value) { - return Object.fromEntries(Object.entries(value).filter(([, entry]) => textValue(entry))); -} - -function sseEventId(payload) { - const topic = textValue(payload?.topic); - const partition = textValue(payload?.partition); - const offset = textValue(payload?.offset); - return topic && partition && offset ? `${topic}:${partition}:${offset}` : null; -} diff --git a/internal/cloud/workbench-projection-cursor.test.ts b/internal/cloud/workbench-projection-cursor.test.ts deleted file mode 100644 index 03d20970..00000000 --- a/internal/cloud/workbench-projection-cursor.test.ts +++ /dev/null @@ -1,153 +0,0 @@ -// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor. -// Responsibility: Independent backend unit tests for AgentRun projection cursor planning. - -import assert from "node:assert/strict"; -import { test } from "bun:test"; - -import { buildAgentRunProjectionEventsFetchPlan, buildWorkbenchProjectionStateUpdate } from "./workbench-projection-cursor.ts"; - -test("AgentRun projection fetch plan uses durable cursor without scanning long trace", () => { - const poisonTrace = {}; - Object.defineProperty(poisonTrace, "events", { - get() { - throw new Error("trace events must not be read to compute AgentRun afterSeq"); - } - }); - const poisonResult = {}; - Object.defineProperty(poisonResult, "events", { - get() { - throw new Error("result events must not be read to compute AgentRun afterSeq"); - } - }); - - const plan = buildAgentRunProjectionEventsFetchPlan({ - projectionState: { - traceId: "trc_cursor_o1", - sourceRunId: "run_cursor_o1", - sourceCommandId: "cmd_cursor_o1", - lastSourceSeq: 3700, - lastProjectedSeq: 129 - }, - agentRun: { - runId: "run_cursor_o1", - commandId: "cmd_cursor_o1", - lastSeq: 12 - }, - trace: poisonTrace, - result: poisonResult, - pageLimit: 500 - }); - - assert.equal(plan.runId, "run_cursor_o1"); - assert.equal(plan.commandId, "cmd_cursor_o1"); - assert.equal(plan.afterSeq, 3700); - assert.equal(plan.limit, 500); - assert.equal(plan.cursorSource, "projection-state"); -}); - -test("AgentRun projection state advances from incremental events response", () => { - const state = buildWorkbenchProjectionStateUpdate({ - currentState: { - traceId: "trc_cursor_update", - sourceRunId: "run_cursor_update", - sourceCommandId: "cmd_cursor_update", - lastSourceSeq: 3700, - lastProjectedSeq: 3700, - sourceLatestSeq: 3700, - projectionStatus: "projecting", - resultSyncState: "not_started", - createdAt: "2026-06-19T12:00:00.000Z" - }, - agentRun: { - runId: "run_cursor_update", - commandId: "cmd_cursor_update", - lastSeq: 3700 - }, - eventsResponse: { - traceLastSeq: 3720, - maxSeq: 3721, - rawEventCount: 20 - }, - now: () => "2026-06-19T12:01:00.000Z" - }); - - assert.equal(state.lastSourceSeq, 3720); - assert.equal(state.lastAgentRunSeq, 3720); - assert.equal(state.lastProjectedSeq, 3720); - assert.equal(state.sourceLatestSeq, 3721); - assert.equal(state.projectionStatus, "projecting"); - assert.equal(state.resultSyncState, "not_started"); - assert.equal(state.createdAt, "2026-06-19T12:00:00.000Z"); - assert.equal(state.updatedAt, "2026-06-19T12:01:00.000Z"); -}); - -test("AgentRun projection state preserves retry params for background resume", () => { - const state = buildWorkbenchProjectionStateUpdate({ - currentState: { - traceId: "trc_cursor_retry_params", - sourceRunId: "run_cursor_retry_params", - sourceCommandId: "cmd_cursor_retry_params", - retryParams: { - message: "preserve this prompt for fresh-session retry", - sessionId: "ses_cursor_retry", - providerProfile: "dsflash-go", - valuesPrinted: false - }, - createdAt: "2026-06-19T12:00:00.000Z" - }, - agentRun: { - runId: "run_cursor_retry_params", - commandId: "cmd_cursor_retry_params", - lastSeq: 3 - }, - now: () => "2026-06-19T12:01:00.000Z" - }); - - assert.equal(state.retryParams.message, "preserve this prompt for fresh-session retry"); - assert.equal(state.retryParams.prompt, "preserve this prompt for fresh-session retry"); - assert.equal(state.retryParams.sessionId, "ses_cursor_retry"); - assert.equal(state.retryParams.providerProfile, "dsflash-go"); - assert.equal(state.retryParams.valuesPrinted, false); -}); - -test("AgentRun projection state resets source cursor when fresh retry switches run", () => { - const state = buildWorkbenchProjectionStateUpdate({ - currentState: { - traceId: "trc_cursor_fresh_retry", - sourceRunId: "run_cursor_failed", - sourceCommandId: "cmd_cursor_failed", - lastSourceSeq: 3700, - lastAgentRunSeq: 3700, - lastProjectedSeq: 3700, - sourceLatestSeq: 3700, - projectionStatus: "terminal", - resultSyncState: "pending", - retryParams: { - message: "retry on a fresh AgentRun session", - sessionId: "ses_cursor_fresh_retry", - providerProfile: "dsflash-go", - valuesPrinted: false - }, - createdAt: "2026-06-19T12:00:00.000Z" - }, - agentRun: { - runId: "run_cursor_fresh_retry", - commandId: "cmd_cursor_fresh_retry", - lastSeq: 0 - }, - projectionStatus: "projecting", - resultSyncState: null, - now: () => "2026-06-19T12:02:00.000Z" - }); - - assert.equal(state.sourceRunId, "run_cursor_fresh_retry"); - assert.equal(state.sourceCommandId, "cmd_cursor_fresh_retry"); - assert.equal(state.lastSourceSeq, 0); - assert.equal(state.lastAgentRunSeq, 0); - assert.equal(state.lastProjectedSeq, 0); - assert.equal(state.sourceLatestSeq, 0); - assert.equal(state.projectionStatus, "projecting"); - assert.equal(state.retryParams.message, "retry on a fresh AgentRun session"); - const plan = buildAgentRunProjectionEventsFetchPlan({ projectionState: state, agentRun: { runId: "run_cursor_fresh_retry", commandId: "cmd_cursor_fresh_retry", lastSeq: 0 } }); - assert.equal(plan.afterSeq, 0); -}); diff --git a/internal/cloud/workbench-projection-cursor.ts b/internal/cloud/workbench-projection-cursor.ts deleted file mode 100644 index 3f286390..00000000 --- a/internal/cloud/workbench-projection-cursor.ts +++ /dev/null @@ -1,205 +0,0 @@ -/* - * SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; PJ2026-010205 HWLAB接入 draft-2026-06-17-r0. - * Responsibility: Pure cursor planning for AgentRun events projection; never derives cursor by scanning trace events or result payloads. - */ - -const DEFAULT_AGENTRUN_EVENTS_PAGE_LIMIT = 500; -const MAX_AGENTRUN_EVENTS_PAGE_LIMIT = 1000; - -export function buildAgentRunProjectionEventsFetchPlan({ projectionState = null, agentRun = null, pageLimit = DEFAULT_AGENTRUN_EVENTS_PAGE_LIMIT } = {}) { - const state = normalizeProjectionState(projectionState); - const mapping = normalizeAgentRunMapping(agentRun); - const runId = textValue(state?.sourceRunId ?? state?.runId ?? mapping.runId); - const commandId = textValue(state?.sourceCommandId ?? state?.commandId ?? mapping.commandId); - const stateSeq = maxNonNegativeInteger(state?.lastSourceSeq, state?.lastAgentRunSeq); - const mappingSeq = maxNonNegativeInteger(mapping.lastSeq); - return { - runId, - commandId, - afterSeq: Math.max(stateSeq, mappingSeq), - limit: boundedPageLimit(pageLimit), - cursorSource: state ? "projection-state" : "agent-run-result", - valuesPrinted: false - }; -} - -export function buildWorkbenchProjectionStateUpdate({ - currentState = null, - result = null, - agentRun = null, - eventsResponse = null, - resultSyncState = null, - projectionStatus = null, - projectionHealth = null, - polling = null, - retryParams = null, - error = null, - now = () => new Date().toISOString() -} = {}) { - const previous = normalizeProjectionState(currentState) ?? {}; - const mapping = normalizeAgentRunMapping(agentRun ?? result?.agentRun); - const timestamp = timestampValue(now); - const traceId = textValue(previous.traceId ?? result?.traceId ?? mapping.traceId); - const runId = textValue(mapping.runId) || textValue(previous.sourceRunId ?? previous.runId); - const commandId = textValue(mapping.commandId) || textValue(previous.sourceCommandId ?? previous.commandId); - const previousRunId = textValue(previous.sourceRunId ?? previous.runId); - const previousCommandId = textValue(previous.sourceCommandId ?? previous.commandId); - const sourceChanged = Boolean((previousRunId && runId && previousRunId !== runId) || (previousCommandId && commandId && previousCommandId !== commandId)); - const previousCursorSeq = sourceChanged ? 0 : maxNonNegativeInteger(previous.lastSourceSeq, previous.lastAgentRunSeq); - const lastSourceSeq = Math.max( - previousCursorSeq, - maxNonNegativeInteger(mapping.lastSeq), - maxNonNegativeInteger(eventsResponse?.traceLastSeq) - ); - const sourceLatestSeq = Math.max( - sourceChanged ? 0 : maxNonNegativeInteger(previous.sourceLatestSeq, previous.upstreamLatestSeq), - maxNonNegativeInteger(eventsResponse?.maxSeq), - lastSourceSeq - ); - const nextRetryParams = normalizeProjectionRetryParams(retryParams) ?? normalizeProjectionRetryParams(previous.retryParams); - const nextPolling = normalizeProjectionPolling(polling); - const nextResultSyncState = textValue(resultSyncState ?? previous.resultSyncState) || "not_started"; - const nextProjectionStatus = textValue(projectionStatus) || inferProjectionStatus({ lastSourceSeq, sourceLatestSeq, resultSyncState: nextResultSyncState, previous }); - const nextProjectionHealth = textValue(projectionHealth) || (error ? "degraded" : "healthy"); - const nextRetryAt = nextPolling?.nextRetryAt ?? (error ? previous.nextRetryAt ?? null : null); - return { - ...previous, - traceId, - sessionId: textValue(previous.sessionId ?? result?.sessionId ?? mapping.sessionId) || null, - conversationId: textValue(previous.conversationId ?? result?.conversationId ?? mapping.conversationId) || null, - threadId: textValue(previous.threadId ?? result?.threadId ?? mapping.threadId) || null, - ownerUserId: textValue(previous.ownerUserId ?? result?.ownerUserId) || null, - ownerRole: textValue(previous.ownerRole ?? result?.ownerRole) || null, - managerUrl: textValue(previous.managerUrl ?? mapping.managerUrl) || null, - backendProfile: textValue(previous.backendProfile ?? mapping.backendProfile) || null, - providerId: textValue(previous.providerId ?? mapping.providerId) || null, - sourceRunId: runId, - sourceCommandId: commandId, - runId, - commandId, - lastSourceSeq, - lastAgentRunSeq: lastSourceSeq, - lastProjectedSeq: sourceChanged ? lastSourceSeq : Math.max(maxNonNegativeInteger(previous.lastProjectedSeq), lastSourceSeq), - sourceLatestSeq, - upstreamLatestSeq: sourceLatestSeq, - projectionStatus: nextProjectionStatus, - projectionHealth: nextProjectionHealth, - resultSyncState: nextResultSyncState, - lastProjectedAt: sourceChanged || lastSourceSeq > maxNonNegativeInteger(previous.lastSourceSeq, previous.lastAgentRunSeq) ? timestamp : previous.lastProjectedAt ?? timestamp, - lastResultSyncAt: nextResultSyncState === "synced" ? timestamp : previous.lastResultSyncAt ?? null, - lastErrorCode: error ? textValue(error.code ?? error.errorCode ?? "projection_sync_failed") : null, - lastErrorMessage: error ? textValue(error.message ?? "Projection sync failed.") : null, - failureCount: error ? maxNonNegativeInteger(previous.failureCount) + 1 : 0, - nextRetryAt, - pollCount: nextPolling?.pollCount ?? maxNonNegativeInteger(previous.pollCount), - noProgressPollCount: nextPolling?.noProgressPollCount ?? (error ? maxNonNegativeInteger(previous.noProgressPollCount) : 0), - backoffMs: nextPolling?.backoffMs ?? 0, - backoffReason: nextPolling?.backoffReason ?? null, - rootCause: nextPolling?.rootCause ?? null, - terminalConsistencyGap: nextPolling?.terminalConsistencyGap ?? null, - retryParams: nextRetryParams, - createdAt: previous.createdAt ?? timestamp, - updatedAt: timestamp, - valuesPrinted: false - }; -} - -export function agentRunResultSyncDeferred({ forceResultSync = false, options = {}, env = process.env } = {}) { - if (forceResultSync) return false; - if (options.deferAgentRunResultSync === true) return true; - const configured = String(env?.HWLAB_CODE_AGENT_AGENTRUN_RESULT_SYNC_IN_RUNNING ?? "").trim().toLowerCase(); - return ["0", "false", "off", "no"].includes(configured); -} - -function normalizeProjectionState(value) { - if (!value || typeof value !== "object" || Array.isArray(value)) return null; - return value; -} - -function normalizeAgentRunMapping(value) { - return value && typeof value === "object" && !Array.isArray(value) ? value : {}; -} - -function normalizeProjectionRetryParams(value) { - if (!value || typeof value !== "object" || Array.isArray(value)) return null; - const message = textValue(value.message ?? value.prompt ?? value.text); - if (!message) return null; - const result = { - message, - prompt: message, - text: message, - sessionId: textValue(value.sessionId) || null, - conversationId: textValue(value.conversationId) || null, - threadId: textValue(value.threadId) || null, - projectId: textValue(value.projectId) || null, - ownerUserId: textValue(value.ownerUserId) || null, - ownerRole: textValue(value.ownerRole) || null, - providerProfile: textValue(value.providerProfile ?? value.codeAgentProviderProfile ?? value.backendProfile) || null, - codeAgentProviderProfile: textValue(value.codeAgentProviderProfile ?? value.providerProfile ?? value.backendProfile) || null, - backendProfile: textValue(value.backendProfile ?? value.providerProfile ?? value.codeAgentProviderProfile) || null, - providerId: textValue(value.providerId) || null, - valuesPrinted: false - }; - for (const [key, item] of Object.entries(result)) { - if (item === null || item === "") delete result[key]; - } - return result; -} - -function normalizeProjectionPolling(value) { - if (!value || typeof value !== "object" || Array.isArray(value)) return null; - const result = { - pollCount: maxNonNegativeInteger(value.pollCount), - noProgressPollCount: maxNonNegativeInteger(value.noProgressPollCount), - backoffMs: maxNonNegativeInteger(value.backoffMs), - backoffReason: clippedText(value.backoffReason, 80), - rootCause: clippedText(value.rootCause, 120), - terminalConsistencyGap: maxNonNegativeInteger(value.terminalConsistencyGap), - nextRetryAt: timestampText(value.nextRetryAt) - }; - return result; -} - -function inferProjectionStatus({ lastSourceSeq, sourceLatestSeq, resultSyncState, previous }) { - if (["pending", "timed_out", "failed"].includes(resultSyncState)) return "terminal"; - if (sourceLatestSeq > lastSourceSeq) return "projecting"; - return textValue(previous?.projectionStatus) || "projecting"; -} - -function boundedPageLimit(value) { - const parsed = Number.parseInt(String(value ?? ""), 10); - if (!Number.isInteger(parsed) || parsed <= 0) return DEFAULT_AGENTRUN_EVENTS_PAGE_LIMIT; - return Math.min(parsed, MAX_AGENTRUN_EVENTS_PAGE_LIMIT); -} - -function maxNonNegativeInteger(...values) { - let max = 0; - for (const value of values) { - const parsed = Number.parseInt(String(value ?? ""), 10); - if (Number.isInteger(parsed) && parsed >= 0 && parsed > max) max = parsed; - } - return max; -} - -function timestampValue(now) { - const value = typeof now === "function" ? now() : now; - const parsed = Date.parse(String(value ?? "")); - return Number.isFinite(parsed) ? new Date(parsed).toISOString() : new Date().toISOString(); -} - -function textValue(value) { - return String(value ?? "").trim(); -} - -function clippedText(value, limit) { - const text = textValue(value); - if (!text) return null; - return text.length > limit ? text.slice(0, limit) : text; -} - -function timestampText(value) { - const text = textValue(value); - if (!text) return null; - const parsed = Date.parse(text); - return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null; -} diff --git a/internal/cloud/workbench-projection-finalizer.ts b/internal/cloud/workbench-projection-finalizer.ts deleted file mode 100644 index c5b72a10..00000000 --- a/internal/cloud/workbench-projection-finalizer.ts +++ /dev/null @@ -1,106 +0,0 @@ -/* - * SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-18-p0-unique-projection; PJ2026-010205 HWLAB接入 draft-2026-06-17-r0. - * 职责: WorkbenchProjectionFinalizer 组件入口。以 checkpoint/轮询驱动 AgentRun facts 追平，不让 GET path 承担 finalize/repair。 - */ -import { appendProjectionDiagnostic } from "./workbench-projection-writer.ts"; - -export function scheduleWorkbenchProjectionFinalizer({ traceId, currentResult = null, traceStore, getCachedResult, isCanceled, isTerminal, syncResult, onTerminal, activitySignature, noResponseTimeoutMs = null, pollIntervalMs = 1000, sleep = defaultSleep, performanceStore = null } = {}) { - if (!traceId || !currentResult?.agentRun?.runId || !currentResult?.agentRun?.commandId || typeof syncResult !== "function") return null; - let lastActivityAt = Date.now(); - let lastActivitySignature = activitySignature?.(currentResult, traceStore?.snapshot?.(traceId)) ?? null; - let idleDegraded = false; - performanceStore?.recordWorkbenchProjectorCandidate?.({ status: "scheduled", reason: "finalizer" }); - - setImmediate(() => { - void (async () => { - let result = currentResult; - let lastError = null; - let nextDelayMs = pollIntervalMs; - while (true) { - const cached = getCachedResult?.(traceId); - if (isCanceled?.(cached)) return; - if (cached && isTerminal?.(cached, traceStore?.snapshot?.(traceId))) { - performanceStore?.recordWorkbenchProjectorCandidate?.({ status: "completed", reason: "cached_terminal" }); - onTerminal?.(cached, { preserveLastTraceId: true }); - return; - } - const syncStartedAt = Date.now(); - try { - const synced = await syncResult({ result }); - performanceStore?.recordWorkbenchProjectorBatch?.({ phase: "candidate_scan", status: "ok", durationMs: Date.now() - syncStartedAt }); - result = synced?.result ?? result; - nextDelayMs = projectionFinalizerDelayMs(synced?.polling, pollIntervalMs); - const runnerTrace = synced?.runnerTrace ?? traceStore?.snapshot?.(traceId); - if (result && isTerminal?.(result, runnerTrace)) { - performanceStore?.recordWorkbenchProjectorCandidate?.({ status: "completed", reason: "synced_terminal" }); - onTerminal?.(result, { preserveLastTraceId: true }); - return; - } - const signature = activitySignature?.(result, runnerTrace); - if (signature && signature !== lastActivitySignature) { - lastActivitySignature = signature; - lastActivityAt = Date.now(); - idleDegraded = false; - } - lastError = null; - } catch (error) { - performanceStore?.recordWorkbenchProjectorBatch?.({ phase: "candidate_scan", status: error?.code === "agentrun_timeout" ? "timeout" : "error", durationMs: Date.now() - syncStartedAt }); - performanceStore?.recordWorkbenchProjectorError?.({ reason: error?.code ?? "finalizer_sync_failed" }); - nextDelayMs = projectionFinalizerDelayMs(error?.polling, pollIntervalMs); - lastError = error; - } - const idleMs = Date.now() - lastActivityAt; - if (noResponseTimeoutMs && idleMs >= noResponseTimeoutMs && !idleDegraded) { - performanceStore?.recordWorkbenchProjectorCandidate?.({ status: "degraded", reason: "no_response_idle_timeout" }); - performanceStore?.recordWorkbenchProjectorError?.({ reason: lastError?.code ?? "no_response_idle_timeout" }); - appendProjectionDiagnostic(traceStore, traceId, { - type: "turn-status", - status: "degraded", - label: "projection-finalizer:no-response-idle-timeout", - errorCode: lastError?.code ?? "no_response_idle_timeout", - degradedReason: lastError ? "projection_activity_stale" : "no_response_idle_timeout", - message: lastError?.message ?? `AgentRun projection has no new activity for ${idleMs}ms; idle threshold ${noResponseTimeoutMs}ms.`, - runId: result?.agentRun?.runId ?? currentResult.agentRun.runId, - commandId: result?.agentRun?.commandId ?? currentResult.agentRun.commandId, - timeoutMs: noResponseTimeoutMs, - idleMs, - lastActivityAt: new Date(lastActivityAt).toISOString(), - waitingFor: "agentrun-result-activity", - terminal: false - }); - idleDegraded = true; - } - await sleep(nextDelayMs); - } - })().catch((error) => { - performanceStore?.recordWorkbenchProjectorError?.({ reason: error?.code ?? "workbench_projection_finalizer_crashed" }); - appendProjectionDiagnostic(traceStore, traceId, { - type: "turn-status", - status: "degraded", - label: "projection-finalizer:sync-crashed", - errorCode: error?.code ?? "workbench_projection_finalizer_crashed", - message: error?.message ?? "Workbench projection finalizer crashed.", - runId: currentResult.agentRun.runId, - commandId: currentResult.agentRun.commandId, - waitingFor: "agentrun-result", - terminal: false - }); - }); - }); - return { scheduled: true, traceId, noResponseTimeoutMs, pollIntervalMs, valuesPrinted: false }; -} - -function defaultSleep(ms) { - return new Promise((resolve) => setTimeout(resolve, Math.max(0, Number(ms) || 0))); -} - -function projectionFinalizerDelayMs(polling = null, fallbackMs = 1000) { - const fallback = positiveInteger(fallbackMs, 1000); - const backoffMs = positiveInteger(polling?.backoffMs, 0); - return backoffMs > 0 ? Math.max(fallback, backoffMs) : fallback; -} - -function positiveInteger(value, fallback) { - const parsed = Number.parseInt(String(value ?? ""), 10); - return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; -} diff --git a/internal/cloud/workbench-projection-outbox-events.ts b/internal/cloud/workbench-projection-outbox-events.ts new file mode 100644 index 00000000..7e5fed25 --- /dev/null +++ b/internal/cloud/workbench-projection-outbox-events.ts @@ -0,0 +1,99 @@ +/* + * Immutable Workbench projection outbox rows are the common SSE and sync-replay authority. + */ + +export const WORKBENCH_REALTIME_AUTHORITY_VERSION = "workbench-realtime-authority-v2"; + +export function projectionOutboxRealtimeEvents(snapshot = {}, { includeSnapshot = false, includeEvents = true } = {}) { + const facts = objectValue(snapshot.facts); + const rows = includeEvents ? factArray(snapshot.events) : []; + const emitted = new Set(); + const output = []; + for (const row of rows) { + const family = textValue(row.entityFamily) || textValue(row.payload?.family) || projectionOutboxFamily(row); + const fact = projectionOutboxFact(row); + if (!fact || !SUPPORTED_FAMILIES.has(family)) continue; + const item = projectionRealtimeItem(row, fact, family); + if (!item) continue; + emitted.add(`${family}:${item.payload.entity.id}`); + output.push(item); + } + if (includeSnapshot) { + const cutoff = nonNegativeInteger(snapshot.cutoffOutboxSeq); + for (const [family, name] of [["messages", "workbench.message.snapshot"], ["turns", "workbench.turn.snapshot"]]) { + for (const fact of factArray(facts[family])) { + const id = projectionFactId(fact, family); + if (!id || emitted.has(`${family}:${id}`)) continue; + const item = projectionRealtimeItem({ outboxSeq: cutoff, entityFamily: family, entityId: id, projectedSeq: fact.projectedSeq, projectionRevision: fact.projectedSeq, createdAt: fact.updatedAt }, fact, family, name); + if (item) output.push(item); + } + } + } + return output.sort((left, right) => nonNegativeInteger(left.payload.cursor?.outboxSeq) - nonNegativeInteger(right.payload.cursor?.outboxSeq)); +} + +function projectionRealtimeItem(row, fact, family, forcedName = null) { + const id = textValue(row.entityId) || projectionFactId(fact, family); + if (!id) return null; + const outboxSeq = nonNegativeInteger(row.outboxSeq); + const version = nonNegativeInteger(row.projectionRevision ?? row.projectedSeq ?? fact.projectedSeq ?? outboxSeq); + const projectionRevision = String(version); + const base = { + contractVersion: "workbench-sync-v1", + realtimeAuthority: WORKBENCH_REALTIME_AUTHORITY_VERSION, + realtimeSource: "projection-outbox", + sessionId: textValue(fact.sessionId ?? row.sessionId), + traceId: textValue(fact.traceId ?? row.traceId), + cursor: { outboxSeq, traceSeq: nonNegativeInteger(row.projectedSeq ?? fact.projectedSeq) }, + entity: { family, id, version, entityVersion: version, outboxSeq, traceSeq: nonNegativeInteger(row.projectedSeq ?? fact.projectedSeq), projectionRevision, authority: WORKBENCH_REALTIME_AUTHORITY_VERSION }, + family, + id, + commitType: textValue(row.commitType), + terminal: row.terminal === true || fact.terminal === true, + sealed: row.sealed === true || fact.sealed === true, + projectionRevision, + outboxEventId: textValue(row.outboxEventId), + eventCreatedAt: textValue(row.createdAt ?? fact.updatedAt ?? fact.createdAt), + valuesRedacted: true + }; + if (family === "messages") return { name: forcedName ?? "workbench.message.snapshot", payload: { ...base, type: "message.snapshot", reason: "projection-outbox", message: fact } }; + if (family === "turns") return { name: forcedName ?? "workbench.turn.snapshot", payload: { ...base, type: "turn.snapshot", reason: "projection-outbox", turn: fact } }; + return { name: forcedName ?? "workbench.trace.event", payload: { ...base, type: "trace.event", event: fact, snapshot: { traceId: base.traceId, sessionId: base.sessionId, status: fact.status ?? null, events: [fact], eventCount: 1 } } }; +} + +function projectionOutboxFact(row) { + const fact = row?.payload?.fact; + return fact && typeof fact === "object" && !Array.isArray(fact) ? fact : null; +} + +function projectionOutboxFamily(row) { + if (row.commitType === "message") return "messages"; + if (row.commitType === "terminal" && row.entityFamily !== "traceEvents") return "turns"; + return "traceEvents"; +} + +function projectionFactId(fact, family) { + if (family === "messages") return textValue(fact.messageId ?? fact.id); + if (family === "turns") return textValue(fact.turnId ?? fact.traceId); + return textValue(fact.id ?? fact.sourceEventId); +} + +function factArray(value) { + return Array.isArray(value) ? value : []; +} + +function objectValue(value) { + return value && typeof value === "object" && !Array.isArray(value) ? value : {}; +} + +function textValue(value) { + const text = typeof value === "string" ? value.trim() : value === null || value === undefined ? "" : String(value).trim(); + return text || ""; +} + +function nonNegativeInteger(value) { + const parsed = Number.parseInt(String(value ?? ""), 10); + return Number.isInteger(parsed) && parsed >= 0 ? parsed : 0; +} + +const SUPPORTED_FAMILIES = new Set(["messages", "turns", "traceEvents"]); diff --git a/internal/cloud/workbench-projection-writer.test.ts b/internal/cloud/workbench-projection-writer.test.ts index 5d4368d7..a3ba0bbd 100644 --- a/internal/cloud/workbench-projection-writer.test.ts +++ b/internal/cloud/workbench-projection-writer.test.ts @@ -1,11 +1,70 @@ // SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor. -// Responsibility: Workbench projection writer/finalizer durable facts regression tests. +// Responsibility: Workbench Kafka projection writer durable facts regression tests. import assert from "node:assert/strict"; import { test } from "bun:test"; import { createCloudRuntimeStore } from "../db/runtime-store.ts"; -import { writeWorkbenchProjectionEvent, writeWorkbenchProjectionSession } from "./workbench-projection-writer.ts"; +import { buildWorkbenchProjectionEventFacts, writeWorkbenchProjectionEvent, writeWorkbenchProjectionSession } from "./workbench-projection-writer.ts"; + +test("Kafka projection keeps assistant text visible through progress, final, and terminal status", () => { + const traceId = "trc_kafka_assistant_lifecycle"; + const sessionId = "ses_kafka_assistant_lifecycle"; + const progress = buildWorkbenchProjectionEventFacts({ + projectedSeq: 1, + projectedAt: "2026-07-10T10:00:01.000Z", + event: { traceId, sessionId, sourceSeq: 1, sourceEventId: "evt-progress", type: "assistant", eventType: "assistant", status: "running", label: "agentrun:assistant:message", assistantText: "first progress", text: "first progress", createdAt: "2026-07-10T10:00:01.000Z" } + }); + const toolProgress = buildWorkbenchProjectionEventFacts({ + projectedSeq: 2, + projectedAt: "2026-07-10T10:00:02.000Z", + previousCheckpoint: progress.facts.checkpoints[0], + event: { traceId, sessionId, sourceSeq: 2, sourceEventId: "evt-tool-progress", type: "tool", eventType: "tool", status: "running", label: "agentrun:tool:completed", createdAt: "2026-07-10T10:00:02.000Z" } + }); + const final = buildWorkbenchProjectionEventFacts({ + projectedSeq: 3, + projectedAt: "2026-07-10T10:00:03.000Z", + previousCheckpoint: toolProgress.facts.checkpoints[0], + event: { traceId, sessionId, sourceSeq: 3, sourceEventId: "evt-final", type: "assistant", eventType: "assistant", status: "completed", label: "agentrun:assistant:message", assistantText: "final answer", text: "final answer", finalResponse: { text: "final answer", status: "completed", traceId }, terminal: true, createdAt: "2026-07-10T10:00:03.000Z" } + }); + const terminal = buildWorkbenchProjectionEventFacts({ + projectedSeq: 4, + projectedAt: "2026-07-10T10:00:04.000Z", + previousCheckpoint: final.facts.checkpoints[0], + event: { traceId, sessionId, sourceSeq: 4, sourceEventId: "evt-terminal", type: "result", eventType: "terminal", status: "completed", label: "agentrun:terminal:completed", message: "AgentRun terminal status completed", terminal: true, createdAt: "2026-07-10T10:00:04.000Z" } + }); + + assert.equal(progress.facts.messages[0].text, "first progress"); + assert.equal(progress.facts.parts[0].text, "first progress"); + assert.equal(progress.facts.checkpoints[0].assistantText, "first progress"); + assert.equal(progress.facts.turns[0].status, "running"); + assert.equal(progress.facts.turns[0].terminal, false); + assert.equal(toolProgress.facts.messages[0].text, "first progress"); + assert.equal(toolProgress.facts.parts[0].text, "first progress"); + assert.equal(toolProgress.facts.checkpoints[0].assistantText, "first progress"); + assert.equal(final.facts.messages[0].text, "final answer"); + assert.equal(final.facts.parts[0].partType, "final_response"); + assert.equal(terminal.facts.messages[0].text, "final answer"); + assert.equal(terminal.facts.parts[0].text, "final answer"); + assert.equal(terminal.facts.checkpoints[0].finalResponse.text, "final answer"); + assert.equal(terminal.facts.turns[0].terminal, true); + assert.equal(terminal.facts.turns[0].sealed, true); + assert.equal(terminal.facts.turns[0].finalResponse.text, "final answer"); + assert.equal(terminal.facts.turns[0].assistantText, "final answer"); +}); + +test("Kafka projection seals failed terminal turn with a failure response", () => { + const result = buildWorkbenchProjectionEventFacts({ + projectedSeq: 1, + projectedAt: "2026-07-10T10:01:00.000Z", + event: { traceId: "trc_kafka_failed", sessionId: "ses_kafka_failed", sourceSeq: 1, sourceEventId: "evt-kafka-failed", type: "result", eventType: "terminal", status: "failed", label: "agentrun:terminal:failed", failureKind: "provider-stream-disconnected", terminal: true, createdAt: "2026-07-10T10:01:00.000Z" } + }); + + assert.equal(result.facts.turns[0].terminal, true); + assert.equal(result.facts.turns[0].sealed, true); + assert.equal(result.facts.turns[0].failureKind, "provider-stream-disconnected"); + assert.match(result.facts.turns[0].finalResponse.text, /provider-stream-disconnected/u); +}); test("workbench projection writer preserves Workbench launch context on Code Agent session facts", async () => { const factWrites = []; diff --git a/internal/cloud/workbench-projection-writer.ts b/internal/cloud/workbench-projection-writer.ts index d5d34076..5fed372c 100644 --- a/internal/cloud/workbench-projection-writer.ts +++ b/internal/cloud/workbench-projection-writer.ts @@ -178,8 +178,15 @@ export async function writeWorkbenchProjectionEvent({ runtimeStore = null, event if (projectedSeq <= 0) throw new Error("Workbench projection allocator returned an invalid projectedSeq."); const eventId = isAssistantMessageTraceEvent(event) ? stableFactId("wte", { traceId, sourceEventId }) : textValue(event.id) || stableFactId("wte", { traceId, sourceEventId }); const previousFinalText = finalResponseTextValue(resolvedPreviousCheckpoint?.finalResponse, resolvedPreviousCheckpoint?.assistantText, resolvedPreviousCheckpoint?.finalText); + const liveAssistantText = isAssistantMessageTraceEvent(event) ? finalResponseTextValue(event.assistantText, event.text, event.reply, event.message) : finalResponseTextValue(event.assistantText, event.reply); const eventTerminalFinalResponse = terminal ? terminalFinalResponse(status, event, { evidence: event, finalResponse: event.finalResponse }) : null; - const messageFinalText = terminal ? finalResponseTextValue(event.finalResponse, event.assistantText, event.reply, eventTerminalFinalResponse, previousFinalText) : null; + const messageFinalText = terminal ? finalResponseTextValue(event.finalResponse, liveAssistantText, previousFinalText, eventTerminalFinalResponse) : null; + const checkpointAssistantText = messageFinalText ?? liveAssistantText ?? previousFinalText ?? null; + const checkpointFinalResponse = terminal && messageFinalText + ? { ...(eventTerminalFinalResponse ?? {}), text: messageFinalText, status, traceId, source: "workbench-event-projection", valuesPrinted: false } + : resolvedPreviousCheckpoint?.finalResponse ?? null; + const terminalSealBlocked = terminal && status === "completed" && !messageFinalText; + const factTerminal = terminal && !terminalSealBlocked; const userMessageFact = sessionId && !suppressedAfterSeal && resolvedPreviousCheckpoint?.userMessage ? normalizeMessageFact(resolvedPreviousCheckpoint.userMessage, 0, { traceId, sessionId, turnId, terminal: false, terminalStatus: "sent", finalText: null, timestamp: projectedAt, timing }) : null; const messageFact = sessionId && !suppressedAfterSeal ? normalizeMessageFact({ messageId: eventProjectionAssistantMessageId(traceId, event), @@ -187,12 +194,12 @@ export async function writeWorkbenchProjectionEvent({ runtimeStore = null, event status, traceId, turnId, - text: terminal && messageFinalText ? messageFinalText : "", + text: messageFinalText ?? liveAssistantText ?? previousFinalText ?? "", projectedSeq, sourceSeq, sourceEventId, - terminal, - sealed: terminal, + terminal: factTerminal, + sealed: factTerminal, timing, startedAt: timing.startedAt, lastEventAt: timing.lastEventAt, @@ -201,7 +208,7 @@ export async function writeWorkbenchProjectionEvent({ runtimeStore = null, event createdAt: occurredAt, updatedAt: projectedAt, source: "workbench-event-projection" - }, userMessageFact ? 1 : 0, { traceId, sessionId, turnId, terminal, terminalStatus: status, finalText: messageFinalText, timestamp: projectedAt, timing }) : null; + }, userMessageFact ? 1 : 0, { traceId, sessionId, turnId, terminal, terminalStatus: status, finalText: messageFinalText, terminalSealBlocked, timestamp: projectedAt, timing }) : null; const facts = { sessions: sessionId && !suppressedAfterSeal ? [{ sessionId, @@ -245,6 +252,29 @@ export async function writeWorkbenchProjectionEvent({ runtimeStore = null, event }], messages: [userMessageFact, messageFact].filter(Boolean), parts: [userMessageFact, messageFact].filter(Boolean).flatMap((message) => messagePartFacts(message, { finalText: message === messageFact ? messageFinalText : null })), + turns: sessionId && !suppressedAfterSeal ? [{ + turnId, + sessionId, + traceId, + messageId: messageFact?.messageId ?? null, + status, + projectedSeq, + sourceSeq, + sourceEventId, + terminal: factTerminal, + sealed: factTerminal, + finalResponse: checkpointFinalResponse, + assistantText: checkpointAssistantText, + failureKind: textValue(event.failureKind ?? event.errorCode) || null, + diagnostic: { timingAuthorityIssue, terminalSealBlocked, authority: "workbench-event-projection", valuesRedacted: true }, + timing, + startedAt: timing.startedAt, + lastEventAt: timing.lastEventAt, + finishedAt: timing.finishedAt, + durationMs: timing.durationMs, + createdAt: occurredAt, + updatedAt: projectedAt + }] : [], checkpoints: !suppressedAfterSeal ? [{ traceId, sessionId, @@ -255,9 +285,11 @@ export async function writeWorkbenchProjectionEvent({ runtimeStore = null, event sourceSeq, sourceEventId, projectionStatus: terminal ? "caught_up" : "projecting", - projectionHealth: "healthy", - terminal, - sealed: terminal, + projectionHealth: terminalSealBlocked ? "degraded" : "healthy", + terminal: factTerminal, + sealed: factTerminal, + assistantText: checkpointAssistantText, + finalResponse: checkpointFinalResponse, timing, startedAt: timing.startedAt, lastEventAt: timing.lastEventAt, @@ -267,6 +299,7 @@ export async function writeWorkbenchProjectionEvent({ runtimeStore = null, event lastEventLabel: textValue(event.label) || null, lastEventStatus: textValue(event.status) || null, timingAuthorityIssue, + terminalSealBlocked, valuesRedacted: true }, updatedAt: projectedAt @@ -322,6 +355,122 @@ export async function writeWorkbenchProjectionEvent({ runtimeStore = null, event } } +export function buildWorkbenchProjectionEventFacts({ event = {}, requestMeta = {}, previousCheckpoint = null, projectedSeq, projectedAt = new Date().toISOString() } = {}) { + const traceId = safeTraceId(event.traceId ?? requestMeta.traceId); + if (!traceId) throw workbenchProjectionError("workbench_projection_trace_required", "Kafka projection event requires traceId."); + const durableProjectedSeq = nonNegativeInteger(projectedSeq); + if (durableProjectedSeq <= 0) throw workbenchProjectionError("workbench_projection_seq_required", "Kafka projection event requires an allocated projectedSeq."); + const sourceSeq = nonNegativeInteger(event.sourceSeq ?? event.seq); + const sessionId = textValue(event.sessionId ?? requestMeta.sessionId) || null; + const turnId = textValue(event.turnId) || traceId; + const eventType = textValue(event.eventType ?? event.type ?? event.label) || "event"; + const terminal = event.terminal === true || TERMINAL_STATUSES.has(normalizeWorkbenchStatus(event.status)); + const status = terminal ? normalizeWorkbenchStatus(event.status) : "running"; + const previousTiming = normalizeTimingProjection(previousCheckpoint?.timing) ?? normalizeTimingProjection(previousCheckpoint); + const sourceOccurredAt = timestampValue(event.createdAt ?? event.occurredAt ?? event.updatedAt ?? projectedAt); + const occurredAt = latestTimestamp(previousTiming?.lastEventAt, sourceOccurredAt) ?? sourceOccurredAt; + if (checkpointIsTerminal(previousCheckpoint) && !terminal) { + return { written: false, suppressedAfterSeal: true, traceId, sessionId, sourceSeq, valuesPrinted: false }; + } + const explicitEventTiming = normalizeTimingProjection(event.timing) ?? normalizeTimingProjection(event); + const explicitDurationMs = durationValue(event.durationMs ?? explicitEventTiming?.durationMs); + const explicitStartedAt = optionalTimestampValue(event.startedAt ?? event.traceStartedAt ?? event.runnerStartedAt ?? explicitEventTiming?.startedAt); + const provisionalLastEventAt = latestTimestamp(previousTiming?.lastEventAt, explicitEventTiming?.lastEventAt, occurredAt); + const provisionalFinishedAt = terminal ? latestTimestamp(explicitEventTiming?.finishedAt, optionalTimestampValue(event.finishedAt), provisionalLastEventAt, occurredAt) : null; + const startedAt = previousTiming?.startedAt ?? explicitStartedAt ?? startedAtFromDuration(provisionalFinishedAt, explicitDurationMs) ?? (sourceSeq <= 1 ? occurredAt : null); + const lastEventAt = provisionalLastEventAt; + const finishedAt = terminal ? latestTimestamp(provisionalFinishedAt, lastEventAt) : null; + const timing = eventTimingProjection({ startedAt, lastEventAt, finishedAt, terminal, durationMs: explicitDurationMs ?? previousTiming?.durationMs }); + const timingAuthorityIssue = terminalTimingAuthorityIssue(timing, { terminal, traceId, source: "kafka", status, label: event.label, sourceSeq }); + const sourceEventId = workbenchSourceEventId(event, { traceId, sourceSeq, occurredAt }); + const eventId = textValue(event.id) || stableFactId("wte", { traceId, sourceEventId }); + const previousFinalText = finalResponseTextValue(previousCheckpoint?.finalResponse, previousCheckpoint?.assistantText, previousCheckpoint?.finalText); + const liveAssistantText = isAssistantMessageTraceEvent(event) ? finalResponseTextValue(event.assistantText, event.text, event.reply, event.message) : finalResponseTextValue(event.assistantText, event.reply); + const eventTerminalFinalResponse = terminal ? terminalFinalResponse(status, event, { evidence: event, finalResponse: event.finalResponse }) : null; + const messageFinalText = terminal ? finalResponseTextValue(event.finalResponse, liveAssistantText, previousFinalText, eventTerminalFinalResponse) : null; + const checkpointAssistantText = messageFinalText ?? liveAssistantText ?? previousFinalText ?? null; + const checkpointFinalResponse = terminal && messageFinalText + ? { ...(eventTerminalFinalResponse ?? {}), text: messageFinalText, status, traceId, source: "agentrun-kafka-projector", valuesPrinted: false } + : previousCheckpoint?.finalResponse ?? null; + const terminalSealBlocked = terminal && status === "completed" && !messageFinalText; + const factTerminal = terminal && !terminalSealBlocked; + const userMessageFact = sessionId && previousCheckpoint?.userMessage ? normalizeMessageFact(previousCheckpoint.userMessage, 0, { traceId, sessionId, turnId, terminal: false, terminalStatus: "sent", finalText: null, timestamp: projectedAt, timing }) : null; + const messageFact = sessionId ? normalizeMessageFact({ + messageId: eventProjectionAssistantMessageId(traceId, event), + role: "agent", + status, + traceId, + turnId, + text: messageFinalText ?? liveAssistantText ?? previousFinalText ?? "", + projectedSeq: durableProjectedSeq, + sourceSeq, + sourceEventId, + terminal: factTerminal, + sealed: factTerminal, + timing, + startedAt: timing.startedAt, + lastEventAt: timing.lastEventAt, + finishedAt: timing.finishedAt, + durationMs: timing.durationMs, + createdAt: occurredAt, + updatedAt: projectedAt, + source: "agentrun-kafka-projector" + }, userMessageFact ? 1 : 0, { traceId, sessionId, turnId, terminal, terminalStatus: status, finalText: messageFinalText, terminalSealBlocked, timestamp: projectedAt, timing }) : null; + const facts = { + sessions: sessionId ? [{ sessionId, status, lastTraceId: traceId, projectedSeq: durableProjectedSeq, sourceSeq, sourceEventId, terminal: factTerminal, sealed: factTerminal, timing, startedAt: timing.startedAt, lastEventAt: timing.lastEventAt, finishedAt: timing.finishedAt, durationMs: timing.durationMs, updatedAt: projectedAt }] : [], + traceEvents: [{ ...event, id: eventId, traceId, sessionId, turnId, messageId: textValue(event.messageId) || null, sourceSeq, sourceEventId, projectedSeq: durableProjectedSeq, eventType, terminal, sealed: terminal, timing, startedAt: timing.startedAt, lastEventAt: timing.lastEventAt, finishedAt: timing.finishedAt, durationMs: timing.durationMs, createdAt: occurredAt, occurredAt, updatedAt: projectedAt }], + messages: [userMessageFact, messageFact].filter(Boolean), + parts: [userMessageFact, messageFact].filter(Boolean).flatMap((message) => messagePartFacts(message, { finalText: message === messageFact ? messageFinalText : null })), + turns: sessionId ? [{ + turnId, + sessionId, + traceId, + messageId: messageFact?.messageId ?? null, + status, + projectedSeq: durableProjectedSeq, + sourceSeq, + sourceEventId, + terminal: factTerminal, + sealed: factTerminal, + finalResponse: checkpointFinalResponse, + assistantText: checkpointAssistantText, + failureKind: textValue(event.failureKind ?? event.errorCode) || null, + diagnostic: { timingAuthorityIssue, terminalSealBlocked, authority: "agentrun-kafka-projector", valuesRedacted: true }, + timing, + startedAt: timing.startedAt, + lastEventAt: timing.lastEventAt, + finishedAt: timing.finishedAt, + durationMs: timing.durationMs, + createdAt: occurredAt, + updatedAt: projectedAt + }] : [], + checkpoints: [{ + traceId, + sessionId, + turnId, + runId: textValue(event.runId) || null, + commandId: textValue(event.commandId) || null, + projectedSeq: durableProjectedSeq, + sourceSeq, + sourceEventId, + projectionStatus: terminal ? "caught_up" : "projecting", + projectionHealth: terminalSealBlocked ? "degraded" : "healthy", + terminal: factTerminal, + sealed: factTerminal, + assistantText: checkpointAssistantText, + finalResponse: checkpointFinalResponse, + timing, + startedAt: timing.startedAt, + lastEventAt: timing.lastEventAt, + finishedAt: timing.finishedAt, + durationMs: timing.durationMs, + diagnostic: { lastEventLabel: textValue(event.label) || null, lastEventStatus: textValue(event.status) || null, timingAuthorityIssue, terminalSealBlocked, authority: "agentrun-kafka-projector", valuesRedacted: true }, + updatedAt: projectedAt + }] + }; + return { written: true, suppressedAfterSeal: false, traceId, sessionId, sourceSeq, sourceEventId, projectedSeq: durableProjectedSeq, facts, terminal, valuesPrinted: false }; +} + function eventProjectionAssistantMessageId(traceId, event = {}) { return eventProjectionMessageId(traceId, "agent", event); } @@ -479,7 +628,7 @@ async function backfillTerminalTraceEvents({ runtimeStore = null, traceStore = d eventType: "terminal", status: "completed", label: "agentrun:result:completed", - message: "AgentRun result is ready for HWLAB short-connection polling.", + message: "AgentRun result is committed to the Workbench projection event stream.", finalResponse: { text: finalText, status: "completed", traceId: safeId, valuesPrinted: false }, terminal: true, sealed: true, @@ -1094,12 +1243,12 @@ function stableFactId(prefix, value) { } function workbenchSourceEventId(event = {}, { traceId, sourceSeq, occurredAt } = {}) { + const explicit = textValue(event.sourceEventId ?? event.id); + if (explicit) return explicit; if (isAssistantMessageTraceEvent(event)) { const messageId = eventProjectionAssistantMessageId(traceId, event); return `${safeTraceId(traceId) ?? traceId}:assistant-message:${messageId}`; } - const explicit = textValue(event.sourceEventId ?? event.id); - if (explicit) return explicit; const source = textValue(event.source); const label = textValue(event.label ?? event.type ?? event.eventType) || "event"; if (source && sourceSeq > 0) return `${source}:${sourceSeq}:${label}`; diff --git a/internal/cloud/workbench-read-model.ts b/internal/cloud/workbench-read-model.ts index fcde5182..580daeb9 100644 --- a/internal/cloud/workbench-read-model.ts +++ b/internal/cloud/workbench-read-model.ts @@ -16,7 +16,6 @@ export function createWorkbenchReadModel(options = {}, actor = null) { getSessionByTraceId: (traceId) => facts.getSessionByTraceId(traceId), resultForTrace: (traceId) => facts.resultForTrace(traceId), traceSnapshot: (traceId) => facts.traceSnapshot(traceId), - traceSnapshotSync: (traceId) => facts.traceSnapshotSync(traceId), canReadOwner: (ownerUserId) => facts.canReadOwner(ownerUserId), projectionDiagnostics: ({ traceId, result = null, trace = null, projection = null, refreshError = null } = {}) => workbenchProjectionDiagnostics({ traceId, result, trace, projection, refreshError }) }; diff --git a/internal/cloud/workbench-realtime-authority-contract.test.ts b/internal/cloud/workbench-realtime-authority-contract.test.ts index 2033c8b9..e6cd55bf 100644 --- a/internal/cloud/workbench-realtime-authority-contract.test.ts +++ b/internal/cloud/workbench-realtime-authority-contract.test.ts @@ -8,7 +8,7 @@ import { createCloudApiServer } from "./server.ts"; const ACTOR = { id: "usr_workbench_realtime_authority", username: "reader", displayName: "Reader", role: "user", status: "active" }; -test("workbench sync delta stays durable while SSE realtime forwards Kafka events", async () => { +test("workbench sync and SSE read the same atomic projection outbox", async () => { const sessionId = "ses_realtime_authority_p1"; const traceId = "trc_realtime_authority_p1"; const outboxRows = [{ @@ -28,20 +28,32 @@ test("workbench sync delta stays durable while SSE realtime forwards Kafka event terminal: true, sealed: true, createdAt: "2026-07-08T12:20:00.000Z", - payload: { type: "terminal", status: "completed", text: "redacted final" } + entityFamily: "turns", + entityId: traceId, + payload: { + family: "turns", + fact: { + turnId: traceId, + sessionId, + traceId, + messageId: "msg_realtime_authority_agent", + status: "completed", + projectedSeq: 8, + terminal: true, + sealed: true, + finalResponse: { text: "redacted final", status: "completed", traceId } + }, + valuesRedacted: true + } }]; const outboxQueries = []; const runtime = createRuntime({ facts: durableFacts({ sessionId, traceId }), outboxRows, outboxQueries }); - const fakeKafka = createFakeKafkaFactory(); const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: runtime, - kafkaFactory: fakeKafka.factory, env: { HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", - HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", - HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", - HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1" + HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } }); await listen(server); @@ -59,23 +71,16 @@ test("workbench sync delta stays durable while SSE realtime forwards Kafka event assert.equal(sync.body.events[0].terminal, true); assert.equal(sync.body.events[0].sealed, true); - setTimeout(() => { void fakeKafka.emit({ - eventType: "hwlab.trace.event.projected", - sessionId: "ses_agentrun_realtime_authority_p1", - traceId, - context: { sourceSeq: 8, runId: "run_realtime_authority_p1", commandId: "cmd_realtime_authority_p1" }, - event: { type: "result", eventType: "terminal", status: "completed", label: "agentrun:terminal:completed", message: "redacted final", terminal: true, sourceSeq: 8 } - }); }, 0); const events = await getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(sessionId)}&traceId=${encodeURIComponent(traceId)}&afterSeq=10`, 2); - assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.trace.event"]); - assert.equal(events[1].data.realtimeSource, "kafka"); + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.turn.snapshot"]); + assert.equal(events[1].data.realtimeSource, "projection-outbox"); assert.equal(events[1].data.realtimeAuthority, "workbench-realtime-authority-v2"); - assert.equal(events[1].data.entity.family, "traceEvents"); - assert.equal(events[1].data.entity.version, 8); - assert.equal(events[1].data.entity.projectionRevision, "kafka:hwlab.event.v1:0:0"); - assert.equal(events[1].data.event.message, "redacted final"); - assert.deepEqual(outboxQueries.map((query) => query.afterSeq), [10]); + assert.equal(events[1].data.entity.family, "turns"); + assert.equal(events[1].data.entity.version, 42); + assert.equal(events[1].data.turn.finalResponse.text, "redacted final"); + assert.deepEqual(outboxQueries.map((query) => query.afterOutboxSeq), [10, 10]); assert.equal(outboxQueries.every((query) => query.sessionId === sessionId), true); + assert.deepEqual(outboxQueries.at(-1).actor, { id: ACTOR.id, role: ACTOR.role }); } finally { await close(server); } @@ -118,6 +123,27 @@ test("workbench sync delta includes all authority families and marks trace rows } }); +test("workbench sync preserves session and trace scope for narrow replay", async () => { + const sessionId = "ses_realtime_authority_narrow"; + const traceId = "trc_realtime_authority_narrow"; + const outboxQueries = []; + const runtime = createRuntime({ facts: durableFacts({ sessionId, traceId }), outboxQueries }); + const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: runtime }); + await listen(server); + + try { + const { port } = server.address(); + const sync = await getJson(port, `/v1/workbench/sync?sessionId=${encodeURIComponent(sessionId)}&traceId=${encodeURIComponent(traceId)}&since=7`); + assert.equal(sync.status, 200); + assert.deepEqual(sync.body.scope, { kind: "trace", sessionId, traceId, since: 7 }); + assert.equal(outboxQueries.length, 1); + assert.equal(outboxQueries[0].sessionId, sessionId); + assert.equal(outboxQueries[0].traceId, traceId); + } finally { + await close(server); + } +}); + test("workbench trace event detail route declares detail-only authority", async () => { const sessionId = "ses_realtime_authority_detail"; const traceId = "trc_realtime_authority_detail"; @@ -168,9 +194,19 @@ function createAccessController() { function createRuntime({ facts, outboxRows = [], outboxQueries = [] }) { return { - async readWorkbenchProjectionOutbox(params = {}) { + async readAtomicWorkbenchProjectionSync(params = {}) { outboxQueries.push({ ...params }); - return outboxRows.filter((row) => Number(row.outboxSeq) > Number(params.afterSeq ?? 0)); + const after = Number(params.afterOutboxSeq ?? params.afterSeq ?? 0); + const events = outboxRows.filter((row) => Number(row.outboxSeq) > after); + const cutoffOutboxSeq = outboxRows.reduce((max, row) => Math.max(max, Number(row.outboxSeq) || 0), 0); + return { + facts: filterFacts(facts, params), + events, + cutoffOutboxSeq, + cursorOutboxSeq: cutoffOutboxSeq, + hasMore: false, + valuesRedacted: true + }; }, async queryWorkbenchFacts(params = {}) { return { @@ -310,44 +346,6 @@ function parseSseBlock(block) { return { event, id, data: JSON.parse(dataLine) }; } -function createFakeKafkaFactory() { - let eachMessage = null; - let offset = 0; - let subscribedTopic = "hwlab.event.v1"; - const consumer = { - connect: async () => undefined, - subscribe: async (input = {}) => { subscribedTopic = String(input.topic ?? subscribedTopic); }, - run: async (input = {}) => { eachMessage = input.eachMessage; }, - stop: async () => undefined, - disconnect: async () => undefined - }; - return { - factory: () => ({ consumer: () => consumer }), - emit: async (value = {}) => { - await waitFor(() => typeof eachMessage === "function"); - await eachMessage({ - topic: subscribedTopic, - partition: 0, - message: { - offset: String(offset++), - key: Buffer.from(value.traceId ?? value.sessionId ?? "fake"), - timestamp: String(Date.now()), - value: Buffer.from(JSON.stringify(value)) - } - }); - } - }; -} - -async function waitFor(predicate, timeoutMs = 1000) { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - if (predicate()) return; - await new Promise((resolve) => setTimeout(resolve, 10)); - } - throw new Error("waitFor timed out"); -} - function listen(server) { return new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); } diff --git a/internal/cloud/workbench-realtime-authority.ts b/internal/cloud/workbench-realtime-authority.ts index 98a1735d..b1703d1a 100644 --- a/internal/cloud/workbench-realtime-authority.ts +++ b/internal/cloud/workbench-realtime-authority.ts @@ -4,12 +4,12 @@ */ import { safeSessionId, safeTraceId, sendJson } from "./server-http-utils.ts"; import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts"; +import { projectionOutboxRealtimeEvents } from "./workbench-projection-outbox-events.ts"; const SYNC_CONTRACT_VERSION = "workbench-sync-v1"; const REALTIME_AUTHORITY_VERSION = "workbench-realtime-authority-v2"; const DEFAULT_SYNC_LIMIT = 100; const MAX_SYNC_LIMIT = 500; -const SYNC_FACT_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "traceEvents", "checkpoints"]); export async function handleWorkbenchSyncHttp(request, response, url, options = {}, actor = null) { if (request.method !== "GET") { @@ -30,12 +30,12 @@ export async function handleWorkbenchSyncHttp(request, response, url, options = return; } const runtime = workbenchRealtimeRuntime(request, options); - if (typeof runtime?.queryWorkbenchFacts !== "function" || typeof runtime?.readWorkbenchProjectionOutbox !== "function") { + if (typeof runtime?.readAtomicWorkbenchProjectionSync !== "function") { sendJson(response, 503, { ok: false, error: { code: "workbench_sync_runtime_unconfigured", - message: "Workbench sync requires durable facts and outbox readers.", + message: "Workbench sync requires one atomic durable projection snapshot reader.", valuesRedacted: true } }); @@ -45,27 +45,25 @@ export async function handleWorkbenchSyncHttp(request, response, url, options = try { const since = realtimeSinceCursor(request, url); const limit = boundedSyncLimit(url.searchParams.get("limit")); - const outboxQuery = { afterSeq: since, limit }; - if (sessionId) outboxQuery.sessionId = sessionId; - else outboxQuery.traceId = traceId; - const rows = await runtime.readWorkbenchProjectionOutbox(outboxQuery); - const events = rows.map(workbenchRealtimeEntityDelta); - const latestOutboxSeq = events.reduce((max, event) => Math.max(max, Number(event.cursor?.outboxSeq ?? 0)), since); - const facts = await runtime.queryWorkbenchFacts({ - families: SYNC_FACT_FAMILIES, - ...(sessionId ? { sessionId } : { traceId }), + const snapshot = await runtime.readAtomicWorkbenchProjectionSync({ + afterOutboxSeq: since, + afterSeq: since, limit, - actor: actor ? { id: actor.id, role: actor.role ?? "user", valuesRedacted: true } : undefined + ...(sessionId ? { sessionId } : {}), + ...(traceId ? { traceId } : {}), + actor: actor ? { id: actor.id, role: actor.role ?? "user" } : undefined }); - if (facts?.error) throw facts.error; - const delta = normalizeSyncFacts(facts?.facts); + const rows = Array.isArray(snapshot?.events) ? snapshot.events : []; + const events = projectionOutboxRealtimeEvents({ events: rows, facts: {} }).map((item) => item.payload); + const latestOutboxSeq = nonNegativeInteger(snapshot?.cursorOutboxSeq); + const delta = normalizeSyncFacts(snapshot?.facts); sendJson(response, 200, { ok: true, status: "succeeded", contractVersion: SYNC_CONTRACT_VERSION, realtimeAuthority: REALTIME_AUTHORITY_VERSION, scope: { - kind: sessionId ? "session" : "trace", + kind: traceId ? "trace" : "session", sessionId, traceId, since @@ -73,7 +71,8 @@ export async function handleWorkbenchSyncHttp(request, response, url, options = cursor: { outboxSeq: latestOutboxSeq, since, - hasMore: rows.length >= limit + cutoffOutboxSeq: nonNegativeInteger(snapshot?.cutoffOutboxSeq), + hasMore: snapshot?.hasMore === true }, events, delta, @@ -101,48 +100,8 @@ export async function handleWorkbenchSyncHttp(request, response, url, options = } } -export function workbenchRealtimeEntityDelta(row = {}) { - const commitType = textValue(row.commitType) || "event"; - const projectionRevision = nonNegativeInteger(row.projectionRevision ?? row.projectedSeq ?? row.outboxSeq); - const family = workbenchRealtimeEntityFamily(row, commitType); - const id = workbenchRealtimeEntityId(row, family); - const outboxSeq = nonNegativeInteger(row.outboxSeq); - const projectedSeq = nonNegativeInteger(row.projectedSeq); - return { - type: "workbench.entity.delta", - family, - id, - entity: { - family, - id, - version: projectionRevision, - entityVersion: projectionRevision - }, - cursor: { - outboxSeq, - traceSeq: projectedSeq - }, - sessionId: safeSessionId(row.sessionId) ?? null, - turnId: textValue(row.turnId) || null, - traceId: safeTraceId(row.traceId) ?? null, - messageId: textValue(row.messageId) || null, - commitType, - eventSeq: row.eventSeq === null || row.eventSeq === undefined ? null : nonNegativeInteger(row.eventSeq), - aggregateId: textValue(row.aggregateId) || null, - aggregateSeq: nonNegativeInteger(row.aggregateSeq), - projectionRevision, - entityVersion: projectionRevision, - terminal: row.terminal === true, - sealed: row.sealed === true, - serverCommittedAt: textValue(row.createdAt) || null, - payload: redactedPayload(row.payload), - valuesRedacted: true - }; -} - function workbenchRealtimeRuntime(request, options = {}) { return options.workbenchRuntime - ?? options.runtimeStore ?? createWorkbenchRuntimeClient({ env: options.env ?? process.env, fetch: options.fetch, logger: options.logger, traceparent: request?.hwlabHttpRequestContext?.traceparent }); } @@ -175,31 +134,6 @@ function normalizeSyncFacts(facts = {}) { }; } -function workbenchRealtimeEntityFamily(row = {}, commitType = "event") { - if (commitType === "terminal" || row.terminal === true || row.sealed === true) return "turns"; - if (commitType === "message" || row.messageId) return "messages"; - if (row.traceId) return "traceEvents"; - if (row.sessionId) return "sessions"; - return "diagnostics"; -} - -function workbenchRealtimeEntityId(row = {}, family = "diagnostics") { - if (family === "messages") return textValue(row.messageId) || textValue(row.turnId) || textValue(row.traceId) || `outbox:${nonNegativeInteger(row.outboxSeq)}`; - if (family === "turns") return textValue(row.turnId) || textValue(row.traceId) || `outbox:${nonNegativeInteger(row.outboxSeq)}`; - if (family === "traceEvents") return textValue(row.sourceEventId) || [textValue(row.traceId), nonNegativeInteger(row.projectedSeq)].filter(Boolean).join(":") || `outbox:${nonNegativeInteger(row.outboxSeq)}`; - if (family === "sessions") return textValue(row.sessionId) || `outbox:${nonNegativeInteger(row.outboxSeq)}`; - return textValue(row.aggregateId) || `outbox:${nonNegativeInteger(row.outboxSeq)}`; -} - -function redactedPayload(payload) { - if (!payload || typeof payload !== "object" || Array.isArray(payload)) return null; - return { - kind: textValue(payload.kind ?? payload.type ?? payload.eventType) || null, - status: textValue(payload.status) || null, - valuesRedacted: true - }; -} - function factArray(value) { return Array.isArray(value) ? value : []; } diff --git a/internal/cloud/workbench-runtime-client.ts b/internal/cloud/workbench-runtime-client.ts index f54b13f9..8e95003b 100644 --- a/internal/cloud/workbench-runtime-client.ts +++ b/internal/cloud/workbench-runtime-client.ts @@ -17,6 +17,7 @@ export function createWorkbenchRuntimeClient(options = {}) { queryWorkbenchFacts: (params = {}) => postRuntime({ fetchFn, baseUrl, timeoutMs, traceparent, path: "/v1/workbench-runtime/facts", body: params }), queryAgentTraceEvents: (params = {}) => postRuntime({ fetchFn, baseUrl, timeoutMs, traceparent, path: "/v1/workbench-runtime/trace-events", body: params }), listWorkbenchSessions: (params = {}) => postRuntime({ fetchFn, baseUrl, timeoutMs, traceparent, path: "/v1/workbench-runtime/sessions", body: params }), + readAtomicWorkbenchProjectionSync: (params = {}) => postRuntime({ fetchFn, baseUrl, timeoutMs, traceparent, path: "/v1/workbench-runtime/sync", body: params }), readWorkbenchProjectionOutbox: (params = {}) => postRuntime({ fetchFn, baseUrl, timeoutMs, traceparent, path: "/v1/workbench-runtime/projection-outbox", body: params }).then((result) => Array.isArray(result?.rows) ? result.rows : []) }; } diff --git a/internal/db/migrations/0008_workbench_kafka_realtime.sql b/internal/db/migrations/0008_workbench_kafka_realtime.sql new file mode 100644 index 00000000..9c225562 --- /dev/null +++ b/internal/db/migrations/0008_workbench_kafka_realtime.sql @@ -0,0 +1,106 @@ +-- Versioned v8 upgrade for immutable Workbench replay and the AgentRun Kafka projector. + +ALTER TABLE workbench_projection_outbox ADD COLUMN IF NOT EXISTS outbox_event_id TEXT; +ALTER TABLE workbench_projection_outbox ADD COLUMN IF NOT EXISTS entity_family TEXT; +ALTER TABLE workbench_projection_outbox ADD COLUMN IF NOT EXISTS entity_id TEXT; + +-- Pre-v8 rows did not store immutable fact payloads. Mark them as cutover rows so +-- recovery advances its cursor without fabricating message/turn/trace entities. +UPDATE workbench_projection_outbox +SET outbox_event_id = COALESCE(outbox_event_id, 'legacy:' || outbox_seq::text), + entity_family = COALESCE(entity_family, 'legacy'), + entity_id = COALESCE(entity_id, 'legacy:' || outbox_seq::text) +WHERE outbox_event_id IS NULL OR entity_family IS NULL OR entity_id IS NULL; + +ALTER TABLE workbench_projection_outbox ALTER COLUMN outbox_event_id SET NOT NULL; +ALTER TABLE workbench_projection_outbox ALTER COLUMN entity_family SET NOT NULL; +ALTER TABLE workbench_projection_outbox ALTER COLUMN entity_id SET NOT NULL; +DROP INDEX IF EXISTS idx_workbench_projection_outbox_event_id; +CREATE UNIQUE INDEX idx_workbench_projection_outbox_event_id ON workbench_projection_outbox(outbox_event_id); + +CREATE TABLE IF NOT EXISTS workbench_kafka_inbox ( + source_topic TEXT NOT NULL, + source_partition INTEGER NOT NULL, + source_offset BIGINT NOT NULL, + source_key TEXT, + source_event_id TEXT, + input_sha256 TEXT NOT NULL, + trace_id TEXT, + session_id TEXT, + run_id TEXT, + command_id TEXT, + source_seq INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'processing', + projected_seq INTEGER, + envelope_json TEXT NOT NULL DEFAULT '{}', + error_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + projected_at TEXT, + PRIMARY KEY (source_topic, source_partition, source_offset) +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_workbench_kafka_inbox_source_event ON workbench_kafka_inbox(source_topic, source_event_id) WHERE source_event_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_workbench_kafka_inbox_status_updated ON workbench_kafka_inbox(status, updated_at, source_topic, source_partition, source_offset); +CREATE INDEX IF NOT EXISTS idx_workbench_kafka_inbox_trace ON workbench_kafka_inbox(trace_id, source_seq); + +CREATE TABLE IF NOT EXISTS workbench_kafka_dlq ( + dlq_seq BIGSERIAL PRIMARY KEY, + source_topic TEXT NOT NULL, + source_partition INTEGER NOT NULL, + source_offset BIGINT NOT NULL, + source_event_id TEXT, + input_sha256 TEXT NOT NULL, + error_code TEXT NOT NULL, + error_message TEXT NOT NULL, + envelope_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_workbench_kafka_dlq_transport ON workbench_kafka_dlq(source_topic, source_partition, source_offset); + +CREATE TABLE IF NOT EXISTS hwlab_kafka_outbox ( + outbox_seq BIGSERIAL PRIMARY KEY, + event_id TEXT NOT NULL UNIQUE, + topic TEXT NOT NULL, + partition_key TEXT NOT NULL, + payload_json TEXT NOT NULL, + headers_json TEXT NOT NULL DEFAULT '{}', + attempt_count INTEGER NOT NULL DEFAULT 0, + next_attempt_at TEXT NOT NULL, + lease_owner TEXT, + lease_expires_at TEXT, + published_at TEXT, + last_error TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_hwlab_kafka_outbox_due ON hwlab_kafka_outbox(published_at, next_attempt_at, lease_expires_at, outbox_seq); + +CREATE OR REPLACE FUNCTION notify_hwlab_workbench_projection() RETURNS trigger AS $$ +BEGIN + PERFORM pg_notify( + 'hwlab_workbench_projection', + json_build_object( + 'sessionId', NEW.session_id, + 'traceId', NEW.trace_id, + 'outboxSeq', NEW.outbox_seq + )::text + ); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_notify_hwlab_workbench_projection ON workbench_projection_outbox; +CREATE TRIGGER trg_notify_hwlab_workbench_projection +AFTER INSERT ON workbench_projection_outbox +FOR EACH ROW EXECUTE FUNCTION notify_hwlab_workbench_projection(); + +INSERT INTO hwlab_schema_migrations (id, schema_version, applied_at, migration_json) +VALUES ( + '0008_workbench_kafka_realtime', + 'runtime-durable-postgres-v8', + CURRENT_TIMESTAMP, + '{"path":"internal/db/migrations/0008_workbench_kafka_realtime.sql","runtime":"cloud-api"}' +) +ON CONFLICT (id) DO UPDATE SET + schema_version = EXCLUDED.schema_version, + migration_json = EXCLUDED.migration_json; diff --git a/internal/db/runtime-store-core.ts b/internal/db/runtime-store-core.ts index 07678476..cdbe503c 100644 --- a/internal/db/runtime-store-core.ts +++ b/internal/db/runtime-store-core.ts @@ -16,7 +16,7 @@ import { deriveProtocolActorFromMeta } from "../audit/index.mjs"; import { - CLOUD_CORE_MIGRATION_ID, + CLOUD_RUNTIME_DURABLE_MIGRATION_ID, CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS @@ -62,7 +62,10 @@ export const postgresCountTables = Object.freeze([ "workbench_event_sequences", "workbench_events", "workbench_session_inputs", - "workbench_projection_checkpoints" + "workbench_projection_checkpoints", + "workbench_kafka_inbox", + "workbench_kafka_dlq", + "hwlab_kafka_outbox" ]); export const postgresRuntimeReadIndexes = Object.freeze([ @@ -1343,7 +1346,7 @@ export function notCheckedRuntimeMigration() { checked: false, ready: false, table: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, - requiredMigrationId: CLOUD_CORE_MIGRATION_ID, + requiredMigrationId: CLOUD_RUNTIME_DURABLE_MIGRATION_ID, requiredSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, appliedMigrationId: null, appliedSchemaVersion: null, diff --git a/internal/db/runtime-store-memory.ts b/internal/db/runtime-store-memory.ts index 61fe2aff..761626bf 100644 --- a/internal/db/runtime-store-memory.ts +++ b/internal/db/runtime-store-memory.ts @@ -16,7 +16,6 @@ import { deriveProtocolActorFromMeta } from "../audit/index.mjs"; import { - CLOUD_CORE_MIGRATION_ID, CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS diff --git a/internal/db/runtime-store-postgres-kafka.test.ts b/internal/db/runtime-store-postgres-kafka.test.ts new file mode 100644 index 00000000..dc56a647 --- /dev/null +++ b/internal/db/runtime-store-postgres-kafka.test.ts @@ -0,0 +1,179 @@ +import assert from "node:assert/strict"; +import { test } from "bun:test"; + +import { buildWorkbenchProjectionEventFacts } from "../cloud/workbench-projection-writer.ts"; +import { projectionOutboxRealtimeEvents } from "../cloud/server-workbench-realtime-http.ts"; +import { + claimHwlabKafkaOutbox, + commitAgentRunKafkaProjection, + completeHwlabKafkaOutbox, + projectionOutboxEventId, + recordFailedAgentRunKafkaMessage, + retryHwlabKafkaOutbox +} from "./runtime-store-postgres-kafka.ts"; + +test("malformed duplicate source event at a new offset is durably DLQed without mutating projected inbox", async () => { + const queries = []; + const client = { + async query(sql, params) { + queries.push({ sql, params }); + if (sql.startsWith("SELECT pg_advisory_xact_lock")) return { rows: [] }; + if (sql.startsWith("SELECT source_topic")) return { rows: [{ source_topic: "agentrun.event.v1", source_partition: 0, source_offset: "10", input_sha256: "prior-hash", status: "projected", projected_seq: 7 }] }; + if (sql.startsWith("INSERT INTO workbench_kafka_dlq")) return { rows: [] }; + throw new Error(`unexpected SQL: ${sql}`); + } + }; + const store = transactionStore(client); + const result = await recordFailedAgentRunKafkaMessage(store, { + transport: { sourceTopic: "agentrun.event.v1", sourcePartition: 0, sourceOffset: "11", sourceKey: "run-1", inputSha256: "new-hash" }, + sourceEventId: "evt-duplicate", + envelope: { eventId: "evt-duplicate", valuesPrinted: false }, + error: { code: "workbench_kafka_schema_invalid", message: "malformed" } + }); + + assert.equal(result.recorded, true); + assert.equal(result.conflict, true); + assert.equal(result.priorStatus, "projected"); + assert.equal(queries[0].params[0], "workbench-kafka-inbox:agentrun.event.v1:event:evt-duplicate"); + assert.equal(queries.some(({ sql }) => sql.startsWith("UPDATE workbench_kafka_inbox")), false); + assert.equal(queries.some(({ sql }) => sql.startsWith("INSERT INTO workbench_kafka_dlq")), true); +}); + +test("projection outbox derives a stable non-null identity when a fact has no source event", () => { + const input = { + fact: { traceId: "trc-derived", projectedSeq: 3, sourceSeq: 2 }, + event: { aggregateId: "trc-derived", aggregateSeq: 3, projectionRevision: 3 }, + family: "traceEvents", + entityId: "wte-derived", + commitType: "event" + }; + const first = projectionOutboxEventId(input); + assert.match(first, /^projection:[a-f0-9]{64}$/u); + assert.equal(projectionOutboxEventId(input), first); + assert.notEqual(projectionOutboxEventId({ ...input, event: { ...input.event, projectionRevision: 4 } }), first); +}); + +test("Kafka projection preserves admission owner and scope fields", async () => { + let persistedSession = null; + const client = { + async query(sql) { + if (sql.startsWith("SELECT pg_advisory_xact_lock")) return { rows: [] }; + if (sql.startsWith("SELECT source_topic")) return { rows: [] }; + if (sql.startsWith("INSERT INTO workbench_kafka_inbox")) return { rows: [] }; + if (sql.startsWith("SELECT checkpoint_json")) return { rows: [] }; + if (sql.startsWith("SELECT owner_user_id")) return { rows: [{ owner_user_id: "usr-owner", project_id: "prj-owner", conversation_id: "cnv-owner", thread_id: "thread-owner", session_json: { sessionId: "ses-owner", ownerUserId: "usr-owner", ownerRole: "user", launchContext: { source: "workbench" } } }] }; + if (sql.startsWith("SELECT GREATEST")) return { rows: [{ max_projected_seq: 0 }] }; + if (sql.startsWith("INSERT INTO hwlab_kafka_outbox")) return { rows: [] }; + if (sql.startsWith("UPDATE workbench_kafka_inbox")) return { rows: [] }; + throw new Error(`unexpected SQL: ${sql}`); + } + }; + const store = { + ...transactionStore(client), + memory: { writeWorkbenchFacts() {} }, + async persistWorkbenchAggregateEvent(event) { return { ...event, eventSeq: 1, aggregateSeq: 1, projectionRevision: 1 }; }, + async persistWorkbenchSessionInputFact() {}, + async persistWorkbenchSessionFact(fact) { persistedSession = fact; }, + async persistWorkbenchMessageFact() {}, + async persistWorkbenchPartFact() {}, + async persistWorkbenchTurnFact() {}, + async persistWorkbenchTraceEventFact() {}, + async persistWorkbenchProjectionCheckpoint() {}, + async persistWorkbenchProjectionOutbox() {} + }; + await commitAgentRunKafkaProjection(store, { + transport: { sourceTopic: "agentrun.event.v1", sourcePartition: 0, sourceOffset: "12", sourceKey: "run-owner", inputSha256: "hash-owner" }, + canonicalEvent: { eventId: "evt-owner", sourceSeq: 1, traceId: "trc-owner", hwlabSessionId: "ses-owner", runId: "run-owner", commandId: "cmd-owner" }, + projectedEvent: { traceId: "trc-owner", sessionId: "ses-owner" }, + factsFactory: ({ projectedSeq, projectedAt }) => ({ + written: true, + facts: { + sessions: [{ sessionId: "ses-owner", status: "running", lastTraceId: "trc-owner", projectedSeq, sourceSeq: 1, sourceEventId: "evt-owner", terminal: false, sealed: false, updatedAt: projectedAt }], + messages: [], parts: [], turns: [], traceEvents: [], checkpoints: [] + } + }), + hwlabEvent: { eventId: "hwlab:evt-owner" }, + hwlabTopic: "hwlab.event.v1", + partitionKey: "trc-owner", + headers: {} + }); + + assert.equal(persistedSession.ownerUserId, "usr-owner"); + assert.equal(persistedSession.ownerRole, "user"); + assert.equal(persistedSession.projectId, "prj-owner"); + assert.equal(persistedSession.conversationId, "cnv-owner"); + assert.equal(persistedSession.threadId, "thread-owner"); + assert.deepEqual(persistedSession.launchContext, { source: "workbench" }); +}); + +test("terminal Kafka transaction persists sealed turn and immutable SSE snapshot", async () => { + let persistedTurn = null; + const outbox = []; + const client = { + async query(sql) { + if (sql.startsWith("SELECT pg_advisory_xact_lock")) return { rows: [] }; + if (sql.startsWith("SELECT source_topic")) return { rows: [] }; + if (sql.startsWith("INSERT INTO workbench_kafka_inbox")) return { rows: [] }; + if (sql.startsWith("SELECT checkpoint_json")) return { rows: [{ checkpoint_json: { traceId: "trc_terminal", sessionId: "ses_terminal", assistantText: "final body", finalResponse: { text: "final body", status: "completed" }, projectedSeq: 2, sourceSeq: 2, terminal: true, sealed: true } }] }; + if (sql.startsWith("SELECT owner_user_id")) return { rows: [{ owner_user_id: "usr-owner", session_json: { sessionId: "ses_terminal", ownerUserId: "usr-owner" } }] }; + if (sql.startsWith("SELECT GREATEST")) return { rows: [{ max_projected_seq: 2 }] }; + if (sql.startsWith("INSERT INTO hwlab_kafka_outbox") || sql.startsWith("UPDATE workbench_kafka_inbox")) return { rows: [] }; + throw new Error(`unexpected SQL: ${sql}`); + } + }; + const store = { + ...transactionStore(client), + memory: { writeWorkbenchFacts() {} }, + async persistWorkbenchAggregateEvent(event) { return { ...event, eventSeq: 3, aggregateSeq: 3, projectionRevision: 3 }; }, + async persistWorkbenchSessionInputFact() {}, async persistWorkbenchSessionFact() {}, async persistWorkbenchMessageFact() {}, async persistWorkbenchPartFact() {}, + async persistWorkbenchTurnFact(fact) { persistedTurn = fact; }, async persistWorkbenchTraceEventFact() {}, async persistWorkbenchProjectionCheckpoint() {}, + async persistWorkbenchProjectionOutbox(record) { outbox.push(record); } + }; + await commitAgentRunKafkaProjection(store, { + transport: { sourceTopic: "agentrun.event.v1", sourcePartition: 0, sourceOffset: "20", inputSha256: "hash-terminal" }, + canonicalEvent: { eventId: "evt-terminal", sourceSeq: 3, traceId: "trc_terminal", hwlabSessionId: "ses_terminal", runId: "run-terminal", commandId: "cmd-terminal" }, + projectedEvent: { traceId: "trc_terminal", sessionId: "ses_terminal", sourceEventId: "evt-terminal", sourceSeq: 3, type: "result", eventType: "terminal", status: "completed", terminal: true, createdAt: "2026-07-10T10:00:03.000Z" }, + factsFactory: ({ projectedSeq, previousCheckpoint, projectedAt }) => buildWorkbenchProjectionEventFacts({ projectedSeq, previousCheckpoint, projectedAt, event: { traceId: "trc_terminal", sessionId: "ses_terminal", sourceEventId: "evt-terminal", sourceSeq: 3, type: "result", eventType: "terminal", status: "completed", terminal: true, createdAt: "2026-07-10T10:00:03.000Z" } }), + hwlabEvent: { eventId: "hwlab:evt-terminal" }, hwlabTopic: "hwlab.event.v1", partitionKey: "trc_terminal", headers: {} + }); + + assert.equal(persistedTurn.terminal, true); + assert.equal(persistedTurn.sealed, true); + assert.equal(persistedTurn.finalResponse.text, "final body"); + const turnRow = outbox.find((row) => row.entityFamily === "turns"); + assert.equal(turnRow.payload.fact.assistantText, "final body"); + const [sse] = projectionOutboxRealtimeEvents({ events: [{ ...turnRow, outboxSeq: 9 }], facts: {} }); + assert.equal(sse.name, "workbench.turn.snapshot"); + assert.equal(sse.payload.turn.finalResponse.text, "final body"); +}); + +test("HWLAB outbox claim and completion use attempt and lease fencing", async () => { + const calls = []; + const row = { outbox_seq: 3, event_id: "evt-3", topic: "hwlab.event.v1", partition_key: "trc-3", payload_json: {}, headers_json: {}, attempt_count: 4, lease_owner: "relay-a", lease_expires_at: "2026-07-10T10:05:00.000Z", next_attempt_at: "2026-07-10T10:00:00.000Z", created_at: "2026-07-10T10:00:00.000Z" }; + const store = { + now: () => "2026-07-10T10:04:00.000Z", + async withDurableTransaction(_label, operation) { return operation({ query: async (sql, params) => { calls.push({ sql, params }); return { rows: [row] }; } }); }, + async query(sql, params) { calls.push({ sql, params }); return { rows: [{ outbox_seq: 3 }] }; } + }; + const [item] = await claimHwlabKafkaOutbox(store, { owner: "relay-a", leaseMs: 30000, limit: 1 }); + await completeHwlabKafkaOutbox(store, item); + await retryHwlabKafkaOutbox(store, item, new Error("retry"), "2026-07-10T10:04:30.000Z"); + + assert.match(calls[0].sql, /attempt_count = o\.attempt_count \+ 1/u); + assert.match(calls[0].sql, /NOT EXISTS/u); + assert.match(calls[1].sql, /lease_owner = \$3 AND attempt_count = \$4 AND lease_expires_at = \$5 AND lease_expires_at > \$1/u); + assert.deepEqual(calls[1].params.slice(2), ["relay-a", 4, "2026-07-10T10:05:00.000Z"]); + assert.match(calls[2].sql, /lease_owner = \$5 AND attempt_count = \$6 AND lease_expires_at = \$7 AND lease_expires_at > \$3/u); +}); + +test("stale HWLAB outbox fencing token cannot complete a claim", async () => { + const store = { now: () => "2026-07-10T10:06:00.000Z", async query() { return { rows: [] }; } }; + await assert.rejects(completeHwlabKafkaOutbox(store, { outboxSeq: 3, leaseOwner: "relay-old", attemptCount: 1, leaseExpiresAt: "2026-07-10T10:05:00.000Z" }), (error) => error?.code === "hwlab_kafka_outbox_lease_conflict"); +}); + +function transactionStore(client) { + return { + now: () => "2026-07-10T10:00:00.000Z", + async withDurableTransaction(_label, operation) { return operation(client); } + }; +} diff --git a/internal/db/runtime-store-postgres-kafka.ts b/internal/db/runtime-store-postgres-kafka.ts new file mode 100644 index 00000000..72ae184c --- /dev/null +++ b/internal/db/runtime-store-postgres-kafka.ts @@ -0,0 +1,410 @@ +/* + * Durable AgentRun Kafka projector storage operations. + * Kafka I/O is intentionally outside every transaction in this module. + */ +import { createHash } from "node:crypto"; + +import { + indexWorkbenchAggregateEvents, + inputFactWithAggregateSeq, + nonNegativeInteger, + normalizeWorkbenchAggregateEventsForFacts, + normalizeWorkbenchFacts, + parseJsonColumn, + stableJson, + textOr +} from "./runtime-store-core.ts"; + +export async function commitAgentRunKafkaProjection(store, params = {}) { + const transport = normalizeTransport(params.transport); + const canonicalEvent = objectValue(params.canonicalEvent); + const projectedEvent = objectValue(params.projectedEvent); + const sourceEventId = requiredText(canonicalEvent.eventId, "canonical AgentRun eventId"); + const traceId = requiredText(projectedEvent.traceId ?? canonicalEvent.traceId, "canonical AgentRun traceId"); + const sessionId = text(projectedEvent.sessionId ?? canonicalEvent.hwlabSessionId); + const sourceSeq = nonNegativeInteger(canonicalEvent.sourceSeq ?? canonicalEvent.event?.seq); + const inputSha256 = requiredText(transport.inputSha256, "Kafka input sha256"); + const now = store.now(); + const result = await store.withDurableTransaction("workbench.kafka-projection.commit", async (client) => { + await lockKafkaInboxIdentity(client, transport, sourceEventId); + await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`workbench-projection:${traceId}`]); + const existing = await client.query( + "SELECT source_topic, source_partition, source_offset, source_event_id, input_sha256, status, projected_seq FROM workbench_kafka_inbox WHERE (source_topic = $1 AND source_partition = $2 AND source_offset = $3) OR (source_topic = $1 AND source_event_id = $4) ORDER BY created_at ASC LIMIT 1", + [transport.sourceTopic, transport.sourcePartition, transport.sourceOffset, sourceEventId] + ); + const prior = existing.rows?.[0] ?? null; + if (prior) { + if (prior.input_sha256 !== inputSha256) { + const diagnostic = { code: "workbench_kafka_inbox_hash_conflict", message: "Kafka transport or source event identity was reused with a different payload hash.", valuesRedacted: true }; + if (prior.status !== "projected") { + await client.query( + "UPDATE workbench_kafka_inbox SET status = 'failed', error_json = $1, updated_at = $2 WHERE source_topic = $3 AND source_partition = $4 AND source_offset = $5", + [stableJson(diagnostic), now, prior.source_topic, prior.source_partition, prior.source_offset] + ); + } + await insertDlq(client, { ...transport, sourceEventId, inputSha256, error: diagnostic, envelope: redactedEnvelope(canonicalEvent), createdAt: now }); + return { conflict: true, diagnostic, duplicate: false, projectedSeq: nonNegativeInteger(prior.projected_seq) }; + } + return { duplicate: true, conflict: false, status: prior.status, projectedSeq: nonNegativeInteger(prior.projected_seq) }; + } + await client.query( + "INSERT INTO workbench_kafka_inbox (source_topic, source_partition, source_offset, source_key, source_event_id, input_sha256, trace_id, session_id, run_id, command_id, source_seq, status, projected_seq, envelope_json, error_json, created_at, updated_at, projected_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,'processing',NULL,$12,'{}',$13,$13,NULL)", + [transport.sourceTopic, transport.sourcePartition, transport.sourceOffset, transport.sourceKey, sourceEventId, inputSha256, traceId, sessionId, text(canonicalEvent.runId), text(canonicalEvent.commandId), sourceSeq, stableJson(redactedEnvelope(canonicalEvent)), now] + ); + const checkpointResult = await client.query("SELECT checkpoint_json FROM workbench_projection_checkpoints WHERE trace_id = $1 FOR UPDATE", [traceId]); + const previousCheckpoint = parseJsonColumn(checkpointResult.rows?.[0]?.checkpoint_json, null); + const sessionResult = sessionId + ? await client.query("SELECT owner_user_id, project_id, conversation_id, thread_id, session_json FROM workbench_sessions WHERE session_id = $1 FOR UPDATE", [sessionId]) + : { rows: [] }; + const existingSession = sessionResult.rows?.[0] ?? null; + const maxResult = await client.query( + "SELECT GREATEST(COALESCE((SELECT MAX(projected_seq) FROM workbench_trace_events WHERE trace_id = $1), 0), COALESCE((SELECT projected_seq FROM workbench_projection_checkpoints WHERE trace_id = $1), 0))::int AS max_projected_seq", + [traceId] + ); + const projectedSeq = nonNegativeInteger(maxResult.rows?.[0]?.max_projected_seq) + 1; + if (typeof params.factsFactory !== "function") throw codedError("workbench_kafka_facts_factory_required", "Kafka projection requires a pure facts factory."); + const built = await params.factsFactory({ projectedSeq, previousCheckpoint, projectedAt: now }); + let facts = null; + let persistedEvents = []; + if (built?.written !== false) { + facts = normalizeWorkbenchFacts(mergeKafkaSessionIdentity(built?.facts ?? {}, existingSession), params.requestMeta ?? {}, now); + const persisted = await persistFacts(store, client, facts, params.requestMeta ?? {}); + persistedEvents = persisted.events; + } + const hwlabEvent = objectValue(params.hwlabEvent); + const hwlabEventId = requiredText(hwlabEvent.eventId, "HWLAB Kafka eventId"); + await client.query( + "INSERT INTO hwlab_kafka_outbox (event_id, topic, partition_key, payload_json, headers_json, attempt_count, next_attempt_at, lease_owner, lease_expires_at, published_at, last_error, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,0,$6,NULL,NULL,NULL,NULL,$6,$6) ON CONFLICT (event_id) DO NOTHING", + [hwlabEventId, requiredText(params.hwlabTopic, "HWLAB Kafka topic"), requiredText(params.partitionKey, "HWLAB Kafka partition key"), stableJson(hwlabEvent), stableJson(objectValue(params.headers)), now] + ); + await client.query( + "UPDATE workbench_kafka_inbox SET status = 'projected', projected_seq = $1, updated_at = $2, projected_at = $2 WHERE source_topic = $3 AND source_partition = $4 AND source_offset = $5", + [projectedSeq, now, transport.sourceTopic, transport.sourcePartition, transport.sourceOffset] + ); + return { duplicate: false, conflict: false, projectedSeq, facts, events: persistedEvents, suppressedAfterSeal: built?.suppressedAfterSeal === true }; + }); + if (result.facts) store.memory.writeWorkbenchFacts({ facts: result.facts }, params.requestMeta ?? {}); + return { ...result, transport, sourceEventId, traceId, sessionId, sourceSeq, valuesPrinted: false }; +} + +export async function recordFailedAgentRunKafkaMessage(store, params = {}) { + const transport = normalizeTransport(params.transport); + const now = store.now(); + const error = redactedError(params.error); + const sourceEventId = text(params.sourceEventId); + const inputSha256 = requiredText(transport.inputSha256, "Kafka input sha256"); + const result = await store.withDurableTransaction("workbench.kafka-projection.dlq", async (client) => { + await lockKafkaInboxIdentity(client, transport, sourceEventId); + const existing = await client.query("SELECT source_topic, source_partition, source_offset, input_sha256, status, projected_seq FROM workbench_kafka_inbox WHERE (source_topic = $1 AND source_partition = $2 AND source_offset = $3) OR ($4::text IS NOT NULL AND source_topic = $1 AND source_event_id = $4) ORDER BY created_at ASC LIMIT 1 FOR UPDATE", [transport.sourceTopic, transport.sourcePartition, transport.sourceOffset, sourceEventId]); + const prior = existing.rows?.[0] ?? null; + if (prior) { + const sameTransport = prior.source_partition === transport.sourcePartition && String(prior.source_offset) === transport.sourceOffset; + const diagnostic = prior.input_sha256 !== inputSha256 + ? { ...error, code: "workbench_kafka_inbox_hash_conflict" } + : sameTransport ? error : { ...error, code: "workbench_kafka_inbox_source_event_duplicate" }; + if (prior.status !== "projected") { + await client.query("UPDATE workbench_kafka_inbox SET status = 'failed', error_json = $1, updated_at = $2 WHERE source_topic = $3 AND source_partition = $4 AND source_offset = $5", [stableJson(diagnostic), now, prior.source_topic, prior.source_partition, prior.source_offset]); + } + await insertDlq(client, { ...transport, sourceEventId, inputSha256, error: diagnostic, envelope: redactedEnvelope(params.envelope), createdAt: now }); + return { duplicate: sameTransport, conflict: !sameTransport || prior.input_sha256 !== inputSha256, priorStatus: prior.status, projectedSeq: nonNegativeInteger(prior.projected_seq), diagnostic }; + } + await client.query( + "INSERT INTO workbench_kafka_inbox (source_topic, source_partition, source_offset, source_key, source_event_id, input_sha256, trace_id, session_id, run_id, command_id, source_seq, status, projected_seq, envelope_json, error_json, created_at, updated_at, projected_at) VALUES ($1,$2,$3,$4,$5,$6,NULL,NULL,NULL,NULL,0,'failed',NULL,$7,$8,$9,$9,NULL) ON CONFLICT (source_topic, source_partition, source_offset) DO UPDATE SET status = 'failed', error_json = EXCLUDED.error_json, updated_at = EXCLUDED.updated_at", + [transport.sourceTopic, transport.sourcePartition, transport.sourceOffset, transport.sourceKey, sourceEventId, inputSha256, stableJson(redactedEnvelope(params.envelope)), stableJson(error), now] + ); + await insertDlq(client, { ...transport, sourceEventId, inputSha256, error, envelope: redactedEnvelope(params.envelope), createdAt: now }); + return { duplicate: false, conflict: false, priorStatus: null, diagnostic: error }; + }); + return { recorded: true, transport, sourceEventId, error: result.diagnostic, ...result, valuesPrinted: false }; +} + +export async function recordIgnoredAgentRunKafkaMessage(store, params = {}) { + const transport = normalizeTransport(params.transport); + const envelope = objectValue(params.envelope); + const sourceEventId = requiredText(envelope.eventId, "canonical AgentRun eventId"); + const inputSha256 = requiredText(transport.inputSha256, "Kafka input sha256"); + const now = store.now(); + return store.withDurableTransaction("workbench.kafka-projection.ignore", async (client) => { + await lockKafkaInboxIdentity(client, transport, sourceEventId); + const existing = await client.query("SELECT input_sha256, status FROM workbench_kafka_inbox WHERE (source_topic = $1 AND source_partition = $2 AND source_offset = $3) OR (source_topic = $1 AND source_event_id = $4) ORDER BY created_at ASC LIMIT 1", [transport.sourceTopic, transport.sourcePartition, transport.sourceOffset, sourceEventId]); + const prior = existing.rows?.[0] ?? null; + if (prior) { + if (prior.input_sha256 !== inputSha256) { + const diagnostic = { code: "workbench_kafka_inbox_hash_conflict", message: "Ignored Kafka identity was reused with a different payload hash.", valuesRedacted: true }; + await insertDlq(client, { ...transport, sourceEventId, inputSha256, error: diagnostic, envelope: redactedEnvelope(envelope), createdAt: now }); + return { ignored: true, duplicate: false, conflict: true, status: prior.status, diagnostic, valuesPrinted: false }; + } + return { ignored: true, duplicate: true, status: prior.status, valuesPrinted: false }; + } + await client.query( + "INSERT INTO workbench_kafka_inbox (source_topic, source_partition, source_offset, source_key, source_event_id, input_sha256, trace_id, session_id, run_id, command_id, source_seq, status, projected_seq, envelope_json, error_json, created_at, updated_at, projected_at) VALUES ($1,$2,$3,$4,$5,$6,NULL,NULL,$7,$8,$9,'ignored',NULL,$10,'{}',$11,$11,NULL)", + [transport.sourceTopic, transport.sourcePartition, transport.sourceOffset, transport.sourceKey, sourceEventId, inputSha256, text(envelope.runId), text(envelope.commandId), nonNegativeInteger(envelope.sourceSeq), stableJson(redactedEnvelope(envelope)), now] + ); + return { ignored: true, duplicate: false, status: "ignored", valuesPrinted: false }; + }); +} + +export async function claimHwlabKafkaOutbox(store, { owner, leaseMs, limit } = {}) { + const leaseOwner = requiredText(owner, "HWLAB Kafka relay owner"); + const boundedLimit = boundedInteger(limit, 100, 1, 500); + const now = store.now(); + const leaseExpiresAt = new Date(Date.parse(now) + boundedInteger(leaseMs, 60_000, 1_000, 3_600_000)).toISOString(); + const result = await store.withDurableTransaction("hwlab.kafka-outbox.claim", (client) => client.query( + `WITH candidates AS ( + SELECT candidate.outbox_seq FROM hwlab_kafka_outbox candidate + WHERE candidate.published_at IS NULL + AND candidate.next_attempt_at <= $1 + AND (candidate.lease_expires_at IS NULL OR candidate.lease_expires_at <= $1) + AND NOT EXISTS ( + SELECT 1 FROM hwlab_kafka_outbox earlier + WHERE earlier.published_at IS NULL + AND earlier.partition_key = candidate.partition_key + AND earlier.outbox_seq < candidate.outbox_seq + ) + ORDER BY candidate.outbox_seq ASC + FOR UPDATE SKIP LOCKED + LIMIT $2 + ) + UPDATE hwlab_kafka_outbox o SET attempt_count = o.attempt_count + 1, lease_owner = $3, lease_expires_at = $4, updated_at = $1 + FROM candidates WHERE o.outbox_seq = candidates.outbox_seq + RETURNING o.*`, + [now, boundedLimit, leaseOwner, leaseExpiresAt] + )); + return (result.rows ?? []).map(outboxRow); +} + +export async function completeHwlabKafkaOutbox(store, item = {}) { + const result = await store.query( + "UPDATE hwlab_kafka_outbox SET published_at = $1, lease_owner = NULL, lease_expires_at = NULL, last_error = NULL, updated_at = $1 WHERE outbox_seq = $2 AND lease_owner = $3 AND attempt_count = $4 AND lease_expires_at = $5 AND lease_expires_at > $1 AND published_at IS NULL RETURNING outbox_seq", + [store.now(), item.outboxSeq, item.leaseOwner, item.attemptCount, item.leaseExpiresAt] + ); + if (!result.rows?.[0]) throw codedError("hwlab_kafka_outbox_lease_conflict", "HWLAB Kafka outbox claim is stale."); + return { completed: true, outboxSeq: Number(result.rows[0].outbox_seq), valuesPrinted: false }; +} + +export async function retryHwlabKafkaOutbox(store, item = {}, error = null, retryAt = null) { + const now = store.now(); + const nextAttemptAt = text(retryAt) ?? now; + const result = await store.query( + "UPDATE hwlab_kafka_outbox SET next_attempt_at = $1, lease_owner = NULL, lease_expires_at = NULL, last_error = $2, updated_at = $3 WHERE outbox_seq = $4 AND lease_owner = $5 AND attempt_count = $6 AND lease_expires_at = $7 AND lease_expires_at > $3 AND published_at IS NULL RETURNING outbox_seq", + [nextAttemptAt, stableJson(redactedError(error)), now, item.outboxSeq, item.leaseOwner, item.attemptCount, item.leaseExpiresAt] + ); + if (!result.rows?.[0]) throw codedError("hwlab_kafka_outbox_lease_conflict", "HWLAB Kafka outbox claim is stale."); + return { retried: true, outboxSeq: Number(result.rows[0].outbox_seq), nextAttemptAt, valuesPrinted: false }; +} + +export async function hwlabKafkaProjectorStatus(store) { + const result = await store.query( + `SELECT + COUNT(*) FILTER (WHERE published_at IS NULL)::int AS backlog_count, + COUNT(*) FILTER (WHERE published_at IS NULL AND attempt_count > 0)::int AS retry_count, + MIN(created_at) FILTER (WHERE published_at IS NULL) AS oldest_pending_at, + (SELECT COUNT(*)::int FROM workbench_kafka_inbox WHERE status = 'failed') AS failed_inbox_count, + (SELECT COUNT(*)::int FROM workbench_kafka_dlq) AS dlq_count + FROM hwlab_kafka_outbox`, [] + ); + const row = result.rows?.[0] ?? {}; + return { backlogCount: Number(row.backlog_count ?? 0), retryCount: Number(row.retry_count ?? 0), oldestPendingAt: row.oldest_pending_at ?? null, failedInboxCount: Number(row.failed_inbox_count ?? 0), dlqCount: Number(row.dlq_count ?? 0), valuesPrinted: false }; +} + +export async function readAtomicWorkbenchProjectionSync(store, params = {}) { + const since = nonNegativeInteger(params.afterOutboxSeq ?? params.afterSeq ?? params.since); + const limit = boundedInteger(params.limit, 100, 1, 500); + const traceId = text(params.traceId); + const sessionId = text(params.sessionId); + const snapshotOnly = params.snapshotOnly === true; + const deltaOnly = params.deltaOnly === true; + if (!traceId && !sessionId) throw codedError("workbench_sync_scope_required", "Atomic Workbench sync requires sessionId or traceId."); + if (snapshotOnly && deltaOnly) throw codedError("workbench_sync_mode_invalid", "snapshotOnly and deltaOnly are mutually exclusive."); + return store.withDurableTransaction("workbench.projection-sync.read", async (client) => { + await client.query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY"); + const filter = traceId ? { column: "trace_id", value: traceId } : { column: "session_id", value: sessionId }; + const cutoffResult = await client.query(`SELECT COALESCE(MAX(outbox_seq), 0)::bigint AS cutoff FROM workbench_projection_outbox WHERE ${filter.column} = $1`, [filter.value]); + const cutoff = Number(cutoffResult.rows?.[0]?.cutoff ?? 0); + const outboxResult = snapshotOnly ? { rows: [] } : await client.query( + `SELECT outbox_seq, outbox_event_id, entity_family, entity_id, event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at FROM workbench_projection_outbox WHERE outbox_seq > $1 AND outbox_seq <= $2 AND ${filter.column} = $3 ORDER BY outbox_seq ASC LIMIT $4`, + [since, cutoff, filter.value, limit + 1] + ); + const allRows = (outboxResult.rows ?? []).map(projectionOutboxRow); + const events = allRows.slice(0, limit); + const facts = deltaOnly ? {} : await readSyncFacts(client, { traceId, sessionId, limit: 500, families: snapshotOnly ? ["sessions", "messages", "turns"] : null }); + const hasMore = allRows.length > limit; + return { facts, events, cutoffOutboxSeq: cutoff, cursorOutboxSeq: snapshotOnly ? cutoff : hasMore ? (events.at(-1)?.outboxSeq ?? since) : cutoff, hasMore, valuesPrinted: false }; + }); +} + +async function persistFacts(store, client, facts, requestMeta) { + const aggregateEvents = normalizeWorkbenchAggregateEventsForFacts(facts, requestMeta, store.now()); + const events = []; + for (const event of aggregateEvents) events.push(await store.persistWorkbenchAggregateEvent(event, client)); + const eventIndex = indexWorkbenchAggregateEvents(events); + const inputFacts = facts.inputs.map((fact) => inputFactWithAggregateSeq(fact, eventIndex)); + for (const fact of inputFacts) await store.persistWorkbenchSessionInputFact(fact, client); + for (const fact of facts.sessions) await store.persistWorkbenchSessionFact(fact, client); + for (const fact of facts.messages) await store.persistWorkbenchMessageFact(fact, client); + for (const fact of facts.parts) await store.persistWorkbenchPartFact(fact, client); + for (const fact of facts.turns) await store.persistWorkbenchTurnFact(fact, client); + for (const fact of facts.traceEvents) await store.persistWorkbenchTraceEventFact(fact, client); + for (const fact of facts.checkpoints) await store.persistWorkbenchProjectionCheckpoint(fact, client); + for (const fact of facts.messages) { + const event = eventIndex.get(`message:${fact.messageId}`) ?? eventIndex.get(`messages:${fact.messageId}`) ?? eventIndex.get(`sourceEvent:${fact.sourceEventId}`) ?? null; + await store.persistWorkbenchProjectionOutbox(outboxRecord(fact, event, "messages", "message", { role: fact.role, status: fact.status }), client); + } + for (const fact of facts.traceEvents) { + const event = eventIndex.get(`traceEvent:${fact.id}`) ?? eventIndex.get(`sourceEvent:${fact.sourceEventId}`) ?? null; + await store.persistWorkbenchProjectionOutbox(outboxRecord(fact, event, "traceEvents", fact.terminal ? "terminal" : "event", { type: fact.eventType }), client); + } + for (const fact of facts.turns.filter((item) => item.terminal || item.sealed)) { + const event = eventIndex.get(`turn:${fact.turnId}`) ?? eventIndex.get(`sourceEvent:${fact.sourceEventId}`) ?? null; + await store.persistWorkbenchProjectionOutbox(outboxRecord(fact, event, "turns", "terminal", { status: fact.status, failureKind: fact.failureKind }), client); + } + return { events, inputFacts }; +} + +function outboxRecord(fact, event, family, commitType, metadata) { + const entityId = text(family === "messages" ? fact.messageId : family === "turns" ? fact.turnId : fact.id) ?? text(fact.traceId) ?? "projection"; + return { + outboxEventId: projectionOutboxEventId({ fact, event, family, entityId, commitType }), + entityFamily: family, + entityId, + eventSeq: event?.eventSeq ?? null, + aggregateId: event?.aggregateId ?? null, + aggregateSeq: event?.aggregateSeq ?? 0, + projectionRevision: event?.projectionRevision ?? fact.projectedSeq, + traceId: fact.traceId, + sessionId: fact.sessionId, + turnId: fact.turnId, + messageId: fact.messageId, + projectedSeq: fact.projectedSeq, + sourceSeq: fact.sourceSeq, + sourceEventId: fact.sourceEventId, + commitType, + terminal: Boolean(fact.terminal), + sealed: Boolean(fact.sealed), + payload: { + family, + fact, + metadata: { ...metadata, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq, valuesRedacted: true }, + valuesRedacted: true + }, + createdAt: fact.occurredAt ?? fact.updatedAt ?? fact.createdAt + }; +} + +export function projectionOutboxEventId({ fact = {}, event = null, family, entityId, commitType }) { + const sourceEventId = text(fact.sourceEventId); + if (sourceEventId) return `${sourceEventId}:${family}:${entityId}:${commitType}`; + const identity = stableJson({ + family, + entityId, + commitType, + eventSeq: event?.eventSeq ?? null, + aggregateId: event?.aggregateId ?? null, + aggregateSeq: event?.aggregateSeq ?? 0, + projectionRevision: event?.projectionRevision ?? fact.projectedSeq ?? 0, + projectedSeq: fact.projectedSeq ?? 0, + sourceSeq: fact.sourceSeq ?? 0 + }); + return `projection:${createHash("sha256").update(identity).digest("hex")}`; +} + +function mergeKafkaSessionIdentity(facts, row) { + if (!row) return facts; + const existing = parseJsonColumn(row.session_json, {}); + const sessions = Array.isArray(facts?.sessions) ? facts.sessions.map((fact) => ({ + ...existing, + ...fact, + ownerUserId: fact.ownerUserId ?? row.owner_user_id ?? existing.ownerUserId ?? null, + projectId: fact.projectId ?? row.project_id ?? existing.projectId ?? null, + conversationId: fact.conversationId ?? row.conversation_id ?? existing.conversationId ?? null, + threadId: fact.threadId ?? row.thread_id ?? existing.threadId ?? null, + ownerRole: fact.ownerRole ?? existing.ownerRole ?? null + })) : []; + return { ...facts, sessions }; +} + +async function readSyncFacts(client, { traceId, sessionId, limit, families = null }) { + const familySet = Array.isArray(families) ? new Set(families) : null; + const specs = [ + ["sessions", "workbench_sessions", "session_json", traceId ? "last_trace_id" : "session_id"], + ["messages", "workbench_messages", "message_json", traceId ? "trace_id" : "session_id"], + ["parts", "workbench_parts", "part_json", traceId ? "trace_id" : "session_id"], + ["turns", "workbench_turns", "turn_json", traceId ? "trace_id" : "session_id"], + ["traceEvents", "workbench_trace_events", "event_json", traceId ? "trace_id" : "session_id"], + ["checkpoints", "workbench_projection_checkpoints", "checkpoint_json", traceId ? "trace_id" : "session_id"] + ].filter(([family]) => !familySet || familySet.has(family)); + const value = traceId ?? sessionId; + const facts = {}; + for (const [family, table, jsonColumn, column] of specs) { + const result = await client.query(`SELECT ${jsonColumn} FROM ${table} WHERE ${column} = $1 ORDER BY updated_at ASC LIMIT $2`, [value, limit]); + facts[family] = (result.rows ?? []).map((row) => parseJsonColumn(row[jsonColumn], null)).filter(Boolean); + } + return facts; +} + +async function insertDlq(client, record) { + await client.query( + "INSERT INTO workbench_kafka_dlq (source_topic, source_partition, source_offset, source_event_id, input_sha256, error_code, error_message, envelope_json, created_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (source_topic, source_partition, source_offset) DO UPDATE SET error_code = EXCLUDED.error_code, error_message = EXCLUDED.error_message, envelope_json = EXCLUDED.envelope_json", + [record.sourceTopic, record.sourcePartition, record.sourceOffset, record.sourceEventId, record.inputSha256, record.error.code, record.error.message, stableJson(record.envelope), record.createdAt] + ); +} + +async function lockKafkaInboxIdentity(client, transport, sourceEventId) { + const identity = sourceEventId + ? `${transport.sourceTopic}:event:${sourceEventId}` + : `${transport.sourceTopic}:transport:${transport.sourcePartition}:${transport.sourceOffset}`; + await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`workbench-kafka-inbox:${identity}`]); +} + +function normalizeTransport(value = {}) { + const input = objectValue(value); + const sourceTopic = requiredText(input.sourceTopic ?? input.topic, "Kafka source topic"); + const sourcePartition = Number(input.sourcePartition ?? input.partition); + const sourceOffset = text(input.sourceOffset ?? input.offset); + if (!Number.isInteger(sourcePartition) || sourcePartition < 0) throw codedError("workbench_kafka_partition_invalid", "Kafka source partition must be a non-negative integer."); + if (!/^\d+$/u.test(sourceOffset ?? "")) throw codedError("workbench_kafka_offset_invalid", "Kafka source offset must be a non-negative integer string."); + return { sourceTopic, sourcePartition, sourceOffset, inputSha256: text(input.inputSha256), sourceKey: text(input.sourceKey), valuesPrinted: false }; +} + +function projectionOutboxRow(row) { + return { outboxSeq: Number(row.outbox_seq), outboxEventId: row.outbox_event_id, entityFamily: row.entity_family, entityId: row.entity_id, eventSeq: row.event_seq == null ? null : Number(row.event_seq), aggregateId: row.aggregate_id, aggregateSeq: Number(row.aggregate_seq ?? 0), projectionRevision: Number(row.projection_revision ?? 0), traceId: row.trace_id, sessionId: row.session_id, turnId: row.turn_id, messageId: row.message_id, projectedSeq: Number(row.projected_seq ?? 0), sourceSeq: Number(row.source_seq ?? 0), sourceEventId: row.source_event_id, commitType: row.commit_type, terminal: Boolean(row.terminal), sealed: Boolean(row.sealed), payload: parseJsonColumn(row.payload_json, {}), createdAt: row.created_at, valuesPrinted: false }; +} + +function outboxRow(row) { + return { outboxSeq: Number(row.outbox_seq), eventId: row.event_id, topic: row.topic, partitionKey: row.partition_key, payload: parseJsonColumn(row.payload_json, {}), headers: parseJsonColumn(row.headers_json, {}), attemptCount: Number(row.attempt_count ?? 0), nextAttemptAt: row.next_attempt_at, leaseOwner: row.lease_owner, leaseExpiresAt: row.lease_expires_at, createdAt: row.created_at, valuesPrinted: false }; +} + +function redactedEnvelope(value) { + const input = objectValue(value); + return { schema: text(input.schema), eventType: text(input.eventType), eventId: text(input.eventId), outboxSeq: nonNegativeInteger(input.outboxSeq), sourceSeq: nonNegativeInteger(input.sourceSeq), traceId: text(input.traceId), sessionId: text(input.sessionId), hwlabSessionId: text(input.hwlabSessionId), runId: text(input.runId), commandId: text(input.commandId), valuesRedacted: true }; +} + +function redactedError(error) { + return { code: text(error?.code) ?? "workbench_kafka_message_invalid", message: String(error?.message ?? error ?? "Kafka message is invalid.").slice(0, 500), valuesRedacted: true }; +} + +function objectValue(value) { + return value && typeof value === "object" && !Array.isArray(value) ? value : {}; +} + +function text(value) { + const normalized = textOr(value, ""); + return normalized || null; +} + +function requiredText(value, label) { + const normalized = text(value); + if (!normalized) throw codedError("workbench_kafka_contract_invalid", `${label} is required.`); + return normalized; +} + +function boundedInteger(value, fallback, min, max) { + const parsed = Number(value); + return Number.isInteger(parsed) && parsed >= min && parsed <= max ? parsed : fallback; +} + +function codedError(code, message) { + const error = new Error(message); + error.code = code; + return error; +} diff --git a/internal/db/runtime-store-postgres-notify.test.ts b/internal/db/runtime-store-postgres-notify.test.ts new file mode 100644 index 00000000..4239b7b2 --- /dev/null +++ b/internal/db/runtime-store-postgres-notify.test.ts @@ -0,0 +1,26 @@ +import assert from "node:assert/strict"; +import { EventEmitter } from "node:events"; +import { test } from "bun:test"; + +import { subscribeWorkbenchProjectionCommits, WORKBENCH_PROJECTION_CHANNEL } from "./runtime-store-postgres-notify.ts"; + +test("Postgres LISTEN fans committed projection scope out and releases its dedicated connection", async () => { + const queries = []; + let released = 0; + const client = new EventEmitter(); + client.query = async (sql) => { queries.push(sql); return { rows: [] }; }; + client.release = () => { released += 1; }; + const pool = { async connect() { return client; } }; + const store = { async getQueryClient() { return pool; }, logger: null }; + const signals = []; + + const unsubscribe = await subscribeWorkbenchProjectionCommits(store, (signal) => signals.push(signal)); + assert.deepEqual(queries, [`LISTEN ${WORKBENCH_PROJECTION_CHANNEL}`]); + + client.emit("notification", { channel: WORKBENCH_PROJECTION_CHANNEL, payload: JSON.stringify({ sessionId: "ses_notify", traceId: "trc_notify", outboxSeq: 12 }) }); + assert.deepEqual(signals, [{ sessionId: "ses_notify", traceId: "trc_notify", outboxSeq: 12, valuesRedacted: true }]); + + await unsubscribe(); + assert.equal(queries.at(-1), `UNLISTEN ${WORKBENCH_PROJECTION_CHANNEL}`); + assert.equal(released, 1); +}); diff --git a/internal/db/runtime-store-postgres-notify.ts b/internal/db/runtime-store-postgres-notify.ts new file mode 100644 index 00000000..4969bcbc --- /dev/null +++ b/internal/db/runtime-store-postgres-notify.ts @@ -0,0 +1,145 @@ +/* PostgreSQL projection notifications wake SSE readers; the outbox remains the payload authority. */ + +const WORKBENCH_PROJECTION_CHANNEL = "hwlab_workbench_projection"; +const RECONNECT_DELAY_MS = 500; +const hubs = new WeakMap(); + +export async function subscribeWorkbenchProjectionCommits(store, listener) { + if (typeof listener !== "function") throw codedError("workbench_projection_listener_invalid", "Projection notification listener must be a function."); + let hub = hubs.get(store); + if (!hub) { + hub = { store, listeners: new Set(), client: null, connectPromise: null, reconnectTimer: null, connectedOnce: false, closing: false }; + hubs.set(store, hub); + } + hub.listeners.add(listener); + hub.closing = false; + await ensureConnected(hub); + let subscribed = true; + return async () => { + if (!subscribed) return; + subscribed = false; + hub.listeners.delete(listener); + if (hub.listeners.size === 0) await closeHub(hub); + }; +} + +async function ensureConnected(hub) { + if (hub.client || hub.connectPromise || hub.closing || hub.listeners.size === 0) return hub.connectPromise; + hub.connectPromise = (async () => { + const queryClient = await hub.store.getQueryClient(); + const client = typeof queryClient?.connect === "function" ? await queryClient.connect() : queryClient; + if (!client || typeof client.query !== "function" || typeof client.on !== "function") { + client?.release?.(); + throw codedError("workbench_projection_notify_unsupported", "Postgres runtime adapter requires a dedicated LISTEN connection."); + } + const onNotification = (message) => { + if (message?.channel !== WORKBENCH_PROJECTION_CHANNEL) return; + const signal = parseSignal(message.payload); + for (const notify of [...hub.listeners]) { + try { notify(signal); } catch {} + } + }; + const onDisconnect = (error) => disconnectHubClient(hub, client, error); + client.on("notification", onNotification); + client.on("error", onDisconnect); + client.on("end", onDisconnect); + client.__hwlabProjectionNotifyHandlers = { onNotification, onDisconnect }; + try { + await client.query(`LISTEN ${WORKBENCH_PROJECTION_CHANNEL}`); + } catch (error) { + detachClient(client, error); + throw error; + } + if (hub.closing || hub.listeners.size === 0) { + detachClient(client); + return; + } + hub.client = client; + if (hub.connectedOnce) emitRecovery(hub); + hub.connectedOnce = true; + })().finally(() => { + hub.connectPromise = null; + }); + return hub.connectPromise; +} + +function disconnectHubClient(hub, client, error) { + if (client && hub.client !== client) return; + if (client) { + hub.client = null; + detachClient(client, error); + } + scheduleReconnect(hub, error); +} + +function scheduleReconnect(hub, error) { + if (hub.closing || hub.listeners.size === 0 || hub.reconnectTimer) return; + hub.store.logger?.warn?.({ event: "workbench_projection_notify_disconnected", errorCode: error?.code ?? "UNKNOWN", valuesRedacted: true }); + hub.reconnectTimer = setTimeout(() => { + hub.reconnectTimer = null; + void ensureConnected(hub).catch((connectError) => scheduleReconnect(hub, connectError)); + }, RECONNECT_DELAY_MS); + hub.reconnectTimer.unref?.(); +} + +async function closeHub(hub) { + hub.closing = true; + if (hub.reconnectTimer) clearTimeout(hub.reconnectTimer); + hub.reconnectTimer = null; + const client = hub.client; + hub.client = null; + if (client) { + try { await client.query(`UNLISTEN ${WORKBENCH_PROJECTION_CHANNEL}`); } catch {} + detachClient(client); + } + hubs.delete(hub.store); +} + +function detachClient(client, error = null) { + const handlers = client?.__hwlabProjectionNotifyHandlers; + if (handlers) { + client.off?.("notification", handlers.onNotification); + client.off?.("error", handlers.onDisconnect); + client.off?.("end", handlers.onDisconnect); + delete client.__hwlabProjectionNotifyHandlers; + } + client?.release?.(error || undefined); +} + +function emitRecovery(hub) { + for (const notify of [...hub.listeners]) { + try { notify({ recovery: true, valuesRedacted: true }); } catch {} + } +} + +function parseSignal(payload) { + try { + const value = JSON.parse(String(payload ?? "{}")); + return { + sessionId: text(value.sessionId), + traceId: text(value.traceId), + outboxSeq: nonNegativeInteger(value.outboxSeq), + valuesRedacted: true + }; + } catch { + return { recovery: true, valuesRedacted: true }; + } +} + +function text(value) { + const normalized = String(value ?? "").trim(); + return normalized || null; +} + +function nonNegativeInteger(value) { + const number = Number(value); + return Number.isInteger(number) && number >= 0 ? number : 0; +} + +function codedError(code, message) { + const error = new Error(message); + error.code = code; + return error; +} + +export { WORKBENCH_PROJECTION_CHANNEL }; diff --git a/internal/db/runtime-store-postgres.ts b/internal/db/runtime-store-postgres.ts index 8bb5bebc..3f5b77c2 100644 --- a/internal/db/runtime-store-postgres.ts +++ b/internal/db/runtime-store-postgres.ts @@ -16,7 +16,7 @@ import { deriveProtocolActorFromMeta } from "../audit/index.mjs"; import { - CLOUD_CORE_MIGRATION_ID, + CLOUD_RUNTIME_DURABLE_MIGRATION_ID, CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS @@ -25,6 +25,17 @@ import { import { CloudRuntimeStore } from "./runtime-store-memory.ts"; import * as runtimeCore from "./runtime-store-core.ts"; +import { + claimHwlabKafkaOutbox as claimHwlabKafkaOutboxOperation, + commitAgentRunKafkaProjection as commitAgentRunKafkaProjectionOperation, + completeHwlabKafkaOutbox as completeHwlabKafkaOutboxOperation, + hwlabKafkaProjectorStatus as hwlabKafkaProjectorStatusOperation, + readAtomicWorkbenchProjectionSync as readAtomicWorkbenchProjectionSyncOperation, + recordFailedAgentRunKafkaMessage as recordFailedAgentRunKafkaMessageOperation, + recordIgnoredAgentRunKafkaMessage as recordIgnoredAgentRunKafkaMessageOperation, + retryHwlabKafkaOutbox as retryHwlabKafkaOutboxOperation +} from "./runtime-store-postgres-kafka.ts"; +import { subscribeWorkbenchProjectionCommits as subscribeWorkbenchProjectionCommitsOperation } from "./runtime-store-postgres-notify.ts"; const { RUNTIME_STORE_KIND, @@ -697,7 +708,7 @@ export class PostgresCloudRuntimeStore { commitType: "message", terminal: Boolean(fact.terminal), sealed: Boolean(fact.sealed), - payload: { role: fact.role, status: fact.status, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq, valuesRedacted: true }, + payload: immutableProjectionOutboxPayload("messages", fact, { role: fact.role, status: fact.status, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq }), createdAt: fact.updatedAt ?? fact.createdAt ?? this.now() }, client); } @@ -718,7 +729,7 @@ export class PostgresCloudRuntimeStore { commitType: fact.terminal ? "terminal" : "event", terminal: Boolean(fact.terminal), sealed: Boolean(fact.sealed), - payload: { type: fact.eventType, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq, valuesRedacted: true }, + payload: immutableProjectionOutboxPayload("traceEvents", fact, { type: fact.eventType, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq }), createdAt: fact.occurredAt ?? fact.updatedAt ?? this.now() }, client); } @@ -740,7 +751,7 @@ export class PostgresCloudRuntimeStore { commitType: "terminal", terminal: true, sealed: true, - payload: { status: fact.status, failureKind: fact.failureKind, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, valuesRedacted: true }, + payload: immutableProjectionOutboxPayload("turns", fact, { status: fact.status, failureKind: fact.failureKind, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null }), createdAt: fact.updatedAt ?? this.now() }, client); } @@ -752,6 +763,49 @@ export class PostgresCloudRuntimeStore { return withPersistence({ written: true, facts: persistedFacts, events: txResult.events }, this.summary()); } + async commitAgentRunKafkaProjection(params = {}) { + await this.assertReadyForWrites(); + return commitAgentRunKafkaProjectionOperation(this, params); + } + + async recordFailedAgentRunKafkaMessage(params = {}) { + await this.assertReadyForWrites(); + return recordFailedAgentRunKafkaMessageOperation(this, params); + } + + async recordIgnoredAgentRunKafkaMessage(params = {}) { + await this.assertReadyForWrites(); + return recordIgnoredAgentRunKafkaMessageOperation(this, params); + } + + async claimHwlabKafkaOutbox(params = {}) { + await this.assertReadyForWrites(); + return claimHwlabKafkaOutboxOperation(this, params); + } + + async completeHwlabKafkaOutbox(item = {}) { + return completeHwlabKafkaOutboxOperation(this, item); + } + + async retryHwlabKafkaOutbox(item = {}, error = null, retryAt = null) { + return retryHwlabKafkaOutboxOperation(this, item, error, retryAt); + } + + async hwlabKafkaProjectorStatus() { + await this.assertReadyForDurableReads("hwlab.kafka-projector.status"); + return hwlabKafkaProjectorStatusOperation(this); + } + + async readAtomicWorkbenchProjectionSync(params = {}) { + await this.assertReadyForDurableReads("workbench.projection-sync.read"); + return readAtomicWorkbenchProjectionSyncOperation(this, params); + } + + async subscribeWorkbenchProjectionCommits(listener) { + await this.assertReadyForDurableReads("workbench.projection-notify.listen"); + return subscribeWorkbenchProjectionCommitsOperation(this, listener); + } + async queryWorkbenchFacts(params = {}) { await this.assertReadyForDurableReads("workbench.facts.query"); const families = workbenchFactFamilySet(params); @@ -1122,7 +1176,7 @@ export class PostgresCloudRuntimeStore { async persistWorkbenchSessionFact(record, client = this) { await client.query( - "INSERT INTO workbench_sessions (session_id, owner_user_id, project_id, conversation_id, thread_id, status, last_trace_id, projected_seq, source_seq, source_event_id, terminal, sealed, session_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (session_id) DO UPDATE SET owner_user_id = EXCLUDED.owner_user_id, project_id = EXCLUDED.project_id, conversation_id = EXCLUDED.conversation_id, thread_id = EXCLUDED.thread_id, status = EXCLUDED.status, last_trace_id = EXCLUDED.last_trace_id, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, session_json = EXCLUDED.session_json, updated_at = EXCLUDED.updated_at", + "INSERT INTO workbench_sessions (session_id, owner_user_id, project_id, conversation_id, thread_id, status, last_trace_id, projected_seq, source_seq, source_event_id, terminal, sealed, session_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (session_id) DO UPDATE SET owner_user_id = COALESCE(EXCLUDED.owner_user_id, workbench_sessions.owner_user_id), project_id = COALESCE(EXCLUDED.project_id, workbench_sessions.project_id), conversation_id = COALESCE(EXCLUDED.conversation_id, workbench_sessions.conversation_id), thread_id = COALESCE(EXCLUDED.thread_id, workbench_sessions.thread_id), status = EXCLUDED.status, last_trace_id = EXCLUDED.last_trace_id, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, session_json = EXCLUDED.session_json, updated_at = EXCLUDED.updated_at", [record.sessionId, record.ownerUserId, record.projectId, record.conversationId, record.threadId, record.status, record.lastTraceId, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record), record.createdAt, record.updatedAt] ); } @@ -1164,8 +1218,8 @@ export class PostgresCloudRuntimeStore { async persistWorkbenchProjectionOutbox(record, client = this) { await client.query( - "INSERT INTO workbench_projection_outbox (event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)", - [record.eventSeq, record.aggregateId, nonNegativeInteger(record.aggregateSeq), nonNegativeInteger(record.projectionRevision), record.traceId, record.sessionId, record.turnId, record.messageId, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.commitType ?? "event", Boolean(record.terminal), Boolean(record.sealed), stableJson(record.payload ?? record), record.createdAt ?? this.now()] + "INSERT INTO workbench_projection_outbox (outbox_event_id, entity_family, entity_id, event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19) ON CONFLICT (outbox_event_id) DO NOTHING", + [record.outboxEventId ?? projectionOutboxEventId(record), record.entityFamily ?? projectionOutboxEntity(record).family, record.entityId ?? projectionOutboxEntity(record).id, record.eventSeq, record.aggregateId, nonNegativeInteger(record.aggregateSeq), nonNegativeInteger(record.projectionRevision), record.traceId, record.sessionId, record.turnId, record.messageId, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.commitType ?? "event", Boolean(record.terminal), Boolean(record.sealed), stableJson(record.payload ?? record), record.createdAt ?? this.now()] ); } @@ -1186,11 +1240,14 @@ export class PostgresCloudRuntimeStore { } params.push(Math.min(Math.max(Number(limit) || 100, 1), 500)); const result = await this.query( - `SELECT outbox_seq, event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at FROM workbench_projection_outbox WHERE ${where} ORDER BY outbox_seq ASC LIMIT $${paramIdx}`, + `SELECT outbox_seq, outbox_event_id, entity_family, entity_id, event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at FROM workbench_projection_outbox WHERE ${where} ORDER BY outbox_seq ASC LIMIT $${paramIdx}`, params ); return (result.rows ?? []).map((row) => ({ outboxSeq: Number(row.outbox_seq), + outboxEventId: row.outbox_event_id, + entityFamily: row.entity_family, + entityId: row.entity_id, eventSeq: row.event_seq === null || row.event_seq === undefined ? null : Number(row.event_seq), aggregateId: row.aggregate_id, aggregateSeq: Number(row.aggregate_seq ?? 0), @@ -1230,17 +1287,17 @@ export class PostgresCloudRuntimeStore { async readMigrationReadiness() { const result = await this.query( `SELECT id, schema_version FROM ${CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE} WHERE id = $1 LIMIT 1`, - [CLOUD_CORE_MIGRATION_ID] + [CLOUD_RUNTIME_DURABLE_MIGRATION_ID] ); const row = result.rows?.[0] ?? null; const ready = - row?.id === CLOUD_CORE_MIGRATION_ID && + row?.id === CLOUD_RUNTIME_DURABLE_MIGRATION_ID && row?.schema_version === CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION; return { checked: true, ready, table: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, - requiredMigrationId: CLOUD_CORE_MIGRATION_ID, + requiredMigrationId: CLOUD_RUNTIME_DURABLE_MIGRATION_ID, requiredSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, appliedMigrationId: row?.id ?? null, appliedSchemaVersion: row?.schema_version ?? null, @@ -1535,3 +1592,28 @@ export class PostgresCloudRuntimeStore { }); } } + +function projectionOutboxEventId(record = {}) { + const sourceEventId = textOr(record.sourceEventId, ""); + const commitType = textOr(record.commitType, "event"); + const family = textOr(record.entityFamily ?? record.payload?.family, "") || (commitType === "message" ? "messages" : record.payload?.type ? "traceEvents" : "turns"); + const entityId = family === "traceEvents" + ? textOr(record.entityId ?? record.payload?.fact?.id ?? record.aggregateId ?? record.sourceEventId ?? record.traceId, "projection") + : textOr(record.entityId ?? record.payload?.fact?.messageId ?? record.payload?.fact?.turnId ?? record.messageId ?? record.turnId ?? record.traceId ?? record.aggregateId, "projection"); + if (sourceEventId) return `${sourceEventId}:${family}:${entityId}:${commitType}`; + const derived = stableJson({ family, entityId, commitType, eventSeq: record.eventSeq ?? null, aggregateId: record.aggregateId ?? null, aggregateSeq: record.aggregateSeq ?? 0, projectionRevision: record.projectionRevision ?? record.projectedSeq ?? 0, traceId: record.traceId ?? null, sessionId: record.sessionId ?? null }); + return `derived:${createHash("sha256").update(derived).digest("hex")}`; +} + +function projectionOutboxEntity(record = {}) { + const commitType = textOr(record.commitType, "event"); + const family = textOr(record.entityFamily ?? record.payload?.family, "") || (commitType === "message" ? "messages" : record.payload?.type ? "traceEvents" : "turns"); + const id = family === "traceEvents" + ? textOr(record.entityId ?? record.payload?.fact?.id ?? record.aggregateId ?? record.sourceEventId ?? record.traceId, "projection") + : textOr(record.entityId ?? record.payload?.fact?.messageId ?? record.payload?.fact?.turnId ?? record.messageId ?? record.turnId ?? record.traceId ?? record.aggregateId, "projection"); + return { family, id }; +} + +function immutableProjectionOutboxPayload(family, fact, metadata = {}) { + return { family, fact, metadata: { ...metadata, valuesRedacted: true }, valuesRedacted: true }; +} diff --git a/internal/db/runtime-store.test.ts b/internal/db/runtime-store.test.ts index d46f4a98..346b17e7 100644 --- a/internal/db/runtime-store.test.ts +++ b/internal/db/runtime-store.test.ts @@ -16,7 +16,7 @@ import { createConfiguredCloudRuntimeStore } from "./runtime-store.ts"; import { - CLOUD_CORE_MIGRATION_ID, + CLOUD_RUNTIME_DURABLE_MIGRATION_ID, CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS } from "./schema.ts"; @@ -1451,8 +1451,8 @@ function createFakePostgresClient({ hwlab_schema_migrations: new Map() }; if (migrationReady) { - state.hwlab_schema_migrations.set(CLOUD_CORE_MIGRATION_ID, { - id: CLOUD_CORE_MIGRATION_ID, + state.hwlab_schema_migrations.set(CLOUD_RUNTIME_DURABLE_MIGRATION_ID, { + id: CLOUD_RUNTIME_DURABLE_MIGRATION_ID, schema_version: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION }); } @@ -1605,7 +1605,7 @@ function createFakePostgresClient({ if (beforeWorkbenchFactRead) await beforeWorkbenchFactRead(sql, params); return workbenchFactRows(state.workbench_projection_checkpoints, "checkpoint_json", sql, params, readErrorCode); } - if (sql.startsWith("SELECT outbox_seq, event_seq, aggregate_id")) { + if (sql.startsWith("SELECT outbox_seq, outbox_event_id, entity_family, entity_id")) { const afterSeq = Number(params[0] ?? 0); const traceFilter = sql.includes("trace_id = $2") ? params[1] : null; const sessionFilter = sql.includes("session_id = $2") ? params[1] : sql.includes("session_id = $3") ? params[2] : null; @@ -1770,25 +1770,30 @@ function createFakePostgresClient({ error.code = outboxErrorCode; throw error; } + const existing = [...state.workbench_projection_outbox.values()].find((record) => record.outbox_event_id === params[0]); + if (existing) return { rows: [] }; const outboxSeq = state.workbench_projection_outbox.size + 1; state.workbench_projection_outbox.set(outboxSeq, { outbox_seq: outboxSeq, - event_seq: params[0], - aggregate_id: params[1], - aggregate_seq: params[2], - projection_revision: params[3], - trace_id: params[4], - session_id: params[5], - turn_id: params[6], - message_id: params[7], - projected_seq: params[8], - source_seq: params[9], - source_event_id: params[10], - commit_type: params[11], - terminal: params[12], - sealed: params[13], - payload_json: params[14], - created_at: params[15] + outbox_event_id: params[0], + entity_family: params[1], + entity_id: params[2], + event_seq: params[3], + aggregate_id: params[4], + aggregate_seq: params[5], + projection_revision: params[6], + trace_id: params[7], + session_id: params[8], + turn_id: params[9], + message_id: params[10], + projected_seq: params[11], + source_seq: params[12], + source_event_id: params[13], + commit_type: params[14], + terminal: params[15], + sealed: params[16], + payload_json: params[17], + created_at: params[18] }); return { rows: [] }; } diff --git a/internal/db/schema.test.ts b/internal/db/schema.test.ts index 44827f0f..fffcfd60 100644 --- a/internal/db/schema.test.ts +++ b/internal/db/schema.test.ts @@ -7,7 +7,7 @@ import { fileURLToPath } from "node:url"; import { TABLES, assertProtocolRecord, assertProtocolRecords } from "../protocol/index.mjs"; import { CLOUD_CORE_MIGRATIONS, - CLOUD_CORE_MIGRATION_ID, + CLOUD_RUNTIME_DURABLE_MIGRATION_ID, CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE_COLUMNS, @@ -41,15 +41,17 @@ test("initial migration skeleton declares every frozen table", async () => { assert.match(sql, /\btimestamp\b/); }); -test("initial migration exposes columns required by the durable runtime adapter", async () => { - const sql = await readFile(path.join(repoRoot, "internal/db/migrations/0001_cloud_core_skeleton.sql"), "utf8"); +test("versioned migration chain exposes columns required by the durable runtime adapter", async () => { + const sql = await readMigrationChain(); assert.ok(requiredRuntimeDurableColumns().length > 0); for (const [table, columns] of Object.entries(CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS)) { const tableMatch = sql.match(new RegExp(`CREATE TABLE IF NOT EXISTS ${table}\\s*\\(([\\s\\S]*?)\\);`, "u")); assert.ok(tableMatch, `missing ${table}`); for (const column of columns) { - assert.match(tableMatch[1], new RegExp(`\\b${column}\\b`), `missing ${table}.${column}`); + const declared = new RegExp(`\\b${column}\\b`).test(tableMatch[1]); + const added = new RegExp(`ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS ${column}\\b`, "u").test(sql); + assert.equal(declared || added, true, `missing ${table}.${column}`); } } @@ -60,7 +62,7 @@ test("initial migration exposes columns required by the durable runtime adapter" for (const column of CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE_COLUMNS) { assert.match(migrationMatch[1], new RegExp(`\\b${column}\\b`), `missing migration ledger column ${column}`); } - assert.match(sql, new RegExp(`'${CLOUD_CORE_MIGRATION_ID}'`, "u")); + assert.match(sql, new RegExp(`'${CLOUD_RUNTIME_DURABLE_MIGRATION_ID}'`, "u")); assert.match(sql, new RegExp(`'${CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION}'`, "u")); assert.equal(CLOUD_CORE_MIGRATIONS[0].runtimeDurableMigrationTable, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE); }); @@ -92,6 +94,35 @@ test("initial migration declares Workbench fact backfill sources", async () => { assert.match(sql, /CREATE UNIQUE INDEX idx_workbench_trace_events_trace_projected_seq/u); }); +test("v8 migration upgrades the existing Workbench outbox before enforcing realtime identity", async () => { + const sql = await readFile(path.join(repoRoot, "internal/db/migrations/0008_workbench_kafka_realtime.sql"), "utf8"); + + for (const column of ["outbox_event_id", "entity_family", "entity_id"]) { + assert.match( + sql, + new RegExp(`ALTER TABLE workbench_projection_outbox ADD COLUMN IF NOT EXISTS ${column} TEXT`, "u"), + `missing additive upgrade for workbench_projection_outbox.${column}` + ); + assert.match( + sql, + new RegExp(`ALTER TABLE workbench_projection_outbox ALTER COLUMN ${column} SET NOT NULL`, "u"), + `missing post-backfill constraint for workbench_projection_outbox.${column}` + ); + } + + assert.match(sql, /SET outbox_event_id = COALESCE\(outbox_event_id, 'legacy:' \|\| outbox_seq::text\)/u); + assert.match(sql, /entity_family = COALESCE\(entity_family, 'legacy'\)/u); + assert.match(sql, /CREATE UNIQUE INDEX idx_workbench_projection_outbox_event_id ON workbench_projection_outbox\(outbox_event_id\)/u); + assert.match(sql, /CREATE TRIGGER trg_notify_hwlab_workbench_projection/u); + assert.match(sql, new RegExp(`'${CLOUD_RUNTIME_DURABLE_MIGRATION_ID}'`, "u")); + assert.match(sql, /ON CONFLICT \(id\) DO UPDATE SET\s+schema_version = EXCLUDED\.schema_version/u); +}); + +async function readMigrationChain() { + const sources = await Promise.all(CLOUD_CORE_MIGRATIONS.map((migration) => readFile(path.join(repoRoot, migration.path), "utf8"))); + return sources.join("\n"); +} + test("protocol record guards catch schema drift before runtime writes", () => { assertProtocolRecord("gatewaySession", { gatewaySessionId: "gws_01J00000000000000000000000", diff --git a/internal/db/schema.ts b/internal/db/schema.ts index fe57c0d2..a3b236aa 100644 --- a/internal/db/schema.ts +++ b/internal/db/schema.ts @@ -5,7 +5,8 @@ import { TABLES } from "../protocol/index.mjs"; export const CLOUD_CORE_MIGRATION_ID = "0001_cloud_core_skeleton"; -export const CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION = "runtime-durable-postgres-v7"; +export const CLOUD_RUNTIME_DURABLE_MIGRATION_ID = "0008_workbench_kafka_realtime"; +export const CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION = "runtime-durable-postgres-v8"; export const CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE = "hwlab_schema_migrations"; export const FROZEN_CLOUD_CORE_TABLES = Object.freeze([...TABLES]); @@ -318,6 +319,9 @@ export const CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS = Object.freeze({ ]), workbench_projection_outbox: Object.freeze([ "outbox_seq", + "outbox_event_id", + "entity_family", + "entity_id", "event_seq", "aggregate_id", "aggregate_seq", @@ -334,6 +338,54 @@ export const CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS = Object.freeze({ "sealed", "payload_json", "created_at" + ]), + workbench_kafka_inbox: Object.freeze([ + "source_topic", + "source_partition", + "source_offset", + "source_key", + "source_event_id", + "input_sha256", + "trace_id", + "session_id", + "run_id", + "command_id", + "source_seq", + "status", + "projected_seq", + "envelope_json", + "error_json", + "created_at", + "updated_at", + "projected_at" + ]), + workbench_kafka_dlq: Object.freeze([ + "dlq_seq", + "source_topic", + "source_partition", + "source_offset", + "source_event_id", + "input_sha256", + "error_code", + "error_message", + "envelope_json", + "created_at" + ]), + hwlab_kafka_outbox: Object.freeze([ + "outbox_seq", + "event_id", + "topic", + "partition_key", + "payload_json", + "headers_json", + "attempt_count", + "next_attempt_at", + "lease_owner", + "lease_expires_at", + "published_at", + "last_error", + "created_at", + "updated_at" ]) }); @@ -343,6 +395,14 @@ export const CLOUD_CORE_MIGRATIONS = Object.freeze([ path: "internal/db/migrations/0001_cloud_core_skeleton.sql", tables: FROZEN_CLOUD_CORE_TABLES, connectsToDatabase: false, + runtimeDurableAdapterSchemaVersion: "runtime-durable-postgres-v7", + runtimeDurableMigrationTable: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE + }, + { + id: CLOUD_RUNTIME_DURABLE_MIGRATION_ID, + path: "internal/db/migrations/0008_workbench_kafka_realtime.sql", + tables: Object.freeze(["workbench_projection_outbox", "workbench_kafka_inbox", "workbench_kafka_dlq", "hwlab_kafka_outbox"]), + connectsToDatabase: false, runtimeDurableAdapterSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, runtimeDurableMigrationTable: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE } diff --git a/internal/workbenchruntime/realtime_sync.go b/internal/workbenchruntime/realtime_sync.go index 13a22086..96b5b4bf 100644 --- a/internal/workbenchruntime/realtime_sync.go +++ b/internal/workbenchruntime/realtime_sync.go @@ -23,6 +23,8 @@ type realtimeSyncQuery struct { TraceID string `json:"traceId"` AfterOutboxSeq int64 `json:"afterOutboxSeq"` Limit int `json:"limit"` + SnapshotOnly bool `json:"snapshotOnly,omitempty"` + DeltaOnly bool `json:"deltaOnly,omitempty"` Actor sessionActor `json:"actor"` } @@ -74,6 +76,12 @@ var realtimeSyncFactSpecs = []realtimeSyncFactSpec{ {Family: "checkpoints", Table: "workbench_projection_checkpoints", JSONColumn: "checkpoint_json", OrderBy: "updated_at ASC, trace_id ASC"}, } +var realtimeSyncSnapshotFactSpecs = []realtimeSyncFactSpec{ + realtimeSyncFactSpecs[0], + realtimeSyncFactSpecs[1], + realtimeSyncFactSpecs[3], +} + const realtimeSyncResolveScopeSQL = ` SELECT sessions.session_id FROM workbench_sessions AS sessions @@ -212,6 +220,9 @@ func normalizeRealtimeSyncQuery(query realtimeSyncQuery) (realtimeSyncQuery, *re if query.Limit < 0 { return query, &realtimeSyncValidationError{Status: http.StatusBadRequest, Code: "workbench_sync_limit_invalid", Message: "limit must be non-negative"} } + if query.SnapshotOnly && query.DeltaOnly { + return query, &realtimeSyncValidationError{Status: http.StatusBadRequest, Code: "workbench_sync_mode_invalid", Message: "snapshotOnly and deltaOnly are mutually exclusive"} + } query.Limit = boundedLimit(query.Limit, realtimeSyncDefaultLimit, realtimeSyncMaxLimit) if query.Actor.Role != "admin" && query.Actor.Role != "user" { return query, &realtimeSyncValidationError{Status: http.StatusBadRequest, Code: "workbench_sync_actor_invalid", Message: "actor.role must be user or admin"} @@ -285,21 +296,32 @@ func readRealtimeSyncSnapshot(ctx context.Context, tx realtimeSyncQueryer, query if err != nil { return realtimeSyncSnapshot{}, err } - allEvents, err := readRealtimeSyncEvents(ctx, tx, sessionID, query.TraceID, query.AfterOutboxSeq, cutoff, query.Limit+1) - if err != nil { - return realtimeSyncSnapshot{}, err + events := make([]realtimeSyncEvent, 0) + hasMore := false + if !query.SnapshotOnly { + allEvents, readErr := readRealtimeSyncEvents(ctx, tx, sessionID, query.TraceID, query.AfterOutboxSeq, cutoff, query.Limit+1) + if readErr != nil { + return realtimeSyncSnapshot{}, readErr + } + hasMore = len(allEvents) > query.Limit + events = allEvents + if hasMore { + events = allEvents[:query.Limit] + } } - hasMore := len(allEvents) > query.Limit - events := allEvents - if hasMore { - events = allEvents[:query.Limit] - } - facts, err := readRealtimeSyncFacts(ctx, tx, sessionID, query.TraceID) - if err != nil { - return realtimeSyncSnapshot{}, err + facts := make(map[string][]any) + if !query.DeltaOnly { + specs := realtimeSyncFactSpecs + if query.SnapshotOnly { + specs = realtimeSyncSnapshotFactSpecs + } + facts, err = readRealtimeSyncFacts(ctx, tx, sessionID, query.TraceID, specs) + if err != nil { + return realtimeSyncSnapshot{}, err + } } cursor := query.AfterOutboxSeq - if cursor > cutoff { + if query.SnapshotOnly || cursor > cutoff { cursor = cutoff } if len(events) > 0 { @@ -408,9 +430,9 @@ func scanRealtimeSyncEvent(scanner realtimeSyncScanner) (realtimeSyncEvent, erro return event, nil } -func readRealtimeSyncFacts(ctx context.Context, tx realtimeSyncQueryer, sessionID string, traceID string) (map[string][]any, error) { - facts := make(map[string][]any, len(realtimeSyncFactSpecs)) - for _, spec := range realtimeSyncFactSpecs { +func readRealtimeSyncFacts(ctx context.Context, tx realtimeSyncQueryer, sessionID string, traceID string, specs []realtimeSyncFactSpec) (map[string][]any, error) { + facts := make(map[string][]any, len(specs)) + for _, spec := range specs { args := []any{sessionID} where := "session_id = $1" if spec.Family != "sessions" { diff --git a/internal/workbenchruntime/realtime_sync_test.go b/internal/workbenchruntime/realtime_sync_test.go index 49497c68..ad52d4f4 100644 --- a/internal/workbenchruntime/realtime_sync_test.go +++ b/internal/workbenchruntime/realtime_sync_test.go @@ -56,6 +56,11 @@ func TestNormalizeRealtimeSyncQueryRequiresScopedActorAndBoundsLimit(t *testing. if validation == nil || validation.Code != "workbench_sync_cursor_invalid" { t.Fatalf("negative cursor validation=%#v", validation) } + + _, validation = normalizeRealtimeSyncQuery(realtimeSyncQuery{SessionID: "ses_sync", SnapshotOnly: true, DeltaOnly: true, Actor: sessionActor{ID: "usr_owner", Role: "user"}}) + if validation == nil || validation.Code != "workbench_sync_mode_invalid" { + t.Fatalf("conflicting sync mode validation=%#v", validation) + } } func TestRealtimeSyncRouteIsPostOnlyAndRejectsUnknownFields(t *testing.T) { @@ -152,6 +157,52 @@ func TestReadRealtimeSyncSnapshotRejectsOtherOwnerBeforeReadingFacts(t *testing. } } +func TestReadRealtimeSyncSnapshotOnlySkipsHistoryAndReadsBoundedCurrentFacts(t *testing.T) { + queryer := newRealtimeSyncFakeQueryer("usr_owner") + snapshot, err := readRealtimeSyncSnapshot(context.Background(), queryer, realtimeSyncQuery{ + SessionID: "ses_sync", + Limit: 100, + SnapshotOnly: true, + Actor: sessionActor{ID: "usr_owner", Role: "user"}, + }) + if err != nil { + t.Fatalf("read snapshot-only sync: %v", err) + } + if len(snapshot.Events) != 0 || snapshot.HasMore || snapshot.CursorOutboxSeq != snapshot.CutoffOutboxSeq { + t.Fatalf("snapshot-only boundary=%#v", snapshot) + } + if len(snapshot.Facts) != len(realtimeSyncSnapshotFactSpecs) { + t.Fatalf("snapshot-only facts=%#v", snapshot.Facts) + } + for _, call := range queryer.calls { + if call.kind == "events" || call.kind == "facts.parts" || call.kind == "facts.traceEvents" || call.kind == "facts.checkpoints" { + t.Fatalf("snapshot-only performed unbounded read: %v", queryer.calls) + } + } +} + +func TestReadRealtimeSyncDeltaOnlyReadsOutboxWithoutFacts(t *testing.T) { + queryer := newRealtimeSyncFakeQueryer("usr_owner") + snapshot, err := readRealtimeSyncSnapshot(context.Background(), queryer, realtimeSyncQuery{ + SessionID: "ses_sync", + AfterOutboxSeq: 1, + Limit: 100, + DeltaOnly: true, + Actor: sessionActor{ID: "usr_owner", Role: "user"}, + }) + if err != nil { + t.Fatalf("read delta-only sync: %v", err) + } + if len(snapshot.Events) == 0 || len(snapshot.Facts) != 0 { + t.Fatalf("delta-only result=%#v", snapshot) + } + for _, call := range queryer.calls { + if strings.HasPrefix(call.kind, "facts.") { + t.Fatalf("delta-only read facts: %v", queryer.calls) + } + } +} + func TestReadRealtimeSyncSnapshotClampsForeignCursorToScopedCutoff(t *testing.T) { queryer := newRealtimeSyncFakeQueryer("usr_owner") queryer.events = [][]any{} diff --git a/scripts/src/dev-runtime-migration.mjs b/scripts/src/dev-runtime-migration.mjs index b3b11958..2c75b53f 100644 --- a/scripts/src/dev-runtime-migration.mjs +++ b/scripts/src/dev-runtime-migration.mjs @@ -4,7 +4,8 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { - CLOUD_CORE_MIGRATION_ID, + CLOUD_CORE_MIGRATIONS, + CLOUD_RUNTIME_DURABLE_MIGRATION_ID, CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE_COLUMNS, @@ -25,7 +26,8 @@ import { ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs"; import { ensureNotRepoReportsPath, tempReportPath } from "./report-paths.mjs"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); -const migrationPath = "internal/db/migrations/0001_cloud_core_skeleton.sql"; +const migrationPaths = CLOUD_CORE_MIGRATIONS.map((migration) => migration.path); +const latestMigrationPath = migrationPaths.at(-1); const defaultReportPath = tempReportPath("dev-runtime-migration-report.json"); const issue = "pikasTech/HWLAB#164"; @@ -55,7 +57,11 @@ export async function buildDevRuntimeMigrationReport(args = {}, options = {}) { const env = options.env ?? process.env; const now = options.now ?? (() => new Date().toISOString()); const root = options.repoRoot ?? repoRoot; - const sql = await readFile(path.join(root, migrationPath), "utf8"); + const migrationSources = await Promise.all(migrationPaths.map(async (migrationPath) => ({ + path: migrationPath, + sql: await readFile(path.join(root, migrationPath), "utf8") + }))); + const sql = migrationSources.map((migration) => migration.sql).join("\n"); const sourceCheck = validateMigrationSource(sql); const db = summarizeDbEnv(env); const report = { @@ -71,8 +77,9 @@ export async function buildDevRuntimeMigrationReport(args = {}, options = {}) { dbEndpointAuthority: DEV_DB_ENV_CONTRACT.endpointAuthority.source }, migration: { - id: CLOUD_CORE_MIGRATION_ID, - path: migrationPath, + id: CLOUD_RUNTIME_DURABLE_MIGRATION_ID, + path: latestMigrationPath, + paths: migrationPaths, sha256: sha256(sql), schemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, ledgerTable: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, @@ -195,7 +202,9 @@ export function validateMigrationSource(sql) { continue; } for (const column of columns) { - if (!new RegExp(`\\b${column}\\b`, "u").test(tableMatch[1])) { + const created = new RegExp(`\\b${column}\\b`, "u").test(tableMatch[1]); + const added = new RegExp(`ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS ${column}\\b`, "u").test(sql); + if (!created && !added) { missing.push(`column:${table}.${column}`); } } @@ -214,8 +223,8 @@ export function validateMigrationSource(sql) { } } - if (!new RegExp(`'${CLOUD_CORE_MIGRATION_ID}'`, "u").test(sql)) { - missing.push(`ledger-id:${CLOUD_CORE_MIGRATION_ID}`); + if (!new RegExp(`'${CLOUD_RUNTIME_DURABLE_MIGRATION_ID}'`, "u").test(sql)) { + missing.push(`ledger-id:${CLOUD_RUNTIME_DURABLE_MIGRATION_ID}`); } if (!new RegExp(`'${CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION}'`, "u").test(sql)) { missing.push(`schema-version:${CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION}`); @@ -227,8 +236,9 @@ export function validateMigrationSource(sql) { return { ready: missing.length === 0, checked: true, - migrationPath, - requiredMigrationId: CLOUD_CORE_MIGRATION_ID, + migrationPath: latestMigrationPath, + migrationPaths, + requiredMigrationId: CLOUD_RUNTIME_DURABLE_MIGRATION_ID, requiredSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, requiredTables: Object.keys(CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS), requiredLedgerTable: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, @@ -456,7 +466,7 @@ function summarizeRuntimeMigration(migration = {}) { checked: Boolean(migration.checked), ready: Boolean(migration.ready), table: migration.table ?? CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, - requiredMigrationId: migration.requiredMigrationId ?? CLOUD_CORE_MIGRATION_ID, + requiredMigrationId: migration.requiredMigrationId ?? CLOUD_RUNTIME_DURABLE_MIGRATION_ID, requiredSchemaVersion: migration.requiredSchemaVersion ?? CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, appliedMigrationId: migration.appliedMigrationId ?? null, appliedSchemaVersion: migration.appliedSchemaVersion ?? null, diff --git a/scripts/src/dev-runtime-migration.test.mjs b/scripts/src/dev-runtime-migration.test.mjs index fc81c8af..1baf9721 100644 --- a/scripts/src/dev-runtime-migration.test.mjs +++ b/scripts/src/dev-runtime-migration.test.mjs @@ -9,7 +9,7 @@ import { parseArgs } from "./dev-runtime-migration.mjs"; import { - CLOUD_CORE_MIGRATION_ID, + CLOUD_RUNTIME_DURABLE_MIGRATION_ID, CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS } from "../../internal/db/schema.ts"; @@ -60,7 +60,7 @@ test("cloud-api image migration entrypoint exposes the same non-secret source ch assert.equal(report.actions.liveDbWriteAttempted, false); assert.equal(report.safety.printsSecretValues, false); assert.equal(report.safety.dbUrlValueRedacted, true); - assert.equal(report.migration.id, CLOUD_CORE_MIGRATION_ID); + assert.equal(report.migration.id, CLOUD_RUNTIME_DURABLE_MIGRATION_ID); assert.equal(JSON.stringify(report).includes("secret-password"), false); assert.equal(JSON.stringify(report).includes("db.example.test"), false); }); @@ -306,7 +306,7 @@ test("apply mode records ledger and verifies durable runtime readiness", async ( assert.equal(report.gates.schema.status, "ready"); assert.equal(report.gates.migration.status, "ready"); assert.equal(report.gates.readiness.status, "ready"); - assert.equal(report.runtime.migration.appliedMigrationId, CLOUD_CORE_MIGRATION_ID); + assert.equal(report.runtime.migration.appliedMigrationId, CLOUD_RUNTIME_DURABLE_MIGRATION_ID); assert.equal(report.runtime.migration.appliedSchemaVersion, CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION); assert.ok(queryClient.calls.some((call) => call.sql === "BEGIN")); assert.ok(queryClient.calls.some((call) => call.sql.includes("INSERT INTO hwlab_schema_migrations"))); @@ -448,7 +448,7 @@ function createFakeQueryClient({ migrationReady, countErrorCode = null, migratio rows: state.migrationReady ? [ { - id: CLOUD_CORE_MIGRATION_ID, + id: CLOUD_RUNTIME_DURABLE_MIGRATION_ID, schema_version: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION } ] diff --git a/web/hwlab-cloud-web/scripts/check.ts b/web/hwlab-cloud-web/scripts/check.ts index 49da44b8..18568f72 100644 --- a/web/hwlab-cloud-web/scripts/check.ts +++ b/web/hwlab-cloud-web/scripts/check.ts @@ -98,7 +98,7 @@ const workbenchHealthRuntimeSource = readWeb("src/utils/workbench-health.ts"); const traceTimelineSource = readWeb("src/components/agent/TraceTimeline.vue"); const messageTraceDebugSource = readWeb("src/components/workbench/MessageTraceDebugPanel.vue"); const conversationPanelSource = readWeb("src/components/workbench/ConversationPanel.vue"); -const serverWorkbenchHttpSource = fs.readFileSync(path.resolve(rootDir, "..", "..", "internal/cloud/server-workbench-http.ts"), "utf8"); +const serverWorkbenchRealtimeHttpSource = fs.readFileSync(path.resolve(rootDir, "..", "..", "internal/cloud/server-workbench-realtime-http.ts"), "utf8"); assert.match(html, /
<\/div>/u, "index.html must expose Vue mount root"); assert.match(html, /src="\/src\/main\.ts"/u, "index.html must load Vue TS entry"); @@ -208,9 +208,9 @@ assert.doesNotMatch(workbenchStoreSource, /refreshActiveTraceFromRest[\s\S]{0,12 assert.doesNotMatch(workbenchStoreSource, /refreshTerminalTraceFromRest[\s\S]{0,1200}refreshSessions\([^;]+force:\s*true/u, "Terminal trace REST refresh must not force-refresh the full session list"); assert.doesNotMatch(workbenchStoreSource, /message\.runnerTrace(?:\?\.|\.)status/u, "Workbench message lifecycle must not be inferred from runnerTrace.status"); assert.doesNotMatch(conversationPanelSource, /message\.runnerTrace(?:\?\.|\.)status/u, "ConversationPanel must not override message completion from runnerTrace.status"); -assert.doesNotMatch(serverWorkbenchHttpSource, /return\s+turnSnapshot\(context\)|\.\.\.turnSnapshot\(context\)|blockedTraceEventProjection\(context\.projection/u, "Realtime turn snapshots must not fall back to legacy trace/result projection"); -assertIncludes(serverWorkbenchHttpSource, "blockedRealtimeTurnSnapshot", "Realtime read-model gaps must produce blocked diagnostic turn snapshots"); -assertIncludes(serverWorkbenchHttpSource, "workbench_facts_session_missing", "Realtime read-model gaps must expose a facts-missing blocker"); +assert.doesNotMatch(serverWorkbenchRealtimeHttpSource, /return\s+turnSnapshot\(context\)|\.\.\.turnSnapshot\(context\)|blockedTraceEventProjection\(context\.projection/u, "Realtime turn snapshots must not fall back to legacy trace/result projection"); +assertIncludes(serverWorkbenchRealtimeHttpSource, "blockedRealtimeTurnSnapshot", "Realtime read-model gaps must produce blocked diagnostic turn snapshots"); +assertIncludes(serverWorkbenchRealtimeHttpSource, "workbench_facts_session_missing", "Realtime read-model gaps must expose a facts-missing blocker"); assertIncludes(appSource, "/v1/workbench/events", "Workbench realtime client must use the RESTful same-origin events endpoint"); assertIncludes(appSource, "/v1/workbench/traces/", "trace detail read must use Workbench read-model trace API"); assert.doesNotMatch(appSource, /\/v1\/agent\/(?:turns|traces|chat\/result)\//u, "Cloud Web must not call legacy Code Agent read-through turn/trace/result APIs"); diff --git a/web/hwlab-cloud-web/src/api/index.ts b/web/hwlab-cloud-web/src/api/index.ts index c941e47a..b8b17ed0 100644 --- a/web/hwlab-cloud-web/src/api/index.ts +++ b/web/hwlab-cloud-web/src/api/index.ts @@ -4,7 +4,7 @@ export { fetchJson, fetchText, type ActivityRef, type ActivityRefSource, type ApiRequestOptions } from "./client"; export { authAPI, HWLAB_WEB_SESSION_COOKIE } from "./auth"; export { workbenchAPI } from "./workbench"; -export { workbenchDebugAPI, connectWorkbenchDebugFakeSse, connectWorkbenchKafkaSseDebug, type WorkbenchDebugFakeSseQueue, type WorkbenchDebugFakeSseResponse, type WorkbenchDebugFakeSseSequence, type WorkbenchDebugFakeSseStream, type WorkbenchKafkaSseDebugEvent, type WorkbenchKafkaSseDebugFilters } from "./workbench-debug"; +export { workbenchDebugAPI, connectWorkbenchDebugFakeSse, type WorkbenchDebugFakeSseQueue, type WorkbenchDebugFakeSseResponse, type WorkbenchDebugFakeSseSequence, type WorkbenchDebugFakeSseStream } from "./workbench-debug"; export { connectWorkbenchEvents, type WorkbenchEventStream, type WorkbenchRealtimeEvent } from "./workbench-events"; export { agentAPI } from "./agent"; export { hwpodAPI } from "./hwpod"; diff --git a/web/hwlab-cloud-web/src/api/workbench-debug.ts b/web/hwlab-cloud-web/src/api/workbench-debug.ts index 68134dc8..653f256e 100644 --- a/web/hwlab-cloud-web/src/api/workbench-debug.ts +++ b/web/hwlab-cloud-web/src/api/workbench-debug.ts @@ -50,39 +50,6 @@ export interface WorkbenchDebugFakeSseStream { close: () => void; } -export interface WorkbenchKafkaSseDebugFilters { - traceId?: string; - sessionId?: string; - runId?: string; - commandId?: string; -} - -export type WorkbenchKafkaSseDebugStreamName = "stdio" | "agentrun" | "hwlab"; - -export interface WorkbenchKafkaSseDebugEvent { - ok?: boolean; - contractVersion?: string; - stream?: string; - topic?: string; - partition?: number; - offset?: string; - key?: string | null; - timestamp?: string | null; - value?: Record; - filters?: WorkbenchKafkaSseDebugFilters; - serverSentAt?: string; - error?: { code?: string; message?: string; [key: string]: unknown }; - valuesPrinted?: boolean; -} - -export interface WorkbenchKafkaSseDebugStreamOptions extends WorkbenchKafkaSseDebugFilters { - stream?: WorkbenchKafkaSseDebugStreamName; - fromBeginning?: boolean; - onEvent: (event: WorkbenchKafkaSseDebugEvent, eventName: string) => void; - onOpen?: () => void; - onError?: (event: Event) => void; -} - const DEBUG_FAKE_SSE_EVENTS = [ "workbench.connected", "workbench.trace.snapshot", @@ -126,39 +93,6 @@ export function connectWorkbenchDebugFakeSse(options: WorkbenchDebugFakeSseStrea }; } -export function connectWorkbenchKafkaSseDebug(options: WorkbenchKafkaSseDebugStreamOptions): WorkbenchDebugFakeSseStream | null { - if (typeof EventSource === "undefined") return null; - const source = new EventSource(kafkaDebugPath("/events", { - stream: options.stream || "hwlab", - traceId: options.traceId, - sessionId: options.sessionId, - runId: options.runId, - commandId: options.commandId, - fromBeginning: options.fromBeginning ? "true" : undefined - }), { withCredentials: true }); - source.onopen = () => options.onOpen?.(); - source.onerror = (event) => options.onError?.(event); - const names = ["hwlab.kafka.connected", "hwlab.kafka.event", "hwlab.kafka.error"]; - const listeners = names.map((name) => { - const listener = (event: MessageEvent) => { - const payload = parseKafkaEvent(event.data); - if (payload) options.onEvent(payload, name); - }; - source.addEventListener(name, listener); - return { name, listener }; - }); - source.onmessage = (event) => { - const payload = parseKafkaEvent(event.data); - if (payload) options.onEvent(payload, "message"); - }; - return { - close() { - for (const { name, listener } of listeners) source.removeEventListener(name, listener); - source.close(); - } - }; -} - function debugPath(suffix: string, params: Record = {}): string { const query = new URLSearchParams(); for (const [key, value] of Object.entries(params)) { @@ -169,16 +103,6 @@ function debugPath(suffix: string, params: Record = {}): string { - const query = new URLSearchParams(); - for (const [key, value] of Object.entries(params)) { - const text = typeof value === "string" ? value.trim() : ""; - if (text) query.set(key, text); - } - const qs = query.toString(); - return `/v1/workbench/debug/kafka-sse${suffix}${qs ? `?${qs}` : ""}`; -} - function parseRealtimeEvent(raw: string): WorkbenchRealtimeEvent | null { try { const value = JSON.parse(raw); @@ -187,12 +111,3 @@ function parseRealtimeEvent(raw: string): WorkbenchRealtimeEvent | null { return null; } } - -function parseKafkaEvent(raw: string): WorkbenchKafkaSseDebugEvent | null { - try { - const value = JSON.parse(raw); - return value && typeof value === "object" ? value as WorkbenchKafkaSseDebugEvent : null; - } catch { - return null; - } -} diff --git a/web/hwlab-cloud-web/src/config/workbench-runtime-policy.ts b/web/hwlab-cloud-web/src/config/workbench-runtime-policy.ts index 800f16bf..ac94a12c 100644 --- a/web/hwlab-cloud-web/src/config/workbench-runtime-policy.ts +++ b/web/hwlab-cloud-web/src/config/workbench-runtime-policy.ts @@ -29,8 +29,6 @@ export interface WorkbenchRuntimePolicy { workbenchRealtimeFlushMaxItemsPerChunk: number; workbenchRealtimeFlushMaxChunkMs: number; workbenchRealtimeFlushYieldMs: number; - workbenchActiveTraceSyncReplayInitialMs: number; - workbenchActiveTraceSyncReplayRepeatMs: number; } const DEFAULT_WORKBENCH_RUNTIME_POLICY: WorkbenchRuntimePolicy = Object.freeze({ @@ -60,9 +58,7 @@ const DEFAULT_WORKBENCH_RUNTIME_POLICY: WorkbenchRuntimePolicy = Object.freeze({ workbenchRealtimeErrorSyncReplayMinMs: 2_000, workbenchRealtimeFlushMaxItemsPerChunk: 4, workbenchRealtimeFlushMaxChunkMs: 8, - workbenchRealtimeFlushYieldMs: 0, - workbenchActiveTraceSyncReplayInitialMs: 2_500, - workbenchActiveTraceSyncReplayRepeatMs: 5_000 + workbenchRealtimeFlushYieldMs: 0 }); export function workbenchRuntimePolicy(input: unknown = runtimePolicyConfig()): WorkbenchRuntimePolicy { @@ -94,9 +90,7 @@ export function workbenchRuntimePolicy(input: unknown = runtimePolicyConfig()): workbenchRealtimeErrorSyncReplayMinMs: nonNegativeNumber(source.workbenchRealtimeErrorSyncReplayMinMs ?? source.workbenchRealtimeErrorGapFillMinMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchRealtimeErrorSyncReplayMinMs), workbenchRealtimeFlushMaxItemsPerChunk: positiveInteger(source.workbenchRealtimeFlushMaxItemsPerChunk, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchRealtimeFlushMaxItemsPerChunk), workbenchRealtimeFlushMaxChunkMs: nonNegativeNumber(source.workbenchRealtimeFlushMaxChunkMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchRealtimeFlushMaxChunkMs), - workbenchRealtimeFlushYieldMs: nonNegativeNumber(source.workbenchRealtimeFlushYieldMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchRealtimeFlushYieldMs), - workbenchActiveTraceSyncReplayInitialMs: nonNegativeNumber(source.workbenchActiveTraceSyncReplayInitialMs ?? source.workbenchActiveTraceRestGapFillInitialMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchActiveTraceSyncReplayInitialMs), - workbenchActiveTraceSyncReplayRepeatMs: nonNegativeNumber(source.workbenchActiveTraceSyncReplayRepeatMs ?? source.workbenchActiveTraceRestGapFillRepeatMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchActiveTraceSyncReplayRepeatMs) + workbenchRealtimeFlushYieldMs: nonNegativeNumber(source.workbenchRealtimeFlushYieldMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchRealtimeFlushYieldMs) }; } diff --git a/web/hwlab-cloud-web/src/stores/workbench-colada.test.ts b/web/hwlab-cloud-web/src/stores/workbench-colada.test.ts index 128e7b1c..185eb3d2 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-colada.test.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-colada.test.ts @@ -61,7 +61,8 @@ test("workbench colada sync replay is the only live repair query scope", () => { const storeSource = fs.readFileSync(path.join(storeDir, "workbench.ts"), "utf8"); assert.match(querySource, /fetchSyncReplay: \(input, options = \{\}\) => runWorkbenchQuery\(queryCache, workbenchColadaKeys\.syncReplay\(input\),/u); - assert.match(storeSource, /workbenchColadaQueries\.fetchSyncReplay\(\{ sessionId, traceId, since: sinceOutboxSeq \}/u); + assert.match(storeSource, /workbenchColadaQueries\.fetchSyncReplay\(\{ sessionId, traceId, since: cursor \}/u); + assert.match(storeSource, /responseCursor\?\.hasMore !== true/u); assert.doesNotMatch(storeSource, /import \{ fetchWorkbenchSyncReplay \} from "@\/api\/workbench-events"/u); assert.match(mutationSource, /workbenchColadaKeys\.syncRoot\(\)/u); @@ -135,7 +136,7 @@ test("workbench sync replay projects durable message and turn facts into observe if (action.type === "message.snapshot") { state = reduceWorkbenchServerState(state, { type: "message.snapshot", sessionId: action.realtimeEvent.sessionId, message: action.realtimeEvent.message }); } else if (action.type === "turn.snapshot") { - state = reduceWorkbenchServerState(state, { type: "turn.status", turn: action.turn as TurnStatusAuthority }); + state = reduceWorkbenchServerState(state, { type: "turn.status", turn: action.turn as unknown as TurnStatusAuthority }); } } diff --git a/web/hwlab-cloud-web/src/stores/workbench-debug-fake-sse.ts b/web/hwlab-cloud-web/src/stores/workbench-debug-fake-sse.ts index 05046984..3182dd45 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-debug-fake-sse.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-debug-fake-sse.ts @@ -79,7 +79,11 @@ function applyTraceEvent(state: WorkbenchDebugFakeSseState, traceId: string | nu function applyTraceSnapshot(state: WorkbenchDebugFakeSseState, traceId: string | null, snapshot: WorkbenchRealtimeEvent["snapshot"]): WorkbenchDebugFakeSseState { const id = firstNonEmptyString(traceId, snapshot?.traceId); if (!id || !snapshot) return state; - const message = state.message ?? debugMessageForTrace(id, { sessionId: snapshot.sessionId, threadId: snapshot.threadId, traceId: id }); + const message = state.message ?? debugMessageForTrace(id, { + sessionId: firstNonEmptyString(snapshot.sessionId), + threadId: firstNonEmptyString(snapshot.threadId), + traceId: id + }); const trace = traceFromRealtime(id, snapshot, Array.isArray(snapshot.events) ? snapshot.events : [], message.runnerTrace ?? null); return { ...state, diff --git a/web/hwlab-cloud-web/src/stores/workbench-message-projection-runtime.test.ts b/web/hwlab-cloud-web/src/stores/workbench-message-projection-runtime.test.ts index e4ef7799..bb1918a8 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-message-projection-runtime.test.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-message-projection-runtime.test.ts @@ -136,15 +136,13 @@ test("workbench active terminal paths seal final response from turn authority", const source = fs.readFileSync(path.join(storeDir, "workbench.ts"), "utf8"); const projectBlock = source.slice(source.indexOf("function projectTurnAuthorityToMessages"), source.indexOf("async function submitMessage")); const realtimeTurnBlock = source.slice(source.indexOf("function applyRealtimeTurnSnapshot"), source.indexOf("function scheduleRealtimeTurnProjection")); - const realtimeTurnProjectionBlock = source.slice(source.indexOf("function flushRealtimeTurnProjection"), source.indexOf("async function refreshTerminalTraceFromSyncReplay")); - const terminalSyncBlock = source.slice(source.indexOf("async function refreshTerminalTraceFromSyncReplay"), source.indexOf("function installRealtimeVisibilityHandler")); + const realtimeTurnProjectionBlock = source.slice(source.indexOf("function flushRealtimeTurnProjection"), source.indexOf("function installRealtimeVisibilityHandler")); const completeBlock = source.slice(source.indexOf("function completeTrace"), source.indexOf("async function hydrateTerminalMessageDiagnostics")); - const realtimeSessionSyncBlock = source.slice(source.indexOf("async function refreshRealtimeSessionFromSyncReplay"), source.indexOf("function completeTrace")); + const crossTabSyncBlock = source.slice(source.indexOf("async function refreshCrossTabSessionFromSyncReplay"), source.indexOf("function completeTrace")); const sessionDetailReadBlock = source.slice(source.indexOf("function fetchSessionDetailPage"), source.indexOf("function sessionMessageProjectionWindowLimit")); const traceDetailReadBlock = source.slice(source.indexOf("async function readTraceEventsForExplicitDetailPages"), source.indexOf("async function fetchTraceDetailEventsPage")); const loadBlock = source.slice(source.indexOf("async function loadWorkbenchSession"), source.indexOf("async function sealRestoredActiveTurnMessages")); const restoreSealBlock = source.slice(source.indexOf("async function sealRestoredActiveTurnMessages"), source.indexOf("function reattachRestoredActiveTrace")); - const activeSyncReplayBlock = source.slice(source.indexOf("async function refreshActiveTraceFromSyncReplay"), source.indexOf("function stopRealtime")); assert.match(projectBlock, /terminalAuthorityMessageFromTurnResult\(result\)/u); assert.match(projectBlock, /type:\s*"message\.upsert"/u); @@ -155,19 +153,14 @@ test("workbench active terminal paths seal final response from turn authority", assert.match(realtimeTurnBlock, /rememberTurnStatus\(traceId, result\)[\s\S]*scheduleRealtimeTurnProjection\(\{ traceId, result, terminalTurn \}\)/u); assert.doesNotMatch(realtimeTurnBlock, new RegExp(`applyTurnStatusSnapshot\\(|${["refresh", "TerminalTraceFromRest"].join("")}\\(`, "u")); assert.match(realtimeTurnProjectionBlock, /syncTurnStatusToMessage\(next\.traceId, next\.result\)/u); - assert.match(realtimeTurnProjectionBlock, /next\.terminalTurn[\s\S]*refreshTerminalTraceFromSyncReplay\(next\.traceId, "realtime-turn-snapshot"\)/u); assert.match(realtimeTurnProjectionBlock, /workbench_realtime_turn_projection_budget/u); - assert.match(terminalSyncBlock, /refreshWorkbenchSyncReplay\(ownerSessionId, id, null, `terminal-sync-replay:\$\{reason\}`\)/u); - assert.doesNotMatch(terminalSyncBlock, /refreshTurnStatusByTraceId|refreshMessageProjectionForTrace|fetchSessionMessagesPage|readTraceEventsForMessage/u); + assert.doesNotMatch(realtimeTurnProjectionBlock, /refreshWorkbenchSyncReplay|refreshTerminalTraceFromSyncReplay/u); assert.match(completeBlock, /projectTurnAuthorityToMessages\(traceId, result, "complete-trace"\)/u); assert.match(completeBlock, /options\.forceRead[\s\S]*refreshMessageProjectionForTrace\(ownerSessionId, traceId, \{ force: true \}\)/u); - assert.match(completeBlock, /else scheduleActiveTraceSyncReplay\(traceId, "complete-trace-sync-replay", 0\)/u); - assert.match(realtimeSessionSyncBlock, /traceIdFromRealtimeRefreshReason\(reason\)/u); - assert.match(realtimeSessionSyncBlock, /refreshWorkbenchSyncReplay\(id, traceId, null, `realtime-session-sync:\$\{reason\}`\)/u); - assert.doesNotMatch(realtimeSessionSyncBlock, /hydrateTurnStatusAuthority|sessionDetailAutoReadDecision|fetchSessionDetailPage|fetchSessionMessagesPage|refreshMessageProjectionForTrace/u); - assert.doesNotMatch(realtimeSessionSyncBlock, /loadWorkbenchSession|applySelectedSessionDetail/u); - assert.match(activeSyncReplayBlock, /refreshWorkbenchSyncReplay\(ownerSessionId, id, null, `active-sync-replay:\$\{reason\}`\)/u); - assert.doesNotMatch(activeSyncReplayBlock, /refreshTurnStatusByTraceId|refreshMessageProjectionForTrace|fetchSessionMessagesPage|readTraceEventsForMessage/u); + assert.doesNotMatch(completeBlock, /scheduleActiveTraceSyncReplay|refreshWorkbenchSyncReplay/u); + assert.match(crossTabSyncBlock, /traceIdFromRealtimeRefreshReason\(reason\)/u); + assert.match(crossTabSyncBlock, /refreshWorkbenchSyncReplay\(id, traceId, null, `cross-tab-sync:\$\{reason\}`\)/u); + assert.doesNotMatch(source, /scheduleActiveTraceSyncReplay|refreshActiveTraceFromSyncReplay|active-sync-replay:repeat|complete-trace-sync-replay/u); assert.match(traceDetailReadBlock, /traceAuthorityById\.value\[traceId\] \?\? message\.runnerTrace/u); assert.match(traceDetailReadBlock, /traceEventsDetailReadDecision\(traceId, afterProjectedSeq, message, options\)/u); assert.match(traceDetailReadBlock, /trace_events_auto_read_skip/u); @@ -183,27 +176,26 @@ test("workbench active terminal paths seal final response from turn authority", assert.doesNotMatch(restoreSealBlock, /force:\s*true/u); }); -test("workbench automatic live repair does not call legacy REST fan-out", () => { +test("workbench active SSE path never starts periodic sync replay", () => { const source = fs.readFileSync(path.join(storeDir, "workbench.ts"), "utf8"); - const automaticBlocks = [ + const activeBlocks = [ source.slice(source.indexOf("async function submitMessage"), source.indexOf("async function cancelAgentMessage")), source.slice(source.indexOf("function reattachTrace"), source.indexOf("function restartRealtime")), - source.slice(source.indexOf("async function refreshActiveTraceFromSyncReplay"), source.indexOf("function stopRealtime")), source.slice(source.indexOf("function flushRealtimeTurnProjection"), source.indexOf("function installRealtimeVisibilityHandler")), - source.slice(source.indexOf("async function refreshRealtimeSessionFromSyncReplay"), source.indexOf("function completeTrace")), + source.slice(source.indexOf("function completeTrace"), source.indexOf("async function hydrateTerminalMessageDiagnostics")), source.slice(source.indexOf("async function sealRestoredActiveTurnMessages"), source.indexOf("function reattachRestoredActiveTrace")) ].join("\n"); - assert.match(automaticBlocks, /refreshWorkbenchSyncReplay/u); - assert.doesNotMatch(automaticBlocks, /fetchWorkbenchTurnStatus|fetchWorkbenchTraceEvents|fetchSessionMessagesPage|fetchSessionDetailPage/u); - assert.doesNotMatch(automaticBlocks, /refreshTurnStatusByTraceId|refreshMessageProjectionForTrace|readTraceEventsForMessage\(/u); + assert.doesNotMatch(activeBlocks, /refreshWorkbenchSyncReplay|scheduleActiveTraceSyncReplay|setTimeout|setInterval/u); + assert.match(source.slice(source.indexOf("function executeRealtimeRecoveryStep"), source.indexOf("function stopRealtime")), /case "sync-replay"[\s\S]*refreshWorkbenchSyncReplay/u); + assert.match(source.slice(source.indexOf("async function refreshCrossTabSessionFromSyncReplay"), source.indexOf("function completeTrace")), /refreshWorkbenchSyncReplay/u); }); test("cross page projection signal always replays durable sync", () => { const source = fs.readFileSync(path.join(storeDir, "workbench.ts"), "utf8"); const signalBlock = source.slice(source.indexOf("function handleWorkbenchProjectionSignal"), source.indexOf("function applyTraceSnapshot")); - assert.match(signalBlock, /refreshRealtimeSessionFromSyncReplay\(sessionId, `cross-tab-session-projection:/u); + assert.match(signalBlock, /refreshCrossTabSessionFromSyncReplay\(sessionId, `cross-tab-session-projection:/u); assert.doesNotMatch(signalBlock, /messages\.value\.some/u); - assert.doesNotMatch(signalBlock, /return;\s*\n\s*void refreshRealtimeSessionFromSyncReplay/u); + assert.doesNotMatch(signalBlock, /return;\s*\n\s*void refreshCrossTabSessionFromSyncReplay/u); }); diff --git a/web/hwlab-cloud-web/src/stores/workbench-server-state.ts b/web/hwlab-cloud-web/src/stores/workbench-server-state.ts index bf77b27f..e4d641ec 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-server-state.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-server-state.ts @@ -200,8 +200,9 @@ function reduceMessageUpsert(state: WorkbenchServerState, sessionId: string | nu const existingSession = state.sessionsById[id]; const existing = state.messagesBySessionId[id] ?? existingSession?.messages ?? []; const index = existing.findIndex((item) => messageMatchesSnapshot(item, message)); - const mergedMessage = index >= 0 ? mergeMessageSnapshot(existing[index], message) : message; - if (index >= 0 && messageEquivalent(existing[index], mergedMessage)) return state; + const current = index >= 0 ? existing[index] : undefined; + const mergedMessage = current ? mergeMessageSnapshot(current, message) : message; + if (current && messageEquivalent(current, mergedMessage)) return state; const mergedMessages = index >= 0 ? existing.map((item, itemIndex) => itemIndex === index ? mergedMessage : item) : [...existing, mergedMessage]; @@ -488,7 +489,7 @@ function valuesEquivalent(left: unknown, right: unknown): boolean { if (leftKeys.length !== rightKeys.length) return false; for (let index = 0; index < leftKeys.length; index += 1) { const key = leftKeys[index]; - if (key !== rightKeys[index]) return false; + if (key === undefined || key !== rightKeys[index]) return false; if (!valuesEquivalent(leftRecord[key], rightRecord[key])) return false; } return true; diff --git a/web/hwlab-cloud-web/src/stores/workbench-session.test.ts b/web/hwlab-cloud-web/src/stores/workbench-session.test.ts index 083a591e..f7eb4ab5 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-session.test.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-session.test.ts @@ -95,9 +95,10 @@ test("turn status refresh selector keeps automatic hydrate bounded to the active test("turn status refresh selector skips sealed terminal traces", () => { const sessionId = "ses_turn_detail_sealed"; - const sealed = agentMessage({ id: "msg_sealed", traceId: "trc_sealed", sessionId, status: "completed", text: "final" }); + const traceId = "trc_sealed"; + const sealed = agentMessage({ id: "msg_sealed", traceId, sessionId, status: "completed", text: "final" }); - assert.deepEqual(selectActiveTurnStatusRefreshTraceIds({ messages: [sealed], currentRequestTraceId: sealed.traceId, turnStatusAuthority: { [sealed.traceId]: turnStatus({ traceId: sealed.traceId, sessionId, status: "completed", running: false, terminal: true }) }, limit: 1 }), []); + assert.deepEqual(selectActiveTurnStatusRefreshTraceIds({ messages: [sealed], currentRequestTraceId: traceId, turnStatusAuthority: { [traceId]: turnStatus({ traceId, sessionId, status: "completed", running: false, terminal: true }) }, limit: 1 }), []); }); test("terminal detail read skips sealed terminal trace authority", () => { diff --git a/web/hwlab-cloud-web/src/stores/workbench-trace-detail.test.ts b/web/hwlab-cloud-web/src/stores/workbench-trace-detail.test.ts index d5999179..51c62aef 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-trace-detail.test.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-trace-detail.test.ts @@ -45,7 +45,7 @@ test("terminal seal strips heavy trace rows while preserving final authority", ( const sealed = terminalSealResultWithoutTraceEvents(result); assert.equal(sealed.status, "completed"); - assert.equal(sealed.finalResponse?.text, "final answer"); + assert.equal((sealed.finalResponse as { text?: string } | null | undefined)?.text, "final answer"); assert.equal(sealed.events, undefined); assert.equal(sealed.traceEvents, undefined); assert.equal(sealed.runnerTrace?.events, undefined); diff --git a/web/hwlab-cloud-web/src/stores/workbench-trace-detail.ts b/web/hwlab-cloud-web/src/stores/workbench-trace-detail.ts index 2e6f1884..7b9784e6 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-trace-detail.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-trace-detail.ts @@ -1,4 +1,8 @@ +import type { WorkbenchRealtimeEvent } from "../api/workbench-events"; +import type { TraceSnapshot } from "../composables/workbench-trace-snapshot"; +import { normalizeProjectionDiagnostic } from "../utils/workbench-error-runtime"; import type { AgentChatResultResponse, ChatMessage, TraceEvent } from "../types"; +import { asAgentRun, firstFiniteNumber, messageTimingPatch, optionalString, recordValue, traceSnapshotError } from "./workbench-message-projection-runtime"; export function traceDetailProjectedSeq(trace: ChatMessage["runnerTrace"]): number { const cursor = traceDetailCursor(trace); @@ -21,6 +25,39 @@ export function terminalSealResultWithoutTraceEvents(result: AgentChatResultResp return { ...rest, runnerTrace: traceRest }; } +export function realtimeSnapshotToTraceSnapshot(traceId: string, snapshot: WorkbenchRealtimeEvent["snapshot"], eventPageEvents?: TraceEvent[]): TraceSnapshot { + const source = snapshot ?? { traceId }; + const events = Array.isArray(eventPageEvents) ? eventPageEvents : Array.isArray(source.events) ? source.events : []; + const { traceId: _sourceTraceId, error: _sourceError, ...sourceRest } = source; + return { + ...sourceRest, + traceId: optionalString(source.traceId, traceId), + status: text(source.status) ?? undefined, + events, + eventCount: firstFiniteNumber(source.eventCount, events.length), + fullTraceLoaded: source.fullTraceLoaded === true, + hasMore: source.hasMore === true, + truncated: source.truncated === true, + nextProjectedSeq: firstFiniteNumber(source.nextProjectedSeq) ?? null, + range: recordValue(source.range) as TraceSnapshot["range"], + agentRun: asAgentRun(source.agentRun) ?? undefined, + traceStatus: text(source.traceStatus) ?? undefined, + retention: source.retention, + terminalEvidence: source.terminalEvidence, + traceSummary: source.traceSummary, + error: traceSnapshotError(source.error), + projection: normalizeProjectionDiagnostic(source.projection ?? source) ?? null, + projectionStatus: text(source.projectionStatus) ?? text(recordValue(source.projection)?.projectionStatus) ?? undefined, + projectionHealth: text(source.projectionHealth) ?? text(recordValue(source.projection)?.projectionHealth) ?? undefined, + staleMs: firstFiniteNumber(source.staleMs, recordValue(source.projection)?.staleMs), + blocker: recordValue(source.blocker) ?? recordValue(recordValue(source.projection)?.blocker) ?? undefined, + ...messageTimingPatch(source), + lastEventLabel: text(source.lastEventLabel) ?? undefined, + eventSource: "trace-api", + updatedAt: text(source.updatedAt) ?? new Date().toISOString() + } as TraceSnapshot; +} + function traceDetailCursor(source: { nextProjectedSeq?: unknown; range?: { toProjectedSeq?: unknown } | null } | null | undefined): number | null { return finiteProjectedSeq(source?.nextProjectedSeq) ?? finiteProjectedSeq(source?.range?.toProjectedSeq); } @@ -41,3 +78,8 @@ function traceEventProjectedSeq(event: TraceEvent): number { const seq = Number(event.projectedSeq); return Number.isFinite(seq) ? Math.trunc(seq) : Number.NaN; } + +function text(value: unknown): string | null { + const normalized = typeof value === "string" ? value.trim() : ""; + return normalized || null; +} diff --git a/web/hwlab-cloud-web/src/stores/workbench.ts b/web/hwlab-cloud-web/src/stores/workbench.ts index 10412fb8..eaaad5fd 100644 --- a/web/hwlab-cloud-web/src/stores/workbench.ts +++ b/web/hwlab-cloud-web/src/stores/workbench.ts @@ -64,7 +64,6 @@ import { traceHasTerminalResponse, traceHasEvents, traceResultHasTerminalEvidence, - traceSnapshotError } from "./workbench-message-projection-runtime"; import { planWorkbenchRealtimeApply, planWorkbenchRealtimeRecovery, type WorkbenchRealtimeApplyStep, type WorkbenchRealtimeRecoveryStep } from "./workbench-realtime-plan"; import { workbenchSyncReplayDiagnostic, workbenchSyncReplayEvents } from "./workbench-realtime-authority"; @@ -72,7 +71,7 @@ import { useWorkbenchColadaMutations } from "./workbench-colada-mutations"; import { useWorkbenchColadaQueries } from "./workbench-colada-queries"; import { useWorkbenchColadaReducer } from "./workbench-colada-reducer"; import { projectionMergeCommitSummary, traceEventsAutoReadDecision, workbenchSessionDetailReadKey, workbenchSessionMessagesReadKey, workbenchTraceEventsReadKey, type TraceEventsReadRangeRecord } from "./workbench-session-messages-read-budget"; -import { terminalSealResultWithoutTraceEvents, traceDetailProjectedSeq, traceNextProjectedSeq } from "./workbench-trace-detail"; +import { realtimeSnapshotToTraceSnapshot, terminalSealResultWithoutTraceEvents, traceDetailProjectedSeq, traceNextProjectedSeq } from "./workbench-trace-detail"; const WORKBENCH_SESSION_PROJECTION_SIGNAL_CHANNEL = "hwlab.workbench.sessionProjection.v1"; const WORKBENCH_SESSION_PROJECTION_SIGNAL_KEY = "hwlab.workbench.sessionProjectionSignal.v1"; @@ -825,7 +824,6 @@ export const useWorkbenchStore = defineStore("workbench", () => { completeTrace(canonicalTraceId, response.data as AgentChatResultResponse); return true; } - scheduleActiveTraceSyncReplay(canonicalTraceId, "submit-sync-replay"); restartRealtime("submit"); return true; } @@ -842,7 +840,6 @@ export const useWorkbenchStore = defineStore("workbench", () => { recordActivity("cancel:accepted"); if (message.status === "running") chatPending.value = false; currentRequest.value = null; - clearActiveTraceSyncReplay(traceId); void clearActiveTrace(traceId, "cancel-agent-message"); scheduleSessionListRefresh(sessionId, runtimePolicy.sessionListTerminalRefreshDelayMs); } @@ -1050,7 +1047,6 @@ export const useWorkbenchStore = defineStore("workbench", () => { })); markWorkbenchTraceProjected(traceId); if (traceResultHasTerminalEvidence(result) && !turnResultIsTerminalForMerge(result)) { - void refreshTerminalTraceFromSyncReplay(traceId, "trace-detail-read-terminal-evidence"); } if (turnResultIsTerminalForMerge(result)) { rememberTurnStatus(traceId, result); @@ -1100,7 +1096,6 @@ export const useWorkbenchStore = defineStore("workbench", () => { const initial: AgentChatResponse = { status: "running", traceId }; if (!messages.value.some((message) => message.traceId === traceId)) appendActiveMessages(makeMessage("agent", "", "running", { traceId, sessionId: selectedSessionId.value ?? undefined, threadId: selectedThreadId.value ?? undefined, title: "Code Agent", traceAutoLifecycle: "running" })); currentRequest.value = { traceId, sessionId: selectedSessionId.value ?? null, threadId: selectedThreadId.value ?? null, status: initial.status }; - scheduleActiveTraceSyncReplay(traceId, "reattach-sync-replay"); restartRealtime("reattach"); } @@ -1144,55 +1139,25 @@ export const useWorkbenchStore = defineStore("workbench", () => { } async function refreshWorkbenchSyncReplay(sessionId: string | null, traceId: string | null, sinceOutboxSeq: number | null, reason: string): Promise { - const result = await workbenchColadaQueries.fetchSyncReplay({ sessionId, traceId, since: sinceOutboxSeq }, { timeoutMs: 8000, activityRef: () => activityRef.value }); - if (!result.ok || !result.data) { - recordWorkbenchRuntimeDiagnostic({ module: "workbench-sync-replay", sessionId, traceId, outcome: "network", diagnostic: { code: "workbench_sync_replay_failed", reason, status: result.status, apiError: result.apiError, valuesRedacted: true } }); - return; + let cursor = firstFiniteNumber(sinceOutboxSeq) ?? 0; + for (;;) { + const result = await workbenchColadaQueries.fetchSyncReplay({ sessionId, traceId, since: cursor }, { timeoutMs: 8000, activityRef: () => activityRef.value }); + if (!result.ok || !result.data) { + recordWorkbenchRuntimeDiagnostic({ module: "workbench-sync-replay", sessionId, traceId, outcome: "network", diagnostic: { code: "workbench_sync_replay_failed", reason, status: result.status, apiError: result.apiError, valuesRedacted: true } }); + return; + } + const events = workbenchSyncReplayEvents(result.data); + for (const event of events) applyRealtimeEvent(event, realtimeEventName(event)); + recordWorkbenchRuntimeDiagnostic({ module: "workbench-sync-replay", sessionId, traceId, outcome: "ok", diagnostic: workbenchSyncReplayDiagnostic(result.data, events, { reason, sinceOutboxSeq: cursor }) }); + const responseCursor = recordValue(result.data.cursor); + if (responseCursor?.hasMore !== true) return; + const nextCursor = firstFiniteNumber(responseCursor.outboxSeq); + if (nextCursor === undefined || nextCursor <= cursor) { + recordWorkbenchRuntimeDiagnostic({ module: "workbench-sync-replay", sessionId, traceId, outcome: "error", diagnostic: { code: "workbench_sync_replay_cursor_stalled", reason, cursor, nextCursor, valuesRedacted: true } }); + return; + } + cursor = nextCursor; } - const events = workbenchSyncReplayEvents(result.data); - for (const event of events) applyRealtimeEvent(event, realtimeEventName(event)); - recordWorkbenchRuntimeDiagnostic({ module: "workbench-sync-replay", sessionId, traceId, outcome: "ok", diagnostic: workbenchSyncReplayDiagnostic(result.data, events, { reason, sinceOutboxSeq }) }); - } - - function scheduleActiveTraceSyncReplay(traceId: string | null | undefined, reason: string, delayMs = runtimePolicy.workbenchActiveTraceSyncReplayInitialMs): void { - const id = firstNonEmptyString(traceId); - if (!id) return; - if (typeof window === "undefined") { - void refreshActiveTraceFromSyncReplay(id, reason); - return; - } - window.setTimeout(() => { void refreshActiveTraceFromSyncReplay(id, reason); }, delayMs); - } - - function clearActiveTraceSyncReplay(traceId: string | null | undefined): void { - void traceId; - } - - async function refreshActiveTraceFromSyncReplay(traceId: string, reason: string): Promise { - const id = firstNonEmptyString(traceId); - if (!id) return; - if (traceTerminalBodyIsVisible(id)) { - clearActiveTraceSyncReplay(id); - recordWorkbenchRuntimeDiagnostic({ module: "workbench-terminal-priority", traceId: id, outcome: "ok", diagnostic: { code: "terminal_low_priority_sync_replay_skip", reason, source: "active-sync-replay", valuesRedacted: true } }); - return; - } - const ownerSessionId = traceOwnerSessionId(id, currentRequest.value?.sessionId ?? null); - if (ownerSessionId === activeSessionId.value) recordActivity(reason); - await refreshWorkbenchSyncReplay(ownerSessionId, id, null, `active-sync-replay:${reason}`); - const ownerMessages = ownerSessionId ? serverState.value.messagesBySessionId[ownerSessionId] ?? [] : messages.value; - if (traceProjectionIsTerminalSealed(id, ownerMessages)) { - clearActiveTraceSyncReplay(id); - return; - } - const message = latestMessageForTrace(id, ownerMessages); - const turn = turnStatusAuthority.value[id]; - const terminal = messageHasTerminalResponse(message); - if (terminal) { - clearActiveTraceSyncReplay(id); - return; - } - const stillActive = currentRequest.value?.traceId === id || isTraceActiveStatus(turn?.status) || isTraceActiveStatus(message?.status); - if (stillActive) scheduleActiveTraceSyncReplay(id, "active-sync-replay:repeat", runtimePolicy.workbenchActiveTraceSyncReplayRepeatMs); } function stopRealtime(): void { @@ -1405,7 +1370,6 @@ export const useWorkbenchStore = defineStore("workbench", () => { const startedAt = performanceNowMs(); if (shouldApplyActiveTraceAuthority(next.traceId, traceResultSessionId(next.result))) { syncTurnStatusToMessage(next.traceId, next.result); - if (next.terminalTurn) void refreshTerminalTraceFromSyncReplay(next.traceId, "realtime-turn-snapshot"); } recordWorkbenchRuntimeDiagnostic({ module: "workbench-turn-status", @@ -1425,25 +1389,6 @@ export const useWorkbenchStore = defineStore("workbench", () => { if (realtimeTurnProjectionQueue.size > 0) scheduleRealtimeTurnProjectionFlush(); } - async function refreshTerminalTraceFromSyncReplay(traceId: string, reason: string): Promise { - const id = firstNonEmptyString(traceId); - if (!id) return; - if (traceTerminalBodyIsVisible(id)) { - clearActiveTraceSyncReplay(id); - recordWorkbenchRuntimeDiagnostic({ module: "workbench-terminal-priority", traceId: id, outcome: "ok", diagnostic: { code: "terminal_low_priority_terminal_sync_skip", reason, source: "terminal-sync-replay", valuesRedacted: true } }); - return; - } - const ownerSessionId = traceOwnerSessionId(id, null); - if (ownerSessionId === activeSessionId.value) recordActivity(reason); - await refreshWorkbenchSyncReplay(ownerSessionId, id, null, `terminal-sync-replay:${reason}`); - const ownerMessages = ownerSessionId ? serverState.value.messagesBySessionId[ownerSessionId] ?? [] : messages.value; - if (traceProjectionIsTerminalSealed(id, ownerMessages)) { - clearActiveTraceSyncReplay(id); - return; - } - if (ownerSessionId) scheduleSessionListRefresh(ownerSessionId, runtimePolicy.sessionListTerminalRefreshDelayMs); - } - function installRealtimeVisibilityHandler(): void { if (typeof document === "undefined") return; document.addEventListener("visibilitychange", () => { @@ -1484,7 +1429,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { const sessionId = normalizeWorkbenchSessionId(record.sessionId); if (!sessionId || sessionId !== activeSessionId.value) return; const traceId = firstNonEmptyString(record.traceId); - void refreshRealtimeSessionFromSyncReplay(sessionId, `cross-tab-session-projection:${firstNonEmptyString(record.reason, traceId, "session") ?? "session"}`); + void refreshCrossTabSessionFromSyncReplay(sessionId, `cross-tab-session-projection:${firstNonEmptyString(record.reason, traceId, "session") ?? "session"}`); } function applyTraceSnapshot(traceId: string, snapshot: TraceSnapshot): void { @@ -1495,10 +1440,8 @@ export const useWorkbenchStore = defineStore("workbench", () => { const authoritySessionId = traceResultSessionId(trace) ?? traceResultSessionId(snapshot); const ownerSessionId = traceOwnerSessionId(traceId, authoritySessionId); if (!ownerSessionId) return; - let matchedMessage = false; updateSessionMessages(ownerSessionId, (source) => source.map((message) => { if (!messageMatchesTraceAuthority(message, traceId, authoritySessionId, ownerSessionId)) return message; - matchedMessage = true; const mergedRunnerTrace = mergeRunnerTrace(message.runnerTrace, trace); const traceStatus = normalizedStatusText(trace.status ?? snapshot.status) ?? null; const clearCompletedDiagnostics = shouldClearCompletedTurnDiagnostics(traceStatus, null); @@ -1509,14 +1452,11 @@ export const useWorkbenchStore = defineStore("workbench", () => { const projection = clearCompletedDiagnostics ? nonBlockingProjection(trace.projection ?? null) : trace.projection ?? runnerTrace.projection ?? message.projection ?? null; return { ...message, ...messageTimingPatchForMerge(message, trace), ...messageStatusPatchForTerminalMerge(message, traceStatus, terminal), runnerTrace, error: clearCompletedDiagnostics ? null : error ?? message.error ?? null, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, updatedAt: new Date().toISOString() }; })); - if (!matchedMessage && ownerSessionId === activeSessionId.value) { - void refreshRealtimeSessionFromSyncReplay(ownerSessionId, `realtime-trace-snapshot:${traceId}`); - } markWorkbenchTraceProjected(traceId); if (!traceProjectionIsTerminalSealed(traceId, serverState.value.messagesBySessionId[ownerSessionId] ?? [])) scheduleSessionListRefresh(ownerSessionId, runtimePolicy.sessionListRealtimeRefreshDelayMs); } - async function refreshRealtimeSessionFromSyncReplay(sessionId: string, reason: string): Promise { + async function refreshCrossTabSessionFromSyncReplay(sessionId: string, reason: string): Promise { const id = normalizeWorkbenchSessionId(sessionId); if (!id || id !== activeSessionId.value) return; const traceId = traceIdFromRealtimeRefreshReason(reason); @@ -1525,7 +1465,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { return; } recordActivity(reason); - await refreshWorkbenchSyncReplay(id, traceId, null, `realtime-session-sync:${reason}`); + await refreshWorkbenchSyncReplay(id, traceId, null, `cross-tab-sync:${reason}`); } function completeTrace(traceId: string, result: AgentChatResultResponse, options: { forceRead?: boolean } = {}): void { @@ -1536,12 +1476,10 @@ export const useWorkbenchStore = defineStore("workbench", () => { projectTurnAuthorityToMessages(traceId, result, "complete-trace"); rememberTurnStatus(traceId, result); markWorkbenchTraceProjected(traceId); - clearActiveTraceSyncReplay(traceId); const ownerMessages = serverState.value.messagesBySessionId[ownerSessionId] ?? []; const terminalMessage = latestMessageForTrace(traceId, ownerMessages); if (!traceProjectionIsTerminalSealed(traceId, ownerMessages)) { if (options.forceRead) void refreshMessageProjectionForTrace(ownerSessionId, traceId, { force: true }); - else scheduleActiveTraceSyncReplay(traceId, "complete-trace-sync-replay", 0); } if (options.forceRead && terminalMessage && !traceProjectionIsTerminalSealed(traceId, ownerMessages)) void readTraceEventsForMessage(terminalMessage, { force: true }); if (ownerSessionId === activeSessionId.value) { @@ -1802,7 +1740,6 @@ export const useWorkbenchStore = defineStore("workbench", () => { } async function clearActiveTrace(traceId: string, reason: string): Promise { - clearActiveTraceSyncReplay(traceId); if (currentRequest.value?.traceId === traceId) currentRequest.value = null; } @@ -2010,7 +1947,7 @@ function normalizeChatMessage(message: ChatMessage): ChatMessage { const text = role === "agent" ? isTerminalMessageStatus(status) ? projectedAgentMessageText({ status, finalText, errorText, baseText }) - : "" + : firstNonEmptyString(baseText, finalText) ?? "" : firstNonEmptyString(baseText, finalText, errorText) ?? ""; const messageId = firstNonEmptyString((message as Record).messageId, message.id) ?? nextProtocolId("msg"); const timingPatch = isTerminalMessageStatus(status) ? terminalMessageTimingPatchForNormalize(message) : messageTimingPatch(message); @@ -2030,39 +1967,6 @@ function activeTraceIdFromMessages(messages: ChatMessage[], turnStatusAuthority: return null; } -function realtimeSnapshotToTraceSnapshot(traceId: string, snapshot: WorkbenchRealtimeEvent["snapshot"], eventPageEvents?: TraceEvent[]): TraceSnapshot { - const source = snapshot ?? { traceId }; - const events = Array.isArray(eventPageEvents) ? eventPageEvents : Array.isArray(source.events) ? source.events : []; - const { traceId: _sourceTraceId, error: _sourceError, ...sourceRest } = source; - return { - ...sourceRest, - traceId: optionalString(source.traceId, traceId), - status: firstNonEmptyString(source.status) ?? undefined, - events, - eventCount: firstFiniteNumber(source.eventCount, events.length), - fullTraceLoaded: source.fullTraceLoaded === true, - hasMore: source.hasMore === true, - truncated: source.truncated === true, - nextProjectedSeq: firstFiniteNumber(source.nextProjectedSeq) ?? null, - range: recordValue(source.range) as TraceSnapshot["range"], - agentRun: asAgentRun(source.agentRun) ?? undefined, - traceStatus: firstNonEmptyString(source.traceStatus) ?? undefined, - retention: source.retention, - terminalEvidence: source.terminalEvidence, - traceSummary: source.traceSummary, - error: traceSnapshotError(source.error), - projection: normalizeProjectionDiagnostic(source.projection ?? source) ?? null, - projectionStatus: firstNonEmptyString(source.projectionStatus, recordValue(source.projection)?.projectionStatus) ?? undefined, - projectionHealth: firstNonEmptyString(source.projectionHealth, recordValue(source.projection)?.projectionHealth) ?? undefined, - staleMs: firstFiniteNumber(source.staleMs, recordValue(source.projection)?.staleMs), - blocker: recordValue(source.blocker) ?? recordValue(recordValue(source.projection)?.blocker) ?? undefined, - ...messageTimingPatch(source), - lastEventLabel: firstNonEmptyString(source.lastEventLabel) ?? undefined, - eventSource: "trace-api", - updatedAt: firstNonEmptyString(source.updatedAt) ?? new Date().toISOString() - } as TraceSnapshot; -} - export async function liveCall(label: string, call: () => Promise>): Promise> { try { return await call(); diff --git a/web/hwlab-cloud-web/src/types/global.d.ts b/web/hwlab-cloud-web/src/types/global.d.ts index 17b5c110..1b38ba95 100644 --- a/web/hwlab-cloud-web/src/types/global.d.ts +++ b/web/hwlab-cloud-web/src/types/global.d.ts @@ -54,14 +54,8 @@ declare global { sessionListTerminalRefreshDelayMs?: number; sessionListMinRefreshIntervalMs?: number; workbenchRealtimeErrorSyncReplayMinMs?: number; - workbenchActiveTraceSyncReplayInitialMs?: number; - workbenchActiveTraceSyncReplayRepeatMs?: number; /** @deprecated Use workbenchRealtimeErrorSyncReplayMinMs. */ workbenchRealtimeErrorGapFillMinMs?: number; - /** @deprecated Use workbenchActiveTraceSyncReplayInitialMs. */ - workbenchActiveTraceRestGapFillInitialMs?: number; - /** @deprecated Use workbenchActiveTraceSyncReplayRepeatMs. */ - workbenchActiveTraceRestGapFillRepeatMs?: number; }; }; opencode?: { diff --git a/web/hwlab-cloud-web/src/views/workbench/WorkbenchDebugView.vue b/web/hwlab-cloud-web/src/views/workbench/WorkbenchDebugView.vue index 11687892..b6ae82fb 100644 --- a/web/hwlab-cloud-web/src/views/workbench/WorkbenchDebugView.vue +++ b/web/hwlab-cloud-web/src/views/workbench/WorkbenchDebugView.vue @@ -3,55 +3,26 @@ // Responsibility: Workbench debug route for backend-driven fake SSE single-step Trace card inspection. import { computed, onBeforeUnmount, onMounted, ref } from "vue"; -import { connectWorkbenchDebugFakeSse, connectWorkbenchKafkaSseDebug, workbenchDebugAPI, type WorkbenchDebugFakeSseQueue, type WorkbenchDebugFakeSseSequence, type WorkbenchDebugFakeSseStream, type WorkbenchKafkaSseDebugEvent, type WorkbenchKafkaSseDebugStreamName } from "@/api"; +import { connectWorkbenchDebugFakeSse, workbenchDebugAPI, type WorkbenchDebugFakeSseQueue, type WorkbenchDebugFakeSseSequence, type WorkbenchDebugFakeSseStream } from "@/api"; import StatusBadge from "@/components/common/StatusBadge.vue"; import WorkbenchMessageCard from "@/components/workbench/WorkbenchMessageCard.vue"; import type { WorkbenchRealtimeEvent } from "@/api/workbench-events"; import { applyWorkbenchDebugFakeSseEvent, createWorkbenchDebugFakeSseState } from "@/stores/workbench-debug-fake-sse"; const queueId = "trace-card"; -const activeTab = ref<"fake" | "kafka">("fake"); const selectedSequenceId = ref("trace-card-basic"); const sequences = ref([]); const queue = ref(null); const projection = ref(createWorkbenchDebugFakeSseState()); const streamStatus = ref<"connecting" | "open" | "error" | "closed">("connecting"); -const kafkaStatus = ref<"idle" | "connecting" | "open" | "error" | "closed">("idle"); -const kafkaStreamName = ref("hwlab"); -const kafkaFromBeginning = ref(false); -const kafkaFilters = ref({ traceId: "", sessionId: "", runId: "", commandId: "" }); -const kafkaEvents = ref([]); -const kafkaTraceStatuses = ref>(createKafkaStatusMap()); -const kafkaTraceEvents = ref>(createKafkaEventMap()); const busy = ref(false); const error = ref(null); const appendDraft = ref(defaultAppendDraft()); let stream: WorkbenchDebugFakeSseStream | null = null; -let kafkaStream: WorkbenchDebugFakeSseStream | null = null; -const kafkaTraceStreams: Partial> = {}; - -const KAFKA_TRACE_STREAMS: Array<{ name: WorkbenchKafkaSseDebugStreamName; label: string; topic: string }> = [ - { name: "stdio", label: "codex stdio", topic: "codex-stdio.raw.v1" }, - { name: "agentrun", label: "AgentRun event", topic: "agentrun.event.v1" }, - { name: "hwlab", label: "HWLAB event", topic: "hwlab.event.v1" } -]; - -type KafkaStreamStatus = "idle" | "connecting" | "open" | "error" | "closed"; - -interface KafkaRawRow { - id: string; - eventName: string; - receivedAt: string; - payload: WorkbenchKafkaSseDebugEvent; -} - const message = computed(() => projection.value.message); const logs = computed(() => projection.value.logs); const queueSummary = computed(() => queue.value ? `${queue.value.cursor}/${queue.value.eventCount}` : "-"); const streamBadgeStatus = computed(() => streamStatus.value === "open" ? "completed" : streamStatus.value === "error" ? "failed" : "running"); -const kafkaBadgeStatus = computed(() => kafkaStatus.value === "open" ? "completed" : kafkaStatus.value === "error" ? "failed" : "running"); -const kafkaTraceEventCount = computed(() => Object.values(kafkaTraceEvents.value).reduce((total, rows) => total + rows.length, 0)); -const kafkaTraceConnectedCount = computed(() => Object.values(kafkaTraceStatuses.value).filter((status) => status === "open").length); const canStep = computed(() => !busy.value && (queue.value?.remaining ?? 0) > 0); onMounted(async () => { @@ -63,11 +34,7 @@ onMounted(async () => { onBeforeUnmount(() => { stream?.close(); stream = null; - kafkaStream?.close(); - kafkaStream = null; - closeKafkaTraceStreams(); streamStatus.value = "closed"; - kafkaStatus.value = "closed"; }); async function refreshDescription(): Promise { @@ -97,96 +64,6 @@ function openStream(): void { if (!stream) streamStatus.value = "error"; } -function openKafkaStream(): void { - kafkaStream?.close(); - kafkaStatus.value = "connecting"; - kafkaStream = connectWorkbenchKafkaSseDebug({ - stream: kafkaStreamName.value, - fromBeginning: kafkaFromBeginning.value, - traceId: cleanFilter(kafkaFilters.value.traceId), - sessionId: cleanFilter(kafkaFilters.value.sessionId), - runId: cleanFilter(kafkaFilters.value.runId), - commandId: cleanFilter(kafkaFilters.value.commandId), - onOpen: () => { kafkaStatus.value = "open"; }, - onError: () => { kafkaStatus.value = "error"; }, - onEvent: (event, eventName) => { - if (eventName === "hwlab.kafka.connected") return; - if (eventName === "hwlab.kafka.error") { - kafkaStatus.value = "error"; - error.value = event.error?.message || "Kafka SSE 事件流错误"; - } - kafkaEvents.value = [{ id: kafkaRowId(event, eventName), eventName, receivedAt: new Date().toISOString(), payload: event }, ...kafkaEvents.value].slice(0, 120); - } - }); - if (!kafkaStream) kafkaStatus.value = "error"; -} - -function closeKafkaStream(): void { - kafkaStream?.close(); - kafkaStream = null; - kafkaStatus.value = "closed"; -} - -function clearKafkaEvents(): void { - kafkaEvents.value = []; -} - -function openKafkaTraceStreams(): void { - closeKafkaTraceStreams(); - kafkaTraceEvents.value = createKafkaEventMap(); - kafkaTraceStatuses.value = createKafkaStatusMap("connecting"); - for (const streamName of KAFKA_TRACE_STREAMS.map((entry) => entry.name)) { - const opened = connectWorkbenchKafkaSseDebug({ - stream: streamName, - fromBeginning: true, - traceId: cleanFilter(kafkaFilters.value.traceId), - sessionId: cleanFilter(kafkaFilters.value.sessionId), - runId: cleanFilter(kafkaFilters.value.runId), - commandId: cleanFilter(kafkaFilters.value.commandId), - onOpen: () => setKafkaTraceStatus(streamName, "open"), - onError: () => setKafkaTraceStatus(streamName, "error"), - onEvent: (event, eventName) => { - if (eventName === "hwlab.kafka.connected") return; - if (eventName === "hwlab.kafka.error") { - setKafkaTraceStatus(streamName, "error"); - error.value = event.error?.message || "Kafka SSE 三流订阅错误"; - } - pushKafkaTraceEvent(streamName, { id: kafkaRowId(event, eventName), eventName, receivedAt: new Date().toISOString(), payload: event }); - } - }); - if (opened) kafkaTraceStreams[streamName] = opened; - else setKafkaTraceStatus(streamName, "error"); - } -} - -function closeKafkaTraceStreams(): void { - for (const streamName of KAFKA_TRACE_STREAMS.map((entry) => entry.name)) { - kafkaTraceStreams[streamName]?.close(); - delete kafkaTraceStreams[streamName]; - } - kafkaTraceStatuses.value = Object.fromEntries(KAFKA_TRACE_STREAMS.map((entry) => [entry.name, kafkaTraceStatuses.value[entry.name] === "idle" ? "idle" : "closed"])) as Record; -} - -function clearKafkaTraceEvents(): void { - kafkaTraceEvents.value = createKafkaEventMap(); -} - -function setKafkaTraceStatus(streamName: WorkbenchKafkaSseDebugStreamName, status: KafkaStreamStatus): void { - kafkaTraceStatuses.value = { ...kafkaTraceStatuses.value, [streamName]: status }; -} - -function pushKafkaTraceEvent(streamName: WorkbenchKafkaSseDebugStreamName, row: KafkaRawRow): void { - kafkaTraceEvents.value = { ...kafkaTraceEvents.value, [streamName]: [row, ...kafkaTraceEvents.value[streamName]].slice(0, 80) }; -} - -function createKafkaStatusMap(status: KafkaStreamStatus = "idle"): Record { - return { stdio: status, agentrun: status, hwlab: status }; -} - -function createKafkaEventMap(): Record { - return { stdio: [], agentrun: [], hwlab: [] }; -} - async function resetQueue(): Promise { busy.value = true; error.value = null; @@ -245,38 +122,6 @@ function parseAppendDraft(raw: string): { ok: true; events: WorkbenchRealtimeEve } } -function cleanFilter(value: string): string | undefined { - const text = value.trim(); - return text ? text : undefined; -} - -function kafkaRowId(event: WorkbenchKafkaSseDebugEvent, eventName: string): string { - const topic = event.topic || "topic"; - const partition = event.partition ?? "p"; - const offset = event.offset || Date.now(); - return `${eventName}:${topic}:${partition}:${offset}:${Math.random().toString(16).slice(2)}`; -} - -function kafkaEventSummary(event: WorkbenchKafkaSseDebugEvent): string { - const value = recordValue(event.value); - const nestedEvent = recordValue(value.event); - return String(value.eventType || value.type || nestedEvent.type || event.error?.code || "event"); -} - -function kafkaEventIds(event: WorkbenchKafkaSseDebugEvent): string { - const value = recordValue(event.value); - const context = recordValue(value.context); - return [value.traceId, value.sessionId, context.runId || value.runId, context.commandId || value.commandId].filter(Boolean).join(" · ") || "-"; -} - -function kafkaEventJson(event: WorkbenchKafkaSseDebugEvent): string { - return JSON.stringify(event.value ?? event, null, 2); -} - -function recordValue(value: unknown): Record { - return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; -} - function defaultAppendDraft(): string { return JSON.stringify({ type: "trace.event", @@ -305,18 +150,12 @@ function defaultAppendDraft(): string {

实时调试台

- - - {{ activeTab === 'fake' ? `Queue ${queueSummary}` : `${kafkaTraceConnectedCount}/3 streams · ${kafkaTraceEventCount + kafkaEvents.length} events` }} + + Queue {{ queueSummary }}
- - -
+
- - - - - - - - - - - -
-

{{ error }}

-
+
等待第一条 fake SSE message.snapshot
@@ -398,62 +200,6 @@ function defaultAppendDraft(): string {
-
-
-
- Trace Streams - {{ kafkaTraceConnectedCount }}/3 open · {{ kafkaTraceEventCount }} events -
-
-
-
-
- {{ streamEntry.label }} - {{ streamEntry.topic }} -
- -
-
等待匹配 traceId 的 {{ streamEntry.topic }} 事件。
-
    -
  1. -
    - {{ row.eventName }} - {{ kafkaEventSummary(row.payload) }} - #{{ row.payload.offset || '-' }} -
    -
    - {{ kafkaEventIds(row.payload) }} - -
    - 
    {{ kafkaEventJson(row.payload) }}
    -
    2. -
-
-
-
- -
-
- Raw SSE Events - {{ kafkaEvents.length }} events -
-
等待后端从 Kafka 订阅并透传 hwlab event。
-
    -
  1. -
    - {{ row.eventName }} - {{ kafkaEventSummary(row.payload) }} - {{ row.payload.topic || '-' }} #{{ row.payload.offset || '-' }} -
    -
    - {{ kafkaEventIds(row.payload) }} - -
    - 
    {{ kafkaEventJson(row.payload) }}
    -
    2. -
-
-
@@ -501,27 +247,6 @@ function defaultAppendDraft(): string { font-weight: 750; } -.workbench-debug-tabs { - display: flex; - gap: 8px; - border-bottom: 1px solid #d8e1eb; -} - -.debug-tab { - border: 0; - border-bottom: 2px solid transparent; - background: transparent; - padding: 10px 12px; - color: #475569; - font-size: 13px; - font-weight: 800; -} - -.debug-tab[data-active="true"] { - border-bottom-color: #0e7490; - color: #0f172a; -} - .workbench-debug-toolbar { flex-wrap: wrap; gap: 10px; @@ -536,24 +261,6 @@ function defaultAppendDraft(): string { font-weight: 750; } -.workbench-debug-toolbar .compact-label { - width: min(220px, 100%); -} - -.kafka-toolbar { - align-items: end; -} - -.debug-check { - display: flex !important; - width: auto !important; - grid-auto-flow: column; - align-items: center; - gap: 7px !important; - padding-bottom: 8px; - color: #334155 !important; -} - .workbench-debug-grid { display: grid; min-height: 0; @@ -676,164 +383,11 @@ function defaultAppendDraft(): string { white-space: nowrap; } -.kafka-debug-layout { - display: grid; - min-height: 0; - grid-template-rows: minmax(280px, 0.9fr) minmax(220px, 0.7fr); - gap: 12px; -} - -.kafka-trace-panel { - display: grid; - min-height: 0; - grid-template-rows: auto minmax(0, 1fr); - gap: 10px; -} - -.kafka-stream-grid { - display: grid; - min-height: 0; - grid-template-columns: repeat(3, minmax(0, 1fr)); - gap: 10px; -} - -.kafka-stream-column { - display: grid; - min-width: 0; - min-height: 0; - grid-template-rows: auto minmax(0, 1fr); - gap: 8px; - border: 1px solid #d8e1eb; - border-radius: 8px; - background: white; - padding: 10px; -} - -.kafka-stream-head { - display: flex; - align-items: start; - justify-content: space-between; - gap: 10px; -} - -.kafka-stream-head div { - display: grid; - min-width: 0; - gap: 3px; -} - -.kafka-stream-head strong { - color: #0f172a; - font-size: 13px; -} - -.kafka-stream-head code { - overflow: hidden; - color: #0f766e; - font-size: 11px; - text-overflow: ellipsis; - white-space: nowrap; -} - -.kafka-debug-panel { - display: grid; - min-height: 0; - grid-template-rows: auto minmax(0, 1fr); - gap: 10px; - border: 1px solid #d8e1eb; - border-radius: 8px; - background: white; - padding: 10px; -} - -.kafka-event-list { - display: grid; - align-content: start; - gap: 8px; - min-height: 0; - margin: 0; - overflow: auto; - padding: 0; - list-style: none; -} - -.kafka-event-list li { - display: grid; - gap: 6px; - border: 1px solid #d8e1eb; - border-left: 3px solid #0e7490; - border-radius: 8px; - background: #f8fafc; - padding: 9px; -} - -.kafka-event-head, -.kafka-event-meta { - display: grid; - grid-template-columns: minmax(150px, 0.5fr) minmax(160px, 0.7fr) minmax(0, 1fr); - gap: 10px; - align-items: center; - color: #334155; - font-size: 12px; -} - -.kafka-event-head code { - color: #0f766e; -} - -.kafka-event-head span, -.kafka-event-meta { - color: #64748b; -} - -.kafka-event-meta { - grid-template-columns: minmax(0, 1fr) auto; -} - -.kafka-event-list pre { - max-height: 220px; - margin: 0; - overflow: auto; - border-radius: 7px; - background: #0f172a; - padding: 9px; - color: #d1fae5; - font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; - font-size: 11px; - line-height: 1.45; - white-space: pre-wrap; - overflow-wrap: anywhere; -} - -.compact-kafka-event-list .kafka-event-head { - grid-template-columns: minmax(92px, 0.6fr) minmax(110px, 0.8fr) minmax(44px, 0.4fr); -} - -.compact-kafka-event-list .kafka-event-meta { - grid-template-columns: minmax(0, 1fr); -} - -.compact-kafka-event-list .kafka-event-meta time { - display: none; -} - -.compact-kafka-event-list pre { - max-height: 160px; -} - @media (max-width: 980px) { .workbench-debug-grid { grid-template-columns: minmax(0, 1fr); } - .kafka-debug-layout { - grid-template-rows: auto auto; - } - - .kafka-stream-grid { - grid-template-columns: minmax(0, 1fr); - } - .debug-side { grid-template-rows: 280px 220px; }