From 2cb3d3aaa62ab886c4b0cfae74c4c8dee0bd1466 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 11 Jul 2026 05:15:35 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=AE=9E=E7=8E=B0=E7=BA=AF=20Kafka=20?= =?UTF-8?q?=E9=A1=B5=E9=9D=A2=E5=88=B7=E6=96=B0=E9=87=8D=E6=94=BE=E4=BA=A4?= =?UTF-8?q?=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../R2.4_Task_Report.md | 27 + .../R3.1_Task_Report.md | 77 +++ .../R3.2_Task_Report.md | 57 +++ .../R3.3_Task_Report.md | 86 ++++ .../pure-kafka-live-single-step-debug.md | 12 +- internal/cloud/code-agent-agentrun-adapter.ts | 12 +- internal/cloud/code-agent-agentrun-result.ts | 7 + internal/cloud/kafka-event-bridge.test.ts | 72 +++ internal/cloud/kafka-event-bridge.ts | 163 +++++- internal/cloud/server-agent-chat.test.ts | 35 +- .../cloud/server-code-agent-admission-http.ts | 6 +- .../cloud/server-code-agent-trace-http.ts | 7 +- .../server-workbench-realtime-http.test.ts | 247 +++++++++- .../cloud/server-workbench-realtime-http.ts | 192 +++++++- .../workbench-kafka-refresh-handoff.test.ts | 336 +++++++++++++ .../cloud/workbench-kafka-refresh-handoff.ts | 462 ++++++++++++++++++ ...kbench-realtime-authority-contract.test.ts | 2 + .../cloud/workbench-realtime-capabilities.ts | 1 + internal/dev-entrypoint/cloud-web-runtime.mjs | 1 + .../dev-entrypoint/cloud-web-runtime.test.mjs | 5 + ...time-realtime-capabilities-config.test.mjs | 13 + tools/hwlab-cli/kafka-regenerate.test.ts | 259 +++++++++- tools/src/hwlab-cli/kafka-regenerate.ts | 257 +++++++++- .../scripts/workbench-e2e-server.ts | 2 +- .../workbench-realtime-runtime.test.ts | 18 +- .../src/api/workbench-events.test.ts | 2 +- .../src/api/workbench-events.ts | 1 + web/hwlab-cloud-web/src/config/runtime.ts | 5 +- .../workbench-kafka-refresh-policy.test.ts | 27 + .../stores/workbench-kafka-refresh-policy.ts | 22 + .../stores/workbench-live-kafka-event.test.ts | 127 ++++- .../src/stores/workbench-live-kafka-event.ts | 49 ++ web/hwlab-cloud-web/src/stores/workbench.ts | 54 +- web/hwlab-cloud-web/src/types/global.d.ts | 1 + 34 files changed, 2547 insertions(+), 97 deletions(-) create mode 100644 docs/MDTODO/details/pure-kafka-live-single-step-debug/R2.4_Task_Report.md create mode 100644 docs/MDTODO/details/pure-kafka-live-single-step-debug/R3.1_Task_Report.md create mode 100644 docs/MDTODO/details/pure-kafka-live-single-step-debug/R3.2_Task_Report.md create mode 100644 docs/MDTODO/details/pure-kafka-live-single-step-debug/R3.3_Task_Report.md create mode 100644 internal/cloud/workbench-kafka-refresh-handoff.test.ts create mode 100644 internal/cloud/workbench-kafka-refresh-handoff.ts create mode 100644 web/hwlab-cloud-web/src/stores/workbench-kafka-refresh-policy.test.ts create mode 100644 web/hwlab-cloud-web/src/stores/workbench-kafka-refresh-policy.ts diff --git a/docs/MDTODO/details/pure-kafka-live-single-step-debug/R2.4_Task_Report.md b/docs/MDTODO/details/pure-kafka-live-single-step-debug/R2.4_Task_Report.md new file mode 100644 index 00000000..2fe1aa6d --- /dev/null +++ b/docs/MDTODO/details/pure-kafka-live-single-step-debug/R2.4_Task_Report.md @@ -0,0 +1,27 @@ +# R2.4 纯 Kafka Trace 可读性闭环报告 + +- 上下文： + - [HWLAB #2474](https://github.com/pikasTech/HWLAB/issues/2474) + - [HWLAB PR #2484](https://github.com/pikasTech/HWLAB/pull/2484) +- 合入提交：`55f85a164d0e219b648c3639f6a1d3fa1463bf49` +- 固定 trace：`trc_fb6031c28d79495e` + +## 完成结果 + +- CLI 直接读取持久 `hwlab.event.v1`，并复用 Web 的 decoder、reducer、row model 和 Markdown renderer。 +- Kafka 扫描只有抵达 captured end-offset barrier 才声明完整；timeout、limit 或缺失 barrier 均返回 typed partial。 +- Final Response 只在 Trace 外层显示一次，不再作为内部助手行重复渲染。 +- tool-call summary 显示命令、输出摘要、耗时和 exit code，不再只有 `commandExecution` 名称。 +- 每个正式 HWLAB SSE ingress 都刷新 `lastEventAt`，terminal 后到达的新事件仍更新时间。 +- 产品链继续保持 projection 关闭，不引入 snapshot、sync、gap-fill、finalizer 或 HTTP 历史拼接。 + +## 验证证据 + +- PR #2484 已合入 `v0.3`，提交 `55f85a16` 属于 `origin/v0.3`。 +- 固定 trace 的后续 retention 重放再次确认： + - `finalResponsePresent=true`； + - `finalResponseInTrace=false`； + - tool rows 为 `111`； + - assistant rows 为 `80`； + - terminal 状态为 `completed`。 +- 页面刷新 retention bootstrap 属于 R3，不把 R2.4 的既有可读性修复改造成第二事实源。 diff --git a/docs/MDTODO/details/pure-kafka-live-single-step-debug/R3.1_Task_Report.md b/docs/MDTODO/details/pure-kafka-live-single-step-debug/R3.1_Task_Report.md new file mode 100644 index 00000000..1700a4db --- /dev/null +++ b/docs/MDTODO/details/pure-kafka-live-single-step-debug/R3.1_Task_Report.md @@ -0,0 +1,77 @@ +# R3.1 纯 Kafka 页面刷新重放状态机调查报告 + +- 上下文： + - [HWLAB #2486](https://github.com/pikasTech/HWLAB/issues/2486) + - [AgentRun #283](https://github.com/pikasTech/agentrun/issues/283) +- 固定离线 trace：`trc_fb6031c28d79495e` +- 目标运行面：NC01/v03 + AgentRun NC01/v0.2 + +## 结论 + +- 当前产品实时链没有偏离纯 Kafka 目标： + - `agentrun.event.v1` 由 HWLAB direct mapper 一对一发布到 `hwlab.event.v1`； + - 产品 SSE 订阅 shared Kafka fanout； + - Web 已让 `hwlab.event.v1` 进入统一 decode、reducer 和卡片投影。 +- 页面刷新缺口位于 transport bootstrap： + - 当前 SSE 只发送连接建立后的 future event； + - 浏览器内存清空后没有从 Kafka retention 恢复历史事件。 +- 用户消息还存在一个上游正式事件缺口： + - AgentRun `EventType` 当前没有 `user_message`； + - command 创建只发布 `backend_status(command-created)`； + - HWLAB mapper 和 Web live projector 也没有 user event 语义； + - 因此历史 trace 不得伪造用户消息，根因由 AgentRun #283 沿同一 Kafka 链修复。 + +## 已确认的可复用基础 + +- `startLiveHwlabKafkaEventBridge()` 已维护单个 shared `hwlab.event.v1` consumer，并向浏览器 listener fanout；浏览器连接不会创建长期 consumer group。 +- `kafkaMessageKey()` 优先使用 `sessionId`，因此同一 HWLAB session 的正式事件使用同一 Kafka key；bootstrap 仍须从实际 transport metadata 证明命中事件只在一个 partition，不能只信任代码假设。 +- `queryKafkaEventStream()` 已具备： + - Kafka admin end-offset snapshot； + - 从 retention 起点读取； + - 按 trace/session 过滤； + - transport `topic/partition/offset` 证据； + - 到达 barrier 后有界退出。 +- 当前查询缺少独立 scanned-record 上限，且默认 limit/timeout 不能作为运行时隐藏策略；refresh 调用必须从 owning YAML 显式传入 group prefix、timeout、scan、event 和 live buffer 预算。 +- SSE scope authorization 已先通过 server-side session ownership 校验；数据库仅用于授权，不参与历史状态重建。 + +## 冻结状态机 + +1. `authorize` + - 解析并校验请求的 session/trace； + - trace-only 请求解析到授权 session； + - 浏览器不能请求任意外部 trace。 +2. `buffer-live` + - 在抓 barrier 前注册 shared live fanout listener； + - 只缓冲当前授权 scope 的 envelope； + - buffer 超过 YAML 预算立即记录 typed failure。 +3. `capture-and-replay` + - 抓取 `hwlab.event.v1` 每个 partition 的 end-offset barrier； + - 从 Kafka retention 起点读取到该 barrier； + - scan、matched event 和 timeout 均使用 YAML 预算； + - 要求扫描完整、至少命中一条、命中事件只属于一个 partition，且每条都有稳定 transport identity 和 event identity。 +4. `handoff` + - replay 按目标 partition offset 递增发送到原有 `hwlab.event.v1` SSE event name； + - replay payload 保持原始 HWLAB envelope，不生成 snapshot 或第二种业务事件； + - flush live buffer 时，barrier 内记录必须已由 replay identity 证明并去重；barrier 后记录按 offset 继续发送； + - flush 完成后切换到 shared live direct delivery，仍保留 barrier/identity 去重以处理 consumer lag。 +5. `fail-closed` + - barrier 缺失、扫描未完成、预算耗尽、多 partition、identity 缺失、buffer 溢出或 barrier overlap 无对应 replay 记录时，通过 `workbench.error` 返回 phase/code/evidence 后关闭 SSE； + - 禁止回退 projector、read model、snapshot、sync、gap-fill、finalizer、polling 或 HTTP 历史。 + +## 可组合能力与配置合同 + +- `directPublish`、`liveKafkaSse`、`kafkaRefreshReplay`、`transactionalProjector`、`projectionOutboxRelay`、`projectionRealtime` 均为独立 boolean capability，不建立二选一 mode。 +- `kafkaRefreshReplay=false` 时保留现有纯 live 行为。 +- `kafkaRefreshReplay=true` 时显式要求 owning YAML 提供： + - bootstrap consumer group prefix； + - timeout； + - scanned-record limit； + - matched-event limit； + - live buffer limit。 +- 代码只校验这些字段为所需类型并执行，不增加运行时数值默认。 + +## 验收边界 + +- 固定历史 trace 用于验证 retained replay、terminal、Final Response 唯一性、tool summary、timing 和 projection=0。 +- 固定历史 trace 生成于 `user_message` 合同之前，不能伪造用户输入；用户消息先用 AgentRun #283 的正式 event fixture 验证一对一 lineage，最终部署后再由 semantic internal web-probe 的真实刷新验收确认。 +- CLI 必须直接调用生产 bootstrap、decoder、reducer 和 Markdown renderer；若无法表达 barrier/handoff/user event 证据，先改进 CLI 再继续。 diff --git a/docs/MDTODO/details/pure-kafka-live-single-step-debug/R3.2_Task_Report.md b/docs/MDTODO/details/pure-kafka-live-single-step-debug/R3.2_Task_Report.md new file mode 100644 index 00000000..b832a47c --- /dev/null +++ b/docs/MDTODO/details/pure-kafka-live-single-step-debug/R3.2_Task_Report.md @@ -0,0 +1,57 @@ +# R3.2 纯 Kafka retention-to-live 实现报告 + +- 上下文： + - [HWLAB #2486](https://github.com/pikasTech/HWLAB/issues/2486) + - [HWLAB #2487](https://github.com/pikasTech/HWLAB/issues/2487) + - [AgentRun #283](https://github.com/pikasTech/agentrun/issues/283) + - [UniDesk PR #1738](https://github.com/pikasTech/unidesk/pull/1738) + +## 完成结果 + +- 用户输入沿唯一权威链成为正式事件： + - AgentRun `user_message`； + - HWLAB direct mapper 一对一映射到正式 `hwlab.event.v1` user event； + - Web reducer 按稳定 `userMessageId` upsert，HTTP-first 与 Kafka-first 均只保留一个用户气泡。 +- 页面刷新使用 retention bootstrap 与 shared live fanout 无缝交接： + - 先订阅 shared live 并有界缓冲； + - 再捕获 topic end-offset barrier； + - 从 Kafka retention 起点完整扫描到 barrier； + - 原样发送历史 `hwlab.event.v1`； + - 按 transport identity、`eventId` 和 `sourceEventId` 去重后切到同一 live writer。 +- fail-closed 覆盖： + - retention 起点移动或不可验证； + - timeout、scan limit、matched limit； + - scope 跨 partition； + - transport 或稳定事件身份冲突； + - live buffer overflow； + - pre-barrier gap； + - SSE write 或客户端断开。 +- `workbench.error` 只披露有界安全诊断： + - completion reason； + - timeout、scan 和 matched limit； + - scanned、matched count； + - partition； + - 不透传未列入白名单的 payload。 +- capability、group prefix、timeout、scan limit、matched-event limit 和 live-buffer limit 均独立且 YAML-first。 +- UniDesk PR #1738 已先合入并 apply 到 NC01/v03 控制面；旧 runtime 未被触发，满足先配置后代码的无窗口顺序。 + +## 架构边界 + +- 数据链保持： + - `codex-stdio.raw.v1`； + - `agentrun.event.v1`； + - HWLAB direct mapper； + - `hwlab.event.v1`； + - Kafka retention bootstrap； + - shared live fanout； + - SSE； + - Web reducer。 +- 未引入 projector、read model、snapshot、sync、gap-fill、finalizer、polling 或 HTTP 历史 fallback。 +- `kafkaRefreshReplay=false` 保留既有 live-only 行为；各 capability 可组合、可独立开关。 + +## 测试 + +- cloud、handoff、CLI 与 AgentRun adapter：`98/98`。 +- Web reducer、runtime policy 与 capability config：`51/51`。 +- 本次变更相关 cloud-web runtime config：`4/4`。 +- 合计：`153/153`。 diff --git a/docs/MDTODO/details/pure-kafka-live-single-step-debug/R3.3_Task_Report.md b/docs/MDTODO/details/pure-kafka-live-single-step-debug/R3.3_Task_Report.md new file mode 100644 index 00000000..c2e0dcd6 --- /dev/null +++ b/docs/MDTODO/details/pure-kafka-live-single-step-debug/R3.3_Task_Report.md @@ -0,0 +1,86 @@ +# R3.3 固定 Trace 生产 handoff 单步重放报告 + +- 上下文： + - [HWLAB #2486](https://github.com/pikasTech/HWLAB/issues/2486) + - [HWLAB #2487](https://github.com/pikasTech/HWLAB/issues/2487) + - [AgentRun #283](https://github.com/pikasTech/agentrun/issues/283) +- 固定 trace：`trc_fb6031c28d79495e` +- 未启动新的真实 AgentRun 运行。 + +## 执行命令 + +```bash +bun tools/hwlab-cli/bin/hwlab-cli.ts kafka render trace \ + --from kafka \ + --trace-id trc_fb6031c28d79495e \ + --input-topic hwlab.event.v1 \ + --group-prefix hwlab-v03-workbench-isolated-debug-refresh-2487-final \ + --limit 500 \ + --timeout-ms 30000 \ + --format markdown \ + --output-markdown /tmp/hwlab-refresh-2487-trc_fb6031c28d79495e-final.md \ + --row-limit 1 \ + --json +``` + +## 生产路径证据 + +- 临时只读源码副本：`/tmp/hwlab-refresh-2487.ACKmyy`。 +- CLI 明确调用生产模块： + - `internal/cloud/workbench-kafka-refresh-handoff:createWorkbenchKafkaRefreshHandoff`。 +- 后续路径与 Web 同构： + - `decodeWorkbenchRealtimeEventFrame`； + - `reduceWorkbenchRealtimeEvent`； + - `planWorkbenchRealtimeApply`； + - `workbenchLiveKafkaProjectionTarget`； + - `projectWorkbenchLiveKafkaUserMessage|projectWorkbenchLiveKafkaMessage`； + - `traceDisplayRows`； + - Markdown 只在最后一步与 HTML 分叉。 +- `liveSubscriptionInstalled=true`，证明 CLI 使用完整生产 handoff 合同，不是直接把查询结果塞给 renderer。 + +## Kafka 与 handoff 结果 + +- 扫描： + - `scanned=2727`； + - `parsed=2727`； + - `invalidJson=0`； + - `completionReason=end-offset`。 +- barrier： + - topic 为 `hwlab.event.v1`； + - partition 为 `0`； + - start offset 为 `0`； + - exclusive end offset 为 `2727`； + - last scanned offset 为 `2726`。 +- trace 命中与应用： + - matched 为 `381`； + - replayed 为 `381`； + - decoded、planned、applied 均为 `381`； + - rejected 为 `0`； + - stable identity missing 与 duplicate 均为 `0`。 +- 终态和可读性： + - terminal 为 `completed`； + - rendered rows 为 `194`； + - tool rows 为 `111`； + - assistant rows 为 `80`； + - `finalResponsePresent=true`； + - `finalResponseInTrace=false`。 + +## Markdown artifact + +- 运行面路径：`/tmp/hwlab-refresh-2487-trc_fb6031c28d79495e-final.md`。 +- 行数：`4181`。 +- 字节数：`225131`。 +- SHA-256：`39f9fe38c05c8ece6a056dcac405fb29fd7b2d81954a5b3b90a47c6ed52c26e7`。 + +## 用户正式事件补充验收 + +- 该固定 trace 生成于 `user_message` 合同上线之前，因此历史记录中的 `userMessageCount=0`；未伪造用户输入。 +- 正式 user event fixture 验证： + - AgentRun `user_message` 一对一映射为稳定 HWLAB user event； + - retention SSE 在 lifecycle 事件前重放用户事件； + - Web reducer 同时覆盖 Kafka-first 和 HTTP-first，稳定为单个用户气泡； + - session replay 保留 steer 用户输入与 target trace 的独立身份； + - 同构 Markdown 的“用户输入”区域只渲染一次。 +- CLI 另有定向合同： + - incomplete scan 即使零匹配也返回 `partial/source_scan_incomplete`； + - 只有完整 barrier 后零匹配才返回 `source_trace_missing`。 diff --git a/docs/MDTODO/pure-kafka-live-single-step-debug.md b/docs/MDTODO/pure-kafka-live-single-step-debug.md index 3e10b748..24bb96e1 100644 --- a/docs/MDTODO/pure-kafka-live-single-step-debug.md +++ b/docs/MDTODO/pure-kafka-live-single-step-debug.md @@ -28,21 +28,21 @@ ### R2.3 [completed] 修复隔离 debug replay 的 producer/barrier 与分层结果合同：保留 source lineage、replayId、offset range，报告 scanned/matched/delivered/decoded/applied/terminal，区分 producer-not-invoked、trace missing、filter mismatch、decoder/reducer rejection 与 terminal missing；仅写 debug topic。上下文：[隔离根因 #2476](https://github.com/pikasTech/HWLAB/issues/2476#issuecomment-4935398125)、[主方案 #2474](https://github.com/pikasTech/HWLAB/issues/2474#issuecomment-4935408473)。依赖：无。验证：empty、matched-no-terminal、reducer-rejected、terminal-complete 四类 typed 结果及 semantic internal web-probe command，完成任务后将详细报告写入[任务报告](./details/pure-kafka-live-single-step-debug/R2.3_Task_Report.md)。 -### R2.4 [in_progress] +### R2.4 [completed] 以 [HWLAB #2474](https://github.com/pikasTech/HWLAB/issues/2474) 和固定 trace trc_fb6031c28d79495e 完成纯 Kafka 集成闭环：先增强 CLI 直接读取持久 hwlab.event 并复用前端 decode、merge、row model、summary 输出同构 Markdown，再修复 Trace 内 final response 重复、tool-call summary 可见性和每个 HWLAB SSE ingress 刷新 lastEventAt；最后受控部署 NC01/v03，以 Kafka 重放、semantic internal web-probe、DOM 与 OTel 验收 projection=0 且无补洞路径，完成任务后将详细报告写入[任务报告](./details/pure-kafka-live-single-step-debug/R2.4_Task_Report.md)。 ## R3 [in_progress] 在纯 Kafka SSE 权威链内实现页面刷新后的可重放引导，明确区别于 projector/read model/snapshot/sync/gap-fill/finalizer 补洞；上下文：[HWLAB #2486](https://github.com/pikasTech/HWLAB/issues/2486)，完成任务后将详细报告写入[任务报告](./details/pure-kafka-live-single-step-debug/R3_Task_Report.md)。 -### R3.1 [in_progress] +### R3.1 [completed] 依据 [HWLAB #2486](https://github.com/pikasTech/HWLAB/issues/2486) 深入分析 Kafka retention、SSE 刷新连接、offset barrier、shared live handoff 与去重身份，形成不引入第二事实源的状态机和失败可见性，完成任务后将详细报告写入[任务报告](./details/pure-kafka-live-single-step-debug/R3.1_Task_Report.md)。 -### R3.2 +### R3.2 [completed] -依据 [HWLAB #2486](https://github.com/pikasTech/HWLAB/issues/2486) 实现 trace/session-scoped Kafka refresh bootstrap 与 shared live fanout 无缝交接，全部开关、group、容量和时限保持 YAML-first，完成任务后将详细报告写入[任务报告](./details/pure-kafka-live-single-step-debug/R3.2_Task_Report.md)。 -### R3.3 +依据 [HWLAB #2486](https://github.com/pikasTech/HWLAB/issues/2486) 与 [AgentRun #283](https://github.com/pikasTech/agentrun/issues/283)，先让用户输入成为 agentrun.event.v1 与 hwlab.event.v1 正式事件，再实现 trace/session-scoped Kafka refresh bootstrap 和 shared live fanout 无缝交接；全部开关、group、容量和时限保持 YAML-first，完成任务后将详细报告写入[任务报告](./details/pure-kafka-live-single-step-debug/R3.2_Task_Report.md)。 +### R3.3 [completed] -使用 [HWLAB #2486](https://github.com/pikasTech/HWLAB/issues/2486) 的固定 trace trc_fb6031c28d79495e 通过 CLI 单步重放验证顺序、barrier、handoff、terminal、final 唯一性、tool summary 和 timing，不启动新真实运行，完成任务后将详细报告写入[任务报告](./details/pure-kafka-live-single-step-debug/R3.3_Task_Report.md)。 +依据 [HWLAB #2486](https://github.com/pikasTech/HWLAB/issues/2486) 与 [AgentRun #283](https://github.com/pikasTech/agentrun/issues/283)，使用固定 trace trc_fb6031c28d79495e 做 CLI 单步重放，并用正式 user_message fixture 补充历史 trace 不具备的用户输入验收；验证顺序、barrier、handoff、terminal、final 唯一性、tool summary 和 timing，不启动新真实运行，完成任务后将详细报告写入[任务报告](./details/pure-kafka-live-single-step-debug/R3.3_Task_Report.md)。 ### R3.4 完成 [HWLAB #2486](https://github.com/pikasTech/HWLAB/issues/2486) 的定向测试、受控 CI/CD 与 semantic internal web-probe 页面刷新验收，确认 projection=0 且无 snapshot/sync/gap-fill/finalizer 路径，完成任务后将详细报告写入[任务报告](./details/pure-kafka-live-single-step-debug/R3.4_Task_Report.md)。 diff --git a/internal/cloud/code-agent-agentrun-adapter.ts b/internal/cloud/code-agent-agentrun-adapter.ts index 268c495d..b7067230 100644 --- a/internal/cloud/code-agent-agentrun-adapter.ts +++ b/internal/cloud/code-agent-agentrun-adapter.ts @@ -1086,6 +1086,7 @@ function buildAgentRunCommandInput({ params, traceId, backendProfile, sessionId, ...agentRunPromptEventFields(promptEvidence), ...(promptEvidence?.prompt ? { promptMetadata: promptEvidence.prompt } : {}), traceId, + userMessageId: agentRunUserMessageId(params, traceId), projectId: firstNonEmpty(params.projectId) || null, conversationId: safeConversationId(params.conversationId) || null, sessionId: sessionId ?? scopedAgentRunSessionIdForParams(params, traceId), @@ -1114,6 +1115,7 @@ function buildAgentRunSteerCommandInput({ params, traceId, steerTraceId, mapped ...agentRunPromptEventFields(promptEvidence), ...(promptEvidence?.prompt ? { promptMetadata: promptEvidence.prompt } : {}), traceId: steerTraceId, + userMessageId: agentRunUserMessageId(params, steerTraceId), targetTraceId: traceId, targetCommandId: mapped.agentRun.commandId, conversationId: safeConversationId(mapped.conversationId ?? params.conversationId) || null, @@ -1138,6 +1140,14 @@ function buildAgentRunDurableDispatchInput({ env, traceId, backendProfile }) { }; } +function agentRunUserMessageId(params, traceId) { + const explicit = firstNonEmpty(params?.userMessageId, params?.messageId); + if (/^msg_[A-Za-z0-9_.:-]+$/u.test(explicit ?? "")) return explicit; + const suffix = safeTraceId(traceId)?.replace(/^trc_/u, "").replace(/[^A-Za-z0-9_.:-]/gu, "_").slice(0, 48); + if (!suffix) throw adapterError("agentrun_user_message_id_missing", "AgentRun command requires a deterministic HWLAB userMessageId"); + return `msg_${suffix}_user`; +} + function assertAgentRunDurableTransientEnv(entries) { const seen = new Set(); for (const entry of Array.isArray(entries) ? entries : []) { @@ -1493,7 +1503,7 @@ function isReusableAgentRunMapping(mapping, backendProfile) { mapping.adapter === ADAPTER_ID && mapping.reuseEligible !== false && mapping.runId && - mapping.jobName && + (mapping.durableDispatch === true || mapping.jobName) && (!mapping.backendProfile || mapping.backendProfile === backendProfile) ); } diff --git a/internal/cloud/code-agent-agentrun-result.ts b/internal/cloud/code-agent-agentrun-result.ts index f4924d13..9de5580e 100644 --- a/internal/cloud/code-agent-agentrun-result.ts +++ b/internal/cloud/code-agent-agentrun-result.ts @@ -96,6 +96,7 @@ export function agentRunSessionEvidence(payload = {}) { commandId: payload.agentRun.commandId ?? null, attemptId: payload.agentRun.attemptId ?? null, runnerId: payload.agentRun.runnerId ?? null, + runnerJobId: payload.agentRun.runnerJobId ?? null, jobName: payload.agentRun.jobName ?? null, namespace: payload.agentRun.namespace ?? null, backendProfile: payload.agentRun.backendProfile ?? null, @@ -105,10 +106,16 @@ export function agentRunSessionEvidence(payload = {}) { runStatus: payload.agentRun.runStatus ?? null, commandState: payload.agentRun.commandState ?? null, terminalStatus: payload.agentRun.terminalStatus ?? null, + dispatchIntentId: payload.agentRun.dispatchIntentId ?? null, + dispatchIntentState: payload.agentRun.dispatchIntentState ?? null, + durableDispatch: payload.agentRun.durableDispatch === true, traceId: payload.agentRun.traceId ?? payload.traceId ?? payload.providerTrace?.traceId ?? null, lastSeq: payload.agentRun.lastSeq ?? 0, providerId: payload.agentRun.providerId ?? agentRunProviderIdOrNull(), reuseEligible: payload.agentRun.reuseEligible ?? false, + runnerJobCount: Number(payload.agentRun.runnerJobCount ?? 0), + runnerReused: payload.agentRun.runnerReused === true, + reused: payload.agentRun.reused === true, sessionId: payload.agentRun.sessionId ?? payload.sessionId ?? null, conversationId: payload.conversationId ?? payload.agentRun.conversationId ?? null, threadId: payload.threadId ?? payload.agentRun.threadId ?? null, diff --git a/internal/cloud/kafka-event-bridge.test.ts b/internal/cloud/kafka-event-bridge.test.ts index dab70c94..8f2f66cf 100644 --- a/internal/cloud/kafka-event-bridge.test.ts +++ b/internal/cloud/kafka-event-bridge.test.ts @@ -11,6 +11,7 @@ import { decodeCanonicalAgentRunKafkaMessage, kafkaEventBridgeConfig, liveKafkaO const PROJECTOR_ENV = Object.freeze({ HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false", HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false", HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "true", HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "true", HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true", @@ -33,6 +34,7 @@ const PROJECTOR_ENV = Object.freeze({ const LIVE_ENV = Object.freeze({ HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false", HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false", @@ -101,6 +103,47 @@ test("projects AgentRun assistant_message Kafka event into HWLAB trace event", ( assert.equal(projected.valuesPrinted, false); }); +test("projects AgentRun user_message into a stable HWLAB user event", () => { + const projected = projectAgentRunKafkaEventToHwlabEvent({ + schema: "agentrun.event.v1", + eventType: "agentrun.event.committed", + eventId: "evt_user_message", + traceId: "trc_user_message", + hwlabSessionId: "ses_user_message", + run: { runId: "run_user_message", sessionId: "ses_agentrun_user_message" }, + command: { commandId: "cmd_user_message" }, + event: { + id: "evt_user_message", + seq: 2, + type: "user_message", + createdAt: "2026-07-10T10:00:00.000Z", + payload: { + commandId: "cmd_user_message", + traceId: "trc_user_message", + targetTraceId: "trc_user_message_target", + userMessageId: "msg_user_message_user", + text: "replay this input", + source: "agentrun-turn-command-payload", + valuesPrinted: false + } + } + }, { source: "hwlab-test", sourceTopic: "agentrun.event.v1", sourcePartition: 1, sourceOffset: "7" }); + + assert.equal(projected.eventId, "hwlab:evt_user_message"); + assert.equal(projected.traceId, "trc_user_message"); + assert.equal(projected.sessionId, "ses_user_message"); + assert.equal(projected.event.type, "user"); + assert.equal(projected.event.eventType, "user"); + assert.equal(projected.event.role, "user"); + assert.equal(projected.event.userMessageId, "msg_user_message_user"); + assert.equal(projected.event.messageId, "msg_user_message_user"); + assert.equal(projected.event.targetTraceId, "trc_user_message_target"); + assert.equal(projected.event.text, "replay this input"); + assert.equal(projected.event.sourceEventId, "evt_user_message"); + assert.equal(projected.sourceEvent.partition, 1); + assert.equal(projected.sourceEvent.offset, "7"); +}); + test("projects final AgentRun assistant text without sealing before terminal_status", () => { const projected = projectAgentRunKafkaEventToHwlabEvent({ schema: "agentrun.event.v1", @@ -355,6 +398,34 @@ test("composable Kafka capabilities reject consumer group collisions", () => { ); }); +test("Kafka refresh replay requires explicit YAML budgets and composes with live SSE", () => { + const refreshEnv = { + ...LIVE_ENV, + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "true", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_GROUP_PREFIX: "hwlab-v03-cloud-api-sse", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS: "30000", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT: "1000000", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_MATCHED_EVENT_LIMIT: "2000", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_LIVE_BUFFER_LIMIT: "2000" + }; + const config = kafkaEventBridgeConfig(refreshEnv); + assert.deepEqual(config.refreshReplay, { + groupIdPrefix: "hwlab-v03-cloud-api-sse", + timeoutMs: 30000, + scanLimit: 1000000, + matchedEventLimit: 2000, + liveBufferLimit: 2000 + }); + assert.throws( + () => kafkaEventBridgeConfig({ ...refreshEnv, HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT: undefined }), + (error: any) => error?.code === "hwlab_kafka_realtime_config_incomplete" && error?.missing?.includes("HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT") + ); + assert.throws( + () => kafkaEventBridgeConfig({ ...refreshEnv, HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false" }), + (error: any) => error?.code === "hwlab_kafka_refresh_live_dependency_missing" + ); +}); + test("projection realtime subscribes to projection commits without enabling Kafka ingest or relay", async () => { let subscribeCount = 0; let unsubscribeCount = 0; @@ -362,6 +433,7 @@ test("projection realtime subscribes to projection commits without enabling Kafk env: { HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false", HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false", HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true" diff --git a/internal/cloud/kafka-event-bridge.ts b/internal/cloud/kafka-event-bridge.ts index 0db463ba..acfdcafe 100644 --- a/internal/cloud/kafka-event-bridge.ts +++ b/internal/cloud/kafka-event-bridge.ts @@ -35,13 +35,23 @@ export function kafkaEventBridgeConfig(env = process.env) { const legacyKafkaEnabled = truthy(env.HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED ?? env.HWLAB_KAFKA_AGENTRUN_CONSUME_ENABLED ?? env.HWLAB_KAFKA_ENABLED); if (!featureEnvPresent && !legacyKafkaEnabled) return null; const capabilities = workbenchRealtimeCapabilities(env); - const kafkaEnabled = capabilities.directPublish || capabilities.liveKafkaSse || capabilities.transactionalProjector || capabilities.projectionOutboxRelay; + if (capabilities.kafkaRefreshReplay && !capabilities.liveKafkaSse) { + throw contractError("hwlab_kafka_refresh_live_dependency_missing", "Kafka refresh replay requires the independently enabled live Kafka SSE capability for handoff."); + } + const kafkaEnabled = capabilities.directPublish || capabilities.liveKafkaSse || capabilities.kafkaRefreshReplay || capabilities.transactionalProjector || capabilities.projectionOutboxRelay; const transactionalEnabled = capabilities.transactionalProjector || capabilities.projectionOutboxRelay || capabilities.projectionRealtime; if (!kafkaEnabled && !transactionalEnabled) return null; const required = [...REQUIRED_KAFKA_ENV]; if (!kafkaEnabled) required.length = 0; if (capabilities.directPublish) required.push("HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID"); if (capabilities.liveKafkaSse) required.push("HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID"); + if (capabilities.kafkaRefreshReplay) required.push( + "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_GROUP_PREFIX", + "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS", + "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT", + "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_MATCHED_EVENT_LIMIT", + "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_LIVE_BUFFER_LIMIT" + ); if (capabilities.transactionalProjector) required.push("HWLAB_KAFKA_PROJECTOR_GROUP_ID", "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS"); if (capabilities.projectionOutboxRelay) required.push( "HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS", @@ -68,6 +78,13 @@ export function kafkaEventBridgeConfig(env = process.env) { sendTimeoutMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS, "HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS"), retryBackoffMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS, "HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS") } : null; + const refreshReplay = capabilities.kafkaRefreshReplay ? { + groupIdPrefix: stringValue(env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_GROUP_PREFIX), + timeoutMs: strictPositiveInteger(env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS, "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS"), + scanLimit: strictPositiveInteger(env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT, "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT"), + matchedEventLimit: strictPositiveInteger(env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_MATCHED_EVENT_LIMIT, "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_MATCHED_EVENT_LIMIT"), + liveBufferLimit: strictPositiveInteger(env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_LIVE_BUFFER_LIMIT, "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_LIVE_BUFFER_LIMIT") + } : null; if (relay && relay.sendTimeoutMs + 5000 >= relay.leaseMs) { const error = new Error("HWLAB Kafka relay lease must exceed producer send timeout by at least 5000ms."); error.code = "hwlab_kafka_relay_lease_invalid"; @@ -93,6 +110,7 @@ export function kafkaEventBridgeConfig(env = process.env) { directPublishGroupId: stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID), projectorGroupId: stringValue(env.HWLAB_KAFKA_PROJECTOR_GROUP_ID), hwlabEventGroupId: stringValue(env.HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID), + refreshReplay, projectorHeartbeatIntervalMs: capabilities.transactionalProjector ? strictPositiveInteger(env.HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS, "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS") : null, relay }; @@ -319,6 +337,7 @@ function startLiveHwlabKafkaEventBridge({ config, env = process.env, logger = co started: true, ...config, ready, + liveReady: ready, async stop() { stopped = true; subscribers.clear(); @@ -339,6 +358,9 @@ function startLiveHwlabKafkaEventBridge({ config, env = process.env, logger = co subscribers.add(listener); return () => subscribers.delete(listener); }, + queryHwlabEventRetention(params = {}) { + return queryKafkaEventStream({ ...params, env, stream: "hwlab", topic: config.hwlabTopic, kafkaFactory }); + }, liveSubscriberCount() { return subscribers.size; }, valuesPrinted: false }; @@ -366,11 +388,13 @@ function liveKafkaLagSummary(lagByPartition) { function combineKafkaEventBridgeComponents(config, components) { const ready = Promise.all(components.map((component) => component.ready)); + const liveOwner = components.find((component) => typeof component.subscribeLiveHwlabEvents === "function"); const componentStartup = () => components.map((component) => component.startupStatus?.() ?? { status: "initializing" }); return { started: true, ...config, ready, + liveReady: liveOwner?.liveReady ?? liveOwner?.ready ?? ready, async stop() { await Promise.allSettled(components.map((component) => component.stop())); }, async status() { const statuses = await Promise.all(components.map((component) => component.status())); @@ -402,8 +426,11 @@ function combineKafkaEventBridgeComponents(config, components) { return () => stops.forEach((stop) => stop()); }, subscribeLiveHwlabEvents(listener) { - const owner = components.find((component) => typeof component.subscribeLiveHwlabEvents === "function"); - return owner?.subscribeLiveHwlabEvents(listener) ?? (() => {}); + return liveOwner?.subscribeLiveHwlabEvents(listener) ?? (() => {}); + }, + queryHwlabEventRetention(params = {}) { + if (typeof liveOwner?.queryHwlabEventRetention !== "function") throw contractError("hwlab_kafka_refresh_query_unconfigured", "Kafka refresh retention query is not configured."); + return liveOwner.queryHwlabEventRetention(params); }, liveSubscriberCount() { const owner = components.find((component) => typeof component.liveSubscriberCount === "function"); @@ -926,34 +953,50 @@ function requireKafkaProjectorStore(runtimeStore, capabilities = {}) { if (missing.length > 0) throw contractError("hwlab_kafka_projector_store_invalid", `Kafka durable capabilities require a runtime store: ${missing.join(", ")}`); } -export async function queryKafkaEventStream({ env = process.env, stream = "hwlab", topic = null, traceId = null, sessionId = null, runId = null, commandId = null, limit = DEFAULT_QUERY_LIMIT, timeoutMs = DEFAULT_QUERY_TIMEOUT_MS, fromBeginning = true, groupIdPrefix = null, kafkaFactory = defaultKafkaFactory } = {}) { +export async function queryKafkaEventStream({ env = process.env, stream = "hwlab", topic = null, traceId = null, sessionId = null, runId = null, commandId = null, limit = DEFAULT_QUERY_LIMIT, scanLimit = null, timeoutMs = DEFAULT_QUERY_TIMEOUT_MS, fromBeginning = true, groupIdPrefix = null, signal = null, kafkaFactory = defaultKafkaFactory } = {}) { const brokers = csv(env.HWLAB_KAFKA_BOOTSTRAP_SERVERS); if (brokers.length === 0) throw new Error("HWLAB_KAFKA_BOOTSTRAP_SERVERS is required for Kafka event queries."); const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID) || DEFAULT_CLIENT_ID; const resolvedTopic = stringValue(topic) || kafkaTopicForStream(stream, env); const maxEvents = Math.max(1, integerValue(limit) || DEFAULT_QUERY_LIMIT); + const maxScannedRecords = scanLimit === null || scanLimit === undefined + ? null + : strictPositiveInteger(scanLimit, "scanLimit"); const budgetMs = Math.max(250, integerValue(timeoutMs) || DEFAULT_QUERY_TIMEOUT_MS); const deadlineMs = Date.now() + budgetMs; const kafka = kafkaFactory({ brokers, clientId: `${clientId}-query` }); - const endOffsetSnapshot = await kafkaTopicEndOffsetSnapshot(kafka, resolvedTopic, deadlineMs); + const endOffsetSnapshot = await kafkaTopicEndOffsetSnapshot(kafka, resolvedTopic, deadlineMs, signal); const endOffsetByPartition = new Map(endOffsetSnapshot.offsets.map((entry) => [entry.partition, entry.endOffset])); + const startOffsetByPartition = new Map(endOffsetSnapshot.offsets.map((entry) => [entry.partition, entry.startOffset])); const reachedEndPartitions = new Set(endOffsetSnapshot.offsets.filter(kafkaOffsetPartitionIsEmpty).map((entry) => entry.partition)); + const verifiedStartPartitions = new Set(reachedEndPartitions); const resolvedGroupIdPrefix = stringValue(groupIdPrefix) || `${clientId}-query`; const groupId = `${resolvedGroupIdPrefix}-${Date.now()}-${randomUUID().slice(0, 8)}`; const consumer = kafka.consumer({ groupId, allowAutoTopicCreation: false }); const events = []; + const firstScannedOffsetByPartition = new Map(); const lastScannedOffsetByPartition = new Map(); let scannedCount = 0; let parsedCount = 0; let invalidJsonCount = 0; let filterRejectedCount = 0; + let excludedPostBarrierCount = 0; let completionReason = "timeout"; let timer = null; let running = false; + let removeAbortListener = null; try { + if (signal?.aborted === true) { + completionReason = "aborted"; + return kafkaQueryResult(); + } if (remainingKafkaQueryBudget(deadlineMs) <= 0) return kafkaQueryResult(); - await withinKafkaQueryBudget(consumer.connect(), deadlineMs, "consumer-connect"); - await withinKafkaQueryBudget(consumer.subscribe({ topic: resolvedTopic, fromBeginning: fromBeginning !== false }), deadlineMs, "consumer-subscribe"); + if (fromBeginning !== false && endOffsetSnapshot.available && endOffsetSnapshot.offsets.some((entry) => entry.startOffset === null)) { + completionReason = "retention-start-unavailable"; + return kafkaQueryResult(); + } + await withinKafkaQueryBudget(consumer.connect(), deadlineMs, "consumer-connect", signal); + await withinKafkaQueryBudget(consumer.subscribe({ topic: resolvedTopic, fromBeginning: fromBeginning !== false }), deadlineMs, "consumer-subscribe", signal); const alreadyAtBoundedEnd = fromBeginning !== false && endOffsetSnapshot.available && reachedEndPartitions.size === endOffsetByPartition.size; if (alreadyAtBoundedEnd) completionReason = "end-offset"; if (alreadyAtBoundedEnd) return kafkaQueryResult(); @@ -967,9 +1010,34 @@ export async function queryKafkaEventStream({ env = process.env, stream = "hwlab }; timer = setTimeout(() => finish("timeout"), Math.max(1, remainingKafkaQueryBudget(deadlineMs))); timer.unref?.(); + if (typeof signal?.addEventListener === "function") { + const onAbort = () => finish("aborted"); + signal.addEventListener("abort", onAbort, { once: true }); + removeAbortListener = () => signal.removeEventListener?.("abort", onAbort); + } running = true; consumer.run({ eachMessage: async ({ topic: messageTopic, partition, message }) => { + if (finished) return; + const endOffset = endOffsetByPartition.get(partition); + if (fromBeginning !== false && endOffsetSnapshot.available && !firstScannedOffsetByPartition.has(partition)) { + const actualStartOffset = stringValue(message.offset); + const expectedStartOffset = startOffsetByPartition.get(partition); + firstScannedOffsetByPartition.set(partition, actualStartOffset); + if (!endOffsetByPartition.has(partition) || !expectedStartOffset || actualStartOffset !== expectedStartOffset) { + finish("retention-start-moved"); + return; + } + verifiedStartPartitions.add(partition); + } + if (fromBeginning !== false && endOffset && kafkaOffsetAtOrBeyond(message.offset, endOffset)) { + excludedPostBarrierCount += 1; + reachedEndPartitions.add(partition); + if (endOffsetSnapshot.available && reachedEndPartitions.size === endOffsetByPartition.size) { + finish(verifiedStartPartitions.size === endOffsetByPartition.size ? "end-offset" : "retention-start-unverified"); + } + return; + } scannedCount += 1; lastScannedOffsetByPartition.set(partition, stringValue(message.offset)); const valueText = message.value ? Buffer.from(message.value).toString("utf8") : ""; @@ -990,21 +1058,29 @@ export async function queryKafkaEventStream({ env = process.env, stream = "hwlab }); } } - const endOffset = endOffsetByPartition.get(partition); if (fromBeginning !== false && endOffset && kafkaOffsetReached(message.offset, endOffset)) reachedEndPartitions.add(partition); - if (endOffsetSnapshot.available && reachedEndPartitions.size === endOffsetByPartition.size) return finish("end-offset"); + if (endOffsetSnapshot.available && reachedEndPartitions.size === endOffsetByPartition.size) { + return finish(verifiedStartPartitions.size === endOffsetByPartition.size ? "end-offset" : "retention-start-unverified"); + } if (events.length >= maxEvents) finish("limit"); + else if (maxScannedRecords !== null && scannedCount >= maxScannedRecords) finish("scan-limit"); } }).catch(reject); }); + } catch (error) { + if (error?.code === "kafka_query_aborted") completionReason = "aborted"; + else throw error; } finally { if (timer) clearTimeout(timer); + removeAbortListener?.(); if (running) await boundedKafkaPromise(consumer.stop(), remainingKafkaQueryCleanupBudget(deadlineMs)).catch(() => undefined); await boundedDisconnect(consumer, remainingKafkaQueryCleanupBudget(deadlineMs)); } return kafkaQueryResult(); function kafkaQueryResult() { + const retentionStartVerified = endOffsetSnapshot.available && verifiedStartPartitions.size === endOffsetByPartition.size; + const complete = completionReason === "end-offset" && retentionStartVerified; return { ok: true, stream, @@ -1016,13 +1092,28 @@ export async function queryKafkaEventStream({ env = process.env, stream = "hwlab matchedCount: events.length, invalidJsonCount, filterRejectedCount, + excludedPostBarrierCount, completionReason, - reachedEndOffsets: completionReason === "end-offset", + completion: { + reason: completionReason, + complete, + barrierReached: complete, + retentionStartVerified, + retentionStartMoved: completionReason === "retention-start-moved", + timeout: completionReason === "timeout", + matchedEventLimitReached: completionReason === "limit", + scanLimitReached: completionReason === "scan-limit", + aborted: completionReason === "aborted", + valuesRedacted: true + }, + reachedEndOffsets: complete, endOffsets: endOffsetSnapshot.offsets, endOffsetsAvailable: endOffsetSnapshot.available, endOffsetsError: endOffsetSnapshot.error, - lastScannedOffsets: [...lastScannedOffsetByPartition.entries()].map(([partition, offset]) => ({ partition, offset })), + firstScannedOffsets: [...firstScannedOffsetByPartition.entries()].map(([partition, offset]) => ({ partition, offset })).sort((left, right) => left.partition - right.partition), + lastScannedOffsets: [...lastScannedOffsetByPartition.entries()].map(([partition, offset]) => ({ partition, offset })).sort((left, right) => left.partition - right.partition), limit: maxEvents, + scanLimit: maxScannedRecords, timeoutMs: budgetMs, filters: compactObject({ traceId, sessionId, runId, commandId }), events, @@ -1031,12 +1122,12 @@ export async function queryKafkaEventStream({ env = process.env, stream = "hwlab } } -async function kafkaTopicEndOffsetSnapshot(kafka, topic, deadlineMs) { +async function kafkaTopicEndOffsetSnapshot(kafka, topic, deadlineMs, signal = null) { if (typeof kafka?.admin !== "function") return { available: false, offsets: [], error: { code: "kafka_admin_unavailable", valuesRedacted: true } }; const admin = kafka.admin(); try { - await withinKafkaQueryBudget(admin.connect(), deadlineMs, "admin-connect"); - const fetched = await withinKafkaQueryBudget(admin.fetchTopicOffsets(topic), deadlineMs, "fetch-topic-offsets"); + await withinKafkaQueryBudget(admin.connect(), deadlineMs, "admin-connect", signal); + const fetched = await withinKafkaQueryBudget(admin.fetchTopicOffsets(topic), deadlineMs, "fetch-topic-offsets", signal); const offsets = Array.isArray(fetched) ? fetched.map((entry) => ({ partition: integerValue(entry.partition), @@ -1070,20 +1161,31 @@ function remainingKafkaQueryCleanupBudget(deadlineMs) { return Math.max(1, remainingKafkaQueryBudget(deadlineMs)); } -async function withinKafkaQueryBudget(promise, deadlineMs, stage) { +async function withinKafkaQueryBudget(promise, deadlineMs, stage, signal = null) { const remainingMs = remainingKafkaQueryBudget(deadlineMs); if (remainingMs <= 0) throw contractError("kafka_query_timeout", `Kafka query timed out before ${stage}.`); + if (signal?.aborted === true) throw contractError("kafka_query_aborted", `Kafka query was aborted before ${stage}.`); let timer = null; + let removeAbortListener = null; try { - return await Promise.race([ + const contenders = [ promise, new Promise((_, reject) => { timer = setTimeout(() => reject(contractError("kafka_query_timeout", `Kafka query timed out during ${stage}.`)), remainingMs); timer.unref?.(); }) - ]); + ]; + if (typeof signal?.addEventListener === "function") { + contenders.push(new Promise((_, reject) => { + const onAbort = () => reject(contractError("kafka_query_aborted", `Kafka query was aborted during ${stage}.`)); + signal.addEventListener("abort", onAbort, { once: true }); + removeAbortListener = () => signal.removeEventListener?.("abort", onAbort); + })); + } + return await Promise.race(contenders); } finally { if (timer) clearTimeout(timer); + removeAbortListener?.(); } } @@ -1095,6 +1197,14 @@ function kafkaOffsetReached(offset, endOffset) { } } +function kafkaOffsetAtOrBeyond(offset, endOffset) { + try { + return BigInt(String(offset)) >= BigInt(String(endOffset)); + } catch { + return false; + } +} + export async function openKafkaEventStream({ env = process.env, stream = "hwlab", topic = null, groupIdPrefix = null, traceId = null, sessionId = null, runId = null, commandId = null, replayId = null, offsetRange = null, fromBeginning = false, onEvent, onRecord = null, onBarrierReady = null, onBarrierComplete = null, onError = null, kafkaFactory = defaultKafkaFactory } = {}) { if (typeof onEvent !== "function") throw new Error("onEvent callback is required for Kafka event streaming."); const brokers = csv(env.HWLAB_KAFKA_BOOTSTRAP_SERVERS); @@ -1213,6 +1323,25 @@ function mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run, valuesPrinted: false }; const type = firstText(sourceEvent.type, payload.type, input.eventType) || "event"; + if (type === "user_message") { + const userMessageId = firstText(payload.userMessageId); + const userText = textPayload(payload, "user message"); + return { + ...base, + type: "user", + eventType: "user", + status: "sent", + label: "agentrun:user:message", + message: userText, + text: userText, + role: "user", + messageId: userMessageId, + userMessageId, + targetTraceId: firstText(payload.targetTraceId), + userSource: firstText(payload.source), + terminal: false + }; + } if (type === "assistant_message") { const authoritative = payload.replyAuthority === true || payload.final === true; const assistantText = textPayload(payload, "assistant message"); diff --git a/internal/cloud/server-agent-chat.test.ts b/internal/cloud/server-agent-chat.test.ts index db888cdf..9bfb3549 100644 --- a/internal/cloud/server-agent-chat.test.ts +++ b/internal/cloud/server-agent-chat.test.ts @@ -31,6 +31,14 @@ import { const TEST_AGENT_ACTOR = { id: "usr_agent_owner", username: "agent-owner", displayName: "Agent Owner", role: "user", status: "active" }; const TEST_AUTH_SESSION = { id: "auth_ses_agent_owner" }; +const TEST_REALTIME_CAPABILITIES_DISABLED = Object.freeze({ + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" +}); async function waitForTraceEvent(traceStore, traceId, predicate, message = "expected trace event") { for (let index = 0; index < 100; index += 1) { @@ -508,6 +516,7 @@ test("cloud api /v1/agent/chat supports short submit and result polling", async const codexHome = await mkdtemp(path.join(os.tmpdir(), "hwlab-agent-short-codex-home-")); const server = createCloudApiServer({ env: { + ...TEST_REALTIME_CAPABILITIES_DISABLED, PATH: process.env.PATH, CODEX_HOME: codexHome, HWLAB_CODE_AGENT_CODEX_WORKSPACE: workspace, @@ -567,8 +576,8 @@ test("cloud api /v1/agent/chat supports short submit and result polling", async message: "用pwd列出你当前的工作目录" }) }); - assert.equal(submit.status, 202); const accepted = await submit.json(); + assert.equal(submit.status, 202, JSON.stringify(accepted)); assert.equal(accepted.accepted, true); assert.equal(accepted.shortConnection, true); assert.equal(accepted.traceId, traceId); @@ -670,6 +679,7 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte assert.equal(body.payload.targetTraceId, "trc_server-test-agentrun-adapter"); assert.equal(body.payload.targetCommandId, "cmd_hwlab_adapter"); assert.equal(body.payload.traceId, "trc_steer_server_test"); + assert.equal(body.payload.userMessageId, "msg_steer_server_test_user"); assert.match(body.payload.prompt, /STEER_MARK/u); assert.equal(body.payload.sessionId, "ses_agentrun_server_test_agentrun"); assert.equal(body.payload.hwlabSessionId, "ses_server-test-agentrun"); @@ -685,6 +695,7 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte assert.equal(body.payload.threadContinuityPolicy, "hwlab-agentrun-v01-reuse-runner-thread"); assert.equal(body.payload.sessionPolicy, "hwlab-agentrun-v01-session-runner-reuse"); const secondTurn = /第二轮/u.test(body.payload.message ?? body.payload.prompt); + assert.equal(body.payload.userMessageId, secondTurn ? "msg_server-test-agentrun-adapter-second_user" : "msg_server-test-agentrun-adapter_user"); if (secondTurn) { assert.equal(body.payload.prompt, "AgentRun adapter smoke 第二轮：总结我们刚才的对话内容"); assert.equal(body.payload.message, "AgentRun adapter smoke 第二轮：总结我们刚才的对话内容"); @@ -832,6 +843,12 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte traceStore, runtimeStore, workbenchRuntime: runtimeStore, + kafkaEventBridge: { + capabilities: { directPublish: false, liveKafkaSse: false, transactionalProjector: true, projectionOutboxRelay: false, projectionRealtime: true }, + ready: Promise.resolve(), + subscribeProjectionCommits() { return () => {}; }, + async stop() {} + }, env: { HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, @@ -840,6 +857,12 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte HWLAB_CODE_AGENT_AGENTRUN_SOURCE_COMMIT: "0123456789abcdef0123456789abcdef01234567", HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE: "deepseek", HWLAB_CODE_AGENT_DEEPSEEK_MODEL: "deepseek-chat", + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "true", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true", HWLAB_ENVIRONMENT: "v03", HWLAB_GITOPS_PROFILE: "v03", HWLAB_BOOT_REF: "v0.3", @@ -898,8 +921,8 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte message: "AgentRun adapter smoke" }) }); - assert.equal(submit.status, 202); const accepted = await submit.json(); + assert.equal(submit.status, 202, JSON.stringify(accepted)); assert.equal(accepted.shortConnection, true); assert.equal(accepted.turnId, traceId); assert.equal(accepted.userMessageId, "msg_server-test-agentrun-adapter_user"); @@ -1040,6 +1063,9 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte assert.equal(steerBody.agentRun.runId, "run_hwlab_adapter"); assert.equal(calls.some((call) => call.method === "POST" && call.path === "/api/v1/runs/run_hwlab_adapter/commands" && call.body?.type === "steer"), false); + const retainedAgentRun = ownerSessions.get("ses_server-test-agentrun")?.session?.agentRun; + assert.equal(retainedAgentRun?.durableDispatch, true, JSON.stringify(retainedAgentRun ?? null)); + assert.equal(retainedAgentRun?.runId, "run_hwlab_adapter"); const secondTraceId = "trc_server-test-agentrun-adapter-second"; const second = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, { method: "POST", @@ -1106,6 +1132,7 @@ test("cloud api rejects billing failure before durable AgentRun admission withou const server = createCloudApiServer({ traceStore, env: { + ...TEST_REALTIME_CAPABILITIES_DISABLED, HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1", @@ -1327,6 +1354,7 @@ test("cloud api AgentRun adapter reports persistent thread resume when a complet runtimeStore, workbenchRuntime: runtimeStore, env: { + ...TEST_REALTIME_CAPABILITIES_DISABLED, HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1", @@ -1499,6 +1527,7 @@ test("cloud api AgentRun adapter exposes invalid tool-call attribution through K runtimeStore, workbenchRuntime: runtimeStore, env: { + ...TEST_REALTIME_CAPABILITIES_DISABLED, HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1", @@ -1704,6 +1733,7 @@ test("cloud api surfaces manager-side evicted-session recovery through Kafka pro runtimeStore, workbenchRuntime: runtimeStore, env: { + ...TEST_REALTIME_CAPABILITIES_DISABLED, HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1", @@ -1861,6 +1891,7 @@ test("cloud api AgentRun adapter maps minimax-m3 provider profile to AgentRun ba runtimeStore, workbenchRuntime: runtimeStore, env: { + ...TEST_REALTIME_CAPABILITIES_DISABLED, HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1", diff --git a/internal/cloud/server-code-agent-admission-http.ts b/internal/cloud/server-code-agent-admission-http.ts index edb557f9..10bd3f0e 100644 --- a/internal/cloud/server-code-agent-admission-http.ts +++ b/internal/cloud/server-code-agent-admission-http.ts @@ -1111,7 +1111,11 @@ async function submitCodeAgentChatTurn({ params, options, traceId }) { throw codeAgentSubmissionUnavailableError(billingPreflight, { params, options, traceId }); } - const dispatchParams = { ...params, userBillingReservation: billingPreflight?.reservation ?? null }; + const dispatchParams = { + ...params, + ...lifecycle, + userBillingReservation: billingPreflight?.reservation ?? null + }; const executionOptions = { ...options, ...(await codeAgentChatExecutionOptions(options, dispatchParams)) }; const dispatchStartedAt = Date.now(); let payload; diff --git a/internal/cloud/server-code-agent-trace-http.ts b/internal/cloud/server-code-agent-trace-http.ts index 5ab319f9..9dd2c08e 100644 --- a/internal/cloud/server-code-agent-trace-http.ts +++ b/internal/cloud/server-code-agent-trace-http.ts @@ -445,12 +445,15 @@ function mergeCodeAgentDiagnosticTrace(projectionTrace, diagnosticTrace, traceId || left.key.localeCompare(right.key)) .map(({ event }, index) => ({ ...event, seq: index + 1, paginationSeq: index + 1 })); const projectionFound = projected.status && projected.status !== "missing"; + const projectionEvents = events.filter((event) => event.authority === "workbench-projection"); + const projectionTerminal = projected.terminal === true || projectionEvents.some((event) => event.terminal === true); + const projectionSealed = projected.sealed === true || projectionEvents.some((event) => event.sealed === true); return { ...projected, traceId, status: projectionFound ? projected.status : events.length > 0 ? "running" : "missing", - terminal: projected.terminal === true, - sealed: projected.sealed === true, + terminal: projectionTerminal, + sealed: projectionSealed, events, eventCount: events.length, diagnosticEventCount: events.filter((event) => event.diagnosticOnly === true).length, diff --git a/internal/cloud/server-workbench-realtime-http.test.ts b/internal/cloud/server-workbench-realtime-http.test.ts index 782f1f18..bc5c0ab4 100644 --- a/internal/cloud/server-workbench-realtime-http.test.ts +++ b/internal/cloud/server-workbench-realtime-http.test.ts @@ -40,17 +40,28 @@ import { const TRANSACTIONAL_REALTIME_ENV = Object.freeze({ HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false", HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false", HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true" }); +const REFRESH_REALTIME_ENV = Object.freeze({ + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "true", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" +}); + function projectionRealtimeBridge(capabilities = {}) { return { started: true, capabilities: { directPublish: false, liveKafkaSse: false, + kafkaRefreshReplay: false, transactionalProjector: false, projectionOutboxRelay: false, projectionRealtime: true, @@ -87,7 +98,7 @@ test("live Kafka SSE transparently fans out one envelope without DB, snapshot, c let releaseBridgeReady; const bridgeReady = new Promise((resolve) => { releaseBridgeReady = resolve; }); const bridge = { - capabilities: { directPublish: true, liveKafkaSse: true, transactionalProjector: false, projectionOutboxRelay: false, projectionRealtime: false }, + capabilities: { directPublish: true, liveKafkaSse: true, kafkaRefreshReplay: false, transactionalProjector: false, projectionOutboxRelay: false, projectionRealtime: false }, ready: bridgeReady, subscribeLiveHwlabEvents(listener) { subscribers.add(listener); @@ -112,6 +123,7 @@ test("live Kafka SSE transparently fans out one envelope without DB, snapshot, c env: { HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false", HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" @@ -177,6 +189,183 @@ test("live Kafka SSE transparently fans out one envelope without DB, snapshot, c } }); +test("Kafka refresh SSE replays retained user and lifecycle envelopes before one connected handoff without waiting for projector readiness", async () => { + const sessionId = "ses_kafka_refresh_replay"; + const traceId = "trc_kafka_refresh_replay"; + const records = [ + refreshRecord(0, sessionId, traceId, "user", { userMessageId: "msg_kafka_refresh_user", messageId: "msg_kafka_refresh_user", text: "retained input" }), + refreshRecord(1, sessionId, traceId, "backend"), + refreshRecord(2, sessionId, traceId, "tool"), + refreshRecord(3, sessionId, traceId, "assistant", { assistantText: "retained answer", text: "retained answer" }), + refreshRecord(4, sessionId, traceId, "result", { terminal: true, status: "completed" }) + ]; + let retentionQuery = null; + let projectionRuntimeCalls = 0; + const bridge = refreshReplayBridge({ + records, + endOffset: "5", + onQuery(params) { retentionQuery = params; }, + ready: new Promise(() => undefined), + liveReady: Promise.resolve() + }); + const server = createCloudApiServer({ + accessController: realtimeAccessController({ sessions: [{ id: sessionId, ownerUserId: ACTOR.id, lastTraceId: traceId }] }), + workbenchRuntime: new Proxy({}, { get() { projectionRuntimeCalls += 1; throw new Error("Kafka refresh must not hydrate projection history"); } }), + kafkaEventBridge: bridge, + env: { ...REFRESH_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000" } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const events = await getSseEvents(server.address().port, `/v1/workbench/events?traceId=${traceId}`, 6); + assert.deepEqual(events.map((event) => event.event), [ + "hwlab.event.v1", + "hwlab.event.v1", + "hwlab.event.v1", + "hwlab.event.v1", + "hwlab.event.v1", + "workbench.connected" + ]); + assert.equal(events[0].data.event.type, "user"); + assert.equal(events[0].data.event.userMessageId, "msg_kafka_refresh_user"); + assert.equal(events[4].data.event.terminal, true); + assert.equal(events[5].data.deliverySemantics, "kafka-retention-then-live"); + assert.equal(events[5].data.liveOnly, false); + assert.equal(events[5].data.replay, true); + assert.equal(events[5].data.lossPossible, false); + assert.deepEqual(events[5].data.filters, { sessionId, traceId }); + assert.equal(events[5].data.refreshReplay.counts.replayed, 5); + assert.equal(retentionQuery.sessionId, sessionId); + assert.equal(retentionQuery.traceId, traceId); + assert.equal(retentionQuery.limit, 50); + assert.equal(retentionQuery.scanLimit, 500); + assert.equal(retentionQuery.timeoutMs, 2500); + assert.equal(retentionQuery.groupIdPrefix, "hwlab-v03-refresh-test"); + assert.equal(retentionQuery.fromBeginning, true); + assert.equal(retentionQuery.signal instanceof AbortSignal, true); + assert.equal(projectionRuntimeCalls, 0); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("Kafka refresh client abort while live readiness is pending never starts retention or live subscription", async () => { + const sessionId = "ses_kafka_refresh_ready_abort"; + let releaseLiveReady: (() => void) | null = null; + const liveReady = new Promise((resolve) => { releaseLiveReady = resolve; }); + const bridge = refreshReplayBridge({ liveReady }); + let liveReadyObserved = 0; + let queryCalls = 0; + let subscribeCalls = 0; + const originalQuery = bridge.queryHwlabEventRetention.bind(bridge); + const originalSubscribe = bridge.subscribeLiveHwlabEvents.bind(bridge); + Object.defineProperty(bridge, "liveReady", { + configurable: true, + get() { + liveReadyObserved += 1; + return liveReady; + } + }); + bridge.queryHwlabEventRetention = async (params) => { + queryCalls += 1; + return await originalQuery(params); + }; + bridge.subscribeLiveHwlabEvents = (listener) => { + subscribeCalls += 1; + return originalSubscribe(listener); + }; + const server = createCloudApiServer({ + accessController: realtimeAccessController({ sessions: [{ id: sessionId, ownerUserId: ACTOR.id }] }), + kafkaEventBridge: bridge, + env: { ...REFRESH_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000" } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const controller = new AbortController(); + const response = fetch(`http://127.0.0.1:${server.address().port}/v1/workbench/events?sessionId=${sessionId}`, { signal: controller.signal }).catch((error) => error); + await waitForCondition(() => liveReadyObserved > 0); + controller.abort(); + await response; + releaseLiveReady?.(); + await new Promise((resolve) => setTimeout(resolve, 10)); + assert.equal(queryCalls, 0); + assert.equal(subscribeCalls, 0); + } finally { + releaseLiveReady?.(); + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("trace-only Kafka refresh binds the authorized session and rejects same-trace foreign-session retention", async () => { + const sessionId = "ses_kafka_refresh_trace_owner"; + const foreignSessionId = "ses_kafka_refresh_trace_foreign"; + const traceId = "trc_kafka_refresh_trace_owner"; + let retentionQuery = null; + const bridge = refreshReplayBridge({ + records: [refreshRecord(0, foreignSessionId, traceId, "assistant", { text: "foreign session" })], + endOffset: "1", + onQuery(params) { retentionQuery = params; } + }); + const server = createCloudApiServer({ + accessController: realtimeAccessController({ sessions: [{ id: sessionId, ownerUserId: ACTOR.id, lastTraceId: traceId }] }), + kafkaEventBridge: bridge, + env: { ...REFRESH_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000" } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const response = await fetch(`http://127.0.0.1:${server.address().port}/v1/workbench/events?traceId=${traceId}`); + assert.equal(response.status, 200); + const events = (await response.text()).trim().split("\n\n").filter(Boolean).map(parseSseBlock); + assert.deepEqual(events.map((event) => event.event), ["workbench.error"]); + assert.equal(events[0].data.sessionId, sessionId); + assert.equal(events[0].data.traceId, traceId); + assert.equal(events[0].data.error.code, "workbench_kafka_refresh_scope_mismatch"); + assert.equal(retentionQuery.sessionId, sessionId); + assert.equal(retentionQuery.traceId, traceId); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("session-scoped Kafka refresh gap emits typed workbench.error and closes without connected or fallback", async () => { + const sessionId = "ses_kafka_refresh_gap"; + const traceId = "trc_kafka_refresh_gap"; + const retained = refreshRecord(0, sessionId, traceId, "user", { userMessageId: "msg_kafka_refresh_gap", messageId: "msg_kafka_refresh_gap", text: "retained input" }); + const missing = refreshRecord(1, sessionId, traceId, "assistant", { assistantText: "missing from scan" }); + const bridge = refreshReplayBridge({ + records: [retained], + endOffset: "2", + beforeQueryResult(listener) { listener(missing.value, { topic: missing.topic, partition: missing.partition, offset: missing.offset }); } + }); + const server = createCloudApiServer({ + accessController: realtimeAccessController({ sessions: [{ id: sessionId, ownerUserId: ACTOR.id, lastTraceId: traceId }] }), + kafkaEventBridge: bridge, + env: { ...REFRESH_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000" } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const response = await fetch(`http://127.0.0.1:${server.address().port}/v1/workbench/events?sessionId=${sessionId}`); + assert.equal(response.status, 200); + const body = await response.text(); + const events = body.trim().split("\n\n").filter(Boolean).map(parseSseBlock); + assert.deepEqual(events.map((event) => event.event), ["hwlab.event.v1", "workbench.error"]); + assert.equal(events[1].data.sessionId, sessionId); + assert.equal(events[1].data.traceId, null); + assert.equal(events[1].data.phase, "flushing"); + assert.equal(events[1].data.error.code, "workbench_kafka_refresh_barrier_gap"); + assert.deepEqual(events[1].data.error.details, { partition: 0 }); + assert.equal(events[1].data.fallback, false); + assert.equal(body.includes("workbench.connected"), false); + assert.equal(body.includes("workbench.trace.snapshot"), false); + assert.equal(body.includes("workbench.message.snapshot"), false); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + test("live Kafka SSE heartbeat keeps the transport open without DB, cursor, snapshot, or replay", async () => { const sessionId = "ses_live_kafka_heartbeat"; let dbCalls = 0; @@ -197,6 +386,7 @@ test("live Kafka SSE heartbeat keeps the transport open without DB, cursor, snap env: { HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false", HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false", @@ -244,6 +434,7 @@ test("live Kafka SSE rejects foreign or inconsistent ownership scopes before fan env: { HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false", HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" @@ -288,6 +479,7 @@ test("live Kafka SSE fails closed when ownership lookup is not configured", asyn env: { HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false", HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" @@ -323,6 +515,7 @@ test("live Kafka SSE lets admin subscribe to another owner's consistent session env: { HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false", HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" @@ -407,6 +600,7 @@ test("live and projection realtime remain independently reachable when both capa env: { HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false", HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true", @@ -1060,6 +1254,57 @@ test("workbench trace event page keeps per-trace terminal status after later ses } }); +function refreshReplayBridge({ records = [], endOffset = "0", onQuery = null, beforeQueryResult = null, ready = Promise.resolve(), liveReady = Promise.resolve() } = {}) { + let liveListener = null; + return { + capabilities: { directPublish: true, liveKafkaSse: true, kafkaRefreshReplay: true, transactionalProjector: false, projectionOutboxRelay: false, projectionRealtime: false }, + refreshReplay: { + groupIdPrefix: "hwlab-v03-refresh-test", + timeoutMs: 2500, + scanLimit: 500, + matchedEventLimit: 50, + liveBufferLimit: 20 + }, + ready, + liveReady, + subscribeLiveHwlabEvents(listener) { liveListener = listener; return () => { if (liveListener === listener) liveListener = null; }; }, + async queryHwlabEventRetention(params) { + onQuery?.(params); + beforeQueryResult?.(liveListener); + return { + topic: "hwlab.event.v1", + events: records, + completionReason: "end-offset", + reachedEndOffsets: true, + endOffsetsAvailable: true, + endOffsets: [{ partition: 0, startOffset: "0", endOffset }], + completion: { reason: "end-offset", complete: true, barrierReached: true, retentionStartVerified: true } + }; + }, + async stop() {} + }; +} + +function refreshRecord(offset, sessionId, traceId, type, extra = {}) { + const sourceEventId = `evt_refresh_${offset}`; + return { + topic: "hwlab.event.v1", + partition: 0, + offset: String(offset), + value: { + schema: "hwlab.event.v1", + eventType: "hwlab.trace.event.projected", + eventId: `hwlab:${sourceEventId}`, + sourceEventId, + traceId, + hwlabSessionId: sessionId, + sessionId, + event: { type, eventType: type === "result" ? "terminal" : type, traceId, sessionId, sourceEventId, status: "running", ...extra }, + valuesPrinted: false + } + }; +} + function realtimeAccessController({ actor = ACTOR, sessions = [] } = {}) { const bySessionId = new Map(sessions.map((session) => [session.id, session])); const byTraceId = new Map(sessions.filter((session) => session.lastTraceId).map((session) => [session.lastTraceId, session])); diff --git a/internal/cloud/server-workbench-realtime-http.ts b/internal/cloud/server-workbench-realtime-http.ts index b83fdb14..b640124d 100644 --- a/internal/cloud/server-workbench-realtime-http.ts +++ b/internal/cloud/server-workbench-realtime-http.ts @@ -15,6 +15,7 @@ import { import { createWorkbenchReadModel } from "./workbench-read-model.ts"; import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts"; import { emitLiveKafkaOtelSpan } from "./kafka-event-bridge.ts"; +import { createWorkbenchKafkaRefreshHandoff, workbenchKafkaRefreshErrorPayload } from "./workbench-kafka-refresh-handoff.ts"; import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts"; import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts"; import { projectionOutboxRealtimeEvents } from "./workbench-projection-outbox-events.ts"; @@ -483,11 +484,22 @@ async function handleLiveKafkaWorkbenchRealtimeHttp(request, response, url, opti sendJson(response, authorization.status, authorization.body); return; } + const authorizedSessionId = authorization.sessionId; const bridge = options.kafkaEventBridge; if (bridge?.capabilities?.liveKafkaSse !== true || typeof bridge?.subscribeLiveHwlabEvents !== "function") { sendJson(response, 503, workbenchError("workbench_live_kafka_unconfigured", "Workbench live Kafka fanout is not configured.")); return; } + if (realtimeCapabilities.kafkaRefreshReplay) { + await handleKafkaRefreshReplayWorkbenchRealtimeHttp(request, response, options, { + bridge, + heartbeatMs, + realtimeCapabilities, + requestedSessionId: authorizedSessionId, + requestedTraceId + }); + return; + } let closed = false; let realtimeCloseReason = "client_close"; @@ -502,7 +514,7 @@ async function handleLiveKafkaWorkbenchRealtimeHttp(request, response, url, opti reason: realtimeCloseReason, signal: realtimeCloseSignal, startedAtMs: realtimeStartedAtMs, - sessionId: requestedSessionId, + sessionId: authorizedSessionId, threadId: null, traceId: requestedTraceId, activeConnectionCount: activeWorkbenchRealtimeConnections.size @@ -520,7 +532,7 @@ async function handleLiveKafkaWorkbenchRealtimeHttp(request, response, url, opti let liveDeliveryStarted = false; let enqueueEvent = null; const unsubscribe = bridge.subscribeLiveHwlabEvents((envelope, transport) => { - if (!liveKafkaEnvelopeMatches(envelope, requestedSessionId, requestedTraceId)) return; + if (!liveKafkaEnvelopeMatches(envelope, authorizedSessionId, requestedTraceId)) return; if (!liveDeliveryStarted || typeof enqueueEvent !== "function") { bufferedEnvelopes.push({ envelope, transport }); return; @@ -530,7 +542,7 @@ async function handleLiveKafkaWorkbenchRealtimeHttp(request, response, url, opti if (typeof unsubscribe === "function") cleanup.push(unsubscribe); try { - await bridge.ready; + await (bridge.liveReady ?? bridge.ready); } catch (error) { closeConnection(); throw error; @@ -601,7 +613,7 @@ async function handleLiveKafkaWorkbenchRealtimeHttp(request, response, url, opti replay: false, replaySupported: false, lossPossible: true, - filters: { sessionId: requestedSessionId, traceId: requestedTraceId } + filters: { sessionId: authorizedSessionId, traceId: requestedTraceId } }); liveDeliveryStarted = true; const readyBufferedEnvelopes = bufferedEnvelopes.splice(0); @@ -628,6 +640,178 @@ async function handleLiveKafkaWorkbenchRealtimeHttp(request, response, url, opti emitWorkbenchRealtimeAcceptedOtelSpan(request, options.env); } +async function handleKafkaRefreshReplayWorkbenchRealtimeHttp(request, response, options, input) { + const { + bridge, + heartbeatMs, + realtimeCapabilities, + requestedSessionId, + requestedTraceId + } = input; + const refreshReplay = bridge?.refreshReplay; + if (!refreshReplay || typeof bridge?.queryHwlabEventRetention !== "function") { + sendJson(response, 503, workbenchError("workbench_kafka_refresh_unconfigured", "Workbench Kafka refresh replay is enabled without its retention query runtime.")); + return; + } + let closed = false; + let realtimeCloseReason = "client_close"; + let realtimeCloseSignal = null; + let failureClosing = false; + const realtimeStartedAtMs = Date.now(); + const cleanup = []; + const isActive = () => !closed && !response.destroyed && !response.writableEnded; + const closeConnection = () => { + if (closed) return; + closed = true; + emitWorkbenchRealtimeClosedOtelSpan(request, options.env, { + reason: realtimeCloseReason, + signal: realtimeCloseSignal, + startedAtMs: realtimeStartedAtMs, + sessionId: requestedSessionId, + threadId: null, + traceId: requestedTraceId, + activeConnectionCount: activeWorkbenchRealtimeConnections.size + }); + for (const item of cleanup.splice(0)) item(); + }; + + response.once("close", closeConnection); + request.once?.("aborted", closeConnection); + request.socket?.once?.("close", closeConnection); + cleanup.push(() => request.off?.("aborted", closeConnection)); + cleanup.push(() => request.socket?.off?.("close", closeConnection)); + + try { + await (bridge.liveReady ?? bridge.ready); + } catch (error) { + for (const item of cleanup.splice(0)) item(); + throw error; + } + if (!isActive()) { + closeConnection(); + return; + } + + response.writeHead(200, { + "content-type": "text/event-stream; charset=utf-8", + "cache-control": "no-store, no-transform", + connection: "keep-alive", + "x-accel-buffering": "no", + "x-content-type-options": "nosniff" + }); + if (typeof response.flushHeaders === "function") response.flushHeaders(); + + const writeEvent = async (name, payload, transport = null) => { + if (!isActive()) return false; + const block = `event: ${name}\ndata: ${JSON.stringify(payload)}\n\n`; + const writable = response.write(block); + if (!writable) await waitForResponseDrain(response, isActive); + const active = isActive(); + if (active && name === "hwlab.event.v1") { + emitLiveKafkaOtelSpan("hwlab.workbench.live_sse.business_event_write", payload, transport, { + env: options.env, + otelSpanEmitter: options.otelSpanEmitter ?? emitCodeAgentOtelSpan + }); + } + return active; + }; + let writeChain = Promise.resolve(true); + const enqueueEvent = (name, payload, transport = null) => { + writeChain = writeChain.then(() => writeEvent(name, payload, transport)); + return writeChain; + }; + const closeWithRefreshFailure = async (error) => { + if (failureClosing || !isActive()) return; + failureClosing = true; + await enqueueEvent("workbench.error", workbenchKafkaRefreshErrorPayload(error, { + sessionId: requestedSessionId, + traceId: requestedTraceId + })); + await writeChain; + if (!response.writableEnded) response.end(); + }; + + const realtimeConnection = { + close(fields = {}) { + if (!isActive()) return false; + realtimeCloseReason = textValue(fields.reason) || "server_shutdown"; + realtimeCloseSignal = textValue(fields.signal) || null; + void enqueueEvent("workbench.server_draining", { + type: "server.draining", + status: "closing", + reason: realtimeCloseReason, + signal: realtimeCloseSignal, + capabilities: realtimeCapabilities, + lossPossible: false + }).finally(() => { + if (!response.writableEnded) response.end(); + }); + return true; + }, + isActive + }; + activeWorkbenchRealtimeConnections.add(realtimeConnection); + cleanup.push(() => activeWorkbenchRealtimeConnections.delete(realtimeConnection)); + + const handoff = createWorkbenchKafkaRefreshHandoff({ + sessionId: requestedSessionId, + traceId: requestedTraceId, + liveBufferLimit: refreshReplay.liveBufferLimit, + subscribeLive: (listener) => bridge.subscribeLiveHwlabEvents(listener), + queryRetention: ({ signal }) => bridge.queryHwlabEventRetention({ + sessionId: requestedSessionId, + traceId: requestedTraceId, + limit: refreshReplay.matchedEventLimit, + scanLimit: refreshReplay.scanLimit, + timeoutMs: refreshReplay.timeoutMs, + groupIdPrefix: refreshReplay.groupIdPrefix, + fromBeginning: true, + signal + }), + deliverEvent: (envelope, transport) => enqueueEvent("hwlab.event.v1", envelope, transport), + deliverConnected: (summary) => enqueueEvent("workbench.connected", { + type: "connected", + status: "connected", + capabilities: realtimeCapabilities, + realtimeSource: "hwlab.event.v1", + deliverySemantics: "kafka-retention-then-live", + liveOnly: false, + replay: true, + replaySupported: true, + lossPossible: false, + filters: { sessionId: requestedSessionId, traceId: requestedTraceId }, + refreshReplay: summary + }), + onFailure: closeWithRefreshFailure + }); + cleanup.push(() => handoff.stop("connection-closed")); + + try { + await handoff.start(); + } catch (error) { + if (isActive()) await closeWithRefreshFailure(error); + return; + } + if (!isActive()) return; + + const heartbeatTimer = setInterval(() => { + void enqueueEvent("workbench.heartbeat", { + type: "heartbeat", + status: "connected", + realtimeSource: "hwlab.event.v1", + deliverySemantics: "kafka-retention-then-live", + liveOnly: false, + replay: true, + lossPossible: false, + serverSentAt: new Date().toISOString(), + valuesPrinted: false + }); + }, heartbeatMs); + heartbeatTimer.unref?.(); + cleanup.push(() => clearInterval(heartbeatTimer)); + emitWorkbenchRealtimeAcceptedOtelSpan(request, options.env); +} + async function authorizeLiveKafkaWorkbenchRealtimeScope(options, actor, { sessionId, traceId }) { const access = options.accessController; if ((sessionId && typeof access?.getAgentSession !== "function") || (traceId && typeof access?.getAgentSessionByTraceId !== "function")) { diff --git a/internal/cloud/workbench-kafka-refresh-handoff.test.ts b/internal/cloud/workbench-kafka-refresh-handoff.test.ts new file mode 100644 index 00000000..7a241f8a --- /dev/null +++ b/internal/cloud/workbench-kafka-refresh-handoff.test.ts @@ -0,0 +1,336 @@ +import assert from "node:assert/strict"; +import { test } from "bun:test"; + +import { createWorkbenchKafkaRefreshHandoff, workbenchKafkaRefreshErrorPayload } from "./workbench-kafka-refresh-handoff.ts"; + +const SESSION_ID = "ses_refresh_handoff"; +const TRACE_ID = "trc_refresh_handoff"; + +test("retention handoff replays once, deduplicates overlap and late pre-barrier live, and drains growth before connected", async () => { + let publishLive: any = null; + const output: string[] = []; + let growDuringFlush = true; + const replay = [record(0), record(1), record(2)]; + const handoff = createWorkbenchKafkaRefreshHandoff({ + sessionId: SESSION_ID, + liveBufferLimit: 8, + subscribeLive(listener: any) { publishLive = listener; return () => { publishLive = null; }; }, + async queryRetention() { + publishLive(record(1).value, transport(1)); + publishLive(record(3).value, transport(3)); + return queryResult(replay, [{ partition: 0, startOffset: "0", endOffset: "3" }]); + }, + async deliverEvent(envelope: any, kafkaTransport: any, delivery: string) { + output.push(`${delivery}:${kafkaTransport.offset}:${envelope.eventId}`); + if (delivery === "buffered-live" && kafkaTransport.offset === "3" && growDuringFlush) { + growDuringFlush = false; + publishLive(record(4).value, transport(4)); + } + return true; + }, + async deliverConnected(summary: any) { output.push(`connected:${summary.counts.deduplicated}`); return true; } + }); + + const summary = await handoff.start(); + assert.equal(handoff.phase(), "live"); + assert.deepEqual(output, [ + "replay:0:hwlab:evt_0_0", + "replay:1:hwlab:evt_0_1", + "replay:2:hwlab:evt_0_2", + "buffered-live:3:hwlab:evt_0_3", + "buffered-live:4:hwlab:evt_0_4", + "connected:1" + ]); + assert.deepEqual(summary.counts, { matched: 3, replayed: 3, buffered: 3, bufferedDelivered: 2, liveDelivered: 0, deduplicated: 1 }); + + publishLive(record(2).value, transport(2)); + publishLive(record(5).value, transport(5)); + await waitFor(() => output.includes("live:5:hwlab:evt_0_5")); + assert.equal(output.filter((item) => item.includes("hwlab:evt_0_2")).length, 1); + assert.equal(handoff.summary().counts.deduplicated, 2); + handoff.stop(); +}); + +test("retention handoff fails closed on a pre-barrier live gap", async () => { + let publishLive: any = null; + const handoff = createWorkbenchKafkaRefreshHandoff({ + sessionId: SESSION_ID, + liveBufferLimit: 4, + subscribeLive(listener: any) { publishLive = listener; return () => {}; }, + async queryRetention() { + publishLive(record(1).value, transport(1)); + return queryResult([record(0)], [{ partition: 0, startOffset: "0", endOffset: "3" }]); + }, + async deliverEvent() { return true; }, + async deliverConnected() { return true; } + }); + + await assert.rejects(handoff.start(), (error: any) => error?.code === "workbench_kafka_refresh_barrier_gap" && error?.phase === "flushing"); + assert.equal(handoff.phase(), "failed"); +}); + +test("connected write failure blocks a concurrently queued live business event", async () => { + let publishLive: any = null; + let connectedStarted = false; + let releaseConnected: (() => void) | null = null; + const connectedGate = new Promise((resolve) => { releaseConnected = resolve; }); + const delivered: string[] = []; + const handoff = createWorkbenchKafkaRefreshHandoff({ + sessionId: SESSION_ID, + liveBufferLimit: 4, + subscribeLive(listener: any) { publishLive = listener; return () => { publishLive = null; }; }, + async queryRetention() { return queryResult([], [{ partition: 0, startOffset: "0", endOffset: "1" }]); }, + async deliverEvent(_envelope: any, kafkaTransport: any, delivery: string) { + delivered.push(`${delivery}:${kafkaTransport.offset}`); + return true; + }, + async deliverConnected() { + connectedStarted = true; + await connectedGate; + return false; + } + }); + + const started = handoff.start(); + await waitFor(() => connectedStarted); + publishLive(record(1).value, transport(1)); + releaseConnected?.(); + await assert.rejects(started, (error: any) => error?.code === "workbench_kafka_refresh_sse_write_failed"); + await Promise.resolve(); + assert.equal(handoff.phase(), "failed"); + assert.deepEqual(delivered, []); +}); + +test("retention handoff fails closed on overflow, cross-partition scope, and missing stable identity", async () => { + let overflowLive: any = null; + const overflow = createWorkbenchKafkaRefreshHandoff({ + sessionId: SESSION_ID, + liveBufferLimit: 1, + subscribeLive(listener: any) { overflowLive = listener; return () => {}; }, + async queryRetention() { + overflowLive(record(3).value, transport(3)); + overflowLive(record(4).value, transport(4)); + return queryResult([], [{ partition: 0, startOffset: "0", endOffset: "3" }]); + }, + async deliverEvent() { return true; }, + async deliverConnected() { return true; } + }); + await assert.rejects(overflow.start(), (error: any) => error?.code === "workbench_kafka_refresh_live_buffer_overflow"); + + const multiPartition = createWorkbenchKafkaRefreshHandoff({ + sessionId: SESSION_ID, + liveBufferLimit: 4, + subscribeLive() { return () => {}; }, + async queryRetention() { return queryResult([record(0, 0), record(0, 1)], [{ partition: 0, startOffset: "0", endOffset: "1" }, { partition: 1, startOffset: "0", endOffset: "1" }]); }, + async deliverEvent() { return true; }, + async deliverConnected() { return true; } + }); + await assert.rejects(multiPartition.start(), (error: any) => error?.code === "workbench_kafka_refresh_multi_partition"); + + const missingIdentityRecord = record(0); + delete missingIdentityRecord.value.eventId; + delete missingIdentityRecord.value.sourceEventId; + const missingIdentity = createWorkbenchKafkaRefreshHandoff({ + sessionId: SESSION_ID, + liveBufferLimit: 4, + subscribeLive() { return () => {}; }, + async queryRetention() { return queryResult([missingIdentityRecord], [{ partition: 0, startOffset: "0", endOffset: "1" }]); }, + async deliverEvent() { return true; }, + async deliverConnected() { return true; } + }); + await assert.rejects(missingIdentity.start(), (error: any) => error?.code === "workbench_kafka_refresh_stable_identity_missing"); +}); + +test("multi-partition topic barrier succeeds when the authorized scope remains on one partition", async () => { + const delivered: string[] = []; + const handoff = createWorkbenchKafkaRefreshHandoff({ + sessionId: SESSION_ID, + liveBufferLimit: 4, + subscribeLive() { return () => {}; }, + async queryRetention() { + return queryResult([record(4, 1)], [ + { partition: 0, startOffset: "3", endOffset: "3" }, + { partition: 1, startOffset: "4", endOffset: "5" } + ]); + }, + async deliverEvent(_envelope: any, kafkaTransport: any) { delivered.push(`${kafkaTransport.partition}:${kafkaTransport.offset}`); return true; }, + async deliverConnected() { return true; } + }); + + const summary = await handoff.start(); + assert.deepEqual(delivered, ["1:4"]); + assert.deepEqual(summary.topicPartitions, [1]); + assert.deepEqual(summary.barrier, [{ partition: 0, endOffset: "3" }, { partition: 1, endOffset: "5" }]); +}); + +test("stable eventId and sourceEventId are independently conflict checked", async () => { + const first = record(0); + const conflicting = record(1); + conflicting.value.eventId = first.value.eventId; + const handoff = createWorkbenchKafkaRefreshHandoff({ + sessionId: SESSION_ID, + liveBufferLimit: 4, + subscribeLive() { return () => {}; }, + async queryRetention() { return queryResult([first, conflicting], [{ partition: 0, startOffset: "0", endOffset: "2" }]); }, + async deliverEvent() { return true; }, + async deliverConnected() { return true; } + }); + await assert.rejects(handoff.start(), (error: any) => error?.code === "workbench_kafka_refresh_stable_identity_conflict"); +}); + +test("session-empty refresh is legal while trace-empty refresh is typed fail-closed", async () => { + let sessionLive: any = null; + const sessionOutput: string[] = []; + const sessionHandoff = createWorkbenchKafkaRefreshHandoff({ + sessionId: SESSION_ID, + liveBufferLimit: 4, + subscribeLive(listener: any) { sessionLive = listener; return () => {}; }, + async queryRetention() { return queryResult([], [{ partition: 0, startOffset: "7", endOffset: "7" }, { partition: 1, startOffset: "10", endOffset: "10" }]); }, + async deliverEvent(_envelope: any, kafkaTransport: any, delivery: string) { sessionOutput.push(`${delivery}:${kafkaTransport.offset}`); return true; }, + async deliverConnected() { sessionOutput.push("connected"); return true; } + }); + const summary = await sessionHandoff.start(); + assert.equal(summary.counts.matched, 0); + assert.deepEqual(sessionOutput, ["connected"]); + sessionLive(record(10, 1).value, transport(10, 1)); + await waitFor(() => sessionOutput.includes("live:10")); + + const traceHandoff = createWorkbenchKafkaRefreshHandoff({ + traceId: TRACE_ID, + liveBufferLimit: 4, + subscribeLive() { return () => {}; }, + async queryRetention() { return queryResult([], [{ partition: 0, startOffset: "7", endOffset: "7" }]); }, + async deliverEvent() { return true; }, + async deliverConnected() { return true; } + }); + await assert.rejects(traceHandoff.start(), (error: any) => error?.code === "workbench_kafka_refresh_trace_empty"); +}); + +test("post-handoff partition failure is reported through a typed session-scoped workbench error", async () => { + let publishLive: any = null; + let failure: any = null; + const handoff = createWorkbenchKafkaRefreshHandoff({ + sessionId: SESSION_ID, + liveBufferLimit: 4, + subscribeLive(listener: any) { publishLive = listener; return () => {}; }, + async queryRetention() { return queryResult([record(0)], [{ partition: 0, startOffset: "0", endOffset: "1" }]); }, + async deliverEvent() { return true; }, + async deliverConnected() { return true; }, + async onFailure(error: any) { failure = error; } + }); + await handoff.start(); + publishLive(record(1, 1).value, transport(1, 1)); + await waitFor(() => Boolean(failure)); + const payload = workbenchKafkaRefreshErrorPayload(failure, { sessionId: SESSION_ID }); + assert.equal(payload.sessionId, SESSION_ID); + assert.equal(payload.traceId, null); + assert.equal(payload.phase, "live"); + assert.equal(payload.error.code, "workbench_kafka_refresh_multi_partition"); + assert.deepEqual(payload.error.details, { expectedPartition: 0, actualPartition: 1 }); + assert.equal(payload.fallback, false); +}); + +test("typed refresh failure exposes only bounded safe scan diagnostics", async () => { + const handoff = createWorkbenchKafkaRefreshHandoff({ + traceId: TRACE_ID, + liveBufferLimit: 4, + subscribeLive() { return () => {}; }, + async queryRetention() { + return { + topic: "hwlab.event.v1", + events: [], + completionReason: "timeout", + completion: { reason: "timeout", complete: false, barrierReached: false, retentionStartVerified: true }, + reachedEndOffsets: false, + endOffsetsAvailable: true, + endOffsets: [{ partition: 3, startOffset: "4", endOffset: "9" }], + limit: 50, + scanLimit: 500, + timeoutMs: 2500, + scannedCount: 17, + matchedCount: 2 + }; + }, + async deliverEvent() { return true; }, + async deliverConnected() { return true; } + }); + let failure: any = null; + await assert.rejects(handoff.start(), (error: any) => { + failure = error; + return error?.code === "workbench_kafka_refresh_timeout"; + }); + failure.details.untrustedPayload = "must-not-leak"; + const payload = workbenchKafkaRefreshErrorPayload(failure, { traceId: TRACE_ID }); + assert.deepEqual(payload.error.details, { + completionReason: "timeout", + matchedEventLimit: 50, + scanLimit: 500, + timeoutMs: 2500, + scannedCount: 17, + matchedCount: 2, + partitions: [3] + }); + assert.equal(JSON.stringify(payload).includes("must-not-leak"), false); +}); + +test("post-handoff topic drift is rejected for the lifetime of the retention proof", async () => { + let publishLive: any = null; + let failure: any = null; + const handoff = createWorkbenchKafkaRefreshHandoff({ + sessionId: SESSION_ID, + liveBufferLimit: 4, + subscribeLive(listener: any) { publishLive = listener; return () => {}; }, + async queryRetention() { return queryResult([], [{ partition: 0, startOffset: "0", endOffset: "1" }]); }, + async deliverEvent() { return true; }, + async deliverConnected() { return true; }, + async onFailure(error: any) { failure = error; } + }); + await handoff.start(); + publishLive(record(1).value, { ...transport(1), topic: "hwlab.event.other.v1" }); + await waitFor(() => Boolean(failure)); + assert.equal(failure.code, "workbench_kafka_refresh_transport_topic_mismatch"); + assert.equal(failure.phase, "live"); + assert.equal(handoff.phase(), "failed"); +}); + +function queryResult(events: any[], endOffsets: any[]) { + return { + topic: "hwlab.event.v1", + events, + completionReason: "end-offset", + reachedEndOffsets: true, + endOffsetsAvailable: true, + endOffsets, + completion: { reason: "end-offset", complete: true, barrierReached: true, retentionStartVerified: true } + }; +} + +function record(offset: number, partition = 0) { + return { ...transport(offset, partition), value: envelope(offset, partition) }; +} + +function transport(offset: number, partition = 0) { + return { topic: "hwlab.event.v1", partition, offset: String(offset) }; +} + +function envelope(offset: number, partition = 0) { + const eventId = `evt_${partition}_${offset}`; + return { + schema: "hwlab.event.v1", + eventType: "hwlab.trace.event.projected", + eventId: `hwlab:${eventId}`, + sourceEventId: eventId, + traceId: TRACE_ID, + hwlabSessionId: SESSION_ID, + sessionId: SESSION_ID, + event: { type: "tool", eventType: "tool", traceId: TRACE_ID, sessionId: SESSION_ID, sourceEventId: eventId } + }; +} + +async function waitFor(predicate: () => boolean, timeoutMs = 250) { + const deadline = Date.now() + timeoutMs; + while (!predicate()) { + if (Date.now() >= deadline) throw new Error("condition timeout"); + await new Promise((resolve) => setTimeout(resolve, 1)); + } +} diff --git a/internal/cloud/workbench-kafka-refresh-handoff.ts b/internal/cloud/workbench-kafka-refresh-handoff.ts new file mode 100644 index 00000000..316d04e9 --- /dev/null +++ b/internal/cloud/workbench-kafka-refresh-handoff.ts @@ -0,0 +1,462 @@ +// SPEC: PJ2026-0104010803 Workbench Kafka retention refresh handoff. +// Responsibility: prove one bounded retained hwlab.event.v1 scan before handing the same SSE writer to shared live fanout. + +export function createWorkbenchKafkaRefreshHandoff(options = {}) { + const sessionId = textValue(options.sessionId); + const traceId = textValue(options.traceId); + const liveBufferLimit = positiveInteger(options.liveBufferLimit); + if (!sessionId && !traceId) throw handoffError("workbench_kafka_refresh_scope_missing", "Kafka refresh handoff requires a session or trace scope.", "idle"); + if (!liveBufferLimit) throw handoffError("workbench_kafka_refresh_buffer_limit_invalid", "Kafka refresh handoff requires an explicit positive live buffer limit.", "idle"); + if (typeof options.subscribeLive !== "function" || typeof options.queryRetention !== "function" || typeof options.deliverEvent !== "function" || typeof options.deliverConnected !== "function") { + throw handoffError("workbench_kafka_refresh_runtime_invalid", "Kafka refresh handoff runtime callbacks are incomplete.", "idle"); + } + + const bufferedLive = []; + const barrierByPartition = new Map(); + const replayProofByTransport = new Map(); + const stableIdentityByTransport = new Map(); + const seenEventIds = new Set(); + const seenSourceEventIds = new Set(); + const sourceEventIdByEventId = new Map(); + const eventIdBySourceEventId = new Map(); + const abortController = new AbortController(); + let phase = "idle"; + let unsubscribe = null; + let failure = null; + let bootstrapSettled = false; + let scopedTopic = null; + let scopedPartition = null; + let deliveryChain = Promise.resolve(true); + let pendingLiveDeliveries = 0; + const counts = { + matched: 0, + replayed: 0, + buffered: 0, + bufferedDelivered: 0, + liveDelivered: 0, + deduplicated: 0 + }; + + function stop(reason = "connection-closed") { + if (phase === "closed") return; + phase = "closed"; + abortController.abort(reason); + unsubscribeLive(); + bufferedLive.length = 0; + } + + async function start() { + if (phase !== "idle") throw handoffError("workbench_kafka_refresh_already_started", "Kafka refresh handoff can only be started once.", phase); + phase = "buffering"; + unsubscribe = options.subscribeLive(onLiveEnvelope); + try { + const queryResult = await options.queryRetention({ signal: abortController.signal }); + if (failure) throw failure; + if (phase === "closed") throw handoffError("workbench_kafka_refresh_stopped", "Kafka refresh handoff stopped before the retention barrier completed.", phase); + validateQueryCompletion(queryResult); + installBarrier(queryResult); + const replayRecords = validateReplayRecords(queryResult); + + phase = "replaying"; + for (const record of replayRecords) { + if (failure) throw failure; + if (!record.deliver) continue; + await deliverSerial(record.envelope, record.transport, "replay"); + counts.replayed += 1; + } + + phase = "flushing"; + while (bufferedLive.length > 0) { + if (failure) throw failure; + const item = bufferedLive.shift(); + const decision = classifyLiveRecord(item.envelope, item.transport); + if (!decision.deliver) continue; + await deliverSerial(decision.envelope, decision.transport, "buffered-live"); + counts.bufferedDelivered += 1; + } + + const summary = handoffSummary(); + const connectedDelivery = deliverConnectedSerial(summary); + phase = "live"; + await connectedDelivery; + if (failure) throw failure; + bootstrapSettled = true; + return summary; + } catch (error) { + const typed = asHandoffError(error, phase); + fail(typed); + throw typed; + } + } + + function onLiveEnvelope(envelope, transport) { + if (phase === "closed" || phase === "failed" || !envelopeMatchesScope(envelope, sessionId, traceId)) return; + if (phase !== "live") { + if (bufferedLive.length >= liveBufferLimit) { + fail(handoffError("workbench_kafka_refresh_live_buffer_overflow", "Kafka refresh live buffer exceeded its YAML-owned capacity before handoff.", phase, { liveBufferLimit })); + return; + } + bufferedLive.push({ envelope, transport }); + counts.buffered += 1; + return; + } + let decision; + try { + decision = classifyLiveRecord(envelope, transport); + } catch (error) { + fail(asHandoffError(error, phase)); + return; + } + if (!decision.deliver) return; + if (pendingLiveDeliveries >= liveBufferLimit) { + fail(handoffError("workbench_kafka_refresh_live_buffer_overflow", "Kafka refresh live delivery queue exceeded its YAML-owned capacity after handoff.", phase, { liveBufferLimit })); + return; + } + pendingLiveDeliveries += 1; + void deliverSerial(decision.envelope, decision.transport, "live") + .then(() => { counts.liveDelivered += 1; }) + .catch((error) => fail(asHandoffError(error, phase))) + .finally(() => { pendingLiveDeliveries = Math.max(0, pendingLiveDeliveries - 1); }); + } + + function validateQueryCompletion(result) { + const reason = textValue(result?.completion?.reason ?? result?.completionReason) || "unknown"; + const complete = result?.completion?.complete === true + && result?.completion?.barrierReached === true + && result?.completion?.retentionStartVerified === true + && result?.reachedEndOffsets === true; + if (!complete) { + const codeByReason = { + timeout: "workbench_kafka_refresh_timeout", + "scan-limit": "workbench_kafka_refresh_scan_limit", + limit: "workbench_kafka_refresh_matched_event_limit", + aborted: "workbench_kafka_refresh_aborted", + "retention-start-moved": "workbench_kafka_refresh_retention_start_moved", + "retention-start-unavailable": "workbench_kafka_refresh_retention_start_unavailable", + "retention-start-unverified": "workbench_kafka_refresh_retention_start_unverified" + }; + throw handoffError(codeByReason[reason] ?? "workbench_kafka_refresh_barrier_incomplete", `Kafka retention scan did not prove its captured barrier (${reason}).`, phase, { + completionReason: reason, + matchedEventLimit: result?.limit, + scanLimit: result?.scanLimit, + timeoutMs: result?.timeoutMs, + scannedCount: result?.scannedCount, + matchedCount: result?.matchedCount, + partitions: Array.isArray(result?.endOffsets) ? result.endOffsets.map((entry) => entry?.partition) : [] + }); + } + if (result?.endOffsetsAvailable !== true || !Array.isArray(result?.endOffsets) || result.endOffsets.length === 0) { + throw handoffError("workbench_kafka_refresh_barrier_missing", "Kafka retention scan completed without a usable end-offset barrier.", phase); + } + } + + function installBarrier(result) { + scopedTopic = textValue(result?.topic); + if (!scopedTopic) { + throw handoffError("workbench_kafka_refresh_transport_identity_missing", "Kafka retention barrier is missing the current HWLAB topic identity.", phase); + } + barrierByPartition.clear(); + for (const entry of result.endOffsets) { + const partition = partitionValue(entry?.partition); + const endOffset = offsetValue(entry?.endOffset); + if (partition === null || endOffset === null || barrierByPartition.has(partition)) { + throw handoffError("workbench_kafka_refresh_barrier_invalid", "Kafka retention barrier contains an invalid or duplicate partition entry.", phase); + } + barrierByPartition.set(partition, endOffset); + } + } + + function validateReplayRecords(result) { + const records = Array.isArray(result?.events) ? result.events : []; + counts.matched = records.length; + if (traceId && records.length === 0) { + throw handoffError("workbench_kafka_refresh_trace_empty", "Trace-scoped Kafka refresh found no retained events after a complete scan.", phase); + } + const replayPartitions = new Set(); + let previousOffset = null; + const normalized = []; + for (const record of records) { + const envelope = record?.value; + if (!envelopeMatchesScope(envelope, sessionId, traceId) || envelope?.schema !== "hwlab.event.v1") { + throw handoffError("workbench_kafka_refresh_scope_mismatch", "Kafka retention query returned an envelope outside the authorized scope.", phase); + } + const transport = requireTransportIdentity(record, scopedTopic, phase); + const stableIdentity = requireStableIdentity(envelope, phase); + const stableKey = stableIdentityKey(stableIdentity); + const barrier = barrierByPartition.get(transport.partition); + if (barrier === undefined || compareOffsets(transport.offset, barrier) >= 0) { + throw handoffError("workbench_kafka_refresh_post_barrier_record", "Kafka retention query returned a record at or beyond its exclusive barrier.", phase); + } + replayPartitions.add(transport.partition); + if (replayPartitions.size > 1) throw handoffError("workbench_kafka_refresh_multi_partition", "One authorized Kafka refresh scope spans multiple HWLAB topic partitions.", phase); + if (previousOffset !== null && compareOffsets(transport.offset, previousOffset) <= 0) { + throw handoffError("workbench_kafka_refresh_order_invalid", "Kafka retention records are not strictly ordered within the authorized partition.", phase); + } + previousOffset = transport.offset; + scopedPartition = transport.partition; + const transportKey = transportIdentityKey(transport); + const priorProof = replayProofByTransport.get(transportKey); + if (priorProof && priorProof !== stableKey) throw handoffError("workbench_kafka_refresh_transport_conflict", "One Kafka transport identity maps to conflicting stable events.", phase); + replayProofByTransport.set(transportKey, stableKey); + const deliver = rememberIdentity(transportKey, stableIdentity, phase); + normalized.push({ envelope, transport, deliver }); + } + return normalized; + } + + function classifyLiveRecord(envelope, rawTransport) { + if (envelope?.schema !== "hwlab.event.v1" || !envelopeMatchesScope(envelope, sessionId, traceId)) { + throw handoffError("workbench_kafka_refresh_scope_mismatch", "Shared live fanout delivered an envelope outside the authorized scope.", phase); + } + const transport = requireTransportIdentity(rawTransport, scopedTopic, phase); + const stableIdentity = requireStableIdentity(envelope, phase); + const stableKey = stableIdentityKey(stableIdentity); + if (scopedPartition !== null && transport.partition !== scopedPartition) { + throw handoffError("workbench_kafka_refresh_multi_partition", "One authorized Kafka refresh scope spans multiple HWLAB topic partitions.", phase, { + expectedPartition: scopedPartition, + actualPartition: transport.partition + }); + } + const barrier = barrierByPartition.get(transport.partition); + if (barrier === undefined) throw handoffError("workbench_kafka_refresh_barrier_partition_missing", "Shared live fanout used a partition absent from the captured barrier.", phase, { partition: transport.partition }); + if (scopedPartition === null) scopedPartition = transport.partition; + const transportKey = transportIdentityKey(transport); + if (compareOffsets(transport.offset, barrier) < 0) { + const replayProof = replayProofByTransport.get(transportKey); + if (replayProof !== stableKey) { + throw handoffError("workbench_kafka_refresh_barrier_gap", "A pre-barrier shared live record was not proven by the retained replay scan.", phase, { partition: transport.partition }); + } + } + return { envelope, transport, deliver: rememberIdentity(transportKey, stableIdentity, phase) }; + } + + function rememberIdentity(transportKey, stableIdentity, currentPhase) { + const stableKey = stableIdentityKey(stableIdentity); + const priorStable = stableIdentityByTransport.get(transportKey); + if (priorStable && priorStable !== stableKey) { + throw handoffError("workbench_kafka_refresh_transport_conflict", "One Kafka transport identity maps to conflicting stable events.", currentPhase); + } + const priorSourceEventId = stableIdentity.eventId ? sourceEventIdByEventId.get(stableIdentity.eventId) : null; + if (priorSourceEventId && stableIdentity.sourceEventId && priorSourceEventId !== stableIdentity.sourceEventId) { + throw handoffError("workbench_kafka_refresh_stable_identity_conflict", "One outer eventId maps to conflicting sourceEventId identities.", currentPhase); + } + const priorEventId = stableIdentity.sourceEventId ? eventIdBySourceEventId.get(stableIdentity.sourceEventId) : null; + if (priorEventId && stableIdentity.eventId && priorEventId !== stableIdentity.eventId) { + throw handoffError("workbench_kafka_refresh_stable_identity_conflict", "One outer sourceEventId maps to conflicting eventId identities.", currentPhase); + } + const duplicate = Boolean( + (stableIdentity.eventId && seenEventIds.has(stableIdentity.eventId)) + || (stableIdentity.sourceEventId && seenSourceEventIds.has(stableIdentity.sourceEventId)) + ); + stableIdentityByTransport.set(transportKey, stableKey); + if (stableIdentity.eventId) { + seenEventIds.add(stableIdentity.eventId); + if (stableIdentity.sourceEventId) sourceEventIdByEventId.set(stableIdentity.eventId, stableIdentity.sourceEventId); + } + if (stableIdentity.sourceEventId) { + seenSourceEventIds.add(stableIdentity.sourceEventId); + if (stableIdentity.eventId) eventIdBySourceEventId.set(stableIdentity.sourceEventId, stableIdentity.eventId); + } + if (duplicate) { + counts.deduplicated += 1; + return false; + } + return true; + } + + function deliverSerial(envelope, transport, delivery) { + const run = async () => { + if (failure || phase === "failed" || phase === "closed") throw failure ?? handoffError("workbench_kafka_refresh_stopped", "Kafka refresh delivery stopped before the queued business event could be written.", phase); + const written = await options.deliverEvent(envelope, transport, delivery); + if (written === false) throw handoffError("workbench_kafka_refresh_sse_write_failed", "Kafka refresh business event could not be written to SSE.", phase); + return true; + }; + deliveryChain = deliveryChain.then(run); + return deliveryChain; + } + + function deliverConnectedSerial(summary) { + const run = async () => { + if (failure || phase === "failed" || phase === "closed") throw failure ?? handoffError("workbench_kafka_refresh_stopped", "Kafka refresh delivery stopped before the connected contract could be written.", phase); + const written = await options.deliverConnected(summary); + if (written === false) throw handoffError("workbench_kafka_refresh_sse_write_failed", "Kafka refresh connected contract could not be written to SSE.", phase); + return true; + }; + deliveryChain = deliveryChain.then(run); + return deliveryChain; + } + + function handoffSummary() { + return { + phase: "live", + topic: scopedTopic, + topicPartitions: scopedPartition === null ? [] : [scopedPartition], + barrier: [...barrierByPartition.entries()].map(([partition, endOffset]) => ({ partition, endOffset })).sort((left, right) => left.partition - right.partition), + counts: { ...counts }, + valuesRedacted: true + }; + } + + function fail(error) { + if (failure || phase === "closed") return; + failure = asHandoffError(error, phase); + phase = "failed"; + abortController.abort(failure.code); + unsubscribeLive(); + bufferedLive.length = 0; + if (bootstrapSettled && typeof options.onFailure === "function") { + void Promise.resolve(options.onFailure(failure)).catch(() => undefined); + } + } + + function unsubscribeLive() { + if (typeof unsubscribe !== "function") return; + const stop = unsubscribe; + unsubscribe = null; + try { stop(); } catch {} + } + + return { + start, + stop, + phase: () => phase, + summary: handoffSummary, + valuesPrinted: false + }; +} + +export function workbenchKafkaRefreshErrorPayload(error, scope = {}) { + const typed = asHandoffError(error, textValue(error?.phase) || "failed"); + const details = safeHandoffDiagnosticDetails(typed.details); + return { + type: "error", + status: "blocked", + phase: typed.phase, + traceId: textValue(scope.traceId), + sessionId: textValue(scope.sessionId), + error: { + code: typed.code, + message: typed.message, + phase: typed.phase, + retryable: true, + details, + valuesRedacted: true + }, + fallback: false, + valuesRedacted: true + }; +} + +function safeHandoffDiagnosticDetails(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const details = {}; + const safeTextKeys = ["completionReason", "expectedTopic", "actualTopic"]; + const safeIntegerKeys = [ + "matchedEventLimit", + "scanLimit", + "timeoutMs", + "scannedCount", + "matchedCount", + "liveBufferLimit", + "partition", + "expectedPartition", + "actualPartition" + ]; + for (const key of safeTextKeys) { + const text = textValue(value[key]); + if (text) details[key] = text.slice(0, 160); + } + for (const key of safeIntegerKeys) { + const number = Number(value[key]); + if (Number.isSafeInteger(number) && number >= 0) details[key] = number; + } + if (Array.isArray(value.partitions)) { + details.partitions = [...new Set(value.partitions + .map((partition) => partitionValue(partition)) + .filter((partition) => partition !== null))] + .slice(0, 32); + } + return Object.keys(details).length > 0 ? details : null; +} + +function envelopeMatchesScope(envelope, sessionId, traceId) { + if (!envelope || typeof envelope !== "object" || Array.isArray(envelope)) return false; + const envelopeSessionId = textValue(envelope.hwlabSessionId ?? envelope.sessionId); + const envelopeTraceId = textValue(envelope.traceId); + if (sessionId && envelopeSessionId !== sessionId) return false; + if (traceId && envelopeTraceId !== traceId) return false; + return true; +} + +function requireTransportIdentity(value, expectedTopic, phase) { + const topic = textValue(value?.topic); + const partition = partitionValue(value?.partition); + const offset = offsetValue(value?.offset); + if (!topic || partition === null || offset === null) { + throw handoffError("workbench_kafka_refresh_transport_identity_missing", "Kafka refresh requires the current HWLAB topic/partition/offset identity.", phase); + } + if (expectedTopic && topic !== expectedTopic) { + throw handoffError("workbench_kafka_refresh_transport_topic_mismatch", "Kafka refresh received a live record from a topic other than the captured HWLAB retention topic.", phase, { expectedTopic, actualTopic: topic }); + } + return { topic, partition, offset }; +} + +function requireStableIdentity(envelope, phase) { + const eventId = textValue(envelope?.eventId); + const sourceEventId = textValue(envelope?.sourceEventId); + if (!eventId && !sourceEventId) throw handoffError("workbench_kafka_refresh_stable_identity_missing", "Kafka refresh requires outer eventId or sourceEventId identity.", phase); + return { eventId, sourceEventId }; +} + +function stableIdentityKey(identity) { + return `event:${identity.eventId ?? ""}|source:${identity.sourceEventId ?? ""}`; +} + +function transportIdentityKey(transport) { + return `${transport.topic}:${transport.partition}:${transport.offset}`; +} + +function partitionValue(value) { + const number = Number(value); + return Number.isInteger(number) && number >= 0 ? number : null; +} + +function offsetValue(value) { + const text = textValue(value); + if (!text || !/^\d+$/u.test(text)) return null; + try { + return BigInt(text) >= 0n ? text : null; + } catch { + return null; + } +} + +function compareOffsets(left, right) { + const leftValue = BigInt(left); + const rightValue = BigInt(right); + return leftValue < rightValue ? -1 : leftValue > rightValue ? 1 : 0; +} + +function positiveInteger(value) { + const number = Number(value); + return Number.isInteger(number) && number > 0 ? number : null; +} + +function textValue(value) { + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +function asHandoffError(error, phase) { + if (error?.code && error?.phase) return error; + return handoffError(error?.code ?? "workbench_kafka_refresh_failed", error instanceof Error ? error.message : String(error ?? "Kafka refresh handoff failed."), phase); +} + +function handoffError(code, message, phase, details = null) { + return Object.assign(new Error(message), { + code, + phase, + details, + statusCode: 503, + valuesRedacted: true + }); +} diff --git a/internal/cloud/workbench-realtime-authority-contract.test.ts b/internal/cloud/workbench-realtime-authority-contract.test.ts index a98aac3d..6a8123bb 100644 --- a/internal/cloud/workbench-realtime-authority-contract.test.ts +++ b/internal/cloud/workbench-realtime-authority-contract.test.ts @@ -10,6 +10,7 @@ const ACTOR = { id: "usr_workbench_realtime_authority", username: "reader", disp const PROJECTION_REALTIME_ENV = Object.freeze({ HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false", HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false", HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true" @@ -17,6 +18,7 @@ const PROJECTION_REALTIME_ENV = Object.freeze({ const LIVE_REALTIME_ENV = Object.freeze({ HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false", HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" diff --git a/internal/cloud/workbench-realtime-capabilities.ts b/internal/cloud/workbench-realtime-capabilities.ts index c62e3624..8d87d200 100644 --- a/internal/cloud/workbench-realtime-capabilities.ts +++ b/internal/cloud/workbench-realtime-capabilities.ts @@ -4,6 +4,7 @@ export const WORKBENCH_REALTIME_CAPABILITY_ENVS = Object.freeze({ directPublish: "HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED", liveKafkaSse: "HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED", + kafkaRefreshReplay: "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED", transactionalProjector: "HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED", projectionOutboxRelay: "HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED", projectionRealtime: "HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED" diff --git a/internal/dev-entrypoint/cloud-web-runtime.mjs b/internal/dev-entrypoint/cloud-web-runtime.mjs index 224d6ca4..a143a5bf 100644 --- a/internal/dev-entrypoint/cloud-web-runtime.mjs +++ b/internal/dev-entrypoint/cloud-web-runtime.mjs @@ -393,6 +393,7 @@ function workbenchRuntimeConfigFromEnv() { const result = { realtimeFeatures: { liveKafkaSse: requiredWorkbenchRealtimeFeature(process.env.HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED, "HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED"), + kafkaRefreshReplay: requiredWorkbenchRealtimeFeature(process.env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED, "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED"), projectionRealtime: requiredWorkbenchRealtimeFeature(process.env.HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED, "HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED") }, debugCapabilities: { diff --git a/internal/dev-entrypoint/cloud-web-runtime.test.mjs b/internal/dev-entrypoint/cloud-web-runtime.test.mjs index afd75ac9..e7713ab0 100644 --- a/internal/dev-entrypoint/cloud-web-runtime.test.mjs +++ b/internal/dev-entrypoint/cloud-web-runtime.test.mjs @@ -431,6 +431,7 @@ test("cloud web serves client deep links through the Vue shell", async () => { HWLAB_CLOUD_WEB_DISPLAY_TIME_LOCALE: "zh-CN", HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: "北京时间", HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "true", HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false", HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_ENABLED: "true", HWLAB_WORKBENCH_RAW_HWLAB_EVENT_WINDOW_ENABLED: "true", @@ -466,6 +467,7 @@ test("cloud web serves client deep links through the Vue shell", async () => { const html = await response.text(); assert.match(html, /
<\/div>/u, route); assert.match(html, /HWLAB_CLOUD_WEB_CONFIG/u, route); + assert.match(html, /"realtimeFeatures":\{"liveKafkaSse":true,"kafkaRefreshReplay":true,"projectionRealtime":false\}/u, route); assert.match(html, /"debugCapabilities":\{"isolatedKafka":true,"rawHwlabEventWindow":\{"enabled":true,"maxEntries":200,"maxRetainedBytes":1048576\}\}/u, route); } @@ -493,6 +495,7 @@ test("cloud web injects trace explorer runtime config from env", async () => { HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: "北京时间", HWLAB_WORKBENCH_TRACE_EXPLORER_URL_TEMPLATE: "/v1/workbench/traces/{trace_id}/events", HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false", HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false", HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_ENABLED: "true", HWLAB_WORKBENCH_RAW_HWLAB_EVENT_WINDOW_ENABLED: "true", @@ -603,6 +606,7 @@ test("cloud web OpenCode frame-url endpoint mints traced tickets", async () => { HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: "北京时间", HWLAB_CLOUD_WEB_OPENCODE_URL: "https://opencode.example.test", HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false", HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false", HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_ENABLED: "true", HWLAB_WORKBENCH_RAW_HWLAB_EVENT_WINDOW_ENABLED: "true", @@ -1049,6 +1053,7 @@ test("cloud web OpenCode proxy accepts short-lived tickets minted by the shell", HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: "北京时间", HWLAB_CLOUD_WEB_OPENCODE_URL: "https://opencode.example.test", HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false", HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false", HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_ENABLED: "true", HWLAB_WORKBENCH_RAW_HWLAB_EVENT_WINDOW_ENABLED: "true", diff --git a/scripts/runtime-realtime-capabilities-config.test.mjs b/scripts/runtime-realtime-capabilities-config.test.mjs index 77c0ed62..51e9da48 100644 --- a/scripts/runtime-realtime-capabilities-config.test.mjs +++ b/scripts/runtime-realtime-capabilities-config.test.mjs @@ -20,6 +20,15 @@ const LIVE_KAFKA_CONTRACT = Object.freeze({ HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID: "hwlab-v03-workbench-live-sse" }); +const UNIDESK_OWNED_REFRESH_REPLAY_ENV = Object.freeze([ + "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED", + "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_GROUP_PREFIX", + "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS", + "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT", + "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_MATCHED_EVENT_LIMIT", + "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_LIVE_BUFFER_LIMIT" +]); + const ISOLATED_DEBUG_CONTRACT = Object.freeze({ HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_ENABLED: "true", HWLAB_KAFKA_HWLAB_DEBUG_EVENT_TOPIC: "hwlab.event.debug.v1", @@ -41,6 +50,9 @@ test("v03 YAML keeps pure Kafka realtime capabilities independently composable", for (const [name, expected] of Object.entries(LIVE_KAFKA_CONTRACT)) { assert.equal(cloudApi.env?.[name], expected, `${name} must preserve the pure Kafka live contract`); } + for (const name of UNIDESK_OWNED_REFRESH_REPLAY_ENV) { + assert.equal(cloudApi.env?.[name], undefined, `${name} is injected from UniDesk config/hwlab-node-lanes.yaml and must not have a second value in HWLAB deploy.yaml`); + } for (const [name, expected] of Object.entries(ISOLATED_DEBUG_CONTRACT)) { assert.equal(cloudApi.env?.[name], expected, `${name} must be independently YAML-owned for debug only`); } @@ -48,6 +60,7 @@ test("v03 YAML keeps pure Kafka realtime capabilities independently composable", assert.equal(cloudApi.env?.HWLAB_SESSION_COOKIE_DOMAIN, undefined, "internal and public origins require host-scoped session cookies"); assert.notEqual(cloudApi.env?.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID, "hwlab-v03-agentrun-event-bridge"); assert.equal(cloudWeb.env?.HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED, "true"); + assert.equal(cloudWeb.env?.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED, undefined, "UniDesk renderer owns the Web refresh capability value"); assert.equal(cloudWeb.env?.HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED, "false"); assert.equal(cloudWeb.env?.HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_ENABLED, "true"); }); diff --git a/tools/hwlab-cli/kafka-regenerate.test.ts b/tools/hwlab-cli/kafka-regenerate.test.ts index 0b0ddce8..fb63e466 100644 --- a/tools/hwlab-cli/kafka-regenerate.test.ts +++ b/tools/hwlab-cli/kafka-regenerate.test.ts @@ -28,12 +28,13 @@ test("offline HWLAB JSONL uses the shared Web pipeline and renders final respons const outputFile = path.join(cwd, "trace.md"); const finalText = "Hi. What do you want to work on?"; const records = [ - hwlabRecord(1, { type: "tool", eventType: "tool", status: "started", label: "agentrun:tool:commandExecution", toolName: "commandExecution", itemId: "tool_rg", command: "/bin/bash -lc 'cd /workspace && rg -n target src'" }), - hwlabRecord(2, { type: "tool", eventType: "tool", status: "completed", label: "agentrun:tool:commandExecution", toolName: "commandExecution", itemId: "tool_rg", command: "/bin/bash -lc 'cd /workspace && rg -n target src'", durationMs: 1234, exitCode: 0, stdoutSummary: "src/main.ts:42:target" }), - hwlabRecord(3, { type: "assistant", eventType: "assistant", status: "running", label: "agentrun:assistant:message", itemId: "assistant_progress", message: "I am checking the workspace.", assistantText: "I am checking the workspace.", assistantSource: "agent-message" }), - hwlabRecord(4, { type: "assistant", eventType: "assistant", status: "running", label: "agentrun:assistant:message", itemId: "assistant_final", message: finalText, assistantText: finalText, assistantSource: "completed-agent-message" }), - hwlabRecord(5, { type: "assistant", eventType: "assistant", status: "running", label: "agentrun:assistant:message", itemId: "assistant_final", message: finalText, assistantText: finalText, assistantSource: "completed-agent-message-final", replyAuthority: true, final: true, finalResponse: { text: finalText } }), - hwlabRecord(6, { type: "result", eventType: "terminal", status: "completed", label: "agentrun:terminal:completed", terminal: true, message: "AgentRun completed" }) + hwlabRecord(1, { type: "user", eventType: "user", status: "sent", label: "agentrun:user:message", userMessageId: "msg_render_trace_user", messageId: "msg_render_trace_user", targetTraceId: "trc_render_trace_target", text: "hi" }), + hwlabRecord(2, { type: "tool", eventType: "tool", status: "started", label: "agentrun:tool:commandExecution", toolName: "commandExecution", itemId: "tool_rg", command: "/bin/bash -lc 'cd /workspace && rg -n target src'" }), + hwlabRecord(3, { type: "tool", eventType: "tool", status: "completed", label: "agentrun:tool:commandExecution", toolName: "commandExecution", itemId: "tool_rg", command: "/bin/bash -lc 'cd /workspace && rg -n target src'", durationMs: 1234, exitCode: 0, stdoutSummary: "src/main.ts:42:target" }), + hwlabRecord(4, { type: "assistant", eventType: "assistant", status: "running", label: "agentrun:assistant:message", itemId: "assistant_progress", message: "I am checking the workspace.", assistantText: "I am checking the workspace.", assistantSource: "agent-message" }), + hwlabRecord(5, { type: "assistant", eventType: "assistant", status: "running", label: "agentrun:assistant:message", itemId: "assistant_final", message: finalText, assistantText: finalText, assistantSource: "completed-agent-message" }), + hwlabRecord(6, { type: "assistant", eventType: "assistant", status: "running", label: "agentrun:assistant:message", itemId: "assistant_final", message: finalText, assistantText: finalText, assistantSource: "completed-agent-message-final", replyAuthority: true, final: true, finalResponse: { text: finalText } }), + hwlabRecord(7, { type: "result", eventType: "terminal", status: "completed", label: "agentrun:terminal:completed", terminal: true, message: "AgentRun completed" }) ]; await writeFile(inputFile, records.map((record) => JSON.stringify(record)).join("\n") + "\n", "utf8"); @@ -50,12 +51,16 @@ test("offline HWLAB JSONL uses the shared Web pipeline and renders final respons ], { cwd, env: {}, now: () => "2026-07-10T12:00:00.000Z" }); assert.equal(result.exitCode, 0); - assert.equal(result.payload.input.scannedCount, 6); - assert.equal(result.payload.input.matchedCount, 6); - assert.equal(result.payload.projection.decodedCount, 6); - assert.equal(result.payload.projection.appliedCount, 6); + assert.equal(result.payload.input.scannedCount, 7); + assert.equal(result.payload.input.matchedCount, 7); + assert.equal(result.payload.projection.decodedCount, 7); + assert.equal(result.payload.projection.appliedCount, 7); + assert.equal(result.payload.projection.eventCount, 6, "formal user input must not create an agent trace event"); + assert.equal(result.payload.projection.userMessageCount, 1); + assert.deepEqual(result.payload.projection.userMessageIds, ["msg_render_trace_user"]); + assert.deepEqual(result.payload.projection.userTargetTraceIds, ["trc_render_trace_target"]); assert.equal(result.payload.projection.terminalObserved, true); - assert.equal(result.payload.projection.lastEventAt, "2026-07-10T12:00:00.005Z"); + assert.equal(result.payload.projection.lastEventAt, "2026-07-10T12:00:00.006Z"); assert.deepEqual(result.payload.identity.observedRunIds, [RUN_ID]); assert.deepEqual(result.payload.identity.observedCommandIds, ["cmd_render_trace"]); assert.equal(result.payload.identity.sourceEventIdentityDuplicates, 0); @@ -73,6 +78,9 @@ test("offline HWLAB JSONL uses the shared Web pipeline and renders final respons assert.match(toolRow.preview, /src\/main\.ts:42:target/u); const markdown = await readFile(outputFile, "utf8"); assert.equal((markdown.match(new RegExp(finalText.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"), "gu")) ?? []).length, 1); + assert.equal((markdown.match(/^hi$/gmu) ?? []).length, 1); + assert.match(markdown, /## 用户输入/u); + assert.match(markdown, /target trace: trc_render_trace_target/u); assert.match(markdown, /## 运行记录/u); assert.match(markdown, /## 最终回复/u); assert.equal(result.markdownOutput?.trim(), markdown.trim()); @@ -215,14 +223,13 @@ test("command-scoped trace keeps attributable commandless final and terminal lif now: () => "2026-07-10T12:00:00.000Z", async readKafka(input) { readerInput = input; + const events = [exactTool, exactProgress, unscopedFinal, unscopedTerminal, runnerShutdown, unscopedBackend]; return { - events: [exactTool, exactProgress, unscopedFinal, unscopedTerminal, runnerShutdown, unscopedBackend], + ...completeKafkaRead(events), scannedCount: 6, parsedCount: 6, matchedCount: 6, - filterRejectedCount: 0, - completionReason: "end-offset", - reachedEndOffsets: true + filterRejectedCount: 0 }; } }); @@ -232,6 +239,10 @@ test("command-scoped trace keeps attributable commandless final and terminal lif assert.equal(readerInput.runId, RUN_ID); assert.equal(result.payload.input.queryMatchedCount, 6); assert.equal(result.payload.input.matchedCount, 4); + assert.equal(result.payload.input.refreshHandoff.status, "succeeded"); + assert.equal(result.payload.input.refreshHandoff.liveSubscriptionInstalled, true); + assert.equal(result.payload.input.refreshHandoff.counts.replayed, 6); + assert.deepEqual(result.payload.input.refreshHandoff.barrier, [{ partition: 0, endOffset: "1907" }]); assert.deepEqual(result.payload.input.commandScope.commandIdCounts, [ { commandId: "cmd_render_trace", count: 2 }, { commandId: "runner-shutdown", count: 1 } @@ -243,6 +254,7 @@ test("command-scoped trace keeps attributable commandless final and terminal lif assert.equal(result.payload.input.commandScope.otherCommandRecordCount, 1); assert.equal(result.payload.projection.eventCount, 4); assert.equal(result.payload.projection.terminalObserved, true); + assert.equal(result.payload.projection.pipeline[0], "createWorkbenchKafkaRefreshHandoff"); assert.equal(result.payload.render.finalResponseInTrace, false); assert.equal(result.payload.validation.commandLifecyclePreserved, true); assert.equal((result.markdownOutput?.match(/Hi\. What do you want to work on\?/gu) ?? []).length, 1); @@ -288,14 +300,13 @@ test("missing commandId cannot be satisfied by commandless lifecycle events from env: {}, now: () => "2026-07-10T12:00:00.000Z", async readKafka() { + const events = [otherCommand, commandlessFinal, commandlessTerminal]; return { - events: [otherCommand, commandlessFinal, commandlessTerminal], + ...completeKafkaRead(events), scannedCount: 3, parsedCount: 3, matchedCount: 3, - filterRejectedCount: 0, - completionReason: "end-offset", - reachedEndOffsets: true + filterRejectedCount: 0 }; } }); @@ -340,6 +351,7 @@ test("Kafka Trace render returns a typed partial result when the captured end of reachedEndOffsets: false, endOffsetsAvailable: true, endOffsets: [{ partition: 0, startOffset: "0", endOffset: "2" }], + completion: { reason: "timeout", complete: false, barrierReached: false, retentionStartVerified: true }, lastScannedOffsets: [{ partition: 0, offset: "0" }] }; } @@ -351,6 +363,8 @@ test("Kafka Trace render returns a typed partial result when the captured end of assert.equal(result.payload.error.code, "source_scan_incomplete"); assert.equal(result.payload.input.scanComplete, false); assert.equal(result.payload.input.completionReason, "timeout"); + assert.equal(result.payload.input.refreshHandoff.status, "failed"); + assert.equal(result.payload.input.refreshHandoff.error.code, "workbench_kafka_refresh_timeout"); assert.equal(result.payload.validation.sourceScanComplete, false); assert.equal(result.payload.validation.commandLifecyclePreserved, false); assert.match(result.markdownOutput ?? "", /source scan: partial \(timeout\)/u); @@ -358,6 +372,50 @@ test("Kafka Trace render returns a typed partial result when the captured end of assert.match(renderKafkaCliText(result.payload), /^partial action=kafka\.render\.trace code=source_scan_incomplete completion=timeout reachedEnd=false/u); }); +test("incomplete zero-match Kafka scan stays partial and never claims the trace is missing", async () => { + const cwd = await mkdtemp(path.join(os.tmpdir(), "hwlab-kafka-trace-empty-partial-")); + const result = await runKafkaCli([ + "render", "trace", + "--from", "kafka", + "--trace-id", TRACE_ID, + "--group-prefix", "hwlab-v03-workbench-isolated-debug", + "--format", "markdown" + ], { + cwd, + env: {}, + now: () => "2026-07-10T12:00:00.000Z", + async readKafka() { + return { + events: [], + scannedCount: 0, + parsedCount: 0, + matchedCount: 0, + filterRejectedCount: 0, + completionReason: "timeout", + reachedEndOffsets: false, + endOffsetsAvailable: true, + endOffsets: [{ partition: 0, startOffset: "0", endOffset: "12" }], + completion: { reason: "timeout", complete: false, barrierReached: false, retentionStartVerified: true }, + lastScannedOffsets: [] + }; + } + }); + + assert.equal(result.exitCode, 2); + assert.equal(result.payload.status, "partial"); + assert.equal(result.payload.error.code, "source_scan_incomplete"); + assert.notEqual(result.payload.error.code, "source_trace_missing"); + assert.equal(result.payload.input.scannedCount, 0); + assert.equal(result.payload.input.matchedCount, 0); + assert.equal(result.payload.input.scanComplete, false); + assert.equal(result.payload.input.refreshHandoff.status, "failed"); + assert.equal(result.payload.input.refreshHandoff.error.code, "workbench_kafka_refresh_timeout"); + assert.equal(result.payload.validation.refreshHandoffComplete, false); + assert.match(result.markdownOutput ?? "", /source scan: partial \(timeout\)/u); + assert.equal(typeof result.payload.output.artifact.path, "string"); + assert.match(renderKafkaCliText(result.payload), /^partial action=kafka\.render\.trace code=source_scan_incomplete completion=timeout reachedEnd=false scanned=0 matched=0 applied=0/u); +}); + test("offline JSONL maps 35 stdio reconstructions to 35 ordered HWLAB debug events without runtime dependencies", async () => { const cwd = await mkdtemp(path.join(os.tmpdir(), "hwlab-kafka-debug-jsonl-")); const inputFile = path.join(cwd, "agentrun-events.jsonl"); @@ -751,9 +809,46 @@ test("Kafka query reports invalid JSON, filter rejection, and bounded end-offset assert.equal(queried.completionReason, "end-offset"); assert.equal(queried.reachedEndOffsets, true); assert.deepEqual(queried.endOffsets, [{ partition: 0, startOffset: "0", endOffset: "3" }]); + assert.deepEqual(queried.firstScannedOffsets, [{ partition: 0, offset: "0" }]); assert.deepEqual(queried.lastScannedOffsets, [{ partition: 0, offset: "2" }]); }); +test("Kafka query fails closed when retention advances between barrier capture and consumer assignment", async () => { + const matching = canonicalRecord(1).value; + const queried = await queryKafkaEventStream({ + env: { HWLAB_KAFKA_BOOTSTRAP_SERVERS: "kafka.test:9092", HWLAB_KAFKA_CLIENT_ID: "hwlab-test" }, + topic: "hwlab.event.v1", + sessionId: SESSION_ID, + limit: 10, + scanLimit: 10, + groupIdPrefix: "hwlab-v03-workbench-isolated-debug", + kafkaFactory() { + return { + admin() { return { async connect() {}, async fetchTopicOffsets() { return [{ partition: 0, low: "0", high: "3" }]; }, async disconnect() {} }; }, + consumer() { + return { + async connect() {}, + async subscribe() {}, + async run({ eachMessage }: any) { + await eachMessage({ topic: "hwlab.event.v1", partition: 0, message: { offset: "2", value: Buffer.from(JSON.stringify(matching)) } }); + }, + async stop() {}, + async disconnect() {} + }; + } + }; + } + }); + + assert.equal(queried.completionReason, "retention-start-moved"); + assert.equal(queried.completion.complete, false); + assert.equal(queried.completion.retentionStartVerified, false); + assert.equal(queried.completion.retentionStartMoved, true); + assert.equal(queried.reachedEndOffsets, false); + assert.equal(queried.scannedCount, 0); + assert.deepEqual(queried.firstScannedOffsets, [{ partition: 0, offset: "2" }]); +}); + test("Kafka query prefers end-offset when the final record also reaches the event limit", async () => { const matching = canonicalRecord(1).value; const queried = await queryKafkaEventStream({ @@ -817,6 +912,107 @@ test("Kafka query treats a retained empty partition as a completed bounded scan" assert.deepEqual(queried.endOffsets, [{ partition: 0, startOffset: "7", endOffset: "7" }]); }); +test("Kafka query stops every partition at the captured exclusive barrier and excludes later records", async () => { + const matching = canonicalRecord(1).value; + const mismatch = structuredClone(matching); + mismatch.sessionId = "ses_agentrun_other"; + mismatch.run.sessionId = "ses_agentrun_other"; + mismatch.event.payload.sessionId = "ses_agentrun_other"; + const queried = await queryKafkaEventStream({ + env: { HWLAB_KAFKA_BOOTSTRAP_SERVERS: "kafka.test:9092", HWLAB_KAFKA_CLIENT_ID: "hwlab-test" }, + topic: "agentrun.event.v1", + sessionId: SESSION_ID, + limit: 10, + scanLimit: 10, + groupIdPrefix: "hwlab-v03-workbench-isolated-debug", + kafkaFactory() { + return { + admin() { return { async connect() {}, async fetchTopicOffsets() { return [{ partition: 0, low: "0", high: "2" }, { partition: 1, low: "0", high: "2" }]; }, async disconnect() {} }; }, + consumer() { + return { + async connect() {}, + async subscribe() {}, + async run({ eachMessage }: any) { + await eachMessage({ topic: "agentrun.event.v1", partition: 0, message: { offset: "0", value: Buffer.from(JSON.stringify(matching)) } }); + await eachMessage({ topic: "agentrun.event.v1", partition: 0, message: { offset: "1", value: Buffer.from(JSON.stringify(mismatch)) } }); + await eachMessage({ topic: "agentrun.event.v1", partition: 0, message: { offset: "2", value: Buffer.from(JSON.stringify(matching)) } }); + await eachMessage({ topic: "agentrun.event.v1", partition: 1, message: { offset: "0", value: Buffer.from(JSON.stringify(mismatch)) } }); + await eachMessage({ topic: "agentrun.event.v1", partition: 1, message: { offset: "1", value: Buffer.from(JSON.stringify(matching)) } }); + await eachMessage({ topic: "agentrun.event.v1", partition: 1, message: { offset: "3", value: Buffer.from(JSON.stringify(matching)) } }); + }, + async stop() {}, + async disconnect() {} + }; + } + }; + } + }); + + assert.equal(queried.completionReason, "end-offset"); + assert.equal(queried.completion.complete, true); + assert.equal(queried.completion.barrierReached, true); + assert.equal(queried.scannedCount, 4); + assert.equal(queried.excludedPostBarrierCount, 1); + assert.deepEqual(queried.events.map((event: any) => [event.partition, event.offset]), [[0, "0"], [1, "1"]]); + assert.deepEqual(queried.lastScannedOffsets, [{ partition: 0, offset: "1" }, { partition: 1, offset: "1" }]); +}); + +test("Kafka query reports scan-limit and abort as typed incomplete completion", async () => { + const matching = canonicalRecord(1).value; + const limited = await queryKafkaEventStream({ + env: { HWLAB_KAFKA_BOOTSTRAP_SERVERS: "kafka.test:9092", HWLAB_KAFKA_CLIENT_ID: "hwlab-test" }, + topic: "agentrun.event.v1", + sessionId: SESSION_ID, + limit: 10, + scanLimit: 1, + timeoutMs: 500, + groupIdPrefix: "hwlab-v03-workbench-isolated-debug", + kafkaFactory() { + return { + admin() { return { async connect() {}, async fetchTopicOffsets() { return [{ partition: 0, low: "0", high: "5" }]; }, async disconnect() {} }; }, + consumer() { return { async connect() {}, async subscribe() {}, async run({ eachMessage }: any) { await eachMessage({ topic: "agentrun.event.v1", partition: 0, message: { offset: "0", value: Buffer.from(JSON.stringify(matching)) } }); }, async stop() {}, async disconnect() {} }; } + }; + } + }); + assert.deepEqual(limited.completion, { + reason: "scan-limit", + complete: false, + barrierReached: false, + retentionStartVerified: true, + retentionStartMoved: false, + timeout: false, + matchedEventLimitReached: false, + scanLimitReached: true, + aborted: false, + valuesRedacted: true + }); + + const controller = new AbortController(); + let disconnected = false; + const abortedPromise = queryKafkaEventStream({ + env: { HWLAB_KAFKA_BOOTSTRAP_SERVERS: "kafka.test:9092", HWLAB_KAFKA_CLIENT_ID: "hwlab-test" }, + topic: "agentrun.event.v1", + sessionId: SESSION_ID, + limit: 10, + scanLimit: 10, + timeoutMs: 500, + signal: controller.signal, + groupIdPrefix: "hwlab-v03-workbench-isolated-debug", + kafkaFactory() { + return { + admin() { return { async connect() {}, async fetchTopicOffsets() { return [{ partition: 0, low: "0", high: "5" }]; }, async disconnect() {} }; }, + consumer() { return { async connect() {}, async subscribe() {}, async run() { await new Promise(() => undefined); }, async stop() {}, async disconnect() { disconnected = true; } }; } + }; + } + }); + setTimeout(() => controller.abort(), 5); + const aborted = await abortedPromise; + assert.equal(aborted.completionReason, "aborted"); + assert.equal(aborted.completion.aborted, true); + assert.equal(aborted.completion.complete, false); + assert.equal(disconnected, true); +}); + test("Kafka query does not report an empty offset response as a completed scan", async () => { const queried = await queryKafkaEventStream({ env: { HWLAB_KAFKA_BOOTSTRAP_SERVERS: "kafka.test:9092", HWLAB_KAFKA_CLIENT_ID: "hwlab-test" }, @@ -932,6 +1128,31 @@ test("non-reversible AgentRun session identity stays only in debug lineage", () assert.equal(mapped.events[0].reconstruction.identityDerivation.kind, "none"); }); +function completeKafkaRead(events: any[]) { + const byPartition = new Map(); + for (const event of events) { + const partition = Number(event.partition); + const offset = BigInt(String(event.offset)); + const offsets = byPartition.get(partition) ?? []; + offsets.push(offset); + byPartition.set(partition, offsets); + } + const endOffsets = [...byPartition.entries()].map(([partition, offsets]) => ({ + partition, + startOffset: String(offsets.reduce((left, right) => left < right ? left : right)), + endOffset: String(offsets.reduce((left, right) => left > right ? left : right) + 1n) + })).sort((left, right) => left.partition - right.partition); + return { + topic: events[0]?.topic ?? "hwlab.event.v1", + events, + completionReason: "end-offset", + reachedEndOffsets: true, + endOffsetsAvailable: true, + endOffsets, + completion: { reason: "end-offset", complete: true, barrierReached: true, retentionStartVerified: true } + }; +} + function reconstructionRecord(seq: number): any { const value = { schema: "agentrun.event.reconstruction.v1", diff --git a/tools/src/hwlab-cli/kafka-regenerate.ts b/tools/src/hwlab-cli/kafka-regenerate.ts index ef26e42e..49f54997 100644 --- a/tools/src/hwlab-cli/kafka-regenerate.ts +++ b/tools/src/hwlab-cli/kafka-regenerate.ts @@ -15,9 +15,10 @@ import { projectAgentRunKafkaMessageToHwlabDebugEvent, queryKafkaEventStream } from "../../../internal/cloud/kafka-event-bridge.ts"; +import { createWorkbenchKafkaRefreshHandoff } from "../../../internal/cloud/workbench-kafka-refresh-handoff.ts"; import { decodeWorkbenchRealtimeEventFrame } from "../../../web/hwlab-cloud-web/src/api/workbench-realtime-codec.ts"; import { reduceWorkbenchRealtimeEvent } from "../../../web/hwlab-cloud-web/src/stores/workbench-event-reducer.ts"; -import { projectWorkbenchLiveKafkaMessage } from "../../../web/hwlab-cloud-web/src/stores/workbench-live-kafka-event.ts"; +import { projectWorkbenchLiveKafkaMessage, projectWorkbenchLiveKafkaUserMessage, workbenchLiveKafkaProjectionTarget } from "../../../web/hwlab-cloud-web/src/stores/workbench-live-kafka-event.ts"; import { planWorkbenchRealtimeApply } from "../../../web/hwlab-cloud-web/src/stores/workbench-realtime-plan.ts"; import { renderTraceRowsMarkdown, traceDisplayRows } from "./trace-renderer.ts"; @@ -328,9 +329,7 @@ export async function renderHwlabKafkaTrace(parsed: ParsedArgs, dependencies: Re const groupPrefix = text(groupPrefixResolution.value); if (groupPrefix) assertDebugGroup(groupPrefix); - const readResult = sourceMode === "jsonl" - ? await readJsonlInput(requiredText(parsed.jsonlFile, "jsonlFile"), dependencies.cwd, inputTopic) - : await dependencies.readKafka({ + const kafkaQuery = { env: dependencies.env, stream: "hwlab", topic: inputTopic, @@ -341,12 +340,120 @@ export async function renderHwlabKafkaTrace(parsed: ParsedArgs, dependencies: Re timeoutMs, fromBeginning: parsed.fromEnd !== true, groupIdPrefix: groupPrefix - }); - const records = Array.isArray(readResult.events) ? readResult.events : []; + }; + const refreshHandoff = sourceMode === "kafka" + ? await replayHwlabKafkaTraceThroughProductionHandoff({ + traceId, + sessionId: sessionId || null, + query: kafkaQuery, + readKafka: dependencies.readKafka + }) + : null; + const readResult = sourceMode === "jsonl" + ? await readJsonlInput(requiredText(parsed.jsonlFile, "jsonlFile"), dependencies.cwd, inputTopic) + : refreshHandoff?.readResult ?? {}; + const records = sourceMode === "kafka" && refreshHandoff?.status === "succeeded" + ? refreshHandoff.records + : Array.isArray(readResult.events) ? readResult.events : []; const scoped = scopeHwlabTraceRecords(records, { traceId, sessionId, runId, commandId }); const sourceScanComplete = sourceMode === "jsonl" || readResult.completionReason === "end-offset"; const commandScopeScanComplete = sourceScanComplete; const matchingRecords = scoped.records; + const stateId = `hwlab-trace-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`; + const artifactPath = path.resolve( + dependencies.cwd, + text(parsed.outputMarkdown) || path.join(".state", "hwlab-cli", "kafka-render", stateId, `${traceId}.md`) + ); + if (!sourceScanComplete && matchingRecords.length === 0) { + const markdown = renderWorkbenchKafkaTraceMarkdown({ + traceId, + sessionId, + status: "unknown", + userMessages: [], + sourceEventCount: 0, + rows: [], + finalResponseText: null, + sourceScanComplete: false, + completionReason: readResult.completionReason ?? null + }); + const artifact = `${markdown.trimEnd()}\n`; + await mkdir(path.dirname(artifactPath), { recursive: true }); + await writeFile(artifactPath, artifact, "utf8"); + return { + markdown, + payload: { + ok: false, + action: "kafka.render.trace", + status: "partial", + error: { + code: "source_scan_incomplete", + message: "Kafka scan ended before the captured topic end offsets were reached", + details: { + completionReason: readResult.completionReason ?? null, + reachedEndOffsets: readResult.reachedEndOffsets ?? false, + artifactPath, + valuesPrinted: false + } + }, + format, + input: { + mode: sourceMode, + topic: inputTopic, + topicSource: inputTopicResolution.source, + groupId: readResult.groupId ?? null, + groupPrefix: groupPrefix || null, + groupPrefixSource: groupPrefixResolution.source, + scannedCount: readResult.scannedCount ?? records.length, + parsedCount: readResult.parsedCount ?? records.length, + queryMatchedCount: readResult.matchedCount ?? records.length, + matchedCount: 0, + invalidJsonCount: readResult.invalidJsonCount ?? 0, + filterRejectedCount: readResult.filterRejectedCount ?? 0, + scopeRejectedCount: scoped.summary.baseRejectedCount + scoped.summary.commandScopeRejectedCount, + commandScope: { ...scoped.summary, scanComplete: false }, + scanComplete: false, + completionReason: readResult.completionReason ?? null, + reachedEndOffsets: readResult.reachedEndOffsets ?? false, + endOffsetsAvailable: readResult.endOffsetsAvailable ?? false, + endOffsets: readResult.endOffsets ?? [], + lastScannedOffsets: readResult.lastScannedOffsets ?? [], + sourceFile: readResult.sourceFile ?? null, + refreshHandoff: refreshHandoff?.evidence ?? null, + valuesPrinted: false + }, + projection: { + pipeline: ["createWorkbenchKafkaRefreshHandoff", "decodeWorkbenchRealtimeEventFrame", "reduceWorkbenchRealtimeEvent", "planWorkbenchRealtimeApply", "workbenchLiveKafkaProjectionTarget", "projectWorkbenchLiveKafkaUserMessage|projectWorkbenchLiveKafkaMessage", "traceDisplayRows"], + decodedCount: 0, + plannedCount: 0, + appliedCount: 0, + rejectedCount: 0, + valuesPrinted: false + }, + output: { + artifact: { path: artifactPath, byteCount: Buffer.byteLength(artifact, "utf8"), sha256: sha256(artifact) }, + valuesPrinted: false + }, + validation: { + sourceScanComplete: false, + refreshHandoffComplete: false, + allMatchedEventsApplied: true, + sharedRowModel: true, + valuesPrinted: false + }, + runtimeDependencies: { + cloudApi: false, + database: false, + transactionalProjector: false, + kafka: sourceMode === "kafka" + }, + next: { + command: `hwlab-cli kafka render trace --from ${sourceMode} --trace-id ${traceId}${sessionId ? ` --session-id ${sessionId}` : ""}${runId ? ` --run-id ${runId}` : ""}${commandId ? ` --command-id ${commandId}` : ""} --format markdown`, + reason: "源扫描未完成；使用新的隔离 debug group 重试，只有完整 barrier 后零匹配才判定 trace 不存在。" + }, + valuesPrinted: false + } + }; + } if (matchingRecords.length === 0) { throw cliError(commandId && scoped.summary.baseMatchedCount > 0 ? "source_command_missing" : "source_trace_missing", commandId ? "no HWLAB Kafka events matched the requested command lifecycle" @@ -398,17 +505,13 @@ export async function renderHwlabKafkaTrace(parsed: ParsedArgs, dependencies: Re traceId, sessionId: text(projection.message.sessionId), status: text(projection.message.status), + userMessages: projection.userMessages, sourceEventCount: events.length, rows, finalResponseText: finalResponseText || null, sourceScanComplete, completionReason: readResult.completionReason ?? (sourceMode === "jsonl" ? "file-end" : null) }); - const stateId = `hwlab-trace-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`; - const artifactPath = path.resolve( - dependencies.cwd, - text(parsed.outputMarkdown) || path.join(".state", "hwlab-cli", "kafka-render", stateId, `${traceId}.md`) - ); await mkdir(path.dirname(artifactPath), { recursive: true }); await writeFile(artifactPath, `${markdown.trimEnd()}\n`, "utf8"); const fullRows = parsed.full === true; @@ -477,6 +580,7 @@ export async function renderHwlabKafkaTrace(parsed: ParsedArgs, dependencies: Re endOffsets: readResult.endOffsets ?? [], lastScannedOffsets: readResult.lastScannedOffsets ?? [], sourceFile: readResult.sourceFile ?? null, + refreshHandoff: refreshHandoff?.evidence ?? null, valuesPrinted: false }, identity: { @@ -498,13 +602,16 @@ export async function renderHwlabKafkaTrace(parsed: ParsedArgs, dependencies: Re valuesPrinted: false }, projection: { - pipeline: ["decodeWorkbenchRealtimeEventFrame", "reduceWorkbenchRealtimeEvent", "planWorkbenchRealtimeApply", "projectWorkbenchLiveKafkaMessage", "traceDisplayRows"], + pipeline: ["createWorkbenchKafkaRefreshHandoff", "decodeWorkbenchRealtimeEventFrame", "reduceWorkbenchRealtimeEvent", "planWorkbenchRealtimeApply", "workbenchLiveKafkaProjectionTarget", "projectWorkbenchLiveKafkaUserMessage|projectWorkbenchLiveKafkaMessage", "traceDisplayRows"], decodedCount: projection.decodedCount, plannedCount: projection.plannedCount, appliedCount: projection.appliedCount, rejectedCount: projection.rejections.length, rejections: projection.rejections.slice(0, 20), eventCount: events.length, + userMessageCount: projection.userMessages.length, + userMessageIds: projection.userMessages.map((item: any) => text(item.messageId ?? item.id)).filter(Boolean), + userTargetTraceIds: projection.userMessages.map((item: any) => text(item.targetTraceId)).filter(Boolean), terminalObserved: projection.message.traceAutoLifecycle === "terminal", status: projection.message.status, lastEventAt: projection.message.timing?.lastEventAt ?? null, @@ -536,6 +643,7 @@ export async function renderHwlabKafkaTrace(parsed: ParsedArgs, dependencies: Re allMatchedEventsApplied: projection.appliedCount === matchingRecords.length, commandLifecyclePreserved: !commandId || (commandScopeScanComplete && scoped.summary.exactCommandCount > 0 && scoped.summary.ambiguousUnscopedLifecycleCount === 0), sourceScanComplete, + refreshHandoffComplete: sourceMode !== "kafka" || refreshHandoff?.status === "succeeded", finalResponseOutsideTrace: !finalResponseInTrace && Boolean(finalResponseText), sharedRowModel: true, valuesPrinted: false @@ -555,8 +663,90 @@ export async function renderHwlabKafkaTrace(parsed: ParsedArgs, dependencies: Re }; } +export async function replayHwlabKafkaTraceThroughProductionHandoff(input: { + traceId: string; + sessionId: string | null; + query: Record; + readKafka: (query: Record) => Promise>; +}) { + let readResult: Record | null = null; + let summary: Record | null = null; + let liveSubscriptionInstalled = false; + const records: JsonRecord[] = []; + const sourceByTransport = new Map(); + const handoff = createWorkbenchKafkaRefreshHandoff({ + sessionId: input.sessionId, + traceId: input.traceId, + liveBufferLimit: 1, + subscribeLive() { + liveSubscriptionInstalled = true; + return () => undefined; + }, + async queryRetention({ signal }: { signal: AbortSignal }) { + readResult = await input.readKafka({ ...input.query, signal }); + for (const record of Array.isArray(readResult.events) ? readResult.events : []) { + sourceByTransport.set(kafkaTransportKey(record), record); + } + return readResult; + }, + async deliverEvent(envelope: JsonRecord, transport: JsonRecord, delivery: string) { + if (delivery !== "replay") throw cliError("kafka_refresh_cli_delivery_invalid", "CLI retention handoff must deliver replay records before its connected barrier", { delivery }); + records.push(sourceByTransport.get(kafkaTransportKey(transport)) ?? { ...transport, value: envelope }); + return true; + }, + async deliverConnected(value: Record) { + summary = value; + return true; + } + }); + try { + summary = await handoff.start(); + return { + status: "succeeded", + readResult: readResult ?? {}, + records, + evidence: { + module: "internal/cloud/workbench-kafka-refresh-handoff:createWorkbenchKafkaRefreshHandoff", + status: "succeeded", + liveSubscriptionInstalled, + phase: summary?.phase ?? "live", + topic: summary?.topic ?? null, + topicPartitions: summary?.topicPartitions ?? [], + barrier: summary?.barrier ?? [], + counts: summary?.counts ?? null, + valuesPrinted: false + } + }; + } catch (error) { + return { + status: "failed", + readResult: readResult ?? {}, + records: [], + evidence: { + module: "internal/cloud/workbench-kafka-refresh-handoff:createWorkbenchKafkaRefreshHandoff", + status: "failed", + liveSubscriptionInstalled, + phase: text((error as any)?.phase) || handoff.phase(), + error: { + code: text((error as any)?.code) || "workbench_kafka_refresh_failed", + message: error instanceof Error ? error.message : String(error), + valuesRedacted: true + }, + valuesPrinted: false + } + }; + } finally { + handoff.stop("cli-complete"); + } +} + +function kafkaTransportKey(value: Record): string { + return `${text(value?.topic)}:${nullableInteger(value?.partition) ?? ""}:${text(value?.offset)}`; +} + export function projectHwlabRecordsThroughWorkbench(records: JsonRecord[], options: { traceId: string; sessionId?: string; observedAt: string }) { let message: any = null; + const userMessagesById = new Map(); let decodedCount = 0; let plannedCount = 0; let appliedCount = 0; @@ -601,6 +791,23 @@ export function projectHwlabRecordsThroughWorkbench(records: JsonRecord[], optio rejections.push({ index, code: "session-mismatch", detail: eventSessionId }); continue; } + if (workbenchLiveKafkaProjectionTarget(traceStep.event as any) === "user") { + const userMessageId = text((traceStep.event as any).userMessageId ?? (traceStep.event as any).messageId); + const userMessage = projectWorkbenchLiveKafkaUserMessage({ + previous: userMessageId ? userMessagesById.get(userMessageId) ?? null : null, + traceId: options.traceId, + sessionId: eventSessionId, + event: traceStep.event as any, + receivedAt: new Date(baseMs + index).toISOString() + }); + if (!userMessage) { + rejections.push({ index, code: "user-projection-invalid" }); + continue; + } + userMessagesById.set(userMessage.id, userMessage); + appliedCount += 1; + continue; + } message = projectWorkbenchLiveKafkaMessage({ previous: message, traceId: options.traceId, @@ -610,10 +817,10 @@ export function projectHwlabRecordsThroughWorkbench(records: JsonRecord[], optio }); appliedCount += 1; } - return { message, decodedCount, plannedCount, appliedCount, rejections }; + return { message, userMessages: [...userMessagesById.values()], decodedCount, plannedCount, appliedCount, rejections }; } -function renderWorkbenchKafkaTraceMarkdown(input: { traceId: string; sessionId: string; status: string; sourceEventCount: number; rows: any[]; finalResponseText: string | null; sourceScanComplete: boolean; completionReason: string | null }) { +function renderWorkbenchKafkaTraceMarkdown(input: { traceId: string; sessionId: string; status: string; userMessages: any[]; sourceEventCount: number; rows: any[]; finalResponseText: string | null; sourceScanComplete: boolean; completionReason: string | null }) { const lines = [ `# Workbench Trace ${input.traceId}`, "", @@ -621,13 +828,25 @@ function renderWorkbenchKafkaTraceMarkdown(input: { traceId: string; sessionId: `- status: ${input.status || "unknown"}`, `- source scan: ${input.sourceScanComplete ? "complete" : `partial (${input.completionReason ?? "unknown"})`}`, `- source events: ${input.sourceEventCount}`, + `- user messages: ${input.userMessages.length}`, `- rendered rows: ${input.rows.length}`, - `- final response placement: outer`, - "", - "## 运行记录", - "", - renderTraceRowsMarkdown(input.rows) + `- final response placement: outer` ]; + if (input.userMessages.length > 0) { + lines.push("", "## 用户输入"); + for (const message of input.userMessages) { + lines.push( + "", + `### ${text(message.messageId ?? message.id) || "用户消息"}`, + "", + `- trace: ${text(message.traceId) || "unknown"}`, + `- target trace: ${text(message.targetTraceId) || "none"}`, + "", + text(message.text) + ); + } + } + lines.push("", "## 运行记录", "", renderTraceRowsMarkdown(input.rows)); if (input.finalResponseText) lines.push("", "## 最终回复", "", input.finalResponseText); return lines.join("\n"); } diff --git a/web/hwlab-cloud-web/scripts/workbench-e2e-server.ts b/web/hwlab-cloud-web/scripts/workbench-e2e-server.ts index 2ce7084c..553359bc 100644 --- a/web/hwlab-cloud-web/scripts/workbench-e2e-server.ts +++ b/web/hwlab-cloud-web/scripts/workbench-e2e-server.ts @@ -102,7 +102,7 @@ const terminalAssistantFinalText = [ ].join("\n"); const canonicalTraceFinalText = "messageProjection 的 sealed final response 才是主消息正文。"; const traceDetailConflictText = "TRACE_DETAIL_CONFLICT_SHOULD_NOT_REPLACE_MESSAGE"; -const runtimeConfigScript = ``; +const runtimeConfigScript = ``; let state = createScenarioState("baseline"); const sseClients = new Set(); diff --git a/web/hwlab-cloud-web/scripts/workbench-realtime-runtime.test.ts b/web/hwlab-cloud-web/scripts/workbench-realtime-runtime.test.ts index ff99b6ab..62a1755d 100644 --- a/web/hwlab-cloud-web/scripts/workbench-realtime-runtime.test.ts +++ b/web/hwlab-cloud-web/scripts/workbench-realtime-runtime.test.ts @@ -83,22 +83,22 @@ test("Workbench API uses metadata-only session detail and bounded messages paths }); test("live SSE transport never sends an afterSeq cursor while projection transport can", () => { - assert.equal(workbenchEventStreamPath({ realtimeCapabilities: { liveKafkaSse: true, projectionRealtime: false }, sessionId: "ses_live", traceId: "trc_live", afterSeq: 42 }), "/v1/workbench/events?sessionId=ses_live&traceId=trc_live"); - assert.equal(workbenchEventStreamPath({ realtimeCapabilities: { liveKafkaSse: false, projectionRealtime: true }, sessionId: "ses_projection", traceId: "trc_projection", afterSeq: 42 }), "/v1/workbench/events?sessionId=ses_projection&traceId=trc_projection&afterSeq=42"); - assert.equal(workbenchEventStreamPath({ realtimeCapabilities: { liveKafkaSse: true, projectionRealtime: true }, sessionId: "ses_both", traceId: "trc_both", afterSeq: 42 }), "/v1/workbench/events?sessionId=ses_both&traceId=trc_both"); + assert.equal(workbenchEventStreamPath({ realtimeCapabilities: { liveKafkaSse: true, kafkaRefreshReplay: false, projectionRealtime: false }, sessionId: "ses_live", traceId: "trc_live", afterSeq: 42 }), "/v1/workbench/events?sessionId=ses_live&traceId=trc_live"); + assert.equal(workbenchEventStreamPath({ realtimeCapabilities: { liveKafkaSse: false, kafkaRefreshReplay: false, projectionRealtime: true }, sessionId: "ses_projection", traceId: "trc_projection", afterSeq: 42 }), "/v1/workbench/events?sessionId=ses_projection&traceId=trc_projection&afterSeq=42"); + assert.equal(workbenchEventStreamPath({ realtimeCapabilities: { liveKafkaSse: true, kafkaRefreshReplay: true, projectionRealtime: true }, sessionId: "ses_both", traceId: "trc_both", afterSeq: 42 }), "/v1/workbench/events?sessionId=ses_both&traceId=trc_both"); assert.equal(workbenchProjectionEventStreamPath({ sessionId: "ses_both", traceId: "trc_both", afterSeq: 42 }), "/v1/workbench/projection-events?sessionId=ses_both&traceId=trc_both&afterSeq=42"); - assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: false, projectionRealtime: false }), false); - assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: true, projectionRealtime: false }), true); - assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: false, projectionRealtime: true }), true); - assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: true, projectionRealtime: true }), true); + assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: false, kafkaRefreshReplay: false, projectionRealtime: false }), false); + assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: true, kafkaRefreshReplay: false, projectionRealtime: false }), true); + assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: false, kafkaRefreshReplay: false, projectionRealtime: true }), true); + assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: true, kafkaRefreshReplay: true, projectionRealtime: true }), true); }); test("live Kafka keeps the pre-submit session SSE key when a turn trace becomes active", () => { - const capabilities = { liveKafkaSse: true, projectionRealtime: false }; + const capabilities = { liveKafkaSse: true, kafkaRefreshReplay: true, projectionRealtime: false }; const beforeSubmit = workbenchRealtimeScopeKey("ses_live", workbenchRealtimeTraceIdForCapabilities(capabilities, null, null)); const afterSubmit = workbenchRealtimeScopeKey("ses_live", workbenchRealtimeTraceIdForCapabilities(capabilities, "trc_current_request", "trc_message")); assert.equal(afterSubmit, beforeSubmit); - assert.equal(workbenchRealtimeTraceIdForCapabilities({ liveKafkaSse: false, projectionRealtime: true }, "trc_current_request", "trc_message"), "trc_current_request"); + assert.equal(workbenchRealtimeTraceIdForCapabilities({ liveKafkaSse: false, kafkaRefreshReplay: false, projectionRealtime: true }, "trc_current_request", "trc_message"), "trc_current_request"); }); test("Error runtime owns Workbench message diagnostic view model", () => { diff --git a/web/hwlab-cloud-web/src/api/workbench-events.test.ts b/web/hwlab-cloud-web/src/api/workbench-events.test.ts index 320a25a9..c5d302e4 100644 --- a/web/hwlab-cloud-web/src/api/workbench-events.test.ts +++ b/web/hwlab-cloud-web/src/api/workbench-events.test.ts @@ -48,7 +48,7 @@ test("product EventSource tees every raw HWLAB frame once before decode and redu globalThis.EventSource = FakeProductEventSource as unknown as typeof EventSource; try { const stream = connectWorkbenchEvents({ - realtimeCapabilities: { liveKafkaSse: true, projectionRealtime: false }, + realtimeCapabilities: { liveKafkaSse: true, kafkaRefreshReplay: true, projectionRealtime: false }, sessionId: "ses_raw_tee", flushYieldMs: 1, onIngress: (frame) => ingress.push(frame), diff --git a/web/hwlab-cloud-web/src/api/workbench-events.ts b/web/hwlab-cloud-web/src/api/workbench-events.ts index 2f731b4b..c1b0aa37 100644 --- a/web/hwlab-cloud-web/src/api/workbench-events.ts +++ b/web/hwlab-cloud-web/src/api/workbench-events.ts @@ -32,6 +32,7 @@ export interface WorkbenchRealtimeEvent { capabilities?: WorkbenchRealtimeCapabilities | null; liveOnly?: boolean | null; replay?: boolean | null; + deliverySemantics?: string | null; lossPossible?: boolean | null; type?: string; contractVersion?: string | null; diff --git a/web/hwlab-cloud-web/src/config/runtime.ts b/web/hwlab-cloud-web/src/config/runtime.ts index 628cb730..5936ba7a 100644 --- a/web/hwlab-cloud-web/src/config/runtime.ts +++ b/web/hwlab-cloud-web/src/config/runtime.ts @@ -12,6 +12,7 @@ export interface OpenCodeFrameConfig { export interface WorkbenchRealtimeCapabilities { liveKafkaSse: boolean; + kafkaRefreshReplay: boolean; projectionRealtime: boolean; } @@ -98,10 +99,10 @@ export function workbenchTraceTimelinePolicy(): WorkbenchTraceTimelinePolicy { export function workbenchRealtimeCapabilities(): WorkbenchRealtimeCapabilities { const features = window.HWLAB_CLOUD_WEB_CONFIG?.workbench?.realtimeFeatures; - if (typeof features?.liveKafkaSse !== "boolean" || typeof features?.projectionRealtime !== "boolean") { + if (typeof features?.liveKafkaSse !== "boolean" || typeof features?.kafkaRefreshReplay !== "boolean" || typeof features?.projectionRealtime !== "boolean") { throw new Error("HWLAB Cloud Web workbench.realtimeFeatures are required"); } - return { liveKafkaSse: features.liveKafkaSse, projectionRealtime: features.projectionRealtime }; + return { liveKafkaSse: features.liveKafkaSse, kafkaRefreshReplay: features.kafkaRefreshReplay, projectionRealtime: features.projectionRealtime }; } export function workbenchDebugCapabilities(): WorkbenchDebugCapabilities { diff --git a/web/hwlab-cloud-web/src/stores/workbench-kafka-refresh-policy.test.ts b/web/hwlab-cloud-web/src/stores/workbench-kafka-refresh-policy.test.ts new file mode 100644 index 00000000..6a72c694 --- /dev/null +++ b/web/hwlab-cloud-web/src/stores/workbench-kafka-refresh-policy.test.ts @@ -0,0 +1,27 @@ +import assert from "node:assert/strict"; +import { test } from "bun:test"; + +import { workbenchHistoryAuthorityPolicy } from "./workbench-kafka-refresh-policy"; + +test("Kafka refresh keeps metadata authorization but disables every HTTP history hydrate", () => { + assert.deepEqual(workbenchHistoryAuthorityPolicy({ liveKafkaSse: true, kafkaRefreshReplay: true, projectionRealtime: true }), { + kafkaRetention: true, + sessionMetadataRead: true, + sessionMessagesHydrate: false, + turnStatusHydrate: false, + traceEventsHydrate: false, + syncReplay: false + }); +}); + +test("independent live-only and projection capabilities preserve their existing history policy", () => { + assert.deepEqual(workbenchHistoryAuthorityPolicy({ liveKafkaSse: true, kafkaRefreshReplay: false, projectionRealtime: false }), { + kafkaRetention: false, + sessionMetadataRead: true, + sessionMessagesHydrate: true, + turnStatusHydrate: true, + traceEventsHydrate: true, + syncReplay: false + }); + assert.equal(workbenchHistoryAuthorityPolicy({ liveKafkaSse: false, kafkaRefreshReplay: false, projectionRealtime: true }).syncReplay, true); +}); diff --git a/web/hwlab-cloud-web/src/stores/workbench-kafka-refresh-policy.ts b/web/hwlab-cloud-web/src/stores/workbench-kafka-refresh-policy.ts new file mode 100644 index 00000000..61597642 --- /dev/null +++ b/web/hwlab-cloud-web/src/stores/workbench-kafka-refresh-policy.ts @@ -0,0 +1,22 @@ +import type { WorkbenchRealtimeCapabilities } from "@/config/runtime"; + +export interface WorkbenchHistoryAuthorityPolicy { + kafkaRetention: boolean; + sessionMetadataRead: true; + sessionMessagesHydrate: boolean; + turnStatusHydrate: boolean; + traceEventsHydrate: boolean; + syncReplay: boolean; +} + +export function workbenchHistoryAuthorityPolicy(capabilities: WorkbenchRealtimeCapabilities): WorkbenchHistoryAuthorityPolicy { + const kafkaRetention = capabilities.liveKafkaSse && capabilities.kafkaRefreshReplay; + return { + kafkaRetention, + sessionMetadataRead: true, + sessionMessagesHydrate: !kafkaRetention, + turnStatusHydrate: !kafkaRetention, + traceEventsHydrate: !kafkaRetention, + syncReplay: !kafkaRetention && capabilities.projectionRealtime + }; +} diff --git a/web/hwlab-cloud-web/src/stores/workbench-live-kafka-event.test.ts b/web/hwlab-cloud-web/src/stores/workbench-live-kafka-event.test.ts index 1d0d2435..2e6d386b 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-live-kafka-event.test.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-live-kafka-event.test.ts @@ -2,7 +2,8 @@ import assert from "node:assert/strict"; import { test } from "bun:test"; import type { ChatMessage, TraceEvent } from "../types"; -import { projectWorkbenchLiveKafkaMessage, workbenchLiveKafkaEnvelope } from "./workbench-live-kafka-event"; +import { projectWorkbenchLiveKafkaMessage, projectWorkbenchLiveKafkaUserMessage, workbenchLiveKafkaEnvelope, workbenchLiveKafkaEventIsUserMessage, workbenchLiveKafkaProjectionTarget, workbenchUserMessageIdForTrace } from "./workbench-live-kafka-event"; +import { createWorkbenchServerState, reduceWorkbenchServerState, selectActiveMessages, type WorkbenchServerState } from "./workbench-server-state"; test("only canonical HWLAB Kafka envelopes bypass terminal priority", () => { assert.equal(workbenchLiveKafkaEnvelope("hwlab.event.v1"), true); @@ -10,6 +11,130 @@ test("only canonical HWLAB Kafka envelopes bypass terminal priority", () => { assert.equal(workbenchLiveKafkaEnvelope(null), false); }); +test("live Kafka user event restores and idempotently updates the user bubble", () => { + const event = traceEvent(1, "user", { + userMessageId: "msg_live_user_user", + messageId: "msg_live_user_user", + text: "persisted user input", + message: "persisted user input" + }); + assert.equal(workbenchLiveKafkaEventIsUserMessage(event), true); + const first = projectWorkbenchLiveKafkaUserMessage({ + previous: null, + traceId: "trc_live_user", + sessionId: "ses_live_user", + event, + receivedAt: "2026-07-10T10:00:05.000Z" + }); + const second = projectWorkbenchLiveKafkaUserMessage({ + previous: first, + traceId: "trc_live_user", + sessionId: "ses_live_user", + event, + receivedAt: "2026-07-10T10:00:06.000Z" + }); + assert.equal(first?.id, "msg_live_user_user"); + assert.equal(first?.role, "user"); + assert.equal(first?.text, "persisted user input"); + assert.equal(second?.id, first?.id); + assert.equal(second?.createdAt, first?.createdAt); + assert.equal(second?.updatedAt, "2026-07-10T10:00:06.000Z"); +}); + +test("HTTP-first and Kafka-first admission converge on one stable optimistic user bubble", () => { + const sessionId = "ses_live_user_race"; + const traceId = "trc_live_user_race"; + const userMessageId = workbenchUserMessageIdForTrace(traceId); + assert.equal(userMessageId, "msg_live_user_race_user"); + const optimistic = { + id: userMessageId, + messageId: userMessageId, + role: "user", + title: "用户", + text: "race input", + status: "sent", + traceId, + sessionId, + createdAt: "2026-07-10T10:00:00.000Z", + updatedAt: "2026-07-10T10:00:00.000Z" + } as ChatMessage; + const event = traceEvent(1, "user", { + traceId, + sessionId, + userMessageId, + messageId: userMessageId, + text: "race input" + }); + + const runOrder = (order: "http-first" | "kafka-first"): ChatMessage[] => { + let state = createWorkbenchServerState(); + state = reduceWorkbenchServerState(state, { type: "message.upsert", sessionId, message: optimistic }); + const applyAdmissionResponse = (source: WorkbenchServerState): WorkbenchServerState => { + const previous = selectActiveMessages(source, sessionId).find((message) => message.role === "user" && message.traceId === traceId); + if (!previous) throw new Error("optimistic user message missing"); + return reduceWorkbenchServerState(source, { + type: "message.upsert", + sessionId, + message: { ...previous, id: userMessageId, messageId: userMessageId, turnId: traceId } + }); + }; + const applyKafkaEvent = (source: WorkbenchServerState): WorkbenchServerState => { + const previous = selectActiveMessages(source, sessionId).find((message) => (message.messageId ?? message.id) === userMessageId) ?? null; + const message = projectWorkbenchLiveKafkaUserMessage({ previous, traceId, sessionId, event, receivedAt: "2026-07-10T10:00:01.000Z" }); + if (!message) throw new Error("formal Kafka user event was not projected"); + return reduceWorkbenchServerState(source, { type: "message.upsert", sessionId, message }); + }; + state = order === "http-first" ? applyKafkaEvent(applyAdmissionResponse(state)) : applyAdmissionResponse(applyKafkaEvent(state)); + return selectActiveMessages(state, sessionId).filter((message) => message.role === "user"); + }; + + for (const order of ["http-first", "kafka-first"] as const) { + const users = runOrder(order); + assert.equal(users.length, 1, `${order} must keep one user bubble`); + assert.equal(users[0]?.id, userMessageId); + assert.equal(users[0]?.messageId, userMessageId); + assert.equal(users[0]?.text, "race input"); + } +}); + +test("session replay keeps steer user input separate from the target agent lifecycle", () => { + const sessionId = "ses_live_session_sequence"; + const targetTraceId = "trc_live_target"; + const steerTraceId = "trc_live_steer"; + const events = [ + traceEvent(1, "user", { traceId: targetTraceId, sessionId, userMessageId: "msg_target_user", messageId: "msg_target_user", text: "initial input" }), + traceEvent(2, "backend", { traceId: targetTraceId, sessionId }), + traceEvent(3, "assistant", { traceId: targetTraceId, sessionId, assistantText: "single final", text: "single final" }), + traceEvent(4, "user", { traceId: steerTraceId, targetTraceId, sessionId, userMessageId: "msg_steer_user", messageId: "msg_steer_user", text: "steer input" }), + traceEvent(5, "tool", { traceId: targetTraceId, sessionId }), + traceEvent(6, "result", { traceId: targetTraceId, sessionId, terminal: true, status: "completed" }) + ]; + const userMessages = new Map(); + const agentMessages = new Map(); + + events.forEach((event, index) => { + const eventTraceId = String(event.traceId); + const receivedAt = `2026-07-10T10:01:0${index}.000Z`; + if (workbenchLiveKafkaProjectionTarget(event) === "user") { + const messageId = String(event.userMessageId); + const projected = projectWorkbenchLiveKafkaUserMessage({ previous: userMessages.get(messageId) ?? null, traceId: eventTraceId, sessionId, event, receivedAt }); + if (projected) userMessages.set(messageId, projected); + return; + } + agentMessages.set(eventTraceId, projectWorkbenchLiveKafkaMessage({ previous: agentMessages.get(eventTraceId) ?? null, traceId: eventTraceId, sessionId, event, receivedAt })); + }); + + assert.deepEqual([...userMessages.keys()], ["msg_target_user", "msg_steer_user"]); + assert.deepEqual([...agentMessages.keys()], [targetTraceId]); + assert.equal(userMessages.get("msg_steer_user")?.targetTraceId, targetTraceId); + assert.equal(agentMessages.has(steerTraceId), false); + const target = agentMessages.get(targetTraceId); + assert.equal(target?.status, "completed"); + assert.equal(target?.text, "single final"); + assert.equal((target?.finalResponse as { text?: string } | null)?.text, "single final"); + assert.equal([...agentMessages.values()].filter((message) => message.finalResponse).length, 1); +}); + test("live Kafka projection refreshes lastEventAt from each ingress receipt", () => { const traceId = "trc_live_timing"; const sessionId = "ses_live_timing"; diff --git a/web/hwlab-cloud-web/src/stores/workbench-live-kafka-event.ts b/web/hwlab-cloud-web/src/stores/workbench-live-kafka-event.ts index 46f6a9cf..e5e2090d 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-live-kafka-event.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-live-kafka-event.ts @@ -20,6 +20,14 @@ export interface WorkbenchLiveKafkaProjectionInput { title?: string; } +export interface WorkbenchLiveKafkaUserProjectionInput { + previous: ChatMessage | null; + traceId: string; + sessionId: string; + event: TraceEvent; + receivedAt: string; +} + export function reduceWorkbenchLiveKafkaMessageState(previous: WorkbenchLiveMessageState, event: TraceEvent): WorkbenchLiveMessageState { if (previous.terminal) return previous; const terminal = workbenchLiveKafkaEventIsTerminal(event); @@ -36,10 +44,51 @@ export function workbenchLiveKafkaMessageId(traceId: string): string { return `msg_live_${traceId}`; } +export function workbenchUserMessageIdForTrace(traceId: string): string { + const suffix = firstNonEmptyString(traceId) + ?.replace(/^trc_/u, "") + .replace(/[^A-Za-z0-9_.:-]/gu, "_") + .slice(0, 48); + if (!suffix) throw new Error("traceId must produce a stable Workbench user message identity"); + return `msg_${suffix}_user`; +} + export function workbenchLiveKafkaEnvelope(schema: unknown): boolean { return firstNonEmptyString(schema) === "hwlab.event.v1"; } +export function workbenchLiveKafkaEventIsUserMessage(event: TraceEvent | null | undefined): boolean { + return firstNonEmptyString(event?.type, event?.eventType) === "user"; +} + +export function workbenchLiveKafkaProjectionTarget(event: TraceEvent | null | undefined): "user" | "agent" { + return workbenchLiveKafkaEventIsUserMessage(event) ? "user" : "agent"; +} + +export function projectWorkbenchLiveKafkaUserMessage(input: WorkbenchLiveKafkaUserProjectionInput): ChatMessage | null { + if (!workbenchLiveKafkaEventIsUserMessage(input.event)) return null; + const messageId = firstNonEmptyString(input.event.userMessageId, input.event.messageId); + const text = firstNonEmptyString(input.event.text, input.event.message); + const receivedAt = timestamp(input.receivedAt); + const createdAt = timestamp(input.event.createdAt) ?? receivedAt; + if (!messageId || !text || !receivedAt || !createdAt) return null; + return { + ...(input.previous ?? {}), + id: messageId, + messageId, + role: "user", + title: "用户", + text, + status: "sent", + traceId: input.traceId, + targetTraceId: firstNonEmptyString(input.event.targetTraceId), + turnId: input.previous?.turnId ?? input.traceId, + sessionId: input.sessionId, + createdAt: input.previous?.createdAt ?? createdAt, + updatedAt: receivedAt + } as ChatMessage; +} + export function workbenchLiveKafkaEventIsTerminal(event: TraceEvent | null | undefined): boolean { return Boolean(event?.terminal === true || firstNonEmptyString(event?.eventType) === "terminal" || firstNonEmptyString(event?.type) === "result"); } diff --git a/web/hwlab-cloud-web/src/stores/workbench.ts b/web/hwlab-cloud-web/src/stores/workbench.ts index 456c5cd4..9dcb5e33 100644 --- a/web/hwlab-cloud-web/src/stores/workbench.ts +++ b/web/hwlab-cloud-web/src/stores/workbench.ts @@ -22,7 +22,8 @@ import { initialWorkbenchSessionIdFromLocation } from "./workbench-projection"; import { cleanupWorkbenchServerStateSessions, selectActiveMessages, selectActiveSession, selectSessionList, selectSessionStatusAuthority, selectTraceAuthorityById, selectTurnStatusAuthority, type WorkbenchServerAction } from "./workbench-server-state"; import { cleanupDroppedWorkbenchSessionCaches, trimWorkbenchSessionCache } from "./workbench-session-cache"; import { reduceWorkbenchRealtimeEvent, workbenchRealtimeEventIsBusinessActivity, type WorkbenchRealtimeAction } from "./workbench-event-reducer"; -import { projectWorkbenchLiveKafkaMessage, workbenchLiveKafkaEnvelope } from "./workbench-live-kafka-event"; +import { projectWorkbenchLiveKafkaMessage, projectWorkbenchLiveKafkaUserMessage, workbenchLiveKafkaEnvelope, workbenchLiveKafkaProjectionTarget, workbenchUserMessageIdForTrace } from "./workbench-live-kafka-event"; +import { workbenchHistoryAuthorityPolicy } from "./workbench-kafka-refresh-policy"; import { messageHasSealedTerminalResult, messageIsSealedTerminal, traceAuthorityIsSealed } from "./workbench-terminal-authority"; import { boundedProjectionMessageLimit, mergeBoundedProjectionMessages, selectProjectionMessageWindow, traceProjectionIsTerminalSealed } from "./workbench-message-projection-budget"; import { @@ -110,6 +111,7 @@ interface RealtimeTurnProjectionItem { export const useWorkbenchStore = defineStore("workbench", () => { const runtimePolicy = workbenchRuntimePolicy(); const realtimeCapabilities = workbenchRealtimeCapabilities(); + const historyAuthorityPolicy = workbenchHistoryAuthorityPolicy(realtimeCapabilities); const debugCapabilities = workbenchDebugCapabilities(); const workbenchColadaReducer = useWorkbenchColadaReducer(); const workbenchColadaQueries = useWorkbenchColadaQueries(); @@ -374,7 +376,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { rememberSessionList(next); applySessionPagination(response.data); sessionsReady.value = true; - await refreshSessionStatusAuthority(next); + if (historyAuthorityPolicy.turnStatusHydrate) await refreshSessionStatusAuthority(next); return; } sessionsReady.value = sessions.value.length > 0; @@ -410,7 +412,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { rememberSessionList(next); applySessionPagination(response.data); sessionsReady.value = true; - await refreshSessionStatusAuthority(next); + if (historyAuthorityPolicy.turnStatusHydrate) await refreshSessionStatusAuthority(next); } function currentSessionListLimit(): number { @@ -761,6 +763,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { const submitEntry = steerMode ? "steer" : "existing"; const traceId = steerMode && composer.value.targetTraceId ? composer.value.targetTraceId : nextProtocolId("trc"); const steerTraceId = steerMode ? nextProtocolId("trc_steer") : null; + const userMessageId = workbenchUserMessageIdForTrace(steerTraceId ?? traceId); const sessionId = composer.value.sessionId; if (realtimeCapabilities.liveKafkaSse && liveRealtimeReadySessionId.value !== sessionId) { error.value = "realtime_connecting"; @@ -772,7 +775,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { const submittedAt = new Date().toISOString(); const previousChatPending = chatPending.value; if (!steerMode) { - const user = makeMessage("user", value, "sent", { traceId, sessionId, threadId, title: "用户", createdAt: submittedAt, updatedAt: submittedAt }); + const user = makeMessage("user", value, "sent", { id: userMessageId, messageId: userMessageId, traceId, sessionId, threadId, title: "用户", createdAt: submittedAt, updatedAt: submittedAt }); const pending = makeMessage("agent", "", "running", { traceId, sessionId, threadId, title: "Code Agent", retryInput: value, traceAutoLifecycle: "running", createdAt: submittedAt, updatedAt: submittedAt, ...optimisticRunningTimingPatch(submittedAt) }); appendActiveMessages(user, pending); projectOptimisticRunningTurn({ sessionId, threadId, traceId, userText: value }); @@ -782,7 +785,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { chatPending.value = true; rememberRecentDraft(value); recordActivity("submit"); - const payload = { message: value, prompt: value, sessionId, threadId: providerThreadId, traceId, providerProfile: providerProfile.value, gatewayShellTimeoutMs: gatewayShellTimeoutMs.value, targetTraceId: composer.value.targetTraceId, steerTraceId }; + const payload = { message: value, prompt: value, sessionId, threadId: providerThreadId, traceId, userMessageId, providerProfile: providerProfile.value, gatewayShellTimeoutMs: gatewayShellTimeoutMs.value, targetTraceId: composer.value.targetTraceId, steerTraceId }; const response = steerMode ? await workbenchColadaMutations.steerAgentMessage({ payload, timeoutMs: codeAgentTimeoutMs.value, activityRef: () => activityRef.value, sessionId, traceId }) : await workbenchColadaMutations.submitAgentMessage({ payload, timeoutMs: codeAgentTimeoutMs.value, activityRef: () => activityRef.value, sessionId, traceId }); @@ -1149,7 +1152,14 @@ export const useWorkbenchStore = defineStore("workbench", () => { if (realtimeCapabilities.liveKafkaSse && eventName === "workbench.connected") { const filters = recordValue(event.filters); const connectedSessionId = normalizeWorkbenchSessionId(filters?.sessionId); - const valid = event.liveOnly === true && event.replay === false && event.capabilities?.liveKafkaSse === true && connectedSessionId === sessionId; + const refreshReplay = realtimeCapabilities.kafkaRefreshReplay; + const deliveryValid = refreshReplay + ? event.deliverySemantics === "kafka-retention-then-live" && event.liveOnly === false && event.replay === true && event.lossPossible === false + : event.deliverySemantics === "live-only" && event.liveOnly === true && event.replay === false; + const valid = deliveryValid + && event.capabilities?.liveKafkaSse === true + && event.capabilities?.kafkaRefreshReplay === refreshReplay + && connectedSessionId === sessionId; liveRealtimeReadySessionId.value = valid ? connectedSessionId : null; if (!valid) error.value = "workbench_live_realtime_contract_invalid"; } @@ -1372,6 +1382,16 @@ export const useWorkbenchStore = defineStore("workbench", () => { const eventSessionId = firstNonEmptyString(event.sessionId); const ownerSessionId = traceOwnerSessionId(traceId, authoritySessionId ?? eventSessionId, true); if (!ownerSessionId) return; + if (workbenchLiveKafkaProjectionTarget(event) === "user") { + const messageId = firstNonEmptyString(event.userMessageId, event.messageId); + const previousUserMessage = messageId + ? (serverState.value.messagesBySessionId[ownerSessionId] ?? []).find((message) => (message.messageId ?? message.id) === messageId) ?? null + : null; + const userMessage = projectWorkbenchLiveKafkaUserMessage({ previous: previousUserMessage, traceId, sessionId: ownerSessionId, event, receivedAt }); + if (userMessage) reduceServerState({ type: "message.upsert", sessionId: ownerSessionId, message: userMessage }); + else error.value = "workbench_live_user_message_invalid"; + return; + } const previousMessage = (serverState.value.messagesBySessionId[ownerSessionId] ?? []).find((message) => messageMatchesTraceAuthority(message, traceId, authoritySessionId ?? eventSessionId, ownerSessionId, true)) ?? null; const message = projectWorkbenchLiveKafkaMessage({ previous: previousMessage, @@ -1626,9 +1646,16 @@ export const useWorkbenchStore = defineStore("workbench", () => { function applyRealtimeProjectionError(event: WorkbenchRealtimeEvent): void { const traceId = firstNonEmptyString(event.traceId, realtimeTraceId()); - if (!traceId) return; - const error = recordValue(event.error) ?? event; - const projection = normalizeProjectionDiagnostic(event) ?? projectionDiagnosticFromFailure({ code: firstNonEmptyString(error.code, "workbench_realtime_error") ?? "workbench_realtime_error", message: firstNonEmptyString(error.message, error.summary, "Workbench 实时状态更新异常，最新运行记录暂不可见。") ?? "Workbench 实时状态更新异常，最新运行记录暂不可见。", health: "degraded", diagnostic: normalizeErrorDiagnostic(error.diagnostic, recordValue(event.projection)?.diagnostic, recordValue(event)?.diagnostic), apiError: normalizeApiErrorRecord(error, "Workbench 实时状态更新异常，最新运行记录暂不可见。") }); + const errorRecord = recordValue(event.error) ?? event; + if (!traceId) { + const code = firstNonEmptyString(errorRecord.code, "workbench_realtime_error") ?? "workbench_realtime_error"; + const phase = firstNonEmptyString(event.phase, errorRecord.phase, "realtime") ?? "realtime"; + const message = firstNonEmptyString(errorRecord.message, errorRecord.summary, "Workbench 实时状态更新异常，最新运行记录暂不可见。") ?? "Workbench 实时状态更新异常，最新运行记录暂不可见。"; + liveRealtimeReadySessionId.value = null; + error.value = `${phase} · ${code} · ${message}`; + return; + } + const projection = normalizeProjectionDiagnostic(event) ?? projectionDiagnosticFromFailure({ code: firstNonEmptyString(errorRecord.code, "workbench_realtime_error") ?? "workbench_realtime_error", message: firstNonEmptyString(errorRecord.message, errorRecord.summary, "Workbench 实时状态更新异常，最新运行记录暂不可见。") ?? "Workbench 实时状态更新异常，最新运行记录暂不可见。", health: "degraded", diagnostic: normalizeErrorDiagnostic(errorRecord.diagnostic, recordValue(event.projection)?.diagnostic, recordValue(event)?.diagnostic), apiError: normalizeApiErrorRecord(errorRecord, "Workbench 实时状态更新异常，最新运行记录暂不可见。") }); applyProjectionDiagnostic(traceId, projection.projectionStatus ? projection : { ...projection, projectionStatus: "blocked", projectionHealth: projection.projectionHealth ?? "degraded" }); } @@ -1877,6 +1904,10 @@ export const useWorkbenchStore = defineStore("workbench", () => { rememberSessionDetail({ ...session, messages: projectedMessages, messageCount: session.messageCount ?? projectedMessages.length }); sessionsReady.value = true; currentRequest.value = null; + if (historyAuthorityPolicy.kafkaRetention) { + restartRealtime("apply-selected-session:kafka-retention"); + return; + } void hydrateTurnStatusAuthority(messages.value); void hydrateTerminalMessageDiagnostics(); void readTraceEventsForMessages(messages.value); @@ -1905,16 +1936,17 @@ export const useWorkbenchStore = defineStore("workbench", () => { if (!requestId) return null; const normalizedRequestId = normalizeWorkbenchSessionId(requestId); const messageLimit = sessionMessageProjectionWindowLimit(); - const eagerMessages = normalizedRequestId ? fetchSessionMessagesPage(normalizedRequestId, { limit: messageLimit, reason: "load-session:eager", force: true }) : null; + const eagerMessages = historyAuthorityPolicy.sessionMessagesHydrate && normalizedRequestId ? fetchSessionMessagesPage(normalizedRequestId, { limit: messageLimit, reason: "load-session:eager", force: true }) : null; const detail = await fetchSessionDetailPage(requestId, { reason: "load-session:detail", force: true }); if (!detail.ok) return null; const detailSession = sessionFromWorkbenchSession(detail.data?.session, { includeMessages: false }); const id = detailSession?.sessionId ?? normalizedRequestId ?? seed?.sessionId; if (!id) return null; - const messages = eagerMessages && id === normalizedRequestId ? await eagerMessages : await fetchSessionMessagesPage(id, { limit: messageLimit, reason: "load-session", force: true }); const fallbackMessages = seed?.sessionId === id ? seed.messages ?? [] : []; const base = detailSession ? { ...detailSession, messages: fallbackMessages } : seed; if (!base) return null; + if (!historyAuthorityPolicy.sessionMessagesHydrate) return { ...base, sessionId: id, messages: [], messageCount: base.messageCount ?? 0 }; + const messages = eagerMessages && id === normalizedRequestId ? await eagerMessages : await fetchSessionMessagesPage(id, { limit: messageLimit, reason: "load-session", force: true }); const page = messages.ok ? messages.data : null; const pageMessages = Array.isArray(page?.messages) ? await sealRestoredActiveTurnMessages(page.messages.map((message) => normalizeChatMessage(message as ChatMessage))) : fallbackMessages; return { ...base, sessionId: id, messages: pageMessages, messageCount: page?.total ?? pageMessages?.length ?? base.messageCount }; diff --git a/web/hwlab-cloud-web/src/types/global.d.ts b/web/hwlab-cloud-web/src/types/global.d.ts index b3ff42a3..3bafc4d3 100644 --- a/web/hwlab-cloud-web/src/types/global.d.ts +++ b/web/hwlab-cloud-web/src/types/global.d.ts @@ -10,6 +10,7 @@ declare global { workbench?: { realtimeFeatures?: { liveKafkaSse?: boolean; + kafkaRefreshReplay?: boolean; projectionRealtime?: boolean; }; debugCapabilities?: {