merge: pull request #491

This commit is contained in:
Codex
2026-05-26 23:07:17 +08:00
10 changed files with 205 additions and 37 deletions
+1
View File
@@ -27,6 +27,7 @@ HWLAB 是硬件实验室运行面和控制面项目。本文是 agent、指挥
## 工作区
- G14 分支固定 source workspace 是 G14 节点上的 `/root/hwlab`,固定使用 `G14` 分支和 `origin git@github.com:pikasTech/HWLAB.git`。在 G14 上进行代码、文档、GitOps render、Tekton/poller/Argo CD 修复或 CI/CD 验证前,必须先确认 `pwd``/root/hwlab``git status --short --branch``G14...origin/G14`;不满足时先停止并修正 workspace。
- G14 开发默认先以 `/root/hwlab` 做固定 repo 预检,再在 `/root/hwlab/.worktree/<task>` 从最新 `origin/G14` 创建独立 worktree 修改和提交;固定 repo 不作为并行任务 scratch 区,详见 [docs/reference/commander-collaboration.md](docs/reference/commander-collaboration.md)。
- G14 k3s 操作必须通过 UniDesk SSH route `G14:k3s` 执行,例如 `bun scripts/cli.ts ssh G14:k3s kubectl get pods -n hwlab-ci`;禁止使用 `ssh G14 k3s ...`。不要把 `/workspace/hwlab``/root/HWLAB`、D601 workspace、master-server checkout 或临时 clone 当作 G14 分支 source truth。
- Runner 和指挥常用工作区是 `/workspace/hwlab`;进入仓库先检查分支与工作树状态,详见 [docs/reference/commander-collaboration.md](docs/reference/commander-collaboration.md)。
- G14 CI/CD 由 `G14` source branch、G14 k3s Tekton 和 `G14-gitops` branch 驱动;需要构建、Playwright、check、发布预检或运行面验证时放到 G14 k3s/runner/CI/CD,不在 master server 跑重型验证。
@@ -16,6 +16,8 @@
- 常规 runner 和指挥官工作区是 `/workspace/hwlab`
- 进入仓库先检查分支和工作树状态。
- G14 人工/指挥官开发必须先在固定 repo `/root/hwlab``pwd``git status --short --branch``git remote -v``fetch` 预检,再在 `/root/hwlab/.worktree/<task>` 从最新 `origin/G14` 创建任务专属 worktree;代码、文档、测试补丁、提交和 PR 准备都在该独立 worktree 中完成。
- 固定 repo `/root/hwlab` 是 source truth 和 worktree 管理入口,不是并行任务 scratch 区;不要在固定 repo 根目录堆叠未提交改动,也不要复用其他任务遗留 `.worktree/<task>`
- D601 发布和 rollout 工作区是 `/home/ubuntu/workspace/hwlab`
- 不要清理、reset 或复用无关 runner worktree 作为发布真相。
+2 -1
View File
@@ -193,7 +193,8 @@ G14 PR、CI、CD 的判断应按以下顺序收敛真相,越靠前越接近最
bun scripts/cli.ts ssh G14:/root/hwlab shell 'git status --short --branch && git remote -v | sed -n "1,4p"'
```
- 只有 `/root/hwlab` 已经是 `G14...origin/G14` 时才继续;否则先从 `origin/G14` 建干净临时 worktree,再开始本轮修改。
- `/root/hwlab` 是固定 source workspace 和 worktree 管理入口;完成预检后,在 `/root/hwlab/.worktree/<task>` 从最新 `origin/G14` 创建任务专属 worktree,再开始本轮代码、文档、测试和提交修改。
- 只有独立 worktree 跟踪正确 base 且 `git status` 只包含本任务文件时才继续;不要在 `/root/hwlab` 根目录直接堆叠并行开发改动,也不要复用其他任务遗留 worktree。
- k3s 只走 `G14:k3s`;不要混用 D601 kubeconfig、master server 执行面或旧 SSH route 语法。
2. Source / PR 预检
@@ -1708,6 +1708,8 @@ test("Codex app-server partial assistant output without turn completion is a tim
assert.equal(Object.hasOwn(payload, "reply"), false);
assert.equal(payload.session.status, "timeout");
assert.equal(payload.providerTrace, undefined);
assert.equal(payload.runnerTrace.events.some((event) => event.label === "assistant:chunk"), false);
assert.equal(payload.runnerTrace.assistantStreams?.[0]?.chunkCount, 1);
assert.ok(payload.runnerTrace.events.some((event) => event.label === "assistant:item_completed"));
assert.ok(payload.runnerTrace.events.some((event) => event.label === "timeout:no_activity"));
assert.equal(payload.runnerTrace.events.some((event) => event.label === "turn:completed:idle_timeout_after_assistant"), false);
+96 -7
View File
@@ -1,7 +1,7 @@
import { randomUUID } from "node:crypto";
const DEFAULT_MAX_TRACES = 256;
const DEFAULT_MAX_EVENTS = 600;
const DEFAULT_MAX_EVENTS = 6000;
const TEXT_LIMIT = 1200;
const SECRET_PATTERN = /\b(?:sk-[A-Za-z0-9_-]+|OPENAI_API_KEY|DATABASE_URL|secretRef:[^\s,;]+|secret|token|password|passwd|credential|private[_ -]?key|kubeconfig|BEGIN [A-Z ]*PRIVATE KEY)\b/giu;
@@ -24,6 +24,7 @@ export function createCodeAgentTraceStore(options = {}) {
finishedAt: null,
nextSeq: 1,
events: [],
assistantStreams: new Map(),
listeners: new Set(),
meta: {
runnerKind: meta.runnerKind ?? null,
@@ -79,10 +80,62 @@ export function createCodeAgentTraceStore(options = {}) {
return normalized;
}
function appendAssistantDelta(traceId, delta = {}, meta = {}) {
const trace = ensure(traceId, meta);
const normalized = normalizeAssistantDelta(delta, {
traceId: trace.traceId,
now: meta.now,
fallbackRunnerKind: trace.meta.runnerKind
});
if (!normalized) return null;
const key = normalized.itemId || "assistant-message";
const previous = trace.assistantStreams.get(key) ?? {
traceId: trace.traceId,
itemId: key,
type: "assistant_message_stream",
status: "streaming",
label: "assistant:stream",
createdAt: normalized.createdAt,
updatedAt: normalized.createdAt,
runnerKind: normalized.runnerKind,
sessionId: normalized.sessionId,
sessionStatus: normalized.sessionStatus,
turn: normalized.turn,
threadId: normalized.threadId,
turnId: normalized.turnId,
waitingFor: normalized.waitingFor,
chunkCount: 0,
text: "",
lastChunk: "",
valuesPrinted: false
};
const text = `${previous.text ?? ""}${normalized.chunk}`;
const stream = dropUndefined({
...previous,
updatedAt: normalized.createdAt,
runnerKind: normalized.runnerKind ?? previous.runnerKind,
sessionId: normalized.sessionId ?? previous.sessionId,
sessionStatus: normalized.sessionStatus ?? previous.sessionStatus,
turn: normalized.turn ?? previous.turn,
threadId: normalized.threadId ?? previous.threadId,
turnId: normalized.turnId ?? previous.turnId,
waitingFor: normalized.waitingFor ?? previous.waitingFor,
chunkCount: previous.chunkCount + 1,
text,
lastChunk: normalized.chunk,
valuesPrinted: false
});
trace.assistantStreams.set(key, stream);
trace.updatedAt = normalized.createdAt;
notify(trace, null);
return stream;
}
function snapshot(traceId, extra = {}) {
const trace = traces.get(cleanTraceId(traceId));
if (!trace) return emptySnapshot(traceId, extra);
const events = trace.events.map((event) => ({ ...event }));
const assistantStreams = assistantStreamsSnapshot(trace);
const lastEvent = events.at(-1) ?? null;
return {
traceId: trace.traceId,
@@ -93,10 +146,11 @@ export function createCodeAgentTraceStore(options = {}) {
finishedAt: trace.finishedAt,
eventCount: events.length,
events,
assistantStreams,
eventLabels: events.map((event) => event.label).filter(Boolean),
lastEvent,
elapsedMs: elapsedMs(trace.startedAt, trace.finishedAt ?? trace.updatedAt),
waitingFor: lastWaitingFor(events),
waitingFor: lastWaitingFor(events) ?? assistantStreams.at(-1)?.waitingFor ?? null,
runnerKind: extra.runnerKind ?? trace.meta.runnerKind ?? null,
workspace: extra.workspace ?? trace.meta.workspace ?? null,
sandbox: extra.sandbox ?? trace.meta.sandbox ?? null,
@@ -136,6 +190,7 @@ export function createCodeAgentTraceStore(options = {}) {
return {
ensure,
append,
appendAssistantDelta,
snapshot,
subscribe,
clear
@@ -166,6 +221,10 @@ export function createCodeAgentTraceRecorder({
}, baseMeta);
}
function appendAssistantDelta(delta = {}) {
return traceStore.appendAssistantDelta(traceId, delta, baseMeta);
}
function snapshot(extra = {}) {
return traceStore.snapshot(traceId, { runnerKind, workspace, sandbox, sessionMode, implementationType, ...extra });
}
@@ -191,6 +250,7 @@ export function createCodeAgentTraceRecorder({
finishedAt: extra.finishedAt ?? current.finishedAt ?? timestampFor(now),
updatedAt: current.updatedAt,
events: current.events,
assistantStreams: current.assistantStreams,
eventLabels: current.eventLabels,
lastEvent: current.lastEvent,
elapsedMs: current.elapsedMs,
@@ -204,6 +264,7 @@ export function createCodeAgentTraceRecorder({
return {
traceId,
append,
appendAssistantDelta,
snapshot,
runnerTrace
};
@@ -229,6 +290,7 @@ export function runnerTraceFromSnapshot(snapshot = {}, extra = {}) {
finishedAt: extra.finishedAt ?? snapshot.finishedAt ?? snapshot.updatedAt ?? null,
updatedAt: snapshot.updatedAt ?? null,
events: Array.isArray(snapshot.events) ? snapshot.events : [],
assistantStreams: Array.isArray(snapshot.assistantStreams) ? snapshot.assistantStreams : [],
eventLabels: Array.isArray(snapshot.eventLabels) ? snapshot.eventLabels : [],
lastEvent: snapshot.lastEvent ?? null,
elapsedMs: snapshot.elapsedMs ?? null,
@@ -276,6 +338,30 @@ function normalizeTraceEvent(event, { traceId, seq, now, fallbackRunnerKind } =
});
}
function normalizeAssistantDelta(delta, { traceId, now, fallbackRunnerKind } = {}) {
const chunk = safeText(delta.chunk ?? delta.delta, 400);
if (!chunk) return null;
return dropUndefined({
traceId,
itemId: safeText(delta.itemId, 180),
chunk,
createdAt: timestampFor(now),
runnerKind: safeText(delta.runnerKind ?? fallbackRunnerKind, 140),
sessionId: safeText(delta.sessionId, 180),
sessionStatus: safeText(delta.sessionStatus, 80),
turn: typeof delta.turn === "number" ? delta.turn : undefined,
threadId: safeText(delta.threadId, 180),
turnId: safeText(delta.turnId, 180),
waitingFor: safeText(delta.waitingFor, 220),
valuesPrinted: false
});
}
function assistantStreamsSnapshot(trace) {
if (!(trace?.assistantStreams instanceof Map)) return [];
return [...trace.assistantStreams.values()].map((stream) => ({ ...stream }));
}
function labelFor({ type, status, toolName }) {
if (type === "session") return `session:${status}`;
if (type === "prompt") return `prompt:${status}`;
@@ -288,6 +374,7 @@ function labelFor({ type, status, toolName }) {
}
function notify(trace, event) {
const assistantStreams = assistantStreamsSnapshot(trace);
const snapshot = {
...emptySnapshot(trace.traceId),
...{
@@ -299,18 +386,19 @@ function notify(trace, event) {
finishedAt: trace.finishedAt,
eventCount: trace.events.length,
events: trace.events.map((item) => ({ ...item })),
assistantStreams,
eventLabels: trace.events.map((item) => item.label).filter(Boolean),
lastEvent: event,
lastEvent: event ?? trace.events.at(-1) ?? null,
elapsedMs: elapsedMs(trace.startedAt, trace.finishedAt ?? trace.updatedAt),
waitingFor: lastWaitingFor(trace.events),
waitingFor: lastWaitingFor(trace.events) ?? assistantStreams.at(-1)?.waitingFor ?? null,
runnerKind: trace.meta.runnerKind,
workspace: trace.meta.workspace,
sandbox: trace.meta.sandbox,
sessionMode: trace.meta.sessionMode,
implementationType: trace.meta.implementationType,
sessionId: event.sessionId ?? null,
sessionStatus: event.sessionStatus ?? null,
sessionLifecycleStatus: event.sessionLifecycleStatus ?? null,
sessionId: event?.sessionId ?? assistantStreams.at(-1)?.sessionId ?? null,
sessionStatus: event?.sessionStatus ?? assistantStreams.at(-1)?.sessionStatus ?? null,
sessionLifecycleStatus: event?.sessionLifecycleStatus ?? null,
outputTruncated: trace.events.some((item) => item.outputTruncated === true),
valuesPrinted: false
}
@@ -335,6 +423,7 @@ function emptySnapshot(traceId, extra = {}) {
finishedAt: null,
eventCount: 0,
events: [],
assistantStreams: [],
eventLabels: [],
lastEvent: null,
elapsedMs: null,
+6 -10
View File
@@ -580,11 +580,9 @@ export function createCodexStdioSessionManager(options = {}) {
sessionStatus: session.status,
turn: session.turn
});
traceRecorder.append({
type: "assistant_message",
status: "chunk",
label: "assistant:chunk",
chunk: assistantContent.slice(0, 400),
traceRecorder.appendAssistantDelta?.({
itemId: turnResult.turnId ?? "assistant-message",
chunk: assistantContent,
sessionId: session.sessionId,
sessionStatus: session.status,
turn: session.turn,
@@ -1705,11 +1703,9 @@ function createAppServerTurnState({ traceRecorder, session } = {}) {
const delta = String(params?.delta ?? "");
assistantText += delta;
if (delta) {
appendTrace({
type: "assistant_message",
status: "chunk",
label: "assistant:chunk",
chunk: delta.slice(0, 400),
traceRecorder?.appendAssistantDelta?.({
itemId: optionalId(params?.itemId),
chunk: delta,
sessionId: session?.sessionId,
sessionStatus: session?.status,
turn: session?.turn,
+33
View File
@@ -27,6 +27,39 @@ const CODEX_STDIO_FEASIBILITY_BLOCKERS = new Set([
"codex_stdio_egress_boundary"
]);
test("code agent trace store keeps assistant deltas outside event count", () => {
const traceStore = createCodeAgentTraceStore({ maxEvents: 6000 });
const traceId = "trc_trace-store-assistant-stream";
traceStore.append(traceId, { type: "request", status: "accepted", label: "request:accepted" });
for (let index = 0; index < 6200; index += 1) {
traceStore.appendAssistantDelta(traceId, {
itemId: "item_assistant_stream",
chunk: `chunk-${index} `,
waitingFor: "turn/completed"
});
}
const snapshot = traceStore.snapshot(traceId);
assert.equal(snapshot.eventCount, 1);
assert.equal(snapshot.events.length, 1);
assert.equal(snapshot.events[0].label, "request:accepted");
assert.equal(snapshot.assistantStreams.length, 1);
assert.equal(snapshot.assistantStreams[0].chunkCount, 6200);
assert.equal(snapshot.waitingFor, "turn/completed");
});
test("code agent trace store retains six thousand regular events", () => {
const traceStore = createCodeAgentTraceStore({ maxEvents: 6000 });
const traceId = "trc_trace-store-regular-events";
for (let index = 0; index < 6001; index += 1) {
traceStore.append(traceId, { type: "trace", status: "observed", label: `event:${index}` });
}
const snapshot = traceStore.snapshot(traceId);
assert.equal(snapshot.eventCount, 6000);
assert.equal(snapshot.events.length, 6000);
assert.equal(snapshot.events[0].label, "event:1");
assert.equal(snapshot.events.at(-1).label, "event:6000");
});
function assertCodexStdioFeasibilityBlocker(value, label = "codex stdio blocker") {
assert.equal(CODEX_STDIO_FEASIBILITY_BLOCKERS.has(value), true, `${label}: ${value}`);
}
+46 -1
View File
@@ -1603,11 +1603,17 @@ function noteCurrentRequestTraceActivity(snapshot) {
const request = state.currentRequest;
if (!request || request.traceId !== snapshot.traceId) return;
const eventCount = Number.isInteger(snapshot.eventCount) ? snapshot.eventCount : snapshot.events.length;
if (eventCount <= (request.traceEventCount ?? 0)) {
const assistantChunkCount = traceAssistantStreamChunkCount(snapshot);
const traceUpdatedAt = Date.parse(snapshot.updatedAt ?? "");
const previousUpdatedAt = Date.parse(request.lastTraceUpdatedAt ?? "");
const hasAssistantActivity = assistantChunkCount > (request.assistantChunkCount ?? 0);
const hasTimestampActivity = Number.isFinite(traceUpdatedAt) && (!Number.isFinite(previousUpdatedAt) || traceUpdatedAt > previousUpdatedAt);
if (eventCount <= (request.traceEventCount ?? 0) && !hasAssistantActivity && !hasTimestampActivity) {
if (snapshot.waitingFor) request.waitingFor = snapshot.waitingFor;
return;
}
request.traceEventCount = eventCount;
request.assistantChunkCount = assistantChunkCount;
request.lastActivityAt = Date.now();
request.lastActivityIso = new Date(request.lastActivityAt).toISOString();
request.lastTraceUpdatedAt = snapshot.updatedAt ?? request.lastTraceUpdatedAt;
@@ -1615,6 +1621,12 @@ function noteCurrentRequestTraceActivity(snapshot) {
request.lastEventLabel = snapshot.lastEvent?.label ?? snapshot.events.at(-1)?.label ?? request.lastEventLabel;
}
function traceAssistantStreamChunkCount(trace) {
return Array.isArray(trace?.assistantStreams)
? trace.assistantStreams.reduce((total, stream) => total + Math.max(0, Number(stream?.chunkCount ?? 0)), 0)
: 0;
}
function latestTraceSnapshot(traceId) {
for (const message of [...state.chatMessages].reverse()) {
if (message.traceId === traceId && message.runnerTrace) return message.runnerTrace;
@@ -1645,6 +1657,7 @@ function runnerTraceFromSnapshot(snapshot, previous = null) {
elapsedMs: snapshot.elapsedMs ?? previous?.elapsedMs,
waitingFor: snapshot.waitingFor ?? previous?.waitingFor,
events: snapshot.events,
assistantStreams: Array.isArray(snapshot.assistantStreams) ? snapshot.assistantStreams : previous?.assistantStreams ?? [],
eventLabels: snapshot.eventLabels ?? snapshot.events.map((item) => item.label).filter(Boolean),
lastEvent: snapshot.lastEvent ?? snapshot.events.at(-1) ?? null,
outputTruncated: snapshot.outputTruncated === true,
@@ -4652,6 +4665,7 @@ function seedTraceMessage(options = {}) {
runnerTrace: {
traceId,
events,
assistantStreams: Array.isArray(options.assistantStreams) ? options.assistantStreams : [],
eventCount: Number.isInteger(options.eventCount) ? options.eventCount : events.length,
eventsCompacted: options.eventsCompacted === true,
eventWindow: options.eventWindow ?? null,
@@ -4818,6 +4832,7 @@ function setConversationScrollTop(top, { user = true } = {}) {
function traceDisplayRows(trace, events) {
const rows = [];
const assistantRows = traceAssistantStreamRows(trace);
let noisyRun = [];
let toolOutputRun = [];
const flushNoisyRun = () => {
@@ -4851,9 +4866,38 @@ function traceDisplayRows(trace, events) {
}
flushNoisyRun();
flushToolOutputRun();
rows.push(...assistantRows);
return rows.length > 0 ? rows : events.map((event) => traceDisplayRow(trace, event, { includeNoise: true })).filter(Boolean);
}
function traceAssistantStreamRows(trace) {
const streams = Array.isArray(trace?.assistantStreams) ? trace.assistantStreams.filter(Boolean) : [];
return streams.map((stream) => traceAssistantStreamRow(trace, stream)).filter(Boolean);
}
function traceAssistantStreamRow(trace, stream) {
const chunkCount = Math.max(0, Number(stream?.chunkCount ?? 0));
if (chunkCount <= 0) return null;
const clock = traceClock(stream.updatedAt ?? stream.createdAt);
const total = formatTraceDuration(traceRelativeMs(trace, { createdAt: stream.updatedAt ?? stream.createdAt }));
return {
seq: null,
tone: "source",
header: `${clock} total=${total} stream assistant message x${chunkCount}`,
body: traceAssistantStreamBody(stream, chunkCount)
};
}
function traceAssistantStreamBody(stream, chunkCount) {
const message = cleanTraceText(stream?.text ?? "");
const waitingFor = stream?.waitingFor ? `waiting=${stream.waitingFor}` : null;
return [
`chunks=${chunkCount} assistant message chunks`,
waitingFor,
message ? `message=${message}` : null
].filter(Boolean).join("\n");
}
function traceToolOutputSummaryRows(trace, events) {
const visibleEvents = events.filter(Boolean);
if (visibleEvents.length === 0) return [];
@@ -5206,6 +5250,7 @@ function messageTraceJson(message, trace, events) {
eventsCompacted: trace?.eventsCompacted === true,
eventWindow: trace?.eventWindow ?? null,
lastEvent: trace?.lastEvent ?? events.at(-1) ?? null,
assistantStreams: Array.isArray(trace?.assistantStreams) ? trace.assistantStreams : [],
events
}, null, 2);
}
+7
View File
@@ -838,6 +838,11 @@ assert.match(app, /function maybeReplayFullTraceForMessage/);
assert.match(app, /function replayFullTrace/);
assert.match(app, /eventsCompacted:\s*snapshot\.eventsCompacted === true/);
assert.match(app, /eventWindow:\s*snapshot\.eventWindow/);
assert.match(app, /assistantStreams:\s*Array\.isArray\(snapshot\.assistantStreams\)/);
assert.match(app, /function traceAssistantStreamRows/);
assert.match(app, /function traceAssistantStreamRow/);
assert.match(app, /function traceAssistantStreamBody/);
assert.match(app, /function traceAssistantStreamChunkCount/);
assert.match(app, /fullTraceLoaded:/);
assert.match(app, /eventCount:\s*Number\.isInteger\(trace\?\.eventCount\)/);
assert.match(app, /traceDetailsOpen:\s*new Map\(\)/);
@@ -897,7 +902,9 @@ assert.match(app, /tool gateway\.shell/);
assert.match(app, /function traceNoiseSummaryRow/);
assert.match(styles, /max-height:\s*min\(520px,\s*54dvh\)/);
assert.match(app, /assistant message x\$\{visibleEvents\.length\}/);
assert.match(app, /assistant message x\$\{chunkCount\}/);
assert.match(app, /chunks=\$\{assistantChunks\.length\} assistant message chunks/);
assert.match(app, /chunks=\$\{chunkCount\} assistant message chunks/);
assert.doesNotMatch(app, /compressed=\$\{assistantChunks\.length\} assistant chunks/);
assert.match(app, /message=\$\{text\}/);
assert.match(app, /waiting first assistant token/);
@@ -107,30 +107,22 @@ test("trace display full means complete readable timeline, not compacted result
status: "started",
message: "accepted"
},
{
label: "assistant:chunk",
type: "assistant_message",
status: "chunk",
chunk: "first readable sentence "
},
{
label: "assistant:chunk",
type: "assistant_message",
status: "chunk",
chunk: "middle detail "
},
{
label: "assistant:chunk",
type: "assistant_message",
status: "chunk",
chunk: "final conclusion"
},
{
label: "item/commandExecution/outputDelta",
type: "tool_call",
status: "output_chunk",
outputSummary: `${gatewayPayload("req_1", "op_one")}${gatewayPayload("req_2", "op_two")}`
}
],
assistantStreams: [
{
label: "assistant:stream",
status: "streaming",
chunkCount: 3,
text: "first readable sentence middle detail final conclusion",
updatedAt: "2026-05-26T12:29:28.000Z",
waitingFor: "turn/completed"
}
]
});
});