fix: reduce device pod cli runner friction

This commit is contained in:
Codex
2026-05-28 09:46:24 +08:00
parent 6bb7bb8c9e
commit 904649533b
7 changed files with 202 additions and 45 deletions
@@ -210,6 +210,8 @@ Workbench 与 Code Agent 的用户请求必须是短连接 submit + 短连接 re
Code Agent backend 的 completed 语义只能来自真实 Codex app-server `turn/completed` 成功事件。`item/agentMessage/delta``item/completed`、已有 assistant 文本、transport close 或 activity idle timeout 都不能单独升级成 `status: "completed"`。如果已经收到部分 assistant 文本但没有收到 `turn/completed`,终态必须是 timeout/partial blocker,并保留 trace、session、thread、partial output 摘要和可重试提示;Workbench 只能显示“部分回复/超时”,不能标 DEV-LIVE reply pass。
`Codex app-server transport closed after partial assistant output but before turn/completed` 不能单独判定为 CI/CD 滚动中断。排查必须同时看三类证据:失败 trace 的 `providerTrace`/`runnerTrace`、对应 `hwlab-cloud-api` Pod 的 restart/age、以及同一时间窗口内的 Kubernetes rollout/kill 事件。若 Pod 在 trace 开始前已经稳定且 `RESTARTS=0`,应归类为 Codex app-server transport/session 失败,提示用户用新 session 重试;只有 trace 时间窗口内存在当前会话所在 Pod 的删除、重建或 restart 证据时,才归类为滚动导致的中断。失败响应也必须尽量保留 `providerTrace`,即使 `terminalStatus=failed`,避免前端把“providerTrace 缺失”误报成未知降级。
cloud-web 同源代理必须把短连接语义原样转发给 cloud-api,至少包括 `Prefer: respond-async``X-HWLAB-Short-Connection``X-Trace-Id`。如果这些 header 在 cloud-web 层被过滤,cloud-api 会把同一个请求当成长同步请求处理,用户入口会表现为 `17666` 卡住或代理超时,而 `17667` 直连 cloud-api 正常。此类问题应先比对同一 trace 在 `17666``17667` 的 submit 行为,再修代理 header 透传,而不是调大前端等待时间。
`/v1/agent/chat/result/<traceId>` 是终态摘要接口,不是完整 trace 下载接口。它可以携带压缩后的 `runnerTrace` 窗口用于传输保活,但 Workbench 用户界面不得把该窗口显示为“压缩窗口”或“显示全部”。只要结果或轮询快照声明 `eventsCompacted=true`,前端必须自动请求 `/v1/agent/chat/trace/<traceId>` 并用完整 trace 替换可视事件线;回放完成前只能显示“完整 trace 回放中/当前已载入”状态。result 响应仍必须保留 `eventCount``lastEvent``providerTrace``threadId/sessionId` 和终态 reply/blocker;完整 trace 只能从 `/v1/agent/chat/trace/<traceId>`、复制 JSON 或下载 trace 入口取得。默认 result trace 窗口上限由 `HWLAB_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT` 控制;不要把数百个大 chunk 原样塞进 result 响应,避免 cloud-web 代理层或浏览器 fetch 把“正常执行中的大响应”表现成 503、非 JSON 或空响应。
+3 -1
View File
@@ -223,6 +223,7 @@ export async function handleCodeAgentChat(params = {}, options = {}) {
if (error.skills !== undefined) payload.skills = error.skills;
if (error.runner !== undefined) payload.runner = error.runner;
if (error.runnerTrace !== undefined) payload.runnerTrace = error.runnerTrace;
if (error.providerTrace !== undefined) payload.providerTrace = error.providerTrace;
if (payload.runnerTrace === undefined) {
payload.runnerTrace = traceRecorder.runnerTrace({
runnerKind: CODEX_STDIO_RUNNER_KIND,
@@ -917,6 +918,7 @@ async function callCodexStdioRunner({ message, conversationId, sessionId, traceI
idleMs: error.idleMs,
lastActivityAt: error.lastActivityAt,
waitingFor: error.waitingFor,
providerTrace: error.providerTrace,
stage: error.stage,
missingTools: error.missingTools,
availability: await describeCodeAgentAvailability(env, { codexStdioManager: manager, workspace })
@@ -1404,7 +1406,7 @@ function errorTaxonomy(code, error = {}) {
layer: "runner",
category: "runner_blocked",
retryable: true,
userMessage: "Codex stdio runner 执行失败,可稍后重试。"
userMessage: "Codex stdio runner 执行失败;若 trace 显示 transport closed after partial assistant output,请新建 session 重试,并结合 providerTrace、runnerTrace 与 pod restart/rollout 事件判断是否为滚动中断。"
},
codex_stdio_network_failed: {
layer: "runner",
@@ -1802,6 +1802,9 @@ test("Codex app-server failed turn is not wrapped as completed assistant output"
assert.ok(payload.longLivedSessionGate.blockers.some((blocker) => blocker.code === "session_failed"));
assert.equal(payload.error.message.includes("sk-test-secret"), false);
assert.equal(payload.error.message.includes("TOKEN=abc123"), false);
assert.equal(payload.providerTrace.protocol, "codex-app-server-jsonrpc-stdio");
assert.equal(payload.providerTrace.terminalStatus, "failed");
assert.equal(payload.providerTrace.toolName, "codex-app-server.thread/start+turn/start");
assert.deepEqual(calls.filter((call) => call.method !== "initialize").map((call) => call.method), ["thread/start", "turn/start"]);
assert.equal(classifyCodexRunnerCapability(payload, { httpStatus: 200 }).capabilityPass, false);
await rm(fakeCodex.root, { recursive: true, force: true });
+24 -14
View File
@@ -527,13 +527,18 @@ export function createCodexStdioSessionManager(options = {}) {
waitingFor: "assistant-message"
});
if (turnResult.terminalStatus !== "completed") {
const providerTrace = codexStdioProviderTrace({ availability, toolName, turnResult, session });
throw codexStdioError("codex_stdio_failed", redactText(turnResult.terminalError || `Codex app-server turn finished with status=${turnResult.terminalStatus || "unknown"}.`), {
availability,
session,
terminalStatus: turnResult.terminalStatus,
threadId: turnResult.threadId,
turnId: turnResult.turnId,
appServerExit: turnResult.appServerExit
appServerExit: turnResult.appServerExit,
idleMs: turnResult.idleMs,
lastActivityAt: turnResult.lastActivityAt,
waitingFor: turnResult.waitingFor,
providerTrace
});
}
const assistantContent = redactText(firstNonEmpty(turnResult.finalResponse, turnResult.assistantText));
@@ -633,19 +638,7 @@ export function createCodexStdioSessionManager(options = {}) {
outputTruncated: sidecar.outputTruncated
}),
capabilityLevel: CODEX_STDIO_CAPABILITY_LEVEL,
providerTrace: {
transport: "stdio",
protocol: CODEX_APP_SERVER_PROTOCOL,
wireApi: CODEX_APP_SERVER_WIRE_API,
runnerKind: CODEX_STDIO_RUNNER_KIND,
implementationType: CODEX_STDIO_IMPLEMENTATION_TYPE,
command: `${availability.command} app-server --listen stdio://`,
toolName,
threadId: turnResult.threadId ?? session.threadId ?? null,
turnId: turnResult.turnId ?? null,
terminalStatus: turnResult.terminalStatus,
valuesPrinted: false
}
providerTrace: codexStdioProviderTrace({ availability, toolName, turnResult, session })
};
} catch (error) {
closeRpcClient();
@@ -2035,6 +2028,23 @@ function summarizeAppServerTurn(turnResult) {
].filter(Boolean).join(" ");
}
function codexStdioProviderTrace({ availability, toolName, turnResult, session } = {}) {
return {
transport: "stdio",
protocol: CODEX_APP_SERVER_PROTOCOL,
wireApi: CODEX_APP_SERVER_WIRE_API,
runnerKind: CODEX_STDIO_RUNNER_KIND,
implementationType: CODEX_STDIO_IMPLEMENTATION_TYPE,
command: `${availability?.command || "/app/node_modules/.bin/codex"} app-server --listen stdio://`,
toolName,
threadId: turnResult?.threadId ?? session?.threadId ?? null,
turnId: turnResult?.turnId ?? null,
terminalStatus: turnResult?.terminalStatus ?? null,
appServerExit: turnResult?.appServerExit ?? null,
valuesPrinted: false
};
}
function codexAppServerTextInput(text) {
return [{ type: "text", text: String(text ?? ""), text_elements: [] }];
}
+12
View File
@@ -36,6 +36,18 @@ not the HWLAB runner contract and cause stale instructions.
- If a command returns `profile-missing`, run profile bootstrap. If it returns a
device-host-cli JSON failure, fix the named device-pod operation or host CLI;
do not bypass the profile with generic gateway shell.
- Workspace search follows a compact ripgrep-like shape:
`device-pod-71-freq:workspace:/ rg "printf|main" projects/01_baseline -g "*.c" -g "*.h" -l`.
The optional path after the pattern is honored as the search root; use it to
avoid scanning vendor PDFs or generated text outside the project under test.
- For debug probe jobs, `download start` is mutating and requires
`--approved --reason`, but `download status <jobId>` is read-only and must be
queried without approval friction. If `download status` includes UART capture,
read the top-level `hostUartCapture` field first before falling back to job
files or direct `tran`.
- UART ids are slash-normalized by the CLI, but write them as `uart/1` in
prompts and commands. Do not write `uart / 1`; that form is only tolerated for
recovery from model spacing drift.
Fast build smoke for `device-pod-71-freq`:
@@ -16,27 +16,33 @@ function parseGlobalCli(argv) {
for (let i = 0; i < argv.length; i += 1) {
const item = argv[i];
if (item === '--profile') {
options.profile = argv[i + 1];
options.profile = cleanArg(argv[i + 1]);
i += 1;
} else if (item.startsWith('--profile=')) {
options.profile = item.slice('--profile='.length);
options.profile = cleanArg(item.slice('--profile='.length));
} else if (item === '--pod-id') {
options.podId = argv[i + 1];
options.podId = cleanArg(argv[i + 1]);
i += 1;
} else if (item.startsWith('--pod-id=')) {
options.podId = item.slice('--pod-id='.length);
options.podId = cleanArg(item.slice('--pod-id='.length));
} else if (item === '--workspace') {
options.workspace = argv[i + 1];
options.workspace = cleanArg(argv[i + 1]);
i += 1;
} else if (item.startsWith('--workspace=')) {
options.workspace = item.slice('--workspace='.length);
options.workspace = cleanArg(item.slice('--workspace='.length));
} else {
args.push(item);
args.push(cleanArg(item));
}
}
return { options, args };
}
function cleanArg(value) {
const text = String(value ?? '');
if ((text.startsWith('"') && text.endsWith('"')) || (text.startsWith("'") && text.endsWith("'"))) return text.slice(1, -1);
return text;
}
function selectedPodId() {
return GLOBAL_CLI.options.podId || process.env.DEVICE_POD_ID || DEFAULT_POD_ID;
}
@@ -182,27 +188,44 @@ function parseArgs(args) {
const out = { _: [] };
for (let i = 0; i < args.length; i += 1) {
const arg = args[i];
if (/^-[A-Za-z]$/u.test(arg)) {
const key = arg.slice(1);
const next = args[i + 1];
if (['g', 't'].includes(key) && next !== undefined && !next.startsWith('-')) {
setOption(out, key, cleanArg(next));
i += 1;
} else {
setOption(out, key, true);
}
continue;
}
if (!arg.startsWith('--')) {
out._.push(arg);
out._.push(cleanArg(arg));
continue;
}
const eq = arg.indexOf('=');
if (eq >= 0) {
out[arg.slice(2, eq)] = arg.slice(eq + 1);
setOption(out, arg.slice(2, eq), cleanArg(arg.slice(eq + 1)));
continue;
}
const key = arg.slice(2);
const next = args[i + 1];
if (next !== undefined && !next.startsWith('--')) {
out[key] = next;
setOption(out, key, cleanArg(next));
i += 1;
} else {
out[key] = true;
setOption(out, key, true);
}
}
return out;
}
function setOption(out, key, value) {
if (out[key] === undefined) out[key] = value;
else if (Array.isArray(out[key])) out[key].push(value);
else out[key] = [out[key], value];
}
function findFile(candidates) {
for (const candidate of candidates.filter(Boolean)) {
if (fs.existsSync(candidate)) return candidate;
@@ -293,26 +316,82 @@ function walkFiles(root, rel, results, limit = 5000) {
}
}
function rgWorkspace(pattern, rel) {
function rgWorkspace(pattern, rel, options = {}) {
const profile = loadProfile();
const root = resolveWorkspacePath(profile, rel || '.');
const files = [];
const stat = fs.statSync(root);
if (stat.isFile()) files.push(path.relative(profile.__workspaceRoot, root));
else walkFiles(root, '.', files);
else {
const nested = [];
const rootRel = path.relative(profile.__workspaceRoot, root) || '.';
walkFiles(root, '.', nested);
files.push(...nested.map((fileRel) => rootRel === '.' ? fileRel : path.join(rootRel, fileRel)));
}
const re = new RegExp(pattern, 'i');
const matches = [];
const fileMatches = [];
const filesWithMatches = Boolean(options.l || options.filesWithMatches || options['files-with-matches'] || options._?.includes('-l'));
const globs = normalizeList(options.g).concat(normalizeList(options.glob), normalizeList(options.iglob));
const types = normalizeList(options.t).concat(normalizeList(options.type));
for (const fileRel of files) {
if (matches.length >= 100) break;
if (!rgFileAllowed(fileRel, { globs, types })) continue;
if (!filesWithMatches && matches.length >= 100) break;
const full = resolveWorkspacePath(profile, fileRel);
const buffer = fs.readFileSync(full);
if (buffer.includes(0)) continue;
const lines = buffer.toString('utf8').split(/\r?\n/u);
lines.forEach((line, index) => {
if (matches.length < 100 && re.test(line)) matches.push({ path: fileRel, line: index + 1, text: line.slice(0, 500) });
});
for (let index = 0; index < lines.length; index += 1) {
const line = lines[index];
if (!re.test(line)) continue;
if (filesWithMatches) {
fileMatches.push(fileRel);
break;
}
if (matches.length < 100) matches.push({ path: fileRel, line: index + 1, text: line.slice(0, 500) });
}
}
ok('workspace.rg', { pattern, root: path.relative(profile.__workspaceRoot, root) || '.', count: matches.length, matches });
ok('workspace.rg', { pattern, root: path.relative(profile.__workspaceRoot, root) || '.', count: filesWithMatches ? fileMatches.length : matches.length, files: filesWithMatches ? fileMatches : undefined, matches: filesWithMatches ? undefined : matches, options: { globs, types, filesWithMatches } });
}
function normalizeList(value) {
if (value === undefined || value === false) return [];
return (Array.isArray(value) ? value : [value]).map((item) => String(item)).filter(Boolean);
}
function rgFileAllowed(fileRel, options = {}) {
const rel = fileRel.replace(/\\/gu, '/');
if (options.types?.length) {
const ext = path.extname(rel).toLowerCase();
const allowed = new Set();
for (const type of options.types) {
if (type === 'c') ['.c', '.h'].forEach((item) => allowed.add(item));
else if (type === 'cc') ['.cc', '.cpp', '.cxx', '.hpp', '.hh', '.hxx'].forEach((item) => allowed.add(item));
}
if (allowed.size && !allowed.has(ext)) return false;
}
if (!options.globs?.length) return true;
return options.globs.some((glob) => matchSimpleGlob(rel, glob));
}
function matchSimpleGlob(fileRel, glob) {
const raw = String(glob || '').replace(/\\/gu, '/');
const target = raw.includes('/') ? fileRel : path.basename(fileRel);
return new RegExp(`^${globToRegExp(raw)}$`, 'iu').test(target);
}
function globToRegExp(glob) {
let out = '';
for (let i = 0; i < glob.length; i += 1) {
const char = glob[i];
if (char === '*' && glob[i + 1] === '*') {
out += '.*';
i += 1;
} else if (char === '*') out += '[^/]*';
else if (char === '?') out += '[^/]';
else out += char.replace(/[.+^${}()|[\]\\]/gu, '\\$&');
}
return out;
}
function workspaceApplyPatch(baseRel, options = {}) {
@@ -748,7 +827,7 @@ async function serialPorts() {
}
function resolveUart(profile, uartId, options = {}) {
const normalized = String(uartId || 'uart/1').replace(/^\/+/, '');
const normalized = String(uartId || 'uart/1').trim().replace(/\\/gu, '/').replace(/\s*\/\s*/gu, '/').replace(/^\/+/, '');
const item = (profile.ioInterface?.uart || []).find((entry) => entry.id === normalized);
if (!item) throw new Error(`uart profile not found: ${normalized}`);
const resolved = { ...item };
@@ -899,7 +978,7 @@ async function main() {
if (group === 'workspace') {
if (command === 'ls') return listWorkspace(rest[0] || '.');
if (command === 'cat') return catWorkspace(rest[0], parseArgs(rest.slice(1)).limit);
if (command === 'rg') return rgWorkspace(rest[0], rest[1] || '.');
if (command === 'rg') return rgWorkspace(rest[0], rest[1] || '.', parseArgs(rest.slice(2)));
if (command === 'apply-patch') return workspaceApplyPatch(rest[0] || '.', parseArgs(rest.slice(1)));
if (command === 'build') {
const sub = rest[0] || 'start';
+59 -10
View File
@@ -39,9 +39,47 @@ function parseTarget(raw) {
if (!raw || raw === '--help' || raw === 'help') return { kind: 'help' };
if (raw === 'health') return { kind: 'health' };
if (raw === 'profile') return { kind: 'profile' };
const match = String(raw).match(/^([^:]+):(workspace|debug-probe|io-probe)(?::(.*))?$/u);
const match = String(raw).trim().match(/^([^:]+):(workspace|debug-probe|io-probe)(?::(.*))?$/u);
if (!match) throw new Error(`invalid target selector: ${raw}`);
return { kind: 'invoke', podId: match[1], surface: match[2], path: match[3] || '' };
return { kind: 'invoke', podId: match[1], surface: match[2], path: normalizeDevicePath(match[3] || '') };
}
function normalizeDevicePath(value) {
const normalized = String(value || '')
.trim()
.replace(/\\/gu, '/')
.replace(/\s*\/\s*/gu, '/')
.replace(/\/+/gu, '/')
.replace(/^\/+/u, '')
.replace(/\/+$/u, '');
return normalized || '.';
}
function joinDevicePath(base, child) {
const left = normalizeDevicePath(base);
const right = normalizeDevicePath(child);
if (!right || right === '.') return left;
if (String(child || '').trim().startsWith('/')) return right;
if (!left || left === '.') return right;
return normalizeDevicePath(`${left}/${right}`);
}
function normalizeProbePath(value) {
return String(value || '').trim().replace(/\\/gu, '/').replace(/\s*\/\s*/gu, '/').replace(/^\/+/u, '');
}
const WORKSPACE_COMMANDS = new Set(['ls', 'cat', 'rg', 'apply-patch', 'build']);
function normalizeWorkspaceSelector(selector, parsed) {
let workspacePath = normalizeDevicePath(selector.path || '.');
if (parsed._.length >= 2 && WORKSPACE_COMMANDS.has(parsed._[1])) {
const maybePath = String(parsed._[0] || '').trim();
if (maybePath === '/' || maybePath === '.' || maybePath.startsWith('/') || maybePath.includes('/')) {
workspacePath = joinDevicePath(workspacePath, maybePath);
parsed._ = parsed._.slice(1);
}
}
return workspacePath;
}
function parseOptions(argv) {
@@ -242,20 +280,27 @@ async function buildHostCommand(selector, parsed) {
const args = [];
const options = hostOptionArgs(parsed);
const operation = parsed._[0] || (selector.surface === 'workspace' ? 'ls' : selector.surface === 'debug-probe' ? 'status' : 'read');
const mutating = isMutating(selector.surface, operation);
const mutating = isMutating(selector.surface, operation, parsed);
if (mutating) requireApproval(parsed, `${selector.surface}.${operation}`);
if (selector.surface === 'workspace') {
const workspacePath = selector.path.replace(/^\/+/, '') || '.';
if (operation === 'ls' || operation === 'cat') args.push('workspace', operation, parsed._[1] ? path.posix.join(workspacePath, parsed._[1]) : workspacePath, ...parsed._.slice(2), ...options);
else if (operation === 'rg') args.push('workspace', 'rg', parsed._[1] || '', workspacePath, ...parsed._.slice(2), ...options);
else if (operation === 'apply-patch') args.push('workspace', 'apply-patch', workspacePath, '--patch-b64', Buffer.from(await resolvePatchText(parsed), 'utf8').toString('base64'));
const workspacePath = normalizeWorkspaceSelector(selector, parsed);
if (operation === 'ls' || operation === 'cat') args.push('workspace', operation, joinDevicePath(workspacePath, parsed._[1] || ''), ...parsed._.slice(2), ...options);
else if (operation === 'rg') {
const pattern = parsed._[1] || '';
const rootArg = parsed._[2] && !String(parsed._[2]).startsWith('-') ? parsed._[2] : '';
const restStart = rootArg ? 3 : 2;
args.push('workspace', 'rg', pattern, joinDevicePath(workspacePath, rootArg), ...parsed._.slice(restStart), ...options);
} else if (operation === 'apply-patch') {
const baseArg = parsed._[1] && !String(parsed._[1]).startsWith('-') ? parsed._[1] : '';
args.push('workspace', 'apply-patch', joinDevicePath(workspacePath, baseArg), '--patch-b64', Buffer.from(await resolvePatchText(parsed), 'utf8').toString('base64'));
}
else if (operation === 'build') args.push('workspace', 'build', ...(parsed._.slice(1).length ? parsed._.slice(1) : ['start']), ...options);
else throw new Error(`unsupported workspace command: ${operation}`);
} else if (selector.surface === 'debug-probe') {
args.push('debug-probe', operation, ...parsed._.slice(1), ...options);
} else if (selector.surface === 'io-probe') {
const probePath = selector.path.replace(/^\/+/, '') || 'uart/1';
const probePath = normalizeProbePath(selector.path) || 'uart/1';
if (probePath.startsWith('inner/')) throw new Error(`unsupported io-probe path in MVP: ${probePath}`);
args.push('io-probe', probePath, operation, ...parsed._.slice(1), ...options);
} else {
@@ -264,9 +309,9 @@ async function buildHostCommand(selector, parsed) {
return { operation, mutating, commandArgs: args };
}
function isMutating(surface, operation) {
function isMutating(surface, operation, parsed = { _: [] }) {
return (surface === 'workspace' && operation === 'apply-patch')
|| (surface === 'debug-probe' && (operation === 'download' || operation === 'reset' || operation === 'launch-flash'))
|| (surface === 'debug-probe' && (operation === 'reset' || operation === 'launch-flash' || (operation === 'download' && !['status', 'wait'].includes(String(parsed._[1] || 'start')))))
|| (surface === 'io-probe' && (operation === 'write' || operation === 'read-after-launch-flash'));
}
function requireApproval(parsed, operation) {
@@ -345,10 +390,14 @@ function hostJsonSummary(hostJson) {
: {};
const jobId = data.jobId || hostJson.jobId || hostJson.job_id || result.jobId || result.job_id || null;
const status = data.status || hostJson.status || result.status || null;
const uartCapture = result.uartCapture || data.uartCapture || hostJson.uartCapture || undefined;
return {
hostAction: hostJson.action || null,
hostJobId: jobId,
hostStatus: status,
hostSuccess: typeof hostJson.ok === 'boolean' ? hostJson.ok : result.success,
hostError: hostJson.error || data.error || result.error || undefined,
hostUartCapture: uartCapture,
hostResult: result && Object.keys(result).length > 0 ? result : undefined
};
}