mirrors/pikasTech-HWLAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: use idle-only AgentRun projection timeout

Browse Source
This commit is contained in:
lyon
2026-06-18 22:36:30 +08:00
parent 522605ecdb
commit edae2b40b4
2 changed files with 69 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/deploy.yaml
+3 -1
View File
@@ -301,8 +301,10 @@ lanes:
- skills/hwlab-agent-runtime/
env:
HWLAB_METRICS_NAMESPACE: hwlab-v03
HWLAB_OBSERVABILITY_CONFIG_REVISION: PJ2026-01060505-20260617-turn-status-budget
HWLAB_OBSERVABILITY_CONFIG_REVISION: PJ2026-01060505-20260618-no-response-timeout
HWLAB_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS: "2500"
HWLAB_CODE_AGENT_AGENTRUN_NO_RESPONSE_TIMEOUT_MS: "90000"
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_POLL_INTERVAL_MS: "1000"
observable: true
hwlab-user-billing:
runtimeKind: go-service
internal/cloud/server-code-agent-http.ts
+66 -20
View File
@@ -791,16 +791,24 @@ function submitCodeAgentChatTurn({ params, options, traceId }) {
function scheduleAgentRunProjectionSync({ traceId, params = {}, options = {}, traceStore = defaultCodeAgentTraceStore, currentResult = null } = {}) {
if (!safeTraceId(traceId) || !currentResult?.agentRun?.runId || !currentResult?.agentRun?.commandId) return null;
const env = options.env ?? process.env;
const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_TIMEOUT_MS, 90_000);
const noResponseTimeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_NO_RESPONSE_TIMEOUT_MS, null);
const pollIntervalMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_POLL_INTERVAL_MS, 1_000);
const refreshOptions = codeAgentTurnStatusRefreshOptions(options);
const startedAt = Date.now();
let lastActivityAt = Date.now();
let lastActivitySignature = agentRunProjectionActivitySignature(currentResult, traceStore.snapshot(traceId));
let idleDegraded = false;
setImmediate(() => {
void (async () => {
let result = currentResult;
let lastError = null;
while (Date.now() - startedAt <= timeoutMs) {
while (true) {
const cached = options.codeAgentChatResults?.get?.(traceId);
if (isCodeAgentResultCanceled(cached)) return;
if (isTraceCommandTerminalStatus(cached?.status)) {
scheduleCodeAgentTerminalTurnStatusEffects({ payload: cached, params, options, preserveLastTraceId: true });
return;
}
try {
const synced = await syncAgentRunChatResult({ traceId, currentResult: result, options: refreshOptions, traceStore });
result = synced.result ?? result;
@@ -808,27 +816,38 @@ function scheduleAgentRunProjectionSync({ traceId, params = {}, options = {}, tr
scheduleCodeAgentTerminalTurnStatusEffects({ payload: result, params, options, preserveLastTraceId: true });
return;
}
const signature = agentRunProjectionActivitySignature(result, synced.runnerTrace ?? traceStore.snapshot(traceId));
if (signature && signature !== lastActivitySignature) {
lastActivitySignature = signature;
lastActivityAt = Date.now();
idleDegraded = false;
}
lastError = null;
} catch (error) {
lastError = error;
}
const elapsedMs = Date.now() - startedAt;
if (elapsedMs >= timeoutMs) break;
await sleepAgentRunProjectionSync(Math.min(pollIntervalMs, Math.max(0, timeoutMs - elapsedMs)));
const idleMs = Date.now() - lastActivityAt;
if (noResponseTimeoutMs && idleMs >= noResponseTimeoutMs && !idleDegraded) {
traceStore.append(traceId, {
type: "turn-status",
status: "degraded",
label: "agentrun:projection:no-response-idle-timeout",
errorCode: lastError?.code ?? "no_response_idle_timeout",
degradedReason: lastError ? "projection_activity_stale" : "no_response_idle_timeout",
message: lastError?.message ?? `AgentRun projection has no new activity for ${idleMs}ms; idle threshold ${noResponseTimeoutMs}ms.`,
runId: result?.agentRun?.runId ?? currentResult.agentRun.runId,
commandId: result?.agentRun?.commandId ?? currentResult.agentRun.commandId,
timeoutMs: noResponseTimeoutMs,
idleMs,
lastActivityAt: new Date(lastActivityAt).toISOString(),
waitingFor: "agentrun-result-activity",
terminal: false,
valuesPrinted: false
});
idleDegraded = true;
}
await sleepAgentRunProjectionSync(pollIntervalMs);
}
traceStore.append(traceId, {
type: "turn-status",
status: "degraded",
label: lastError ? "agentrun:projection:sync-failed" : "agentrun:projection:sync-timeout",
errorCode: lastError?.code ?? null,
message: lastError?.message ?? "AgentRun projection sync did not reach terminal within the configured budget.",
runId: result?.agentRun?.runId ?? currentResult.agentRun.runId,
commandId: result?.agentRun?.commandId ?? currentResult.agentRun.commandId,
timeoutMs,
waitingFor: "agentrun-result",
terminal: false,
valuesPrinted: false
});
})().catch((error) => {
traceStore.append(traceId, {
type: "turn-status",
@@ -844,7 +863,34 @@ function scheduleAgentRunProjectionSync({ traceId, params = {}, options = {}, tr
});
});
});
return { scheduled: true, traceId, timeoutMs, pollIntervalMs, valuesPrinted: false };
return { scheduled: true, traceId, noResponseTimeoutMs, pollIntervalMs, valuesPrinted: false };
}
function agentRunProjectionActivitySignature(result, runnerTrace) {
const agentRun = result?.agentRun && typeof result.agentRun === "object" ? result.agentRun : {};
const trace = runnerTrace && typeof runnerTrace === "object" ? runnerTrace : result?.runnerTrace;
const events = Array.isArray(trace?.events) ? trace.events : [];
const lastEvent = trace?.lastEvent ?? events.at(-1) ?? null;
return [
result?.status,
result?.updatedAt,
agentRun.status,
agentRun.runStatus,
agentRun.commandState,
agentRun.terminalStatus,
agentRun.lastSeq,
agentRun.updatedAt,
trace?.status,
trace?.eventCount ?? events.length,
trace?.updatedAt,
trace?.lastEventLabel,
lastEvent?.seq,
lastEvent?.sourceSeq,
lastEvent?.label,
lastEvent?.type,
lastEvent?.status,
lastEvent?.createdAt
].map((value) => value ?? "").join("|");
}
function sleepAgentRunProjectionSync(ms) {