diff --git a/internal/db/runtime-store.test.ts b/internal/db/runtime-store.test.ts index 429f6762..0b623cfe 100644 --- a/internal/db/runtime-store.test.ts +++ b/internal/db/runtime-store.test.ts @@ -328,6 +328,56 @@ test("configured postgres runtime consumes pg pool idle errors as degraded readi assert.equal(JSON.stringify(warnings).includes("fixture secret host"), false); }); +test("configured postgres runtime only resets pg pool once while reset is in flight", async () => { + let releaseReset; + let endCalls = 0; + const warnings = []; + const store = createConfiguredCloudRuntimeStore({ + env: { + HWLAB_CLOUD_RUNTIME_ADAPTER: "postgres", + HWLAB_CLOUD_RUNTIME_DURABLE: "true", + HWLAB_CLOUD_DB_POOL_RESET_COOLDOWN_MS: "5000" + }, + dbUrl: "postgres://hwlab_user:fixture-pass@db.example.test:5432/hwlab?sslmode=require", + logger: { warn(entry) { warnings.push(entry); } }, + pgModuleLoader: async () => ({ + Pool: class FakePool { + constructor() { + this.totalCount = 16; + this.idleCount = 0; + this.waitingCount = 4; + } + + async query() { + return { rows: [] }; + } + + async end() { + endCalls += 1; + await new Promise((resolve) => { releaseReset = resolve; }); + } + } + }) + }); + + await store.readiness(); + const classified = { + blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED, + connection: { queryResult: "connect_timeout", errorCode: "ETIMEDOUT" } + }; + + assert.equal(store.resetPostgresPoolAfterTransientFailure(classified, { attempt: 1, maxAttempts: 5, retrying: true }), true); + const activePool = { totalCount: 16, idleCount: 0, waitingCount: 6, async end() { endCalls += 1; } }; + store.pool = activePool; + assert.equal(store.resetPostgresPoolAfterTransientFailure(classified, { attempt: 2, maxAttempts: 5, retrying: true }), false); + assert.equal(store.pool, activePool); + await Promise.resolve(); + assert.equal(endCalls, 1); + assert.equal(warnings.some((entry) => entry.event === "postgres_runtime_pool_reset_skipped" && entry.reason === "in_flight"), true); + releaseReset?.(); + await Promise.resolve(); +}); + test("configured postgres runtime classifies pg_hba no-encryption rejection as SSL before auth", async () => { const store = createConfiguredCloudRuntimeStore({ env: { @@ -1070,7 +1120,8 @@ test("configured postgres runtime writes Workbench facts without re-running bloc const store = createConfiguredCloudRuntimeStore({ env: { HWLAB_CLOUD_RUNTIME_ADAPTER: "postgres", - HWLAB_CLOUD_RUNTIME_DURABLE: "true" + HWLAB_CLOUD_RUNTIME_DURABLE: "true", + HWLAB_CLOUD_DB_QUERY_RETRY_MAX_ATTEMPTS: "1" }, dbUrl: "postgres://hwlab_redacted@db.example.invalid:5432/hwlab", queryClient, @@ -1079,7 +1130,9 @@ test("configured postgres runtime writes Workbench facts without re-running bloc const readiness = await store.readiness(); assert.equal(readiness.blocker, RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED); - assert.equal(readiness.connection.queryResult, "query_blocked"); + assert.equal(readiness.connection.queryResult, "too_many_connections"); + assert.equal(readiness.retryable, true); + assert.equal(readiness.transient, true); queryClient.calls.length = 0; const write = await store.writeWorkbenchFacts({ diff --git a/internal/db/runtime-store.ts b/internal/db/runtime-store.ts index 88eaf41a..226aae21 100644 --- a/internal/db/runtime-store.ts +++ b/internal/db/runtime-store.ts @@ -139,9 +139,11 @@ const RUNTIME_DB_QUERY_TIMEOUT_MS_ENV = "HWLAB_CLOUD_DB_QUERY_TIMEOUT_MS"; const RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS_ENV = "HWLAB_CLOUD_DB_QUERY_RETRY_MAX_ATTEMPTS"; const RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS_ENV = "HWLAB_CLOUD_DB_QUERY_RETRY_INITIAL_DELAY_MS"; const RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS_ENV = "HWLAB_CLOUD_DB_QUERY_RETRY_MAX_DELAY_MS"; +const RUNTIME_DB_POOL_RESET_COOLDOWN_MS_ENV = "HWLAB_CLOUD_DB_POOL_RESET_COOLDOWN_MS"; const DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS = 5; const DEFAULT_RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS = 250; const DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS = 5_000; +const DEFAULT_RUNTIME_DB_POOL_RESET_COOLDOWN_MS = 10_000; function normalizePositiveInteger(value, fallback) { const number = Number(value); @@ -826,6 +828,8 @@ export class PostgresCloudRuntimeStore { this.pgModuleLoader = pgModuleLoader; this.logger = logger; this.pool = null; + this.postgresPoolResetInFlight = null; + this.postgresPoolResetLastAtMs = 0; this.runtimeReadIndexesReady = false; this.runtimeReadIndexesReadyPromise = null; this.memory = new CloudRuntimeStore({ now }); @@ -1941,7 +1945,43 @@ export class PostgresCloudRuntimeStore { if (classified?.connection?.queryResult !== "connect_timeout") return false; const pool = this.pool; if (!pool) return false; + const now = Date.now(); + const cooldownMs = this.runtimePoolResetCooldownMs(); + if (this.postgresPoolResetInFlight) { + this.logger?.warn?.({ + event: "postgres_runtime_pool_reset_skipped", + reason: "in_flight", + blocker: classified.blocker, + queryResult: classified.connection?.queryResult ?? "query_blocked", + errorCode: classified.connection?.errorCode ?? "UNKNOWN", + retryAttempt: Number.isFinite(Number(retry.attempt)) ? Number(retry.attempt) : null, + retryMaxAttempts: Number.isFinite(Number(retry.maxAttempts)) ? Number(retry.maxAttempts) : null, + cooldownMs, + poolStats: postgresPoolStats(pool), + valuesRedacted: true, + endpointRedacted: true + }); + return false; + } + if (now - this.postgresPoolResetLastAtMs < cooldownMs) { + this.logger?.warn?.({ + event: "postgres_runtime_pool_reset_skipped", + reason: "cooldown", + blocker: classified.blocker, + queryResult: classified.connection?.queryResult ?? "query_blocked", + errorCode: classified.connection?.errorCode ?? "UNKNOWN", + retryAttempt: Number.isFinite(Number(retry.attempt)) ? Number(retry.attempt) : null, + retryMaxAttempts: Number.isFinite(Number(retry.maxAttempts)) ? Number(retry.maxAttempts) : null, + cooldownMs, + poolResetLastAtMs: this.postgresPoolResetLastAtMs, + poolStats: postgresPoolStats(pool), + valuesRedacted: true, + endpointRedacted: true + }); + return false; + } this.pool = null; + this.postgresPoolResetLastAtMs = now; this.logger?.warn?.({ event: "postgres_runtime_pool_reset_after_transient_failure", blocker: classified.blocker, @@ -1950,22 +1990,32 @@ export class PostgresCloudRuntimeStore { retryAttempt: Number.isFinite(Number(retry.attempt)) ? Number(retry.attempt) : null, retryMaxAttempts: Number.isFinite(Number(retry.maxAttempts)) ? Number(retry.maxAttempts) : null, retrying: retry.retrying === true, + cooldownMs, + poolStats: postgresPoolStats(pool), valuesRedacted: true, endpointRedacted: true }); if (typeof pool.end === "function") { - Promise.resolve() + this.postgresPoolResetInFlight = Promise.resolve() .then(() => pool.end()) .catch((error) => this.logger?.warn?.({ event: "postgres_runtime_pool_reset_failed", errorCode: error?.code ?? "UNKNOWN", valuesRedacted: true, endpointRedacted: true - })); + })) + .finally(() => { + this.postgresPoolResetInFlight = null; + }); } return true; } + runtimePoolResetCooldownMs() { + const env = this.env ?? {}; + return normalizePositiveInteger(env[RUNTIME_DB_POOL_RESET_COOLDOWN_MS_ENV], DEFAULT_RUNTIME_DB_POOL_RESET_COOLDOWN_MS); + } + async query(sql, params = []) { const retry = this.runtimeQueryRetryPolicy(); for (let attempt = 1; attempt <= retry.maxAttempts; attempt += 1) { @@ -2094,7 +2144,7 @@ export class PostgresCloudRuntimeStore { }); } - blockedSummary({ blocker, reason, schema, migration, connection, gates }) { + blockedSummary({ blocker, reason, schema, migration, connection, gates, retryable, transient, retryAfterMs }) { const blockedSchema = schema ?? { ready: false, checked: false, @@ -2111,6 +2161,9 @@ export class PostgresCloudRuntimeStore { status: "blocked", blocker, reason, + retryable: retryable === true ? true : undefined, + transient: transient === true ? true : undefined, + retryAfterMs: Number.isFinite(Number(retryAfterMs)) ? Math.max(0, Math.trunc(Number(retryAfterMs))) : undefined, liveRuntimeEvidence: false, fixtureEvidence: false, connection: { @@ -3167,6 +3220,20 @@ export function classifyRuntimeDbError(error) { } }; } + if (code === "53300") { + return { + blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED, + reason: "Postgres runtime adapter reached the database, but the server has exhausted available connection slots", + retryable: true, + transient: true, + retryAfterMs: 2000, + connection: { + queryAttempted: true, + queryResult: "too_many_connections", + errorCode: code + } + }; + } return { blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED, reason: "Postgres runtime adapter could not complete a live runtime schema query", @@ -3467,6 +3534,16 @@ function parseJsonColumn(value, fallback) { } } +function postgresPoolStats(pool) { + if (!pool || typeof pool !== "object") return null; + return { + totalCount: Number.isFinite(Number(pool.totalCount)) ? Number(pool.totalCount) : null, + idleCount: Number.isFinite(Number(pool.idleCount)) ? Number(pool.idleCount) : null, + waitingCount: Number.isFinite(Number(pool.waitingCount)) ? Number(pool.waitingCount) : null, + valuesRedacted: true + }; +} + function toCountKey(table) { if (table === "users") return "users"; if (table === "user_sessions") return "userSessions"; diff --git a/web/hwlab-cloud-web/src/components/common/ApiErrorDiagnostic.vue b/web/hwlab-cloud-web/src/components/common/ApiErrorDiagnostic.vue index d0343267..a39a625a 100644 --- a/web/hwlab-cloud-web/src/components/common/ApiErrorDiagnostic.vue +++ b/web/hwlab-cloud-web/src/components/common/ApiErrorDiagnostic.vue @@ -40,8 +40,9 @@ const facts = computed(() => { ].filter((item): item is { label: string; value: string } => Boolean(item)); }); const primaryTraceFact = computed(() => facts.value.find((item) => item.label === "trace_id") ?? null); +const primaryTraceId = computed(() => primaryTraceFact.value?.value ?? null); const summaryText = computed(() => { - const parts = [props.showMessage && message.value ? message.value : null, primaryTraceFact.value ? `trace_id=${primaryTraceFact.value.value}` : null].filter(Boolean); + const parts = [props.showMessage && message.value ? message.value : null].filter(Boolean); return parts.join(" ") || (props.title ? "" : "诊断详情"); }); const hasContent = computed(() => Boolean(message.value || facts.value.length)); @@ -102,6 +103,10 @@ function copyDiagnostic(): void { +

+ Trace ID + {{ primaryTraceId }} +

@@ -113,3 +118,21 @@ function copyDiagnostic(): void {
+ + diff --git a/web/hwlab-cloud-web/src/components/mdtodo/MdtodoPageShell.vue b/web/hwlab-cloud-web/src/components/mdtodo/MdtodoPageShell.vue index aab75033..8bf46173 100644 --- a/web/hwlab-cloud-web/src/components/mdtodo/MdtodoPageShell.vue +++ b/web/hwlab-cloud-web/src/components/mdtodo/MdtodoPageShell.vue @@ -514,6 +514,8 @@ function setError(err: unknown): void { :mutation-error="mutation.taskMutationError.value" :launch-loading="launch.launchLoading.value" :launch-error="launch.launchError.value" + :launch-api-error="launch.launchApiError.value" + :launch-diagnostic="launch.launchDiagnostic.value" :launch-enabled="workbenchLaunchEnabled" :launch-blocker="workbenchLaunchBlocker" :provider-profile="workbench.providerProfile" diff --git a/web/hwlab-cloud-web/src/components/mdtodo/MdtodoTaskPanel.vue b/web/hwlab-cloud-web/src/components/mdtodo/MdtodoTaskPanel.vue index 2b2021cd..4a01c22b 100644 --- a/web/hwlab-cloud-web/src/components/mdtodo/MdtodoTaskPanel.vue +++ b/web/hwlab-cloud-web/src/components/mdtodo/MdtodoTaskPanel.vue @@ -5,9 +5,11 @@ import { computed } from "vue"; import type { MdtodoTaskDetailRecord, MdtodoTaskLinkRecord, MdtodoTaskRecord, ProjectNavigationResponse } from "@/api"; import type { ProviderProfile } from "@/types"; +import type { ApiError, ErrorDiagnostic } from "@/types"; import type { ProviderProfileOption } from "@/stores/workbench-session"; import LoadingState from "@/components/common/LoadingState.vue"; import EmptyState from "@/components/common/EmptyState.vue"; +import ApiErrorDiagnostic from "@/components/common/ApiErrorDiagnostic.vue"; const props = defineProps<{ task: MdtodoTaskRecord | null; @@ -28,6 +30,8 @@ const props = defineProps<{ mutationError: string | null; launchLoading: boolean; launchError: string | null; + launchApiError: ApiError | null; + launchDiagnostic: ErrorDiagnostic | null; launchEnabled: boolean; launchBlocker: string | null; providerProfile: ProviderProfile; @@ -104,7 +108,7 @@ function normalizeTaskText(value?: string | null): string {

{{ mutationMessage }}

{{ mutationError }}

-

{{ launchError }}

+

{{ launchBlocker }}

diff --git a/web/hwlab-cloud-web/src/composables/mdtodo/useMdtodoWorkbenchLaunch.ts b/web/hwlab-cloud-web/src/composables/mdtodo/useMdtodoWorkbenchLaunch.ts index 0a1188db..b2a77bfe 100644 --- a/web/hwlab-cloud-web/src/composables/mdtodo/useMdtodoWorkbenchLaunch.ts +++ b/web/hwlab-cloud-web/src/composables/mdtodo/useMdtodoWorkbenchLaunch.ts @@ -3,6 +3,8 @@ import { ref } from "vue"; import { agentAPI, workbenchAPI, type MdtodoTaskLinkRecord, type MdtodoTaskRecord, type ProjectNavigationResponse } from "@/api"; +import { apiErrorContextFromUnknown } from "@/api/client"; +import type { ApiError, ErrorDiagnostic } from "@/types"; import type { Router } from "vue-router"; export interface WorkbenchLaunchContext { @@ -62,8 +64,16 @@ export interface WorkbenchLaunchExecutionContext { export function useMdtodoWorkbenchLaunch(router: Router) { const launchLoading = ref(false); const launchError = ref(null); + const launchApiError = ref(null); + const launchDiagnostic = ref(null); const launchResult = ref<{ sessionId: string; workbenchUrl: string } | null>(null); + function setLaunchError(error: string | null, apiError: ApiError | null = null, diagnostic: ErrorDiagnostic | null = null): void { + launchError.value = error; + launchApiError.value = apiError; + launchDiagnostic.value = diagnostic; + } + function buildPrompt(task: MdtodoTaskRecord, body: string, links: MdtodoTaskLinkRecord[], fileName: string, launchContext?: WorkbenchLaunchContext | null): string { const reportLines = links .filter((link) => link.kind === "markdown-report") @@ -143,16 +153,16 @@ export function useMdtodoWorkbenchLaunch(router: Router) { if (!task.taskRef || !task.projectId) return null; const enabled = navigation?.navigation?.capabilities?.workbenchLaunch === true; if (!enabled) { - launchError.value = "Workbench Launch capability unavailable"; + setLaunchError("Workbench Launch capability unavailable"); return null; } const selectedProviderProfile = providerProfile.trim(); if (!selectedProviderProfile) { - launchError.value = "模型通道未选择"; + setLaunchError("模型通道未选择"); return null; } launchLoading.value = true; - launchError.value = null; + setLaunchError(null); try { const launchContext = { ...buildContext(task, body, links, fileName), providerProfile: selectedProviderProfile }; const response = await workbenchAPI.launch({ @@ -181,7 +191,8 @@ export function useMdtodoWorkbenchLaunch(router: Router) { launchResult.value = result; return result; } catch (err) { - launchError.value = err && typeof err === "object" && "error" in err ? String((err as { error?: string }).error || "Workbench launch 失败") : "Workbench launch 失败"; + const context = apiErrorContextFromUnknown(err, "Workbench launch 失败"); + setLaunchError(context.error || "Workbench launch 失败", context.apiError, context.diagnostic); return null; } finally { launchLoading.value = false; @@ -199,6 +210,8 @@ export function useMdtodoWorkbenchLaunch(router: Router) { return { launchLoading, launchError, + launchApiError, + launchDiagnostic, launchResult, buildPrompt, buildContext,