Merge pull request #2305 from pikasTech/fix/2302-mdtodo-trace-pool

fix: expose MDTODO Workbench trace diagnostics

internal/db/runtime-store.test.ts

@@ -328,6 +328,56 @@ test("configured postgres runtime consumes pg pool idle errors as degraded readi
   assert.equal(JSON.stringify(warnings).includes("fixture secret host"), false);
 });
 
+test("configured postgres runtime only resets pg pool once while reset is in flight", async () => {
+  let releaseReset;
+  let endCalls = 0;
+  const warnings = [];
+  const store = createConfiguredCloudRuntimeStore({
+    env: {
+      HWLAB_CLOUD_RUNTIME_ADAPTER: "postgres",
+      HWLAB_CLOUD_RUNTIME_DURABLE: "true",
+      HWLAB_CLOUD_DB_POOL_RESET_COOLDOWN_MS: "5000"
+    },
+    dbUrl: "postgres://hwlab_user:fixture-pass@db.example.test:5432/hwlab?sslmode=require",
+    logger: { warn(entry) { warnings.push(entry); } },
+    pgModuleLoader: async () => ({
+      Pool: class FakePool {
+        constructor() {
+          this.totalCount = 16;
+          this.idleCount = 0;
+          this.waitingCount = 4;
+        }
+
+        async query() {
+          return { rows: [] };
+        }
+
+        async end() {
+          endCalls += 1;
+          await new Promise((resolve) => { releaseReset = resolve; });
+        }
+      }
+    })
+  });
+
+  await store.readiness();
+  const classified = {
+    blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED,
+    connection: { queryResult: "connect_timeout", errorCode: "ETIMEDOUT" }
+  };
+
+  assert.equal(store.resetPostgresPoolAfterTransientFailure(classified, { attempt: 1, maxAttempts: 5, retrying: true }), true);
+  const activePool = { totalCount: 16, idleCount: 0, waitingCount: 6, async end() { endCalls += 1; } };
+  store.pool = activePool;
+  assert.equal(store.resetPostgresPoolAfterTransientFailure(classified, { attempt: 2, maxAttempts: 5, retrying: true }), false);
+  assert.equal(store.pool, activePool);
+  await Promise.resolve();
+  assert.equal(endCalls, 1);
+  assert.equal(warnings.some((entry) => entry.event === "postgres_runtime_pool_reset_skipped" && entry.reason === "in_flight"), true);
+  releaseReset?.();
+  await Promise.resolve();
+});
+
 test("configured postgres runtime classifies pg_hba no-encryption rejection as SSL before auth", async () => {
   const store = createConfiguredCloudRuntimeStore({
     env: {
@@ -1070,7 +1120,8 @@ test("configured postgres runtime writes Workbench facts without re-running bloc
   const store = createConfiguredCloudRuntimeStore({
     env: {
       HWLAB_CLOUD_RUNTIME_ADAPTER: "postgres",
-      HWLAB_CLOUD_RUNTIME_DURABLE: "true"
+      HWLAB_CLOUD_RUNTIME_DURABLE: "true",
+      HWLAB_CLOUD_DB_QUERY_RETRY_MAX_ATTEMPTS: "1"
     },
     dbUrl: "postgres://hwlab_redacted@db.example.invalid:5432/hwlab",
     queryClient,
@@ -1079,7 +1130,9 @@ test("configured postgres runtime writes Workbench facts without re-running bloc
 
   const readiness = await store.readiness();
   assert.equal(readiness.blocker, RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED);
-  assert.equal(readiness.connection.queryResult, "query_blocked");
+  assert.equal(readiness.connection.queryResult, "too_many_connections");
+  assert.equal(readiness.retryable, true);
+  assert.equal(readiness.transient, true);
   queryClient.calls.length = 0;
 
   const write = await store.writeWorkbenchFacts({

internal/db/runtime-store.ts

@@ -139,9 +139,11 @@ const RUNTIME_DB_QUERY_TIMEOUT_MS_ENV = "HWLAB_CLOUD_DB_QUERY_TIMEOUT_MS";
 const RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS_ENV = "HWLAB_CLOUD_DB_QUERY_RETRY_MAX_ATTEMPTS";
 const RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS_ENV = "HWLAB_CLOUD_DB_QUERY_RETRY_INITIAL_DELAY_MS";
 const RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS_ENV = "HWLAB_CLOUD_DB_QUERY_RETRY_MAX_DELAY_MS";
+const RUNTIME_DB_POOL_RESET_COOLDOWN_MS_ENV = "HWLAB_CLOUD_DB_POOL_RESET_COOLDOWN_MS";
 const DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS = 5;
 const DEFAULT_RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS = 250;
 const DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS = 5_000;
+const DEFAULT_RUNTIME_DB_POOL_RESET_COOLDOWN_MS = 10_000;
 
 function normalizePositiveInteger(value, fallback) {
   const number = Number(value);
@@ -826,6 +828,8 @@ export class PostgresCloudRuntimeStore {
     this.pgModuleLoader = pgModuleLoader;
     this.logger = logger;
     this.pool = null;
+    this.postgresPoolResetInFlight = null;
+    this.postgresPoolResetLastAtMs = 0;
     this.runtimeReadIndexesReady = false;
     this.runtimeReadIndexesReadyPromise = null;
     this.memory = new CloudRuntimeStore({ now });
@@ -1941,7 +1945,43 @@ export class PostgresCloudRuntimeStore {
     if (classified?.connection?.queryResult !== "connect_timeout") return false;
     const pool = this.pool;
     if (!pool) return false;
+    const now = Date.now();
+    const cooldownMs = this.runtimePoolResetCooldownMs();
+    if (this.postgresPoolResetInFlight) {
+      this.logger?.warn?.({
+        event: "postgres_runtime_pool_reset_skipped",
+        reason: "in_flight",
+        blocker: classified.blocker,
+        queryResult: classified.connection?.queryResult ?? "query_blocked",
+        errorCode: classified.connection?.errorCode ?? "UNKNOWN",
+        retryAttempt: Number.isFinite(Number(retry.attempt)) ? Number(retry.attempt) : null,
+        retryMaxAttempts: Number.isFinite(Number(retry.maxAttempts)) ? Number(retry.maxAttempts) : null,
+        cooldownMs,
+        poolStats: postgresPoolStats(pool),
+        valuesRedacted: true,
+        endpointRedacted: true
+      });
+      return false;
+    }
+    if (now - this.postgresPoolResetLastAtMs < cooldownMs) {
+      this.logger?.warn?.({
+        event: "postgres_runtime_pool_reset_skipped",
+        reason: "cooldown",
+        blocker: classified.blocker,
+        queryResult: classified.connection?.queryResult ?? "query_blocked",
+        errorCode: classified.connection?.errorCode ?? "UNKNOWN",
+        retryAttempt: Number.isFinite(Number(retry.attempt)) ? Number(retry.attempt) : null,
+        retryMaxAttempts: Number.isFinite(Number(retry.maxAttempts)) ? Number(retry.maxAttempts) : null,
+        cooldownMs,
+        poolResetLastAtMs: this.postgresPoolResetLastAtMs,
+        poolStats: postgresPoolStats(pool),
+        valuesRedacted: true,
+        endpointRedacted: true
+      });
+      return false;
+    }
     this.pool = null;
+    this.postgresPoolResetLastAtMs = now;
     this.logger?.warn?.({
       event: "postgres_runtime_pool_reset_after_transient_failure",
       blocker: classified.blocker,
@@ -1950,22 +1990,32 @@ export class PostgresCloudRuntimeStore {
       retryAttempt: Number.isFinite(Number(retry.attempt)) ? Number(retry.attempt) : null,
       retryMaxAttempts: Number.isFinite(Number(retry.maxAttempts)) ? Number(retry.maxAttempts) : null,
       retrying: retry.retrying === true,
+      cooldownMs,
+      poolStats: postgresPoolStats(pool),
       valuesRedacted: true,
       endpointRedacted: true
     });
     if (typeof pool.end === "function") {
-      Promise.resolve()
+      this.postgresPoolResetInFlight = Promise.resolve()
         .then(() => pool.end())
         .catch((error) => this.logger?.warn?.({
           event: "postgres_runtime_pool_reset_failed",
           errorCode: error?.code ?? "UNKNOWN",
           valuesRedacted: true,
           endpointRedacted: true
-        }));
+        }))
+        .finally(() => {
+          this.postgresPoolResetInFlight = null;
+        });
     }
     return true;
   }
 
+  runtimePoolResetCooldownMs() {
+    const env = this.env ?? {};
+    return normalizePositiveInteger(env[RUNTIME_DB_POOL_RESET_COOLDOWN_MS_ENV], DEFAULT_RUNTIME_DB_POOL_RESET_COOLDOWN_MS);
+  }
+
   async query(sql, params = []) {
     const retry = this.runtimeQueryRetryPolicy();
     for (let attempt = 1; attempt <= retry.maxAttempts; attempt += 1) {
@@ -2094,7 +2144,7 @@ export class PostgresCloudRuntimeStore {
     });
   }
 
-  blockedSummary({ blocker, reason, schema, migration, connection, gates }) {
+  blockedSummary({ blocker, reason, schema, migration, connection, gates, retryable, transient, retryAfterMs }) {
     const blockedSchema = schema ?? {
       ready: false,
       checked: false,
@@ -2111,6 +2161,9 @@ export class PostgresCloudRuntimeStore {
       status: "blocked",
       blocker,
       reason,
+      retryable: retryable === true ? true : undefined,
+      transient: transient === true ? true : undefined,
+      retryAfterMs: Number.isFinite(Number(retryAfterMs)) ? Math.max(0, Math.trunc(Number(retryAfterMs))) : undefined,
       liveRuntimeEvidence: false,
       fixtureEvidence: false,
       connection: {
@@ -3167,6 +3220,20 @@ export function classifyRuntimeDbError(error) {
       }
     };
   }
+  if (code === "53300") {
+    return {
+      blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED,
+      reason: "Postgres runtime adapter reached the database, but the server has exhausted available connection slots",
+      retryable: true,
+      transient: true,
+      retryAfterMs: 2000,
+      connection: {
+        queryAttempted: true,
+        queryResult: "too_many_connections",
+        errorCode: code
+      }
+    };
+  }
   return {
     blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED,
     reason: "Postgres runtime adapter could not complete a live runtime schema query",
@@ -3467,6 +3534,16 @@ function parseJsonColumn(value, fallback) {
   }
 }
 
+function postgresPoolStats(pool) {
+  if (!pool || typeof pool !== "object") return null;
+  return {
+    totalCount: Number.isFinite(Number(pool.totalCount)) ? Number(pool.totalCount) : null,
+    idleCount: Number.isFinite(Number(pool.idleCount)) ? Number(pool.idleCount) : null,
+    waitingCount: Number.isFinite(Number(pool.waitingCount)) ? Number(pool.waitingCount) : null,
+    valuesRedacted: true
+  };
+}
+
 function toCountKey(table) {
   if (table === "users") return "users";
   if (table === "user_sessions") return "userSessions";

web/hwlab-cloud-web/src/components/common/ApiErrorDiagnostic.vue

@@ -40,8 +40,9 @@ const facts = computed(() => {
   ].filter((item): item is { label: string; value: string } => Boolean(item));
 });
 const primaryTraceFact = computed(() => facts.value.find((item) => item.label === "trace_id") ?? null);
+const primaryTraceId = computed(() => primaryTraceFact.value?.value ?? null);
 const summaryText = computed(() => {
-  const parts = [props.showMessage && message.value ? message.value : null, primaryTraceFact.value ? `trace_id=${primaryTraceFact.value.value}` : null].filter(Boolean);
+  const parts = [props.showMessage && message.value ? message.value : null].filter(Boolean);
   return parts.join(" ") || (props.title ? "" : "诊断详情");
 });
 const hasContent = computed(() => Boolean(message.value || facts.value.length));
@@ -102,6 +103,10 @@ function copyDiagnostic(): void {
       </div>
       <button v-if="facts.length" class="api-error-diagnostic-toggle" type="button" :aria-expanded="expanded ? 'true' : 'false'" aria-label="诊断详情" title="诊断详情" @click="expanded = !expanded">!</button>
     </header>
+    <p v-if="primaryTraceId" class="api-error-trace-line" data-testid="api-error-trace-id">
+      <span>Trace ID</span>
+      <code>{{ primaryTraceId }}</code>
+    </p>
     <div v-if="facts.length && expanded" class="api-error-diagnostic-details">
       <button class="table-action" type="button" @click="copyDiagnostic">{{ copied ? "已复制" : "复制诊断" }}</button>
       <dl class="api-error-diagnostic-grid">
@@ -113,3 +118,21 @@ function copyDiagnostic(): void {
     </div>
   </section>
 </template>
+
+<style scoped>
+.api-error-diagnostic { display: grid; min-width: 0; gap: 6px; }
+.api-error-diagnostic-summary { display: flex; min-width: 0; align-items: start; justify-content: space-between; gap: 8px; }
+.api-error-diagnostic-summary-text { min-width: 0; }
+.api-error-diagnostic-summary-text strong { display: block; margin-bottom: 2px; font-size: 12px; font-weight: 850; }
+.api-error-diagnostic-summary-text p { margin: 0; overflow-wrap: anywhere; }
+.api-error-diagnostic-toggle { display: inline-flex; width: 22px; height: 22px; flex: 0 0 auto; align-items: center; justify-content: center; border: 1px solid #fed7aa; border-radius: 999px; background: #fff7ed; color: #9a3412; font: inherit; font-size: 12px; font-weight: 850; }
+.api-error-trace-line { display: flex; min-width: 0; flex-wrap: wrap; gap: 6px; align-items: center; margin: 0; color: #7c2d12; font-size: 12px; }
+.api-error-trace-line span { font-weight: 850; }
+.api-error-trace-line code { min-width: 0; max-width: 100%; overflow-wrap: anywhere; border: 1px solid #fed7aa; border-radius: 4px; background: #fffbeb; color: #7c2d12; padding: 1px 5px; }
+.api-error-diagnostic-details { display: grid; min-width: 0; gap: 6px; }
+.api-error-diagnostic-grid { display: grid; min-width: 0; gap: 5px; margin: 0; }
+.api-error-diagnostic-grid div { display: grid; min-width: 0; grid-template-columns: 112px minmax(0, 1fr); gap: 8px; align-items: start; }
+.api-error-diagnostic-grid dt { color: #7c2d12; font-weight: 800; }
+.api-error-diagnostic-grid dd { min-width: 0; margin: 0; }
+.api-error-diagnostic-grid code { overflow-wrap: anywhere; }
+</style>

web/hwlab-cloud-web/src/components/mdtodo/MdtodoPageShell.vue

@@ -514,6 +514,8 @@ function setError(err: unknown): void {
           :mutation-error="mutation.taskMutationError.value"
           :launch-loading="launch.launchLoading.value"
           :launch-error="launch.launchError.value"
+          :launch-api-error="launch.launchApiError.value"
+          :launch-diagnostic="launch.launchDiagnostic.value"
           :launch-enabled="workbenchLaunchEnabled"
           :launch-blocker="workbenchLaunchBlocker"
           :provider-profile="workbench.providerProfile"

web/hwlab-cloud-web/src/components/mdtodo/MdtodoTaskPanel.vue

@@ -5,9 +5,11 @@
 import { computed } from "vue";
 import type { MdtodoTaskDetailRecord, MdtodoTaskLinkRecord, MdtodoTaskRecord, ProjectNavigationResponse } from "@/api";
 import type { ProviderProfile } from "@/types";
+import type { ApiError, ErrorDiagnostic } from "@/types";
 import type { ProviderProfileOption } from "@/stores/workbench-session";
 import LoadingState from "@/components/common/LoadingState.vue";
 import EmptyState from "@/components/common/EmptyState.vue";
+import ApiErrorDiagnostic from "@/components/common/ApiErrorDiagnostic.vue";
 
 const props = defineProps<{
   task: MdtodoTaskRecord | null;
@@ -28,6 +30,8 @@ const props = defineProps<{
   mutationError: string | null;
   launchLoading: boolean;
   launchError: string | null;
+  launchApiError: ApiError | null;
+  launchDiagnostic: ErrorDiagnostic | null;
   launchEnabled: boolean;
   launchBlocker: string | null;
   providerProfile: ProviderProfile;
@@ -104,7 +108,7 @@ function normalizeTaskText(value?: string | null): string {
       <div class="task-status-stack">
         <p v-if="mutationMessage" class="source-message" data-testid="mdtodo-task-mutation-message">{{ mutationMessage }}</p>
         <p v-if="mutationError" class="source-error" data-testid="mdtodo-task-mutation-error">{{ mutationError }}</p>
-        <p v-if="launchError" class="launch-blocker" data-testid="mdtodo-workbench-launch-error">{{ launchError }}</p>
+        <ApiErrorDiagnostic v-if="launchError" class="launch-blocker" data-testid="mdtodo-workbench-launch-error" title="Workbench launch" :error="launchError" :api-error="launchApiError" :diagnostic="launchDiagnostic" compact />
         <p v-else-if="launchBlocker" class="launch-blocker" data-testid="mdtodo-workbench-launch-blocker">{{ launchBlocker }}</p>
       </div>
 

web/hwlab-cloud-web/src/composables/mdtodo/useMdtodoWorkbenchLaunch.ts

@@ -3,6 +3,8 @@
 
 import { ref } from "vue";
 import { agentAPI, workbenchAPI, type MdtodoTaskLinkRecord, type MdtodoTaskRecord, type ProjectNavigationResponse } from "@/api";
+import { apiErrorContextFromUnknown } from "@/api/client";
+import type { ApiError, ErrorDiagnostic } from "@/types";
 import type { Router } from "vue-router";
 
 export interface WorkbenchLaunchContext {
@@ -62,8 +64,16 @@ export interface WorkbenchLaunchExecutionContext {
 export function useMdtodoWorkbenchLaunch(router: Router) {
   const launchLoading = ref(false);
   const launchError = ref<string | null>(null);
+  const launchApiError = ref<ApiError | null>(null);
+  const launchDiagnostic = ref<ErrorDiagnostic | null>(null);
   const launchResult = ref<{ sessionId: string; workbenchUrl: string } | null>(null);
 
+  function setLaunchError(error: string | null, apiError: ApiError | null = null, diagnostic: ErrorDiagnostic | null = null): void {
+    launchError.value = error;
+    launchApiError.value = apiError;
+    launchDiagnostic.value = diagnostic;
+  }
+
   function buildPrompt(task: MdtodoTaskRecord, body: string, links: MdtodoTaskLinkRecord[], fileName: string, launchContext?: WorkbenchLaunchContext | null): string {
     const reportLines = links
       .filter((link) => link.kind === "markdown-report")
@@ -143,16 +153,16 @@ export function useMdtodoWorkbenchLaunch(router: Router) {
     if (!task.taskRef || !task.projectId) return null;
     const enabled = navigation?.navigation?.capabilities?.workbenchLaunch === true;
     if (!enabled) {
-      launchError.value = "Workbench Launch capability unavailable";
+      setLaunchError("Workbench Launch capability unavailable");
       return null;
     }
     const selectedProviderProfile = providerProfile.trim();
     if (!selectedProviderProfile) {
-      launchError.value = "模型通道未选择";
+      setLaunchError("模型通道未选择");
       return null;
     }
     launchLoading.value = true;
-    launchError.value = null;
+    setLaunchError(null);
     try {
       const launchContext = { ...buildContext(task, body, links, fileName), providerProfile: selectedProviderProfile };
       const response = await workbenchAPI.launch({
@@ -181,7 +191,8 @@ export function useMdtodoWorkbenchLaunch(router: Router) {
       launchResult.value = result;
       return result;
     } catch (err) {
-      launchError.value = err && typeof err === "object" && "error" in err ? String((err as { error?: string }).error || "Workbench launch 失败") : "Workbench launch 失败";
+      const context = apiErrorContextFromUnknown(err, "Workbench launch 失败");
+      setLaunchError(context.error || "Workbench launch 失败", context.apiError, context.diagnostic);
       return null;
     } finally {
       launchLoading.value = false;
@@ -199,6 +210,8 @@ export function useMdtodoWorkbenchLaunch(router: Router) {
   return {
     launchLoading,
     launchError,
+    launchApiError,
+    launchDiagnostic,
     launchResult,
     buildPrompt,
     buildContext,