fix: expose mdtodo workbench trace diagnostics
This commit is contained in:
UniDesk Codex
@@ -328,6 +328,56 @@ test("configured postgres runtime consumes pg pool idle errors as degraded readi
|
||||
assert.equal(JSON.stringify(warnings).includes("fixture secret host"), false);
|
||||
});
|
||||
|
||||
test("configured postgres runtime only resets pg pool once while reset is in flight", async () => {
|
||||
let releaseReset;
|
||||
let endCalls = 0;
|
||||
const warnings = [];
|
||||
const store = createConfiguredCloudRuntimeStore({
|
||||
env: {
|
||||
HWLAB_CLOUD_RUNTIME_ADAPTER: "postgres",
|
||||
HWLAB_CLOUD_RUNTIME_DURABLE: "true",
|
||||
HWLAB_CLOUD_DB_POOL_RESET_COOLDOWN_MS: "5000"
|
||||
},
|
||||
dbUrl: "postgres://hwlab_user:fixture-pass@db.example.test:5432/hwlab?sslmode=require",
|
||||
logger: { warn(entry) { warnings.push(entry); } },
|
||||
pgModuleLoader: async () => ({
|
||||
Pool: class FakePool {
|
||||
constructor() {
|
||||
this.totalCount = 16;
|
||||
this.idleCount = 0;
|
||||
this.waitingCount = 4;
|
||||
}
|
||||
|
||||
async query() {
|
||||
return { rows: [] };
|
||||
}
|
||||
|
||||
async end() {
|
||||
endCalls += 1;
|
||||
await new Promise((resolve) => { releaseReset = resolve; });
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
await store.readiness();
|
||||
const classified = {
|
||||
blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED,
|
||||
connection: { queryResult: "connect_timeout", errorCode: "ETIMEDOUT" }
|
||||
};
|
||||
|
||||
assert.equal(store.resetPostgresPoolAfterTransientFailure(classified, { attempt: 1, maxAttempts: 5, retrying: true }), true);
|
||||
const activePool = { totalCount: 16, idleCount: 0, waitingCount: 6, async end() { endCalls += 1; } };
|
||||
store.pool = activePool;
|
||||
assert.equal(store.resetPostgresPoolAfterTransientFailure(classified, { attempt: 2, maxAttempts: 5, retrying: true }), false);
|
||||
assert.equal(store.pool, activePool);
|
||||
await Promise.resolve();
|
||||
assert.equal(endCalls, 1);
|
||||
assert.equal(warnings.some((entry) => entry.event === "postgres_runtime_pool_reset_skipped" && entry.reason === "in_flight"), true);
|
||||
releaseReset?.();
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
test("configured postgres runtime classifies pg_hba no-encryption rejection as SSL before auth", async () => {
|
||||
const store = createConfiguredCloudRuntimeStore({
|
||||
env: {
|
||||
@@ -1070,7 +1120,8 @@ test("configured postgres runtime writes Workbench facts without re-running bloc
|
||||
const store = createConfiguredCloudRuntimeStore({
|
||||
env: {
|
||||
HWLAB_CLOUD_RUNTIME_ADAPTER: "postgres",
|
||||
HWLAB_CLOUD_RUNTIME_DURABLE: "true"
|
||||
HWLAB_CLOUD_RUNTIME_DURABLE: "true",
|
||||
HWLAB_CLOUD_DB_QUERY_RETRY_MAX_ATTEMPTS: "1"
|
||||
},
|
||||
dbUrl: "postgres://hwlab_redacted@db.example.invalid:5432/hwlab",
|
||||
queryClient,
|
||||
@@ -1079,7 +1130,9 @@ test("configured postgres runtime writes Workbench facts without re-running bloc
|
||||
|
||||
const readiness = await store.readiness();
|
||||
assert.equal(readiness.blocker, RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED);
|
||||
assert.equal(readiness.connection.queryResult, "query_blocked");
|
||||
assert.equal(readiness.connection.queryResult, "too_many_connections");
|
||||
assert.equal(readiness.retryable, true);
|
||||
assert.equal(readiness.transient, true);
|
||||
queryClient.calls.length = 0;
|
||||
|
||||
const write = await store.writeWorkbenchFacts({
|
||||
|
||||
@@ -139,9 +139,11 @@ const RUNTIME_DB_QUERY_TIMEOUT_MS_ENV = "HWLAB_CLOUD_DB_QUERY_TIMEOUT_MS";
|
||||
const RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS_ENV = "HWLAB_CLOUD_DB_QUERY_RETRY_MAX_ATTEMPTS";
|
||||
const RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS_ENV = "HWLAB_CLOUD_DB_QUERY_RETRY_INITIAL_DELAY_MS";
|
||||
const RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS_ENV = "HWLAB_CLOUD_DB_QUERY_RETRY_MAX_DELAY_MS";
|
||||
const RUNTIME_DB_POOL_RESET_COOLDOWN_MS_ENV = "HWLAB_CLOUD_DB_POOL_RESET_COOLDOWN_MS";
|
||||
const DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS = 5;
|
||||
const DEFAULT_RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS = 250;
|
||||
const DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS = 5_000;
|
||||
const DEFAULT_RUNTIME_DB_POOL_RESET_COOLDOWN_MS = 10_000;
|
||||
|
||||
function normalizePositiveInteger(value, fallback) {
|
||||
const number = Number(value);
|
||||
@@ -826,6 +828,8 @@ export class PostgresCloudRuntimeStore {
|
||||
this.pgModuleLoader = pgModuleLoader;
|
||||
this.logger = logger;
|
||||
this.pool = null;
|
||||
this.postgresPoolResetInFlight = null;
|
||||
this.postgresPoolResetLastAtMs = 0;
|
||||
this.runtimeReadIndexesReady = false;
|
||||
this.runtimeReadIndexesReadyPromise = null;
|
||||
this.memory = new CloudRuntimeStore({ now });
|
||||
@@ -1941,7 +1945,43 @@ export class PostgresCloudRuntimeStore {
|
||||
if (classified?.connection?.queryResult !== "connect_timeout") return false;
|
||||
const pool = this.pool;
|
||||
if (!pool) return false;
|
||||
const now = Date.now();
|
||||
const cooldownMs = this.runtimePoolResetCooldownMs();
|
||||
if (this.postgresPoolResetInFlight) {
|
||||
this.logger?.warn?.({
|
||||
event: "postgres_runtime_pool_reset_skipped",
|
||||
reason: "in_flight",
|
||||
blocker: classified.blocker,
|
||||
queryResult: classified.connection?.queryResult ?? "query_blocked",
|
||||
errorCode: classified.connection?.errorCode ?? "UNKNOWN",
|
||||
retryAttempt: Number.isFinite(Number(retry.attempt)) ? Number(retry.attempt) : null,
|
||||
retryMaxAttempts: Number.isFinite(Number(retry.maxAttempts)) ? Number(retry.maxAttempts) : null,
|
||||
cooldownMs,
|
||||
poolStats: postgresPoolStats(pool),
|
||||
valuesRedacted: true,
|
||||
endpointRedacted: true
|
||||
});
|
||||
return false;
|
||||
}
|
||||
if (now - this.postgresPoolResetLastAtMs < cooldownMs) {
|
||||
this.logger?.warn?.({
|
||||
event: "postgres_runtime_pool_reset_skipped",
|
||||
reason: "cooldown",
|
||||
blocker: classified.blocker,
|
||||
queryResult: classified.connection?.queryResult ?? "query_blocked",
|
||||
errorCode: classified.connection?.errorCode ?? "UNKNOWN",
|
||||
retryAttempt: Number.isFinite(Number(retry.attempt)) ? Number(retry.attempt) : null,
|
||||
retryMaxAttempts: Number.isFinite(Number(retry.maxAttempts)) ? Number(retry.maxAttempts) : null,
|
||||
cooldownMs,
|
||||
poolResetLastAtMs: this.postgresPoolResetLastAtMs,
|
||||
poolStats: postgresPoolStats(pool),
|
||||
valuesRedacted: true,
|
||||
endpointRedacted: true
|
||||
});
|
||||
return false;
|
||||
}
|
||||
this.pool = null;
|
||||
this.postgresPoolResetLastAtMs = now;
|
||||
this.logger?.warn?.({
|
||||
event: "postgres_runtime_pool_reset_after_transient_failure",
|
||||
blocker: classified.blocker,
|
||||
@@ -1950,22 +1990,32 @@ export class PostgresCloudRuntimeStore {
|
||||
retryAttempt: Number.isFinite(Number(retry.attempt)) ? Number(retry.attempt) : null,
|
||||
retryMaxAttempts: Number.isFinite(Number(retry.maxAttempts)) ? Number(retry.maxAttempts) : null,
|
||||
retrying: retry.retrying === true,
|
||||
cooldownMs,
|
||||
poolStats: postgresPoolStats(pool),
|
||||
valuesRedacted: true,
|
||||
endpointRedacted: true
|
||||
});
|
||||
if (typeof pool.end === "function") {
|
||||
Promise.resolve()
|
||||
this.postgresPoolResetInFlight = Promise.resolve()
|
||||
.then(() => pool.end())
|
||||
.catch((error) => this.logger?.warn?.({
|
||||
event: "postgres_runtime_pool_reset_failed",
|
||||
errorCode: error?.code ?? "UNKNOWN",
|
||||
valuesRedacted: true,
|
||||
endpointRedacted: true
|
||||
}));
|
||||
}))
|
||||
.finally(() => {
|
||||
this.postgresPoolResetInFlight = null;
|
||||
});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
runtimePoolResetCooldownMs() {
|
||||
const env = this.env ?? {};
|
||||
return normalizePositiveInteger(env[RUNTIME_DB_POOL_RESET_COOLDOWN_MS_ENV], DEFAULT_RUNTIME_DB_POOL_RESET_COOLDOWN_MS);
|
||||
}
|
||||
|
||||
async query(sql, params = []) {
|
||||
const retry = this.runtimeQueryRetryPolicy();
|
||||
for (let attempt = 1; attempt <= retry.maxAttempts; attempt += 1) {
|
||||
@@ -2094,7 +2144,7 @@ export class PostgresCloudRuntimeStore {
|
||||
});
|
||||
}
|
||||
|
||||
blockedSummary({ blocker, reason, schema, migration, connection, gates }) {
|
||||
blockedSummary({ blocker, reason, schema, migration, connection, gates, retryable, transient, retryAfterMs }) {
|
||||
const blockedSchema = schema ?? {
|
||||
ready: false,
|
||||
checked: false,
|
||||
@@ -2111,6 +2161,9 @@ export class PostgresCloudRuntimeStore {
|
||||
status: "blocked",
|
||||
blocker,
|
||||
reason,
|
||||
retryable: retryable === true ? true : undefined,
|
||||
transient: transient === true ? true : undefined,
|
||||
retryAfterMs: Number.isFinite(Number(retryAfterMs)) ? Math.max(0, Math.trunc(Number(retryAfterMs))) : undefined,
|
||||
liveRuntimeEvidence: false,
|
||||
fixtureEvidence: false,
|
||||
connection: {
|
||||
@@ -3167,6 +3220,20 @@ export function classifyRuntimeDbError(error) {
|
||||
}
|
||||
};
|
||||
}
|
||||
if (code === "53300") {
|
||||
return {
|
||||
blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED,
|
||||
reason: "Postgres runtime adapter reached the database, but the server has exhausted available connection slots",
|
||||
retryable: true,
|
||||
transient: true,
|
||||
retryAfterMs: 2000,
|
||||
connection: {
|
||||
queryAttempted: true,
|
||||
queryResult: "too_many_connections",
|
||||
errorCode: code
|
||||
}
|
||||
};
|
||||
}
|
||||
return {
|
||||
blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED,
|
||||
reason: "Postgres runtime adapter could not complete a live runtime schema query",
|
||||
@@ -3467,6 +3534,16 @@ function parseJsonColumn(value, fallback) {
|
||||
}
|
||||
}
|
||||
|
||||
function postgresPoolStats(pool) {
|
||||
if (!pool || typeof pool !== "object") return null;
|
||||
return {
|
||||
totalCount: Number.isFinite(Number(pool.totalCount)) ? Number(pool.totalCount) : null,
|
||||
idleCount: Number.isFinite(Number(pool.idleCount)) ? Number(pool.idleCount) : null,
|
||||
waitingCount: Number.isFinite(Number(pool.waitingCount)) ? Number(pool.waitingCount) : null,
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function toCountKey(table) {
|
||||
if (table === "users") return "users";
|
||||
if (table === "user_sessions") return "userSessions";
|
||||
|
||||
@@ -40,8 +40,9 @@ const facts = computed(() => {
|
||||
].filter((item): item is { label: string; value: string } => Boolean(item));
|
||||
});
|
||||
const primaryTraceFact = computed(() => facts.value.find((item) => item.label === "trace_id") ?? null);
|
||||
const primaryTraceId = computed(() => primaryTraceFact.value?.value ?? null);
|
||||
const summaryText = computed(() => {
|
||||
const parts = [props.showMessage && message.value ? message.value : null, primaryTraceFact.value ? `trace_id=${primaryTraceFact.value.value}` : null].filter(Boolean);
|
||||
const parts = [props.showMessage && message.value ? message.value : null].filter(Boolean);
|
||||
return parts.join(" ") || (props.title ? "" : "诊断详情");
|
||||
});
|
||||
const hasContent = computed(() => Boolean(message.value || facts.value.length));
|
||||
@@ -102,6 +103,10 @@ function copyDiagnostic(): void {
|
||||
</div>
|
||||
<button v-if="facts.length" class="api-error-diagnostic-toggle" type="button" :aria-expanded="expanded ? 'true' : 'false'" aria-label="诊断详情" title="诊断详情" @click="expanded = !expanded">!</button>
|
||||
</header>
|
||||
<p v-if="primaryTraceId" class="api-error-trace-line" data-testid="api-error-trace-id">
|
||||
<span>Trace ID</span>
|
||||
<code>{{ primaryTraceId }}</code>
|
||||
</p>
|
||||
<div v-if="facts.length && expanded" class="api-error-diagnostic-details">
|
||||
<button class="table-action" type="button" @click="copyDiagnostic">{{ copied ? "已复制" : "复制诊断" }}</button>
|
||||
<dl class="api-error-diagnostic-grid">
|
||||
@@ -113,3 +118,21 @@ function copyDiagnostic(): void {
|
||||
</div>
|
||||
</section>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.api-error-diagnostic { display: grid; min-width: 0; gap: 6px; }
|
||||
.api-error-diagnostic-summary { display: flex; min-width: 0; align-items: start; justify-content: space-between; gap: 8px; }
|
||||
.api-error-diagnostic-summary-text { min-width: 0; }
|
||||
.api-error-diagnostic-summary-text strong { display: block; margin-bottom: 2px; font-size: 12px; font-weight: 850; }
|
||||
.api-error-diagnostic-summary-text p { margin: 0; overflow-wrap: anywhere; }
|
||||
.api-error-diagnostic-toggle { display: inline-flex; width: 22px; height: 22px; flex: 0 0 auto; align-items: center; justify-content: center; border: 1px solid #fed7aa; border-radius: 999px; background: #fff7ed; color: #9a3412; font: inherit; font-size: 12px; font-weight: 850; }
|
||||
.api-error-trace-line { display: flex; min-width: 0; flex-wrap: wrap; gap: 6px; align-items: center; margin: 0; color: #7c2d12; font-size: 12px; }
|
||||
.api-error-trace-line span { font-weight: 850; }
|
||||
.api-error-trace-line code { min-width: 0; max-width: 100%; overflow-wrap: anywhere; border: 1px solid #fed7aa; border-radius: 4px; background: #fffbeb; color: #7c2d12; padding: 1px 5px; }
|
||||
.api-error-diagnostic-details { display: grid; min-width: 0; gap: 6px; }
|
||||
.api-error-diagnostic-grid { display: grid; min-width: 0; gap: 5px; margin: 0; }
|
||||
.api-error-diagnostic-grid div { display: grid; min-width: 0; grid-template-columns: 112px minmax(0, 1fr); gap: 8px; align-items: start; }
|
||||
.api-error-diagnostic-grid dt { color: #7c2d12; font-weight: 800; }
|
||||
.api-error-diagnostic-grid dd { min-width: 0; margin: 0; }
|
||||
.api-error-diagnostic-grid code { overflow-wrap: anywhere; }
|
||||
</style>
|
||||
|
||||
@@ -514,6 +514,8 @@ function setError(err: unknown): void {
|
||||
:mutation-error="mutation.taskMutationError.value"
|
||||
:launch-loading="launch.launchLoading.value"
|
||||
:launch-error="launch.launchError.value"
|
||||
:launch-api-error="launch.launchApiError.value"
|
||||
:launch-diagnostic="launch.launchDiagnostic.value"
|
||||
:launch-enabled="workbenchLaunchEnabled"
|
||||
:launch-blocker="workbenchLaunchBlocker"
|
||||
:provider-profile="workbench.providerProfile"
|
||||
|
||||
@@ -5,9 +5,11 @@
|
||||
import { computed } from "vue";
|
||||
import type { MdtodoTaskDetailRecord, MdtodoTaskLinkRecord, MdtodoTaskRecord, ProjectNavigationResponse } from "@/api";
|
||||
import type { ProviderProfile } from "@/types";
|
||||
import type { ApiError, ErrorDiagnostic } from "@/types";
|
||||
import type { ProviderProfileOption } from "@/stores/workbench-session";
|
||||
import LoadingState from "@/components/common/LoadingState.vue";
|
||||
import EmptyState from "@/components/common/EmptyState.vue";
|
||||
import ApiErrorDiagnostic from "@/components/common/ApiErrorDiagnostic.vue";
|
||||
|
||||
const props = defineProps<{
|
||||
task: MdtodoTaskRecord | null;
|
||||
@@ -28,6 +30,8 @@ const props = defineProps<{
|
||||
mutationError: string | null;
|
||||
launchLoading: boolean;
|
||||
launchError: string | null;
|
||||
launchApiError: ApiError | null;
|
||||
launchDiagnostic: ErrorDiagnostic | null;
|
||||
launchEnabled: boolean;
|
||||
launchBlocker: string | null;
|
||||
providerProfile: ProviderProfile;
|
||||
@@ -104,7 +108,7 @@ function normalizeTaskText(value?: string | null): string {
|
||||
<div class="task-status-stack">
|
||||
<p v-if="mutationMessage" class="source-message" data-testid="mdtodo-task-mutation-message">{{ mutationMessage }}</p>
|
||||
<p v-if="mutationError" class="source-error" data-testid="mdtodo-task-mutation-error">{{ mutationError }}</p>
|
||||
<p v-if="launchError" class="launch-blocker" data-testid="mdtodo-workbench-launch-error">{{ launchError }}</p>
|
||||
<ApiErrorDiagnostic v-if="launchError" class="launch-blocker" data-testid="mdtodo-workbench-launch-error" title="Workbench launch" :error="launchError" :api-error="launchApiError" :diagnostic="launchDiagnostic" compact />
|
||||
<p v-else-if="launchBlocker" class="launch-blocker" data-testid="mdtodo-workbench-launch-blocker">{{ launchBlocker }}</p>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
|
||||
import { ref } from "vue";
|
||||
import { agentAPI, workbenchAPI, type MdtodoTaskLinkRecord, type MdtodoTaskRecord, type ProjectNavigationResponse } from "@/api";
|
||||
import { apiErrorContextFromUnknown } from "@/api/client";
|
||||
import type { ApiError, ErrorDiagnostic } from "@/types";
|
||||
import type { Router } from "vue-router";
|
||||
|
||||
export interface WorkbenchLaunchContext {
|
||||
@@ -62,8 +64,16 @@ export interface WorkbenchLaunchExecutionContext {
|
||||
export function useMdtodoWorkbenchLaunch(router: Router) {
|
||||
const launchLoading = ref(false);
|
||||
const launchError = ref<string | null>(null);
|
||||
const launchApiError = ref<ApiError | null>(null);
|
||||
const launchDiagnostic = ref<ErrorDiagnostic | null>(null);
|
||||
const launchResult = ref<{ sessionId: string; workbenchUrl: string } | null>(null);
|
||||
|
||||
function setLaunchError(error: string | null, apiError: ApiError | null = null, diagnostic: ErrorDiagnostic | null = null): void {
|
||||
launchError.value = error;
|
||||
launchApiError.value = apiError;
|
||||
launchDiagnostic.value = diagnostic;
|
||||
}
|
||||
|
||||
function buildPrompt(task: MdtodoTaskRecord, body: string, links: MdtodoTaskLinkRecord[], fileName: string, launchContext?: WorkbenchLaunchContext | null): string {
|
||||
const reportLines = links
|
||||
.filter((link) => link.kind === "markdown-report")
|
||||
@@ -143,16 +153,16 @@ export function useMdtodoWorkbenchLaunch(router: Router) {
|
||||
if (!task.taskRef || !task.projectId) return null;
|
||||
const enabled = navigation?.navigation?.capabilities?.workbenchLaunch === true;
|
||||
if (!enabled) {
|
||||
launchError.value = "Workbench Launch capability unavailable";
|
||||
setLaunchError("Workbench Launch capability unavailable");
|
||||
return null;
|
||||
}
|
||||
const selectedProviderProfile = providerProfile.trim();
|
||||
if (!selectedProviderProfile) {
|
||||
launchError.value = "模型通道未选择";
|
||||
setLaunchError("模型通道未选择");
|
||||
return null;
|
||||
}
|
||||
launchLoading.value = true;
|
||||
launchError.value = null;
|
||||
setLaunchError(null);
|
||||
try {
|
||||
const launchContext = { ...buildContext(task, body, links, fileName), providerProfile: selectedProviderProfile };
|
||||
const response = await workbenchAPI.launch({
|
||||
@@ -181,7 +191,8 @@ export function useMdtodoWorkbenchLaunch(router: Router) {
|
||||
launchResult.value = result;
|
||||
return result;
|
||||
} catch (err) {
|
||||
launchError.value = err && typeof err === "object" && "error" in err ? String((err as { error?: string }).error || "Workbench launch 失败") : "Workbench launch 失败";
|
||||
const context = apiErrorContextFromUnknown(err, "Workbench launch 失败");
|
||||
setLaunchError(context.error || "Workbench launch 失败", context.apiError, context.diagnostic);
|
||||
return null;
|
||||
} finally {
|
||||
launchLoading.value = false;
|
||||
@@ -199,6 +210,8 @@ export function useMdtodoWorkbenchLaunch(router: Router) {
|
||||
return {
|
||||
launchLoading,
|
||||
launchError,
|
||||
launchApiError,
|
||||
launchDiagnostic,
|
||||
launchResult,
|
||||
buildPrompt,
|
||||
buildContext,
|
||||
|
||||
Reference in New Issue
Block a user