add provider profile real test action

2026-07-09 13:57:35 +02:00
parent b4acb18524
commit fe2948387b
9 changed files with 214 additions and 4 deletions
@@ -294,6 +294,45 @@ test("provider profile remove delegates through HWLAB and preserves redaction",
  }
 });

+test("provider profile real test delegates prompt and model through AgentRun validation", async () => {
+  const calls: Array<AgentRunCall> = [];
+  const agentRunServer = await startAgentRunServer(calls);
+  const { port: agentRunPort } = agentRunServer.address() as { port: number };
+  const server = createCloudApiServer({
+    env: testEnv(agentRunPort),
+    accessController: fakeAccessController({ actor: adminActor })
+  });
+  await listen(server);
+  const { port } = server.address() as { port: number };
+
+  try {
+    const response = await fetch(`http://127.0.0.1:${port}/v1/admin/provider-profiles/gpt.pika/test`, {
+      method: "POST",
+      headers: { "content-type": "application/json", cookie: "hwlab_session=admin" },
+      body: JSON.stringify({ model: "gpt-5.4-mini", prompt: "只回复 OK" })
+    });
+    const body = await response.json();
+    const text = JSON.stringify(body);
+    assert.equal(response.status, 200);
+    assert.equal(body.ok, true);
+    assert.equal(body.profile, "gpt.pika");
+    assert.equal(body.data.profile, "gpt.pika");
+    assert.equal(body.data.backendProfile, "gpt-pika");
+    assert.equal(body.data.model, "gpt-5.4-mini");
+    assert.equal(body.data.validationId, "val_1234567890abcdef1234567890abcdef");
+    assert.equal(text.includes("sk-test"), false);
+    assert.equal(calls[0].method, "POST");
+    assert.equal(calls[0].path, "/api/v1/provider-profiles/gpt-pika/validate");
+    assert.equal(calls[0].authorization, "Bearer test-agentrun-manager-key");
+    assert.equal(calls[0].body.model, "gpt-5.4-mini");
+    assert.equal(calls[0].body.prompt, "只回复 OK");
+    assert.equal(calls[0].body.delegatedBy.system, "hwlab-v02");
+  } finally {
+    await close(server);
+    await close(agentRunServer);
+  }
+});
+
 function testEnv(port: number) {
  return {
    HWLAB_CODE_AGENT_AGENTRUN_MGR_URL: `http://127.0.0.1:${port}`,
@@ -354,6 +393,10 @@ async function startAgentRunServer(calls: Array<AgentRunCall>) {
      response.end(`${JSON.stringify({ ok: true, traceId: "trc_remove", data: { action: "provider-profile-removed", mutation: true, profile: "deepseek", configured: false, removed: true, builtinCapabilityRetained: true, secretRef: { namespace: "agentrun-v01", name: "agentrun-v01-provider-deepseek", keys: ["auth.json", "config.toml"] }, deletedResourceVersion: "rv-removed", credentialHashSuffix: "abc12345", configHashSuffix: "def67890", updatedAt: "2026-06-05T08:02:00.000Z", valuesPrinted: false } })}\n`);
      return;
    }
+    if (request.method === "POST" && url.pathname === "/api/v1/provider-profiles/gpt-pika/validate") {
+      response.end(`${JSON.stringify({ ok: true, traceId: "trc_test_submit", data: { action: "provider-profile-validation-started", validationId: "val_1234567890abcdef1234567890abcdef", profile: "gpt-pika", model: body?.model, runId: "run_profile_test", commandId: "cmd_profile_test", jobName: "agentrun-nc01-v02-runner-profile-test", status: "running", valuesPrinted: false } })}\n`);
+      return;
+    }
    response.end(`${JSON.stringify({ ok: false, error: { code: "unexpected", message: `${request.method} ${url.pathname}` } })}\n`);
  });
  await listen(server);
@@ -107,6 +107,7 @@ export async function handleProviderProfilesHttp(request, response, url, options
      const body = await readOptionalJsonObject(request, options.bodyLimitBytes);
      const delegatedBody = {
        ...(text(body.prompt) ? { prompt: text(body.prompt) } : {}),
+        ...(text(body.model) ? { model: text(body.model) } : {}),
        delegatedBy: delegatedBy(auth, requestId),
        reason: text(body.reason) || "hwlab-provider-management"
      };
@@ -147,7 +148,7 @@ function parseProviderProfileRoute(pathname, method) {
    if (method !== "GET") throw routeError(405, "method_not_allowed", "Provider profiles collection only supports GET");
    return { kind: "list", publicProfile: null, agentRunProfile: null };
  }
-  const match = pathname.match(/^\/v1\/admin\/provider-profiles\/([^/]+)(?:\/(credential|config|validate|validations\/([^/]+)))?$/u);
+  const match = pathname.match(/^\/v1\/admin\/provider-profiles\/([^/]+)(?:\/(credential|config|validate|test|validations\/([^/]+)))?$/u);
  if (!match) throw routeError(404, "not_found", "Provider profile route is not implemented");
  const profile = normalizeProfile(decodeURIComponent(match[1]));
  const suffix = match[2] ?? "";
@@ -163,7 +164,7 @@ function parseProviderProfileRoute(pathname, method) {
    if (method !== "GET" && method !== "PUT") throw routeError(405, "method_not_allowed", "Provider profile config only supports GET and PUT");
    return { kind: "config", method, ...profile };
  }
-  if (suffix === "validate") {
+  if (suffix === "validate" || suffix === "test") {
    if (method !== "POST") throw routeError(405, "method_not_allowed", "Provider profile validation only supports POST");
    return { kind: "validate", ...profile };
  }
@@ -102,6 +102,7 @@ function isProviderProfileManagementProxyRoute(method, pathname) {
  if (method === "PUT" && parts.length === 5 && parts[4] === "credential") return true;
  if (method === "PUT" && parts.length === 5 && parts[4] === "config") return true;
  if (method === "POST" && parts.length === 5 && parts[4] === "validate") return true;
+  if (method === "POST" && parts.length === 5 && parts[4] === "test") return true;
  return false;
 }

@@ -190,6 +190,12 @@ test("cloud web proxies authenticated provider profile management write routes",
    publicRoute: false,
    routeKey: "POST /v1/admin/provider-profiles/deepseek/validate"
  });
+  assert.deepEqual(cloudWebProxyRoutePolicy("POST", "/v1/admin/provider-profiles/deepseek/test"), {
+    proxy: true,
+    authRequired: true,
+    publicRoute: false,
+    routeKey: "POST /v1/admin/provider-profiles/deepseek/test"
+  });
  assert.deepEqual(cloudWebProxyRoutePolicy("DELETE", "/v1/admin/provider-profiles/deepseek"), {
    proxy: true,
    authRequired: true,
@@ -230,4 +236,3 @@ test("cloud web proxies authenticated Workbench launch writes to cloud-api", ()
    routeKey: "POST /v1/workbench/launches"
  });
 });
-
@@ -295,7 +295,18 @@ test("cloud web proxies provider profile management write routes", async () => {
    });
    assert.equal(validateResponse.status, 200);

-    assert.equal(upstreamRequests.length, 2);
+    const testBody = JSON.stringify({ model: "gpt-5.4-mini", prompt: "只回复 OK" });
+    const testResponse = await fetch(`${serverUrl(cloudWeb)}/v1/admin/provider-profiles/deepseek/test`, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        authorization: "Bearer hwl_live_admin"
+      },
+      body: testBody
+    });
+    assert.equal(testResponse.status, 200);
+
+    assert.equal(upstreamRequests.length, 3);
    assert.deepEqual(upstreamRequests[0], {
      method: "PUT",
      url: "/v1/admin/provider-profiles/deepseek/credential",
@@ -308,6 +319,12 @@ test("cloud web proxies provider profile management write routes", async () => {
      authorization: "Bearer hwl_live_admin",
      body: "{}"
    });
+    assert.deepEqual(upstreamRequests[2], {
+      method: "POST",
+      url: "/v1/admin/provider-profiles/deepseek/test",
+      authorization: "Bearer hwl_live_admin",
+      body: testBody
+    });
  } finally {
    await close(cloudWeb);
    await close(upstream);
@@ -0,0 +1,27 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { normalizeProviderValidation } from "../src/stores/provider-profiles-view.ts";
+
+test("Provider validation exposes real test model and final response", () => {
+  const validation = normalizeProviderValidation({
+    data: {
+      validationId: "val_profile_test",
+      profile: "gpt.pika",
+      model: "gpt-5.4-mini",
+      status: "completed",
+      runId: "run_profile_test",
+      commandId: "cmd_profile_test",
+      result: {
+        finalResponse: { text: "HWLAB provider profile test OK" },
+        elapsedMs: 1234,
+        terminalStatus: "completed"
+      },
+      valuesPrinted: false
+    }
+  });
+  assert.equal(validation.profile, "gpt.pika");
+  assert.equal(validation.model, "gpt-5.4-mini");
+  assert.equal(validation.outputText, "HWLAB provider profile test OK");
+  assert.equal(validation.elapsedMs, "1234");
+});
@@ -9,5 +9,6 @@ export const providerProfilesAPI = {
  config: (profile: ProviderProfile): Promise<ApiResult<ProviderProfilesResponse>> => fetchJson(`/v1/admin/provider-profiles/${encodeURIComponent(profile)}/config`, { timeoutMs: 12000, timeoutName: "provider profile config" }),
  setConfig: (profile: ProviderProfile, configToml: string): Promise<ApiResult<ProviderProfilesResponse>> => fetchJson(`/v1/admin/provider-profiles/${encodeURIComponent(profile)}/config`, { method: "PUT", body: JSON.stringify({ configToml }), timeoutMs: 30000, timeoutName: "provider profile config update" }),
  validate: (profile: ProviderProfile): Promise<ApiResult<ProviderProfilesResponse>> => fetchJson(`/v1/admin/provider-profiles/${encodeURIComponent(profile)}/validate`, { method: "POST", body: JSON.stringify({}), timeoutMs: 30000, timeoutName: "provider profile validation submit" }),
+  test: (profile: ProviderProfile, model: string, prompt: string): Promise<ApiResult<ProviderProfilesResponse>> => fetchJson(`/v1/admin/provider-profiles/${encodeURIComponent(profile)}/test`, { method: "POST", body: JSON.stringify({ model, prompt }), timeoutMs: 30000, timeoutName: "provider profile real test submit" }),
  validation: (profile: ProviderProfile, validationId: string): Promise<ApiResult<ProviderProfilesResponse & ProviderProfileValidation>> => fetchJson(`/v1/admin/provider-profiles/${encodeURIComponent(profile)}/validations/${encodeURIComponent(validationId)}`, { timeoutMs: 12000, timeoutName: "provider profile validation" })
 };
@@ -26,6 +26,7 @@ export interface ProviderConfigView {
 export interface ProviderValidationView {
  validationId: string;
  profile: string;
+  model: string;
  status: string;
  runId: string;
  commandId: string;
@@ -33,6 +34,8 @@ export interface ProviderValidationView {
  traceId: string;
  failureKind: string;
  terminalStatus: string;
+  outputText: string;
+  elapsedMs: string;
  valuesPrinted: boolean;
 }

@@ -69,6 +72,7 @@ export function normalizeProviderValidation(payload: (ProviderProfilesResponse &
  return {
    validationId: text(data.validationId ?? record?.validationId),
    profile: text(data.profile ?? record?.profile) || fallbackProfile,
+    model: text(data.model ?? result?.model ?? objectRecord(result?.command)?.model),
    status: text(data.status ?? record?.status) || "unknown",
    runId: text(data.runId ?? agentRun?.runId),
    commandId: text(data.commandId ?? agentRun?.commandId),
@@ -76,6 +80,8 @@ export function normalizeProviderValidation(payload: (ProviderProfilesResponse &
    traceId: text(data.traceId ?? agentRun?.traceId),
    failureKind: text(data.failureKind ?? result?.failureKind ?? objectRecord(data.error)?.code),
    terminalStatus: text(data.terminalStatus ?? result?.terminalStatus),
+    outputText: finalTextFromResult(result),
+    elapsedMs: scalarText(result?.elapsedMs ?? result?.durationMs ?? agentRun?.elapsedMs),
    valuesPrinted: data.valuesPrinted === true || record?.valuesPrinted === true
  };
 }
@@ -145,8 +151,24 @@ function text(value: unknown): string {
  return typeof value === "string" ? value.trim() : "";
 }

+function scalarText(value: unknown): string {
+  if (typeof value === "number" && Number.isFinite(value)) return String(Math.round(value));
+  return text(value);
+}
+
 function booleanValue(value: unknown): boolean {
  if (typeof value === "boolean") return value;
  if (typeof value === "string") return ["1", "true", "yes", "ready", "configured"].includes(value.toLowerCase());
  return Boolean(value);
 }
+
+function finalTextFromResult(result: Record<string, unknown> | null): string {
+  if (!result) return "";
+  return text(
+    objectRecord(result.finalResponse)?.text ??
+    objectRecord(result.reply)?.content ??
+    objectRecord(result.message)?.content ??
+    result.assistantText ??
+    result.finalText
+  );
+}
@@ -31,10 +31,14 @@ const selectedProfile = ref<ProviderProfileRow | null>(null);
 const credentialDialogOpen = ref(false);
 const configDialogOpen = ref(false);
 const removeDialogOpen = ref(false);
+const testDialogOpen = ref(false);
 const apiKeyInput = ref("");
 const configText = ref("");
+const testModel = ref("");
+const testPrompt = ref("");
 const configInfo = ref<ProviderConfigView | null>(null);
 const validation = ref<ProviderValidationView | null>(null);
+const testResult = ref<ProviderValidationView | null>(null);
 const validationError = ref<string | null>(null);
 const validationApiError = ref<ApiError | null>(null);
 const validationDiagnostic = ref<ErrorDiagnostic | null>(null);
@@ -42,6 +46,7 @@ const validatingProfile = ref<string | null>(null);
 const credentialForm = useForm();
 const configForm = useForm();
 const removeForm = useForm();
+const testForm = useForm();
 const page = ref(1);
 const pageSize = 10;

@@ -72,6 +77,17 @@ function openRemove(row: ProviderProfileRow): void {
  removeForm.error.value = null;
 }

+function openTest(row: ProviderProfileRow): void {
+  selectedProfile.value = row;
+  testModel.value = defaultModelFor(row);
+  testPrompt.value = `请用一句中文回复：HWLAB provider profile ${row.profile} 测试通过。`;
+  testResult.value = null;
+  testForm.error.value = null;
+  testForm.apiError.value = null;
+  testForm.diagnostic.value = null;
+  testDialogOpen.value = true;
+}
+
 async function saveCredential(): Promise<void> {
  const row = selectedProfile.value;
  if (!row || !apiKeyInput.value.trim()) return;
@@ -164,6 +180,26 @@ async function validateProfile(row: ProviderProfileRow): Promise<void> {
  void table.reload();
 }

+async function runRealTest(): Promise<void> {
+  const row = selectedProfile.value;
+  if (!row || !testModel.value.trim() || !testPrompt.value.trim()) return;
+  testResult.value = null;
+  await testForm.submit(async () => {
+    const submit = await providerProfilesAPI.test(row.profile, testModel.value.trim(), testPrompt.value.trim());
+    if (!submit.ok) throw submit;
+    let current = normalizeProviderValidation(submit.data, row.profile);
+    testResult.value = current;
+    const validationId = current.validationId;
+    for (let attempt = 0; validationId && attempt < 45 && !validationComplete(current.status); attempt += 1) {
+      await delay(2000);
+      const poll = await providerProfilesAPI.validation(row.profile, validationId);
+      if (!poll.ok) throw poll;
+      current = normalizeProviderValidation(poll.data, row.profile);
+      testResult.value = current;
+    }
+  }, "Profile test finished");
+}
+
 function delay(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
 }
@@ -174,6 +210,13 @@ function displayDate(value: string): string {
  if (Number.isNaN(date.getTime())) return value;
  return date.toLocaleString("zh-CN", { month: "2-digit", day: "2-digit", hour: "2-digit", minute: "2-digit" });
 }
+
+function defaultModelFor(row: ProviderProfileRow): string {
+  const source = row.source || {};
+  const configSummary = source.configSummary && typeof source.configSummary === "object" ? source.configSummary as Record<string, unknown> : null;
+  const candidates = [configSummary?.model, source.model, row.profile === "gpt.pika" ? "gpt-5.4-mini" : null, row.profile === "deepseek" ? "deepseek-chat" : null, row.profile === "dsflash-go" ? "deepseek-v4-flash" : null, row.profile === "minimax-m3" ? "MiniMax-M3" : null, "gpt-5.4-mini"];
+  return candidates.find((value): value is string => typeof value === "string" && value.trim().length > 0)?.trim() ?? "gpt-5.4-mini";
+}
 </script>

 <template>
@@ -215,6 +258,7 @@ function displayDate(value: string): string {
          <div class="table-actions-inline">
            <button class="table-action" type="button" @click="openCredential(row as ProviderProfileRow)">Key</button>
            <button class="table-action" type="button" @click="openConfig(row as ProviderProfileRow)">Config</button>
+            <button class="table-action" type="button" data-testid="provider-profile-test-open" @click="openTest(row as ProviderProfileRow)">测试</button>
            <button class="table-action" type="button" :disabled="validatingProfile !== null" @click="validateProfile(row as ProviderProfileRow)">{{ validatingProfile === (row as ProviderProfileRow).profile ? "验证中" : "Validate" }}</button>
            <button class="table-action danger" type="button" @click="openRemove(row as ProviderProfileRow)">Remove</button>
          </div>
@@ -273,7 +317,56 @@ function displayDate(value: string): string {
      </template>
    </BaseDialog>

+    <BaseDialog :open="testDialogOpen" title="真实 Responses 测试" :description="selectedProfile?.profile || ''" wide @close="testDialogOpen = false">
+      <form class="admin-form" data-testid="provider-profile-test-dialog" @submit.prevent="runRealTest">
+        <label>
+          <span>模型</span>
+          <input id="provider-profile-test-model" v-model="testModel" data-testid="provider-profile-test-model" autocomplete="off" placeholder="gpt-5.4-mini">
+        </label>
+        <label>
+          <span>测试提示词</span>
+          <textarea id="provider-profile-test-prompt" v-model="testPrompt" data-testid="provider-profile-test-prompt" rows="5" spellcheck="false" />
+        </label>
+        <ApiErrorDiagnostic v-if="testForm.error.value" :error="testForm.error.value" :api-error="testForm.apiError.value" :diagnostic="testForm.diagnostic.value" compact />
+        <section v-if="testResult" class="muted-box" data-testid="provider-profile-test-result">
+          <strong>{{ testResult.status }}</strong>
+          <dl class="field-grid">
+            <div><dt>profile</dt><dd>{{ testResult.profile || selectedProfile?.profile || "-" }}</dd></div>
+            <div><dt>model</dt><dd>{{ testResult.model || testModel || "-" }}</dd></div>
+            <div><dt>validationId</dt><dd>{{ testResult.validationId || "-" }}</dd></div>
+            <div><dt>runId</dt><dd>{{ testResult.runId || "-" }}</dd></div>
+            <div><dt>commandId</dt><dd>{{ testResult.commandId || "-" }}</dd></div>
+            <div><dt>jobName</dt><dd>{{ testResult.jobName || "-" }}</dd></div>
+            <div><dt>elapsedMs</dt><dd>{{ testResult.elapsedMs || "-" }}</dd></div>
+            <div><dt>failureKind</dt><dd>{{ testResult.failureKind || testResult.terminalStatus || "-" }}</dd></div>
+          </dl>
+          <pre v-if="testResult.outputText" class="provider-test-output">{{ testResult.outputText }}</pre>
+          <p v-else>等待终端回复，或当前结果未包含 final response。</p>
+        </section>
+        <p v-if="testForm.notice.value" class="form-notice">{{ testForm.notice.value }}</p>
+      </form>
+      <template #footer>
+        <button class="table-action" type="button" :disabled="testForm.loading.value" @click="testDialogOpen = false">关闭</button>
+        <button class="table-action" type="button" data-testid="provider-profile-test-submit" :disabled="testForm.loading.value || !testModel.trim() || !testPrompt.trim()" @click="runRealTest">{{ testForm.loading.value ? "测试中" : "测试" }}</button>
+      </template>
+    </BaseDialog>
+
    <ConfirmDialog :open="removeDialogOpen" title="Remove provider profile" :message="`删除 ${selectedProfile?.profile || 'profile'} 的管理配置；内建能力仍由后端决定是否保留。`" confirm-label="删除" :pending="removeForm.loading.value" @close="removeDialogOpen = false" @confirm="removeProfile" />
    <ApiErrorDiagnostic v-if="removeForm.error.value" class="admin-action-error" :error="removeForm.error.value" :api-error="removeForm.apiError.value" :diagnostic="removeForm.diagnostic.value" compact />
  </section>
 </template>
+
+<style scoped>
+.provider-test-output {
+  max-height: 180px;
+  overflow: auto;
+  margin: 12px 0 0;
+  padding: 10px 12px;
+  border: 1px solid var(--border-muted, #d8dee8);
+  border-radius: 6px;
+  background: var(--surface-muted, #f7f9fc);
+  color: var(--text-primary, #172033);
+  white-space: pre-wrap;
+  word-break: break-word;
+}
+</style>