pikasTech-HWLAB/tools/src/hwlab-caserun-diagnostics.ts

import { readFile, readdir, stat } from "node:fs/promises";
import path from "node:path";

import {
  archiveCaseRunArtifacts,
  syncCaseRegistryRel,
  writeCaseAggregateFromRegistry,
} from "./hwlab-caserun-closeout.ts";
import { agentToolCallSummaryFromResult, arrayOfObjects, clean, clipText, text } from "./hwlab-caserun-evidence.ts";
import type { CaseContext, PreparedCaseRun } from "./hwlab-caserun-shared.ts";

const MAX_DIAGNOSTIC_ROWS = 30;

export async function refreshCaseRegistryArtifacts(context: CaseContext, caseRepo: string) {
  const caseId = text(context.parsed.caseId ?? context.rest[1]);
  const runId = text(context.parsed.runId ?? context.rest[2]);
  if (!caseId) throw cliError("missing_required_value", "caseId is required", { field: "caseId" });
  if (!runId) throw cliError("missing_required_value", "runId is required", { field: "runId" });
  const caseRepoRunDir = path.join(caseRepo, "runs", caseId, runId);
  const stored = await loadRegistryArtifacts(caseRepo, caseId, runId);
  const source = text(context.parsed.source ?? context.parsed.from) || "local";
  if (source !== "local" && source !== "runtime") throw cliError("unsupported_case_refresh_source", "case refresh source must be local or runtime", { source });
  const run = registryPreparedRun(caseRepo, caseId, runId, caseRepoRunDir, stored.run, stored.evidence, stored.manifest, context.now());
  const evidence = {
    ...stored.evidence,
    caseId,
    runId,
    autoEvaluation: false,
    refresh: { source, authority: "registry", runtimeAccess: source === "runtime" },
  };
  const offlineContext = { ...context, env: { ...context.env, HWLAB_API_KEY: undefined }, parsed: { ...context.parsed, noCaseRepoGitSync: true } };
  const archiveContext = source === "runtime" ? { ...context, parsed: { ...context.parsed, caseRefreshSource: "runtime" } } : offlineContext;
  const first = await archiveCaseRunArtifacts(archiveContext, run, evidence, { sync: false });
  const aggregateContext = { ...offlineContext, parsed: { ...offlineContext.parsed, caseId, runId }, rest: ["aggregate", caseId, runId] };
  await writeCaseAggregateFromRegistry(aggregateContext, caseRepo, { sync: false });
  const final = await archiveCaseRunArtifacts(offlineContext, run, evidence, { sync: false });
  const sync = context.parsed.noCaseRepoGitSync === true
    ? null
    : await syncCaseRegistryRel(context, caseRepo, `runs/${caseId}/${runId}`, `data: refresh caserun artifacts ${runId}`);
  const trace = (final.manifest.trace ?? first.manifest.trace ?? {}) as Record<string, any>;
  return ok("case.refresh", {
    caseId,
    runId,
    caseRepo,
    caseRepoRunDir,
    source,
    refreshed: {
      files: final.caseRepoFiles,
      traceId: trace.traceId,
      sourceEventCount: trace.sourceEventCount,
      renderedRowCount: trace.renderedRowCount,
      commandCount: trace.commandCount,
      hwpodCommandCount: trace.hwpodCommandCount,
      hwpodBuildCommandCount: trace.hwpodBuildCommandCount,
      manifestPath: final.artifactManifestPath,
      registrySync: sync,
      provenance: evidence.refresh,
    },
    aggregationOnly: false,
    autoEvaluation: false,
  });
}

export async function inspectCaseRegistryRun(context: CaseContext, caseRepo: string, view: "inspect" | "trace-summary") {
  const artifacts = await loadDiagnosticByRunId(caseRepo, text(context.parsed.runId ?? context.rest[2]), text(context.parsed.caseId ?? context.parsed.case));
  const diagnostic = diagnosticFromArtifacts(artifacts);
  const payload = view === "trace-summary" ? traceSummary(diagnostic) : diagnostic;
  return ok(`case.run.${view}`, { ...payload, inspectionOnly: true, serviceRuntime: false, autoEvaluation: false });
}

export async function skillUsageCaseRegistryRun(context: CaseContext, caseRepo: string) {
  const skill = text(context.parsed.skill ?? context.parsed.skillName ?? context.rest[3]);
  if (!skill) throw cliError("missing_required_value", "skill is required", { field: "skill" });
  const artifacts = await loadDiagnosticByRunId(caseRepo, text(context.parsed.runId ?? context.rest[2]), text(context.parsed.caseId ?? context.parsed.case));
  const pattern = new RegExp(`(?:\\.agents[\\\\/]skills[\\\\/]${escapeRegExp(skill)}[\\\\/]SKILL\\.md|\\b${escapeRegExp(skill)}\\b)`, "iu");
  const bundleSources = [
    ["artifact-manifest.json", artifacts.manifest.resourceBundle ?? artifacts.manifest.provenance?.resourceBundle],
    ["evidence.json", artifacts.evidence.resourceBundle ?? artifacts.evidence.provenance?.resourceBundle],
  ] as const;
  const evidence = bundleSources
    .filter(([, value]) => value && pattern.test(JSON.stringify(value)))
    .map(([source, value]) => ({ source, materialized: value.materialized === true, preview: clipText(JSON.stringify(value).replace(/\s+/gu, " "), 500) }));
  const rows = traceRows(artifacts);
  const allMatches = rows.filter((row) => pattern.test(`${row.header ?? ""}\n${row.body ?? ""}`));
  const matches = allMatches.slice(0, MAX_DIAGNOSTIC_ROWS).map((row) => ({ rowId: row.rowId, seq: row.seq, header: row.header, preview: clipText(row.body, 500) }));
  return ok("case.run.skill-usage", {
    caseId: artifacts.caseId,
    runId: artifacts.runId,
    caseRepo,
    caseRepoRunDir: artifacts.caseRepoRunDir,
    skill,
    prompt: { mentionsSkill: pattern.test(artifacts.promptText), path: path.join(artifacts.caseRepoRunDir, "agent-prompt.md") },
    trace: { readSkillFile: allMatches.length > 0, matchCount: allMatches.length, matches },
    resourceBundle: { materialized: evidence.some((item) => item.materialized), evidence },
    inspectionOnly: true,
    serviceRuntime: false,
    autoEvaluation: false,
  });
}

async function loadDiagnosticByRunId(caseRepo: string, runId: string, explicitCaseId: string) {
  if (!runId) throw cliError("missing_required_value", "runId is required", { field: "runId" });
  const caseId = explicitCaseId || await resolveCaseId(caseRepo, runId);
  return loadRegistryArtifacts(caseRepo, caseId, runId);
}

async function loadRegistryArtifacts(caseRepo: string, caseId: string, runId: string) {
  const caseRepoRunDir = path.join(caseRepo, "runs", caseId, runId);
  const [run, evidence, manifest, agentTrace, agentMessages, finalResponseText, promptText] = await Promise.all([
    readJson(path.join(caseRepoRunDir, "run.json")),
    readJson(path.join(caseRepoRunDir, "evidence.json")),
    readJson(path.join(caseRepoRunDir, "artifact-manifest.json")),
    readJson(path.join(caseRepoRunDir, "agent-trace.json")),
    readJson(path.join(caseRepoRunDir, "agent-messages.json")),
    readText(path.join(caseRepoRunDir, "final-response.md")),
    readText(path.join(caseRepoRunDir, "agent-prompt.md")),
  ]);
  if (!run && !evidence && !manifest) throw cliError("case_registry_run_not_found", "case registry run artifacts were not found", { caseRepoRunDir, caseId, runId });
  return { caseRepo, caseId, runId, caseRepoRunDir, run: run ?? {}, evidence: evidence ?? {}, manifest: manifest ?? {}, agentTrace, agentMessages, finalResponseText, promptText };
}

async function resolveCaseId(caseRepo: string, runId: string) {
  const root = path.join(caseRepo, "runs");
  const entries = await readdir(root).catch(() => []);
  const matches: string[] = [];
  for (const entry of entries.filter((item) => !item.startsWith("."))) {
    if (await isDirectory(path.join(root, entry, runId))) matches.push(entry);
  }
  if (matches.length === 0) throw cliError("case_registry_run_not_found", "runId was not found in case registry runs", { caseRepo, runId });
  if (matches.length > 1) throw cliError("case_registry_run_ambiguous", "runId exists under multiple caseIds; pass --case-id", { caseRepo, runId, caseIds: matches });
  return matches[0];
}

function registryPreparedRun(caseRepo: string, caseId: string, runId: string, caseRepoRunDir: string, rawRun: any, rawEvidence: any, manifest: any, now: string): PreparedCaseRun {
  const run = rawRun ?? {};
  const evidence = rawEvidence ?? {};
  const subject = { ...(run.subject ?? {}), ...(evidence.subject ?? {}), ...(manifest?.subject ?? {}) };
  const agent = { ...(run.agent ?? {}), ...(evidence.agent ?? {}), ...(manifest?.agent ?? {}) };
  return {
    ...run,
    caseId,
    runId,
    runDir: caseRepoRunDir,
    caseRepo,
    caseDir: run.caseDir ?? "",
    caseFile: run.caseFile ?? "",
    sourceSpecPath: run.sourceSpecPath ?? path.join(caseRepoRunDir, ".hwlab", "hwpod-spec.yaml"),
    specPath: run.specPath ?? path.join(caseRepoRunDir, ".hwlab", "hwpod-spec.yaml"),
    apiUrl: run.apiUrl ?? evidence.apiUrl ?? "",
    compileOnly: run.compileOnly !== false,
    createdAt: run.createdAt ?? now,
    updatedAt: now,
    definition: run.definition ?? {},
    subject: { repoLocalPath: subject.repoLocalPath ?? "", commitId: subject.commitId ?? "", subdir: subject.subdir ?? "", worktreePath: subject.worktreePath ?? "", workspacePath: subject.workspacePath ?? subject.worktreePath ?? "", setup: subject.setup ?? {} },
    agent: agent.traceId ? agent : null,
    agentTrace: evidence.agentTrace ?? manifest?.trace ?? run.agentTrace,
    agentDiff: evidence.agentDiff ?? manifest?.diff ?? run.agentDiff,
  } as PreparedCaseRun;
}

function diagnosticFromArtifacts(artifacts: any) {
  const agent = { ...(artifacts.run.agent ?? {}), ...(artifacts.evidence.agent ?? {}), ...(artifacts.manifest.agent ?? {}) };
  const trace = { ...(artifacts.run.agentTrace ?? {}), ...(artifacts.evidence.agentTrace ?? {}), ...(artifacts.manifest.trace ?? {}) };
  const summary = boundedToolCallSummary(trace.toolCallSummary ?? agent.toolCallSummary ?? agentToolCallSummaryFromResult(artifacts.agentTrace?.body ?? artifacts.agentTrace) ?? null);
  const rows = traceRows(artifacts);
  const agentFinal = clean({ ...(artifacts.evidence.agentFinal ?? {}), ...(artifacts.manifest.agentFinal ?? {}) });
  const commands = summary?.items ?? rows.filter((row) => /commandExecution|tool_call|^tool:/iu.test(`${row.rowId ?? ""} ${row.header ?? ""}`)).map((row) => ({ rowId: row.rowId, seq: row.seq, command: row.command ?? row.body, status: row.status ?? row.tone }));
  return clean({
    caseId: artifacts.caseId,
    runId: artifacts.runId,
    caseRepo: artifacts.caseRepo,
    caseRepoRunDir: artifacts.caseRepoRunDir,
    status: artifacts.run.status ?? artifacts.evidence.status ?? "recorded",
    stage: artifacts.run.stage ?? artifacts.run._control?.stage,
    compileOnly: artifacts.evidence.compileOnly ?? artifacts.run.compileOnly,
    traceId: agent.traceId ?? trace.traceId,
    sessionId: agent.sessionId ?? trace.sessionId,
    conversationId: agent.conversationId ?? trace.conversationId,
    threadId: agent.threadId ?? trace.threadId,
    providerProfile: agent.providerProfile,
    requestedProviderProfile: agent.requestedProviderProfile,
    resolvedBackendProfile: agent.resolvedBackendProfile,
    provider: agent.provider,
    model: agent.model,
    backend: agent.backend,
    terminalStatus: trace.terminalStatus ?? agent.terminalStatus,
    agentResultStatus: agent.result?.status ?? trace.status,
    commandStatus: agent.commandStatus ?? summary?.commandStatus,
    agentRunStatus: agent.agentRunStatus ?? summary?.agentRunStatus,
    toolCallSummary: summary,
    sourceEventCount: trace.sourceEventCount ?? artifacts.agentMessages?.sourceEventCount,
    renderedRowCount: trace.renderedRowCount ?? artifacts.agentMessages?.renderedRowCount ?? rows.length,
    commandCount: trace.commandCount ?? summary?.count ?? commands.length,
    hwpodCommandCount: trace.hwpodCommandCount,
    hwpodBuildCommandCount: trace.hwpodBuildCommandCount,
    agentFinal,
    postValidation: artifacts.manifest.postValidation ?? artifacts.evidence.postValidation,
    refresh: artifacts.manifest.refresh ?? artifacts.evidence.refresh,
    finalResponse: { present: agentFinal.present === true, reason: agentFinal.reason, path: path.join(artifacts.caseRepoRunDir, "final-response.md"), textPreview: clipText(artifacts.finalResponseText, 800) },
    traceCommands: commands.slice(0, MAX_DIAGNOSTIC_ROWS),
    paths: { runJson: path.join(artifacts.caseRepoRunDir, "run.json"), evidenceJson: path.join(artifacts.caseRepoRunDir, "evidence.json"), agentTraceJson: path.join(artifacts.caseRepoRunDir, "agent-trace.json"), agentMessagesJson: path.join(artifacts.caseRepoRunDir, "agent-messages.json"), artifactManifestJson: path.join(artifacts.caseRepoRunDir, "artifact-manifest.json"), aggregateMd: path.join(artifacts.caseRepoRunDir, "aggregate.md") },
  });
}

function traceSummary(diagnostic: any) {
  return clean({
    caseId: diagnostic.caseId,
    runId: diagnostic.runId,
    caseRepo: diagnostic.caseRepo,
    caseRepoRunDir: diagnostic.caseRepoRunDir,
    traceId: diagnostic.traceId,
    providerProfile: diagnostic.providerProfile,
    requestedProviderProfile: diagnostic.requestedProviderProfile,
    resolvedBackendProfile: diagnostic.resolvedBackendProfile,
    provider: diagnostic.provider,
    model: diagnostic.model,
    backend: diagnostic.backend,
    terminalStatus: diagnostic.terminalStatus,
    agentResultStatus: diagnostic.agentResultStatus,
    commandStatus: diagnostic.commandStatus,
    agentRunStatus: diagnostic.agentRunStatus,
    toolCallSummary: diagnostic.toolCallSummary,
    sourceEventCount: diagnostic.sourceEventCount,
    renderedRowCount: diagnostic.renderedRowCount,
    commandCount: diagnostic.commandCount,
    hwpodCommandCount: diagnostic.hwpodCommandCount,
    hwpodBuildCommandCount: diagnostic.hwpodBuildCommandCount,
    traceCommands: diagnostic.traceCommands,
    paths: diagnostic.paths,
  });
}

function boundedToolCallSummary(summary: any) {
  if (!summary || typeof summary !== "object") return null;
  return clean({
    ...summary,
    items: arrayOfObjects(summary.items).slice(0, MAX_DIAGNOSTIC_ROWS).map((item) => clean({
      ...item,
      command: clipText(item.command, 1000),
      detail: clipText(item.detail, 1000),
    })),
  });
}

function traceRows(artifacts: any) {
  const body = artifacts.agentTrace?.body && typeof artifacts.agentTrace.body === "object" ? artifacts.agentTrace.body : artifacts.agentTrace ?? {};
  return arrayOfObjects(artifacts.agentMessages?.rows ?? body.rows ?? body.renderedRows ?? body.traceRows);
}

async function readJson(file: string) { try { return JSON.parse(await readFile(file, "utf8")); } catch { return null; } }
async function readText(file: string) { try { return await readFile(file, "utf8"); } catch { return ""; } }
async function isDirectory(file: string) { try { return (await stat(file)).isDirectory(); } catch { return false; } }
function escapeRegExp(value: string) { return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); }
function cliError(code: string, message: string, details: Record<string, unknown> = {}) { return Object.assign(new Error(message), { code, details }); }
function ok<T extends Record<string, unknown>>(action: string, extra: T) { return { ok: true, action, status: "completed", ...extra }; }