// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p8-web-probe-sentinel-recovery.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p9-multi-web-probe-sentinel.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p10-monitor-web-aggregation.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p12-cadence-scheduler-monitor-web.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-28-p13-1206-multi-runner-boundaries.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-30-p14-sentinel-cicd-visibility.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p16-cicd-source-snapshot.
// SPEC: PJ2026-01060703 CI/CD branch follower draft-2026-07-03-p0-branch-follower.
// Responsibility: YAML-first CI/CD, image, GitOps and Argo command plan for the web-probe sentinel.
import { createHash, randomUUID } from "node:crypto";
import { existsSync, readFileSync } from "node:fs";
import { join } from "node:path";
import { repoRoot, rootPath } from "./config";
import { runCommand, type CommandResult } from "./command";
import { runtimeReuseService, summarizeRuntimeReuseConfig, type RuntimeReuseConfig } from "./cicd-reuse-config";
import type { CicdDeliveryAuthority } from "./cicd-delivery-authority";
import { startJob } from "./jobs";
import { webProbeSentinelConfigPlan, withWebProbeSentinelConfigRendered } from "./hwlab-node-web-sentinel-config";
import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref";
import { effectiveWebProbeSentinelPublicExposure, requireSentinelIdForRegistry, resolveWebProbeSentinel, webProbeSentinelRegistryRows } from "./hwlab-node-web-sentinel-resolver";
import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
import {
  guardedSentinelDeliveryAction,
  publishCurrentNext,
  renderBlockedSentinelDeliveryMutation,
  sentinelReadOnlyNext,
  sentinelSourceOverrideFromOptions,
  sentinelSourceResolveMode,
  webProbeSentinelDeliveryAuthority,
  type SentinelSourceOverride,
} from "./hwlab-node/sentinel-delivery-authority";
import type { RenderedCliResult } from "./output";
import { probeSentinelRuntimeHealthEndpoint, runSentinelDashboard, runSentinelErrorDetail, runSentinelInspect, runSentinelMaintenance, runSentinelReport, runSentinelValidate } from "./hwlab-node-web-sentinel-p5";
import { runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe";
import { emitWebProbeSentinelSpan, webProbeSentinelOtelSummary } from "./hwlab-node-web-sentinel-otel";
import { exportMonitorSqliteSnapshot, planMonitorSqliteSources, verifyMonitorSqliteSnapshot, type MonitorSqliteSource } from "./monitor-sqlite-migration";
import { releaseAMonitorManifests, releaseAMonitorMigrationArtifactGaps, releaseAMonitorMigrationJob, releaseAMonitorMigrationPool } from "./hwlab-node-web-probe-monitor";
import {
  arrayAt,
  arrayAtNullable,
  booleanAt,
  booleanAtNullable,
  compactCommand,
  configRefFile,
  displayPath,
  finiteNumberOrNull,
  isRecord,
  manifestObjectSummary,
  monitorWebCicdPlan,
  nonEmptyString,
  numberAt,
  numberAtNullable,
  parseEnvFile,
  parseJsonObject,
  readConfigFile,
  record,
  recordTarget,
  rendered,
  renderAsyncJobResult,
  renderControlPlaneResult,
  renderImageResult,
  renderPublishCurrentResult,
  renderPublishResult,
  resolveSentinelChildJson,
  safeJobSegment,
  safeKubernetesSegment,
  secretSourcePaths,
  sentinelCliSuffix,
  sentinelPipelineRunName,
  sentinelProgressEvent,
  sentinelSourceSnapshotRef,
  sentinelSourceSnapshotStageRefPrefix,
  sha256,
  shellQuote,
  short,
  stringAt,
  stringAtNullable,
  stringField,
  stringTarget,
  table,
  text,
  tomlEscape,
  valueAtPath,
  type ChildCliResult,
  type CompactCommandResult,
  type SentinelCicdState,
  type SentinelImagePlan,
  type SentinelObservedExpectation,
  type SentinelObservedStatus,
  type SentinelRemoteJobResult,
  type SourceHead,
  type WebProbeSentinelDashboardAction,
  type WebProbeSentinelOptions,
  type WebProbeSentinelReportView,
  type WebProbeSentinelSourceAuthority,
} from "./hwlab-node-web-sentinel-cicd-shared";
import {
  applySentinelArgoApplication,
  controlPlaneWaitWarningSeconds,
  createK8sJobScript,
  probeK8sJobScript,
  publishSatisfiedByObservedWarnings,
  runSentinelPublishJob,
  runSentinelSourceMirrorSyncJob,
  sentinelBlockedRemoteResult,
  sentinelCicdElapsedWarnings,
  sentinelGitMirrorCacheVolumeFromTarget,
  sentinelPayloadFromLogs,
  sentinelRemoteJobTimeoutWarnings,
  sentinelSourceMirrorSyncShellFromConfig,
  sentinelSourceMirrorAlreadyPresentResult,
  sourceMirrorAlreadyReadyWarnings,
} from "./hwlab-node-web-sentinel-cicd-jobs";

export type {
  ChildCliResult,
  CompactCommandResult,
  SentinelCicdState,
  WebProbeSentinelDashboardAction,
  WebProbeSentinelOptions,
  WebProbeSentinelReportView,
  WebProbeSentinelSourceAuthority,
  WebProbeSentinelSourceOverrideOptions,
} from "./hwlab-node-web-sentinel-cicd-shared";
export {
  arrayAt,
  compactCommand,
  displayPath,
  isRecord,
  nonEmptyString,
  numberAt,
  numberAtNullable,
  parseEnvFile,
  parseJsonObject,
  record,
  recordTarget,
  rendered,
  renderAsyncJobResult,
  safeJobSegment,
  secretSourcePaths,
  sentinelCliSuffix,
  shellQuote,
  short,
  stringAt,
  stringAtNullable,
  table,
  text,
};
export {
  guardedSentinelDeliveryAction,
  webProbeSentinelDeliveryAuthority,
  webProbeSentinelDeliveryAuthorityFromCatalog,
  webProbeSentinelDeliveryMutationDecision,
} from "./hwlab-node/sentinel-delivery-authority";
export type { SentinelSourceOverride } from "./hwlab-node/sentinel-delivery-authority";


const SPEC_REF = "PJ2026-01060508 Web哨兵 draft-2026-07-01-p16-cicd-source-snapshot";

export type SourceResolveMode = "cached" | "sync";

export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options: WebProbeSentinelOptions): RenderedCliResult {
  if (options.kind === "migration") return runSentinelMigration(spec, options);
  if (options.kind === "config") return withWebProbeSentinelConfigRendered(webProbeSentinelConfigPlan(spec, options.action, options.sentinelId));
  const guardedAction = guardedSentinelDeliveryAction(options);
  if (guardedAction !== null) {
    const blocked = renderBlockedSentinelDeliveryMutation(spec, options, guardedAction.command, guardedAction.action);
    if (blocked !== null) return blocked;
  }
  requireSentinelIdForRegistry(spec, options.sentinelId, `web-probe sentinel ${options.kind}`);
  const localCommit = options.kind === "validate" && options.localOnly
    ? runCommand(["git", "rev-parse", "HEAD"], repoRoot, { timeoutMs: 5_000 }).stdout.trim()
    : null;
  const localSourceOverride = localCommit === null ? null : {
    commit: localCommit,
    stageRef: null,
    mirrorCommit: localCommit,
    sourceAuthority: "gitea-snapshot" as const,
  };
  const state = loadSentinelCicdState(spec, options.sentinelId, options.timeoutSeconds, sentinelSourceResolveMode(options), localSourceOverride ?? sentinelSourceOverrideFromOptions(options));
  if (options.kind === "image") return runSentinelImage(state, options);
  if (options.kind === "control-plane") return runSentinelControlPlane(state, options);
  if (options.kind === "publish") return runSentinelPublishCurrent(state, options);
  if (options.kind === "maintenance") return runSentinelMaintenance(state, options);
  if (options.kind === "validate") return runSentinelValidate(state, options);
  if (options.kind === "inspect") return runSentinelInspect(state, options);
  if (options.kind === "dashboard") return runSentinelDashboard(state, options);
  if (options.kind === "error") return runSentinelErrorDetail(state, options);
  return runSentinelReport(state, options);
}

function runSentinelMigration(spec: HwlabRuntimeLaneSpec, options: Extract<WebProbeSentinelOptions, { kind: "migration" }>): RenderedCliResult {
  const sources = resolvedMigrationSources(spec, options.sentinelId).map((source) => ({ ...source, legacyArtifactGaps: releaseAMonitorMigrationArtifactGaps(spec, source) }));
  const command = `web-probe sentinel migration ${options.action}`;
  if (options.action === "job-plan" || options.action === "job-run") return runSentinelMigrationJob(spec, options, sources, command);
  if (options.action === "plan") {
    const projection = planMonitorSqliteSources(sources);
    return rendered(true, command, renderMigrationCompact(projection), projection);
  }
  if (options.action === "export") {
    if (sources.length !== 1) throw new Error("web-probe sentinel migration export requires --sentinel when more than one resolved registry source exists");
    if (options.snapshotPath === null) throw new Error("web-probe sentinel migration export requires --snapshot");
    const snapshot = exportMonitorSqliteSnapshot(sources[0]!, options.snapshotPath);
    const verified = verifyMonitorSqliteSnapshot(snapshot);
    const projection = { ...verified, command, snapshot: { fingerprint: snapshot.fingerprint, integrity: snapshot.integrity, createdAt: snapshot.createdAt }, valuesRedacted: true };
    return rendered(verified.ok === true, command, renderMigrationCompact(projection), projection);
  }
  if (options.snapshotPath === null) throw new Error(`web-probe sentinel migration ${options.action} requires --snapshot`);
  const snapshot = JSON.parse(readFileSync(options.snapshotPath, "utf8")) as Parameters<typeof verifyMonitorSqliteSnapshot>[0];
  const source = sources.find((item) => sameMigrationSource(item, snapshot.source));
  if (source === undefined) throw new Error("migration snapshot source is not in the current resolved registry");
  const verification = verifyMonitorSqliteSnapshot(snapshot);
  if (verification.ok !== true) throw new Error("migration snapshot verification failed");
  if (options.action === "import") {
    if (!options.confirm) throw new Error("web-probe sentinel migration import requires --confirm");
    if (!process.env.DATABASE_URL) throw new Error("monitor migration DATABASE_URL must be injected by the controlled Secret consumer");
    const pool = releaseAMonitorMigrationPool(spec, source);
    const result = runCommand(["bun", "scripts/monitor-sqlite-migration-import.ts", options.snapshotPath], repoRoot, { timeoutMs: 120_000, env: { ...process.env, MONITOR_CENTRAL_PG_POOL_MAX: String(pool.poolMax), MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS: String(pool.idleTimeoutSeconds) } });
    const projection = parseJsonObject(result.stdout) ?? { ok: false, command, error: "monitor-migration-import-unparseable", stderr: short(result.stderr), valuesRedacted: true };
    return rendered(projection.ok === true, command, renderMigrationCompact(projection), projection);
  }
  if (!process.env.DATABASE_URL) throw new Error("monitor migration verify requires DATABASE_URL projected from hwlab-web-probe-monitor-db/DATABASE_URL");
  const pool = releaseAMonitorMigrationPool(spec, source);
  const result = runCommand(["bun", "scripts/monitor-sqlite-migration-verify.ts", options.snapshotPath], repoRoot, { timeoutMs: 120_000, env: { ...process.env, MONITOR_CENTRAL_PG_POOL_MAX: String(pool.poolMax), MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS: String(pool.idleTimeoutSeconds) } });
  const projection = parseJsonObject(result.stdout) ?? { ok: false, command, error: "monitor-migration-verify-unparseable", stderr: short(result.stderr), valuesRedacted: true };
  return rendered(projection.ok === true, command, renderMigrationCompact(projection), projection);
}

function runSentinelMigrationJob(spec: HwlabRuntimeLaneSpec, options: Extract<WebProbeSentinelOptions, { kind: "migration" }>, sources: readonly MonitorSqliteSource[], command: string): RenderedCliResult {
  if (sources.length !== 1) throw new Error(`web-probe sentinel migration ${options.action} requires --sentinel when more than one resolved registry source exists`);
  const state = loadSentinelCicdState(spec, sources[0]!.sentinelId, options.timeoutSeconds, "cached");
  const manifest = releaseAMonitorMigrationJob(spec, state.image.ref, sources[0]!, { enabled: true, nameSuffix: Date.now().toString(36) });
  if (manifest === null) throw new Error("Release A migration Job helper did not render an enabled Job");
  const metadata = manifest.metadata as Record<string, unknown>;
  const namespace = String(metadata.namespace);
  const jobName = String(metadata.name);
  const plan = { ok: true, command, mutation: false, namespace, jobName, image: state.image.ref, source: sources[0], manifest: { apiVersion: manifest.apiVersion, kind: manifest.kind, metadata: { name: jobName, namespace } }, valuesRedacted: true };
  if (options.action === "job-plan") return rendered(true, command, renderMigrationCompact(plan), plan);
  if (!options.confirm || !options.wait) throw new Error("web-probe sentinel migration job-run requires --confirm --wait");
  const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(options.timeoutSeconds, 60) * 1000 });
  if (created.exitCode !== 0) return migrationJobFailed(command, plan, "create-failed", {}, { podPhase: "CreateFailed", logsTail: "migration Job creation failed" });
  const startedAt = Date.now();
  let lastProbe = created;
  while (Date.now() - startedAt < Math.max(5_000, Math.min(options.timeoutSeconds * 1000, 120_000))) {
    lastProbe = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(options.timeoutSeconds, 60) * 1000 });
    const probe = resolveSentinelChildJson(lastProbe, "web-probe-sentinel-migration-job-probe").parsed ?? {};
    if (probe.succeeded === true || probe.failed === true) {
      const payload = migrationJobPayloadFromProbe(probe);
      if (probe.succeeded === true && payload.ok === true) return rendered(true, command, "ok=true", { ...plan, ok: true, mutation: true, phase: "succeeded", result: payload, valuesRedacted: true });
      return migrationJobFailed(command, plan, probe.succeeded === true ? "result-failed" : "failed", payload, probe, migrationJobFailureNext(state, namespace, jobName));
    }
    runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
  }
  return migrationJobFailed(command, plan, "timeout", {}, { podPhase: "Timeout", logsTail: "migration Job did not reach a successful result before timeout" }, migrationJobFailureNext(state, namespace, jobName));
}

export function migrationJobPayloadFromProbe(probe: Record<string, unknown>): Record<string, unknown> {
  const failedContainer = migrationFailedContainerName(probe);
  const container = probe.succeeded === true ? "verify" : failedContainer;
  const expectedCommand = container === "export" ? "web-probe sentinel migration export" : container === "import" ? "web-probe sentinel migration import" : container === "verify" ? "web-probe sentinel migration verify" : null;
  const containerLogs = isRecord(probe.containerLogs) ? probe.containerLogs : {};
  const payload = container === null ? {} : sentinelPayloadFromLogs(String(containerLogs[container] ?? ""));
  if (expectedCommand !== null && migrationPayloadCommandMatches(payload.command, expectedCommand) && (payload.ok === true || payload.ok === false)) return payload;
  return { ok: false, error: { code: "monitor-migration-job-result-missing", message: expectedCommand === null ? "migration Job failed without an identifiable terminated container result" : `migration Job ${container} container did not emit the expected structured result` }, valuesRedacted: true };
}

function migrationPayloadCommandMatches(value: unknown, expected: string): boolean {
  return typeof value === "string" && (value === expected || value.startsWith(`${expected} `));
}

function migrationFailedContainerName(probe: Record<string, unknown>): string | null {
  const containers = Array.isArray(probe.containers) ? probe.containers.filter(isRecord) : [];
  return stringAtNullable(containers.find((container) => finiteNumberOrNull(container.exitCode) !== null && finiteNumberOrNull(container.exitCode) !== 0), "name");
}

export function migrationJobFailed(command: string, plan: Record<string, unknown>, phase: string, payload: Record<string, unknown>, probe: Record<string, unknown>, next?: Record<string, string>): RenderedCliResult {
  return rendered(false, command, "ok=false", { ...plan, ok: false, mutation: true, phase, failure: migrationJobFailureSummary(probe, payload), next, valuesRedacted: true });
}

export function migrationJobFailureSummary(probe: Record<string, unknown>, payload: Record<string, unknown>): Record<string, unknown> {
  const containers = Array.isArray(probe.containers) ? probe.containers.filter(isRecord).map((item) => ({ name: stringAtNullable(item, "name"), phase: stringAtNullable(item, "phase"), exitCode: finiteNumberOrNull(item.exitCode), reason: stringAtNullable(item, "reason"), message: redactedMigrationJobText(stringAtNullable(item, "message") ?? "") })).filter((item) => item.phase === "terminated" || item.exitCode !== null || item.reason !== null) : [];
  const failed = containers.find((item) => item.exitCode !== null && item.exitCode !== 0) ?? containers[0] ?? null;
  const logsTail = redactedMigrationJobText(String(probe.logsTail ?? ""));
  const nestedError = isRecord(payload.error) ? payload.error : null;
  return { phase: stringAtNullable(probe, "podPhase") ?? "Failed", container: failed?.name ?? null, exitCode: failed?.exitCode ?? null, code: stringAtNullable(nestedError, "code") ?? null, status: failed?.reason ?? stringAtNullable(nestedError, "code") ?? stringAtNullable(probe, "reason") ?? "JobFailed", message: redactedMigrationJobText(stringAtNullable(nestedError, "message") ?? stringAtNullable(payload, "message") ?? failed?.message ?? "migration Job failed before a structured command result was emitted"), logsTail, valuesRedacted: true };
}

function redactedMigrationJobText(value: string): string {
  return short(redactMigrationJobLogTail(value));
}

function redactMigrationJobLogTail(value: string): string {
  return value
    .replace(/\b[a-z][a-z0-9+.-]*:\/\/[^\s"']*@[^\s"']+/giu, "<redacted-url>")
    .replace(/\bBearer\s+[^\s"']+/giu, "Bearer <redacted>")
    .replace(/(["']?(?:DATABASE_URL|password|token|secret)["']?\s*[=:]\s*["']?)[^\s,"'}\]]+/giu, "$1<redacted>")
    .replace(/([?&](?:password|token|secret)=[^&\s"']+)/giu, "<redacted-query>");
}

function migrationJobFailureNext(state: SentinelCicdState, namespace: string, jobName: string): Record<string, string> {
  return { logs: `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs job/${jobName} --all-containers=true --tail=120`, describe: `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe job/${jobName}` };
}

export function sameMigrationSource(left: MonitorSqliteSource, right: MonitorSqliteSource): boolean {
  return left.sentinelId === right.sentinelId && left.node === right.node && left.lane === right.lane && left.path === right.path && left.runtimeConfigRef === right.runtimeConfigRef && left.stateRoot === right.stateRoot && left.ownerPvc === right.ownerPvc && left.artifactSources.length === right.artifactSources.length && left.artifactSources.every((source, index) => source.ownerPvc === right.artifactSources[index]?.ownerPvc && source.mountPath === right.artifactSources[index]?.mountPath && source.hostPath === right.artifactSources[index]?.hostPath) && monitorGapSet(left.legacyArtifactGaps) === monitorGapSet(right.legacyArtifactGaps);
}

function monitorGapSet(gaps: readonly { readonly runId: string; readonly artifactCount: number; readonly stateDir: string; readonly reportSha256: string }[] | undefined): string {
  return [...(gaps ?? [])].map((gap) => [gap.runId, gap.artifactCount, gap.stateDir, gap.reportSha256] as const).sort((left, right) => left[0].localeCompare(right[0])).map((gap) => JSON.stringify(gap)).join("\n");
}

function resolvedMigrationSources(spec: HwlabRuntimeLaneSpec, sentinelId: string | null): readonly MonitorSqliteSource[] {
  const ids = sentinelId === null ? webProbeSentinelRegistryRows(spec).filter((item) => item.enabled).map((item) => item.id) : [sentinelId];
  return ids.map((id) => {
    const sentinel = resolveWebProbeSentinel(spec, id);
    const runtime = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.runtime), sentinel.configRefs.runtime);
    const stateRoot = stringAt(runtime, "stateRoot");
    const ownerPvc = stringAt(runtime, "pvcName");
    const artifactSources = arrayAt(runtime, "artifactSources").map((item, index) => {
      const source = recordTarget(item, `${sentinel.configRefs.runtime}.artifactSources[${index}]`);
      const hostPath = stringAtNullable(source, "hostPath");
      return { ownerPvc: stringAt(source, "ownerPvc"), mountPath: stringAt(source, "mountPath"), ...(hostPath === null ? {} : { hostPath }) };
    });
    if (!artifactSources.some((item) => item.ownerPvc === ownerPvc && item.mountPath === stateRoot)) throw new Error(`${sentinel.configRefs.runtime}.artifactSources must include the resolved index PVC and stateRoot`);
    return { sentinelId: sentinel.id, node: spec.nodeId, lane: spec.lane, path: stringAt(runtime, "sqlite.path"), runtimeConfigRef: sentinel.configRefs.runtime, stateRoot, ownerPvc, artifactSources };
  });
}

function renderMigrationCompact(value: Record<string, unknown>): string {
  const reconciliation = record(value.reconciliation);
  return [`ok=${value.ok === true}`, `command=${String(value.command ?? "web-probe sentinel migration")}`, `runs=${String(reconciliation.runs ?? "-")}`, `findings=${String(reconciliation.findings ?? "-")}`, `metadata=${String(reconciliation.metadata ?? "-")}`, `payloads=${String(reconciliation.payloads ?? "-")}`, `locators=${String(reconciliation.locators ?? "-")}`, `latest=${String(reconciliation.latest ?? "-")}`].join(" ");
}

function runSentinelImage(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "image" }>): RenderedCliResult {
  const command = `web-probe sentinel image ${options.action}`;
  const deliveryAuthority = webProbeSentinelDeliveryAuthority(state.spec);
  if (options.action === "build" && options.confirm) {
    if (!options.wait) return renderAsyncSentinelJob(state, "image", "build", options.timeoutSeconds);
    return runSentinelImageBuildConfirmed(state, options);
  }
  const sourceMirror = options.action === "status" ? probeSourceMirror(state, options.timeoutSeconds) : null;
  const registry = options.action === "status" ? probeImageRegistry(state, options.timeoutSeconds) : null;
  const sourceMirrorReady = options.action !== "status" || record(sourceMirror).ok === true;
  const registryReady = options.action !== "status" || record(registry?.probe).present === true;
  const result = {
    ok: state.configReady && state.sourceHead.ok && sourceMirrorReady && registryReady,
    command,
    node: state.spec.nodeId,
    lane: state.spec.lane,
    sentinelId: state.sentinelId,
    mode: options.action === "status" ? "status" : options.confirm ? "confirm" : "dry-run",
    mutation: false,
    specRef: SPEC_REF,
    source: state.sourceHead,
    sourceMirror,
    image: state.image,
    registry,
    blocker: sourceMirrorReady
      ? registryReady ? null : { code: "sentinel-image-missing", reason: "expected sentinel image tag is not present in the node-local registry" }
      : { code: "sentinel-source-mirror-not-ready", reason: "source.gitMirrorReadUrl does not expose the selected source commit yet" },
    deliveryAuthority,
    next: deliveryAuthority.kind === "legacy-manual"
      ? {
        status: `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`,
        dryRun: `bun scripts/cli.ts web-probe sentinel image build --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --dry-run`,
        confirm: `bun scripts/cli.ts web-probe sentinel image build --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm`,
        controlPlanePlan: `bun scripts/cli.ts web-probe sentinel control-plane plan --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --dry-run`,
      }
      : sentinelReadOnlyNext(state, deliveryAuthority),
    valuesRedacted: true,
  };
  return rendered(result.ok, command, renderImageResult(result));
}

function runSentinelControlPlane(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "control-plane" }>): RenderedCliResult {
  const command = `web-probe sentinel control-plane ${options.action}`;
  const deliveryAuthority = webProbeSentinelDeliveryAuthority(state.spec);
  const mutationAction = options.action === "apply" || options.action === "trigger-current";
  if (options.confirm && mutationAction) {
    if (!options.wait) return renderAsyncSentinelJob(state, "control-plane", options.action, options.timeoutSeconds);
    return runSentinelControlPlaneConfirmed(state, options);
  }
  const observed = options.action === "status"
    ? options.wait
      ? waitForSentinelObservedStatus(state, options.timeoutSeconds)
      : collectSentinelObservedStatus(state, options.timeoutSeconds)
    : null;
  const observedReady = options.action !== "status" || sentinelObservedReady(record(observed));
  const observedWarnings = options.action === "status" ? sentinelObservedWarnings(record(observed)) : [];
  const pipelineRun = sentinelPipelineRunName(state, options.rerun);
  const statusDiagnosis = options.action === "status" && !observedReady ? sentinelObservedStatusDiagnosis(state, observed, pipelineRun) : null;
  const result = {
    ok: state.configReady && state.sourceHead.ok && (options.action === "status" || observedReady),
    command,
    node: state.spec.nodeId,
    lane: state.spec.lane,
    sentinelId: state.sentinelId,
    mode: options.action === "status" ? "status" : options.confirm ? "confirm" : "dry-run",
    mutation: false,
    deliveryAuthority,
    specRef: SPEC_REF,
    source: state.sourceHead,
    image: state.image,
    pipelineRun,
    gitops: {
      path: stringField(state.cicd, "gitopsPath"),
      targetRevision: stringAt(state.cicd, "argo.targetRevision"),
      manifestObjects: state.manifests.length,
      manifestSha256: state.manifestSha256,
    },
    argo: {
      namespace: stringAt(state.cicd, "argo.namespace"),
      projectName: stringAt(state.cicd, "argo.projectName"),
      applicationName: stringAt(state.cicd, "argo.applicationName"),
    },
    maintenance: {
      startCommand: stringAt(state.cicd, "maintenance.startCommand"),
      stopCommand: stringAt(state.cicd, "maintenance.stopCommand"),
      serviceUnavailablePolicy: stringAt(state.cicd, "targetValidation.serviceUnavailablePolicy"),
    },
    validation: {
      scenarioId: stringAt(state.cicd, "targetValidation.scenarioId"),
      maxSeconds: numberAt(state.cicd, "targetValidation.maxSeconds"),
      controlPlaneWaitMaxSeconds: controlPlaneWaitWarningSeconds(state),
      quickVerifyMode: "manual-validate",
      automaticSecondPath: false,
    },
    manifests: {
      objects: manifestObjectSummary(state.manifests),
      sha256: state.manifestSha256,
    },
    observability: webProbeSentinelOtelSummary({
      node: state.spec.nodeId,
      lane: state.spec.lane,
      sentinelId: state.sentinelId,
      namespace: stringAt(state.runtime, "namespace"),
      runtime: state.runtime,
      cicd: state.cicd,
    }),
    observed,
    deliveryStatus: options.action === "status" ? sentinelDeliveryStatus(state, record(observed), pipelineRun) : null,
    statusDiagnosis,
    drillDown: controlPlaneDrillDown(state, pipelineRun, null),
    warnings: mergeWarnings(observedWarnings, record(statusDiagnosis).warning),
    blocker: options.action === "status" || observedReady
      ? null
      : record(statusDiagnosis).blocker ?? { code: "sentinel-control-plane-observed-not-ready", reason: "one or more source, registry, GitOps, Argo, runtime or cadence checks did not pass", valuesRedacted: true },
    recoveryNext: record(statusDiagnosis).recoveryNext ?? null,
    next: controlPlaneNext(state, options.action),
    valuesRedacted: true,
  };
  return rendered(result.ok, command, renderControlPlaneResult(result), options.action === "status" ? record(result.deliveryStatus) : undefined);
}

function runSentinelPublishCurrent(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "publish" }>): RenderedCliResult {
  const command = "web-probe sentinel publish-current";
  if (options.confirm) {
    if (!options.wait) return renderAsyncSentinelJob(state, "publish", options.action, options.timeoutSeconds);
    return runSentinelPublishCurrentConfirmed(state, options);
  }
  const result = {
    ok: state.configReady && state.sourceHead.ok,
    command,
    node: state.spec.nodeId,
    lane: state.spec.lane,
    sentinelId: state.sentinelId,
    mode: "dry-run",
    mutation: false,
    specRef: SPEC_REF,
    source: state.sourceHead,
    image: state.image,
    pipelineRun: sentinelPipelineRunName(state, options.rerun),
    gitops: {
      path: stringAt(state.cicd, "gitopsPath"),
      targetRevision: stringAt(state.cicd, "argo.targetRevision"),
      manifestSha256: state.manifestSha256,
    },
    argo: {
      namespace: stringAt(state.cicd, "argo.namespace"),
      applicationName: stringAt(state.cicd, "argo.applicationName"),
    },
    budget: publishCurrentBudget(state),
    validationPlan: publishCurrentHealthValidationPlan(state),
    stageBudgets: publishCurrentStageBudgets(state),
    drillDown: controlPlaneDrillDown(state, sentinelPipelineRunName(state, options.rerun), null),
    blocker: state.configReady && state.sourceHead.ok ? null : { code: "sentinel-publish-current-plan-blocked", reason: "sentinel config or source head is not ready" },
    next: publishCurrentNext(state),
    valuesRedacted: true,
  };
  return rendered(result.ok === true, command, renderPublishCurrentResult(result));
}

function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "publish" }>): RenderedCliResult {
  const interruptContext: Record<string, unknown> = {
    phase: "starting",
    pipelineRun: sentinelPipelineRunName(state, options.rerun),
    sourceCommit: state.sourceHead.commit,
    node: state.spec.nodeId,
    lane: state.spec.lane,
    sentinelId: state.sentinelId,
    valuesRedacted: true,
  };
  const uninstallInterruptHandler = installSentinelPublishInterruptHandler(state, interruptContext);
  try {
    return runSentinelPublishCurrentConfirmedInner(state, options, interruptContext);
  } finally {
    uninstallInterruptHandler();
  }
}

function runSentinelPublishCurrentConfirmedInner(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "publish" }>, interruptContext: Record<string, unknown>): RenderedCliResult {
  const startedAt = Date.now();
  const command = "web-probe sentinel publish-current";
  const budget = publishCurrentBudget(state);
  const budgetSeconds = Math.min(options.timeoutSeconds, numberAt(budget, "maxSeconds"));
  const deadline = startedAt + budgetSeconds * 1000;
  const remainingBudgetSeconds = () => strictRemainingSeconds(deadline, budgetSeconds);
  let controlResult: Record<string, unknown> | null = null;
  const inCluster = Boolean(process.env.KUBERNETES_SERVICE_HOST && process.env.KUBERNETES_SERVICE_PORT);
  if (!inCluster && !options.manualRecovery) {
    const next = publishCurrentNext(state);
    const result = {
      ok: false,
      command,
      node: state.spec.nodeId,
      lane: state.spec.lane,
      sentinelId: state.sentinelId,
      mode: "debug-step-only-blocked",
      mutation: false,
      deliverySource: "auto-pac-required",
      specRef: SPEC_REF,
      source: state.sourceHead,
      image: state.image,
      pipelineRun: sentinelPipelineRunName(state, options.rerun),
      budget,
      validationPlan: publishCurrentHealthValidationPlan(state),
      stageBudgets: publishCurrentStageBudgets(state),
      warnings: [
        "operator-side publish-current is debug/test/recovery only; formal delivery must use platform-infra pipelines-as-code closeout/status/history.",
      ],
      blocker: {
        code: "sentinel-publish-current-debug-step-only",
        reason: "this operator-side command no longer performs formal Web sentinel delivery; use PaC closeout for automatic branch following",
        valuesRedacted: true,
      },
      recoveryNext: {
        reason: "manual recovery is only allowed after PaC is terminally failed, half-applied, or explicitly over budget",
        pipelineRun: null,
        digestRef: null,
        gitopsCommit: null,
        pacCloseout: next.pacCloseout,
        publishCurrent: next.manualRecovery,
        nextStatus: next.pacStatus,
        gitMirrorStatus: next.gitMirrorStatus,
        gitMirrorSync: null,
        gitMirrorFlush: null,
        controlPlaneApply: null,
        valuesRedacted: true,
      },
      next,
      valuesRedacted: true,
    };
    return rendered(false, command, renderPublishCurrentResult(result));
  }
  sentinelProgressEvent("sentinel.publish.progress", {
    phase: "publish-current-start",
    status: "running",
    pipelineRun: interruptContext.pipelineRun,
    sourceCommit: state.sourceHead.commit,
    node: state.spec.nodeId,
    lane: state.spec.lane,
    sentinelId: state.sentinelId,
  });
  if (inCluster) {
    const publish = runSentinelPublishJob(state, true, Math.max(1, remainingBudgetSeconds()), options.rerun);
    const elapsedMs = Date.now() - startedAt;
    const ok = record(publish).ok === true;
    const publishFirstBreak = record(record(publish).firstBreak);
    const publishError = record(record(publish).error);
    if (!ok) {
      sentinelProgressEvent("sentinel.publish.failure", {
        phase: publishFirstBreak.phase ?? record(publish).phase ?? "sentinel-publish",
        status: "failed",
        traceId: record(publish).traceId ?? null,
        firstBreak: Object.keys(publishFirstBreak).length === 0 ? null : publishFirstBreak,
        error: Object.keys(publishError).length === 0 ? null : publishError,
        pipelineRun: record(publish).jobName ?? sentinelPipelineRunName(state, options.rerun),
        sourceCommit: state.sourceHead.commit,
        node: state.spec.nodeId,
        lane: state.spec.lane,
      });
    }
    const result = {
      ok,
      command,
      node: state.spec.nodeId,
      lane: state.spec.lane,
      sentinelId: state.sentinelId,
      mode: "pac-publish-only",
      mutation: true,
      specRef: SPEC_REF,
      source: state.sourceHead,
      image: state.image,
      pipelineRun: record(publish).jobName ?? sentinelPipelineRunName(state, options.rerun),
      traceId: record(publish).traceId ?? null,
      firstBreak: Object.keys(publishFirstBreak).length === 0 ? null : publishFirstBreak,
      error: Object.keys(publishError).length === 0 ? null : publishError,
      controlPlane: {
        ok,
        mode: "pac-publish-only",
        mutation: true,
        source: state.sourceHead,
        publish,
        argo: {
          namespace: stringAt(state.cicd, "argo.namespace"),
          applicationName: stringAt(state.cicd, "argo.applicationName"),
        },
        valuesRedacted: true,
      },
      health: { ok: false, skipped: true, reason: "pac-publish-only-runtime-closeout-read-by-platform-infra-status", valuesRedacted: true },
      elapsedMs,
      timings: publishCurrentStageTimings({ publish }, null, elapsedMs),
      slowStages: publishCurrentSlowStages(state, publishCurrentStageTimings({ publish }, null, elapsedMs), budgetSeconds),
      withinBudget: elapsedMs <= budgetSeconds * 1000,
      drillDown: controlPlaneDrillDown(state, record(publish).jobName ?? sentinelPipelineRunName(state, options.rerun), null),
      warnings: [
        "PaC in-cluster publish-current submits the native sentinel publish PipelineRun and leaves Argo/runtime closeout to platform-infra pipelines-as-code status.",
        ...sentinelCicdElapsedWarnings(record(publish).elapsedMs, "sentinel publish", controlPlaneWaitWarningSeconds(state)),
        ...sentinelRemoteJobTimeoutWarnings(publish, "sentinel publish"),
      ],
      blocker: ok ? null : {
        code: publishFirstBreak.code ?? publishError.code ?? "sentinel-pac-publish-not-ready",
        reason: publishFirstBreak.reason ?? publishError.message ?? "sentinel publish PipelineRun did not complete successfully",
      },
      recoveryNext: controlPlaneRecoveryNext(state, ok, publish, null, false),
      next: publishCurrentNext(state),
      valuesRedacted: true,
    };
    return rendered(ok, command, renderPublishCurrentResult(result));
  }
  if (state.configReady && state.sourceHead.ok && remainingBudgetSeconds() >= 5) {
    interruptContext.phase = "already-current-registry-probe";
    const registryProbe = probeImageRegistry(state, Math.max(1, Math.min(remainingBudgetSeconds(), 5)));
    if (record(record(registryProbe).probe).present === true && remainingBudgetSeconds() >= 5) {
      const preflightStartedAt = Date.now();
      const preflightTimeoutSeconds = Math.max(1, Math.min(remainingBudgetSeconds(), 10));
      interruptContext.phase = "already-current-observed-probe";
      const preflightObserved = withObservedWait(
        collectSentinelObservedStatus(state, preflightTimeoutSeconds, undefined, true),
        preflightStartedAt,
        preflightTimeoutSeconds,
        true,
      );
      if (sentinelObservedReady(preflightObserved)) {
        controlResult = sentinelAlreadyCurrentControlResult(state, preflightObserved, Date.now() - preflightStartedAt);
      }
    }
  }
  interruptContext.phase = "control-plane-trigger-current";
  controlResult ??= sentinelControlPlaneConfirmedResult(state, {
    kind: "control-plane",
    action: "trigger-current",
    node: options.node,
    lane: options.lane,
    sentinelId: options.sentinelId,
    dryRun: false,
    confirm: true,
    wait: true,
    timeoutSeconds: Math.max(1, remainingBudgetSeconds()),
    rerun: options.rerun,
    sourceOverride: options.sourceOverride,
  });
  let health: Record<string, unknown>;
  let healthElapsedMs: number | null = null;
  if (controlResult.ok !== true) {
    health = { ok: false, skipped: true, reason: "control-plane-blocked", valuesRedacted: true };
  } else if (remainingBudgetSeconds() < 2) {
    health = { ok: false, skipped: true, reason: "end-to-end-budget-exhausted-before-health", valuesRedacted: true };
  } else {
    interruptContext.phase = "health-endpoint-validation";
    const healthStartedAt = Date.now();
    health = probeSentinelRuntimeHealthEndpoint(state, remainingBudgetSeconds());
    healthElapsedMs = Date.now() - healthStartedAt;
    health = { ...health, elapsedMs: healthElapsedMs, valuesRedacted: true };
  }
  const elapsedMs = Date.now() - startedAt;
  const timings = publishCurrentStageTimings(controlResult, health, elapsedMs);
  const slowStages = publishCurrentSlowStages(state, timings, budgetSeconds);
  const withinBudget = elapsedMs <= budgetSeconds * 1000;
  const healthOk = health.ok === true;
  const ok = controlResult.ok === true && healthOk && withinBudget;
  const blocker = ok ? null : publishCurrentBlocker(controlResult, health, withinBudget);
  const result = {
    ok,
    command,
    node: state.spec.nodeId,
    lane: state.spec.lane,
    sentinelId: state.sentinelId,
    mode: controlResult.mode === "already-current" ? "already-current" : "confirm-wait",
    mutation: controlResult.mutation !== false,
    specRef: SPEC_REF,
    source: state.sourceHead,
    image: state.image,
    pipelineRun: record(controlResult).pipelineRun ?? sentinelPipelineRunName(state, options.rerun),
    controlPlane: controlResult,
    health,
    budget,
    validationPlan: publishCurrentHealthValidationPlan(state),
    stageBudgets: publishCurrentStageBudgets(state),
    elapsedMs,
    withinBudget,
    timings,
    slowStages,
    drillDown: controlPlaneDrillDown(state, record(controlResult).pipelineRun ?? record(controlResult.publish).jobName ?? sentinelPipelineRunName(state, options.rerun), record(controlResult.flush)),
    warnings: mergeWarnings(controlResult.warnings, publishCurrentBudgetWarnings(slowStages, withinBudget, budgetSeconds, elapsedMs)),
    blocker,
    next: publishCurrentNext(state),
    valuesRedacted: true,
  };
  interruptContext.phase = "completed";
  return rendered(ok, command, renderPublishCurrentResult(result));
}

function strictRemainingSeconds(deadline: number, cap: number): number {
  return Math.max(0, Math.min(cap, Math.ceil((deadline - Date.now()) / 1000)));
}

function withObservedWait(observed: SentinelObservedStatus, startedAt: number, timeoutSeconds: number, includeGitMirror: boolean): SentinelObservedStatus {
  const elapsedMs = Date.now() - startedAt;
  return {
    ...observed,
    wait: {
      polls: 1,
      elapsedMs,
      timeoutMs: Math.max(1, timeoutSeconds) * 1000,
      ready: sentinelObservedReady(observed),
      includeGitMirror,
      valuesRedacted: true,
    },
  };
}

function sentinelAlreadyCurrentControlResult(state: SentinelCicdState, observed: SentinelObservedStatus, elapsedMs: number): Record<string, unknown> {
  const registryProbe = record(record(observed.registry).probe);
  const gitops = record(observed.gitops);
  const argo = record(observed.argo);
  const digest = text(registryProbe.digest);
  const digestRef = digest === "-" ? null : `${state.image.repository}@${digest}`;
  const stageTimings = {
    sourceFetchMs: 0,
    monitorWebVerifyMs: 0,
    imageBuildMs: 0,
    gitopsMs: 0,
    totalMs: 0,
    valuesRedacted: true,
  };
  return {
    ok: true,
    command: "web-probe sentinel control-plane trigger-current",
    node: state.spec.nodeId,
    lane: state.spec.lane,
    mode: "already-current",
    mutation: false,
    specRef: SPEC_REF,
    source: state.sourceHead,
    image: state.image,
    pipelineRun: "already-current",
    gitops: {
      path: stringAt(state.cicd, "gitopsPath"),
      targetRevision: stringAt(state.cicd, "argo.targetRevision"),
      manifestObjects: state.manifests.length,
      manifestSha256: state.manifestSha256,
    },
    argo: {
      namespace: stringAt(state.cicd, "argo.namespace"),
      projectName: stringAt(state.cicd, "argo.projectName"),
      applicationName: stringAt(state.cicd, "argo.applicationName"),
    },
    validation: {
      scenarioId: stringAt(state.cicd, "targetValidation.scenarioId"),
      maxSeconds: numberAt(state.cicd, "targetValidation.maxSeconds"),
      controlPlaneWaitMaxSeconds: controlPlaneWaitWarningSeconds(state),
      quickVerifyMode: "manual-validate",
      automaticSecondPath: false,
    },
    manifests: {
      objects: manifestObjectSummary(state.manifests),
      sha256: state.manifestSha256,
    },
    sourceMirrorSync: { ok: true, phase: "already-current", jobName: "-", elapsedMs: 0, valuesRedacted: true },
    publish: {
      ok: true,
      phase: "already-current",
      resourceKind: "PipelineRun",
      jobName: "already-current",
      elapsedMs: 0,
      payload: {
        ok: true,
        status: "already-current",
        sourceCommit: state.sourceHead.commit,
        imageRef: state.image.ref,
        digestRef,
        gitopsCommit: gitops.revision ?? argo.revision ?? null,
        stageTimings,
        completedStages: ["already-current"],
        valuesRedacted: true,
      },
      diagnostics: {
        domain: "publish",
        resourceKind: "PipelineRun",
        pipelineRun: "already-current",
        currentPhase: "already-current",
        completedStages: ["already-current:skipped"],
        stageTimings,
        valuesRedacted: true,
      },
      valuesRedacted: true,
    },
    flush: { ok: true, skipped: true, reason: "already-current", valuesRedacted: true },
    runtimeSecretsApply: { ok: true, skipped: true, reason: "already-current-observed-ready", valuesRedacted: true },
    publicExposureApply: { ok: true, skipped: true, reason: "already-current-observed-ready", valuesRedacted: true },
    argoApply: { ok: true, skipped: true, reason: "already-current-observed-ready", valuesRedacted: true },
    observed,
    targetValidation: null,
    elapsedMs,
    warnings: [
      "publish-current already-current fast path: source mirror, registry, GitOps, Argo and runtime already match the selected source; skipped Tekton publish and used health endpoint validation only.",
      ...sentinelObservedWarnings(observed),
      ...targetValidationDeferredWarnings(state, false, controlPlaneWaitWarningSeconds(state)),
    ],
    blocker: null,
    recoveryNext: controlPlaneRecoveryNext(state, true, {}, { ok: true }, observed),
    next: controlPlaneNext(state, "trigger-current"),
    valuesRedacted: true,
  };
}

export function loadSentinelCicdState(
  spec: HwlabRuntimeLaneSpec,
  sentinelId: string | null,
  timeoutSeconds: number,
  sourceResolveMode: SourceResolveMode,
  sourceOverride: SentinelSourceOverride | null = null,
  reuseConfig: RuntimeReuseConfig | null = null,
): SentinelCicdState {
  const sentinel = resolveWebProbeSentinel(spec, sentinelId);
  const configPlan = webProbeSentinelConfigPlan(spec, "status", sentinel.id);
  const runtime = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.runtime), sentinel.configRefs.runtime);
  const cicd = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.cicd), sentinel.configRefs.cicd);
  const scenarios = readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.scenarios);
  const rawPublicExposure = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.publicExposure), sentinel.configRefs.publicExposure);
  const publicExposure = effectiveWebProbeSentinelPublicExposure(spec, sentinel.id, rawPublicExposure);
  const secrets = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.secrets), sentinel.configRefs.secrets);
  const controlPlaneRef = stringField(cicd, "controlPlaneConfigRef");
  const controlPlaneTarget = recordTarget(readWebProbeSentinelConfigRefTarget(spec, controlPlaneRef), controlPlaneRef);
  const controlPlaneConfig = recordTarget(readConfigFile(configRefFile(controlPlaneRef)), configRefFile(controlPlaneRef));
  const nodeId = stringField(controlPlaneTarget, "node");
  const controlPlaneNode = recordTarget(valueAtPath(controlPlaneConfig, `nodes.${nodeId}`), `${configRefFile(controlPlaneRef)}#nodes.${nodeId}`);
  validateSentinelSourceAuthority(cicd);
  const effectiveCicd = applySentinelRuntimeReuseConfig(cicd, reuseConfig);
  const sourceHead = sourceOverride === null
    ? resolveSourceHead(spec, effectiveCicd, controlPlaneTarget, controlPlaneNode, timeoutSeconds, sourceResolveMode)
    : sourceHeadFromOverride(effectiveCicd, sourceOverride);
  const image = sentinelImagePlan(spec, effectiveCicd, sourceHead);
  const manifests = renderSentinelManifests(spec, sentinel.id, runtime, effectiveCicd, scenarios, publicExposure, secrets, image, sourceHead);
  const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`;
  return {
    spec,
    sentinelId: sentinel.id,
    configRefs: sentinel.configRefs,
    configReady: configPlan.ok,
    runtime,
    cicd: effectiveCicd,
    scenarios,
    publicExposure,
    secrets,
    controlPlaneTarget,
    controlPlaneNode,
    sourceHead,
    image,
    manifests,
    manifestSha256: sha256(manifestYaml),
    valuesRedacted: true,
  };
}

function applySentinelRuntimeReuseConfig(cicd: Record<string, unknown>, reuseConfig: RuntimeReuseConfig | null): Record<string, unknown> {
  const service = runtimeReuseService(reuseConfig, ["web-probe-sentinel", "monitor-web"]);
  const envReuse = service?.envReuse;
  if (envReuse === undefined || envReuse === null) return cicd;
  if (envReuse.enabled === false) return cicd;
  const monitorWeb = record(cicd.monitorWeb);
  const existingEnvReuse = record(monitorWeb.envReuse);
  return {
    ...cicd,
    monitorWeb: {
      ...monitorWeb,
      envReuse: {
        ...existingEnvReuse,
        ...(envReuse.mode === null ? {} : { mode: envReuse.mode }),
        ...(envReuse.nodeDepsPath === null ? {} : { nodeDepsPath: envReuse.nodeDepsPath }),
        source: summarizeRuntimeReuseConfig(reuseConfig),
      },
    },
  };
}

function sourceHeadFromOverride(cicd: Record<string, unknown>, override: SentinelSourceOverride): SourceHead {
  if (!/^[0-9a-f]{40}$/iu.test(override.commit)) throw new Error(`sentinel source override commit must be a full sha, got ${override.commit}`);
  const stageRef = override.stageRef ?? sentinelSourceSnapshotRef(cicd, override.commit);
  if (!stageRef.startsWith("refs/")) throw new Error(`sentinel source override stageRef must be a git ref, got ${stageRef}`);
  const mirrorCommit = override.mirrorCommit ?? override.commit;
  if (!/^[0-9a-f]{40}$/iu.test(mirrorCommit)) throw new Error(`sentinel source override mirrorCommit must be a full sha, got ${mirrorCommit}`);
  return {
    ok: true,
    repository: stringAt(cicd, "source.repository"),
    branch: stringAt(cicd, "source.branch"),
    commit: override.commit,
    stageRef,
    mirrorCommit,
    sourceAuthority: override.sourceAuthority,
    latestDrift: mirrorCommit !== override.commit,
    result: {
      exitCode: 0,
      timedOut: false,
      stdoutBytes: 0,
      stderrBytes: 0,
      stdoutPreview: `source supplied by ${override.sourceAuthority}`,
      stderrPreview: "",
    },
  };
}

function resolveSourceHead(
  spec: HwlabRuntimeLaneSpec,
  cicd: Record<string, unknown>,
  controlPlaneTarget: Record<string, unknown>,
  controlPlaneNode: Record<string, unknown>,
  timeoutSeconds: number,
  mode: SourceResolveMode,
): SourceHead {
  const repository = stringAt(cicd, "source.repository");
  const branch = stringAt(cicd, "source.branch");
  const resolved = mode === "sync"
    ? resolveSourceHeadWithK8sSnapshot(spec, cicd, controlPlaneTarget, controlPlaneNode, timeoutSeconds)
    : probeSourceMirrorCache(cicd, controlPlaneNode, timeoutSeconds, null);
  const probe = record(resolved.probe);
  const commit = nonEmptyString(probe.sourceCommit) ?? nonEmptyString(probe.commit) ?? nonEmptyString(probe.mirrorCommit);
  const stageRef = nonEmptyString(probe.stageRef) ?? (commit === null ? null : sentinelSourceSnapshotRef(cicd, commit));
  const mirrorCommit = nonEmptyString(probe.mirrorCommit) ?? nonEmptyString(probe.commit);
  return {
    ok: resolved.ok === true && commit !== null,
    repository,
    branch,
    commit,
    stageRef,
    mirrorCommit,
    sourceAuthority: mode === "sync" ? "git-mirror-snapshot" : "git-mirror-cache",
    latestDrift: commit !== null && mirrorCommit !== null && commit !== mirrorCommit,
    result: compactCommand(resolved.result),
  };
}

function resolveSourceHeadWithK8sSnapshot(
  spec: HwlabRuntimeLaneSpec,
  cicd: Record<string, unknown>,
  controlPlaneTarget: Record<string, unknown>,
  controlPlaneNode: Record<string, unknown>,
  timeoutSeconds: number,
): { ok: boolean; probe: Record<string, unknown>; result: CommandResult } {
  const namespace = stringAt(cicd, "builder.namespace");
  const prefix = `${stringAt(cicd, "builder.jobPrefix")}-source-resolve`;
  const jobName = `${prefix}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63);
  const manifest = sentinelSourceMirrorResolveJobManifest(spec, cicd, controlPlaneTarget, controlPlaneNode, jobName);
  const created = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
  if (created.exitCode !== 0) return { ok: false, probe: { ok: false, status: "create-failed", jobName, valuesRedacted: true }, result: created };
  const startedAt = Date.now();
  const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, 120_000));
  let lastCapture = created;
  while (Date.now() - startedAt < timeoutMs) {
    const probeCapture = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
    lastCapture = probeCapture;
    const probeResolution = resolveSentinelChildJson(probeCapture, "web-probe-sentinel-source-resolve-job-probe");
    const probe = probeResolution.parsed ?? {};
    const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? ""));
    if (probe.succeeded === true) return { ok: payload.ok === true, probe: { ...payload, stdoutRecovery: probeResolution.diagnostics, valuesRedacted: true }, result: probeCapture };
    if (probe.failed === true) return { ok: false, probe: Object.keys(payload).length === 0 ? { ok: false, status: "failed", jobName, stdoutRecovery: probeResolution.diagnostics, valuesRedacted: true } : { ...payload, stdoutRecovery: probeResolution.diagnostics, valuesRedacted: true }, result: probeCapture };
    runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
  }
  return { ok: false, probe: { ok: false, status: "timeout", jobName, valuesRedacted: true }, result: lastCapture };
}

function sentinelSourceMirrorResolveJobManifest(
  spec: HwlabRuntimeLaneSpec,
  cicd: Record<string, unknown>,
  controlPlaneTarget: Record<string, unknown>,
  controlPlaneNode: Record<string, unknown>,
  jobName: string,
): Record<string, unknown> {
  const namespace = stringAt(cicd, "builder.namespace");
  const labels = {
    "app.kubernetes.io/name": "web-probe-sentinel-source-resolve",
    "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
    "unidesk.ai/spec-ref": "PJ2026-01060508",
    "unidesk.ai/node": spec.nodeId,
    "unidesk.ai/lane": spec.lane,
  };
  return {
    apiVersion: "batch/v1",
    kind: "Job",
    metadata: { name: jobName, namespace, labels },
    spec: {
      backoffLimit: 0,
      activeDeadlineSeconds: numberAt(cicd, "builder.activeDeadlineSeconds"),
      ttlSecondsAfterFinished: numberAt(cicd, "builder.ttlSecondsAfterFinished"),
      template: {
        metadata: { labels },
        spec: {
          restartPolicy: "Never",
          volumes: [
            sentinelGitMirrorCacheVolumeFromTarget(controlPlaneTarget),
            { name: "git-ssh", secret: { secretName: stringAt(cicd, "builder.gitSshSecretName"), defaultMode: 256 } },
          ],
          containers: [{
            name: "resolve",
            image: sentinelSourceResolverImage(spec, cicd),
            imagePullPolicy: "IfNotPresent",
            command: ["/bin/sh", "-ec", sentinelSourceMirrorSyncShellFromConfig(cicd, controlPlaneNode, jobName, null)],
            volumeMounts: [
              { name: "cache", mountPath: "/cache" },
              { name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
            ],
          }],
        },
      },
    },
  };
}

function sentinelSourceResolverImage(spec: HwlabRuntimeLaneSpec, cicd: Record<string, unknown>): string {
  const baseImageRef = stringAt(cicd, "image.baseImageRef");
  return stringTarget(readWebProbeSentinelConfigRefTarget(spec, baseImageRef), baseImageRef);
}

function validateSentinelSourceAuthority(cicd: Record<string, unknown>): void {
  const mode = stringAt(cicd, "sourceAuthority.mode");
  const resolver = stringAt(cicd, "sourceAuthority.resolver");
  const allowHostGit = booleanAt(cicd, "sourceAuthority.allowHostGit");
  const allowGithubDirectInPipeline = booleanAt(cicd, "sourceAuthority.allowGithubDirectInPipeline");
  const missingObjectPolicy = stringAt(cicd, "sourceSnapshot.missingObjectPolicy");
  const supported = (mode === "gitMirrorSnapshot" && resolver === "k8s-git-mirror") || (mode === "giteaSnapshot" && resolver === "gitea-mirror");
  if (!supported) throw new Error("sourceAuthority must be gitMirrorSnapshot/k8s-git-mirror or giteaSnapshot/gitea-mirror");
  if (allowHostGit !== false) throw new Error("sourceAuthority.allowHostGit must be false");
  if (allowGithubDirectInPipeline !== false) throw new Error("sourceAuthority.allowGithubDirectInPipeline must be false");
  if (missingObjectPolicy !== "fail-fast") throw new Error("sourceSnapshot.missingObjectPolicy must be fail-fast");
  sentinelSourceSnapshotStageRefPrefix(cicd);
}


function sentinelImagePlan(spec: HwlabRuntimeLaneSpec, cicd: Record<string, unknown>, sourceHead: SourceHead): SentinelImagePlan {
  const repository = stringAt(cicd, "image.repository");
  const tag = sourceHead.commit === null ? "source-unresolved" : sourceHead.commit.slice(0, 12);
  const baseImageRef = stringAt(cicd, "image.baseImageRef");
  const baseImage = stringTarget(readWebProbeSentinelConfigRefTarget(spec, baseImageRef), baseImageRef);
  const entrypoint = stringAt(cicd, "source.entrypoint");
  const monitorWeb = monitorWebCicdPlan(spec, cicd);
  const dockerfile = sentinelDockerfile(baseImage, entrypoint);
  return {
    repository,
    tag,
    ref: `${repository}:${tag}`,
    digestRef: null,
    baseImage,
    buildContext: stringAt(cicd, "source.buildContext"),
    entrypoint,
    dockerfileSha256: sha256(dockerfile),
    dockerfilePreview: dockerfile,
    monitorWeb,
  };
}

function sentinelDockerfile(baseImage: string, entrypoint: string): string {
  return [
    `FROM ${baseImage}`,
    "WORKDIR /app",
    "ENV NODE_ENV=production",
    "COPY .unidesk-sentinel-bin/trans /usr/local/bin/trans",
    "COPY . /app",
    `ENTRYPOINT ["bun", "${entrypoint}"]`,
    "",
  ].join("\n");
}
function publishCurrentBudget(state: SentinelCicdState): Record<string, unknown> {
  const budget = recordTarget(valueAtPath(state.cicd, "publishCurrent.endToEndBudget"), "publishCurrent.endToEndBudget");
  return {
    maxSeconds: numberAt(budget, "maxSeconds"),
    valuesRedacted: true,
  };
}

function publishCurrentStageBudgets(state: SentinelCicdState): Record<string, unknown> {
  const budgets = recordTarget(valueAtPath(state.cicd, "publishCurrent.stageBudgets"), "publishCurrent.stageBudgets");
  return {
    sourceSyncSeconds: numberAt(budgets, "sourceSyncSeconds"),
    sourceFetchSeconds: numberAt(budgets, "sourceFetchSeconds"),
    monitorWebVerifySeconds: numberAt(budgets, "monitorWebVerifySeconds"),
    imageBuildSeconds: numberAt(budgets, "imageBuildSeconds"),
    gitopsSeconds: numberAt(budgets, "gitopsSeconds"),
    argoRuntimeSeconds: numberAt(budgets, "argoRuntimeSeconds"),
    dashboardVerifySeconds: numberAt(budgets, "dashboardVerifySeconds"),
    valuesRedacted: true,
  };
}

function publishCurrentDashboardPlan(state: SentinelCicdState): Record<string, unknown> {
  const dashboard = recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard");
  const viewport = stringAt(dashboard, "viewport");
  if (!/^[1-9][0-9]{1,4}x[1-9][0-9]{1,4}$/u.test(viewport)) throw new Error(`publishCurrent.dashboard.viewport must look like 1440x900, got ${viewport}`);
  return {
    enabled: booleanAt(dashboard, "enabled"),
    required: booleanAt(dashboard, "required"),
    viewport,
    timeoutMs: numberAt(dashboard, "timeoutMs"),
    waitTimeoutMs: numberAt(dashboard, "waitTimeoutMs"),
    commandTimeoutSeconds: numberAt(dashboard, "commandTimeoutSeconds"),
    fullPage: booleanAt(dashboard, "fullPage"),
    valuesRedacted: true,
  };
}

function publishCurrentDashboardEnabled(state: SentinelCicdState): boolean {
  return booleanAt(recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard"), "enabled");
}

function publishCurrentDashboardRequired(state: SentinelCicdState): boolean {
  return booleanAt(recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard"), "required");
}

function publishCurrentHealthValidationPlan(state: SentinelCicdState): Record<string, unknown> {
  return {
    enabled: true,
    required: true,
    endpoint: stringAt(state.runtime, "healthPath"),
    source: "runtime.healthPath",
    browser: false,
    playwright: false,
    webProbe: false,
    valuesRedacted: true,
  };
}

function publishCurrentDashboardReserveSeconds(state: SentinelCicdState): number {
  if (!publishCurrentDashboardEnabled(state)) return 0;
  const dashboard = publishCurrentDashboardPlan(state);
  const budgets = publishCurrentStageBudgets(state);
  return Math.max(0, Math.min(
    numberAt(budgets, "dashboardVerifySeconds"),
    numberAt(dashboard, "commandTimeoutSeconds"),
    Math.ceil(numberAt(dashboard, "waitTimeoutMs") / 1000),
  ));
}

function publishCurrentDashboardOptions(state: SentinelCicdState, timeoutSeconds: number): Extract<WebProbeSentinelOptions, { kind: "dashboard" }> {
  const dashboard = publishCurrentDashboardPlan(state);
  const remainingMs = Math.max(1000, Math.trunc(timeoutSeconds * 1000));
  const waitTimeoutMs = Math.min(numberAt(dashboard, "waitTimeoutMs"), remainingMs);
  const timeoutMs = Math.min(numberAt(dashboard, "timeoutMs"), waitTimeoutMs);
  return {
    kind: "dashboard",
    action: "verify",
    node: state.spec.nodeId,
    lane: state.spec.lane,
    sentinelId: state.sentinelId,
    viewport: stringAt(dashboard, "viewport"),
    localDir: "/tmp",
    name: null,
    runId: null,
    errorId: null,
    timeoutMs,
    waitTimeoutMs,
    timeoutSeconds: Math.min(numberAt(dashboard, "commandTimeoutSeconds"), Math.max(1, Math.trunc(timeoutSeconds))),
    commandTimeoutSeconds: Math.min(numberAt(dashboard, "commandTimeoutSeconds"), Math.max(1, Math.trunc(timeoutSeconds))),
    fullPage: booleanAt(dashboard, "fullPage"),
    raw: false,
  };
}

function publishCurrentStageTimings(controlResult: Record<string, unknown>, health: Record<string, unknown> | null, elapsedMs: number): Record<string, unknown> {
  const publish = record(controlResult.publish);
  const payload = record(publish.payload);
  const payloadStageTimings = record(payload.stageTimings);
  const diagnosticStageTimings = record(record(publish.diagnostics).stageTimings);
  const stageTimings = Object.keys(payloadStageTimings).length > 0 ? payloadStageTimings : diagnosticStageTimings;
  const observedWait = record(record(controlResult.observed).wait);
  const healthRecord = record(health);
  return {
    sourceSyncMs: finiteNumberOrNull(record(controlResult.sourceMirrorSync).elapsedMs),
    sourceFetchMs: finiteNumberOrNull(stageTimings.sourceFetchMs),
    monitorWebVerifyMs: finiteNumberOrNull(stageTimings.monitorWebVerifyMs),
    imageBuildMs: finiteNumberOrNull(stageTimings.imageBuildMs),
    gitopsMs: finiteNumberOrNull(stageTimings.gitopsMs),
    argoRuntimeMs: finiteNumberOrNull(observedWait.elapsedMs),
    healthValidationMs: finiteNumberOrNull(healthRecord.elapsedMs),
    totalMs: elapsedMs,
    valuesRedacted: true,
  };
}

function publishCurrentSlowStages(state: SentinelCicdState, timings: Record<string, unknown>, budgetSeconds: number): Record<string, unknown>[] {
  const budgets = publishCurrentStageBudgets(state);
  const stageMap: Array<[string, string, string, string]> = [
    ["source-sync", "sourceSyncMs", "sourceSyncSeconds", "check git-mirror pre-sync, node-local mirror health and SSH-over-proxy latency"],
    ["source-fetch", "sourceFetchMs", "sourceFetchSeconds", "keep sparse checkout paths narrow and verify node-local git mirror object availability"],
    ["monitor-web-verify", "monitorWebVerifyMs", "monitorWebVerifySeconds", "keep monitor-web verification copy-only and avoid rebuilding frontend assets during publish"],
    ["image-build", "imageBuildMs", "imageBuildSeconds", "verify env reuse node_modules hit, BuildKit layer cache, copy-only Containerfile and image-build proxy route"],
    ["gitops", "gitopsMs", "gitopsSeconds", "inspect GitOps mirror cache, commit/writeback latency and post-flush state"],
    ["argo-runtime", "argoRuntimeMs", "argoRuntimeSeconds", "inspect Argo refresh, runtime Deployment readiness and image digest alignment probes"],
    ["health-validation", "healthValidationMs", "dashboardVerifySeconds", "inspect runtime health endpoint latency and service routing"],
  ];
  const slow = stageMap.flatMap(([stage, timingKey, budgetKey, suggestion]) => {
    const elapsed = finiteNumberOrNull(timings[timingKey]);
    const stageBudgetSeconds = finiteNumberOrNull(budgets[budgetKey]);
    if (elapsed === null || stageBudgetSeconds === null || elapsed <= stageBudgetSeconds * 1000) return [];
    return [{ stage, elapsedMs: elapsed, budgetSeconds: stageBudgetSeconds, suggestion, valuesRedacted: true }];
  });
  const total = finiteNumberOrNull(timings.totalMs);
  if (total !== null && total > budgetSeconds * 1000) {
    slow.push({
      stage: "total",
      elapsedMs: total,
      budgetSeconds,
      suggestion: "stop blind waiting; use the stage table to optimize the largest source sync, BuildKit/cache, GitOps/Argo or health endpoint segment before rerun",
      valuesRedacted: true,
    });
  }
  return slow;
}

function publishCurrentBudgetWarnings(slowStages: readonly Record<string, unknown>[], withinBudget: boolean, budgetSeconds: number, elapsedMs: number): string[] {
  const warnings = slowStages.map((stage) => `${text(stage.stage)} exceeded configured ${text(stage.budgetSeconds)}s budget (${Math.round((finiteNumberOrNull(stage.elapsedMs) ?? 0) / 1000)}s): ${text(stage.suggestion)}`);
  if (!withinBudget) warnings.unshift(`publish-current exceeded configured ${budgetSeconds}s end-to-end budget (${Math.round(elapsedMs / 1000)}s); no further blind wait was attempted.`);
  return warnings;
}

function publishCurrentBlocker(controlResult: Record<string, unknown>, health: Record<string, unknown>, withinBudget: boolean): Record<string, unknown> {
  if (controlResult.ok !== true) {
    const blocker = record(controlResult.blocker);
    return {
      code: blocker.code ?? "sentinel-publish-current-control-plane-blocked",
      reason: blocker.reason ?? "control-plane publish did not converge",
      valuesRedacted: true,
    };
  }
  if (health.ok !== true) {
    const degradedReason = text(health.degradedReason);
    return {
      code: health.skipped === true ? text(health.reason) : "sentinel-publish-current-health-endpoint-failed",
      reason: health.skipped === true
        ? "health endpoint validation did not run"
        : degradedReason === "-"
          ? "health endpoint validation did not pass"
          : `health endpoint validation did not pass: ${degradedReason}`,
      valuesRedacted: true,
    };
  }
  if (!withinBudget) {
    return {
      code: "sentinel-publish-current-over-budget",
      reason: "runtime and health endpoint converged, but the one-click CI/CD path exceeded the YAML end-to-end budget",
      valuesRedacted: true,
    };
  }
  return { code: "sentinel-publish-current-blocked", reason: "publish-current did not satisfy all checks", valuesRedacted: true };
}

function renderSentinelManifests(
  spec: HwlabRuntimeLaneSpec,
  sentinelId: string,
  runtime: Record<string, unknown>,
  cicd: Record<string, unknown>,
  scenarios: unknown,
  publicExposure: Record<string, unknown>,
  secrets: Record<string, unknown>,
  image: SentinelImagePlan,
  sourceHead: SourceHead,
): readonly Record<string, unknown>[] {
  const namespace = stringAt(runtime, "namespace");
  const labels = {
    "app.kubernetes.io/name": stringAt(runtime, "deploymentName"),
    "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
    "app.kubernetes.io/managed-by": "unidesk",
    "unidesk.ai/spec-ref": "PJ2026-01060508",
    "unidesk.ai/node": spec.nodeId,
    "unidesk.ai/lane": spec.lane,
    "unidesk.ai/web-probe-sentinel-id": sentinelId,
  };
  const deploymentName = stringAt(runtime, "deploymentName");
  const serviceName = stringAt(runtime, "serviceName");
  const servicePort = numberAt(runtime, "servicePort");
  const pvcStorage = stringAt(runtime, "pvcStorage");
  const stateRoot = stringAt(runtime, "stateRoot");
  const sourceCommitAnnotations = sentinelSourceCommitAnnotations(sourceHead.commit);
  const sourceCommitMetadata = sourceCommitAnnotations === null ? {} : { annotations: sourceCommitAnnotations };
  const sentinelEnv = sentinelContainerEnv(sentinelId, runtime, cicd, secrets, sourceHead.commit);
  const kubernetesApiEgress = sentinelKubernetesApiEgress(runtime);
  const cadenceJob = sentinelCadenceCronJobPlan(spec, sentinelId, runtime, cicd, scenarios, image.ref, sentinelEnv);
  if (cadenceJob !== null) {
    emitWebProbeSentinelSpan({
      node: spec.nodeId,
      lane: spec.lane,
      sentinelId,
      namespace,
      runtime,
      cicd,
    }, "web_probe_sentinel.cadence.cronjob_rendered", {
      cronJobName: record(cadenceJob.metadata).name ?? null,
      namespace,
      cadence: record(cadenceJob.metadata).annotations === undefined ? null : record(record(cadenceJob.metadata).annotations)["unidesk.ai/cadence"],
      schedule: record(cadenceJob.spec).schedule ?? null,
      valuesRedacted: true,
    });
  }
  return [
    {
      apiVersion: "v1",
      kind: "ServiceAccount",
      metadata: { name: stringAt(runtime, "serviceAccountName"), namespace, labels },
    },
    {
      apiVersion: "rbac.authorization.k8s.io/v1",
      kind: "Role",
      metadata: { name: `${deploymentName}-manual-trigger`, namespace, labels },
      rules: [
        { apiGroups: ["batch"], resources: ["cronjobs"], verbs: ["get"] },
        { apiGroups: ["batch"], resources: ["jobs"], verbs: ["create", "get", "list"] },
      ],
    },
    {
      apiVersion: "rbac.authorization.k8s.io/v1",
      kind: "RoleBinding",
      metadata: { name: `${deploymentName}-manual-trigger`, namespace, labels },
      subjects: [{ kind: "ServiceAccount", name: stringAt(runtime, "serviceAccountName"), namespace }],
      roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: `${deploymentName}-manual-trigger` },
    },
    {
      apiVersion: "v1",
      kind: "PersistentVolumeClaim",
      metadata: { name: stringAt(runtime, "pvcName"), namespace, labels },
      spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: pvcStorage } } },
    },
    {
      apiVersion: "v1",
      kind: "ConfigMap",
      metadata: { name: `${deploymentName}-config`, namespace, labels },
      data: {
        "config-summary.json": JSON.stringify({
          specRef: SPEC_REF,
          node: spec.nodeId,
          lane: spec.lane,
          sentinelId,
          publicBaseUrl: stringAt(publicExposure, "publicBaseUrl"),
          routePrefix: stringAtNullable(publicExposure, "routePrefix") ?? "/",
          gitopsPath: stringAt(cicd, "gitopsPath"),
          valuesRedacted: true,
        }, null, 2),
      },
    },
    {
      apiVersion: "apps/v1",
      kind: "Deployment",
      metadata: { name: deploymentName, namespace, labels, ...sourceCommitMetadata },
      spec: {
        replicas: numberAt(runtime, "replicas"),
        selector: { matchLabels: { "app.kubernetes.io/name": deploymentName } },
        template: {
          metadata: { labels, ...sourceCommitMetadata },
          spec: {
            serviceAccountName: stringAt(runtime, "serviceAccountName"),
            containers: [{
              name: "sentinel",
              image: image.ref,
              imagePullPolicy: "IfNotPresent",
              args: [
                "--node",
                spec.nodeId,
                "--lane",
                spec.lane,
                "--sentinel",
                sentinelId,
                "--state-root",
                stateRoot,
                "--host",
                stringAt(runtime, "listenHost"),
                "--port",
                String(servicePort),
              ],
              env: sentinelEnv,
              ports: [{ name: "http", containerPort: servicePort }],
              readinessProbe: { httpGet: { path: stringAt(runtime, "healthPath"), port: "http" } },
              livenessProbe: { httpGet: { path: stringAt(runtime, "healthPath"), port: "http" } },
              volumeMounts: [{ name: "state", mountPath: stateRoot }],
            }],
            volumes: [{ name: "state", persistentVolumeClaim: { claimName: stringAt(runtime, "pvcName") } }],
          },
        },
      },
    },
    {
      apiVersion: "v1",
      kind: "Service",
      metadata: { name: serviceName, namespace, labels },
      spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": deploymentName }, ports: [{ name: "http", port: servicePort, targetPort: "http" }] },
    },
    ...(cadenceJob === null ? [] : [cadenceJob]),
    ...releaseAMonitorManifests(spec, image.ref, resolvedMigrationSources(spec, sentinelId)[0] ?? (() => { throw new Error("Release A monitor requires one resolved migration source"); })()),
    {
      apiVersion: "apps/v1",
      kind: "Deployment",
      metadata: { name: stringAt(publicExposure, "frpc.deploymentName"), namespace, labels: { ...labels, "app.kubernetes.io/component": "tunnel" } },
      spec: {
        replicas: 1,
        selector: { matchLabels: { "app.kubernetes.io/name": stringAt(publicExposure, "frpc.deploymentName") } },
        template: {
          metadata: {
            labels: { ...labels, "app.kubernetes.io/name": stringAt(publicExposure, "frpc.deploymentName"), "app.kubernetes.io/component": "tunnel" },
            annotations: {
              "unidesk.ai/public-base-url": stringAt(publicExposure, "publicBaseUrl"),
              "unidesk.ai/frp-server": `${stringAt(publicExposure, "frpc.serverAddr")}:${numberAt(publicExposure, "frpc.serverPort")}`,
              "unidesk.ai/frp-remote-port": String(numberAt(publicExposure, "frpc.httpProxy.remotePort")),
            },
          },
          spec: {
            containers: [{
              name: "frpc",
              image: stringAt(publicExposure, "frpc.image"),
              imagePullPolicy: "IfNotPresent",
              args: ["-c", "/etc/frp/frpc.toml"],
              volumeMounts: [{ name: "frpc-config", mountPath: "/etc/frp/frpc.toml", subPath: stringAt(publicExposure, "frpc.secretKey"), readOnly: true }],
            }],
            volumes: [{ name: "frpc-config", secret: { secretName: stringAt(publicExposure, "frpc.secretName") } }],
          },
        },
      },
    },
    {
      apiVersion: "networking.k8s.io/v1",
      kind: "NetworkPolicy",
      metadata: { name: `${deploymentName}-egress`, namespace, labels },
      spec: {
        podSelector: { matchLabels: { "app.kubernetes.io/name": deploymentName } },
        policyTypes: ["Ingress", "Egress"],
        ingress: [{ from: [{ namespaceSelector: {} }], ports: [{ protocol: "TCP", port: servicePort }] }],
        egress: [
          { to: [{ namespaceSelector: {} }] },
          ...kubernetesApiEgress,
        ],
      },
    },
    {
      apiVersion: "argoproj.io/v1alpha1",
      kind: "Application",
      metadata: { name: stringAt(cicd, "argo.applicationName"), namespace: stringAt(cicd, "argo.namespace"), labels },
      spec: {
        project: stringAt(cicd, "argo.projectName"),
        source: {
          repoURL: stringAt(cicd, "argo.repoURL"),
          targetRevision: stringAt(cicd, "argo.targetRevision"),
          path: stringAt(cicd, "gitopsPath"),
        },
        destination: { server: "https://kubernetes.default.svc", namespace },
        syncPolicy: { automated: { prune: true, selfHeal: true } },
      },
    },
  ];
}

function sentinelKubernetesApiEgress(runtime: Record<string, unknown>): readonly Record<string, unknown>[] {
  if (booleanAtNullable(runtime, "kubernetesApi.egress.enabled") !== true) return [];
  const rules = arrayAtNullable(runtime, "kubernetesApi.egress.rules");
  const rows = rules.length > 0 ? rules : [{ cidr: stringAt(runtime, "kubernetesApi.egress.cidr"), port: numberAt(runtime, "kubernetesApi.egress.port") }];
  return rows.map((rule) => ({
    to: [{ ipBlock: { cidr: stringAt(rule, "cidr") } }],
    ports: [{ protocol: "TCP", port: numberAt(rule, "port") }],
  }));
}

function sentinelSourceCommitAnnotations(sourceCommit: string | null): Record<string, string> | null {
  if (sourceCommit === null) return null;
  return {
    "unidesk.ai/source-commit": sourceCommit,
    "hwlab.pikastech.local/source-commit": sourceCommit,
  };
}

function sentinelContainerEnv(sentinelId: string, runtime: Record<string, unknown>, cicd: Record<string, unknown>, secrets: Record<string, unknown>, sourceCommit: string | null): readonly Record<string, unknown>[] {
  const env: Record<string, unknown>[] = [{ name: "UNIDESK_WEB_PROBE_SENTINEL_ID", value: sentinelId }];
  const otelEnabled = booleanAtNullable(runtime, "observability.otel.enabled") ?? booleanAtNullable(cicd, "observability.otel.enabled") ?? false;
  const otelEndpoint = stringAtNullable(runtime, "observability.otel.tracesEndpoint")
    ?? stringAtNullable(runtime, "observability.otel.endpoint")
    ?? stringAtNullable(cicd, "observability.otel.tracesEndpoint")
    ?? stringAtNullable(cicd, "observability.otel.endpoint");
  const otelServiceName = stringAtNullable(runtime, "observability.otel.serviceName") ?? stringAtNullable(cicd, "observability.otel.serviceName");
  const otelSampler = stringAtNullable(runtime, "observability.otel.sampler") ?? stringAtNullable(cicd, "observability.otel.sampler");
  const otelSamplerArg = stringAtNullable(runtime, "observability.otel.samplerArg") ?? stringAtNullable(cicd, "observability.otel.samplerArg");
  const sourcesByPurpose = new Map<string, Record<string, unknown>>();
  for (const source of arrayAt(secrets, "sources").map(record)) {
    const purpose = stringAtNullable(source, "purpose");
    if (purpose !== null) sourcesByPurpose.set(purpose, source);
  }
  const used = new Set(env.map((item) => String(item.name ?? "")));
  const pushEnv = (item: Record<string, unknown>): void => {
    const name = String(item.name ?? "");
    if (name.length === 0 || used.has(name)) return;
    used.add(name);
    env.push(item);
  };
  if (sourceCommit !== null) {
    pushEnv({ name: "UNIDESK_SOURCE_COMMIT", value: sourceCommit });
    pushEnv({ name: "WEB_PROBE_SENTINEL_SOURCE_COMMIT", value: sourceCommit });
  }
  if (otelEnabled) {
    if (otelEndpoint !== null) pushEnv({ name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", value: otelEndpoint });
    if (otelServiceName !== null) pushEnv({ name: "OTEL_SERVICE_NAME", value: otelServiceName });
    if (otelSampler !== null) pushEnv({ name: "OTEL_TRACES_SAMPLER", value: otelSampler });
    if (otelSamplerArg !== null) pushEnv({ name: "OTEL_TRACES_SAMPLER_ARG", value: otelSamplerArg });
  }
  for (const runtimeSecret of arrayAt(secrets, "runtimeSecrets").map(record)) {
    const secretName = stringAtNullable(runtimeSecret, "name");
    if (secretName === null) continue;
    for (const item of arrayAt(runtimeSecret, "data").map(record)) {
      const targetKey = stringAtNullable(item, "targetKey");
      const sourcePurpose = stringAtNullable(item, "sourcePurpose");
      const sourceKey = sourcePurpose === null ? null : stringAtNullable(sourcesByPurpose.get(sourcePurpose), "sourceKey");
      const sourceKeyEnvName = sourcePurpose === "bootstrap-admin" || sourcePurpose === "prompt-set" ? sourceKey : null;
      if (targetKey !== null && sourceKeyEnvName !== null && /^[A-Za-z_][A-Za-z0-9_]*$/u.test(sourceKeyEnvName)) {
        pushEnv({ name: sourceKeyEnvName, valueFrom: { secretKeyRef: { name: secretName, key: targetKey } } });
      }
      const envName = sourcePurpose === null || targetKey === null ? null : accountSecretEnvName(sourcePurpose, targetKey);
      if (envName === null) continue;
      pushEnv({ name: envName, valueFrom: { secretKeyRef: { name: secretName, key: targetKey } } });
    }
  }
  return env;
}

function sentinelCadenceCronJobPlan(
  spec: HwlabRuntimeLaneSpec,
  sentinelId: string,
  runtime: Record<string, unknown>,
  cicd: Record<string, unknown>,
  scenarios: unknown,
  imageRef: string,
  sentinelEnv: readonly Record<string, unknown>[],
): Record<string, unknown> | null {
  const scheduler = record(valueAtPath(cicd, "targetValidation.cadenceScheduler"));
  const cadenceSchedulerEnabled = booleanAtNullable(cicd, "targetValidation.cadenceScheduler.enabled") === true;
  if (!cadenceSchedulerEnabled) return null;
  const scenarioId = stringAtNullable(cicd, "targetValidation.scenarioId");
  if (scenarioId === null) return null;
  const scenario = scenarioRows(scenarios).find((item) => item.id === scenarioId && item.enabled !== false) ?? null;
  if (scenario === null) return null;
  const cadenceSeconds = typeof scenario.cadence === "string" ? parseDurationSeconds(scenario.cadence) : null;
  const schedule = cadenceSeconds === null ? null : cronScheduleForCadenceSeconds(cadenceSeconds);
  if (schedule === null) return null;
  const namespace = stringAt(runtime, "namespace");
  const deploymentName = stringAt(runtime, "deploymentName");
  const serviceAccountName = stringAt(runtime, "serviceAccountName");
  const timeoutSeconds = numberAt(cicd, "targetValidation.maxSeconds");
  const activeDeadlineSlackSeconds = numberAt(scheduler, "activeDeadlineSlackSeconds");
  const mainServerHost = stringAtNullable(cicd, "scheduler.mainServerHost");
  const name = sentinelCadenceCronJobName(deploymentName);
  const concurrencyPolicy = stringAt(scheduler, "concurrencyPolicy");
  if (!["Allow", "Forbid", "Replace"].includes(concurrencyPolicy)) throw new Error("targetValidation.cadenceScheduler.concurrencyPolicy must be Allow, Forbid or Replace");
  const labels = {
    "app.kubernetes.io/name": name,
    "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
    "app.kubernetes.io/component": "cadence-scheduler",
    "app.kubernetes.io/managed-by": "unidesk",
    "unidesk.ai/spec-ref": "PJ2026-01060508",
    "unidesk.ai/node": spec.nodeId,
    "unidesk.ai/lane": spec.lane,
    "unidesk.ai/web-probe-sentinel-id": sentinelId,
  };
  return {
    apiVersion: "batch/v1",
    kind: "CronJob",
    metadata: {
      name,
      namespace,
      labels,
      annotations: {
        "unidesk.ai/cadence": String(scenario.cadence),
        "unidesk.ai/target-validation-max-seconds": String(timeoutSeconds),
        "unidesk.ai/source": "targetValidation.cadenceScheduler",
      },
    },
    spec: {
      schedule,
      concurrencyPolicy,
      successfulJobsHistoryLimit: numberAt(scheduler, "successfulJobsHistoryLimit"),
      failedJobsHistoryLimit: numberAt(scheduler, "failedJobsHistoryLimit"),
      startingDeadlineSeconds: numberAt(scheduler, "startingDeadlineSeconds"),
      jobTemplate: {
        spec: {
          activeDeadlineSeconds: timeoutSeconds + activeDeadlineSlackSeconds,
          ttlSecondsAfterFinished: numberAt(scheduler, "ttlSecondsAfterFinished"),
          backoffLimit: numberAt(scheduler, "backoffLimit"),
          template: {
            metadata: { labels },
            spec: {
              restartPolicy: "Never",
              serviceAccountName,
              containers: [{
                name: "quick-verify",
                image: imageRef,
                imagePullPolicy: "IfNotPresent",
                command: ["bun", "scripts/cli.ts"],
                args: [
                  "web-probe",
                  "sentinel",
                  "validate",
                  "--node",
                  spec.nodeId,
                  "--lane",
                  spec.lane,
                  "--sentinel",
                  sentinelId,
                  "--quick-verify",
                  "--confirm",
                  "--wait",
                  "--timeout-seconds",
                  String(timeoutSeconds),
                ],
                env: [
                  ...sentinelEnv,
                  { name: "UNIDESK_WEB_PROBE_SENTINEL_DIRECT_SERVICE", value: "1" },
                  ...(mainServerHost === null ? [] : [{ name: "UNIDESK_MAIN_SERVER_HOST", value: mainServerHost }]),
                ],
              }],
            },
          },
        },
      },
    },
  };
}

function sentinelCadenceCronJobName(deploymentName: string): string {
  return safeKubernetesSegment(`${deploymentName}-quick-verify`, 52);
}

function scenarioRows(value: unknown): Record<string, unknown>[] {
  if (Array.isArray(value)) return value.map(record);
  if (!isRecord(value)) return [];
  if (Array.isArray(value.scenarios)) return value.scenarios.map(record);
  if (isRecord(value.workflow)) return [value.workflow];
  return [value];
}

function parseDurationSeconds(value: string): number | null {
  const match = /^(\d+)(ms|s|m|h)$/u.exec(value.trim());
  if (match === null) return null;
  const amount = Number(match[1]);
  const unit = match[2];
  if (unit === "ms") return Math.max(60, Math.ceil(amount / 1000));
  if (unit === "s") return Math.max(60, amount);
  if (unit === "m") return amount * 60;
  if (unit === "h") return amount * 3600;
  return null;
}

function cronScheduleForCadenceSeconds(seconds: number): string | null {
  if (!Number.isFinite(seconds) || seconds <= 0) return null;
  if (seconds <= 60) return "* * * * *";
  if (seconds % 60 === 0) {
    const minutes = Math.trunc(seconds / 60);
    if (minutes >= 1 && minutes <= 59) return `*/${minutes} * * * *`;
    if (minutes % 60 === 0) {
      const hours = Math.trunc(minutes / 60);
      if (hours >= 1 && hours <= 23) return `0 */${hours} * * *`;
      if (hours === 24) return "0 0 * * *";
    }
  }
  return null;
}

function accountSecretEnvName(sourcePurpose: string, targetKey: string): string | null {
  if (!/^account-[a-z0-9-]+$/u.test(sourcePurpose) || !targetKey.endsWith(".json")) return null;
  const segment = sourcePurpose.toUpperCase().replace(/[^A-Z0-9]+/gu, "_").replace(/^_+|_+$/gu, "");
  return segment.length === 0 ? null : `HWLAB_WEB_${segment}_JSON`;
}

export function quickVerifyAccountEnv(state: SentinelCicdState): { ok: boolean; env: NodeJS.ProcessEnv; summary: Record<string, unknown> } {
  const sourcesByPurpose = new Map<string, Record<string, unknown>>();
  for (const source of arrayAt(state.secrets, "sources").map(record)) {
    const purpose = stringAtNullable(source, "purpose");
    if (purpose !== null) sourcesByPurpose.set(purpose, source);
  }
  const env: NodeJS.ProcessEnv = {};
  const items: Record<string, unknown>[] = [];
  const missing: Record<string, unknown>[] = [];
  for (const runtimeSecret of arrayAt(state.secrets, "runtimeSecrets").map(record)) {
    const secretName = stringAtNullable(runtimeSecret, "name");
    for (const item of arrayAt(runtimeSecret, "data").map(record)) {
      const targetKey = stringAtNullable(item, "targetKey");
      const sourcePurpose = stringAtNullable(item, "sourcePurpose");
      const envName = sourcePurpose === null || targetKey === null ? null : accountSecretEnvName(sourcePurpose, targetKey);
      if (envName === null || sourcePurpose === null || targetKey === null) continue;
      const source = sourcesByPurpose.get(sourcePurpose);
      const runtimeValue = process.env[envName];
      if (source === undefined) {
        if (runtimeValue !== undefined && runtimeValue.length > 0) {
          env[envName] = runtimeValue;
          items.push({
            envName,
            secretName,
            targetKey,
            sourcePurpose,
            sourceMode: "runtime-env",
            fingerprint: `sha256:${createHash("sha256").update(runtimeValue).digest("hex").slice(0, 16)}`,
            valuesRedacted: true,
          });
          continue;
        }
        missing.push({ envName, secretName, targetKey, sourcePurpose, reason: "source-purpose-missing", valuesRedacted: true });
        continue;
      }
      const sourceRef = stringAt(source, "sourceRef");
      const sourceKey = stringAt(source, "sourceKey");
      const material = readSentinelSecretSourceValue(source);
      if (!material.ok) {
        if (runtimeValue !== undefined && runtimeValue.length > 0) {
          env[envName] = runtimeValue;
          items.push({
            envName,
            secretName,
            targetKey,
            sourcePurpose,
            sourceRef,
            sourceKey,
            sourceMode: "runtime-env",
            fingerprint: `sha256:${createHash("sha256").update(runtimeValue).digest("hex").slice(0, 16)}`,
            valuesRedacted: true,
          });
          continue;
        }
        missing.push({ envName, secretName, targetKey, sourcePurpose, sourceRef, sourceKey, reason: material.error, sourcePath: material.sourcePath, valuesRedacted: true });
        continue;
      }
      const value = stringAt(material, "value");
      env[envName] = value;
      items.push({
        envName,
        secretName,
        targetKey,
        sourcePurpose,
        sourceRef,
        sourceKey,
        fingerprint: `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`,
        valuesRedacted: true,
      });
    }
  }
  const summary = {
    ok: missing.length === 0,
    envCount: items.length,
    items,
    missing,
    valuesRedacted: true,
  };
  return { ok: missing.length === 0, env, summary };
}

function normalizeRoutePrefix(value: string | null): string {
  if (value === null || value.trim() === "" || value.trim() === "/") return "/";
  const prefixed = value.trim().startsWith("/") ? value.trim() : `/${value.trim()}`;
  return prefixed.replace(/\/+$/u, "") || "/";
}

function probeImageRegistry(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
  const endpoint = stringAt(state.controlPlaneNode, "registry.endpoint");
  const repoTag = state.image.ref.replace(`${endpoint}/`, "");
  const repo = repoTag.slice(0, repoTag.lastIndexOf(":"));
  const tag = repoTag.slice(repoTag.lastIndexOf(":") + 1);
  const registryMode = stringAtNullable(state.controlPlaneNode, "registry.mode") ?? "host-docker";
  const endpointIsLoopback = endpoint === "127.0.0.1:5000" || endpoint.startsWith("127.0.0.1:");
  const probeMode = registryMode === "k8s-workload" && !endpointIsLoopback ? "k8s-service" : "node-loopback";
  const url = probeMode === "k8s-service"
    ? `http://${stringAt(state.controlPlaneNode, "registry.serviceName")}.${stringAt(state.controlPlaneNode, "registry.namespace")}.svc.cluster.local:${numberAtNullable(state.controlPlaneNode, "registry.containerPort") ?? 5000}/v2/${repo}/manifests/${tag}`
    : `http://${endpoint}/v2/${repo}/manifests/${tag}`;
  const script = [
    "set +e",
    `url=${shellQuote(url)}`,
    `probe_mode=${shellQuote(probeMode)}`,
    "headers=$(mktemp)",
    "accept='application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json'",
    "if command -v curl >/dev/null 2>&1 && curl -fsSI -H \"Accept: $accept\" --max-time 5 \"$url\" >\"$headers\" 2>/tmp/web-probe-sentinel-image.err; then present=true; else present=false; fi",
    "digest=$(awk 'BEGIN{IGNORECASE=1} /^docker-content-digest:/ {gsub(/\\r/,\"\",$2); print $2; exit}' \"$headers\" 2>/dev/null)",
    "python3 - \"$present\" \"$digest\" \"$url\" \"$probe_mode\" <<'PY'",
    "import json, sys",
    "print(json.dumps({'present': sys.argv[1] == 'true', 'digest': sys.argv[2] or None, 'url': sys.argv[3], 'mode': sys.argv[4], 'valuesRedacted': True}))",
    "PY",
  ].join("\n");
  const result = probeMode === "k8s-service"
    ? runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", stringAt(state.cicd, "builder.namespace"), "exec", "deploy/git-mirror-http", "--", "sh", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 })
    : runCommand(["trans", stringAt(state.controlPlaneNode, "route"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
  const probeResolution = resolveSentinelChildJson(result, "web-probe-sentinel-image-registry-probe");
  return {
    ok: result.exitCode === 0,
    probe: probeResolution.parsed,
    stdoutRecovery: probeResolution.diagnostics,
    result: compactCommand(result),
  };
}

function runSentinelImageBuildConfirmed(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "image" }>): RenderedCliResult {
  const startedAt = Date.now();
  const command = "web-probe sentinel image build";
  const sourceMirrorProbe = probeSourceMirror(state, Math.min(options.timeoutSeconds, 20));
  const sourceMirrorSync = record(sourceMirrorProbe).ok === true ? sentinelSourceMirrorAlreadyPresentResult(state, sourceMirrorProbe) : runSentinelSourceMirrorSyncJob(state, options.timeoutSeconds);
  const sourceMirrorReady = sourceMirrorSync.ok === true;
  const publish = sourceMirrorReady
    ? runSentinelPublishJob(state, false, options.timeoutSeconds, false)
    : sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started");
  const registry = probeImageRegistry(state, options.timeoutSeconds);
  const registryReady = record(registry.probe).present === true;
  const ok = state.configReady && state.sourceHead.ok && sourceMirrorReady && publish.ok === true && registryReady;
  const elapsedMs = Date.now() - startedAt;
  const cicdWaitWarningSeconds = controlPlaneWaitWarningSeconds(state);
  const result = {
    ok,
    command,
    node: state.spec.nodeId,
    lane: state.spec.lane,
    mode: "confirm-wait",
    mutation: true,
    specRef: SPEC_REF,
    source: state.sourceHead,
    image: state.image,
    registry,
    sourceMirrorSync,
    publish,
    elapsedMs,
    warnings: [
      ...sentinelCicdElapsedWarnings(elapsedMs, "sentinel image build confirm-wait", cicdWaitWarningSeconds),
      ...sentinelCicdElapsedWarnings(record(sourceMirrorSync).elapsedMs, "sentinel source mirror sync", cicdWaitWarningSeconds),
      ...sentinelCicdElapsedWarnings(record(publish).elapsedMs, "sentinel publish", cicdWaitWarningSeconds),
      ...sentinelRemoteJobTimeoutWarnings(sourceMirrorSync, "sentinel source mirror sync"),
      ...sentinelRemoteJobTimeoutWarnings(publish, "sentinel publish"),
      ...sourceMirrorAlreadyReadyWarnings(state, sourceMirrorSync),
    ],
    blocker: ok
      ? null
      : !sourceMirrorReady
        ? { code: "sentinel-source-mirror-sync-failed", reason: "source mirror sync did not complete; investigate git mirror/proxy before image publish" }
        : publish.ok !== true
          ? { code: "sentinel-image-publish-failed", reason: "remote image publish job failed before registry validation" }
          : { code: "sentinel-image-registry-missing", reason: "image publish completed but expected registry tag is not visible" },
    next: {
      status: `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`,
      controlPlaneTrigger: `bun scripts/cli.ts web-probe sentinel control-plane trigger-current --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm`,
    },
    valuesRedacted: true,
  };
  return rendered(result.ok, command, renderImageResult(result));
}

function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "control-plane" }>): RenderedCliResult {
  const result = sentinelControlPlaneConfirmedResult(state, options);
  return rendered(result.ok === true, String(result.command), renderControlPlaneResult(result));
}

function sentinelControlPlaneConfirmedResult(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "control-plane" }>): Record<string, unknown> {
  const startedAt = Date.now();
  const command = `web-probe sentinel control-plane ${options.action}`;
  const applyOnly = options.action === "apply";
  const cicdWaitWarningSeconds = controlPlaneWaitWarningSeconds(state);
  const waitBudgetSeconds = Math.max(1, Math.min(options.timeoutSeconds, cicdWaitWarningSeconds));
  const deadline = startedAt + waitBudgetSeconds * 1000;
  const remainingCicdWaitSeconds = () => strictRemainingSeconds(deadline, waitBudgetSeconds);
  const remainingCommandSeconds = () => Math.max(1, remainingCicdWaitSeconds());
  const giteaSource = state.sourceHead.sourceAuthority === "gitea-snapshot" || state.sourceHead.sourceAuthority === "gitea-cache";
  const sourceMirrorProbe = applyOnly || giteaSource ? null : probeSourceMirror(state, Math.min(remainingCommandSeconds(), 20));
  const sourceMirrorSync = applyOnly
    ? null
    : giteaSource
      ? sentinelSourceMirrorAlreadyPresentResult(state, { ok: true, mode: "gitea-controlled-mirror-snapshot", stageRef: state.sourceHead.stageRef, sourceCommit: state.sourceHead.commit, valuesRedacted: true })
      : record(sourceMirrorProbe).ok === true ? sentinelSourceMirrorAlreadyPresentResult(state, sourceMirrorProbe) : runSentinelSourceMirrorSyncJob(state, remainingCommandSeconds());
  const sourceMirrorReady = applyOnly || record(sourceMirrorSync).ok === true;
  const publish = applyOnly
    ? null
    : sourceMirrorReady
      ? runSentinelPublishJob(state, true, remainingCommandSeconds(), options.rerun)
      : sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started");
  const publishWaitBudgetExhausted = !applyOnly && sourceMirrorReady && record(publish).ok !== true && remainingCicdWaitSeconds() <= 8;
  const flush = !applyOnly && !publishWaitBudgetExhausted && record(publish).ok === true
    ? startSentinelGitMirrorFlushAsync(state)
    : null;
  const runtimeSecretsApply = publishWaitBudgetExhausted
    ? sentinelSkippedControlStep("publish-wait-budget-exhausted")
    : applySentinelRuntimeSecrets(state, remainingCommandSeconds());
  const publicExposureApply = publishWaitBudgetExhausted
    ? sentinelSkippedControlStep("publish-wait-budget-exhausted")
    : applySentinelPublicExposure(state, remainingCommandSeconds());
  const argoApply = publishWaitBudgetExhausted
    ? sentinelSkippedControlStep("publish-wait-budget-exhausted")
    : applySentinelArgoApplication(state, remainingCommandSeconds());
  const observed = publishWaitBudgetExhausted
    ? sentinelSkippedObservedStatus("publish-wait-budget-exhausted")
    : waitForSentinelObservedStatus(state, remainingCommandSeconds(), undefined, false);
  const observedReady = sentinelObservedReady(observed);
  const publishReady = applyOnly || record(publish).ok === true || observedReady;
  const flushReady = applyOnly || record(flush).ok === true || observedReady;
  const targetValidation = null;
  const targetValidationBlocked = false;
  const ok = state.configReady
    && state.sourceHead.ok
    && sourceMirrorReady
    && publishReady
    && flushReady
    && record(runtimeSecretsApply).ok === true
    && record(publicExposureApply).ok === true
    && record(argoApply).ok === true
    && observedReady;
  const elapsedMs = Date.now() - startedAt;
  const blocker = ok ? null : {
    code: !sourceMirrorReady
      ? "sentinel-source-mirror-sync-failed"
      : !publishReady
        ? "sentinel-image-gitops-publish-not-ready"
        : record(runtimeSecretsApply).ok === false
          ? "sentinel-runtime-secret-sync-failed"
          : "sentinel-control-plane-not-ready",
    reason: !sourceMirrorReady
      ? "source mirror sync did not complete; investigate git mirror/proxy before control-plane publish"
      : !publishReady
        ? "Tekton publish did not finish and runtime observation has not proven the selected source/image/GitOps state yet"
        : record(runtimeSecretsApply).ok === false
          ? "one or more YAML-declared runtime Secrets were not synced from sourceRef"
          : "one or more publicExposure, Argo or runtime observation checks did not pass",
  };
  const publishPipelineRun = applyOnly ? sentinelPipelineRunName(state, options.rerun) : record(publish).jobName ?? sentinelPipelineRunName(state, options.rerun);
  const result = {
    ok,
    command,
    node: state.spec.nodeId,
    lane: state.spec.lane,
    mode: "confirm-wait",
    mutation: true,
    specRef: SPEC_REF,
    source: state.sourceHead,
    image: state.image,
    pipelineRun: publishPipelineRun,
    gitops: {
      path: stringAt(state.cicd, "gitopsPath"),
      targetRevision: stringAt(state.cicd, "argo.targetRevision"),
      manifestObjects: state.manifests.length,
      manifestSha256: state.manifestSha256,
    },
    argo: {
      namespace: stringAt(state.cicd, "argo.namespace"),
      projectName: stringAt(state.cicd, "argo.projectName"),
      applicationName: stringAt(state.cicd, "argo.applicationName"),
    },
    validation: {
      scenarioId: stringAt(state.cicd, "targetValidation.scenarioId"),
      maxSeconds: numberAt(state.cicd, "targetValidation.maxSeconds"),
      controlPlaneWaitMaxSeconds: cicdWaitWarningSeconds,
      quickVerifyMode: applyOnly ? "not-applicable" : "manual-validate",
      automaticSecondPath: false,
    },
    manifests: {
      objects: manifestObjectSummary(state.manifests),
      sha256: state.manifestSha256,
    },
    sourceMirrorSync,
    publish,
    flush,
    runtimeSecretsApply,
    publicExposureApply,
    argoApply,
    observed,
    targetValidation,
    elapsedMs,
    drillDown: controlPlaneDrillDown(state, publishPipelineRun, flush),
    warnings: Array.from(new Set([
      ...sentinelCicdElapsedWarnings(elapsedMs, "sentinel control-plane confirm-wait", cicdWaitWarningSeconds),
      ...sentinelCicdElapsedWarnings(record(sourceMirrorSync).elapsedMs, "sentinel source mirror sync", cicdWaitWarningSeconds),
      ...sentinelCicdElapsedWarnings(record(publish).elapsedMs, "sentinel publish", cicdWaitWarningSeconds),
      ...sentinelRemoteJobTimeoutWarnings(sourceMirrorSync, "sentinel source mirror sync"),
      ...sentinelRemoteJobTimeoutWarnings(publish, "sentinel publish"),
      ...sentinelCicdElapsedWarnings(record(flush).result === undefined ? null : record(record(flush).result).durationMs, "sentinel git-mirror flush", cicdWaitWarningSeconds),
      ...asyncGitMirrorFlushWarnings(flush, cicdWaitWarningSeconds),
      ...publishSatisfiedByObservedWarnings(publish, flush, observedReady),
      ...sourceMirrorAlreadyReadyWarnings(state, sourceMirrorSync),
      ...sentinelObservedWarnings(observed),
      ...(publishWaitBudgetExhausted ? [`sentinel publish consumed the configured ${waitBudgetSeconds}s confirm-wait budget; skipped runtime Secret, public exposure, Argo apply and observed wait to avoid blind over-budget waiting. Use the reported status/log drill-down after the PipelineRun advances.`] : []),
      ...targetValidationDeferredWarnings(state, applyOnly, cicdWaitWarningSeconds),
      ...(Array.isArray(record(targetValidation).warnings) ? record(targetValidation).warnings.map(text) : []),
      ...(targetValidationBlocked ? ["targetValidation is blocked; top-level STATUS only covers sentinel control-plane rollout. HWLAB business recovery remains pending; rerun quick verify after internal DB switch completes, without public fallback or a second execution path."] : []),
    ])),
    blocker,
    recoveryNext: controlPlaneRecoveryNext(state, ok, publish, flush, observed),
    next: controlPlaneNext(state, options.action),
    valuesRedacted: true,
  };
  return result;
}

function renderAsyncSentinelJob(state: SentinelCicdState, domain: "image" | "control-plane" | "publish", action: string, timeoutSeconds: number): RenderedCliResult {
  const args = domain === "image"
    ? ["web-probe", "sentinel", "image", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)]
    : domain === "control-plane"
      ? ["web-probe", "sentinel", "control-plane", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)]
      : ["web-probe", "sentinel", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)];
  const job = startJob(`hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_${domain}_${action}`, ["bun", "scripts/cli.ts", ...args], `Run HWLAB ${state.spec.lane} web-probe sentinel ${state.sentinelId} ${domain} ${action} for node ${state.spec.nodeId}`);
  const command = domain === "publish" ? `web-probe sentinel ${action}` : `web-probe sentinel ${domain} ${action}`;
  const result = {
    ok: true,
    command,
    node: state.spec.nodeId,
    lane: state.spec.lane,
    mode: "async-job",
    mutation: true,
    reason: "confirmed sentinel publish/build can exceed the short interactive window; use job status for bounded progress.",
    job,
    next: {
      status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`,
      wait: ["bun", "scripts/cli.ts", ...args].join(" "),
    },
    valuesRedacted: true,
  };
  return rendered(true, command, renderAsyncJobResult(result));
}

function startSentinelGitMirrorFlushAsync(state: SentinelCicdState): Record<string, unknown> {
  const args = ["hwlab", "nodes", "git-mirror", "flush", "--node", state.spec.nodeId, "--lane", state.spec.lane, "--confirm", "--wait"];
  const job = startJob(
    `hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_git_mirror_flush`,
    ["bun", "scripts/cli.ts", ...args],
    `Flush HWLAB ${state.spec.lane} git mirror after web-probe sentinel ${state.sentinelId} GitOps publish for node ${state.spec.nodeId}`,
  );
  return {
    ok: true,
    mode: "async-job",
    job,
    next: {
      status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`,
      wait: ["bun", "scripts/cli.ts", ...args].join(" "),
      gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${state.spec.nodeId} --lane ${state.spec.lane}`,
    },
    valuesRedacted: true,
  };
}

function asyncGitMirrorFlushWarnings(flush: unknown, budgetSeconds: number): string[] {
  const item = record(flush);
  if (item.mode !== "async-job") return [];
  const next = record(item.next);
  return [`sentinel git-mirror flush is running asynchronously to keep control-plane confirm-wait under configured ${Math.round(budgetSeconds)}s; follow ${next.status ?? next.gitMirrorStatus ?? "the reported job status"} for GitHub mirror closeout.`];
}

function sentinelSkippedControlStep(reason: string): Record<string, unknown> {
  return { ok: false, skipped: true, reason, valuesRedacted: true };
}

function sentinelSkippedObservedStatus(reason: string): SentinelObservedStatus {
  const skipped = { ok: false, skipped: true, reason, valuesRedacted: true };
  return {
    sourceMirror: skipped,
    registry: skipped,
    gitMirror: { skipped: true, reason, valuesRedacted: true },
    gitops: skipped,
    argo: skipped,
    runtime: skipped,
    cadence: skipped,
    wait: {
      polls: 0,
      elapsedMs: 0,
      timeoutMs: 0,
      ready: false,
      includeGitMirror: false,
      reason,
      valuesRedacted: true,
    },
  };
}

function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation, includeGitMirror = true): SentinelObservedStatus {
  const registry = probeImageRegistry(state, timeoutSeconds);
  const gitops = probeGitopsRuntimeManifest(state, timeoutSeconds);
  const effectiveExpectation = {
    gitopsRevision: expectation?.gitopsRevision ?? nonEmptyString(gitops.revision),
    runtimeImage: expectation?.runtimeImage ?? nonEmptyString(gitops.image) ?? expectedRuntimeImageFromRegistry(state, registry),
  };
  return {
    sourceMirror: probeSourceMirror(state, timeoutSeconds),
    registry,
    gitMirror: includeGitMirror
      ? runChildCli(["hwlab", "nodes", "git-mirror", "status", "--node", state.spec.nodeId, "--lane", state.spec.lane], timeoutSeconds)
      : { ok: true, skipped: true, reason: "deferred-to-async-flush", valuesRedacted: true },
    gitops,
    argo: probeArgoApplication(state, timeoutSeconds, effectiveExpectation.gitopsRevision),
    runtime: probeRuntimeObjects(state, timeoutSeconds, effectiveExpectation.runtimeImage),
    cadence: probeCadenceCronJob(state, timeoutSeconds, nonEmptyString(gitops.schedule)),
  };
}

function waitForSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation, includeGitMirror = true): SentinelObservedStatus {
  const startedAt = Date.now();
  const timeoutMs = Math.max(1_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000));
  let observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror);
  let polls = 1;
  while (!sentinelObservedReady(observed) && Date.now() - startedAt < timeoutMs) {
    const pollSeconds = numberAtNullable(state.cicd, "confirmWait.pollSeconds") ?? 2;
    runCommand(["sleep", String(pollSeconds)], repoRoot, { timeoutMs: (pollSeconds + 1) * 1_000 });
    observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror);
    polls += 1;
  }
  return {
    ...observed,
    wait: {
      polls,
      elapsedMs: Date.now() - startedAt,
      timeoutMs,
      ready: sentinelObservedReady(observed),
      includeGitMirror,
      valuesRedacted: true,
    },
  };
}

function sentinelObservedReady(value: Record<string, unknown> | SentinelObservedStatus): boolean {
  const observed = record(value);
  const gitMirror = record(observed.gitMirror);
  const gitMirrorReady = gitMirror.skipped === true || gitMirror.ok === true;
  return record(observed.sourceMirror).ok === true
    && record(record(observed.registry).probe).present === true
    && gitMirrorReady
    && record(observed.gitops).ok === true
    && record(observed.argo).ok === true
    && record(observed.runtime).ok === true
    && record(observed.cadence).ok === true;
}

export function sentinelDeliveryStatus(state: SentinelCicdState, observed: Record<string, unknown>, pipelineRun: string): Record<string, unknown> {
  const gitops = record(observed.gitops);
  const runtimeDeployment = record(record(observed.runtime).probe).deployment;
  const cadenceProbe = record(record(observed.cadence).probe);
  const argo = record(observed.argo);
  const desiredSourceCommit = nonEmptyString(gitops.sourceCommit) ?? state.sourceHead.commit;
  const runtimeSourceCommit = nonEmptyString(record(runtimeDeployment).sourceCommit);
  const desiredSchedule = nonEmptyString(gitops.schedule);
  const runtimeSchedule = nonEmptyString(cadenceProbe.schedule);
  const converging = desiredSourceCommit !== runtimeSourceCommit || desiredSchedule !== runtimeSchedule || !sentinelObservedReady(observed);
  return {
    desiredSourceCommit,
    runtimeSourceCommit,
    pipelineRun,
    gitopsRevision: gitops.revision ?? null,
    argoSyncStatus: argo.syncStatus ?? null,
    argoHealthStatus: argo.healthStatus ?? null,
    runtimeReady: record(observed.runtime).ok === true,
    desiredSchedule,
    runtimeSchedule,
    converging,
    warning: converging ? "GitOps 期望状态尚未完全收敛；这是只读、非阻塞 warning。" : null,
    blocking: false,
    outcome: converging ? "converging" : "converged",
    mutation: false,
    valuesRedacted: true,
  };
}

function sentinelObservedWarnings(value: Record<string, unknown> | SentinelObservedStatus | null): string[] {
  const observed = record(value);
  const argo = record(observed.argo);
  const cadence = record(observed.cadence);
  return mergeWarnings(argo.warning, cadence.warning);
}

function sentinelObservedStatusDiagnosis(state: SentinelCicdState, value: unknown, pipelineRun: string): Record<string, unknown> | null {
  const observed = record(value);
  if (sentinelObservedReady(observed)) return null;
  const sourceMirror = record(observed.sourceMirror);
  const registryProbe = record(record(observed.registry).probe);
  const gitMirror = record(observed.gitMirror);
  const gitops = record(observed.gitops);
  const argo = record(observed.argo);
  const runtimeDeployment = record(record(record(observed.runtime).probe).deployment);
  const cadence = record(observed.cadence);
  const sourceReady = sourceMirror.ok === true;
  const registryPresent = registryProbe.present === true;
  const gitMirrorReady = gitMirror.skipped === true || gitMirror.ok === true;
  const gitopsReady = gitops.ok === true;
  const argoReady = argo.ok === true;
  const runtimeReady = record(observed.runtime).ok === true;
  const cadenceReady = cadence.ok === true;
  const deliveryAuthority = webProbeSentinelDeliveryAuthority(state.spec);
  const legacyDelivery = deliveryAuthority.kind === "legacy-manual";
  const code = sourceReady && !registryPresent
    ? "sentinel-publish-half-state-registry-missing"
    : registryPresent && !gitopsReady
      ? "sentinel-gitops-manifest-not-updated"
      : gitopsReady && (!argoReady || !runtimeReady)
        ? "sentinel-runtime-not-aligned"
        : !gitMirrorReady
          ? "sentinel-git-mirror-not-in-sync"
          : !cadenceReady
            ? "sentinel-cadence-not-ready"
        : "sentinel-control-plane-observed-not-ready";
  const legacyReason = code === "sentinel-publish-half-state-registry-missing"
    ? "source mirror contains the selected commit, but the expected registry tag is missing; inspect PaC closeout/history before considering explicit manual recovery"
    : code === "sentinel-gitops-manifest-not-updated"
      ? "registry contains the selected image, but GitOps manifest is not yet updated to a digest-pinned runtime image"
      : code === "sentinel-runtime-not-aligned"
        ? "GitOps manifest is present, but Argo or runtime objects have not converged to the selected image yet"
        : code === "sentinel-git-mirror-not-in-sync"
          ? "runtime is aligned, but git-mirror status is not in sync; run the controlled git-mirror sync/status path before treating the control plane as fully healthy"
          : code === "sentinel-cadence-not-ready"
            ? "runtime is aligned, but cadence CronJob validation did not pass"
        : "one or more source, registry, GitOps, Argo, runtime or cadence checks did not pass";
  const reason = legacyDelivery
    ? legacyReason
    : `Web sentinel manual release observation is not aligned (${code}); inspect read-only status/history, fix the owning YAML, planner, controller, or source, then rerun release plan.`;
  const next = publishCurrentNext(state);
  const controlNext = controlPlaneNext(state, "trigger-current");
  const shouldRetryPublish = code === "sentinel-publish-half-state-registry-missing" || code === "sentinel-gitops-manifest-not-updated";
  return {
    code,
    phase: code === "sentinel-publish-half-state-registry-missing"
      ? "source-ready-registry-missing"
      : code === "sentinel-gitops-manifest-not-updated"
        ? "registry-ready-gitops-pending"
        : code === "sentinel-runtime-not-aligned"
          ? "gitops-ready-runtime-pending"
          : code === "sentinel-git-mirror-not-in-sync"
            ? "runtime-ready-git-mirror-pending"
            : code === "sentinel-cadence-not-ready"
              ? "runtime-ready-cadence-pending"
              : "observed-not-ready",
    reason,
    sourceMirror: sourceReady ? `ready ${short(record(sourceMirror.probe).commit ?? record(sourceMirror.probe).expectedCommit)}` : `blocked ${short(record(sourceMirror.probe).commit)}/${short(record(sourceMirror.probe).expectedCommit)}`,
    registry: registryPresent ? `present ${short(registryProbe.digest)}` : "missing -",
    gitMirror: gitMirrorReady ? "ready" : "pending",
    gitops: gitopsReady ? `ready ${short(gitops.image)}` : `pending ${short(gitops.image)}`,
    argo: `${argo.syncStatus ?? "-"} ${argo.healthStatus ?? "-"} ${short(argo.revision)}/${short(argo.expectedRevision)}`,
    runtime: `ready=${runtimeDeployment.readyReplicas ?? "-"}/${runtimeDeployment.desiredReplicas ?? "-"} image=${short(runtimeDeployment.image)} expected=${short(runtimeDeployment.expectedImage)}`,
    pipelineRun,
    deliveryAuthority,
    warning: !legacyDelivery
      ? `发布链未对齐；检查 ${next.pacStatus ?? next.status} 与 ${next.pacHistory ?? next.status}，按 ${next.fixManualRelease} 修复后重新 plan，范围准确时再手动 trigger。`
      : code === "sentinel-publish-half-state-registry-missing"
      ? `source mirror already exposes ${short(state.sourceHead.commit)} but registry tag ${state.image.tag} is missing; run ${next.pacCloseout} and inspect ${next.pacHistory}.`
      : code === "sentinel-git-mirror-not-in-sync"
        ? `runtime is aligned but git-mirror is not in sync; run ${next.gitMirrorSync} and then recheck ${next.controlPlaneStatus}.`
      : null,
    blocker: { code, reason, valuesRedacted: true },
    recoveryNext: legacyDelivery
      ? {
        reason,
        pipelineRun,
        digestRef: registryPresent ? expectedRuntimeImageFromRegistry(state, record(observed.registry)) : null,
        gitopsCommit: gitops.revision ?? null,
        pacCloseout: next.pacCloseout,
        publishCurrent: shouldRetryPublish ? next.manualRecovery : null,
        nextStatus: next.pacStatus,
        gitMirrorStatus: next.gitMirrorStatus,
        gitMirrorSync: !gitMirrorReady ? next.gitMirrorSync : null,
        gitMirrorFlush: null,
        controlPlaneApply: shouldRetryPublish || code === "sentinel-runtime-not-aligned" ? controlNext.apply : null,
        valuesRedacted: true,
      }
      : {
        reason,
        status: next.status,
        pacStatus: next.pacStatus,
        pacHistory: next.pacHistory,
        fixManualRelease: next.fixManualRelease,
        valuesRedacted: true,
      },
    valuesRedacted: true,
  };
}

function probeSourceMirror(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
  const result = probeSourceMirrorCache(state.cicd, state.controlPlaneNode, timeoutSeconds, state.sourceHead.commit);
  return { ...result, result: compactCommand(result.result) };
}

function probeSourceMirrorCache(cicd: Record<string, unknown>, controlPlaneNode: Record<string, unknown>, timeoutSeconds: number, expectedCommit: string | null): { ok: boolean; probe: Record<string, unknown>; result: CommandResult } {
  const namespace = stringAt(cicd, "builder.namespace");
  const repository = stringAt(cicd, "source.repository");
  const branch = stringAt(cicd, "source.branch");
  const stageRef = expectedCommit === null ? "" : sentinelSourceSnapshotRef(cicd, expectedCommit);
  const script = [
    "set +e",
    `repo_path=${shellQuote(`/cache/${repository}.git`)}`,
    `branch=${shellQuote(branch)}`,
    `expected=${shellQuote(expectedCommit ?? "")}`,
    `stage_ref=${shellQuote(stageRef)}`,
    "commit=$(kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" rev-parse \\\"refs/heads/$branch\\\" 2>/dev/null\" 2>/dev/null)",
    "rc=$?",
    "object_rc=1",
    "expected_object_rc=1",
    "stage_object_rc=1",
    "contains_rc=1",
    "if [ \"$rc\" -eq 0 ]; then",
    "  kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" cat-file -e \\\"$commit^{commit}\\\" 2>/dev/null\" >/dev/null 2>&1",
    "  object_rc=$?",
    "fi",
    "if [ -n \"$expected\" ]; then",
    "  kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" cat-file -e \\\"$expected^{commit}\\\" 2>/dev/null\" >/dev/null 2>&1",
    "  expected_object_rc=$?",
    "fi",
    "if [ -n \"$stage_ref\" ]; then",
    "  kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" rev-parse --verify \\\"$stage_ref^{commit}\\\" >/dev/null 2>&1\" >/dev/null 2>&1",
    "  stage_object_rc=$?",
    "fi",
    "if [ \"$rc\" -eq 0 ] && [ \"$expected_object_rc\" -eq 0 ]; then",
    "  kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" merge-base --is-ancestor \\\"$expected\\\" \\\"$commit\\\" 2>/dev/null\" >/dev/null 2>&1",
    "  contains_rc=$?",
    "fi",
    "node - \"$rc\" \"$object_rc\" \"$expected_object_rc\" \"$stage_object_rc\" \"$contains_rc\" \"$commit\" \"$expected\" \"$stage_ref\" \"$repo_path\" \"$branch\" <<'NODE'",
    "const [rc, objectRc, expectedObjectRc, stageObjectRc, containsRc, commit, expected, stageRef, repoPath, branch] = process.argv.slice(2);",
    "const present = Number(rc) === 0 && /^[0-9a-f]{40}$/i.test(commit || '');",
    "const objectPresent = present && Number(objectRc) === 0;",
    "const expectedObjectPresent = !expected || Number(expectedObjectRc) === 0;",
    "const stageObjectPresent = !stageRef || Number(stageObjectRc) === 0;",
    "const containsExpected = !expected || commit === expected || Number(containsRc) === 0;",
    "const relation = !expected ? 'unconstrained' : commit === expected ? 'equal' : containsExpected ? 'mirror-ahead' : expectedObjectPresent ? 'diverged-or-behind' : 'expected-object-missing';",
    "console.log(JSON.stringify({ ok: objectPresent && expectedObjectPresent && containsExpected, mode: 'internal-git-mirror-cache', present, objectPresent, expectedObjectPresent, stageObjectPresent, containsExpected, relation, commit: present ? commit : null, sourceCommit: expected || (present ? commit : null), mirrorCommit: present ? commit : null, expectedCommit: expected || null, stageRef: stageRef || null, branch, repoPath, persistentMirrorPresent: objectPresent && expectedObjectPresent, readUrl: process.env.SOURCE_GIT_MIRROR_READ_URL || null, valuesRedacted: true }));",
    "NODE",
  ].join("\n");
  const result = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", `export SOURCE_GIT_MIRROR_READ_URL=${shellQuote(stringAt(cicd, "source.gitMirrorReadUrl"))}\n${script}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
  const probeResolution = resolveSentinelChildJson(result, "web-probe-sentinel-source-mirror-cache-probe");
  const probe = probeResolution.parsed ?? {};
  return { ok: result.exitCode === 0 && probe.ok === true, probe: { ...probe, stdoutRecovery: probeResolution.diagnostics, valuesRedacted: true }, result };
}

function probeArgoApplication(state: SentinelCicdState, timeoutSeconds: number, expectedRevision: string | null): Record<string, unknown> {
  const namespace = stringAt(state.cicd, "argo.namespace");
  const applicationName = stringAt(state.cicd, "argo.applicationName");
  const jsonpath = "{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}{.status.sync.revision}{\"\\n\"}";
  const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", namespace, "get", "application", applicationName, "-o", `jsonpath=${jsonpath}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
  const [syncStatusRaw, healthStatusRaw, revisionRaw] = result.stdout.trim().split(/\r?\n/u);
  const syncStatus = nonEmptyString(syncStatusRaw);
  const healthStatus = nonEmptyString(healthStatusRaw);
  const revision = nonEmptyString(revisionRaw);
  const revisionMatches = expectedRevision === null || revision === expectedRevision;
  const healthy = result.exitCode === 0 && syncStatus === "Synced" && healthStatus === "Healthy";
  const diagnostics = result.exitCode === 0 && !healthy
    ? probeArgoApplicationDiagnostics(state, timeoutSeconds, namespace, applicationName)
    : null;
  return {
    ok: healthy,
    present: result.exitCode === 0,
    syncStatus,
    healthStatus,
    revision,
    expectedRevision,
    revisionMatches,
    revisionPolicy: "non-blocking-branch-head-drift",
    warning: healthy && !revisionMatches
      ? "Argo app is Synced/Healthy but status.sync.revision differs from current GitOps branch HEAD; in multi-sentinel GitOps this can happen when another sentinel path advances the branch. Runtime image/manifest checks remain authoritative for rollout readiness."
      : null,
    diagnostics,
    result: compactCommand(result),
  };
}

function probeArgoApplicationDiagnostics(state: SentinelCicdState, timeoutSeconds: number, namespace: string, applicationName: string): Record<string, unknown> {
  const route = stringAt(state.controlPlaneNode, "kubeRoute");
  const appResult = runCommand(["trans", route, "kubectl", "-n", namespace, "get", "application", applicationName, "-o", "json"], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
  const appResolution = resolveSentinelChildJson(appResult, "web-probe-sentinel-argo-application-json");
  const parsed = appResolution.parsed;
  const status = record(parsed?.status);
  const resourcesRaw = Array.isArray(status.resources) ? status.resources : [];
  const resources = resourcesRaw
    .filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item))
    .map((item) => {
      const health = record(item.health);
      return {
        kind: item.kind ?? null,
        namespace: item.namespace ?? null,
        name: item.name ?? null,
        status: item.status ?? null,
        healthStatus: health.status ?? null,
        healthMessage: short(health.message),
      };
    });
  const problemResources = resources
    .filter((item) => {
      const sync = nonEmptyString(item.status);
      const health = nonEmptyString(item.healthStatus);
      return (sync !== null && sync !== "Synced") || (health !== null && health !== "Healthy");
    })
    .slice(0, 12);
  const conditions = (Array.isArray(status.conditions) ? status.conditions : [])
    .filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item))
    .slice(-8)
    .map((item) => ({
      type: item.type ?? null,
      message: short(item.message),
      lastTransitionTime: item.lastTransitionTime ?? null,
    }));
  const operationState = record(status.operationState);
  const eventResult = runCommand(["trans", route, "kubectl", "-n", namespace, "get", "events", "--field-selector", `involvedObject.name=${applicationName}`, "--sort-by=.lastTimestamp", "-o", "json"], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
  const eventResolution = resolveSentinelChildJson(eventResult, "web-probe-sentinel-argo-events-json");
  const eventsJson = eventResolution.parsed;
  const events = (Array.isArray(eventsJson?.items) ? eventsJson.items : [])
    .filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item))
    .slice(-8)
    .map((item) => ({
      type: item.type ?? null,
      reason: item.reason ?? null,
      message: short(item.message),
      count: item.count ?? null,
      lastTimestamp: item.lastTimestamp ?? item.eventTime ?? null,
    }));
  return {
    ok: appResult.exitCode === 0,
    resourceCount: resources.length,
    problemResourceCount: problemResources.length,
    problemResources,
    conditions,
    operationState: {
      phase: operationState.phase ?? null,
      message: short(operationState.message),
      startedAt: operationState.startedAt ?? null,
      finishedAt: operationState.finishedAt ?? null,
    },
    events,
    result: compactCommand(appResult),
    eventsResult: compactCommand(eventResult),
    stdoutRecovery: {
      application: appResolution.diagnostics,
      events: eventResolution.diagnostics,
      valuesRedacted: true,
    },
    drillDown: {
      application: `trans ${route} kubectl -n ${namespace} get application ${applicationName} -o json`,
      events: `trans ${route} kubectl -n ${namespace} get events --field-selector involvedObject.name=${applicationName} --sort-by=.lastTimestamp`,
    },
  };
}

function probeGitopsRuntimeManifest(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
  const namespace = stringAt(state.cicd, "builder.namespace");
  const repository = stringAt(state.controlPlaneTarget, "source.repository");
  const branch = stringAt(state.cicd, "argo.targetRevision");
  const manifestPath = `${stringAt(state.cicd, "gitopsPath")}/web-probe-sentinel.yaml`;
  const repoPath = `/cache/${repository}.git`;
  const inner = [
    "set -eu",
    `git --git-dir=${shellQuote(repoPath)} rev-parse ${shellQuote(`refs/heads/${branch}`)}`,
    `git --git-dir=${shellQuote(repoPath)} show ${shellQuote(`refs/heads/${branch}:${manifestPath}`)}`,
  ].join("\n");
  const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", namespace, "exec", "deploy/git-mirror-http", "--", "sh", "-lc", inner], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
  const [revisionLine, ...manifestLines] = result.stdout.split(/\r?\n/u);
  const revision = /^[0-9a-f]{40}$/iu.test(revisionLine?.trim() ?? "") ? revisionLine.trim() : null;
  const manifest = manifestLines.join("\n");
  const image = nonEmptyString(manifest.match(/image:\s*([^,\s}\]]+)/u)?.[1]);
  const sourceCommit = nonEmptyString(manifest.match(/unidesk\.ai\/source-commit:\s*["']?([0-9a-f]{40})/iu)?.[1]);
  const schedule = nonEmptyString(manifest.match(/\bschedule:\s*["']?([^,"'\r\n#]+)/u)?.[1]?.trim());
  const imageMatchesRepository = image !== null && image.startsWith(`${state.image.repository}@sha256:`);
  const compact = compactCommand(result);
  return {
    ok: result.exitCode === 0 && revision !== null && imageMatchesRepository,
    revision,
    branch,
    manifestPath,
    image,
    sourceCommit,
    schedule,
    imageMatchesRepository,
    result: { ...compact, stdoutPreview: `${revision ?? "-"} ${image ?? "-"}` },
    valuesRedacted: true,
  };
}

function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number, expectedImage: string | null): Record<string, unknown> {
  const namespace = stringAt(state.runtime, "namespace");
  const deploymentName = stringAt(state.runtime, "deploymentName");
  const serviceName = stringAt(state.runtime, "serviceName");
  const pvcName = stringAt(state.runtime, "pvcName");
  const configMapName = `${deploymentName}-config`;
  const serviceAccountName = stringAt(state.runtime, "serviceAccountName");
  const script = [
    "set +e",
    `namespace=${shellQuote(namespace)}`,
    `deployment=${shellQuote(deploymentName)}`,
    `service=${shellQuote(serviceName)}`,
    `pvc=${shellQuote(pvcName)}`,
    `configmap=${shellQuote(configMapName)}`,
    `serviceaccount=${shellQuote(serviceAccountName)}`,
    "tmp=$(mktemp -d)",
    "kubectl -n \"$namespace\" get deploy \"$deployment\" -o json >\"$tmp/deploy.json\" 2>/dev/null; echo $? >\"$tmp/deploy.rc\"",
    "kubectl -n \"$namespace\" get svc \"$service\" -o json >\"$tmp/svc.json\" 2>/dev/null; echo $? >\"$tmp/svc.rc\"",
    "kubectl -n \"$namespace\" get pvc \"$pvc\" -o json >\"$tmp/pvc.json\" 2>/dev/null; echo $? >\"$tmp/pvc.rc\"",
    "kubectl -n \"$namespace\" get cm \"$configmap\" -o json >\"$tmp/cm.json\" 2>/dev/null; echo $? >\"$tmp/cm.rc\"",
    "kubectl -n \"$namespace\" get sa \"$serviceaccount\" -o json >\"$tmp/sa.json\" 2>/dev/null; echo $? >\"$tmp/sa.rc\"",
    `expected_image=${shellQuote(expectedImage ?? "")}`,
    "node - \"$tmp\" \"$expected_image\" <<'NODE'",
    "const fs = require('node:fs');",
    "const dir = process.argv[2];",
    "const expectedImage = process.argv[3] || null;",
    "function rc(name){ return Number(fs.readFileSync(`${dir}/${name}.rc`, 'utf8').trim()); }",
    "function json(name){ try { return JSON.parse(fs.readFileSync(`${dir}/${name}.json`, 'utf8')); } catch { return null; } }",
    "const dep = json('deploy');",
    "const deploymentPresent = rc('deploy') === 0;",
    "const desired = Number(dep?.spec?.replicas ?? 0);",
    "const ready = Number(dep?.status?.readyReplicas ?? 0);",
    "const updated = Number(dep?.status?.updatedReplicas ?? 0);",
    "const image = dep?.spec?.template?.spec?.containers?.[0]?.image ?? null;",
    "const sourceCommit = dep?.spec?.template?.metadata?.annotations?.['unidesk.ai/source-commit'] ?? dep?.metadata?.annotations?.['unidesk.ai/source-commit'] ?? null;",
    "const imageMatches = expectedImage === null || image === expectedImage;",
    "const payload = {",
    "  deployment: { present: deploymentPresent, desiredReplicas: desired, readyReplicas: ready, updatedReplicas: updated, image, expectedImage, imageMatches, sourceCommit },",
    "  service: { present: rc('svc') === 0 },",
    "  pvc: { present: rc('pvc') === 0, phase: json('pvc')?.status?.phase ?? null },",
    "  configMap: { present: rc('cm') === 0 },",
    "  serviceAccount: { present: rc('sa') === 0 },",
    "  valuesRedacted: true",
    "};",
    "payload.ok = payload.deployment.present && imageMatches && ready >= Math.max(1, desired) && updated >= Math.max(1, desired) && payload.service.present && payload.pvc.present && payload.pvc.phase === 'Bound' && payload.configMap.present && payload.serviceAccount.present;",
    "console.log(JSON.stringify(payload));",
    "NODE",
  ].join("\n");
  const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
  const probeResolution = resolveSentinelChildJson(result, "web-probe-sentinel-runtime-objects-probe");
  const probe = probeResolution.parsed;
  return { ok: result.exitCode === 0 && probe?.ok === true, probe, stdoutRecovery: probeResolution.diagnostics, result: compactCommand(result) };
}

function probeCadenceCronJob(state: SentinelCicdState, timeoutSeconds: number, desiredSchedule: string | null = null): Record<string, unknown> {
  const expected = state.manifests.find((item) => item.kind === "CronJob") ?? null;
  if (expected === null) {
    return { ok: true, skipped: true, reason: "targetValidation.cadenceScheduler.disabled", valuesRedacted: true };
  }
  const metadata = record(expected.metadata);
  const spec = record(expected.spec);
  const namespace = stringAt(metadata, "namespace");
  const name = stringAt(metadata, "name");
  const expectedSchedule = desiredSchedule ?? stringAt(spec, "schedule");
  const script = [
    "set +e",
    `namespace=${shellQuote(namespace)}`,
    `cronjob=${shellQuote(name)}`,
    `sentinel=${shellQuote(state.sentinelId)}`,
    `expected_schedule=${shellQuote(expectedSchedule)}`,
    "tmp=$(mktemp -d)",
    "kubectl -n \"$namespace\" get cronjob \"$cronjob\" -o json >\"$tmp/cronjob.json\" 2>/dev/null; echo $? >\"$tmp/cronjob.rc\"",
    "kubectl -n \"$namespace\" get jobs -l \"unidesk.ai/web-probe-sentinel-id=$sentinel,app.kubernetes.io/component=cadence-scheduler\" -o json >\"$tmp/jobs.json\" 2>/dev/null; echo $? >\"$tmp/jobs.rc\"",
    "node - \"$tmp\" \"$namespace\" \"$cronjob\" \"$expected_schedule\" <<'NODE'",
    "const fs = require('node:fs');",
    "const [dir, namespace, cronJobName, expectedSchedule] = process.argv.slice(2);",
    "function rc(name){ try { return Number(fs.readFileSync(`${dir}/${name}.rc`, 'utf8').trim()); } catch { return 1; } }",
    "function json(name){ try { return JSON.parse(fs.readFileSync(`${dir}/${name}.json`, 'utf8')); } catch { return null; } }",
    "const cron = json('cronjob');",
    "const jobs = Array.isArray(json('jobs')?.items) ? json('jobs').items : [];",
    "const present = rc('cronjob') === 0 && !!cron;",
    "const schedule = cron?.spec?.schedule || null;",
    "const scheduleMatches = present && schedule === expectedSchedule;",
    "const suspended = cron?.spec?.suspend === true;",
    "const active = Array.isArray(cron?.status?.active) ? cron.status.active.length : 0;",
    "const sortedJobs = jobs.slice().sort((a,b)=>String(b?.metadata?.creationTimestamp||'').localeCompare(String(a?.metadata?.creationTimestamp||''))).slice(0,8);",
    "let code = null;",
    "if (!present) code = 'sentinel-cadence-cronjob-missing';",
    "else if (!scheduleMatches) code = 'sentinel-cadence-cronjob-schedule-mismatch';",
    "else if (suspended) code = 'sentinel-cadence-cronjob-suspended';",
    "const latestJob = sortedJobs[0] || null;",
    "console.log(JSON.stringify({ ok: code === null, code, present, namespace, name: cronJobName, schedule, expectedSchedule, scheduleMatches, suspended, lastScheduleTime: cron?.status?.lastScheduleTime || null, lastSuccessfulTime: cron?.status?.lastSuccessfulTime || null, active, jobCount: jobs.length, latestJobs: sortedJobs.map((job)=>({ name: job?.metadata?.name || null, createdAt: job?.metadata?.creationTimestamp || null, active: Number(job?.status?.active || 0), succeeded: Number(job?.status?.succeeded || 0), failed: Number(job?.status?.failed || 0), completionTime: job?.status?.completionTime || null, valuesRedacted:true })), latestJobName: latestJob?.metadata?.name || null, valuesRedacted: true }));",
    "NODE",
  ].join("\n");
  const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
  const probeResolution = resolveSentinelChildJson(result, "web-probe-sentinel-cadence-cronjob-probe");
  const probe = probeResolution.parsed;
  const ok = result.exitCode === 0 && probe?.ok === true;
  emitWebProbeSentinelSpan({
    node: state.spec.nodeId,
    lane: state.spec.lane,
    sentinelId: state.sentinelId,
    namespace,
    runtime: state.runtime,
    cicd: state.cicd,
  }, "web_probe_sentinel.cadence.cronjob_observed", {
    cronJobName: name,
    namespace,
    schedule: expectedSchedule,
    status: ok ? "ok" : text(probe?.code ?? "unknown"),
    jobName: probe?.latestJobName ?? null,
    failureKind: probe?.code ?? null,
    valuesRedacted: true,
  }, ok);
  return {
    ok,
    probe,
    stdoutRecovery: probeResolution.diagnostics,
    result: compactCommand(result),
    warning: ok ? null : `cadence CronJob is not ready: ${text(probe?.code ?? "probe-failed")}`,
    valuesRedacted: true,
  };
}

function expectedRuntimeImageFromRegistry(state: SentinelCicdState, registry: Record<string, unknown>): string | null {
  const digest = nonEmptyString(record(record(registry).probe).digest);
  if (digest === null) return null;
  return `${state.image.repository}@${digest}`;
}

function targetValidationDeferredWarnings(state: SentinelCicdState, applyOnly: boolean, budgetSeconds: number): string[] {
  if (applyOnly) return [];
  const next = sentinelP5Next(state);
  return [`targetValidation quick verify is outside the CI/CD validation gate; run ${next.quickVerify} only as separate post-deploy evidence if needed.`];
}

export function targetValidationElapsedWarnings(value: unknown, subject: string, budgetSeconds: number): string[] {
  const elapsedMs = typeof value === "number" && Number.isFinite(value) ? value : null;
  const budgetMs = Math.max(1, Math.trunc(budgetSeconds)) * 1000;
  if (elapsedMs === null || elapsedMs <= budgetMs) return [];
  return [`${subject} exceeded configured ${Math.round(budgetMs / 1000)}s targetValidation budget (${Math.round(elapsedMs / 1000)}s); non-blocking timing alert, only Code Agent multi-round business failures should block acceptance.`];
}

export function mergeWarnings(...items: readonly (readonly unknown[] | unknown)[]): string[] {
  const warnings: string[] = [];
  for (const item of items) {
    const values = Array.isArray(item) ? item : [item];
    for (const value of values) {
      if (value === undefined || value === null || value === "") continue;
      const warning = text(value).trim();
      if (warning.length > 0 && warning !== "-" && !warnings.includes(warning)) warnings.push(warning);
    }
  }
  return warnings;
}

export function withWarnings(payload: Record<string, unknown>, warnings: readonly unknown[]): Record<string, unknown> {
  const merged = mergeWarnings(payload.warnings, warnings);
  return merged.length === 0 ? payload : { ...payload, warnings: merged, valuesRedacted: true };
}

function installSentinelPublishInterruptHandler(state: SentinelCicdState, context: Record<string, unknown>): () => void {
  let handled = false;
  const handler = (signal: string) => {
    if (handled) return;
    handled = true;
    const exitCode = signal === "SIGINT" ? 130 : 143;
    const next = publishCurrentNext(state);
    sentinelProgressEvent("sentinel.publish.interrupted", {
      signal,
      exitCode,
      phase: context.phase ?? "unknown",
      pipelineRun: context.pipelineRun ?? sentinelPipelineRunName(state, false),
      sourceCommit: context.sourceCommit ?? state.sourceHead.commit,
      node: state.spec.nodeId,
      lane: state.spec.lane,
      sentinelId: state.sentinelId,
      recoveryNext: {
        pacCloseout: next.pacCloseout,
        pacStatus: next.pacStatus,
        retry: next.manualRecovery,
        gitMirrorStatus: next.gitMirrorStatus,
        gitMirrorSync: next.gitMirrorSync,
        gitMirrorFlush: next.gitMirrorFlush,
        valuesRedacted: true,
      },
      valuesRedacted: true,
    });
    process.exit(exitCode);
  };
  process.once("SIGTERM", handler);
  process.once("SIGINT", handler);
  return () => {
    process.off("SIGTERM", handler);
    process.off("SIGINT", handler);
  };
}

function confirmBlocked(action: string, state: SentinelCicdState): Record<string, unknown> {
  return {
    code: "sentinel-cicd-confirm-requires-tekton-pipelinerun",
    action,
    reason: "Confirmed publish uses a Tekton PipelineRun for image build and GitOps writeback; dry-run refuses to report a deployment mutation before that CI run is submitted.",
    sourceGitMirrorReadUrl: stringAt(state.cicd, "source.gitMirrorReadUrl"),
    requiredNextImplementation: [
      "clone source from source.gitMirrorReadUrl at selected commit",
      "build and push digest-pinned image through Tekton PipelineRun on the selected node",
      "publish manifests to the HWLAB gitops branch/path through git-mirror",
      "flush/recheck git-mirror and let Argo reconcile the Application",
    ],
    valuesRedacted: true,
  };
}

function controlPlaneNext(state: SentinelCicdState, action: WebProbeSentinelControlPlaneAction): Record<string, string> {
  const deliveryAuthority = webProbeSentinelDeliveryAuthority(state.spec);
  if (deliveryAuthority.kind !== "legacy-manual") return sentinelReadOnlyNext(state, deliveryAuthority);
  const node = state.spec.nodeId;
  const lane = state.spec.lane;
  const suffix = sentinelCliSuffix(state);
  const consumer = sentinelPacConsumerId(state.spec);
  const sourceCommit = state.sourceHead.commit === null ? "" : ` --source-commit ${state.sourceHead.commit}`;
  return {
    pacCloseout: `bun scripts/cli.ts platform-infra pipelines-as-code closeout --target ${node} --consumer ${consumer}${sourceCommit} --wait`,
    pacStatus: `bun scripts/cli.ts platform-infra pipelines-as-code status --target ${node} --consumer ${consumer}`,
    pacHistory: `bun scripts/cli.ts platform-infra pipelines-as-code history --target ${node} --consumer ${consumer} --limit 10`,
    plan: `bun scripts/cli.ts web-probe sentinel control-plane plan --node ${node} --lane ${lane}${suffix} --dry-run`,
    status: `bun scripts/cli.ts web-probe sentinel control-plane status --node ${node} --lane ${lane}${suffix}`,
    image: `bun scripts/cli.ts web-probe sentinel image status --node ${node} --lane ${lane}${suffix}`,
    triggerCurrent: `bun scripts/cli.ts web-probe sentinel control-plane trigger-current --node ${node} --lane ${lane}${suffix} --dry-run`,
    apply: `bun scripts/cli.ts web-probe sentinel control-plane apply --node ${node} --lane ${lane}${suffix} --confirm --wait`,
    validate: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix}`,
    quickVerify: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix} --quick-verify --confirm --wait`,
    gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${node} --lane ${lane}`,
    gitMirrorSync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${node} --lane ${lane} --confirm`,
    gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${node} --lane ${lane} --confirm --wait`,
    issue: "https://github.com/pikasTech/unidesk/issues/1285",
    currentAction: action,
  };
}

function controlPlaneRecoveryNext(state: SentinelCicdState, ok: boolean, publish: unknown, flush: unknown, observed: unknown): Record<string, unknown> | null {
  const payload = record(record(publish).payload);
  if (ok || nonEmptyString(payload.digestRef) === null) return null;
  const deliveryAuthority = webProbeSentinelDeliveryAuthority(state.spec);
  if (deliveryAuthority.kind !== "legacy-manual") {
    return {
      reason: "平台维护后发布链仍未对齐；只读检查 sentinel/PaC 状态并修复 owning YAML、planner 或 controller，重新 plan；不得裸 publish、sync 或 flush。",
      ...sentinelReadOnlyNext(state, deliveryAuthority),
      valuesRedacted: true,
    };
  }
  const next = controlPlaneNext(state, "apply");
  const flushRecord = record(flush);
  const observedRecord = record(observed);
  return {
    reason: "publish produced an image digest, but GitOps/git-mirror/Argo/runtime alignment is not complete yet",
    pipelineRun: record(publish).jobName ?? null,
    digestRef: payload.digestRef,
    gitopsCommit: payload.gitopsCommit ?? null,
    flushMode: flushRecord.mode ?? null,
    observedReady: sentinelObservedReady(observedRecord),
    pacCloseout: next.pacCloseout,
    publishCurrent: `bun scripts/cli.ts web-probe sentinel publish-current --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm --wait --manual-recovery --reason <reason>`,
    nextStatus: next.pacStatus,
    gitMirrorStatus: next.gitMirrorStatus,
    gitMirrorSync: next.gitMirrorSync,
    gitMirrorFlush: next.gitMirrorFlush,
    controlPlaneApply: next.apply,
    valuesRedacted: true,
  };
}

function controlPlaneDrillDown(state: SentinelCicdState, pipelineRun: unknown, flush: unknown): Record<string, unknown> {
  const pipelineRunName = nonEmptyString(pipelineRun);
  const deliveryAuthority = webProbeSentinelDeliveryAuthority(state.spec);
  const next = controlPlaneNext(state, "status");
  const pacBase = "bun scripts/cli.ts platform-infra pipelines-as-code";
  const flushRecord = record(flush);
  const flushNext = record(flushRecord.next);
  const flushJob = record(flushRecord.job);
  const readOnly = {
    pipelineRun: pipelineRunName,
    pipelineRunStatus: pipelineRunName === null
      ? null
      : `${pacBase} history --target ${state.spec.nodeId} --id ${pipelineRunName}`,
    pipelineRunFull: pipelineRunName === null
      ? null
      : `${pacBase} history --target ${state.spec.nodeId} --id ${pipelineRunName} --full`,
    consumerStatus: `${pacBase} status --target ${state.spec.nodeId} --consumer sentinel-${state.spec.nodeId.toLowerCase()}-${state.spec.lane}`,
    controlPlaneStatus: next.controlPlaneStatus ?? next.status,
    fixManualRelease: next.fixManualRelease ?? PAC_AUTOMATIC_DELIVERY_REFERENCE,
    valuesRedacted: true,
  };
  if (deliveryAuthority.kind !== "legacy-manual") return readOnly;
  return {
    ...readOnly,
    imageStatus: `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`,
    gitMirrorStatus: next.gitMirrorStatus,
    gitMirrorFlush: next.gitMirrorFlush,
    asyncFlushJob: nonEmptyString(flushJob.id),
    asyncFlushStatus: nonEmptyString(flushNext.status),
  };
}

function applySentinelRuntimeSecrets(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
  const sourcesByPurpose = new Map<string, Record<string, unknown>>();
  for (const source of arrayAt(state.secrets, "sources").map(record)) {
    const purpose = stringAtNullable(source, "purpose");
    if (purpose !== null) sourcesByPurpose.set(purpose, source);
  }
  const desired: Array<{ namespace: string; name: string; data: Array<{ key: string; value: string; sourcePurpose: string; sourceRef: string; sourceKey: string; fingerprint: string }> }> = [];
  const missing: Record<string, unknown>[] = [];
  const skipped: Record<string, unknown>[] = [];
  for (const runtimeSecret of arrayAt(state.secrets, "runtimeSecrets").map(record)) {
    const name = stringAt(runtimeSecret, "name");
    const namespace = stringAt(runtimeSecret, "namespace");
    const data: Array<{ key: string; value: string; sourcePurpose: string; sourceRef: string; sourceKey: string; fingerprint: string }> = [];
    for (const item of arrayAt(runtimeSecret, "data").map(record)) {
      const sourcePurpose = stringAt(item, "sourcePurpose");
      const targetKey = stringAt(item, "targetKey");
      if (sourcePurpose === "frp-token") {
        skipped.push({ name, namespace, targetKey, sourcePurpose, reason: "managed-by-publicExposure-frpc", valuesRedacted: true });
        continue;
      }
      const source = sourcesByPurpose.get(sourcePurpose);
      if (source === undefined) {
        missing.push({ name, namespace, targetKey, sourcePurpose, reason: "source-purpose-missing", valuesRedacted: true });
        continue;
      }
      const sourceRef = stringAt(source, "sourceRef");
      const sourceKey = stringAt(source, "sourceKey");
      const material = readSentinelSecretSourceValue(source);
      if (!material.ok) {
        missing.push({ name, namespace, targetKey, sourcePurpose, sourceRef, sourceKey, reason: material.error, sourcePath: material.sourcePath, valuesRedacted: true });
        continue;
      }
      const value = stringAt(material, "value");
      data.push({
        key: targetKey,
        value,
        sourcePurpose,
        sourceRef,
        sourceKey,
        fingerprint: `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`,
      });
    }
    if (data.length > 0) desired.push({ namespace, name, data });
  }
  if (missing.length > 0) return { ok: false, phase: "local-source", missing, skipped, valuesRedacted: true };
  if (desired.length === 0) return { ok: true, phase: "skipped-no-runtime-secrets", secretCount: 0, keyCount: 0, skippedKeyCount: skipped.length, skipped, valuesRedacted: true };
  const manifests = desired.map((secret) => ({
    apiVersion: "v1",
    kind: "Secret",
    metadata: {
      name: secret.name,
      namespace: secret.namespace,
      labels: {
        "app.kubernetes.io/managed-by": "unidesk",
        "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
        "unidesk.ai/node": state.spec.nodeId,
        "unidesk.ai/lane": state.spec.lane,
        "unidesk.ai/web-probe-sentinel-id": state.sentinelId,
      },
    },
    type: "Opaque",
    data: Object.fromEntries(secret.data.map((item) => [item.key, Buffer.from(item.value, "utf8").toString("base64")])),
  }));
  const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`;
  const summary = desired.map((secret) => ({
    name: secret.name,
    namespace: secret.namespace,
    keys: secret.data.map((item) => item.key).sort(),
    sources: secret.data.map((item) => ({ key: item.key, sourcePurpose: item.sourcePurpose, sourceRef: item.sourceRef, sourceKey: item.sourceKey, fingerprint: item.fingerprint, valuesRedacted: true })),
    valuesRedacted: true,
  }));
  const summaryB64 = Buffer.from(JSON.stringify(summary), "utf8").toString("base64");
  const script = [
    "set +e",
    `summary_b64=${shellQuote(summaryB64)}`,
    "tmp=$(mktemp -d)",
    "trap 'rm -rf \"$tmp\"' EXIT",
    "manifest=\"$tmp/runtime-secrets.yaml\"",
    "cat >\"$manifest\"",
    "kubectl apply --server-side --force-conflicts --field-manager=unidesk-web-probe-sentinel-runtime-secrets -f \"$manifest\" >/tmp/web-probe-sentinel-runtime-secrets.out 2>/tmp/web-probe-sentinel-runtime-secrets.err",
    "apply_rc=$?",
    "python3 - \"$summary_b64\" \"$apply_rc\" <<'PY'",
    "import base64, json, subprocess, sys",
    "summary = json.loads(base64.b64decode(sys.argv[1]).decode('utf-8'))",
    "apply_rc = int(sys.argv[2])",
    "items = []",
    "ok = apply_rc == 0",
    "for secret in summary:",
    "    proc = subprocess.run(['kubectl', '-n', secret['namespace'], 'get', 'secret', secret['name'], '-o', 'json'], text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)",
    "    data = json.loads(proc.stdout).get('data', {}) if proc.returncode == 0 and proc.stdout else {}",
    "    keys = {key: key in data for key in secret['keys']}",
    "    present = proc.returncode == 0 and all(keys.values())",
    "    ok = ok and present",
    "    items.append({'name': secret['name'], 'namespace': secret['namespace'], 'present': present, 'keys': keys, 'sources': secret['sources'], 'valuesRedacted': True})",
    "print(json.dumps({'ok': ok, 'applyExitCode': apply_rc, 'secretCount': len(summary), 'keyCount': sum(len(item['keys']) for item in summary), 'items': items, 'valuesRedacted': True}, ensure_ascii=False))",
    "PY",
  ].join("\n");
  const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { input: manifestYaml, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
  const parsedResolution = resolveSentinelChildJson(result, "web-probe-sentinel-runtime-secrets-apply");
  const parsed = parsedResolution.parsed;
  return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), skippedKeyCount: skipped.length, skipped, stdoutRecovery: parsedResolution.diagnostics, result: compactCommand(result), valuesRedacted: true };
}

function readSentinelSecretSourceValue(source: Record<string, unknown>): Record<string, unknown> {
  const sourceRef = stringAt(source, "sourceRef");
  const sourceKey = stringAt(source, "sourceKey");
  const sourceLine = numberAtNullable(source, "sourceLine");
  const paths = secretSourcePaths(sourceRef);
  const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? rootPath(".state", "secrets", sourceRef);
  if (!existsSync(sourcePath)) return { ok: false, error: "secret-source-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true };
  const textValue = readFileSync(sourcePath, "utf8");
  const value = sourceLine === null ? parseEnvFile(textValue)[sourceKey] : textValue.split(/\r?\n/u)[sourceLine - 1]?.replace(/\r$/u, "");
  if (value === undefined || value.length === 0) return { ok: false, error: sourceLine === null ? "secret-source-key-missing" : "secret-source-line-missing", sourceRef, sourceKey, sourceLine, sourcePath: displayPath(sourcePath), valuesRedacted: true };
  const format = stringAtNullable(source, "format");
  if (format === null) return { ok: true, value, sourceRef, sourceKey, sourceLine, sourcePath: displayPath(sourcePath), valuesRedacted: true };
  if (format === "web-account-json") {
    const username = readSentinelWebAccountUsername(source);
    if (!username.ok) return { ok: false, error: username.error, sourceRef, sourceKey, sourceLine, sourcePath: displayPath(sourcePath), valuesRedacted: true };
    return { ok: true, value: JSON.stringify({ username: username.value, password: value }), sourceRef, sourceKey, sourceLine, format, sourcePath: displayPath(sourcePath), valuesRedacted: true };
  }
  return { ok: false, error: "unsupported-secret-source-format", sourceRef, sourceKey, format, sourcePath: displayPath(sourcePath), valuesRedacted: true };
}

function readSentinelWebAccountUsername(source: Record<string, unknown>): { ok: true; value: string } | { ok: false; error: string } {
  const username = stringAtNullable(source, "username");
  if (username !== null) return { ok: true, value: username };
  const sourceRef = stringAtNullable(source, "usernameSourceRef");
  const sourceLine = numberAtNullable(source, "usernameSourceLine");
  if (sourceRef === null || sourceLine === null) return { ok: false, error: "web-account-json-username-missing" };
  const paths = secretSourcePaths(sourceRef);
  const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? rootPath(".state", "secrets", sourceRef);
  if (!existsSync(sourcePath)) return { ok: false, error: "web-account-json-username-source-missing" };
  const value = readFileSync(sourcePath, "utf8").split(/\r?\n/u)[sourceLine - 1]?.replace(/\r$/u, "") ?? "";
  return value.length === 0 ? { ok: false, error: "web-account-json-username-line-missing" } : { ok: true, value };
}

function applySentinelPublicExposure(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
  const material = readSentinelFrpcMaterial(state);
  if (!material.ok) return { ok: false, hostname: stringAt(state.publicExposure, "hostname"), material, valuesRedacted: true };
  const secret = applySentinelFrpcSecret(state, stringAt(material, "frpcToml"), timeoutSeconds);
  const caddy = applySentinelCaddyBlock(state, timeoutSeconds);
  return {
    ok: secret.ok === true && caddy.ok === true,
    hostname: stringAt(state.publicExposure, "hostname"),
    publicBaseUrl: stringAt(state.publicExposure, "publicBaseUrl"),
    material: {
      ok: true,
      sourceRef: material.sourceRef,
      sourcePath: material.sourcePath,
      fingerprint: material.fingerprint,
      valuesRedacted: true,
    },
    secret,
    caddy,
    valuesRedacted: true,
  };
}

function readSentinelFrpcMaterial(state: SentinelCicdState): Record<string, unknown> {
  const sourceRef = stringAt(state.publicExposure, "frpc.tokenSourceRef");
  const sourceKey = stringAt(state.publicExposure, "frpc.tokenSourceKey");
  const paths = secretSourcePaths(sourceRef);
  const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? rootPath(".state", "secrets", sourceRef);
  if (!existsSync(sourcePath)) return { ok: false, error: "frp-token-source-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true };
  const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
  const token = values[sourceKey];
  if (token === undefined || token.length === 0) return { ok: false, error: "frp-token-key-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true };
  const proxy = record(valueAtPath(state.publicExposure, "frpc.httpProxy"));
  const frpcToml = [
    `serverAddr = "${tomlEscape(stringAt(state.publicExposure, "frpc.serverAddr"))}"`,
    `serverPort = ${numberAt(state.publicExposure, "frpc.serverPort")}`,
    "loginFailExit = true",
    `auth.token = "${tomlEscape(token)}"`,
    "",
    "[[proxies]]",
    `name = "${tomlEscape(stringAt(proxy, "name"))}"`,
    'type = "tcp"',
    `localIP = "${tomlEscape(stringAt(proxy, "localIP"))}"`,
    `localPort = ${numberAt(proxy, "localPort")}`,
    `remotePort = ${numberAt(proxy, "remotePort")}`,
    "",
  ].join("\n");
  return {
    ok: true,
    sourceRef,
    sourceKey,
    sourcePath: displayPath(sourcePath),
    frpcToml,
    fingerprint: `sha256:${createHash("sha256").update(`${token}\n${frpcToml}`).digest("hex").slice(0, 16)}`,
    valuesRedacted: true,
  };
}

function applySentinelFrpcSecret(state: SentinelCicdState, frpcToml: string, timeoutSeconds: number): Record<string, unknown> {
  const namespace = stringAt(state.runtime, "namespace");
  const secretName = stringAt(state.publicExposure, "frpc.secretName");
  const secretKey = stringAt(state.publicExposure, "frpc.secretKey");
  const script = [
    "set +e",
    `namespace=${shellQuote(namespace)}`,
    `secret=${shellQuote(secretName)}`,
    `key=${shellQuote(secretKey)}`,
    "tmp=$(mktemp -d)",
    "trap 'rm -rf \"$tmp\"' EXIT",
    "cat >\"$tmp/frpc.toml\"",
    "kubectl -n \"$namespace\" create secret generic \"$secret\" --from-file=\"$key=$tmp/frpc.toml\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=unidesk-web-probe-sentinel-public-exposure -f - >/tmp/web-probe-sentinel-frpc-secret.out 2>/tmp/web-probe-sentinel-frpc-secret.err",
    "rc=$?",
    "present=no",
    "bytes=0",
    "if kubectl -n \"$namespace\" get secret \"$secret\" -o jsonpath=\"{.data.$key}\" >/tmp/web-probe-sentinel-frpc-secret.data 2>/dev/null; then present=yes; bytes=$(base64 -d </tmp/web-probe-sentinel-frpc-secret.data 2>/dev/null | wc -c | tr -d ' '); fi",
    "node - \"$rc\" \"$namespace\" \"$secret\" \"$key\" \"$present\" \"$bytes\" <<'NODE'",
    "const [rc, namespace, secret, key, present, bytes] = process.argv.slice(2);",
    "console.log(JSON.stringify({ok:Number(rc)===0&&present==='yes',namespace,secret,key,present:present==='yes',bytes:Number(bytes||0),applyExitCode:Number(rc),valuesRedacted:true}));",
    "NODE",
  ].join("\n");
  const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { input: frpcToml, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
  const parsedResolution = resolveSentinelChildJson(result, "web-probe-sentinel-frpc-secret-apply");
  const parsed = parsedResolution.parsed;
  return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), stdoutRecovery: parsedResolution.diagnostics, result: compactCommand(result), valuesRedacted: true };
}

function applySentinelCaddyBlock(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
  const hostname = stringAt(state.publicExposure, "hostname");
  const owner = stringAt(state.publicExposure, "caddy.managedBlockOwner");
  const configPath = stringAt(state.publicExposure, "caddy.configPath");
  const serviceName = stringAt(state.publicExposure, "caddy.serviceName");
  const responseHeaderTimeoutSeconds = numberAt(state.publicExposure, "caddy.responseHeaderTimeoutSeconds");
  const remotePort = numberAt(state.publicExposure, "frpc.httpProxy.remotePort");
  const routePrefix = normalizeRoutePrefix(stringAtNullable(state.publicExposure, "routePrefix"));
  const rootOrder = stringAtNullable(state.publicExposure, "caddy.rootOrder") ?? "normal";
  const monitorRoot = record(state.publicExposure.monitorRoot);
  const cleanupOwner = monitorRoot.enabled === false ? stringAtNullable(monitorRoot, "caddyManagedBlockOwner") : null;
  const proxyLines = [
    `reverse_proxy 127.0.0.1:${remotePort} {`,
    "    transport http {",
    `        response_header_timeout ${responseHeaderTimeoutSeconds}s`,
    "    }",
    "}",
  ];
  const block = [
    routePrefix === "/" ? "handle {" : `handle_path ${routePrefix}* {`,
    ...proxyLines.map((line) => `    ${line}`),
    "}",
    "",
  ].join("\n");
  const blockB64 = Buffer.from(block, "utf8").toString("base64");
  const script = [
    "set +e",
    `hostname=${shellQuote(hostname)}`,
    `owner=${shellQuote(owner)}`,
    `config_path=${shellQuote(configPath)}`,
    `service=${shellQuote(serviceName)}`,
    `route_prefix=${shellQuote(routePrefix)}`,
    `root_order=${shellQuote(rootOrder)}`,
    `cleanup_owner=${shellQuote(cleanupOwner ?? "")}`,
    `block_b64=${shellQuote(blockB64)}`,
    "marker=\"unidesk managed $owner\"",
    "tmp=$(mktemp -d)",
    "trap 'rm -rf \"$tmp\"' EXIT",
    "block=\"$tmp/block\"",
    "next=\"$tmp/Caddyfile\"",
    "printf '%s' \"$block_b64\" | base64 -d >\"$block\"",
    "if [ -f \"$config_path\" ]; then cp \"$config_path\" \"$next\"; else : >\"$next\"; fi",
    "python3 - \"$next\" \"$block\" \"$marker\" \"$hostname\" \"$route_prefix\" \"$root_order\" \"$cleanup_owner\" <<'PY' >/tmp/web-probe-sentinel-caddy-python.out 2>/tmp/web-probe-sentinel-caddy-python.err",
    "import pathlib, re, sys",
    "config = pathlib.Path(sys.argv[1])",
    "block = pathlib.Path(sys.argv[2]).read_text(encoding='utf-8')",
    "marker = sys.argv[3]",
    "hostname = sys.argv[4]",
    "route_prefix = sys.argv[5]",
    "root_order = sys.argv[6]",
    "cleanup_owner = sys.argv[7]",
    "text = config.read_text(encoding='utf-8') if config.exists() else ''",
    "begin = f'# BEGIN {marker}'",
    "end = f'# END {marker}'",
    "def managed_pattern(marker_text):",
    "    return re.compile(rf'(?ms)^[ \\t]*# BEGIN {re.escape(marker_text)}\\n.*?^[ \\t]*# END {re.escape(marker_text)}\\n*')",
    "pattern = managed_pattern(marker)",
    "def collect_nested_managed(segment):",
    "    preserved = []",
    "    lines = segment.splitlines()",
    "    index = 0",
    "    while index < len(lines):",
    "        stripped = lines[index].strip()",
    "        if stripped.startswith('# BEGIN ') and stripped != begin:",
    "            owner_text = stripped[len('# BEGIN '):]",
    "            end_line = '# END ' + owner_text",
    "            block_lines = [lines[index]]",
    "            index += 1",
    "            while index < len(lines):",
    "                block_lines.append(lines[index])",
    "                if lines[index].strip() == end_line:",
    "                    break",
    "                index += 1",
    "            preserved.append('\\n'.join(block_lines).rstrip() + '\\n')",
    "        index += 1",
    "    return preserved",
    "preserved_blocks = []",
    "for match in pattern.finditer(text):",
    "    preserved_blocks.extend(collect_nested_managed(match.group(0)))",
    "text = pattern.sub('', text)",
    "if cleanup_owner:",
    "    cleanup_marker = f'unidesk managed {cleanup_owner}'",
    "    if cleanup_marker != marker:",
    "        text = managed_pattern(cleanup_marker).sub('', text)",
    "def site_span(src, host):",
    "    match = re.search(rf'(?m)^([ \\t]*){re.escape(host)}[ \\t]*\\{{[ \\t]*\\n', src)",
    "    if not match:",
    "        return None",
    "    depth = 1",
    "    index = match.end()",
    "    while index < len(src):",
    "        char = src[index]",
    "        if char == '{':",
    "            depth += 1",
    "        elif char == '}':",
    "            depth -= 1",
    "            if depth == 0:",
    "                end_index = index + 1",
    "                if end_index < len(src) and src[end_index] == '\\n':",
    "                    end_index += 1",
    "                return match.start(), end_index, index, match.end()",
    "        index += 1",
    "    raise ValueError(f'unclosed Caddy site block for {host}')",
    "handler = '\\n'.join(('    ' + line) if line else '' for line in block.rstrip().splitlines())",
    "managed = f'    {begin}\\n{handler}\\n    {end}\\n'",
    "def fallback_insert_pos(site, relative_open, close_rel):",
    "    for match in re.finditer(r'(?m)^[ \\t]*handle[ \\t]*\\{[ \\t]*\\n', site[relative_open:close_rel]):",
    "        pos = relative_open + match.start()",
    "        prev_end = pos - 1",
    "        if prev_end >= 0 and site[prev_end] == '\\n':",
    "            prev_end -= 1",
    "        if prev_end >= 0:",
    "            prev_start = site.rfind('\\n', 0, prev_end + 1) + 1",
    "            if site[prev_start:prev_end + 1].strip().startswith('# BEGIN '):",
    "                return prev_start",
    "        return pos",
    "    return relative_open",
    "def append_before_close(site, close_rel, addition):",
    "    prefix = site[:close_rel]",
    "    suffix = site[close_rel:]",
    "    if prefix and not prefix.endswith('\\n'):",
    "        prefix += '\\n'",
    "    return prefix + addition + suffix",
    "span = site_span(text, hostname)",
    "if span is None:",
    "    body = ''.join(preserved_blocks) + managed",
    "    text = text.rstrip() + '\\n\\n' + f'{hostname} {{\\n{body}}}\\n'",
    "else:",
    "    start, stop, close_index, open_end = span",
    "    site = text[start:stop]",
    "    relative_open = open_end - start",
    "    close_rel = close_index - start",
    "    additions = ''.join(preserved_blocks) + managed",
    "    if route_prefix == '/' and root_order == 'active':",
    "        replacement = site[:relative_open] + additions + site[relative_open:]",
    "    elif route_prefix == '/':",
    "        replacement = append_before_close(site, close_rel, additions)",
    "    else:",
    "        insert_at = fallback_insert_pos(site, relative_open, close_rel)",
    "        replacement = site[:insert_at] + additions + site[insert_at:]",
    "    text = text[:start] + replacement + text[stop:]",
    "config.write_text(text, encoding='utf-8')",
    "PY",
    "python_rc=$?",
    "validate_rc=1",
    "reload_rc=",
    "if [ \"$python_rc\" = 0 ]; then sudo caddy validate --config \"$next\" --adapter caddyfile >/tmp/web-probe-sentinel-caddy-validate.out 2>/tmp/web-probe-sentinel-caddy-validate.err; validate_rc=$?; fi",
    "if [ \"$validate_rc\" = 0 ]; then sudo install -m 0644 \"$next\" \"$config_path\" >/tmp/web-probe-sentinel-caddy-install.out 2>/tmp/web-probe-sentinel-caddy-install.err && (sudo systemctl reload \"$service\" >/tmp/web-probe-sentinel-caddy-reload.out 2>/tmp/web-probe-sentinel-caddy-reload.err || sudo systemctl restart \"$service\" >>/tmp/web-probe-sentinel-caddy-reload.out 2>>/tmp/web-probe-sentinel-caddy-reload.err); reload_rc=$?; fi",
    "probe_rc=1",
    "probe_status=",
    "if [ \"$reload_rc\" = 0 ]; then",
    "  probe_path=\"$route_prefix\"",
    "  if [ \"$probe_path\" = \"/\" ]; then probe_url=\"https://$hostname/\"; else probe_url=\"https://$hostname$probe_path/\"; fi",
    "  probe_status=$(curl -k -sS -o /tmp/web-probe-sentinel-caddy-probe.out -w '%{http_code}' --max-time 10 --resolve \"$hostname:443:127.0.0.1\" \"$probe_url\" 2>/tmp/web-probe-sentinel-caddy-probe.err)",
    "  probe_rc=$?",
    "fi",
    "after_present=no",
    "grep -Fq \"# BEGIN $marker\" \"$config_path\" 2>/dev/null && after_present=yes",
    "active=$(systemctl is-active \"$service\" 2>/dev/null || true)",
    "err=$(cat /tmp/web-probe-sentinel-caddy-python.err /tmp/web-probe-sentinel-caddy-validate.err /tmp/web-probe-sentinel-caddy-install.err /tmp/web-probe-sentinel-caddy-reload.err /tmp/web-probe-sentinel-caddy-probe.err 2>/dev/null | tr '\\n' ';' | cut -c1-1000 || true)",
    "python3 - \"$python_rc\" \"$validate_rc\" \"$reload_rc\" \"$probe_rc\" \"$probe_status\" \"$after_present\" \"$active\" \"$hostname\" \"$config_path\" \"$err\" <<'PY'",
    "import json, sys",
    "python_rc, validate_rc, reload_rc, probe_rc, probe_status, after_present, active, hostname, config_path, error_preview = sys.argv[1:11]",
    "def num(value):",
    "    if value == '':",
    "        return None",
    "    try:",
    "        return int(value)",
    "    except ValueError:",
    "        return None",
    "http_status = num(probe_status)",
    "probe_ok = num(probe_rc) == 0 and http_status is not None and 200 <= http_status < 400",
    "payload = {",
    "    'ok': num(python_rc) == 0 and num(validate_rc) == 0 and num(reload_rc) == 0 and probe_ok and after_present == 'yes',",
    "    'hostname': hostname,",
    "    'configPath': config_path,",
    "    'pythonExitCode': num(python_rc),",
    "    'validateExitCode': num(validate_rc),",
    "    'reloadExitCode': num(reload_rc),",
    "    'routeProbeExitCode': num(probe_rc),",
    "    'routeProbeHttpStatus': http_status,",
    "    'routeProbeOk': probe_ok,",
    "    'afterBlockPresent': after_present == 'yes',",
    "    'active': active,",
    "    'errorPreview': error_preview,",
    "    'valuesRedacted': True,",
    "}",
    "print(json.dumps(payload, ensure_ascii=False))",
    "PY",
  ].join("\n");
  const result = runCommand(["trans", stringAt(state.publicExposure, "caddy.route"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
  const parsedResolution = resolveSentinelChildJson(result, "web-probe-sentinel-caddy-apply");
  const parsed = parsedResolution.parsed;
  return { ok: result.exitCode === 0 && parsed?.ok === true, routePrefix, rootOrder, ...record(parsed), stdoutRecovery: parsedResolution.diagnostics, result: compactCommand(result), valuesRedacted: true };
}
