fix: separate runner kubeconfig gap from D601 status (#144)

Commander review: direction is correct. This PR separates runner kubeconfig/read-only observability gaps from D601 public endpoint/k3s availability and keeps M3 blocked until real operation/trace/audit/evidence are observed. It does not touch PROD or restart services.
2026-05-22 20:44:30 +08:00
parent bf47a95a97
commit 85da2dcb19
8 changed files with 669 additions and 141 deletions
@@ -2,6 +2,7 @@
 import assert from "node:assert/strict";
 import { execFileSync } from "node:child_process";
 import { randomUUID } from "node:crypto";
+import { accessSync, constants as fsConstants } from "node:fs";
 import { access, mkdir, readFile, writeFile } from "node:fs/promises";
 import { request as httpRequest } from "node:http";
 import { request as httpsRequest } from "node:https";
@@ -9,7 +10,7 @@ import path from "node:path";
 import { fileURLToPath } from "node:url";

 import { activeReportLifecycle } from "../internal/dev-report-lifecycle.mjs";
-import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../internal/protocol/index.mjs";
+import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT, ENVIRONMENT_DEV } from "../internal/protocol/index.mjs";

 const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
 const defaultReportPath = "reports/dev-gate/dev-m3-hardware-loop.json";
@@ -19,6 +20,7 @@ const deployWorkloadsPath = "deploy/k8s/base/workloads.yaml";
 const requiredManifestCardinalityCommand = "node scripts/validate-dev-m3-cardinality.mjs";
 const requiredM3BoxIds = Object.freeze(["boxsimu_1", "boxsimu_2"]);
 const requiredM3BoxResources = Object.freeze(["res_boxsimu_1", "res_boxsimu_2"]);
+const runnerK3sKubeconfigPath = "/etc/rancher/k3s/k3s.yaml";

 const serviceTargets = Object.freeze([
  {
@@ -156,12 +158,57 @@ function commandResult(command, args, { timeoutMs = 5000, maxChars = 3000 } = {}
  }
 }

+function runnerKubeconfigCommandText(args) {
+  return [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args].join(" ");
+}
+
 function commandExists(command) {
  const result = commandResult("bash", ["-lc", `command -v ${command}`], { timeoutMs: 2000 });
  return result.exitCode === 0 && result.stdout.trim().length > 0;
 }

-function collectLocalRuntimeObservations() {
+function fileReadable(filePath) {
+  try {
+    accessSync(filePath, fsConstants.R_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function collectD601RunnerObservability(ingress) {
+  const args = ["-n", namespace, "get", "pods", "-o", "name"];
+  const probe = commandResult("env", [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args], {
+    timeoutMs: 5000,
+    maxChars: 2000
+  });
+  const stderr = oneLine(probe.stderr || "") || "empty";
+  const stdoutLines = String(probe.stdout ?? "")
+    .split("\n")
+    .map((line) => line.trim())
+    .filter(Boolean);
+  const d601PublicEndpointsReachable = ingress.probes.some((item) => item.ok && isHwlabDevIdentity(item));
+  const runnerKubeconfigReadable = fileReadable(runnerK3sKubeconfigPath);
+
+  return {
+    runnerKubeconfigReadable,
+    runnerKubeconfigProbeOk: probe.exitCode === 0,
+    runnerKubeconfigProbeExitCode: probe.exitCode,
+    runnerKubeconfigProbeStderr: stderr,
+    d601PublicEndpointsReachable,
+    d601K3sUnavailable: false,
+    classification: "runner_permission_mount_gap",
+    sourceIssue: "pikasTech/HWLAB#46",
+    command: runnerKubeconfigCommandText(args),
+    stdoutSummary: probe.exitCode === 0 ? `${stdoutLines.length} pod name(s) observed` : undefined,
+    summary: "Runner /etc/rancher/k3s/k3s.yaml is not treated as readable from this container; a kubectl success may come from alternate in-cluster config, so file readability and command exit code stay separate.",
+    inferenceRule: "runnerKubeconfigReadable=false must not imply d601K3sUnavailable=true; public 16666/16667 endpoint reachability is tracked separately.",
+    secretValuesRead: false,
+    secretResourcesRead: false
+  };
+}
+
+function collectLocalRuntimeObservations(d601Observability) {
  const observations = [];

  if (commandExists("docker")) {
@@ -206,6 +253,13 @@ function collectLocalRuntimeObservations() {
    });
  }

+  observations.push({
+    id: "runner-k3s-kubeconfig-readonly",
+    status: d601Observability.runnerKubeconfigProbeOk ? "observed" : "blocked",
+    command: d601Observability.command,
+    summary: `${d601Observability.summary} runnerKubeconfigReadable=${d601Observability.runnerKubeconfigReadable}; exitCode=${d601Observability.runnerKubeconfigProbeExitCode}; stderr=${d601Observability.runnerKubeconfigProbeStderr}.`
+  });
+
  return observations;
 }

@@ -587,6 +641,7 @@ function baseReport({ commitId, observedAt }) {
    ],
    runtimeTarget: {
      endpoint: DEV_ENDPOINT,
+      frontendEndpoint: DEV_FRONTEND_ENDPOINT,
      namespace,
      environment: ENVIRONMENT_DEV,
      requiredBoxSimulators: 2,
@@ -623,6 +678,17 @@ function baseReport({ commitId, observedAt }) {
    ],
    readOnlySupplementalEvidence: [],
    localRuntimeObservations: [],
+    d601Observability: {
+      runnerKubeconfigReadable: false,
+      runnerKubeconfigProbeOk: false,
+      runnerKubeconfigProbeExitCode: null,
+      runnerKubeconfigProbeStderr: "not_observed",
+      d601PublicEndpointsReachable: false,
+      d601K3sUnavailable: false,
+      classification: "not_observed",
+      sourceIssue: "pikasTech/HWLAB#46",
+      inferenceRule: "runnerKubeconfigReadable=false must not imply d601K3sUnavailable=true; public 16666/16667 endpoint reachability is tracked separately."
+    },
    liveOperation: {
      status: "not_run",
      operationId: "not_observed",
@@ -683,7 +749,8 @@ async function main() {
  report.readOnlySupplementalEvidence = await collectReadOnlySupplementalEvidence();

  const ingress = await observeDevIngress();
-  report.localRuntimeObservations = collectLocalRuntimeObservations();
+  report.d601Observability = collectD601RunnerObservability(ingress);
+  report.localRuntimeObservations = collectLocalRuntimeObservations(report.d601Observability);
  report.liveChecks.push({
    id: "dev-ingress-health",
    status: ingress.accepted ? "pass" : "blocked",
@@ -727,21 +794,22 @@ async function main() {
  if (missing.length > 0) {
    addNotRunM3Checks(report, "Stopped before M3 direct checks because service target URLs were not provided.");
    report.blockers.push({
-      type: "runtime_blocker",
+      type: "observability_blocker",
      scope: "m3-service-discovery",
      status: "open",
-      classification: "hardware loop",
-      summary: `DEV ingress is reachable, but M3 smoke lacks direct DEV service URLs: ${missing.join(", ")}.`
+      classification: "runner permission/mount gap",
+      sourceIssue: "pikasTech/HWLAB#46",
+      summary: `DEV ingress is reachable on frozen :16667, but this runner cannot use ${runnerK3sKubeconfigPath} as readable kubeconfig to discover direct M3 service URLs (${missing.join(", ")}); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline.`
    });
    report.summary = {
      status: "blocked",
-      classification: "hardware loop",
+      classification: "runner permission/mount gap",
      observedAt,
      result: "DEV ingress was reachable, but live M3 simulator and patch-panel targets were not discoverable."
    };
    await writeReport(report, reportPath);
    console.log(`[dev-m3-smoke] status=blocked report=${relativePath(reportPath)}`);
-    console.log("[dev-m3-smoke] blocker=runtime_blocker scope=m3-service-discovery");
+    console.log("[dev-m3-smoke] blocker=observability_blocker scope=m3-service-discovery");
    return;
  }

@@ -7,6 +7,8 @@ import path from "node:path";
 import { fileURLToPath } from "node:url";
 import { promisify } from "node:util";

+import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT } from "../../internal/protocol/index.mjs";
+
 const execFileAsync = promisify(execFile);
 const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
 const namespace = "hwlab-dev";
@@ -26,6 +28,12 @@ const kubeconfigCandidatePaths = [
  "/etc/rancher/k3s/k3s.yaml",
  "/var/lib/rancher/k3s/server/cred/admin.kubeconfig"
 ];
+const runnerK3sKubeconfigPath = "/etc/rancher/k3s/k3s.yaml";
+const publicEndpointProbeTargets = [
+  { id: "frontend-16666", url: `${DEV_FRONTEND_ENDPOINT}/` },
+  { id: "api-health-16667", url: `${DEV_ENDPOINT}/health` },
+  { id: "api-live-16667", url: `${DEV_ENDPOINT}/health/live` }
+];
 const sshEnvVars = [
  "HWLAB_D601_SSH_HOST",
  "HWLAB_D601_SSH_TARGET",
@@ -106,6 +114,10 @@ function commandText(command, args) {
  return [command, ...args].join(" ");
 }

+function runnerKubeconfigCommandText(args) {
+  return [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args].join(" ");
+}
+
 async function run(command, args = [], options = {}) {
  try {
    const result = await execFileAsync(command, args, {
@@ -176,6 +188,89 @@ async function fileProbe(filePath) {
  }
 }

+async function collectRunnerKubeconfigProbe(timeoutMs) {
+  const metadata = await fileProbe(runnerK3sKubeconfigPath);
+  const args = ["-n", namespace, "get", "pods", "-o", "name"];
+  const probe = await run("env", [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args], {
+    timeoutMs
+  });
+  const stdoutLines = probe.stdout
+    ? probe.stdout.split("\n").map((line) => line.trim()).filter(Boolean)
+    : [];
+
+  return {
+    path: runnerK3sKubeconfigPath,
+    mode: "read-only",
+    resource: "pods",
+    command: runnerKubeconfigCommandText(args),
+    runnerKubeconfigReadable: metadata.readable === true,
+    runnerKubeconfigProbeOk: probe.ok,
+    runnerKubeconfigProbeExitCode: probe.exitCode,
+    runnerKubeconfigProbeStderr: oneLine(probe.stderr || probe.error || "") || "empty",
+    stdoutSummary: probe.ok ? `${stdoutLines.length} pod name(s) observed` : undefined,
+    classification: metadata.readable
+      ? probe.ok
+        ? "readable"
+        : "readonly_observability_gap"
+      : "runner_permission_mount_gap",
+    sourceIssue: issue,
+    secretValuesRead: false,
+    secretResourcesRead: false,
+    note: "This probe forces KUBECONFIG=/etc/rancher/k3s/k3s.yaml for a read-only kubectl get pods command and records only exit code plus redacted stderr/stdout summary. A successful kubectl exit code does not make the kubeconfig file readable; kubectl may use an alternate in-cluster fallback."
+  };
+}
+
+async function probePublicEndpoint(target, timeoutMs) {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const response = await fetch(target.url, {
+      method: "GET",
+      signal: controller.signal,
+      redirect: "manual"
+    });
+    const body = await response.text();
+    let json = null;
+    try {
+      json = body ? JSON.parse(body) : null;
+    } catch {}
+    return {
+      id: target.id,
+      url: target.url,
+      reached: true,
+      ok: response.status >= 200 && response.status < 400,
+      status: response.status,
+      serviceId: json?.serviceId,
+      environment: json?.environment,
+      bodySummary: json
+        ? `json serviceId=${json.serviceId ?? "unknown"} status=${json.status ?? "unknown"}`
+        : oneLine(body.slice(0, 160))
+    };
+  } catch (error) {
+    return {
+      id: target.id,
+      url: target.url,
+      reached: false,
+      ok: false,
+      error: error instanceof Error ? error.message : String(error)
+    };
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+async function collectPublicEndpoints(timeoutMs) {
+  const probes = await Promise.all(
+    publicEndpointProbeTargets.map((target) => probePublicEndpoint(target, timeoutMs))
+  );
+  return {
+    activeFrontendEndpoint: DEV_FRONTEND_ENDPOINT,
+    activeApiEndpoint: DEV_ENDPOINT,
+    d601PublicEndpointsReachable: probes.every((probe) => probe.ok === true),
+    probes
+  };
+}
+
 async function collectBinary(name, versionArgs) {
  const executablePath = await findExecutable(name);
  const result = {
@@ -391,6 +486,51 @@ async function collectClusterObservability(binaries, timeoutMs) {
  };
 }

+function probeLooksLikeClusterServerUnavailable(probe) {
+  const text = oneLine(`${probe?.stderr ?? ""} ${probe?.error ?? ""}`).toLowerCase();
+  return [
+    "connection refused",
+    "no route to host",
+    "i/o timeout",
+    "context deadline exceeded",
+    "unable to connect to the server",
+    "the server is currently unable to handle the request"
+  ].some((needle) => text.includes(needle));
+}
+
+function inferD601K3sUnavailable({ cluster, publicEndpoints }) {
+  if (cluster.readable === true || publicEndpoints.d601PublicEndpointsReachable === true) {
+    return false;
+  }
+
+  return (cluster.executors ?? []).some((executor) =>
+    Object.values(executor.probes ?? {}).some((probe) => {
+      if (probe && typeof probe === "object" && !Array.isArray(probe) && Object.hasOwn(probe, "ok")) {
+        return probeLooksLikeClusterServerUnavailable(probe);
+      }
+      return Object.values(probe ?? {}).some((nested) => probeLooksLikeClusterServerUnavailable(nested));
+    })
+  );
+}
+
+function d601StatusInference({ runnerKubeconfig, publicEndpoints, d601K3sUnavailable }) {
+  return {
+    runnerKubeconfigReadable: runnerKubeconfig.runnerKubeconfigReadable,
+    runnerKubeconfigProbeOk: runnerKubeconfig.runnerKubeconfigProbeOk,
+    runnerKubeconfigProbeExitCode: runnerKubeconfig.runnerKubeconfigProbeExitCode,
+    runnerKubeconfigProbeStderr: runnerKubeconfig.runnerKubeconfigProbeStderr,
+    d601PublicEndpointsReachable: publicEndpoints.d601PublicEndpointsReachable,
+    d601K3sUnavailable,
+    conclusion: d601K3sUnavailable
+      ? "Direct read-only probes indicate D601 k3s may be unavailable."
+      : "Runner kubeconfig readability is an observability signal only; it is not used to infer D601 k3s or public DEV endpoint availability.",
+    blockerClassification: runnerKubeconfig.runnerKubeconfigReadable
+      ? "none"
+      : "runner_permission_mount_gap",
+    sourceIssue: issue
+  };
+}
+
 function addBlocker(blockers, blocker) {
  assert.ok(blockerTypes.has(blocker.type), `unknown blocker type ${blocker.type}`);
  const key = `${blocker.type}:${blocker.scope}`;
@@ -399,7 +539,7 @@ function addBlocker(blockers, blocker) {
  }
 }

-function buildBlockers({ binaries, kubeconfig, sshBridge, cluster }) {
+function buildBlockers({ binaries, kubeconfig, runnerKubeconfig, sshBridge, cluster }) {
  const blockers = [];
  const hasDirectClient = binaries.kubectl.available || binaries.k3s.available;
  if (!hasDirectClient) {
@@ -412,12 +552,24 @@ function buildBlockers({ binaries, kubeconfig, sshBridge, cluster }) {
  }
  if (!kubeconfig.some((item) => item.readable)) {
    addBlocker(blockers, {
-      type: "environment_blocker",
-      scope: "d601-kubeconfig-path",
-      summary: "No readable KUBECONFIG/default k3s kubeconfig path was found by metadata checks.",
+      type: "observability_blocker",
+      scope: "runner-kubeconfig-readonly-gap",
+      classification: "runner_permission_mount_gap",
+      sourceIssue: issue,
+      summary: "No readable KUBECONFIG/default k3s kubeconfig path was found by metadata checks; this is a runner permission/mount gap and does not prove D601 k3s is unavailable.",
      nextTask: "Mount a read-only kubeconfig for hwlab-dev, or document the approved k3s local kubeconfig path without exposing token material."
    });
  }
+  if (!runnerKubeconfig.runnerKubeconfigReadable) {
+    addBlocker(blockers, {
+      type: "observability_blocker",
+      scope: "runner-kubeconfig-readonly-gap",
+      classification: "runner_permission_mount_gap",
+      sourceIssue: issue,
+      summary: "The runner cannot read /etc/rancher/k3s/k3s.yaml; classify this as a runner permission/mount gap, not a D601 global outage. Alternate read-only cluster probes and public DEV endpoint probes are reported separately.",
+      nextTask: "Provide the approved read-only runner kubeconfig mount or document the intended alternate KUBECONFIG path, then rerun the read-only report."
+    });
+  }
  if (!sshBridge.configured && !cluster.readable) {
    addBlocker(blockers, {
      type: "environment_blocker",
@@ -431,7 +583,9 @@ function buildBlockers({ binaries, kubeconfig, sshBridge, cluster }) {
    addBlocker(blockers, {
      type: "observability_blocker",
      scope: "hwlab-dev-readonly-rbac",
-      summary: `Read-only cluster probes did not return pods, services, and ConfigMaps for ${namespace}; attempted executors: ${attempted}.`,
+      classification: "readonly_observability_gap",
+      sourceIssue: issue,
+      summary: `Read-only runner probes did not return pods, services, and ConfigMaps for ${namespace}; attempted executors: ${attempted}. This is an observability gap unless independent evidence proves D601 k3s unavailable.`,
      nextTask: "Grant or mount read-only access that can get namespace, pods, services, and ConfigMaps in hwlab-dev, then rerun this preflight."
    });
  }
@@ -440,7 +594,7 @@ function buildBlockers({ binaries, kubeconfig, sshBridge, cluster }) {

 function runnerImageFollowUp(blockers) {
  const scopes = new Set(blockers.map((blocker) => blocker.scope));
-  const needed = scopes.has("d601-k3s-client-binary") || scopes.has("d601-kubeconfig-path");
+  const needed = scopes.has("d601-k3s-client-binary") || scopes.has("runner-kubeconfig-readonly-gap");
  return {
    needed,
    suggestedUniDeskIssue: needed
@@ -457,13 +611,21 @@ async function writeReport(report, reportPath) {

 async function buildReport(args) {
  const binaries = await collectBinaries();
-  const [kubeconfig, sshBridge, cluster] = await Promise.all([
+  const [kubeconfig, runnerKubeconfig, sshBridge, cluster, publicEndpoints] = await Promise.all([
    collectKubeconfigPaths(),
+    collectRunnerKubeconfigProbe(args.timeoutMs),
    collectSshBridge(binaries),
-    collectClusterObservability(binaries, args.timeoutMs)
+    collectClusterObservability(binaries, args.timeoutMs),
+    collectPublicEndpoints(args.timeoutMs)
  ]);
-  const blockers = buildBlockers({ binaries, kubeconfig, sshBridge, cluster });
+  const d601K3sUnavailable = inferD601K3sUnavailable({ cluster, publicEndpoints });
+  const blockers = buildBlockers({ binaries, kubeconfig, runnerKubeconfig, sshBridge, cluster });
  const conclusion = cluster.readable && blockers.length === 0 ? "ready" : "blocked";
+  const statusInference = d601StatusInference({
+    runnerKubeconfig,
+    publicEndpoints,
+    d601K3sUnavailable
+  });
  return {
    $schema: "https://hwlab.pikastech.local/schemas/d601-k3s-readonly-observability-report.schema.json",
    $id: "https://hwlab.pikastech.local/reports/d601-k3s-readonly-observability.json",
@@ -497,15 +659,24 @@ async function buildReport(args) {
      "node scripts/d601-k3s-readonly-observability.mjs"
    ],
    conclusion,
+    runnerKubeconfigReadable: statusInference.runnerKubeconfigReadable,
+    runnerKubeconfigProbeOk: statusInference.runnerKubeconfigProbeOk,
+    runnerKubeconfigProbeExitCode: statusInference.runnerKubeconfigProbeExitCode,
+    runnerKubeconfigProbeStderr: statusInference.runnerKubeconfigProbeStderr,
+    d601PublicEndpointsReachable: statusInference.d601PublicEndpointsReachable,
+    d601K3sUnavailable: statusInference.d601K3sUnavailable,
+    statusInference,
    environment: {
      binaries,
      kubeconfig,
+      runnerKubeconfig,
      sshBridge
    },
+    publicEndpoints,
    cluster,
    blockers,
    runnerImageFollowUp: runnerImageFollowUp(blockers),
-    notes: "This report uses command availability checks, file metadata checks, kubectl/k3s kubectl read-only get probes, and auth can-i checks only. It does not print kubeconfig contents, token material, Secret resources, ConfigMap values, SSH config contents, or private keys."
+    notes: "This report uses command availability checks, file metadata checks, public DEV endpoint GET probes, kubectl/k3s kubectl read-only get probes, and auth can-i checks only. It does not print kubeconfig contents, token material, Secret resources, ConfigMap values, SSH config contents, or private keys. A failed runner /etc/rancher/k3s/k3s.yaml probe is classified as a runner permission/mount or read-only observability gap and is not evidence that D601 k3s is globally unavailable."
  };
 }

@@ -517,6 +688,11 @@ function printSummary(args, report) {
    namespace: report.namespace,
    report: args.writeReport ? args.reportPath : null,
    readable: report.cluster.readable,
+    runnerKubeconfigReadable: report.runnerKubeconfigReadable,
+    runnerKubeconfigProbeOk: report.runnerKubeconfigProbeOk,
+    runnerKubeconfigProbeExitCode: report.runnerKubeconfigProbeExitCode,
+    d601PublicEndpointsReachable: report.d601PublicEndpointsReachable,
+    d601K3sUnavailable: report.d601K3sUnavailable,
    attemptedExecutors: report.cluster.attemptedExecutors,
    blockers: report.blockers.map((blocker) => ({
      type: blocker.type,
@@ -263,13 +263,19 @@ function applyPriority(blocker) {
    };
  }

-  if (blocker.scope === "kubectl" || blocker.scope === "d601-k3s" || blocker.scope.startsWith("d601-")) {
+  if (
+    blocker.scope === "kubectl" ||
+    blocker.scope === "d601-k3s" ||
+    blocker.scope === "runner-kubeconfig-readonly-gap" ||
+    blocker.scope === "m3-service-discovery" ||
+    blocker.scope.startsWith("d601-")
+  ) {
    return {
      ...blocker,
      priority: "P1",
      unblockOrder: 4,
-      unblocks: [issue(34), issue(33), issue(36), issue(38)],
-      rationale: "D601 hwlab-dev cluster state must be observable read-only before any live apply or M3/M4/M5 acceptance."
+      unblocks: [issue(34), issue(33), issue(36), issue(38), issue(46), issue(64)],
+      rationale: "Runner read-only observability must be repaired without treating the runner gap as proof that D601 k3s or public DEV endpoints are unavailable."
    };
  }

@@ -279,7 +285,7 @@ function applyPriority(blocker) {
      priority: "P1",
      unblockOrder: 4,
      unblocks: [issue(34), issue(33), issue(36), issue(38), issue(39)],
-      rationale: "D601 has a client path, but read-only hwlab-dev pods/services/configmaps must be observable before live acceptance evidence can be trusted."
+      rationale: "D601 has a client path, but read-only hwlab-dev pods/services/configmaps must be observable before live acceptance evidence can be trusted; this is not evidence that D601 is globally offline."
    };
  }

@@ -462,17 +468,13 @@ function collectM2Evidence(reports) {
    }),
    reportEvidence({
      milestone: "M2",
-      level: "BLOCKED",
+      level: d601.cluster?.readable === true || d601.d601PublicEndpointsReachable === true ? "DEV-LIVE" : "BLOCKED",
      report: d601,
      status: d601.conclusion,
      category: "d601-observability",
-      evidence: [
-        `kubectl=${d601.environment?.binaries?.kubectl?.available === true ? "available" : "missing"}`,
-        `k3s=${d601.environment?.binaries?.k3s?.available === true ? "available" : "missing"}`,
-        `clusterReadable=${d601.cluster?.readable === true ? "yes" : "no"}`
-      ],
+      evidence: d601ObservationSummary(d601),
      commands: d601.validationCommands,
-      summary: "D601 k3s observability is blocked; cluster state has not been read."
+      summary: d601ObservabilitySummary(d601)
    })
  ];
 }
@@ -522,7 +524,10 @@ function collectM3Evidence(reports) {

 function m3TrustedLoopSummary(m3Report) {
  const operationSummary = m3Report.liveOperation?.summary ?? "No live M3 hardware operation was observed.";
-  const blockerSummary = m3Report.blockers?.find((blocker) => blocker.scope === "m3-hardware-loop-runtime")?.summary;
+  const blockerSummary = m3Report.blockers?.find((blocker) =>
+    blocker.scope === "m3-hardware-loop-runtime" ||
+    blocker.scope === "m3-service-discovery"
+  )?.summary;
  if (statusIsPass(m3Report.liveOperation?.status) || !blockerSummary) {
    return operationSummary;
  }
@@ -658,6 +663,32 @@ function publicEndpointEvidence(m2Report) {
  });
 }

+function d601ObservationSummary(d601) {
+  return [
+    `runnerKubeconfigReadable=${d601.runnerKubeconfigReadable === true}`,
+    `runnerKubeconfigProbeExitCode=${d601.runnerKubeconfigProbeExitCode ?? "unknown"}`,
+    `runnerKubeconfigProbeStderr=${d601.runnerKubeconfigProbeStderr || "empty"}`,
+    `d601PublicEndpointsReachable=${d601.d601PublicEndpointsReachable === true}`,
+    `d601K3sUnavailable=${d601.d601K3sUnavailable === true}`
+  ];
+}
+
+function d601ObservabilitySummary(d601) {
+  if (d601.d601PublicEndpointsReachable === true && d601.runnerKubeconfigReadable === false) {
+    const clusterNote = d601.cluster?.readable === true
+      ? " Alternate read-only cluster probes are readable."
+      : "";
+    return `D601 public DEV endpoints are reachable, but the runner cannot read /etc/rancher/k3s/k3s.yaml; classify as #46 runner permission/mount or read-only observability gap, not D601 global offline.${clusterNote} ${d601ObservationSummary(d601).join(", ")}.`;
+  }
+  if (d601.cluster?.readable === true) {
+    return `D601 hwlab-dev cluster is readable from this runner; ${d601ObservationSummary(d601).join(", ")}.`;
+  }
+  if (d601.d601K3sUnavailable === true) {
+    return `Direct read-only probes indicate D601 k3s may be unavailable; ${d601ObservationSummary(d601).join(", ")}.`;
+  }
+  return `D601 read-only observability is blocked without proving D601 unavailable; ${d601ObservationSummary(d601).join(", ")}.`;
+}
+
 function isStaleLegacyIngressBlocker(blocker, sourceReport, reports) {
  const text = `${blocker.scope ?? ""} ${blocker.summary ?? ""}`;
  const staleLegacyPort = deprecatedLegacyPublicEndpoints.some((endpoint) => text.includes(endpoint));
@@ -792,8 +823,6 @@ function buildDoD(reports, milestones, blockers) {
  const preflight = reports.devPreflight;
  const artifactIdentity = preflight.artifactIdentity;
  const d601 = reports.d601Observability;
-  const d601HasClient = d601.environment?.binaries?.kubectl?.available === true ||
-    d601.environment?.binaries?.k3s?.available === true;
  const edgeLive = hasCurrentLiveEdgeEvidence(reports.devEdgeHealth);
  const artifactCurrent = artifactIdentity.publishVerified === true &&
    artifactIdentity.targetCoverage?.covered !== false &&
@@ -827,13 +856,9 @@ function buildDoD(reports, milestones, blockers) {
      },
      {
        id: "d601-k3s-observability",
-        status: d601.cluster?.readable ? "pass" : "blocked",
-        evidenceLevel: d601.cluster?.readable ? "DEV-LIVE" : "BLOCKED",
-        summary: d601.cluster?.readable
-          ? "D601 hwlab-dev cluster was readable."
-          : d601HasClient
-            ? "D601 kubectl/k3s clients are present, but read-only hwlab-dev pods/services/configmaps probes are still blocked."
-            : "D601 runner lacks kubectl/k3s/kubeconfig observability for hwlab-dev."
+        status: d601.runnerKubeconfigReadable === false ? "blocked" : d601.cluster?.readable ? "pass" : "blocked",
+        evidenceLevel: d601.cluster?.readable || d601.d601PublicEndpointsReachable ? "DEV-LIVE" : "BLOCKED",
+        summary: d601ObservabilitySummary(d601)
      },
      {
        id: "dev-edge-frp-16667",
@@ -928,6 +953,21 @@ function buildCurrentDevLayering(reports, blockers) {
      evidence: reports.devM5Gate.devPreconditions?.evidence?.filter((line) => line.includes("/health/live") || line.includes("DB")) ?? [],
      nextRequired: "Provide live DB connection evidence through redacted health output; route reachability alone is insufficient."
    },
+    d601RunnerObservability: {
+      label: "D601 runner observability",
+      status: reports.d601Observability.runnerKubeconfigReadable === false
+        ? "blocked"
+        : reports.d601Observability.cluster?.readable === true
+          ? "pass"
+          : "blocked",
+      evidenceLevel: reports.d601Observability.cluster?.readable === true ||
+        reports.d601Observability.d601PublicEndpointsReachable === true
+        ? "DEV-LIVE"
+        : "BLOCKED",
+      summary: d601ObservabilitySummary(reports.d601Observability),
+      evidence: d601ObservationSummary(reports.d601Observability),
+      nextRequired: "Treat #46 runner kubeconfig/readonly gaps separately from D601 service health; rerun read-only observability after the mount or permission path is repaired."
+    },
    m3HardwareTrustedLoop: {
      label: "M3 hardware trusted loop",
      status: m3Live ? "pass" : "blocked",
@@ -968,21 +1008,36 @@ function buildMilestoneBlockerClassification(reports) {
  const cloudDb = cloudApiDbStatus(reports);
  const m3Live = statusIsPass(reports.devM3Hardware.liveOperation?.status);
  const m4Live = statusIsPass(reports.devM4Agent.livePreflight?.status);
+  const m3ServiceDiscoveryBlocked = reports.devM3Hardware.blockers?.some((blocker) =>
+    blocker.scope === "m3-service-discovery"
+  ) === true;
+  const m3BlockerClass = m3Live
+    ? "cleared"
+    : m3ServiceDiscoveryBlocked
+      ? "runner-readonly-observability-gap"
+      : "hardware-loop-runtime";

  return [
    {
      milestone: "M3",
      status: m3Live ? "pass" : "blocked",
      currentLevel: m3Live ? "DEV-LIVE" : "BLOCKED",
-      blockerClass: m3Live ? "cleared" : "hardware-loop-runtime",
-      dependency: "Real DEV trusted loop through res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1.",
+      blockerClass: m3BlockerClass,
+      dependency: m3ServiceDiscoveryBlocked
+        ? "Runner read-only service discovery must be repaired before the real DEV trusted loop can be observed; this does not mean D601/k3s is globally unavailable."
+        : "Real DEV trusted loop through res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1.",
      evidence: [
        `operationId=${reports.devM3Hardware.liveOperation?.operationId ?? "not_observed"}`,
        `traceId=${reports.devM3Hardware.liveOperation?.traceId ?? "not_observed"}`,
        `auditId=${reports.devM3Hardware.liveOperation?.auditId ?? "not_observed"}`,
-        `evidenceId=${reports.devM3Hardware.liveOperation?.evidenceId ?? "not_observed"}`
+        `evidenceId=${reports.devM3Hardware.liveOperation?.evidenceId ?? "not_observed"}`,
+        `runnerKubeconfigReadable=${reports.devM3Hardware.d601Observability?.runnerKubeconfigReadable === true}`,
+        `d601PublicEndpointsReachable=${reports.devM3Hardware.d601Observability?.d601PublicEndpointsReachable === true}`,
+        `d601K3sUnavailable=${reports.devM3Hardware.d601Observability?.d601K3sUnavailable === true}`
      ],
-      nextRequired: "Run the bounded DEV M3 live smoke only after patch-panel topology can prove operation, trace, audit, and evidence IDs.",
+      nextRequired: m3ServiceDiscoveryBlocked
+        ? "Repair the #46 runner readonly kubeconfig/service-discovery gap, then run the bounded DEV M3 live smoke only when direct simulator and patch-panel targets are discoverable."
+        : "Run the bounded DEV M3 live smoke only after patch-panel topology can prove operation, trace, audit, and evidence IDs.",
      nonPromotionReason: "Public frontend, route, and artifact evidence do not prove the required hardware loop."
    },
    {
@@ -1071,6 +1126,7 @@ function fallbackAction(scope) {
  if (scope.includes("cloud-api-db")) return "Configure DEV cloud-api DB env readiness and rerun health/preflight without exposing secrets.";
  if (scope === "db-live") return "Repair DEV cloud-api DB live readiness, then rerun the read-only health and M4 preflight reports without exposing secret values.";
  if (scope === "m3-hardware-loop-runtime") return "Prove the real DEV M3 trusted loop res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 with operation, trace, audit, and evidence identifiers.";
+  if (scope === "runner-kubeconfig-readonly-gap" || scope === "m3-service-discovery") return "Repair the #46 runner kubeconfig mount/permission or document the approved alternate read-only KUBECONFIG/service-discovery path; do not classify this as D601 global offline.";
  if (scope.includes("kubectl") || scope.includes("k3s")) return "Provide read-only kubectl/kubeconfig observability for hwlab-dev.";
  return "Resolve the blocker and attach source/local/dry-run/DEV-live evidence at the correct level.";
 }
@@ -1083,6 +1139,7 @@ function evidenceRequiredFor(scope) {
  if (scope.includes("cloud-api-db")) return "Cloud API health/live output showing DB env ready and redacted secret references, without secret material.";
  if (scope === "db-live") return "Cloud API /health/live output with ready=true, connected=true, liveDbEvidence=true, and redacted secret references.";
  if (scope === "m3-hardware-loop-runtime") return "Operation, trace, audit, and evidence IDs from the real DEV DO1 -> patch-panel -> DI1 trusted loop.";
+  if (scope === "runner-kubeconfig-readonly-gap" || scope === "m3-service-discovery") return "Read-only report with runnerKubeconfigReadable, runnerKubeconfigProbeExitCode/stderr, d601PublicEndpointsReachable, and d601K3sUnavailable recorded separately, plus direct M3 service target discovery before any DO write.";
  if (scope.includes("edge") || scope.includes("ingress") || scope.includes("frp")) return "Read-only DEV route observation for :16667/frp/edge/router with HWLAB service identity and artifact identity.";
  return "A committed report with the exact evidence level and command used.";
 }
@@ -1271,6 +1271,13 @@ async function validateDevM3Report(report, label) {

  assertObject(report.runtimeTarget, `${label}.runtimeTarget`);
  assert.equal(report.runtimeTarget.endpoint, "http://74.48.78.17:16667", `${label}.runtimeTarget.endpoint`);
+  if (Object.hasOwn(report.runtimeTarget, "frontendEndpoint")) {
+    assert.equal(
+      report.runtimeTarget.frontendEndpoint,
+      "http://74.48.78.17:16666",
+      `${label}.runtimeTarget.frontendEndpoint`
+    );
+  }
  assert.equal(report.runtimeTarget.namespace, "hwlab-dev", `${label}.runtimeTarget.namespace`);
  assert.equal(report.runtimeTarget.environment, "dev", `${label}.runtimeTarget.environment`);
  assert.equal(report.runtimeTarget.requiredBoxSimulators, 2, `${label}.runtimeTarget.requiredBoxSimulators`);
@@ -1322,6 +1329,47 @@ async function validateDevM3Report(report, label) {
    }
  }

+  if (Object.hasOwn(report, "d601Observability")) {
+    assertObject(report.d601Observability, `${label}.d601Observability`);
+    for (const field of [
+      "runnerKubeconfigReadable",
+      "runnerKubeconfigProbeExitCode",
+      "runnerKubeconfigProbeStderr",
+      "d601PublicEndpointsReachable",
+      "d601K3sUnavailable",
+      "classification",
+      "sourceIssue",
+      "inferenceRule"
+    ]) {
+      assert.ok(Object.hasOwn(report.d601Observability, field), `${label}.d601Observability missing ${field}`);
+    }
+    assertBoolean(report.d601Observability.runnerKubeconfigReadable, `${label}.d601Observability.runnerKubeconfigReadable`);
+    if (Object.hasOwn(report.d601Observability, "runnerKubeconfigProbeOk")) {
+      assertBoolean(report.d601Observability.runnerKubeconfigProbeOk, `${label}.d601Observability.runnerKubeconfigProbeOk`);
+    }
+    assert.ok(
+      Number.isInteger(report.d601Observability.runnerKubeconfigProbeExitCode) ||
+        report.d601Observability.runnerKubeconfigProbeExitCode === null,
+      `${label}.d601Observability.runnerKubeconfigProbeExitCode`
+    );
+    assertString(report.d601Observability.runnerKubeconfigProbeStderr, `${label}.d601Observability.runnerKubeconfigProbeStderr`);
+    assertBoolean(report.d601Observability.d601PublicEndpointsReachable, `${label}.d601Observability.d601PublicEndpointsReachable`);
+    assertBoolean(report.d601Observability.d601K3sUnavailable, `${label}.d601Observability.d601K3sUnavailable`);
+    if (
+      report.d601Observability.runnerKubeconfigReadable === false &&
+      report.d601Observability.d601PublicEndpointsReachable === true
+    ) {
+      assert.equal(
+        report.d601Observability.d601K3sUnavailable,
+        false,
+        `${label}.d601Observability runner gap must not imply D601 k3s unavailable`
+      );
+    }
+    assertString(report.d601Observability.classification, `${label}.d601Observability.classification`);
+    assertString(report.d601Observability.sourceIssue, `${label}.d601Observability.sourceIssue`);
+    assertString(report.d601Observability.inferenceRule, `${label}.d601Observability.inferenceRule`);
+  }
+
  if (Object.hasOwn(report, "readOnlySupplementalEvidence")) {
    assertArray(report.readOnlySupplementalEvidence, `${label}.readOnlySupplementalEvidence`);
    for (const [index, evidence] of report.readOnlySupplementalEvidence.entries()) {