#!/usr/bin/env node import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; import { randomUUID } from "node:crypto"; import { accessSync, constants as fsConstants } from "node:fs"; import { access, mkdir, readFile, writeFile } from "node:fs/promises"; import { request as httpRequest } from "node:http"; import { request as httpsRequest } from "node:https"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { activeReportLifecycle } from "../internal/dev-report-lifecycle.mjs"; import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT, ENVIRONMENT_DEV } from "../internal/protocol/index.mjs"; import { ensureNotRepoReportsPath, tempReportPath } from "./src/report-paths.mjs"; import { readStructuredFile } from "./src/structured-config.mjs"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const defaultReportPath = tempReportPath("dev-m3-hardware-loop.json"); const namespace = "hwlab-dev"; const issue = "pikasTech/HWLAB#38"; const deployWorkloadsPath = "deploy/k8s/base/workloads.yaml"; const deployServicesPath = "deploy/k8s/base/services.yaml"; const dryRunPlanCommand = "node scripts/dev-m3-hardware-loop-smoke.mjs --dry-run"; const liveSmokeCommand = "node scripts/dev-m3-hardware-loop-smoke.mjs --live --confirm-dev --expect-non-prod"; const legacyLiveSmokeCommand = "node scripts/dev-m3-hardware-loop-smoke.mjs --live --confirm-dev --confirmed-non-production"; const requiredM3BoxIds = Object.freeze(["boxsimu_1", "boxsimu_2"]); const requiredM3BoxResources = Object.freeze(["res_boxsimu_1", "res_boxsimu_2"]); const requiredM3Connection = Object.freeze({ fromResourceId: "res_boxsimu_1", fromPort: "DO1", toResourceId: "res_boxsimu_2", toPort: "DI1" }); const requiredM3GatewayIds = Object.freeze(["gwsimu_1", "gwsimu_2"]); const runnerK3sKubeconfigPath = "/etc/rancher/k3s/k3s.yaml"; const m3FailureClassifications = Object.freeze({ targetMissing: "target_missing", identityNotDistinct: "identity_not_distinct", patchPanelWiringMissing: "patch_panel_wiring_missing", patchPanelRouteOwnerInvalid: "patch_panel_route_owner_invalid", operationFailed: "operation_failed", evidenceMissing: "evidence_missing" }); const serviceTargets = Object.freeze([ { id: "box-simu-1", serviceId: "hwlab-box-simu", urlEnv: "HWLAB_DEV_BOX_SIMU_1_URL", statusPath: "/status" }, { id: "box-simu-2", serviceId: "hwlab-box-simu", urlEnv: "HWLAB_DEV_BOX_SIMU_2_URL", statusPath: "/status" }, { id: "gateway-simu-1", serviceId: "hwlab-gateway-simu", urlEnv: "HWLAB_DEV_GATEWAY_SIMU_1_URL", statusPath: "/status" }, { id: "gateway-simu-2", serviceId: "hwlab-gateway-simu", urlEnv: "HWLAB_DEV_GATEWAY_SIMU_2_URL", statusPath: "/status" }, { id: "patch-panel", serviceId: "hwlab-patch-panel", urlEnv: "HWLAB_DEV_PATCH_PANEL_URL", statusPath: "/status", wiringPath: "/wiring", routePath: "/signals/route" } ]); const kubernetesM3ServiceIdentities = Object.freeze({ "hwlab-box-simu": Object.freeze({ serviceId: "hwlab-box-simu" }), "hwlab-box-simu-1": Object.freeze({ serviceId: "hwlab-box-simu", targetId: "box-simu-1", instanceId: "res_boxsimu_1" }), "hwlab-box-simu-2": Object.freeze({ serviceId: "hwlab-box-simu", targetId: "box-simu-2", instanceId: "res_boxsimu_2" }), "hwlab-gateway-simu": Object.freeze({ serviceId: "hwlab-gateway-simu" }), "hwlab-gateway-simu-1": Object.freeze({ serviceId: "hwlab-gateway-simu", targetId: "gateway-simu-1", instanceId: "gwsimu_1" }), "hwlab-gateway-simu-2": Object.freeze({ serviceId: "hwlab-gateway-simu", targetId: "gateway-simu-2", instanceId: "gwsimu_2" }), "hwlab-patch-panel": Object.freeze({ serviceId: "hwlab-patch-panel", targetId: "patch-panel" }) }); function parseArgs(argv) { const flags = new Set(); const values = new Map(); for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (!arg.startsWith("--")) { throw new Error(`unexpected positional argument ${arg}`); } if (arg === "--output") { const value = argv[index + 1]; assert.ok(value && !value.startsWith("--"), "--output requires a path"); values.set("output", value); index += 1; continue; } flags.add(arg); } return { flags, values, dryRun: flags.has("--dry-run") || flags.has("--plan") || flags.has("--source-read-only"), live: flags.has("--live") }; } function classifyDirectTargetBlockerScope(scope) { if (scope === "m3-patch-panel-wiring") { return m3FailureClassifications.patchPanelWiringMissing; } if (scope === "m3-box-simu-identity" || scope === "m3-gateway-simu-identity") { return m3FailureClassifications.identityNotDistinct; } if (scope === "m3-service-discovery" || scope === "m3-direct-target-missing") { return m3FailureClassifications.targetMissing; } return m3FailureClassifications.operationFailed; } function classifyLiveOperationError(error) { const message = error instanceof Error ? error.message : String(error); return /\b(auditId|evidenceId)\b/u.test(message) ? m3FailureClassifications.evidenceMissing : m3FailureClassifications.operationFailed; } function requireSafetyGates({ flags, dryRun, live }) { const forbiddenFlags = [ "--prod", "--real-hardware", "--read-secret", "--read-token", "--force-push", "--restart-runtime", "--restart-unidesk", "--restart-code-queue", "--restart-backend-core" ]; for (const flag of forbiddenFlags) { assert.ok(!flags.has(flag), `${flag} is forbidden for DEV M3 smoke`); } const confirmedLive = live && flags.has("--confirm-dev") && (flags.has("--expect-non-prod") || flags.has("--confirmed-non-production")) && !dryRun && !flags.has("--allow-live"); if (!confirmedLive) { return { mode: "dry-run", liveWriteAllowed: false }; } return { mode: "live", liveWriteAllowed: true }; } function isoNow() { return new Date().toISOString(); } function relativePath(absolutePath) { return path.relative(repoRoot, absolutePath).replaceAll(path.sep, "/"); } function resolveReportPath(value) { return ensureNotRepoReportsPath(repoRoot, value ?? defaultReportPath, "report path"); } function currentCommit() { try { return execFileSync("git", ["rev-parse", "--short=12", "HEAD"], { cwd: repoRoot, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); } catch { return "unknown"; } } function oneLine(value) { return String(value).replace(/\s+/g, " ").trim(); } function commandResult(command, args, { timeoutMs = 5000, maxChars = 3000 } = {}) { try { const stdout = execFileSync(command, args, { cwd: repoRoot, encoding: "utf8", timeout: timeoutMs, stdio: ["ignore", "pipe", "pipe"] }); return { command: [command, ...args].join(" "), exitCode: 0, stdout: stdout.slice(0, maxChars) }; } catch (error) { return { command: [command, ...args].join(" "), exitCode: Number.isInteger(error.status) ? error.status : 1, stdout: String(error.stdout ?? "").slice(0, maxChars), stderr: String(error.stderr || error.message || "").slice(0, maxChars) }; } } function runnerKubeconfigCommandText(args) { return [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args].join(" "); } function commandExists(command) { const result = commandResult("bash", ["-lc", `command -v ${command}`], { timeoutMs: 2000 }); return result.exitCode === 0 && result.stdout.trim().length > 0; } function fileReadable(filePath) { try { accessSync(filePath, fsConstants.R_OK); return true; } catch { return false; } } function collectD601RunnerObservability(ingress) { const args = ["-n", namespace, "get", "pods", "-o", "name"]; const probe = commandResult("env", [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args], { timeoutMs: 5000, maxChars: 2000 }); const stderr = oneLine(probe.stderr || "") || "empty"; const stdoutLines = String(probe.stdout ?? "") .split("\n") .map((line) => line.trim()) .filter(Boolean); const d601PublicEndpointsReachable = ingress.probes.some((item) => item.ok && isHwlabDevIdentity(item)); const runnerKubeconfigReadable = fileReadable(runnerK3sKubeconfigPath); const classification = probe.exitCode === 0 ? "read_only_kubectl_available_kubeconfig_file_unreadable" : "runner_permission_mount_gap"; return { runnerKubeconfigReadable, runnerKubeconfigProbeOk: probe.exitCode === 0, runnerKubeconfigProbeExitCode: probe.exitCode, runnerKubeconfigProbeStderr: stderr, d601PublicEndpointsReachable, d601K3sUnavailable: false, classification, sourceIssue: "pikasTech/HWLAB#46", command: runnerKubeconfigCommandText(args), stdoutSummary: probe.exitCode === 0 ? `${stdoutLines.length} pod name(s) observed` : undefined, summary: probe.exitCode === 0 ? "Runner /etc/rancher/k3s/k3s.yaml is not readable through fs access, but read-only kubectl observation succeeds; file readability, kubectl reachability, and service discovery are recorded separately." : "Runner /etc/rancher/k3s/k3s.yaml is not treated as readable from this container; kubectl read-only observation did not succeed, so this is a runner permission/mount gap, not proof that D601 is globally offline.", inferenceRule: "runnerKubeconfigReadable=false must not imply d601K3sUnavailable=true; public 16666/16667 endpoint reachability is tracked separately.", secretValuesRead: false, secretResourcesRead: false }; } function collectLocalRuntimeObservations(d601Observability) { const observations = []; if (commandExists("docker")) { const docker = commandResult("docker", [ "ps", "--format", "{{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}" ]); const hwlabLines = docker.stdout .split("\n") .filter((line) => line.includes("hwlab-")) .join("\n"); observations.push({ id: "docker-hwlab-containers", status: hwlabLines ? "observed" : "not_observed", command: docker.command, summary: oneLine(hwlabLines || "No running docker container name/image/port line contained hwlab-.") }); } else { observations.push({ id: "docker-hwlab-containers", status: "unavailable", command: "command -v docker", summary: "docker CLI is not available in this runner." }); } if (commandExists("kubectl")) { const args = ["get", "deploy,po,svc", "-n", namespace, "-o", "wide"]; const kubectl = commandResult("env", [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args]); observations.push({ id: "kubectl-hwlab-dev", status: kubectl.exitCode === 0 ? "observed" : "blocked", command: runnerKubeconfigCommandText(args), summary: oneLine(kubectl.exitCode === 0 ? kubectl.stdout : kubectl.stderr || kubectl.stdout) }); } else { observations.push({ id: "kubectl-hwlab-dev", status: "unavailable", command: "command -v kubectl", summary: "kubectl is not available in this runner, so hwlab-dev namespace state was not observed." }); } observations.push({ id: "runner-k3s-kubeconfig-readonly", status: d601Observability.runnerKubeconfigProbeOk ? "observed" : "blocked", command: d601Observability.command, summary: `${d601Observability.summary} runnerKubeconfigReadable=${d601Observability.runnerKubeconfigReadable}; exitCode=${d601Observability.runnerKubeconfigProbeExitCode}; stderr=${d601Observability.runnerKubeconfigProbeStderr}.` }); return observations; } function splitList(value) { return String(value ?? "") .split(",") .map((item) => item.trim()) .filter(Boolean); } function hasSameMembers(actual, expected) { return actual.length === expected.length && expected.every((item) => actual.includes(item)); } function listItems(document) { return document?.kind === "List" ? document.items ?? [] : [document].filter(Boolean); } function serviceByName(services) { return new Map( listItems(services) .filter((item) => item?.kind === "Service") .map((item) => [item.metadata?.name, item]) ); } async function collectReadOnlySupplementalEvidence() { const [deploy, workloads, services] = await Promise.all([ readStructuredFile(repoRoot, "deploy/deploy.yaml"), readStructuredFile(repoRoot, deployWorkloadsPath), readStructuredFile(repoRoot, deployServicesPath) ]); const targets = ["hwlab-box-simu", "hwlab-gateway-simu", "hwlab-patch-panel"]; const observed = listItems(workloads) .filter((item) => targets.includes(item.metadata?.name)) .map((item) => { const container = item.spec?.template?.spec?.containers?.[0] ?? {}; const env = Object.fromEntries((container.env ?? []).map((entry) => [entry.name, entry.value])); return { name: item.metadata.name, kind: item.kind, namespace: item.metadata.namespace, replicas: item.spec?.replicas ?? null, image: container.image, env }; }); const box = observed.find((item) => item.name === "hwlab-box-simu"); const gateway = observed.find((item) => item.name === "hwlab-gateway-simu"); const patchPanel = observed.find((item) => item.name === "hwlab-patch-panel"); const patchPanelManifest = (deploy.services ?? []).find((service) => service.serviceId === "hwlab-patch-panel"); const boxIds = splitList(box?.env.HWLAB_BOX_ID); const boxResourceIds = splitList(box?.env.HWLAB_BOX_RESOURCE_ID); const boxGatewaySessionIds = splitList(box?.env.HWLAB_GATEWAY_SESSION_ID); const gatewayIds = splitList(gateway?.env.HWLAB_GATEWAY_ID); const gatewayBoxResources = splitList(gateway?.env.HWLAB_BOX_RESOURCES); const gatewayBoxUrls = splitList(gateway?.env.HWLAB_BOX_URLS); const serviceMap = serviceByName(services); const hasInstanceServices = [ "hwlab-box-simu-1", "hwlab-box-simu-2", "hwlab-gateway-simu-1", "hwlab-gateway-simu-2" ].every((name) => serviceMap.has(name)); const patchPanelEndpointMap = patchPanel?.env.HWLAB_PATCH_PANEL_ENDPOINT_MAP ? JSON.parse(patchPanel.env.HWLAB_PATCH_PANEL_ENDPOINT_MAP) : {}; const patchPanelWiringConfig = patchPanel?.env.HWLAB_PATCH_PANEL_WIRING_CONFIG ? JSON.parse(patchPanel.env.HWLAB_PATCH_PANEL_WIRING_CONFIG) : null; const patchPanelHasManifestRoute = patchPanelWiringConfig?.status === "active" && patchPanelWiringConfig?.constraints?.propagation === "patch-panel-only" && patchPanelWiringConfig?.constraints?.localLoopbackSubstituteAllowed === false && (patchPanelWiringConfig?.connections ?? []).some( (connection) => connection.from?.resourceId === requiredM3Connection.fromResourceId && connection.from?.port === requiredM3Connection.fromPort && connection.to?.resourceId === requiredM3Connection.toResourceId && connection.to?.port === requiredM3Connection.toPort && connection.mode === "exclusive" ); const boxReadyForM3 = box?.kind === "StatefulSet" && box?.replicas === 2 && hasSameMembers(boxIds, requiredM3BoxIds) && hasSameMembers(boxResourceIds, requiredM3BoxResources) && hasSameMembers(boxGatewaySessionIds, ["gws_gwsimu_1", "gws_gwsimu_2"]); const gatewayReadyForM3 = gateway?.kind === "StatefulSet" && gateway?.replicas === 2 && hasSameMembers(gatewayIds, requiredM3GatewayIds) && hasSameMembers(gatewayBoxResources, requiredM3BoxResources) && hasSameMembers(gatewayBoxUrls, [ "http://hwlab-box-simu-1.hwlab-dev.svc.cluster.local:7201", "http://hwlab-box-simu-2.hwlab-dev.svc.cluster.local:7201" ]); const patchPanelReadyForM3 = patchPanel?.replicas === 1 && patchPanelManifest?.m3Route?.fromResourceId === requiredM3Connection.fromResourceId && patchPanelManifest?.m3Route?.fromPort === requiredM3Connection.fromPort && patchPanelManifest?.m3Route?.patchPanelServiceId === "hwlab-patch-panel" && patchPanelManifest?.m3Route?.toResourceId === requiredM3Connection.toResourceId && patchPanelManifest?.m3Route?.toPort === requiredM3Connection.toPort && patchPanelEndpointMap.res_boxsimu_2 === "http://hwlab-box-simu-2.hwlab-dev.svc.cluster.local:7201" && patchPanelHasManifestRoute; const manifestReadyForM3 = boxReadyForM3 && gatewayReadyForM3 && patchPanelReadyForM3 && hasInstanceServices; return [ { id: "deploy-skeleton-m3-cardinality", status: manifestReadyForM3 ? "manifest-ready" : "gap", blockerClass: manifestReadyForM3 ? null : "runtime_blocker", source: ["deploy/deploy.yaml", deployWorkloadsPath, deployServicesPath], summary: manifestReadyForM3 ? "Checked-in DEV deploy skeleton declares distinct indexed M3 identities for two box simulators, two gateway simulators, one patch panel, and the M3 DO1 -> DI1 endpoint map." : "Read-only static comparison found the checked-in DEV deploy skeleton does not declare distinct indexed M3 simulator identities and patch-panel M3 wiring; proving live DEV still requires a separate deploy/runtime task or authorized runtime observation.", evidence: { required: { "hwlab-box-simu": 2, "hwlab-gateway-simu": 2, "hwlab-patch-panel": 1, wiring: "res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1" }, observed: { "hwlab-box-simu": { replicas: box?.replicas ?? null, env: box?.env ?? null }, "hwlab-gateway-simu": { replicas: gateway?.replicas ?? null, env: gateway?.env ?? null }, "hwlab-patch-panel": { replicas: patchPanel?.replicas ?? null, firstClassRoute: patchPanelManifest?.m3Route ?? null, env: patchPanel?.env ?? null, firstClassWiringConfig: patchPanelWiringConfig }, instanceServices: [...serviceMap.keys()].filter((name) => name.startsWith("hwlab-box-simu-") || name.startsWith("hwlab-gateway-simu-") ) } }, validationCommand: null, requiredFollowUp: manifestReadyForM3 ? "Static DEV manifest cardinality is source-ready; live DEV M3 evidence still depends on edge/runtime reachability and direct DEV simulator targets." : "Create a separate authorized DEV deploy/runtime task to provision or expose two box-simu instances, two gateway-simu instances, M3 wiring, and audit/evidence endpoints; do not perform that mutation in this read-only continuation." } ]; } async function probeJson(url, options = {}) { const method = options.method ?? "GET"; const startedAt = isoNow(); const timeoutMs = options.timeoutMs ?? 8000; const body = options.body ? JSON.stringify(options.body) : null; const headers = body ? { "content-type": "application/json", "content-length": Buffer.byteLength(body) } : {}; return new Promise((resolve) => { const parsed = new URL(url); const requestFn = parsed.protocol === "https:" ? httpsRequest : httpRequest; const req = requestFn( parsed, { method, headers, timeout: timeoutMs }, (response) => { let text = ""; response.setEncoding("utf8"); response.on("data", (chunk) => { text += chunk; }); response.on("end", () => { let json = null; try { json = text ? JSON.parse(text) : null; } catch { json = null; } const status = response.statusCode ?? 0; resolve({ url, method, startedAt, observedAt: isoNow(), reached: true, ok: status >= 200 && status < 300, status, statusText: response.statusMessage ?? "", json, bodyPreview: json ? undefined : text.slice(0, 500) }); }); } ); req.on("timeout", () => { req.destroy(new Error(`request timed out after ${timeoutMs}ms`)); }); req.on("error", (error) => { resolve({ url, method, startedAt, observedAt: isoNow(), reached: false, ok: false, error: { name: error.name, message: error.message, code: error.code } }); }); if (body) { req.write(body); } req.end(); }); } function joinUrl(baseUrl, routePath) { return new URL(routePath, baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`).toString(); } function endpointReference(target, routePath) { const baseUrl = process.env[target.urlEnv]; if (baseUrl) { return { url: joinUrl(baseUrl, routePath), urlSource: `env:${target.urlEnv}` }; } return { url: `$${target.urlEnv}${routePath}`, urlSource: `required-env:${target.urlEnv}` }; } function plannedServiceEndpoint(target, routePath, { method = "GET", phase, mutatesDevState = false }) { const endpoint = endpointReference(target, routePath); return { phase, targetId: target.id, serviceId: target.serviceId, method, path: routePath, ...endpoint, mutatesDevState, calledInDryRun: false }; } function createLiveOperationPlan() { const box1 = serviceTargets.find((target) => target.id === "box-simu-1"); const box2 = serviceTargets.find((target) => target.id === "box-simu-2"); const patchPanel = serviceTargets.find((target) => target.id === "patch-panel"); const endpointPlan = [ { phase: "dev-ingress-precondition", targetId: "dev-api-edge", serviceId: "hwlab-cloud-api", method: "GET", path: "/health/live", url: joinUrl(DEV_ENDPOINT, "/health/live"), urlSource: "frozen-dev-endpoint", mutatesDevState: false, calledInDryRun: false }, { phase: "dev-ingress-precondition", targetId: "dev-api-edge-json-rpc", serviceId: "hwlab-cloud-api", method: "POST", path: "/json-rpc", url: joinUrl(DEV_ENDPOINT, "/json-rpc"), urlSource: "frozen-dev-endpoint", mutatesDevState: false, calledInDryRun: false, requestShape: "system.health" }, { phase: "dev-ingress-precondition", targetId: "dev-frontend", serviceId: "hwlab-cloud-web", method: "GET", path: "/health/live", url: joinUrl(DEV_FRONTEND_ENDPOINT, "/health/live"), urlSource: "frozen-dev-frontend-endpoint", mutatesDevState: false, calledInDryRun: false } ]; for (const target of serviceTargets) { endpointPlan.push( plannedServiceEndpoint(target, "/health/live", { phase: "direct-target-precondition" }), plannedServiceEndpoint(target, target.statusPath, { phase: "direct-target-precondition" }) ); if (target.wiringPath) { endpointPlan.push( plannedServiceEndpoint(target, target.wiringPath, { phase: "patch-panel-route-precondition" }) ); } } endpointPlan.push( plannedServiceEndpoint(box1, "/ports/write", { method: "POST", phase: "live-write-source-do1", mutatesDevState: true }), plannedServiceEndpoint(patchPanel, patchPanel.routePath, { method: "POST", phase: "live-write-patch-panel-route", mutatesDevState: true }), plannedServiceEndpoint(box2, "/status", { phase: "live-read-target-di1" }) ); return { mode: "plan-only", evidenceLevel: "DRY-RUN", route: "res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1", dryRunCallsLiveEndpoints: false, liveWriteWillRun: false, liveWriteRequiresHumanApproval: true, liveCommand: liveSmokeCommand, legacyLiveCommand: legacyLiveSmokeCommand, liveWritePreconditions: [ "operator has authorized a bounded DEV M3 live smoke window", "command includes --live, --confirm-dev, and --expect-non-prod", "PROD, real hardware, secret reads, service restarts, and force pushes remain forbidden", "DEV ingress identifies HWLAB DEV on the frozen 16666/16667 boundary", "two distinct hwlab-box-simu targets report res_boxsimu_1 and res_boxsimu_2", "two distinct hwlab-gateway-simu targets report distinct DEV gateway sessions", "one hwlab-patch-panel reports active wiring for res_boxsimu_1:DO1 -> res_boxsimu_2:DI1", "cross-device propagation is owned by hwlab-patch-panel; box loopback, SOURCE, LOCAL, fixture, and DRY-RUN output cannot be marked DEV-LIVE" ], endpointPlan, evidenceFields: [ "operationId", "traceId", "auditId", "evidenceId", "sourceResourceId=res_boxsimu_1", "sourcePort=DO1", "routeOwner=hwlab-patch-panel", "routeResponse.propagatedBy=hwlab-patch-panel", "targetResourceId=res_boxsimu_2", "targetPort=DI1", "targetState.ports.DI1.propagatedBy=hwlab-patch-panel" ], failureClassifications: Object.values(m3FailureClassifications), refusalPolicy: [ "no arguments defaults to refusal", "commands missing --live, --confirm-dev, or non-production confirmation stay source/read-only", "non-production confirmation is --expect-non-prod; legacy --confirmed-non-production remains accepted", "a dry-run, source, local, fixture, or read-only blocker report leaves liveOperation.status as not_run or blocked, never pass" ] }; } function isHwlabDevIdentity(probe) { const body = probe.json; if (!probe.ok || !body || typeof body !== "object" || Array.isArray(body)) { return false; } if (typeof body.serviceId !== "string" || !body.serviceId.startsWith("hwlab-")) { return false; } return body.environment === undefined || body.environment === ENVIRONMENT_DEV; } async function observeDevIngress() { const paths = ["/health/live", "/health", "/live", "/v1"]; const probes = []; for (const routePath of paths) { probes.push(await probeJson(joinUrl(DEV_ENDPOINT, routePath))); } probes.push(await probeJson(joinUrl(DEV_FRONTEND_ENDPOINT, "/health/live"))); probes.push(await probeJson(joinUrl(DEV_FRONTEND_ENDPOINT, "/health"))); probes.push(await probeJson(joinUrl(DEV_ENDPOINT, "/json-rpc"), { method: "POST", body: { jsonrpc: "2.0", id: `req_dev_m3_${randomUUID()}`, method: "system.health", params: {}, meta: { traceId: `trc_dev_m3_${randomUUID()}`, actorId: "system_code_queue_d601", serviceId: "hwlab-cloud-api", environment: ENVIRONMENT_DEV } } })); const accepted = probes.find(isHwlabDevIdentity); return { accepted: Boolean(accepted), acceptedPath: accepted ? new URL(accepted.url).pathname : null, probes }; } function targetMapFromEnvironment() { const missing = []; const targets = new Map(); for (const target of serviceTargets) { const baseUrl = process.env[target.urlEnv]; if (!baseUrl) { missing.push(target.urlEnv); continue; } targets.set(target.id, { ...target, baseUrl }); } return { missing, targets }; } function serviceDefinitionByServiceId(serviceId) { return serviceTargets.find((target) => target.serviceId === serviceId); } function kubernetesServiceIdentity(item) { const serviceName = item?.metadata?.labels?.["kubernetes.io/service-name"]; const mapped = kubernetesM3ServiceIdentities[serviceName]; if (!mapped) { return null; } const labels = item?.metadata?.labels ?? {}; return { serviceName, serviceId: mapped.serviceId, targetId: mapped.targetId ?? null, instanceId: labels["hwlab.pikastech.local/instance-id"] ?? mapped.instanceId ?? null }; } function serviceEndpointUrl(address, port) { return `http://${address}:${port}`; } function endpointReady(endpoint) { return endpoint.conditions?.ready !== false && endpoint.conditions?.serving !== false; } function collectKubernetesDirectTargetDiscovery() { if (!commandExists("kubectl")) { return { status: "unavailable", source: "kubectl", command: "command -v kubectl", services: {}, summary: "kubectl is not available in this runner, so Kubernetes endpoint-slice service discovery was not attempted." }; } const args = ["-n", namespace, "get", "endpointslices.discovery.k8s.io", "-o", "json"]; const probe = commandResult("env", [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args], { timeoutMs: 8000, maxChars: 100000 }); const command = runnerKubeconfigCommandText(args); if (probe.exitCode !== 0) { return { status: "blocked", source: "kubernetes-endpointslices", command, exitCode: probe.exitCode, services: {}, summary: oneLine(probe.stderr || probe.stdout || "kubectl endpoint-slice discovery failed.") }; } let document; try { document = JSON.parse(probe.stdout); } catch (error) { return { status: "blocked", source: "kubernetes-endpointslices", command, exitCode: probe.exitCode, services: {}, summary: `kubectl endpoint-slice discovery returned non-JSON output: ${error.message}` }; } const services = {}; for (const item of document.items ?? []) { const identity = kubernetesServiceIdentity(item); if (!identity) { continue; } const { serviceName, serviceId, targetId, instanceId } = identity; const port = item.ports?.find((entry) => entry.name === "http")?.port ?? item.ports?.[0]?.port; if (!Number.isInteger(port)) { continue; } const endpoints = []; for (const endpoint of item.endpoints ?? []) { if (!endpointReady(endpoint)) { continue; } for (const address of endpoint.addresses ?? []) { endpoints.push({ url: serviceEndpointUrl(address, port), address, port, targetRef: endpoint.targetRef?.name ?? "unknown", serviceName, serviceId, targetId, instanceId }); } } services[serviceName] = { endpointSlice: item.metadata?.name ?? "unknown", serviceName, serviceId, targetId, instanceId, port, count: endpoints.length, endpoints }; } return { status: "observed", source: "kubernetes-endpointslices", command, exitCode: probe.exitCode, services, summary: `Observed endpoint slices for ${Object.keys(services).join(", ") || "no M3 services"}.` }; } function candidateTargetsFromKubernetes(discovery) { const candidates = []; for (const [serviceName, service] of Object.entries(discovery.services ?? {})) { const serviceId = service.serviceId ?? serviceName; const definition = serviceDefinitionByServiceId(serviceId); if (!definition) { continue; } let index = 1; for (const endpoint of service.endpoints ?? []) { candidates.push({ ...definition, id: service.targetId ?? endpoint.targetId ?? `${definition.id}-candidate-${index}`, baseUrl: endpoint.url, discoverySource: discovery.source, serviceName, instanceId: endpoint.instanceId ?? service.instanceId ?? null, targetRef: endpoint.targetRef, address: endpoint.address, port: endpoint.port }); index += 1; } } return candidates; } function candidateTargetsFromEnvironment(targets) { return [...targets.values()].map((target) => ({ ...target, discoverySource: "environment" })); } function resourceIdOfBoxStatus(status) { return status?.resource?.resourceId ?? status?.resourceId ?? null; } function gatewayIdentity(status) { return `${status?.gatewayId ?? "unknown"}:${status?.session?.gatewaySessionId ?? status?.gatewaySessionId ?? "unknown"}`; } function summarizeCandidate(candidate) { const body = candidate.status?.json ?? {}; return { id: candidate.id, serviceId: candidate.serviceId, serviceName: candidate.serviceName, instanceId: candidate.instanceId, baseUrl: candidate.baseUrl, targetRef: candidate.targetRef, ok: candidate.ok, statusCode: candidate.status?.status ?? null, healthCode: candidate.health?.status ?? null, boxId: body.boxId ?? null, resourceId: resourceIdOfBoxStatus(body), gatewayId: body.gatewayId ?? null, gatewaySessionId: body.session?.gatewaySessionId ?? body.gatewaySessionId ?? null, patchPanelState: body.state ?? null }; } function formatConnection(connection) { if (!connection) return "none"; if (connection.from && connection.to) { return `${connection.from.resourceId}:${connection.from.port}->${connection.to.resourceId}:${connection.to.port}`; } return `${connection.fromResourceId}:${connection.fromPort}->${connection.toResourceId}:${connection.toPort}`; } function hasRequiredM3Connection({ status, wiring }) { const activeConnections = status?.activeConnections ?? []; const wiringConnections = wiring?.connections ?? []; const owner = Boolean( status?.serviceId === "hwlab-patch-panel" && status?.metadata?.propagation === "patch-panel-only" && wiring?.constraints?.propagation === "patch-panel-only" && wiring?.constraints?.localLoopbackSubstituteAllowed === false ); const active = activeConnections.find( (connection) => connection.fromResourceId === requiredM3Connection.fromResourceId && connection.fromPort === requiredM3Connection.fromPort && connection.toResourceId === requiredM3Connection.toResourceId && connection.toPort === requiredM3Connection.toPort ); const configured = wiringConnections.find( (connection) => connection.from?.resourceId === requiredM3Connection.fromResourceId && connection.from?.port === requiredM3Connection.fromPort && connection.to?.resourceId === requiredM3Connection.toResourceId && connection.to?.port === requiredM3Connection.toPort && connection.mode === "exclusive" ); return { active, configured, owner }; } async function probeDirectCandidate(candidate) { const health = await probeJson(joinUrl(candidate.baseUrl, "/health/live")); const status = await probeJson(joinUrl(candidate.baseUrl, candidate.statusPath)); return { ...candidate, health, status, ok: health.ok && status.ok && health.json?.serviceId === candidate.serviceId }; } async function resolveDirectM3Targets({ environmentTargets, kubernetesDiscovery }) { const allEnvironmentTargetsAvailable = environmentTargets.missing.length === 0; const candidates = allEnvironmentTargetsAvailable ? candidateTargetsFromEnvironment(environmentTargets.targets) : candidateTargetsFromKubernetes(kubernetesDiscovery); const probed = []; for (const candidate of candidates) { probed.push(await probeDirectCandidate(candidate)); } const boxCandidates = probed.filter((candidate) => candidate.serviceId === "hwlab-box-simu" && candidate.ok); const gatewayCandidates = probed.filter((candidate) => candidate.serviceId === "hwlab-gateway-simu" && candidate.ok); const patchCandidates = probed.filter((candidate) => candidate.serviceId === "hwlab-patch-panel" && candidate.ok); const box1 = boxCandidates.find((candidate) => resourceIdOfBoxStatus(candidate.status.json) === "res_boxsimu_1"); const box2 = boxCandidates.find( (candidate) => resourceIdOfBoxStatus(candidate.status.json) === "res_boxsimu_2" && candidate.baseUrl !== box1?.baseUrl ); const distinctBoxResources = new Set(boxCandidates.map((candidate) => resourceIdOfBoxStatus(candidate.status.json)).filter(Boolean)); const distinctGatewayIdentities = new Set(gatewayCandidates.map((candidate) => gatewayIdentity(candidate.status.json))); const selectedGateways = []; for (const candidate of gatewayCandidates) { if (!selectedGateways.some((item) => gatewayIdentity(item.status.json) === gatewayIdentity(candidate.status.json))) { selectedGateways.push(candidate); } } const patch = patchCandidates[0] ?? null; let patchWiring = null; let m3Connection = { active: null, configured: null }; if (patch) { patchWiring = await probeJson(joinUrl(patch.baseUrl, patch.wiringPath)); m3Connection = hasRequiredM3Connection({ status: patch.status.json, wiring: patchWiring.json }); } const blockers = []; const serviceDiscoveryBlocked = !allEnvironmentTargetsAvailable && kubernetesDiscovery.status !== "observed"; if (serviceDiscoveryBlocked) { blockers.push({ type: "observability_blocker", scope: "m3-service-discovery", status: "open", classification: classifyDirectTargetBlockerScope("m3-service-discovery"), sourceIssue: "pikasTech/HWLAB#46", summary: `DEV ingress is reachable on frozen :16667, but direct M3 target discovery via ${kubernetesDiscovery.source} is ${kubernetesDiscovery.status}; environment URLs missing ${environmentTargets.missing.join(", ")}; discovery detail=${kubernetesDiscovery.summary}.` }); } else if (boxCandidates.length < 2 || gatewayCandidates.length < 2 || patchCandidates.length < 1) { blockers.push({ type: "observability_blocker", scope: "m3-direct-target-missing", status: "open", classification: classifyDirectTargetBlockerScope("m3-direct-target-missing"), sourceIssue: "pikasTech/HWLAB#64", summary: `Read-only service discovery did not expose enough callable DEV M3 direct targets: box-simu=${boxCandidates.length}/2, gateway-simu=${gatewayCandidates.length}/2, patch-panel=${patchCandidates.length}/1.` }); } if (boxCandidates.length >= 2 && (!box1 || !box2 || distinctBoxResources.size < 2)) { blockers.push({ type: "runtime_blocker", scope: "m3-box-simu-identity", status: "open", classification: classifyDirectTargetBlockerScope("m3-box-simu-identity"), sourceIssue: "pikasTech/HWLAB#64", summary: `DEV exposes two box-simu endpoints, but live identities are not the required distinct resources res_boxsimu_1 and res_boxsimu_2; observed resources=${[...distinctBoxResources].join(", ") || "none"}.` }); } if (gatewayCandidates.length >= 2 && distinctGatewayIdentities.size < 2) { blockers.push({ type: "runtime_blocker", scope: "m3-gateway-simu-identity", status: "open", classification: classifyDirectTargetBlockerScope("m3-gateway-simu-identity"), sourceIssue: "pikasTech/HWLAB#64", summary: `DEV exposes two gateway-simu endpoints, but live gateway identities are not distinct; observed identities=${[...distinctGatewayIdentities].join(", ") || "none"}.` }); } if (patch && (!m3Connection.active || !m3Connection.configured || !m3Connection.owner)) { const observedActive = (patch.status.json?.activeConnections ?? []).map(formatConnection).join(", ") || "none"; const observedConfigured = (patchWiring?.json?.connections ?? []).map(formatConnection).join(", ") || "none"; const ownerSummary = `serviceId=${patch.status.json?.serviceId ?? "unknown"} propagation=${patch.status.json?.metadata?.propagation ?? "unknown"} wiringPropagation=${patchWiring?.json?.constraints?.propagation ?? "unknown"} loopbackAllowed=${String(patchWiring?.json?.constraints?.localLoopbackSubstituteAllowed ?? "unknown")}`; blockers.push({ type: "runtime_blocker", scope: "m3-patch-panel-wiring", status: "open", classification: m3Connection.owner ? classifyDirectTargetBlockerScope("m3-patch-panel-wiring") : m3FailureClassifications.patchPanelRouteOwnerInvalid, sourceIssue: "pikasTech/HWLAB#64", summary: m3Connection.owner ? `DEV patch-panel is callable, but live wiring does not contain ${requiredM3Connection.fromResourceId}:${requiredM3Connection.fromPort} -> ${requiredM3Connection.toResourceId}:${requiredM3Connection.toPort}; active=${observedActive}; configured=${observedConfigured}.` : `DEV patch-panel route owner is not the required hwlab-patch-panel patch-panel-only owner; ${ownerSummary}; active=${observedActive}; configured=${observedConfigured}.` }); } const ok = blockers.length === 0 && box1 && box2 && selectedGateways.length >= 2 && patch && m3Connection.active && m3Connection.configured && m3Connection.owner; const targets = new Map(); if (ok) { targets.set("box-simu-1", { ...serviceTargets.find((item) => item.id === "box-simu-1"), baseUrl: box1.baseUrl }); targets.set("box-simu-2", { ...serviceTargets.find((item) => item.id === "box-simu-2"), baseUrl: box2.baseUrl }); targets.set("gateway-simu-1", { ...serviceTargets.find((item) => item.id === "gateway-simu-1"), baseUrl: selectedGateways[0].baseUrl }); targets.set("gateway-simu-2", { ...serviceTargets.find((item) => item.id === "gateway-simu-2"), baseUrl: selectedGateways[1].baseUrl }); targets.set("patch-panel", { ...serviceTargets.find((item) => item.id === "patch-panel"), baseUrl: patch.baseUrl }); } return { ok: Boolean(ok), source: allEnvironmentTargetsAvailable ? "environment" : kubernetesDiscovery.source, environmentMissing: environmentTargets.missing, candidates: probed.map(summarizeCandidate), counts: { boxSimu: boxCandidates.length, gatewaySimu: gatewayCandidates.length, patchPanel: patchCandidates.length, distinctBoxResources: distinctBoxResources.size, distinctGatewayIdentities: distinctGatewayIdentities.size }, selected: { boxSimu1: box1 ? summarizeCandidate(box1) : null, boxSimu2: box2 ? summarizeCandidate(box2) : null, gateways: selectedGateways.slice(0, 2).map(summarizeCandidate), patchPanel: patch ? summarizeCandidate(patch) : null }, patchPanelWiring: patch ? { url: patchWiring?.url ?? joinUrl(patch.baseUrl, patch.wiringPath), ok: patchWiring?.ok === true, requiredConnection: requiredM3Connection, activeObserved: (patch.status.json?.activeConnections ?? []).map(formatConnection), configuredObserved: (patchWiring?.json?.connections ?? []).map(formatConnection), hasRequiredActiveConnection: Boolean(m3Connection.active), hasRequiredConfiguredConnection: Boolean(m3Connection.configured), hasRequiredRouteOwner: Boolean(m3Connection.owner), routeOwnerObserved: { serviceId: patch.status.json?.serviceId ?? null, propagation: patch.status.json?.metadata?.propagation ?? null, wiringPropagation: patchWiring?.json?.constraints?.propagation ?? null, localLoopbackSubstituteAllowed: patchWiring?.json?.constraints?.localLoopbackSubstituteAllowed ?? null }, configSource: patch.status.json?.metadata?.configSource ?? "unknown", syncableConnectionCount: patch.status.json?.metadata?.syncableConnectionCount ?? null } : null, blockers, targets }; } function addBlockedM3ChecksFromResolution(report, resolution) { const identityBlocked = resolution.blockers.some((blocker) => blocker.scope === "m3-box-simu-identity" || blocker.scope === "m3-gateway-simu-identity" ); const missingBlocked = resolution.blockers.some((blocker) => blocker.scope === "m3-direct-target-missing" || blocker.scope === "m3-service-discovery" ); const wiringBlocked = resolution.blockers.some((blocker) => blocker.scope === "m3-patch-panel-wiring"); const candidatesEvidence = [JSON.stringify({ source: resolution.source, counts: resolution.counts, selected: resolution.selected })]; report.liveChecks.push( { id: "two-box-simu-online", status: missingBlocked || identityBlocked ? "blocked" : "pass", blockerClass: missingBlocked ? "observability_blocker" : identityBlocked ? "runtime_blocker" : undefined, summary: missingBlocked || identityBlocked ? "Read-only direct target probes did not prove two distinct live DEV box-simu resources res_boxsimu_1 and res_boxsimu_2." : "Read-only direct target probes proved both required DEV box-simu resources.", evidence: candidatesEvidence }, { id: "two-gateway-simu-online", status: missingBlocked || identityBlocked ? "blocked" : "pass", blockerClass: missingBlocked ? "observability_blocker" : identityBlocked ? "runtime_blocker" : undefined, summary: missingBlocked || identityBlocked ? "Read-only direct target probes did not prove two distinct live DEV gateway-simu identities." : "Read-only direct target probes proved two distinct DEV gateway-simu identities.", evidence: candidatesEvidence }, { id: "patch-panel-healthy", status: resolution.selected.patchPanel ? "pass" : "blocked", blockerClass: resolution.selected.patchPanel ? undefined : "observability_blocker", summary: resolution.selected.patchPanel ? "DEV patch-panel direct target returned live status." : "DEV patch-panel direct target was not callable.", evidence: [JSON.stringify(resolution.selected.patchPanel ?? { status: "not_observed" })] }, { id: "wiring-do1-di1-applied", status: wiringBlocked ? "blocked" : resolution.ok ? "pass" : "not_run", blockerClass: wiringBlocked ? "runtime_blocker" : undefined, summary: wiringBlocked ? "DEV patch-panel wiring is active but does not contain the required res_boxsimu_1:DO1 -> res_boxsimu_2:DI1 route." : resolution.ok ? "DEV patch-panel active wiring matched res_boxsimu_1:DO1 -> res_boxsimu_2:DI1." : "Patch-panel wiring check was not conclusive because earlier direct target checks were blocked.", evidence: [JSON.stringify(resolution.patchPanelWiring ?? { status: "not_observed" })] }, { id: "direct-call-do-write-di-read", status: "not_run", summary: "Not attempted because read-only DEV M3 preconditions were blocked; no box-simu loopback or SOURCE/DRY-RUN substitute was used.", evidence: ["No live DEV write was sent."] }, { id: "audit-evidence-traceable", status: "not_run", summary: "Not attempted because the patch-panel-bound live operation was not safe to trigger.", evidence: ["operationId=not_observed", "traceId=not_observed", "auditId=not_observed", "evidenceId=not_observed"] } ); } function assertBoxState(state, label) { assert.equal(state?.serviceId, "hwlab-box-simu", `${label} serviceId`); assert.equal(state.live, true, `${label} live`); assert.equal(state.resource?.environment ?? state.environment, ENVIRONMENT_DEV, `${label} environment`); assert.ok(state.ports?.DO1, `${label} requires DO1`); assert.ok(state.ports?.DI1, `${label} requires DI1`); assert.equal(state.constraints?.crossDevicePropagation, "patch-panel-only", `${label} propagation`); assert.equal(state.constraints?.localLoopbackEnabled, false, `${label} local loopback`); } function assertGatewayState(state, label) { assert.equal(state?.serviceId, "hwlab-gateway-simu", `${label} serviceId`); assert.equal(state.live, true, `${label} live`); assert.equal(state.session?.status, "connected", `${label} session status`); assert.equal(state.session?.environment, ENVIRONMENT_DEV, `${label} environment`); assert.ok(Array.isArray(state.boxes) && state.boxes.length >= 1, `${label} boxes`); } function assertPatchPanelStatus(state) { assert.equal(state?.serviceId, "hwlab-patch-panel", "patch-panel serviceId"); assert.equal(state.state, "active", "patch-panel state"); assert.equal(state.environment, ENVIRONMENT_DEV, "patch-panel environment"); assert.ok(Array.isArray(state.activeConnections), "patch-panel activeConnections"); } function findM3Connection({ status, wiring, box1, box2 }) { const fromResourceId = box1.resource?.resourceId; const toResourceId = box2.resource?.resourceId; const activeConnections = status.activeConnections ?? []; const wiringConnections = wiring.connections ?? []; const active = activeConnections.find( (connection) => connection.fromResourceId === fromResourceId && connection.fromPort === "DO1" && connection.toResourceId === toResourceId && connection.toPort === "DI1" ); const configured = wiringConnections.find( (connection) => connection.from?.resourceId === fromResourceId && connection.from?.port === "DO1" && connection.to?.resourceId === toResourceId && connection.to?.port === "DI1" && connection.mode === "exclusive" ); return { active, configured }; } async function runLiveM3Targets(targets) { const observed = {}; const probes = []; for (const [id, target] of targets.entries()) { const health = await probeJson(joinUrl(target.baseUrl, "/health/live")); const status = await probeJson(joinUrl(target.baseUrl, target.statusPath)); probes.push({ id, kind: "health", probe: health }, { id, kind: "status", probe: status }); assert.ok(health.ok, `${id} health failed`); assert.ok(status.ok, `${id} status failed`); assert.equal(health.json?.serviceId, target.serviceId, `${id} health serviceId`); observed[id] = status.json; } assertBoxState(observed["box-simu-1"], "box-simu-1"); assertBoxState(observed["box-simu-2"], "box-simu-2"); assertGatewayState(observed["gateway-simu-1"], "gateway-simu-1"); assertGatewayState(observed["gateway-simu-2"], "gateway-simu-2"); assertPatchPanelStatus(observed["patch-panel"]); const patchTarget = targets.get("patch-panel"); const wiring = await probeJson(joinUrl(patchTarget.baseUrl, patchTarget.wiringPath)); probes.push({ id: "patch-panel", kind: "wiring", probe: wiring }); assert.ok(wiring.ok, "patch-panel wiring failed"); assert.equal(wiring.json?.status, "active", "wiring status"); assert.equal(wiring.json?.constraints?.propagation, "patch-panel-only", "wiring propagation"); const connection = findM3Connection({ status: observed["patch-panel"], wiring: wiring.json, box1: observed["box-simu-1"], box2: observed["box-simu-2"] }); assert.ok(connection.active, "patch-panel active DO1 -> DI1 connection missing"); assert.ok(connection.configured, "wiring config DO1 -> DI1 connection missing"); const operationId = `op_dev_m3_${randomUUID()}`; const traceId = `trc_dev_m3_${randomUUID()}`; const box1Target = targets.get("box-simu-1"); const box2Target = targets.get("box-simu-2"); const write = await probeJson(joinUrl(box1Target.baseUrl, "/ports/write"), { method: "POST", body: { port: "DO1", value: true, operationId, traceId } }); probes.push({ id: "box-simu-1", kind: "do.write", probe: write }); assert.ok(write.ok, "box-simu-1 DO1 write failed"); assert.equal(write.json?.accepted, true, "box-simu-1 DO1 write accepted"); const route = await probeJson(joinUrl(patchTarget.baseUrl, patchTarget.routePath), { method: "POST", body: { fromResourceId: observed["box-simu-1"].resource.resourceId, fromPort: "DO1", value: true, operationId, traceId } }); probes.push({ id: "patch-panel", kind: "signals.route", probe: route }); assert.ok(route.ok, "patch-panel signal route failed"); assert.equal(route.json?.accepted, true, "patch-panel signal route accepted"); assert.equal(route.json?.propagatedBy, "hwlab-patch-panel", "patch-panel signal route owner"); assert.equal(route.json?.deliveryCount, 1, "patch-panel signal route delivery count"); const after = await probeJson(joinUrl(box2Target.baseUrl, "/status")); probes.push({ id: "box-simu-2", kind: "di.read", probe: after }); assert.ok(after.ok, "box-simu-2 DI1 read failed"); assert.equal(after.json?.ports?.DI1?.value, true, "box-simu-2 DI1 must read true"); const deliveryResponse = route.json?.deliveries?.[0]?.response; const auditId = route.json?.auditId ?? deliveryResponse?.auditId ?? write.json?.auditId ?? after.json?.auditId; const evidenceId = route.json?.evidenceId ?? deliveryResponse?.evidenceId ?? write.json?.evidenceId ?? after.json?.evidenceId; assert.equal(typeof auditId, "string", "live M3 result must include auditId"); assert.equal(typeof evidenceId, "string", "live M3 result must include evidenceId"); return { operationId, traceId, auditId, evidenceId, probes, observed }; } function baseReport({ commitId, observedAt }) { return { $schema: "https://hwlab.pikastech.local/schemas/dev-m3-hardware-loop-report.schema.json", $id: "https://hwlab.pikastech.local/dev-gate/dev-m3-hardware-loop.json", reportVersion: "v1", issue, taskId: "dev-m3-hardware-loop", commitId, acceptanceLevel: "dev_m3_hardware_loop", devOnly: true, prodDisabled: true, reportLifecycle: activeReportLifecycle("Current M3 DEV hardware trusted-loop report; DEV-LIVE requires res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 with operation, audit, and evidence ids."), sourceContract: { status: "pass", documents: [ "docs/reference/MVP-e2e-acceptance.md", "docs/reference/m3-loop-rollout-runbook.md", "fixtures/mvp/m3-hardware-loop/topology.json" ], summary: "Source contracts define DEV-only M3 simulator wiring and prohibit fixture output as live evidence." }, validationCommands: [ "node --check scripts/dev-m3-hardware-loop-smoke.mjs", dryRunPlanCommand, liveSmokeCommand, legacyLiveSmokeCommand, "node --check scripts/validate-dev-gate-report.mjs", "node scripts/validate-dev-gate-report.mjs" ], runtimeTarget: { endpoint: DEV_ENDPOINT, frontendEndpoint: DEV_FRONTEND_ENDPOINT, namespace, environment: ENVIRONMENT_DEV, requiredBoxSimulators: 2, requiredGatewaySimulators: 2, requiredPatchPanels: 1, requiredRoute: "res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1", realHardwareAllowed: false, prodAllowed: false }, safetyGates: { liveFlagRequired: true, confirmDevRequired: true, expectNonProdRequired: true, confirmedNonProductionRequired: true, defaultSourceReadOnly: true, dryRunPlanSupported: true, dryRunCallsLiveEndpoints: false, prodForbidden: true, realHardwareForbidden: true, secretReadForbidden: true, forcePushForbidden: true, unideskRuntimeSubstitutionForbidden: true }, liveChecks: [ { id: "static-contract-parse", status: "pass", summary: "DEV M3 smoke contract files were readable before live probing.", evidence: [ "docs/reference/MVP-e2e-acceptance.md", "docs/reference/m3-loop-rollout-runbook.md" ] }, { id: "endpoint-freeze", status: "pass", summary: `DEV endpoint is frozen at ${DEV_ENDPOINT}.`, evidence: [DEV_ENDPOINT] } ], readOnlySupplementalEvidence: [], localRuntimeObservations: [], d601Observability: { runnerKubeconfigReadable: false, runnerKubeconfigProbeOk: false, runnerKubeconfigProbeExitCode: null, runnerKubeconfigProbeStderr: "not_observed", d601PublicEndpointsReachable: false, d601K3sUnavailable: false, classification: "not_observed", sourceIssue: "pikasTech/HWLAB#46", inferenceRule: "runnerKubeconfigReadable=false must not imply d601K3sUnavailable=true; public 16666/16667 endpoint reachability is tracked separately." }, liveOperation: { status: "not_run", operationId: "not_observed", traceId: "not_observed", auditId: "not_observed", evidenceId: "not_observed", summary: "No live DEV operation has run yet." }, blockers: [], summary: { status: "not_run", observedAt, result: "DEV M3 smoke has not completed." } }; } async function runDryRunPlan({ values }) { await ensureContractFiles(); const reportPath = resolveReportPath(values.get("output")); const observedAt = isoNow(); const report = baseReport({ commitId: currentCommit(), observedAt }); const plan = createLiveOperationPlan(); report.dryRunPlan = plan; report.readOnlySupplementalEvidence = await collectReadOnlySupplementalEvidence(); report.liveChecks.push( { id: "dry-run-live-write-plan", status: "not_run", summary: "Plan-only mode enumerated live write prerequisites and endpoints without calling DEV endpoints or mutating DEV state.", evidence: [ `endpointCount=${plan.endpointPlan.length}`, `mutatingEndpointCount=${plan.endpointPlan.filter((endpoint) => endpoint.mutatesDevState).length}`, "dryRunCallsLiveEndpoints=false" ] }, { id: "two-box-simu-online", status: "not_run", summary: "Plan-only mode requires two distinct live DEV box-simu identities before any write; no DEV target was probed.", evidence: ["required=res_boxsimu_1,res_boxsimu_2"] }, { id: "two-gateway-simu-online", status: "not_run", summary: "Plan-only mode requires two distinct live DEV gateway-simu identities before any write; no DEV target was probed.", evidence: ["required=gwsimu_1,gwsimu_2"] }, { id: "patch-panel-healthy", status: "not_run", summary: "Plan-only mode requires a callable DEV hwlab-patch-panel before any write; no DEV target was probed.", evidence: ["required=hwlab-patch-panel"] }, { id: "wiring-do1-di1-applied", status: "not_run", summary: "Plan-only mode requires patch-panel-owned res_boxsimu_1:DO1 -> res_boxsimu_2:DI1 wiring before any write.", evidence: [`route=${plan.route}`] }, { id: "direct-call-do-write-di-read", status: "not_run", summary: "Plan-only mode did not send /ports/write or /signals/route; this output is DRY-RUN only and cannot be labeled DEV-LIVE.", evidence: ["No live DEV write was sent."] }, { id: "audit-evidence-traceable", status: "not_run", summary: "Plan-only mode listed the required operation, trace, audit, and evidence fields but did not create them.", evidence: plan.evidenceFields } ); report.liveOperation = { status: "not_run", operationId: "not_observed", traceId: "not_observed", auditId: "not_observed", evidenceId: "not_observed", summary: "Dry-run plan only; no DEV-LIVE operation was attempted or claimed." }; report.blockers.push({ type: "safety_blocker", scope: "m3-live-write-authorization", status: "open", classification: m3FailureClassifications.targetMissing, summary: "Plan-only mode is intentionally non-mutating; run the bounded live smoke only after explicit DEV/non-PROD approval and after read-only preconditions identify the exact HWLAB targets." }); report.summary = { status: "blocked", classification: m3FailureClassifications.targetMissing, observedAt, result: "DRY-RUN plan emitted live write prerequisites, endpoint plan, and required evidence fields; it did not call DEV endpoints or produce DEV-LIVE evidence." }; await writeReport(report, reportPath); console.log(`[dev-m3-smoke] mode=dry-run status=blocked report=${relativePath(reportPath)}`); console.log("[dev-m3-smoke] dry-run: no DEV endpoint calls and no live write were sent"); } function addNotRunM3Checks(report, reason) { for (const id of [ "two-box-simu-online", "two-gateway-simu-online", "patch-panel-healthy", "wiring-do1-di1-applied", "direct-call-do-write-di-read", "audit-evidence-traceable" ]) { report.liveChecks.push({ id, status: "not_run", summary: reason, evidence: ["No live DEV evidence collected for this check."] }); } } async function ensureContractFiles() { await access(path.join(repoRoot, "docs/reference/MVP-e2e-acceptance.md")); await access(path.join(repoRoot, "docs/reference/m3-loop-rollout-runbook.md")); await access(path.join(repoRoot, "fixtures/mvp/m3-hardware-loop/topology.json")); } async function writeReport(report, reportPath) { const absoluteReportPath = ensureNotRepoReportsPath(repoRoot, reportPath, "report path"); await mkdir(path.dirname(absoluteReportPath), { recursive: true }); await writeFile(absoluteReportPath, `${JSON.stringify(report, null, 2)}\n`, "utf8"); } async function main() { const args = parseArgs(process.argv.slice(2)); const safety = requireSafetyGates(args); if (safety.mode === "dry-run") { await runDryRunPlan(args); return; } const { values } = args; await ensureContractFiles(); const reportPath = resolveReportPath(values.get("output")); const observedAt = isoNow(); const report = baseReport({ commitId: currentCommit(), observedAt }); report.readOnlySupplementalEvidence = await collectReadOnlySupplementalEvidence(); const ingress = await observeDevIngress(); report.d601Observability = collectD601RunnerObservability(ingress); report.localRuntimeObservations = collectLocalRuntimeObservations(report.d601Observability); report.liveChecks.push({ id: "dev-ingress-health", status: ingress.accepted ? "pass" : "blocked", blockerClass: ingress.accepted ? undefined : "network_blocker", summary: ingress.accepted ? `DEV ingress returned HWLAB identity at ${ingress.acceptedPath}.` : `DEV ingress at ${DEV_ENDPOINT} did not return reachable HWLAB DEV health.`, evidence: ingress.probes.map((probe) => JSON.stringify(probe)) }); if (!ingress.accepted) { addNotRunM3Checks(report, "Stopped at DEV ingress per the DEV smoke matrix."); report.liveOperation = { status: "not_run", operationId: "not_observed", traceId: "not_observed", auditId: "not_observed", evidenceId: "not_observed", summary: "No live do.write -> di.read operation was attempted because DEV ingress is blocked." }; report.blockers.push({ type: "network_blocker", scope: "frp", status: "open", classification: m3FailureClassifications.targetMissing, summary: "Blocked at the #33 DEV runtime readiness condition: public DEV ingress does not accept HWLAB health requests, so #36/M3 hardware-loop checks were not reached." }); report.summary = { status: "blocked", classification: m3FailureClassifications.targetMissing, observedAt, result: "No live DEV M3 evidence was produced; fixture-backed local evidence was not used as a substitute." }; await writeReport(report, reportPath); console.log(`[dev-m3-smoke] status=blocked report=${relativePath(reportPath)}`); console.log("[dev-m3-smoke] blocker=network_blocker scope=dev-ingress-health"); return; } const environmentTargets = targetMapFromEnvironment(); const kubernetesDiscovery = environmentTargets.missing.length === 0 ? { status: "not_run", source: "environment", command: "environment direct target variables", services: {}, summary: "All direct target URLs were supplied by environment variables." } : collectKubernetesDirectTargetDiscovery(); report.directTargetDiscovery = { environmentMissing: environmentTargets.missing, kubernetes: { status: kubernetesDiscovery.status, source: kubernetesDiscovery.source, command: kubernetesDiscovery.command, exitCode: kubernetesDiscovery.exitCode ?? null, services: kubernetesDiscovery.services, summary: kubernetesDiscovery.summary } }; const directTargets = await resolveDirectM3Targets({ environmentTargets, kubernetesDiscovery }); report.directTargetDiscovery = { ...report.directTargetDiscovery, source: directTargets.source, counts: directTargets.counts, selected: directTargets.selected, candidates: directTargets.candidates, patchPanelWiring: directTargets.patchPanelWiring, summary: directTargets.ok ? "Direct DEV M3 targets and patch-panel M3 wiring were discovered; live operation can be attempted." : "Direct DEV M3 targets were probed read-only, but required identities or patch-panel M3 wiring were not satisfied." }; if (!directTargets.ok) { addBlockedM3ChecksFromResolution(report, directTargets); report.liveOperation = { status: "not_run", operationId: "not_observed", traceId: "not_observed", auditId: "not_observed", evidenceId: "not_observed", summary: "No live DEV operation was attempted because read-only direct target checks did not prove the required patch-panel-owned M3 route." }; report.blockers.push(...directTargets.blockers); report.summary = { status: "blocked", classification: directTargets.blockers[0]?.classification ?? m3FailureClassifications.targetMissing, observedAt, result: "DEV ingress and direct targets were reachable, but M3 DEV-LIVE was not triggered because required identities or patch-panel wiring were missing." }; await writeReport(report, reportPath); console.log(`[dev-m3-smoke] status=blocked report=${relativePath(reportPath)}`); console.log(`[dev-m3-smoke] blocker=${directTargets.blockers[0]?.type ?? "runtime_blocker"} scope=${directTargets.blockers[0]?.scope ?? "m3-direct-targets"}`); return; } try { const live = await runLiveM3Targets(directTargets.targets); report.liveChecks.push( { id: "two-box-simu-online", status: "pass", summary: "Both DEV box simulators reported live status with DO1 and DI1 ports.", evidence: ["box-simu-1 /status", "box-simu-2 /status"] }, { id: "two-gateway-simu-online", status: "pass", summary: "Both DEV gateway simulators reported connected sessions.", evidence: ["gateway-simu-1 /status", "gateway-simu-2 /status"] }, { id: "patch-panel-healthy", status: "pass", summary: "DEV patch-panel reported active status.", evidence: ["patch-panel /status"] }, { id: "wiring-do1-di1-applied", status: "pass", summary: "Patch-panel active wiring matched box-simu-1 DO1 -> box-simu-2 DI1.", evidence: ["patch-panel /wiring"] }, { id: "direct-call-do-write-di-read", status: "pass", summary: "Live DEV direct call observed do.write true followed by di.read true.", evidence: [`operationId=${live.operationId}`, `traceId=${live.traceId}`] }, { id: "audit-evidence-traceable", status: "pass", summary: "Live DEV direct call returned audit and evidence identifiers.", evidence: [`auditId=${live.auditId}`, `evidenceId=${live.evidenceId}`] } ); report.liveOperation = { status: "pass", operationId: live.operationId, traceId: live.traceId, auditId: live.auditId, evidenceId: live.evidenceId, summary: "Live DEV M3 do.write true -> di.read true succeeded with traceable audit/evidence identifiers." }; report.rawLiveProbeCount = live.probes.length; report.summary = { status: "pass", observedAt, result: "Live DEV M3 hardware loop passed without fixture substitution." }; await writeReport(report, reportPath); console.log(`[dev-m3-smoke] status=pass report=${relativePath(reportPath)}`); console.log( `[dev-m3-smoke] operationId=${live.operationId} auditId=${live.auditId} evidenceId=${live.evidenceId}` ); } catch (error) { const classification = classifyLiveOperationError(error); addNotRunM3Checks(report, "A live DEV M3 runtime contract check failed before all checks passed."); report.blockers.push({ type: "runtime_blocker", scope: "m3-hardware-loop-runtime", status: "open", classification, summary: error instanceof Error ? error.message : String(error) }); report.summary = { status: "blocked", classification, observedAt, result: "Live DEV M3 runtime was reachable but did not satisfy the M3 hardware-loop contract." }; await writeReport(report, reportPath); console.log(`[dev-m3-smoke] status=blocked report=${relativePath(reportPath)}`); console.log("[dev-m3-smoke] blocker=runtime_blocker scope=m3-hardware-loop-runtime"); } } export { classifyDirectTargetBlockerScope, classifyLiveOperationError, candidateTargetsFromKubernetes, createLiveOperationPlan, collectKubernetesDirectTargetDiscovery, hasRequiredM3Connection, kubernetesServiceIdentity, m3FailureClassifications, parseArgs, requireSafetyGates }; if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { try { await main(); } catch (error) { console.error(`[dev-m3-smoke] ${error instanceof Error ? error.message : String(error)}`); process.exitCode = 1; } }