Codex Agent
1846 lines
68 KiB
JavaScript
1846 lines
68 KiB
JavaScript
#!/usr/bin/env node
|
|
import assert from "node:assert/strict";
|
|
import { execFileSync } from "node:child_process";
|
|
import { randomUUID } from "node:crypto";
|
|
import { accessSync, constants as fsConstants } from "node:fs";
|
|
import { access, mkdir, readFile, writeFile } from "node:fs/promises";
|
|
import { request as httpRequest } from "node:http";
|
|
import { request as httpsRequest } from "node:https";
|
|
import path from "node:path";
|
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
|
|
import { activeReportLifecycle } from "../internal/dev-report-lifecycle.mjs";
|
|
import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT, ENVIRONMENT_DEV } from "../internal/protocol/index.mjs";
|
|
import { ensureNotRepoReportsPath, tempReportPath } from "./src/report-paths.mjs";
|
|
import { readStructuredFile } from "./src/structured-config.mjs";
|
|
|
|
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
const defaultReportPath = tempReportPath("dev-m3-hardware-loop.json");
|
|
const namespace = "hwlab-dev";
|
|
const issue = "pikasTech/HWLAB#38";
|
|
const deployWorkloadsPath = "deploy/k8s/base/workloads.yaml";
|
|
const deployServicesPath = "deploy/k8s/base/services.yaml";
|
|
const dryRunPlanCommand = "node scripts/dev-m3-hardware-loop-smoke.mjs --dry-run";
|
|
const liveSmokeCommand = "node scripts/dev-m3-hardware-loop-smoke.mjs --live --confirm-dev --expect-non-prod";
|
|
const legacyLiveSmokeCommand = "node scripts/dev-m3-hardware-loop-smoke.mjs --live --confirm-dev --confirmed-non-production";
|
|
const requiredM3BoxIds = Object.freeze(["boxsimu_1", "boxsimu_2"]);
|
|
const requiredM3BoxResources = Object.freeze(["res_boxsimu_1", "res_boxsimu_2"]);
|
|
const requiredM3Connection = Object.freeze({
|
|
fromResourceId: "res_boxsimu_1",
|
|
fromPort: "DO1",
|
|
toResourceId: "res_boxsimu_2",
|
|
toPort: "DI1"
|
|
});
|
|
const requiredM3GatewayIds = Object.freeze(["gwsimu_1", "gwsimu_2"]);
|
|
const runnerK3sKubeconfigPath = "/etc/rancher/k3s/k3s.yaml";
|
|
|
|
const m3FailureClassifications = Object.freeze({
|
|
targetMissing: "target_missing",
|
|
identityNotDistinct: "identity_not_distinct",
|
|
patchPanelWiringMissing: "patch_panel_wiring_missing",
|
|
patchPanelRouteOwnerInvalid: "patch_panel_route_owner_invalid",
|
|
operationFailed: "operation_failed",
|
|
evidenceMissing: "evidence_missing"
|
|
});
|
|
|
|
const serviceTargets = Object.freeze([
|
|
{
|
|
id: "box-simu-1",
|
|
serviceId: "hwlab-box-simu",
|
|
urlEnv: "HWLAB_DEV_BOX_SIMU_1_URL",
|
|
statusPath: "/status"
|
|
},
|
|
{
|
|
id: "box-simu-2",
|
|
serviceId: "hwlab-box-simu",
|
|
urlEnv: "HWLAB_DEV_BOX_SIMU_2_URL",
|
|
statusPath: "/status"
|
|
},
|
|
{
|
|
id: "gateway-simu-1",
|
|
serviceId: "hwlab-gateway-simu",
|
|
urlEnv: "HWLAB_DEV_GATEWAY_SIMU_1_URL",
|
|
statusPath: "/status"
|
|
},
|
|
{
|
|
id: "gateway-simu-2",
|
|
serviceId: "hwlab-gateway-simu",
|
|
urlEnv: "HWLAB_DEV_GATEWAY_SIMU_2_URL",
|
|
statusPath: "/status"
|
|
},
|
|
{
|
|
id: "patch-panel",
|
|
serviceId: "hwlab-patch-panel",
|
|
urlEnv: "HWLAB_DEV_PATCH_PANEL_URL",
|
|
statusPath: "/status",
|
|
wiringPath: "/wiring",
|
|
routePath: "/signals/route"
|
|
}
|
|
]);
|
|
|
|
const kubernetesM3ServiceIdentities = Object.freeze({
|
|
"hwlab-box-simu": Object.freeze({
|
|
serviceId: "hwlab-box-simu"
|
|
}),
|
|
"hwlab-box-simu-1": Object.freeze({
|
|
serviceId: "hwlab-box-simu",
|
|
targetId: "box-simu-1",
|
|
instanceId: "res_boxsimu_1"
|
|
}),
|
|
"hwlab-box-simu-2": Object.freeze({
|
|
serviceId: "hwlab-box-simu",
|
|
targetId: "box-simu-2",
|
|
instanceId: "res_boxsimu_2"
|
|
}),
|
|
"hwlab-gateway-simu": Object.freeze({
|
|
serviceId: "hwlab-gateway-simu"
|
|
}),
|
|
"hwlab-gateway-simu-1": Object.freeze({
|
|
serviceId: "hwlab-gateway-simu",
|
|
targetId: "gateway-simu-1",
|
|
instanceId: "gwsimu_1"
|
|
}),
|
|
"hwlab-gateway-simu-2": Object.freeze({
|
|
serviceId: "hwlab-gateway-simu",
|
|
targetId: "gateway-simu-2",
|
|
instanceId: "gwsimu_2"
|
|
}),
|
|
"hwlab-patch-panel": Object.freeze({
|
|
serviceId: "hwlab-patch-panel",
|
|
targetId: "patch-panel"
|
|
})
|
|
});
|
|
|
|
function parseArgs(argv) {
|
|
const flags = new Set();
|
|
const values = new Map();
|
|
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const arg = argv[index];
|
|
if (!arg.startsWith("--")) {
|
|
throw new Error(`unexpected positional argument ${arg}`);
|
|
}
|
|
if (arg === "--output") {
|
|
const value = argv[index + 1];
|
|
assert.ok(value && !value.startsWith("--"), "--output requires a path");
|
|
values.set("output", value);
|
|
index += 1;
|
|
continue;
|
|
}
|
|
flags.add(arg);
|
|
}
|
|
|
|
return {
|
|
flags,
|
|
values,
|
|
dryRun: flags.has("--dry-run") || flags.has("--plan") || flags.has("--source-read-only"),
|
|
live: flags.has("--live")
|
|
};
|
|
}
|
|
|
|
function classifyDirectTargetBlockerScope(scope) {
|
|
if (scope === "m3-patch-panel-wiring") {
|
|
return m3FailureClassifications.patchPanelWiringMissing;
|
|
}
|
|
if (scope === "m3-box-simu-identity" || scope === "m3-gateway-simu-identity") {
|
|
return m3FailureClassifications.identityNotDistinct;
|
|
}
|
|
if (scope === "m3-service-discovery" || scope === "m3-direct-target-missing") {
|
|
return m3FailureClassifications.targetMissing;
|
|
}
|
|
return m3FailureClassifications.operationFailed;
|
|
}
|
|
|
|
function classifyLiveOperationError(error) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
return /\b(auditId|evidenceId)\b/u.test(message)
|
|
? m3FailureClassifications.evidenceMissing
|
|
: m3FailureClassifications.operationFailed;
|
|
}
|
|
|
|
function requireSafetyGates({ flags, dryRun, live }) {
|
|
const forbiddenFlags = [
|
|
"--prod",
|
|
"--real-hardware",
|
|
"--read-secret",
|
|
"--read-token",
|
|
"--force-push",
|
|
"--restart-runtime",
|
|
"--restart-unidesk",
|
|
"--restart-code-queue",
|
|
"--restart-backend-core"
|
|
];
|
|
|
|
for (const flag of forbiddenFlags) {
|
|
assert.ok(!flags.has(flag), `${flag} is forbidden for DEV M3 smoke`);
|
|
}
|
|
|
|
const confirmedLive =
|
|
live &&
|
|
flags.has("--confirm-dev") &&
|
|
(flags.has("--expect-non-prod") || flags.has("--confirmed-non-production")) &&
|
|
!dryRun &&
|
|
!flags.has("--allow-live");
|
|
|
|
if (!confirmedLive) {
|
|
return {
|
|
mode: "dry-run",
|
|
liveWriteAllowed: false
|
|
};
|
|
}
|
|
|
|
return {
|
|
mode: "live",
|
|
liveWriteAllowed: true
|
|
};
|
|
}
|
|
|
|
function isoNow() {
|
|
return new Date().toISOString();
|
|
}
|
|
|
|
function relativePath(absolutePath) {
|
|
return path.relative(repoRoot, absolutePath).replaceAll(path.sep, "/");
|
|
}
|
|
|
|
function resolveReportPath(value) {
|
|
return ensureNotRepoReportsPath(repoRoot, value ?? defaultReportPath, "report path");
|
|
}
|
|
|
|
function currentCommit() {
|
|
try {
|
|
return execFileSync("git", ["rev-parse", "--short=12", "HEAD"], {
|
|
cwd: repoRoot,
|
|
encoding: "utf8",
|
|
stdio: ["ignore", "pipe", "pipe"]
|
|
}).trim();
|
|
} catch {
|
|
return "unknown";
|
|
}
|
|
}
|
|
|
|
function oneLine(value) {
|
|
return String(value).replace(/\s+/g, " ").trim();
|
|
}
|
|
|
|
function commandResult(command, args, { timeoutMs = 5000, maxChars = 3000 } = {}) {
|
|
try {
|
|
const stdout = execFileSync(command, args, {
|
|
cwd: repoRoot,
|
|
encoding: "utf8",
|
|
timeout: timeoutMs,
|
|
stdio: ["ignore", "pipe", "pipe"]
|
|
});
|
|
return {
|
|
command: [command, ...args].join(" "),
|
|
exitCode: 0,
|
|
stdout: stdout.slice(0, maxChars)
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
command: [command, ...args].join(" "),
|
|
exitCode: Number.isInteger(error.status) ? error.status : 1,
|
|
stdout: String(error.stdout ?? "").slice(0, maxChars),
|
|
stderr: String(error.stderr || error.message || "").slice(0, maxChars)
|
|
};
|
|
}
|
|
}
|
|
|
|
function runnerKubeconfigCommandText(args) {
|
|
return [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args].join(" ");
|
|
}
|
|
|
|
function commandExists(command) {
|
|
const result = commandResult("bash", ["-lc", `command -v ${command}`], { timeoutMs: 2000 });
|
|
return result.exitCode === 0 && result.stdout.trim().length > 0;
|
|
}
|
|
|
|
function fileReadable(filePath) {
|
|
try {
|
|
accessSync(filePath, fsConstants.R_OK);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
function collectD601RunnerObservability(ingress) {
|
|
const args = ["-n", namespace, "get", "pods", "-o", "name"];
|
|
const probe = commandResult("env", [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args], {
|
|
timeoutMs: 5000,
|
|
maxChars: 2000
|
|
});
|
|
const stderr = oneLine(probe.stderr || "") || "empty";
|
|
const stdoutLines = String(probe.stdout ?? "")
|
|
.split("\n")
|
|
.map((line) => line.trim())
|
|
.filter(Boolean);
|
|
const d601PublicEndpointsReachable = ingress.probes.some((item) => item.ok && isHwlabDevIdentity(item));
|
|
const runnerKubeconfigReadable = fileReadable(runnerK3sKubeconfigPath);
|
|
const classification = probe.exitCode === 0
|
|
? "read_only_kubectl_available_kubeconfig_file_unreadable"
|
|
: "runner_permission_mount_gap";
|
|
|
|
return {
|
|
runnerKubeconfigReadable,
|
|
runnerKubeconfigProbeOk: probe.exitCode === 0,
|
|
runnerKubeconfigProbeExitCode: probe.exitCode,
|
|
runnerKubeconfigProbeStderr: stderr,
|
|
d601PublicEndpointsReachable,
|
|
d601K3sUnavailable: false,
|
|
classification,
|
|
sourceIssue: "pikasTech/HWLAB#46",
|
|
command: runnerKubeconfigCommandText(args),
|
|
stdoutSummary: probe.exitCode === 0 ? `${stdoutLines.length} pod name(s) observed` : undefined,
|
|
summary: probe.exitCode === 0
|
|
? "Runner /etc/rancher/k3s/k3s.yaml is not readable through fs access, but read-only kubectl observation succeeds; file readability, kubectl reachability, and service discovery are recorded separately."
|
|
: "Runner /etc/rancher/k3s/k3s.yaml is not treated as readable from this container; kubectl read-only observation did not succeed, so this is a runner permission/mount gap, not proof that D601 is globally offline.",
|
|
inferenceRule: "runnerKubeconfigReadable=false must not imply d601K3sUnavailable=true; public 16666/16667 endpoint reachability is tracked separately.",
|
|
secretValuesRead: false,
|
|
secretResourcesRead: false
|
|
};
|
|
}
|
|
|
|
function collectLocalRuntimeObservations(d601Observability) {
|
|
const observations = [];
|
|
|
|
if (commandExists("docker")) {
|
|
const docker = commandResult("docker", [
|
|
"ps",
|
|
"--format",
|
|
"{{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}"
|
|
]);
|
|
const hwlabLines = docker.stdout
|
|
.split("\n")
|
|
.filter((line) => line.includes("hwlab-"))
|
|
.join("\n");
|
|
observations.push({
|
|
id: "docker-hwlab-containers",
|
|
status: hwlabLines ? "observed" : "not_observed",
|
|
command: docker.command,
|
|
summary: oneLine(hwlabLines || "No running docker container name/image/port line contained hwlab-.")
|
|
});
|
|
} else {
|
|
observations.push({
|
|
id: "docker-hwlab-containers",
|
|
status: "unavailable",
|
|
command: "command -v docker",
|
|
summary: "docker CLI is not available in this runner."
|
|
});
|
|
}
|
|
|
|
if (commandExists("kubectl")) {
|
|
const args = ["get", "deploy,po,svc", "-n", namespace, "-o", "wide"];
|
|
const kubectl = commandResult("env", [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args]);
|
|
observations.push({
|
|
id: "kubectl-hwlab-dev",
|
|
status: kubectl.exitCode === 0 ? "observed" : "blocked",
|
|
command: runnerKubeconfigCommandText(args),
|
|
summary: oneLine(kubectl.exitCode === 0 ? kubectl.stdout : kubectl.stderr || kubectl.stdout)
|
|
});
|
|
} else {
|
|
observations.push({
|
|
id: "kubectl-hwlab-dev",
|
|
status: "unavailable",
|
|
command: "command -v kubectl",
|
|
summary: "kubectl is not available in this runner, so hwlab-dev namespace state was not observed."
|
|
});
|
|
}
|
|
|
|
observations.push({
|
|
id: "runner-k3s-kubeconfig-readonly",
|
|
status: d601Observability.runnerKubeconfigProbeOk ? "observed" : "blocked",
|
|
command: d601Observability.command,
|
|
summary: `${d601Observability.summary} runnerKubeconfigReadable=${d601Observability.runnerKubeconfigReadable}; exitCode=${d601Observability.runnerKubeconfigProbeExitCode}; stderr=${d601Observability.runnerKubeconfigProbeStderr}.`
|
|
});
|
|
|
|
return observations;
|
|
}
|
|
|
|
function splitList(value) {
|
|
return String(value ?? "")
|
|
.split(",")
|
|
.map((item) => item.trim())
|
|
.filter(Boolean);
|
|
}
|
|
|
|
function hasSameMembers(actual, expected) {
|
|
return actual.length === expected.length && expected.every((item) => actual.includes(item));
|
|
}
|
|
|
|
function listItems(document) {
|
|
return document?.kind === "List" ? document.items ?? [] : [document].filter(Boolean);
|
|
}
|
|
|
|
function serviceByName(services) {
|
|
return new Map(
|
|
listItems(services)
|
|
.filter((item) => item?.kind === "Service")
|
|
.map((item) => [item.metadata?.name, item])
|
|
);
|
|
}
|
|
|
|
async function collectReadOnlySupplementalEvidence() {
|
|
const [deploy, workloads, services] = await Promise.all([
|
|
readStructuredFile(repoRoot, "deploy/deploy.yaml"),
|
|
readStructuredFile(repoRoot, deployWorkloadsPath),
|
|
readStructuredFile(repoRoot, deployServicesPath)
|
|
]);
|
|
const targets = ["hwlab-box-simu", "hwlab-gateway-simu", "hwlab-patch-panel"];
|
|
const observed = listItems(workloads)
|
|
.filter((item) => targets.includes(item.metadata?.name))
|
|
.map((item) => {
|
|
const container = item.spec?.template?.spec?.containers?.[0] ?? {};
|
|
const env = Object.fromEntries((container.env ?? []).map((entry) => [entry.name, entry.value]));
|
|
return {
|
|
name: item.metadata.name,
|
|
kind: item.kind,
|
|
namespace: item.metadata.namespace,
|
|
replicas: item.spec?.replicas ?? null,
|
|
image: container.image,
|
|
env
|
|
};
|
|
});
|
|
|
|
const box = observed.find((item) => item.name === "hwlab-box-simu");
|
|
const gateway = observed.find((item) => item.name === "hwlab-gateway-simu");
|
|
const patchPanel = observed.find((item) => item.name === "hwlab-patch-panel");
|
|
const patchPanelManifest = (deploy.services ?? []).find((service) => service.serviceId === "hwlab-patch-panel");
|
|
const boxIds = splitList(box?.env.HWLAB_BOX_ID);
|
|
const boxResourceIds = splitList(box?.env.HWLAB_BOX_RESOURCE_ID);
|
|
const boxGatewaySessionIds = splitList(box?.env.HWLAB_GATEWAY_SESSION_ID);
|
|
const gatewayIds = splitList(gateway?.env.HWLAB_GATEWAY_ID);
|
|
const gatewayBoxResources = splitList(gateway?.env.HWLAB_BOX_RESOURCES);
|
|
const gatewayBoxUrls = splitList(gateway?.env.HWLAB_BOX_URLS);
|
|
const serviceMap = serviceByName(services);
|
|
const hasInstanceServices = [
|
|
"hwlab-box-simu-1",
|
|
"hwlab-box-simu-2",
|
|
"hwlab-gateway-simu-1",
|
|
"hwlab-gateway-simu-2"
|
|
].every((name) => serviceMap.has(name));
|
|
const patchPanelEndpointMap = patchPanel?.env.HWLAB_PATCH_PANEL_ENDPOINT_MAP
|
|
? JSON.parse(patchPanel.env.HWLAB_PATCH_PANEL_ENDPOINT_MAP)
|
|
: {};
|
|
const patchPanelWiringConfig = patchPanel?.env.HWLAB_PATCH_PANEL_WIRING_CONFIG
|
|
? JSON.parse(patchPanel.env.HWLAB_PATCH_PANEL_WIRING_CONFIG)
|
|
: null;
|
|
const patchPanelHasManifestRoute =
|
|
patchPanelWiringConfig?.status === "active" &&
|
|
patchPanelWiringConfig?.constraints?.propagation === "patch-panel-only" &&
|
|
patchPanelWiringConfig?.constraints?.localLoopbackSubstituteAllowed === false &&
|
|
(patchPanelWiringConfig?.connections ?? []).some(
|
|
(connection) =>
|
|
connection.from?.resourceId === requiredM3Connection.fromResourceId &&
|
|
connection.from?.port === requiredM3Connection.fromPort &&
|
|
connection.to?.resourceId === requiredM3Connection.toResourceId &&
|
|
connection.to?.port === requiredM3Connection.toPort &&
|
|
connection.mode === "exclusive"
|
|
);
|
|
const boxReadyForM3 =
|
|
box?.kind === "StatefulSet" &&
|
|
box?.replicas === 2 &&
|
|
hasSameMembers(boxIds, requiredM3BoxIds) &&
|
|
hasSameMembers(boxResourceIds, requiredM3BoxResources) &&
|
|
hasSameMembers(boxGatewaySessionIds, ["gws_gwsimu_1", "gws_gwsimu_2"]);
|
|
const gatewayReadyForM3 =
|
|
gateway?.kind === "StatefulSet" &&
|
|
gateway?.replicas === 2 &&
|
|
hasSameMembers(gatewayIds, requiredM3GatewayIds) &&
|
|
hasSameMembers(gatewayBoxResources, requiredM3BoxResources) &&
|
|
hasSameMembers(gatewayBoxUrls, [
|
|
"http://hwlab-box-simu-1.hwlab-dev.svc.cluster.local:7201",
|
|
"http://hwlab-box-simu-2.hwlab-dev.svc.cluster.local:7201"
|
|
]);
|
|
const patchPanelReadyForM3 =
|
|
patchPanel?.replicas === 1 &&
|
|
patchPanelManifest?.m3Route?.fromResourceId === requiredM3Connection.fromResourceId &&
|
|
patchPanelManifest?.m3Route?.fromPort === requiredM3Connection.fromPort &&
|
|
patchPanelManifest?.m3Route?.patchPanelServiceId === "hwlab-patch-panel" &&
|
|
patchPanelManifest?.m3Route?.toResourceId === requiredM3Connection.toResourceId &&
|
|
patchPanelManifest?.m3Route?.toPort === requiredM3Connection.toPort &&
|
|
patchPanelEndpointMap.res_boxsimu_2 === "http://hwlab-box-simu-2.hwlab-dev.svc.cluster.local:7201" &&
|
|
patchPanelHasManifestRoute;
|
|
const manifestReadyForM3 = boxReadyForM3 && gatewayReadyForM3 && patchPanelReadyForM3 && hasInstanceServices;
|
|
|
|
return [
|
|
{
|
|
id: "deploy-skeleton-m3-cardinality",
|
|
status: manifestReadyForM3 ? "manifest-ready" : "gap",
|
|
blockerClass: manifestReadyForM3 ? null : "runtime_blocker",
|
|
source: ["deploy/deploy.yaml", deployWorkloadsPath, deployServicesPath],
|
|
summary: manifestReadyForM3
|
|
? "Checked-in DEV deploy skeleton declares distinct indexed M3 identities for two box simulators, two gateway simulators, one patch panel, and the M3 DO1 -> DI1 endpoint map."
|
|
: "Read-only static comparison found the checked-in DEV deploy skeleton does not declare distinct indexed M3 simulator identities and patch-panel M3 wiring; proving live DEV still requires a separate deploy/runtime task or authorized runtime observation.",
|
|
evidence: {
|
|
required: {
|
|
"hwlab-box-simu": 2,
|
|
"hwlab-gateway-simu": 2,
|
|
"hwlab-patch-panel": 1,
|
|
wiring: "res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1"
|
|
},
|
|
observed: {
|
|
"hwlab-box-simu": {
|
|
replicas: box?.replicas ?? null,
|
|
env: box?.env ?? null
|
|
},
|
|
"hwlab-gateway-simu": {
|
|
replicas: gateway?.replicas ?? null,
|
|
env: gateway?.env ?? null
|
|
},
|
|
"hwlab-patch-panel": {
|
|
replicas: patchPanel?.replicas ?? null,
|
|
firstClassRoute: patchPanelManifest?.m3Route ?? null,
|
|
env: patchPanel?.env ?? null,
|
|
firstClassWiringConfig: patchPanelWiringConfig
|
|
},
|
|
instanceServices: [...serviceMap.keys()].filter((name) =>
|
|
name.startsWith("hwlab-box-simu-") || name.startsWith("hwlab-gateway-simu-")
|
|
)
|
|
}
|
|
},
|
|
validationCommand: null,
|
|
requiredFollowUp: manifestReadyForM3
|
|
? "Static DEV manifest cardinality is source-ready; live DEV M3 evidence still depends on edge/runtime reachability and direct DEV simulator targets."
|
|
: "Create a separate authorized DEV deploy/runtime task to provision or expose two box-simu instances, two gateway-simu instances, M3 wiring, and audit/evidence endpoints; do not perform that mutation in this read-only continuation."
|
|
}
|
|
];
|
|
}
|
|
|
|
async function probeJson(url, options = {}) {
|
|
const method = options.method ?? "GET";
|
|
const startedAt = isoNow();
|
|
const timeoutMs = options.timeoutMs ?? 8000;
|
|
const body = options.body ? JSON.stringify(options.body) : null;
|
|
const headers = body
|
|
? {
|
|
"content-type": "application/json",
|
|
"content-length": Buffer.byteLength(body)
|
|
}
|
|
: {};
|
|
|
|
return new Promise((resolve) => {
|
|
const parsed = new URL(url);
|
|
const requestFn = parsed.protocol === "https:" ? httpsRequest : httpRequest;
|
|
const req = requestFn(
|
|
parsed,
|
|
{
|
|
method,
|
|
headers,
|
|
timeout: timeoutMs
|
|
},
|
|
(response) => {
|
|
let text = "";
|
|
response.setEncoding("utf8");
|
|
response.on("data", (chunk) => {
|
|
text += chunk;
|
|
});
|
|
response.on("end", () => {
|
|
let json = null;
|
|
try {
|
|
json = text ? JSON.parse(text) : null;
|
|
} catch {
|
|
json = null;
|
|
}
|
|
const status = response.statusCode ?? 0;
|
|
resolve({
|
|
url,
|
|
method,
|
|
startedAt,
|
|
observedAt: isoNow(),
|
|
reached: true,
|
|
ok: status >= 200 && status < 300,
|
|
status,
|
|
statusText: response.statusMessage ?? "",
|
|
json,
|
|
bodyPreview: json ? undefined : text.slice(0, 500)
|
|
});
|
|
});
|
|
}
|
|
);
|
|
|
|
req.on("timeout", () => {
|
|
req.destroy(new Error(`request timed out after ${timeoutMs}ms`));
|
|
});
|
|
|
|
req.on("error", (error) => {
|
|
resolve({
|
|
url,
|
|
method,
|
|
startedAt,
|
|
observedAt: isoNow(),
|
|
reached: false,
|
|
ok: false,
|
|
error: {
|
|
name: error.name,
|
|
message: error.message,
|
|
code: error.code
|
|
}
|
|
});
|
|
});
|
|
|
|
if (body) {
|
|
req.write(body);
|
|
}
|
|
req.end();
|
|
});
|
|
}
|
|
|
|
function joinUrl(baseUrl, routePath) {
|
|
return new URL(routePath, baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`).toString();
|
|
}
|
|
|
|
function endpointReference(target, routePath) {
|
|
const baseUrl = process.env[target.urlEnv];
|
|
if (baseUrl) {
|
|
return {
|
|
url: joinUrl(baseUrl, routePath),
|
|
urlSource: `env:${target.urlEnv}`
|
|
};
|
|
}
|
|
|
|
return {
|
|
url: `$${target.urlEnv}${routePath}`,
|
|
urlSource: `required-env:${target.urlEnv}`
|
|
};
|
|
}
|
|
|
|
function plannedServiceEndpoint(target, routePath, { method = "GET", phase, mutatesDevState = false }) {
|
|
const endpoint = endpointReference(target, routePath);
|
|
return {
|
|
phase,
|
|
targetId: target.id,
|
|
serviceId: target.serviceId,
|
|
method,
|
|
path: routePath,
|
|
...endpoint,
|
|
mutatesDevState,
|
|
calledInDryRun: false
|
|
};
|
|
}
|
|
|
|
function createLiveOperationPlan() {
|
|
const box1 = serviceTargets.find((target) => target.id === "box-simu-1");
|
|
const box2 = serviceTargets.find((target) => target.id === "box-simu-2");
|
|
const patchPanel = serviceTargets.find((target) => target.id === "patch-panel");
|
|
const endpointPlan = [
|
|
{
|
|
phase: "dev-ingress-precondition",
|
|
targetId: "dev-api-edge",
|
|
serviceId: "hwlab-cloud-api",
|
|
method: "GET",
|
|
path: "/health/live",
|
|
url: joinUrl(DEV_ENDPOINT, "/health/live"),
|
|
urlSource: "frozen-dev-endpoint",
|
|
mutatesDevState: false,
|
|
calledInDryRun: false
|
|
},
|
|
{
|
|
phase: "dev-ingress-precondition",
|
|
targetId: "dev-api-edge-json-rpc",
|
|
serviceId: "hwlab-cloud-api",
|
|
method: "POST",
|
|
path: "/json-rpc",
|
|
url: joinUrl(DEV_ENDPOINT, "/json-rpc"),
|
|
urlSource: "frozen-dev-endpoint",
|
|
mutatesDevState: false,
|
|
calledInDryRun: false,
|
|
requestShape: "system.health"
|
|
},
|
|
{
|
|
phase: "dev-ingress-precondition",
|
|
targetId: "dev-frontend",
|
|
serviceId: "hwlab-cloud-web",
|
|
method: "GET",
|
|
path: "/health/live",
|
|
url: joinUrl(DEV_FRONTEND_ENDPOINT, "/health/live"),
|
|
urlSource: "frozen-dev-frontend-endpoint",
|
|
mutatesDevState: false,
|
|
calledInDryRun: false
|
|
}
|
|
];
|
|
|
|
for (const target of serviceTargets) {
|
|
endpointPlan.push(
|
|
plannedServiceEndpoint(target, "/health/live", {
|
|
phase: "direct-target-precondition"
|
|
}),
|
|
plannedServiceEndpoint(target, target.statusPath, {
|
|
phase: "direct-target-precondition"
|
|
})
|
|
);
|
|
if (target.wiringPath) {
|
|
endpointPlan.push(
|
|
plannedServiceEndpoint(target, target.wiringPath, {
|
|
phase: "patch-panel-route-precondition"
|
|
})
|
|
);
|
|
}
|
|
}
|
|
|
|
endpointPlan.push(
|
|
plannedServiceEndpoint(box1, "/ports/write", {
|
|
method: "POST",
|
|
phase: "live-write-source-do1",
|
|
mutatesDevState: true
|
|
}),
|
|
plannedServiceEndpoint(patchPanel, patchPanel.routePath, {
|
|
method: "POST",
|
|
phase: "live-write-patch-panel-route",
|
|
mutatesDevState: true
|
|
}),
|
|
plannedServiceEndpoint(box2, "/status", {
|
|
phase: "live-read-target-di1"
|
|
})
|
|
);
|
|
|
|
return {
|
|
mode: "plan-only",
|
|
evidenceLevel: "DRY-RUN",
|
|
route: "res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1",
|
|
dryRunCallsLiveEndpoints: false,
|
|
liveWriteWillRun: false,
|
|
liveWriteRequiresHumanApproval: true,
|
|
liveCommand: liveSmokeCommand,
|
|
legacyLiveCommand: legacyLiveSmokeCommand,
|
|
liveWritePreconditions: [
|
|
"operator has authorized a bounded DEV M3 live smoke window",
|
|
"command includes --live, --confirm-dev, and --expect-non-prod",
|
|
"PROD, real hardware, secret reads, service restarts, and force pushes remain forbidden",
|
|
"DEV ingress identifies HWLAB DEV on the frozen 16666/16667 boundary",
|
|
"two distinct hwlab-box-simu targets report res_boxsimu_1 and res_boxsimu_2",
|
|
"two distinct hwlab-gateway-simu targets report distinct DEV gateway sessions",
|
|
"one hwlab-patch-panel reports active wiring for res_boxsimu_1:DO1 -> res_boxsimu_2:DI1",
|
|
"cross-device propagation is owned by hwlab-patch-panel; box loopback, SOURCE, LOCAL, fixture, and DRY-RUN output cannot be marked DEV-LIVE"
|
|
],
|
|
endpointPlan,
|
|
evidenceFields: [
|
|
"operationId",
|
|
"traceId",
|
|
"auditId",
|
|
"evidenceId",
|
|
"sourceResourceId=res_boxsimu_1",
|
|
"sourcePort=DO1",
|
|
"routeOwner=hwlab-patch-panel",
|
|
"routeResponse.propagatedBy=hwlab-patch-panel",
|
|
"targetResourceId=res_boxsimu_2",
|
|
"targetPort=DI1",
|
|
"targetState.ports.DI1.propagatedBy=hwlab-patch-panel"
|
|
],
|
|
failureClassifications: Object.values(m3FailureClassifications),
|
|
refusalPolicy: [
|
|
"no arguments defaults to refusal",
|
|
"commands missing --live, --confirm-dev, or non-production confirmation stay source/read-only",
|
|
"non-production confirmation is --expect-non-prod; legacy --confirmed-non-production remains accepted",
|
|
"a dry-run, source, local, fixture, or read-only blocker report leaves liveOperation.status as not_run or blocked, never pass"
|
|
]
|
|
};
|
|
}
|
|
|
|
function isHwlabDevIdentity(probe) {
|
|
const body = probe.json;
|
|
if (!probe.ok || !body || typeof body !== "object" || Array.isArray(body)) {
|
|
return false;
|
|
}
|
|
if (typeof body.serviceId !== "string" || !body.serviceId.startsWith("hwlab-")) {
|
|
return false;
|
|
}
|
|
return body.environment === undefined || body.environment === ENVIRONMENT_DEV;
|
|
}
|
|
|
|
async function observeDevIngress() {
|
|
const paths = ["/health/live", "/health", "/live", "/v1"];
|
|
const probes = [];
|
|
for (const routePath of paths) {
|
|
probes.push(await probeJson(joinUrl(DEV_ENDPOINT, routePath)));
|
|
}
|
|
probes.push(await probeJson(joinUrl(DEV_FRONTEND_ENDPOINT, "/health/live")));
|
|
probes.push(await probeJson(joinUrl(DEV_FRONTEND_ENDPOINT, "/health")));
|
|
probes.push(await probeJson(joinUrl(DEV_ENDPOINT, "/json-rpc"), {
|
|
method: "POST",
|
|
body: {
|
|
jsonrpc: "2.0",
|
|
id: `req_dev_m3_${randomUUID()}`,
|
|
method: "system.health",
|
|
params: {},
|
|
meta: {
|
|
traceId: `trc_dev_m3_${randomUUID()}`,
|
|
actorId: "system_code_queue_d601",
|
|
serviceId: "hwlab-cloud-api",
|
|
environment: ENVIRONMENT_DEV
|
|
}
|
|
}
|
|
}));
|
|
|
|
const accepted = probes.find(isHwlabDevIdentity);
|
|
return {
|
|
accepted: Boolean(accepted),
|
|
acceptedPath: accepted ? new URL(accepted.url).pathname : null,
|
|
probes
|
|
};
|
|
}
|
|
|
|
function targetMapFromEnvironment() {
|
|
const missing = [];
|
|
const targets = new Map();
|
|
|
|
for (const target of serviceTargets) {
|
|
const baseUrl = process.env[target.urlEnv];
|
|
if (!baseUrl) {
|
|
missing.push(target.urlEnv);
|
|
continue;
|
|
}
|
|
targets.set(target.id, {
|
|
...target,
|
|
baseUrl
|
|
});
|
|
}
|
|
|
|
return { missing, targets };
|
|
}
|
|
|
|
function serviceDefinitionByServiceId(serviceId) {
|
|
return serviceTargets.find((target) => target.serviceId === serviceId);
|
|
}
|
|
|
|
function kubernetesServiceIdentity(item) {
|
|
const serviceName = item?.metadata?.labels?.["kubernetes.io/service-name"];
|
|
const mapped = kubernetesM3ServiceIdentities[serviceName];
|
|
if (!mapped) {
|
|
return null;
|
|
}
|
|
|
|
const labels = item?.metadata?.labels ?? {};
|
|
return {
|
|
serviceName,
|
|
serviceId: mapped.serviceId,
|
|
targetId: mapped.targetId ?? null,
|
|
instanceId: labels["hwlab.pikastech.local/instance-id"] ?? mapped.instanceId ?? null
|
|
};
|
|
}
|
|
|
|
function serviceEndpointUrl(address, port) {
|
|
return `http://${address}:${port}`;
|
|
}
|
|
|
|
function endpointReady(endpoint) {
|
|
return endpoint.conditions?.ready !== false && endpoint.conditions?.serving !== false;
|
|
}
|
|
|
|
function collectKubernetesDirectTargetDiscovery() {
|
|
if (!commandExists("kubectl")) {
|
|
return {
|
|
status: "unavailable",
|
|
source: "kubectl",
|
|
command: "command -v kubectl",
|
|
services: {},
|
|
summary: "kubectl is not available in this runner, so Kubernetes endpoint-slice service discovery was not attempted."
|
|
};
|
|
}
|
|
|
|
const args = ["-n", namespace, "get", "endpointslices.discovery.k8s.io", "-o", "json"];
|
|
const probe = commandResult("env", [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args], {
|
|
timeoutMs: 8000,
|
|
maxChars: 100000
|
|
});
|
|
const command = runnerKubeconfigCommandText(args);
|
|
|
|
if (probe.exitCode !== 0) {
|
|
return {
|
|
status: "blocked",
|
|
source: "kubernetes-endpointslices",
|
|
command,
|
|
exitCode: probe.exitCode,
|
|
services: {},
|
|
summary: oneLine(probe.stderr || probe.stdout || "kubectl endpoint-slice discovery failed.")
|
|
};
|
|
}
|
|
|
|
let document;
|
|
try {
|
|
document = JSON.parse(probe.stdout);
|
|
} catch (error) {
|
|
return {
|
|
status: "blocked",
|
|
source: "kubernetes-endpointslices",
|
|
command,
|
|
exitCode: probe.exitCode,
|
|
services: {},
|
|
summary: `kubectl endpoint-slice discovery returned non-JSON output: ${error.message}`
|
|
};
|
|
}
|
|
|
|
const services = {};
|
|
|
|
for (const item of document.items ?? []) {
|
|
const identity = kubernetesServiceIdentity(item);
|
|
if (!identity) {
|
|
continue;
|
|
}
|
|
const { serviceName, serviceId, targetId, instanceId } = identity;
|
|
const port = item.ports?.find((entry) => entry.name === "http")?.port ?? item.ports?.[0]?.port;
|
|
if (!Number.isInteger(port)) {
|
|
continue;
|
|
}
|
|
const endpoints = [];
|
|
for (const endpoint of item.endpoints ?? []) {
|
|
if (!endpointReady(endpoint)) {
|
|
continue;
|
|
}
|
|
for (const address of endpoint.addresses ?? []) {
|
|
endpoints.push({
|
|
url: serviceEndpointUrl(address, port),
|
|
address,
|
|
port,
|
|
targetRef: endpoint.targetRef?.name ?? "unknown",
|
|
serviceName,
|
|
serviceId,
|
|
targetId,
|
|
instanceId
|
|
});
|
|
}
|
|
}
|
|
services[serviceName] = {
|
|
endpointSlice: item.metadata?.name ?? "unknown",
|
|
serviceName,
|
|
serviceId,
|
|
targetId,
|
|
instanceId,
|
|
port,
|
|
count: endpoints.length,
|
|
endpoints
|
|
};
|
|
}
|
|
|
|
return {
|
|
status: "observed",
|
|
source: "kubernetes-endpointslices",
|
|
command,
|
|
exitCode: probe.exitCode,
|
|
services,
|
|
summary: `Observed endpoint slices for ${Object.keys(services).join(", ") || "no M3 services"}.`
|
|
};
|
|
}
|
|
|
|
function candidateTargetsFromKubernetes(discovery) {
|
|
const candidates = [];
|
|
for (const [serviceName, service] of Object.entries(discovery.services ?? {})) {
|
|
const serviceId = service.serviceId ?? serviceName;
|
|
const definition = serviceDefinitionByServiceId(serviceId);
|
|
if (!definition) {
|
|
continue;
|
|
}
|
|
let index = 1;
|
|
for (const endpoint of service.endpoints ?? []) {
|
|
candidates.push({
|
|
...definition,
|
|
id: service.targetId ?? endpoint.targetId ?? `${definition.id}-candidate-${index}`,
|
|
baseUrl: endpoint.url,
|
|
discoverySource: discovery.source,
|
|
serviceName,
|
|
instanceId: endpoint.instanceId ?? service.instanceId ?? null,
|
|
targetRef: endpoint.targetRef,
|
|
address: endpoint.address,
|
|
port: endpoint.port
|
|
});
|
|
index += 1;
|
|
}
|
|
}
|
|
return candidates;
|
|
}
|
|
|
|
function candidateTargetsFromEnvironment(targets) {
|
|
return [...targets.values()].map((target) => ({
|
|
...target,
|
|
discoverySource: "environment"
|
|
}));
|
|
}
|
|
|
|
function resourceIdOfBoxStatus(status) {
|
|
return status?.resource?.resourceId ?? status?.resourceId ?? null;
|
|
}
|
|
|
|
function gatewayIdentity(status) {
|
|
return `${status?.gatewayId ?? "unknown"}:${status?.session?.gatewaySessionId ?? status?.gatewaySessionId ?? "unknown"}`;
|
|
}
|
|
|
|
function summarizeCandidate(candidate) {
|
|
const body = candidate.status?.json ?? {};
|
|
return {
|
|
id: candidate.id,
|
|
serviceId: candidate.serviceId,
|
|
serviceName: candidate.serviceName,
|
|
instanceId: candidate.instanceId,
|
|
baseUrl: candidate.baseUrl,
|
|
targetRef: candidate.targetRef,
|
|
ok: candidate.ok,
|
|
statusCode: candidate.status?.status ?? null,
|
|
healthCode: candidate.health?.status ?? null,
|
|
boxId: body.boxId ?? null,
|
|
resourceId: resourceIdOfBoxStatus(body),
|
|
gatewayId: body.gatewayId ?? null,
|
|
gatewaySessionId: body.session?.gatewaySessionId ?? body.gatewaySessionId ?? null,
|
|
patchPanelState: body.state ?? null
|
|
};
|
|
}
|
|
|
|
function formatConnection(connection) {
|
|
if (!connection) return "none";
|
|
if (connection.from && connection.to) {
|
|
return `${connection.from.resourceId}:${connection.from.port}->${connection.to.resourceId}:${connection.to.port}`;
|
|
}
|
|
return `${connection.fromResourceId}:${connection.fromPort}->${connection.toResourceId}:${connection.toPort}`;
|
|
}
|
|
|
|
function hasRequiredM3Connection({ status, wiring }) {
|
|
const activeConnections = status?.activeConnections ?? [];
|
|
const wiringConnections = wiring?.connections ?? [];
|
|
const owner = Boolean(
|
|
status?.serviceId === "hwlab-patch-panel" &&
|
|
status?.metadata?.propagation === "patch-panel-only" &&
|
|
wiring?.constraints?.propagation === "patch-panel-only" &&
|
|
wiring?.constraints?.localLoopbackSubstituteAllowed === false
|
|
);
|
|
const active = activeConnections.find(
|
|
(connection) =>
|
|
connection.fromResourceId === requiredM3Connection.fromResourceId &&
|
|
connection.fromPort === requiredM3Connection.fromPort &&
|
|
connection.toResourceId === requiredM3Connection.toResourceId &&
|
|
connection.toPort === requiredM3Connection.toPort
|
|
);
|
|
const configured = wiringConnections.find(
|
|
(connection) =>
|
|
connection.from?.resourceId === requiredM3Connection.fromResourceId &&
|
|
connection.from?.port === requiredM3Connection.fromPort &&
|
|
connection.to?.resourceId === requiredM3Connection.toResourceId &&
|
|
connection.to?.port === requiredM3Connection.toPort &&
|
|
connection.mode === "exclusive"
|
|
);
|
|
return { active, configured, owner };
|
|
}
|
|
|
|
async function probeDirectCandidate(candidate) {
|
|
const health = await probeJson(joinUrl(candidate.baseUrl, "/health/live"));
|
|
const status = await probeJson(joinUrl(candidate.baseUrl, candidate.statusPath));
|
|
return {
|
|
...candidate,
|
|
health,
|
|
status,
|
|
ok: health.ok && status.ok && health.json?.serviceId === candidate.serviceId
|
|
};
|
|
}
|
|
|
|
async function resolveDirectM3Targets({ environmentTargets, kubernetesDiscovery }) {
|
|
const allEnvironmentTargetsAvailable = environmentTargets.missing.length === 0;
|
|
const candidates = allEnvironmentTargetsAvailable
|
|
? candidateTargetsFromEnvironment(environmentTargets.targets)
|
|
: candidateTargetsFromKubernetes(kubernetesDiscovery);
|
|
const probed = [];
|
|
|
|
for (const candidate of candidates) {
|
|
probed.push(await probeDirectCandidate(candidate));
|
|
}
|
|
|
|
const boxCandidates = probed.filter((candidate) => candidate.serviceId === "hwlab-box-simu" && candidate.ok);
|
|
const gatewayCandidates = probed.filter((candidate) => candidate.serviceId === "hwlab-gateway-simu" && candidate.ok);
|
|
const patchCandidates = probed.filter((candidate) => candidate.serviceId === "hwlab-patch-panel" && candidate.ok);
|
|
const box1 = boxCandidates.find((candidate) => resourceIdOfBoxStatus(candidate.status.json) === "res_boxsimu_1");
|
|
const box2 = boxCandidates.find(
|
|
(candidate) =>
|
|
resourceIdOfBoxStatus(candidate.status.json) === "res_boxsimu_2" &&
|
|
candidate.baseUrl !== box1?.baseUrl
|
|
);
|
|
const distinctBoxResources = new Set(boxCandidates.map((candidate) => resourceIdOfBoxStatus(candidate.status.json)).filter(Boolean));
|
|
const distinctGatewayIdentities = new Set(gatewayCandidates.map((candidate) => gatewayIdentity(candidate.status.json)));
|
|
const selectedGateways = [];
|
|
for (const candidate of gatewayCandidates) {
|
|
if (!selectedGateways.some((item) => gatewayIdentity(item.status.json) === gatewayIdentity(candidate.status.json))) {
|
|
selectedGateways.push(candidate);
|
|
}
|
|
}
|
|
const patch = patchCandidates[0] ?? null;
|
|
let patchWiring = null;
|
|
let m3Connection = { active: null, configured: null };
|
|
|
|
if (patch) {
|
|
patchWiring = await probeJson(joinUrl(patch.baseUrl, patch.wiringPath));
|
|
m3Connection = hasRequiredM3Connection({
|
|
status: patch.status.json,
|
|
wiring: patchWiring.json
|
|
});
|
|
}
|
|
|
|
const blockers = [];
|
|
const serviceDiscoveryBlocked =
|
|
!allEnvironmentTargetsAvailable && kubernetesDiscovery.status !== "observed";
|
|
if (serviceDiscoveryBlocked) {
|
|
blockers.push({
|
|
type: "observability_blocker",
|
|
scope: "m3-service-discovery",
|
|
status: "open",
|
|
classification: classifyDirectTargetBlockerScope("m3-service-discovery"),
|
|
sourceIssue: "pikasTech/HWLAB#46",
|
|
summary: `DEV ingress is reachable on frozen :16667, but direct M3 target discovery via ${kubernetesDiscovery.source} is ${kubernetesDiscovery.status}; environment URLs missing ${environmentTargets.missing.join(", ")}; discovery detail=${kubernetesDiscovery.summary}.`
|
|
});
|
|
} else if (boxCandidates.length < 2 || gatewayCandidates.length < 2 || patchCandidates.length < 1) {
|
|
blockers.push({
|
|
type: "observability_blocker",
|
|
scope: "m3-direct-target-missing",
|
|
status: "open",
|
|
classification: classifyDirectTargetBlockerScope("m3-direct-target-missing"),
|
|
sourceIssue: "pikasTech/HWLAB#64",
|
|
summary: `Read-only service discovery did not expose enough callable DEV M3 direct targets: box-simu=${boxCandidates.length}/2, gateway-simu=${gatewayCandidates.length}/2, patch-panel=${patchCandidates.length}/1.`
|
|
});
|
|
}
|
|
|
|
if (boxCandidates.length >= 2 && (!box1 || !box2 || distinctBoxResources.size < 2)) {
|
|
blockers.push({
|
|
type: "runtime_blocker",
|
|
scope: "m3-box-simu-identity",
|
|
status: "open",
|
|
classification: classifyDirectTargetBlockerScope("m3-box-simu-identity"),
|
|
sourceIssue: "pikasTech/HWLAB#64",
|
|
summary: `DEV exposes two box-simu endpoints, but live identities are not the required distinct resources res_boxsimu_1 and res_boxsimu_2; observed resources=${[...distinctBoxResources].join(", ") || "none"}.`
|
|
});
|
|
}
|
|
|
|
if (gatewayCandidates.length >= 2 && distinctGatewayIdentities.size < 2) {
|
|
blockers.push({
|
|
type: "runtime_blocker",
|
|
scope: "m3-gateway-simu-identity",
|
|
status: "open",
|
|
classification: classifyDirectTargetBlockerScope("m3-gateway-simu-identity"),
|
|
sourceIssue: "pikasTech/HWLAB#64",
|
|
summary: `DEV exposes two gateway-simu endpoints, but live gateway identities are not distinct; observed identities=${[...distinctGatewayIdentities].join(", ") || "none"}.`
|
|
});
|
|
}
|
|
|
|
if (patch && (!m3Connection.active || !m3Connection.configured || !m3Connection.owner)) {
|
|
const observedActive = (patch.status.json?.activeConnections ?? []).map(formatConnection).join(", ") || "none";
|
|
const observedConfigured = (patchWiring?.json?.connections ?? []).map(formatConnection).join(", ") || "none";
|
|
const ownerSummary = `serviceId=${patch.status.json?.serviceId ?? "unknown"} propagation=${patch.status.json?.metadata?.propagation ?? "unknown"} wiringPropagation=${patchWiring?.json?.constraints?.propagation ?? "unknown"} loopbackAllowed=${String(patchWiring?.json?.constraints?.localLoopbackSubstituteAllowed ?? "unknown")}`;
|
|
blockers.push({
|
|
type: "runtime_blocker",
|
|
scope: "m3-patch-panel-wiring",
|
|
status: "open",
|
|
classification: m3Connection.owner
|
|
? classifyDirectTargetBlockerScope("m3-patch-panel-wiring")
|
|
: m3FailureClassifications.patchPanelRouteOwnerInvalid,
|
|
sourceIssue: "pikasTech/HWLAB#64",
|
|
summary: m3Connection.owner
|
|
? `DEV patch-panel is callable, but live wiring does not contain ${requiredM3Connection.fromResourceId}:${requiredM3Connection.fromPort} -> ${requiredM3Connection.toResourceId}:${requiredM3Connection.toPort}; active=${observedActive}; configured=${observedConfigured}.`
|
|
: `DEV patch-panel route owner is not the required hwlab-patch-panel patch-panel-only owner; ${ownerSummary}; active=${observedActive}; configured=${observedConfigured}.`
|
|
});
|
|
}
|
|
|
|
const ok =
|
|
blockers.length === 0 &&
|
|
box1 &&
|
|
box2 &&
|
|
selectedGateways.length >= 2 &&
|
|
patch &&
|
|
m3Connection.active &&
|
|
m3Connection.configured &&
|
|
m3Connection.owner;
|
|
|
|
const targets = new Map();
|
|
if (ok) {
|
|
targets.set("box-simu-1", { ...serviceTargets.find((item) => item.id === "box-simu-1"), baseUrl: box1.baseUrl });
|
|
targets.set("box-simu-2", { ...serviceTargets.find((item) => item.id === "box-simu-2"), baseUrl: box2.baseUrl });
|
|
targets.set("gateway-simu-1", { ...serviceTargets.find((item) => item.id === "gateway-simu-1"), baseUrl: selectedGateways[0].baseUrl });
|
|
targets.set("gateway-simu-2", { ...serviceTargets.find((item) => item.id === "gateway-simu-2"), baseUrl: selectedGateways[1].baseUrl });
|
|
targets.set("patch-panel", { ...serviceTargets.find((item) => item.id === "patch-panel"), baseUrl: patch.baseUrl });
|
|
}
|
|
|
|
return {
|
|
ok: Boolean(ok),
|
|
source: allEnvironmentTargetsAvailable ? "environment" : kubernetesDiscovery.source,
|
|
environmentMissing: environmentTargets.missing,
|
|
candidates: probed.map(summarizeCandidate),
|
|
counts: {
|
|
boxSimu: boxCandidates.length,
|
|
gatewaySimu: gatewayCandidates.length,
|
|
patchPanel: patchCandidates.length,
|
|
distinctBoxResources: distinctBoxResources.size,
|
|
distinctGatewayIdentities: distinctGatewayIdentities.size
|
|
},
|
|
selected: {
|
|
boxSimu1: box1 ? summarizeCandidate(box1) : null,
|
|
boxSimu2: box2 ? summarizeCandidate(box2) : null,
|
|
gateways: selectedGateways.slice(0, 2).map(summarizeCandidate),
|
|
patchPanel: patch ? summarizeCandidate(patch) : null
|
|
},
|
|
patchPanelWiring: patch
|
|
? {
|
|
url: patchWiring?.url ?? joinUrl(patch.baseUrl, patch.wiringPath),
|
|
ok: patchWiring?.ok === true,
|
|
requiredConnection: requiredM3Connection,
|
|
activeObserved: (patch.status.json?.activeConnections ?? []).map(formatConnection),
|
|
configuredObserved: (patchWiring?.json?.connections ?? []).map(formatConnection),
|
|
hasRequiredActiveConnection: Boolean(m3Connection.active),
|
|
hasRequiredConfiguredConnection: Boolean(m3Connection.configured),
|
|
hasRequiredRouteOwner: Boolean(m3Connection.owner),
|
|
routeOwnerObserved: {
|
|
serviceId: patch.status.json?.serviceId ?? null,
|
|
propagation: patch.status.json?.metadata?.propagation ?? null,
|
|
wiringPropagation: patchWiring?.json?.constraints?.propagation ?? null,
|
|
localLoopbackSubstituteAllowed: patchWiring?.json?.constraints?.localLoopbackSubstituteAllowed ?? null
|
|
},
|
|
configSource: patch.status.json?.metadata?.configSource ?? "unknown",
|
|
syncableConnectionCount: patch.status.json?.metadata?.syncableConnectionCount ?? null
|
|
}
|
|
: null,
|
|
blockers,
|
|
targets
|
|
};
|
|
}
|
|
|
|
function addBlockedM3ChecksFromResolution(report, resolution) {
|
|
const identityBlocked = resolution.blockers.some((blocker) =>
|
|
blocker.scope === "m3-box-simu-identity" || blocker.scope === "m3-gateway-simu-identity"
|
|
);
|
|
const missingBlocked = resolution.blockers.some((blocker) =>
|
|
blocker.scope === "m3-direct-target-missing" || blocker.scope === "m3-service-discovery"
|
|
);
|
|
const wiringBlocked = resolution.blockers.some((blocker) => blocker.scope === "m3-patch-panel-wiring");
|
|
const candidatesEvidence = [JSON.stringify({ source: resolution.source, counts: resolution.counts, selected: resolution.selected })];
|
|
|
|
report.liveChecks.push(
|
|
{
|
|
id: "two-box-simu-online",
|
|
status: missingBlocked || identityBlocked ? "blocked" : "pass",
|
|
blockerClass: missingBlocked ? "observability_blocker" : identityBlocked ? "runtime_blocker" : undefined,
|
|
summary: missingBlocked || identityBlocked
|
|
? "Read-only direct target probes did not prove two distinct live DEV box-simu resources res_boxsimu_1 and res_boxsimu_2."
|
|
: "Read-only direct target probes proved both required DEV box-simu resources.",
|
|
evidence: candidatesEvidence
|
|
},
|
|
{
|
|
id: "two-gateway-simu-online",
|
|
status: missingBlocked || identityBlocked ? "blocked" : "pass",
|
|
blockerClass: missingBlocked ? "observability_blocker" : identityBlocked ? "runtime_blocker" : undefined,
|
|
summary: missingBlocked || identityBlocked
|
|
? "Read-only direct target probes did not prove two distinct live DEV gateway-simu identities."
|
|
: "Read-only direct target probes proved two distinct DEV gateway-simu identities.",
|
|
evidence: candidatesEvidence
|
|
},
|
|
{
|
|
id: "patch-panel-healthy",
|
|
status: resolution.selected.patchPanel ? "pass" : "blocked",
|
|
blockerClass: resolution.selected.patchPanel ? undefined : "observability_blocker",
|
|
summary: resolution.selected.patchPanel
|
|
? "DEV patch-panel direct target returned live status."
|
|
: "DEV patch-panel direct target was not callable.",
|
|
evidence: [JSON.stringify(resolution.selected.patchPanel ?? { status: "not_observed" })]
|
|
},
|
|
{
|
|
id: "wiring-do1-di1-applied",
|
|
status: wiringBlocked ? "blocked" : resolution.ok ? "pass" : "not_run",
|
|
blockerClass: wiringBlocked ? "runtime_blocker" : undefined,
|
|
summary: wiringBlocked
|
|
? "DEV patch-panel wiring is active but does not contain the required res_boxsimu_1:DO1 -> res_boxsimu_2:DI1 route."
|
|
: resolution.ok
|
|
? "DEV patch-panel active wiring matched res_boxsimu_1:DO1 -> res_boxsimu_2:DI1."
|
|
: "Patch-panel wiring check was not conclusive because earlier direct target checks were blocked.",
|
|
evidence: [JSON.stringify(resolution.patchPanelWiring ?? { status: "not_observed" })]
|
|
},
|
|
{
|
|
id: "direct-call-do-write-di-read",
|
|
status: "not_run",
|
|
summary: "Not attempted because read-only DEV M3 preconditions were blocked; no box-simu loopback or SOURCE/DRY-RUN substitute was used.",
|
|
evidence: ["No live DEV write was sent."]
|
|
},
|
|
{
|
|
id: "audit-evidence-traceable",
|
|
status: "not_run",
|
|
summary: "Not attempted because the patch-panel-bound live operation was not safe to trigger.",
|
|
evidence: ["operationId=not_observed", "traceId=not_observed", "auditId=not_observed", "evidenceId=not_observed"]
|
|
}
|
|
);
|
|
}
|
|
|
|
function assertBoxState(state, label) {
|
|
assert.equal(state?.serviceId, "hwlab-box-simu", `${label} serviceId`);
|
|
assert.equal(state.live, true, `${label} live`);
|
|
assert.equal(state.resource?.environment ?? state.environment, ENVIRONMENT_DEV, `${label} environment`);
|
|
assert.ok(state.ports?.DO1, `${label} requires DO1`);
|
|
assert.ok(state.ports?.DI1, `${label} requires DI1`);
|
|
assert.equal(state.constraints?.crossDevicePropagation, "patch-panel-only", `${label} propagation`);
|
|
assert.equal(state.constraints?.localLoopbackEnabled, false, `${label} local loopback`);
|
|
}
|
|
|
|
function assertGatewayState(state, label) {
|
|
assert.equal(state?.serviceId, "hwlab-gateway-simu", `${label} serviceId`);
|
|
assert.equal(state.live, true, `${label} live`);
|
|
assert.equal(state.session?.status, "connected", `${label} session status`);
|
|
assert.equal(state.session?.environment, ENVIRONMENT_DEV, `${label} environment`);
|
|
assert.ok(Array.isArray(state.boxes) && state.boxes.length >= 1, `${label} boxes`);
|
|
}
|
|
|
|
function assertPatchPanelStatus(state) {
|
|
assert.equal(state?.serviceId, "hwlab-patch-panel", "patch-panel serviceId");
|
|
assert.equal(state.state, "active", "patch-panel state");
|
|
assert.equal(state.environment, ENVIRONMENT_DEV, "patch-panel environment");
|
|
assert.ok(Array.isArray(state.activeConnections), "patch-panel activeConnections");
|
|
}
|
|
|
|
function findM3Connection({ status, wiring, box1, box2 }) {
|
|
const fromResourceId = box1.resource?.resourceId;
|
|
const toResourceId = box2.resource?.resourceId;
|
|
const activeConnections = status.activeConnections ?? [];
|
|
const wiringConnections = wiring.connections ?? [];
|
|
|
|
const active = activeConnections.find(
|
|
(connection) =>
|
|
connection.fromResourceId === fromResourceId &&
|
|
connection.fromPort === "DO1" &&
|
|
connection.toResourceId === toResourceId &&
|
|
connection.toPort === "DI1"
|
|
);
|
|
const configured = wiringConnections.find(
|
|
(connection) =>
|
|
connection.from?.resourceId === fromResourceId &&
|
|
connection.from?.port === "DO1" &&
|
|
connection.to?.resourceId === toResourceId &&
|
|
connection.to?.port === "DI1" &&
|
|
connection.mode === "exclusive"
|
|
);
|
|
|
|
return { active, configured };
|
|
}
|
|
|
|
async function runLiveM3Targets(targets) {
|
|
const observed = {};
|
|
const probes = [];
|
|
|
|
for (const [id, target] of targets.entries()) {
|
|
const health = await probeJson(joinUrl(target.baseUrl, "/health/live"));
|
|
const status = await probeJson(joinUrl(target.baseUrl, target.statusPath));
|
|
probes.push({ id, kind: "health", probe: health }, { id, kind: "status", probe: status });
|
|
assert.ok(health.ok, `${id} health failed`);
|
|
assert.ok(status.ok, `${id} status failed`);
|
|
assert.equal(health.json?.serviceId, target.serviceId, `${id} health serviceId`);
|
|
observed[id] = status.json;
|
|
}
|
|
|
|
assertBoxState(observed["box-simu-1"], "box-simu-1");
|
|
assertBoxState(observed["box-simu-2"], "box-simu-2");
|
|
assertGatewayState(observed["gateway-simu-1"], "gateway-simu-1");
|
|
assertGatewayState(observed["gateway-simu-2"], "gateway-simu-2");
|
|
assertPatchPanelStatus(observed["patch-panel"]);
|
|
|
|
const patchTarget = targets.get("patch-panel");
|
|
const wiring = await probeJson(joinUrl(patchTarget.baseUrl, patchTarget.wiringPath));
|
|
probes.push({ id: "patch-panel", kind: "wiring", probe: wiring });
|
|
assert.ok(wiring.ok, "patch-panel wiring failed");
|
|
assert.equal(wiring.json?.status, "active", "wiring status");
|
|
assert.equal(wiring.json?.constraints?.propagation, "patch-panel-only", "wiring propagation");
|
|
|
|
const connection = findM3Connection({
|
|
status: observed["patch-panel"],
|
|
wiring: wiring.json,
|
|
box1: observed["box-simu-1"],
|
|
box2: observed["box-simu-2"]
|
|
});
|
|
assert.ok(connection.active, "patch-panel active DO1 -> DI1 connection missing");
|
|
assert.ok(connection.configured, "wiring config DO1 -> DI1 connection missing");
|
|
|
|
const operationId = `op_dev_m3_${randomUUID()}`;
|
|
const traceId = `trc_dev_m3_${randomUUID()}`;
|
|
const box1Target = targets.get("box-simu-1");
|
|
const box2Target = targets.get("box-simu-2");
|
|
|
|
const write = await probeJson(joinUrl(box1Target.baseUrl, "/ports/write"), {
|
|
method: "POST",
|
|
body: {
|
|
port: "DO1",
|
|
value: true,
|
|
operationId,
|
|
traceId
|
|
}
|
|
});
|
|
probes.push({ id: "box-simu-1", kind: "do.write", probe: write });
|
|
assert.ok(write.ok, "box-simu-1 DO1 write failed");
|
|
assert.equal(write.json?.accepted, true, "box-simu-1 DO1 write accepted");
|
|
|
|
const route = await probeJson(joinUrl(patchTarget.baseUrl, patchTarget.routePath), {
|
|
method: "POST",
|
|
body: {
|
|
fromResourceId: observed["box-simu-1"].resource.resourceId,
|
|
fromPort: "DO1",
|
|
value: true,
|
|
operationId,
|
|
traceId
|
|
}
|
|
});
|
|
probes.push({ id: "patch-panel", kind: "signals.route", probe: route });
|
|
assert.ok(route.ok, "patch-panel signal route failed");
|
|
assert.equal(route.json?.accepted, true, "patch-panel signal route accepted");
|
|
assert.equal(route.json?.propagatedBy, "hwlab-patch-panel", "patch-panel signal route owner");
|
|
assert.equal(route.json?.deliveryCount, 1, "patch-panel signal route delivery count");
|
|
|
|
const after = await probeJson(joinUrl(box2Target.baseUrl, "/status"));
|
|
probes.push({ id: "box-simu-2", kind: "di.read", probe: after });
|
|
assert.ok(after.ok, "box-simu-2 DI1 read failed");
|
|
assert.equal(after.json?.ports?.DI1?.value, true, "box-simu-2 DI1 must read true");
|
|
|
|
const deliveryResponse = route.json?.deliveries?.[0]?.response;
|
|
const auditId = route.json?.auditId ?? deliveryResponse?.auditId ?? write.json?.auditId ?? after.json?.auditId;
|
|
const evidenceId = route.json?.evidenceId ?? deliveryResponse?.evidenceId ?? write.json?.evidenceId ?? after.json?.evidenceId;
|
|
assert.equal(typeof auditId, "string", "live M3 result must include auditId");
|
|
assert.equal(typeof evidenceId, "string", "live M3 result must include evidenceId");
|
|
|
|
return {
|
|
operationId,
|
|
traceId,
|
|
auditId,
|
|
evidenceId,
|
|
probes,
|
|
observed
|
|
};
|
|
}
|
|
|
|
function baseReport({ commitId, observedAt }) {
|
|
return {
|
|
$schema: "https://hwlab.pikastech.local/schemas/dev-m3-hardware-loop-report.schema.json",
|
|
$id: "https://hwlab.pikastech.local/dev-gate/dev-m3-hardware-loop.json",
|
|
reportVersion: "v1",
|
|
issue,
|
|
taskId: "dev-m3-hardware-loop",
|
|
commitId,
|
|
acceptanceLevel: "dev_m3_hardware_loop",
|
|
devOnly: true,
|
|
prodDisabled: true,
|
|
reportLifecycle: activeReportLifecycle("Current M3 DEV hardware trusted-loop report; DEV-LIVE requires res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 with operation, audit, and evidence ids."),
|
|
sourceContract: {
|
|
status: "pass",
|
|
documents: [
|
|
"docs/reference/MVP-e2e-acceptance.md",
|
|
"docs/reference/m3-loop-rollout-runbook.md",
|
|
"fixtures/mvp/m3-hardware-loop/topology.json"
|
|
],
|
|
summary: "Source contracts define DEV-only M3 simulator wiring and prohibit fixture output as live evidence."
|
|
},
|
|
validationCommands: [
|
|
"node --check scripts/dev-m3-hardware-loop-smoke.mjs",
|
|
dryRunPlanCommand,
|
|
liveSmokeCommand,
|
|
legacyLiveSmokeCommand,
|
|
"node --check scripts/validate-dev-gate-report.mjs",
|
|
"node scripts/validate-dev-gate-report.mjs"
|
|
],
|
|
runtimeTarget: {
|
|
endpoint: DEV_ENDPOINT,
|
|
frontendEndpoint: DEV_FRONTEND_ENDPOINT,
|
|
namespace,
|
|
environment: ENVIRONMENT_DEV,
|
|
requiredBoxSimulators: 2,
|
|
requiredGatewaySimulators: 2,
|
|
requiredPatchPanels: 1,
|
|
requiredRoute: "res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1",
|
|
realHardwareAllowed: false,
|
|
prodAllowed: false
|
|
},
|
|
safetyGates: {
|
|
liveFlagRequired: true,
|
|
confirmDevRequired: true,
|
|
expectNonProdRequired: true,
|
|
confirmedNonProductionRequired: true,
|
|
defaultSourceReadOnly: true,
|
|
dryRunPlanSupported: true,
|
|
dryRunCallsLiveEndpoints: false,
|
|
prodForbidden: true,
|
|
realHardwareForbidden: true,
|
|
secretReadForbidden: true,
|
|
forcePushForbidden: true,
|
|
unideskRuntimeSubstitutionForbidden: true
|
|
},
|
|
liveChecks: [
|
|
{
|
|
id: "static-contract-parse",
|
|
status: "pass",
|
|
summary: "DEV M3 smoke contract files were readable before live probing.",
|
|
evidence: [
|
|
"docs/reference/MVP-e2e-acceptance.md",
|
|
"docs/reference/m3-loop-rollout-runbook.md"
|
|
]
|
|
},
|
|
{
|
|
id: "endpoint-freeze",
|
|
status: "pass",
|
|
summary: `DEV endpoint is frozen at ${DEV_ENDPOINT}.`,
|
|
evidence: [DEV_ENDPOINT]
|
|
}
|
|
],
|
|
readOnlySupplementalEvidence: [],
|
|
localRuntimeObservations: [],
|
|
d601Observability: {
|
|
runnerKubeconfigReadable: false,
|
|
runnerKubeconfigProbeOk: false,
|
|
runnerKubeconfigProbeExitCode: null,
|
|
runnerKubeconfigProbeStderr: "not_observed",
|
|
d601PublicEndpointsReachable: false,
|
|
d601K3sUnavailable: false,
|
|
classification: "not_observed",
|
|
sourceIssue: "pikasTech/HWLAB#46",
|
|
inferenceRule: "runnerKubeconfigReadable=false must not imply d601K3sUnavailable=true; public 16666/16667 endpoint reachability is tracked separately."
|
|
},
|
|
liveOperation: {
|
|
status: "not_run",
|
|
operationId: "not_observed",
|
|
traceId: "not_observed",
|
|
auditId: "not_observed",
|
|
evidenceId: "not_observed",
|
|
summary: "No live DEV operation has run yet."
|
|
},
|
|
blockers: [],
|
|
summary: {
|
|
status: "not_run",
|
|
observedAt,
|
|
result: "DEV M3 smoke has not completed."
|
|
}
|
|
};
|
|
}
|
|
|
|
async function runDryRunPlan({ values }) {
|
|
await ensureContractFiles();
|
|
|
|
const reportPath = resolveReportPath(values.get("output"));
|
|
const observedAt = isoNow();
|
|
const report = baseReport({
|
|
commitId: currentCommit(),
|
|
observedAt
|
|
});
|
|
const plan = createLiveOperationPlan();
|
|
|
|
report.dryRunPlan = plan;
|
|
report.readOnlySupplementalEvidence = await collectReadOnlySupplementalEvidence();
|
|
report.liveChecks.push(
|
|
{
|
|
id: "dry-run-live-write-plan",
|
|
status: "not_run",
|
|
summary: "Plan-only mode enumerated live write prerequisites and endpoints without calling DEV endpoints or mutating DEV state.",
|
|
evidence: [
|
|
`endpointCount=${plan.endpointPlan.length}`,
|
|
`mutatingEndpointCount=${plan.endpointPlan.filter((endpoint) => endpoint.mutatesDevState).length}`,
|
|
"dryRunCallsLiveEndpoints=false"
|
|
]
|
|
},
|
|
{
|
|
id: "two-box-simu-online",
|
|
status: "not_run",
|
|
summary: "Plan-only mode requires two distinct live DEV box-simu identities before any write; no DEV target was probed.",
|
|
evidence: ["required=res_boxsimu_1,res_boxsimu_2"]
|
|
},
|
|
{
|
|
id: "two-gateway-simu-online",
|
|
status: "not_run",
|
|
summary: "Plan-only mode requires two distinct live DEV gateway-simu identities before any write; no DEV target was probed.",
|
|
evidence: ["required=gwsimu_1,gwsimu_2"]
|
|
},
|
|
{
|
|
id: "patch-panel-healthy",
|
|
status: "not_run",
|
|
summary: "Plan-only mode requires a callable DEV hwlab-patch-panel before any write; no DEV target was probed.",
|
|
evidence: ["required=hwlab-patch-panel"]
|
|
},
|
|
{
|
|
id: "wiring-do1-di1-applied",
|
|
status: "not_run",
|
|
summary: "Plan-only mode requires patch-panel-owned res_boxsimu_1:DO1 -> res_boxsimu_2:DI1 wiring before any write.",
|
|
evidence: [`route=${plan.route}`]
|
|
},
|
|
{
|
|
id: "direct-call-do-write-di-read",
|
|
status: "not_run",
|
|
summary: "Plan-only mode did not send /ports/write or /signals/route; this output is DRY-RUN only and cannot be labeled DEV-LIVE.",
|
|
evidence: ["No live DEV write was sent."]
|
|
},
|
|
{
|
|
id: "audit-evidence-traceable",
|
|
status: "not_run",
|
|
summary: "Plan-only mode listed the required operation, trace, audit, and evidence fields but did not create them.",
|
|
evidence: plan.evidenceFields
|
|
}
|
|
);
|
|
report.liveOperation = {
|
|
status: "not_run",
|
|
operationId: "not_observed",
|
|
traceId: "not_observed",
|
|
auditId: "not_observed",
|
|
evidenceId: "not_observed",
|
|
summary: "Dry-run plan only; no DEV-LIVE operation was attempted or claimed."
|
|
};
|
|
report.blockers.push({
|
|
type: "safety_blocker",
|
|
scope: "m3-live-write-authorization",
|
|
status: "open",
|
|
classification: m3FailureClassifications.targetMissing,
|
|
summary: "Plan-only mode is intentionally non-mutating; run the bounded live smoke only after explicit DEV/non-PROD approval and after read-only preconditions identify the exact HWLAB targets."
|
|
});
|
|
report.summary = {
|
|
status: "blocked",
|
|
classification: m3FailureClassifications.targetMissing,
|
|
observedAt,
|
|
result: "DRY-RUN plan emitted live write prerequisites, endpoint plan, and required evidence fields; it did not call DEV endpoints or produce DEV-LIVE evidence."
|
|
};
|
|
|
|
await writeReport(report, reportPath);
|
|
console.log(`[dev-m3-smoke] mode=dry-run status=blocked report=${relativePath(reportPath)}`);
|
|
console.log("[dev-m3-smoke] dry-run: no DEV endpoint calls and no live write were sent");
|
|
}
|
|
|
|
function addNotRunM3Checks(report, reason) {
|
|
for (const id of [
|
|
"two-box-simu-online",
|
|
"two-gateway-simu-online",
|
|
"patch-panel-healthy",
|
|
"wiring-do1-di1-applied",
|
|
"direct-call-do-write-di-read",
|
|
"audit-evidence-traceable"
|
|
]) {
|
|
report.liveChecks.push({
|
|
id,
|
|
status: "not_run",
|
|
summary: reason,
|
|
evidence: ["No live DEV evidence collected for this check."]
|
|
});
|
|
}
|
|
}
|
|
|
|
async function ensureContractFiles() {
|
|
await access(path.join(repoRoot, "docs/reference/MVP-e2e-acceptance.md"));
|
|
await access(path.join(repoRoot, "docs/reference/m3-loop-rollout-runbook.md"));
|
|
await access(path.join(repoRoot, "fixtures/mvp/m3-hardware-loop/topology.json"));
|
|
}
|
|
|
|
async function writeReport(report, reportPath) {
|
|
const absoluteReportPath = ensureNotRepoReportsPath(repoRoot, reportPath, "report path");
|
|
await mkdir(path.dirname(absoluteReportPath), { recursive: true });
|
|
await writeFile(absoluteReportPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
|
}
|
|
|
|
async function main() {
|
|
const args = parseArgs(process.argv.slice(2));
|
|
const safety = requireSafetyGates(args);
|
|
if (safety.mode === "dry-run") {
|
|
await runDryRunPlan(args);
|
|
return;
|
|
}
|
|
|
|
const { values } = args;
|
|
await ensureContractFiles();
|
|
|
|
const reportPath = resolveReportPath(values.get("output"));
|
|
const observedAt = isoNow();
|
|
const report = baseReport({
|
|
commitId: currentCommit(),
|
|
observedAt
|
|
});
|
|
report.readOnlySupplementalEvidence = await collectReadOnlySupplementalEvidence();
|
|
|
|
const ingress = await observeDevIngress();
|
|
report.d601Observability = collectD601RunnerObservability(ingress);
|
|
report.localRuntimeObservations = collectLocalRuntimeObservations(report.d601Observability);
|
|
report.liveChecks.push({
|
|
id: "dev-ingress-health",
|
|
status: ingress.accepted ? "pass" : "blocked",
|
|
blockerClass: ingress.accepted ? undefined : "network_blocker",
|
|
summary: ingress.accepted
|
|
? `DEV ingress returned HWLAB identity at ${ingress.acceptedPath}.`
|
|
: `DEV ingress at ${DEV_ENDPOINT} did not return reachable HWLAB DEV health.`,
|
|
evidence: ingress.probes.map((probe) => JSON.stringify(probe))
|
|
});
|
|
|
|
if (!ingress.accepted) {
|
|
addNotRunM3Checks(report, "Stopped at DEV ingress per the DEV smoke matrix.");
|
|
report.liveOperation = {
|
|
status: "not_run",
|
|
operationId: "not_observed",
|
|
traceId: "not_observed",
|
|
auditId: "not_observed",
|
|
evidenceId: "not_observed",
|
|
summary: "No live do.write -> di.read operation was attempted because DEV ingress is blocked."
|
|
};
|
|
report.blockers.push({
|
|
type: "network_blocker",
|
|
scope: "frp",
|
|
status: "open",
|
|
classification: m3FailureClassifications.targetMissing,
|
|
summary: "Blocked at the #33 DEV runtime readiness condition: public DEV ingress does not accept HWLAB health requests, so #36/M3 hardware-loop checks were not reached."
|
|
});
|
|
report.summary = {
|
|
status: "blocked",
|
|
classification: m3FailureClassifications.targetMissing,
|
|
observedAt,
|
|
result: "No live DEV M3 evidence was produced; fixture-backed local evidence was not used as a substitute."
|
|
};
|
|
await writeReport(report, reportPath);
|
|
console.log(`[dev-m3-smoke] status=blocked report=${relativePath(reportPath)}`);
|
|
console.log("[dev-m3-smoke] blocker=network_blocker scope=dev-ingress-health");
|
|
return;
|
|
}
|
|
|
|
const environmentTargets = targetMapFromEnvironment();
|
|
const kubernetesDiscovery = environmentTargets.missing.length === 0
|
|
? {
|
|
status: "not_run",
|
|
source: "environment",
|
|
command: "environment direct target variables",
|
|
services: {},
|
|
summary: "All direct target URLs were supplied by environment variables."
|
|
}
|
|
: collectKubernetesDirectTargetDiscovery();
|
|
report.directTargetDiscovery = {
|
|
environmentMissing: environmentTargets.missing,
|
|
kubernetes: {
|
|
status: kubernetesDiscovery.status,
|
|
source: kubernetesDiscovery.source,
|
|
command: kubernetesDiscovery.command,
|
|
exitCode: kubernetesDiscovery.exitCode ?? null,
|
|
services: kubernetesDiscovery.services,
|
|
summary: kubernetesDiscovery.summary
|
|
}
|
|
};
|
|
const directTargets = await resolveDirectM3Targets({
|
|
environmentTargets,
|
|
kubernetesDiscovery
|
|
});
|
|
report.directTargetDiscovery = {
|
|
...report.directTargetDiscovery,
|
|
source: directTargets.source,
|
|
counts: directTargets.counts,
|
|
selected: directTargets.selected,
|
|
candidates: directTargets.candidates,
|
|
patchPanelWiring: directTargets.patchPanelWiring,
|
|
summary: directTargets.ok
|
|
? "Direct DEV M3 targets and patch-panel M3 wiring were discovered; live operation can be attempted."
|
|
: "Direct DEV M3 targets were probed read-only, but required identities or patch-panel M3 wiring were not satisfied."
|
|
};
|
|
|
|
if (!directTargets.ok) {
|
|
addBlockedM3ChecksFromResolution(report, directTargets);
|
|
report.liveOperation = {
|
|
status: "not_run",
|
|
operationId: "not_observed",
|
|
traceId: "not_observed",
|
|
auditId: "not_observed",
|
|
evidenceId: "not_observed",
|
|
summary: "No live DEV operation was attempted because read-only direct target checks did not prove the required patch-panel-owned M3 route."
|
|
};
|
|
report.blockers.push(...directTargets.blockers);
|
|
report.summary = {
|
|
status: "blocked",
|
|
classification: directTargets.blockers[0]?.classification ?? m3FailureClassifications.targetMissing,
|
|
observedAt,
|
|
result: "DEV ingress and direct targets were reachable, but M3 DEV-LIVE was not triggered because required identities or patch-panel wiring were missing."
|
|
};
|
|
await writeReport(report, reportPath);
|
|
console.log(`[dev-m3-smoke] status=blocked report=${relativePath(reportPath)}`);
|
|
console.log(`[dev-m3-smoke] blocker=${directTargets.blockers[0]?.type ?? "runtime_blocker"} scope=${directTargets.blockers[0]?.scope ?? "m3-direct-targets"}`);
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const live = await runLiveM3Targets(directTargets.targets);
|
|
report.liveChecks.push(
|
|
{
|
|
id: "two-box-simu-online",
|
|
status: "pass",
|
|
summary: "Both DEV box simulators reported live status with DO1 and DI1 ports.",
|
|
evidence: ["box-simu-1 /status", "box-simu-2 /status"]
|
|
},
|
|
{
|
|
id: "two-gateway-simu-online",
|
|
status: "pass",
|
|
summary: "Both DEV gateway simulators reported connected sessions.",
|
|
evidence: ["gateway-simu-1 /status", "gateway-simu-2 /status"]
|
|
},
|
|
{
|
|
id: "patch-panel-healthy",
|
|
status: "pass",
|
|
summary: "DEV patch-panel reported active status.",
|
|
evidence: ["patch-panel /status"]
|
|
},
|
|
{
|
|
id: "wiring-do1-di1-applied",
|
|
status: "pass",
|
|
summary: "Patch-panel active wiring matched box-simu-1 DO1 -> box-simu-2 DI1.",
|
|
evidence: ["patch-panel /wiring"]
|
|
},
|
|
{
|
|
id: "direct-call-do-write-di-read",
|
|
status: "pass",
|
|
summary: "Live DEV direct call observed do.write true followed by di.read true.",
|
|
evidence: [`operationId=${live.operationId}`, `traceId=${live.traceId}`]
|
|
},
|
|
{
|
|
id: "audit-evidence-traceable",
|
|
status: "pass",
|
|
summary: "Live DEV direct call returned audit and evidence identifiers.",
|
|
evidence: [`auditId=${live.auditId}`, `evidenceId=${live.evidenceId}`]
|
|
}
|
|
);
|
|
report.liveOperation = {
|
|
status: "pass",
|
|
operationId: live.operationId,
|
|
traceId: live.traceId,
|
|
auditId: live.auditId,
|
|
evidenceId: live.evidenceId,
|
|
summary: "Live DEV M3 do.write true -> di.read true succeeded with traceable audit/evidence identifiers."
|
|
};
|
|
report.rawLiveProbeCount = live.probes.length;
|
|
report.summary = {
|
|
status: "pass",
|
|
observedAt,
|
|
result: "Live DEV M3 hardware loop passed without fixture substitution."
|
|
};
|
|
await writeReport(report, reportPath);
|
|
console.log(`[dev-m3-smoke] status=pass report=${relativePath(reportPath)}`);
|
|
console.log(
|
|
`[dev-m3-smoke] operationId=${live.operationId} auditId=${live.auditId} evidenceId=${live.evidenceId}`
|
|
);
|
|
} catch (error) {
|
|
const classification = classifyLiveOperationError(error);
|
|
addNotRunM3Checks(report, "A live DEV M3 runtime contract check failed before all checks passed.");
|
|
report.blockers.push({
|
|
type: "runtime_blocker",
|
|
scope: "m3-hardware-loop-runtime",
|
|
status: "open",
|
|
classification,
|
|
summary: error instanceof Error ? error.message : String(error)
|
|
});
|
|
report.summary = {
|
|
status: "blocked",
|
|
classification,
|
|
observedAt,
|
|
result: "Live DEV M3 runtime was reachable but did not satisfy the M3 hardware-loop contract."
|
|
};
|
|
await writeReport(report, reportPath);
|
|
console.log(`[dev-m3-smoke] status=blocked report=${relativePath(reportPath)}`);
|
|
console.log("[dev-m3-smoke] blocker=runtime_blocker scope=m3-hardware-loop-runtime");
|
|
}
|
|
}
|
|
|
|
export {
|
|
classifyDirectTargetBlockerScope,
|
|
classifyLiveOperationError,
|
|
candidateTargetsFromKubernetes,
|
|
createLiveOperationPlan,
|
|
collectKubernetesDirectTargetDiscovery,
|
|
hasRequiredM3Connection,
|
|
kubernetesServiceIdentity,
|
|
m3FailureClassifications,
|
|
parseArgs,
|
|
requireSafetyGates
|
|
};
|
|
|
|
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
try {
|
|
await main();
|
|
} catch (error) {
|
|
console.error(`[dev-m3-smoke] ${error instanceof Error ? error.message : String(error)}`);
|
|
process.exitCode = 1;
|
|
}
|
|
}
|