Lyon
85da2dcb19
Commander review: direction is correct. This PR separates runner kubeconfig/read-only observability gaps from D601 public endpoint/k3s availability and keeps M3 blocked until real operation/trace/audit/evidence are observed. It does not touch PROD or restart services.
902 lines
31 KiB
JavaScript
902 lines
31 KiB
JavaScript
#!/usr/bin/env node
|
|
import assert from "node:assert/strict";
|
|
import { execFileSync } from "node:child_process";
|
|
import { randomUUID } from "node:crypto";
|
|
import { accessSync, constants as fsConstants } from "node:fs";
|
|
import { access, mkdir, readFile, writeFile } from "node:fs/promises";
|
|
import { request as httpRequest } from "node:http";
|
|
import { request as httpsRequest } from "node:https";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
import { activeReportLifecycle } from "../internal/dev-report-lifecycle.mjs";
|
|
import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT, ENVIRONMENT_DEV } from "../internal/protocol/index.mjs";
|
|
|
|
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
const defaultReportPath = "reports/dev-gate/dev-m3-hardware-loop.json";
|
|
const namespace = "hwlab-dev";
|
|
const issue = "pikasTech/HWLAB#38";
|
|
const deployWorkloadsPath = "deploy/k8s/base/workloads.yaml";
|
|
const requiredManifestCardinalityCommand = "node scripts/validate-dev-m3-cardinality.mjs";
|
|
const requiredM3BoxIds = Object.freeze(["boxsimu_1", "boxsimu_2"]);
|
|
const requiredM3BoxResources = Object.freeze(["res_boxsimu_1", "res_boxsimu_2"]);
|
|
const runnerK3sKubeconfigPath = "/etc/rancher/k3s/k3s.yaml";
|
|
|
|
const serviceTargets = Object.freeze([
|
|
{
|
|
id: "box-simu-1",
|
|
serviceId: "hwlab-box-simu",
|
|
urlEnv: "HWLAB_DEV_BOX_SIMU_1_URL",
|
|
statusPath: "/status"
|
|
},
|
|
{
|
|
id: "box-simu-2",
|
|
serviceId: "hwlab-box-simu",
|
|
urlEnv: "HWLAB_DEV_BOX_SIMU_2_URL",
|
|
statusPath: "/status"
|
|
},
|
|
{
|
|
id: "gateway-simu-1",
|
|
serviceId: "hwlab-gateway-simu",
|
|
urlEnv: "HWLAB_DEV_GATEWAY_SIMU_1_URL",
|
|
statusPath: "/status"
|
|
},
|
|
{
|
|
id: "gateway-simu-2",
|
|
serviceId: "hwlab-gateway-simu",
|
|
urlEnv: "HWLAB_DEV_GATEWAY_SIMU_2_URL",
|
|
statusPath: "/status"
|
|
},
|
|
{
|
|
id: "patch-panel",
|
|
serviceId: "hwlab-patch-panel",
|
|
urlEnv: "HWLAB_DEV_PATCH_PANEL_URL",
|
|
statusPath: "/status",
|
|
wiringPath: "/wiring",
|
|
routePath: "/signals/route"
|
|
}
|
|
]);
|
|
|
|
function parseArgs(argv) {
|
|
const flags = new Set();
|
|
const values = new Map();
|
|
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const arg = argv[index];
|
|
if (!arg.startsWith("--")) {
|
|
throw new Error(`unexpected positional argument ${arg}`);
|
|
}
|
|
if (arg === "--output") {
|
|
const value = argv[index + 1];
|
|
assert.ok(value && !value.startsWith("--"), "--output requires a path");
|
|
values.set("output", value);
|
|
index += 1;
|
|
continue;
|
|
}
|
|
flags.add(arg);
|
|
}
|
|
|
|
return { flags, values };
|
|
}
|
|
|
|
function requireSafetyGates(flags) {
|
|
const forbiddenFlags = [
|
|
"--prod",
|
|
"--real-hardware",
|
|
"--read-secret",
|
|
"--read-token",
|
|
"--force-push",
|
|
"--restart-runtime",
|
|
"--restart-unidesk",
|
|
"--restart-code-queue",
|
|
"--restart-backend-core"
|
|
];
|
|
|
|
for (const flag of forbiddenFlags) {
|
|
assert.ok(!flags.has(flag), `${flag} is forbidden for DEV M3 smoke`);
|
|
}
|
|
|
|
const live = flags.has("--live") || flags.has("--allow-live");
|
|
assert.ok(live, "DEV M3 smoke requires --live or --allow-live");
|
|
assert.ok(flags.has("--confirm-dev"), "live DEV smoke requires --confirm-dev");
|
|
assert.ok(
|
|
flags.has("--confirmed-non-production"),
|
|
"live DEV smoke requires --confirmed-non-production"
|
|
);
|
|
}
|
|
|
|
function isoNow() {
|
|
return new Date().toISOString();
|
|
}
|
|
|
|
function relativePath(absolutePath) {
|
|
return path.relative(repoRoot, absolutePath).replaceAll(path.sep, "/");
|
|
}
|
|
|
|
function resolveReportPath(value) {
|
|
const candidate = path.resolve(repoRoot, value ?? defaultReportPath);
|
|
assert.ok(candidate.startsWith(`${repoRoot}${path.sep}`), "report path must stay inside repository");
|
|
return candidate;
|
|
}
|
|
|
|
function currentCommit() {
|
|
try {
|
|
return execFileSync("git", ["rev-parse", "--short=12", "HEAD"], {
|
|
cwd: repoRoot,
|
|
encoding: "utf8",
|
|
stdio: ["ignore", "pipe", "pipe"]
|
|
}).trim();
|
|
} catch {
|
|
return "unknown";
|
|
}
|
|
}
|
|
|
|
function oneLine(value) {
|
|
return String(value).replace(/\s+/g, " ").trim();
|
|
}
|
|
|
|
function commandResult(command, args, { timeoutMs = 5000, maxChars = 3000 } = {}) {
|
|
try {
|
|
const stdout = execFileSync(command, args, {
|
|
cwd: repoRoot,
|
|
encoding: "utf8",
|
|
timeout: timeoutMs,
|
|
stdio: ["ignore", "pipe", "pipe"]
|
|
});
|
|
return {
|
|
command: [command, ...args].join(" "),
|
|
exitCode: 0,
|
|
stdout: stdout.slice(0, maxChars)
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
command: [command, ...args].join(" "),
|
|
exitCode: Number.isInteger(error.status) ? error.status : 1,
|
|
stdout: String(error.stdout ?? "").slice(0, maxChars),
|
|
stderr: String(error.stderr || error.message || "").slice(0, maxChars)
|
|
};
|
|
}
|
|
}
|
|
|
|
function runnerKubeconfigCommandText(args) {
|
|
return [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args].join(" ");
|
|
}
|
|
|
|
function commandExists(command) {
|
|
const result = commandResult("bash", ["-lc", `command -v ${command}`], { timeoutMs: 2000 });
|
|
return result.exitCode === 0 && result.stdout.trim().length > 0;
|
|
}
|
|
|
|
function fileReadable(filePath) {
|
|
try {
|
|
accessSync(filePath, fsConstants.R_OK);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
function collectD601RunnerObservability(ingress) {
|
|
const args = ["-n", namespace, "get", "pods", "-o", "name"];
|
|
const probe = commandResult("env", [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args], {
|
|
timeoutMs: 5000,
|
|
maxChars: 2000
|
|
});
|
|
const stderr = oneLine(probe.stderr || "") || "empty";
|
|
const stdoutLines = String(probe.stdout ?? "")
|
|
.split("\n")
|
|
.map((line) => line.trim())
|
|
.filter(Boolean);
|
|
const d601PublicEndpointsReachable = ingress.probes.some((item) => item.ok && isHwlabDevIdentity(item));
|
|
const runnerKubeconfigReadable = fileReadable(runnerK3sKubeconfigPath);
|
|
|
|
return {
|
|
runnerKubeconfigReadable,
|
|
runnerKubeconfigProbeOk: probe.exitCode === 0,
|
|
runnerKubeconfigProbeExitCode: probe.exitCode,
|
|
runnerKubeconfigProbeStderr: stderr,
|
|
d601PublicEndpointsReachable,
|
|
d601K3sUnavailable: false,
|
|
classification: "runner_permission_mount_gap",
|
|
sourceIssue: "pikasTech/HWLAB#46",
|
|
command: runnerKubeconfigCommandText(args),
|
|
stdoutSummary: probe.exitCode === 0 ? `${stdoutLines.length} pod name(s) observed` : undefined,
|
|
summary: "Runner /etc/rancher/k3s/k3s.yaml is not treated as readable from this container; a kubectl success may come from alternate in-cluster config, so file readability and command exit code stay separate.",
|
|
inferenceRule: "runnerKubeconfigReadable=false must not imply d601K3sUnavailable=true; public 16666/16667 endpoint reachability is tracked separately.",
|
|
secretValuesRead: false,
|
|
secretResourcesRead: false
|
|
};
|
|
}
|
|
|
|
function collectLocalRuntimeObservations(d601Observability) {
|
|
const observations = [];
|
|
|
|
if (commandExists("docker")) {
|
|
const docker = commandResult("docker", [
|
|
"ps",
|
|
"--format",
|
|
"{{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}"
|
|
]);
|
|
const hwlabLines = docker.stdout
|
|
.split("\n")
|
|
.filter((line) => line.includes("hwlab-"))
|
|
.join("\n");
|
|
observations.push({
|
|
id: "docker-hwlab-containers",
|
|
status: hwlabLines ? "observed" : "not_observed",
|
|
command: docker.command,
|
|
summary: oneLine(hwlabLines || "No running docker container name/image/port line contained hwlab-.")
|
|
});
|
|
} else {
|
|
observations.push({
|
|
id: "docker-hwlab-containers",
|
|
status: "unavailable",
|
|
command: "command -v docker",
|
|
summary: "docker CLI is not available in this runner."
|
|
});
|
|
}
|
|
|
|
if (commandExists("kubectl")) {
|
|
const kubectl = commandResult("kubectl", ["get", "deploy,po,svc", "-n", namespace, "-o", "wide"]);
|
|
observations.push({
|
|
id: "kubectl-hwlab-dev",
|
|
status: kubectl.exitCode === 0 ? "observed" : "blocked",
|
|
command: kubectl.command,
|
|
summary: oneLine(kubectl.exitCode === 0 ? kubectl.stdout : kubectl.stderr || kubectl.stdout)
|
|
});
|
|
} else {
|
|
observations.push({
|
|
id: "kubectl-hwlab-dev",
|
|
status: "unavailable",
|
|
command: "command -v kubectl",
|
|
summary: "kubectl is not available in this runner, so hwlab-dev namespace state was not observed."
|
|
});
|
|
}
|
|
|
|
observations.push({
|
|
id: "runner-k3s-kubeconfig-readonly",
|
|
status: d601Observability.runnerKubeconfigProbeOk ? "observed" : "blocked",
|
|
command: d601Observability.command,
|
|
summary: `${d601Observability.summary} runnerKubeconfigReadable=${d601Observability.runnerKubeconfigReadable}; exitCode=${d601Observability.runnerKubeconfigProbeExitCode}; stderr=${d601Observability.runnerKubeconfigProbeStderr}.`
|
|
});
|
|
|
|
return observations;
|
|
}
|
|
|
|
function splitList(value) {
|
|
return String(value ?? "")
|
|
.split(",")
|
|
.map((item) => item.trim())
|
|
.filter(Boolean);
|
|
}
|
|
|
|
function hasSameMembers(actual, expected) {
|
|
return actual.length === expected.length && expected.every((item) => actual.includes(item));
|
|
}
|
|
|
|
async function collectReadOnlySupplementalEvidence() {
|
|
const raw = await readFile(path.join(repoRoot, deployWorkloadsPath), "utf8");
|
|
const workloads = JSON.parse(raw);
|
|
const targets = ["hwlab-box-simu", "hwlab-gateway-simu", "hwlab-patch-panel"];
|
|
const observed = workloads.items
|
|
.filter((item) => targets.includes(item.metadata?.name))
|
|
.map((item) => {
|
|
const container = item.spec?.template?.spec?.containers?.[0] ?? {};
|
|
const env = Object.fromEntries((container.env ?? []).map((entry) => [entry.name, entry.value]));
|
|
return {
|
|
name: item.metadata.name,
|
|
kind: item.kind,
|
|
namespace: item.metadata.namespace,
|
|
replicas: item.spec?.replicas ?? null,
|
|
image: container.image,
|
|
env
|
|
};
|
|
});
|
|
|
|
const box = observed.find((item) => item.name === "hwlab-box-simu");
|
|
const gateway = observed.find((item) => item.name === "hwlab-gateway-simu");
|
|
const patchPanel = observed.find((item) => item.name === "hwlab-patch-panel");
|
|
const boxIds = splitList(box?.env.HWLAB_BOX_ID);
|
|
const gatewayBoxResources = splitList(gateway?.env.HWLAB_BOX_RESOURCES);
|
|
const boxReadyForM3 = box?.replicas === 2 && hasSameMembers(boxIds, requiredM3BoxIds);
|
|
const gatewayReadyForM3 =
|
|
gateway?.replicas === 2 && hasSameMembers(gatewayBoxResources, requiredM3BoxResources);
|
|
const patchPanelReadyForM3 = patchPanel?.replicas === 1;
|
|
const manifestReadyForM3 = boxReadyForM3 && gatewayReadyForM3 && patchPanelReadyForM3;
|
|
|
|
return [
|
|
{
|
|
id: "deploy-skeleton-m3-cardinality",
|
|
status: manifestReadyForM3 ? "manifest-ready" : "gap",
|
|
blockerClass: manifestReadyForM3 ? null : "runtime_blocker",
|
|
source: deployWorkloadsPath,
|
|
summary: manifestReadyForM3
|
|
? "Checked-in DEV deploy skeleton declares M3 cardinality: two box simulators, two gateway simulators, and one patch panel."
|
|
: "Read-only static comparison found the checked-in DEV deploy skeleton does not declare the required M3 simulator and patch-panel cardinality; proving two of each simulator in live DEV requires a separate deploy/runtime task or authorized runtime observation.",
|
|
evidence: {
|
|
required: {
|
|
"hwlab-box-simu": 2,
|
|
"hwlab-gateway-simu": 2,
|
|
"hwlab-patch-panel": 1,
|
|
wiring: "box-simu-1 DO1 -> box-simu-2 DI1"
|
|
},
|
|
observed: {
|
|
"hwlab-box-simu": {
|
|
replicas: box?.replicas ?? null,
|
|
env: box?.env ?? null
|
|
},
|
|
"hwlab-gateway-simu": {
|
|
replicas: gateway?.replicas ?? null,
|
|
env: gateway?.env ?? null
|
|
},
|
|
"hwlab-patch-panel": {
|
|
replicas: patchPanel?.replicas ?? null,
|
|
env: patchPanel?.env ?? null
|
|
}
|
|
}
|
|
},
|
|
validationCommand: requiredManifestCardinalityCommand,
|
|
requiredFollowUp: manifestReadyForM3
|
|
? "Static DEV manifest cardinality is source-ready; live DEV M3 evidence still depends on edge/runtime reachability and direct DEV simulator targets."
|
|
: "Create a separate authorized DEV deploy/runtime task to provision or expose two box-simu instances, two gateway-simu instances, M3 wiring, and audit/evidence endpoints; do not perform that mutation in this read-only continuation."
|
|
}
|
|
];
|
|
}
|
|
|
|
async function probeJson(url, options = {}) {
|
|
const method = options.method ?? "GET";
|
|
const startedAt = isoNow();
|
|
const timeoutMs = options.timeoutMs ?? 8000;
|
|
const body = options.body ? JSON.stringify(options.body) : null;
|
|
const headers = body
|
|
? {
|
|
"content-type": "application/json",
|
|
"content-length": Buffer.byteLength(body)
|
|
}
|
|
: {};
|
|
|
|
return new Promise((resolve) => {
|
|
const parsed = new URL(url);
|
|
const requestFn = parsed.protocol === "https:" ? httpsRequest : httpRequest;
|
|
const req = requestFn(
|
|
parsed,
|
|
{
|
|
method,
|
|
headers,
|
|
timeout: timeoutMs
|
|
},
|
|
(response) => {
|
|
let text = "";
|
|
response.setEncoding("utf8");
|
|
response.on("data", (chunk) => {
|
|
text += chunk;
|
|
});
|
|
response.on("end", () => {
|
|
let json = null;
|
|
try {
|
|
json = text ? JSON.parse(text) : null;
|
|
} catch {
|
|
json = null;
|
|
}
|
|
const status = response.statusCode ?? 0;
|
|
resolve({
|
|
url,
|
|
method,
|
|
startedAt,
|
|
observedAt: isoNow(),
|
|
reached: true,
|
|
ok: status >= 200 && status < 300,
|
|
status,
|
|
statusText: response.statusMessage ?? "",
|
|
json,
|
|
bodyPreview: json ? undefined : text.slice(0, 500)
|
|
});
|
|
});
|
|
}
|
|
);
|
|
|
|
req.on("timeout", () => {
|
|
req.destroy(new Error(`request timed out after ${timeoutMs}ms`));
|
|
});
|
|
|
|
req.on("error", (error) => {
|
|
resolve({
|
|
url,
|
|
method,
|
|
startedAt,
|
|
observedAt: isoNow(),
|
|
reached: false,
|
|
ok: false,
|
|
error: {
|
|
name: error.name,
|
|
message: error.message,
|
|
code: error.code
|
|
}
|
|
});
|
|
});
|
|
|
|
if (body) {
|
|
req.write(body);
|
|
}
|
|
req.end();
|
|
});
|
|
}
|
|
|
|
function joinUrl(baseUrl, routePath) {
|
|
return new URL(routePath, baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`).toString();
|
|
}
|
|
|
|
function isHwlabDevIdentity(probe) {
|
|
const body = probe.json;
|
|
if (!probe.ok || !body || typeof body !== "object" || Array.isArray(body)) {
|
|
return false;
|
|
}
|
|
if (typeof body.serviceId !== "string" || !body.serviceId.startsWith("hwlab-")) {
|
|
return false;
|
|
}
|
|
return body.environment === undefined || body.environment === ENVIRONMENT_DEV;
|
|
}
|
|
|
|
async function observeDevIngress() {
|
|
const paths = ["/health/live", "/health", "/live", "/v1"];
|
|
const probes = [];
|
|
for (const routePath of paths) {
|
|
probes.push(await probeJson(joinUrl(DEV_ENDPOINT, routePath)));
|
|
}
|
|
|
|
const accepted = probes.find(isHwlabDevIdentity);
|
|
return {
|
|
accepted: Boolean(accepted),
|
|
acceptedPath: accepted ? new URL(accepted.url).pathname : null,
|
|
probes
|
|
};
|
|
}
|
|
|
|
function targetMapFromEnvironment() {
|
|
const missing = [];
|
|
const targets = new Map();
|
|
|
|
for (const target of serviceTargets) {
|
|
const baseUrl = process.env[target.urlEnv];
|
|
if (!baseUrl) {
|
|
missing.push(target.urlEnv);
|
|
continue;
|
|
}
|
|
targets.set(target.id, {
|
|
...target,
|
|
baseUrl
|
|
});
|
|
}
|
|
|
|
return { missing, targets };
|
|
}
|
|
|
|
function assertBoxState(state, label) {
|
|
assert.equal(state?.serviceId, "hwlab-box-simu", `${label} serviceId`);
|
|
assert.equal(state.live, true, `${label} live`);
|
|
assert.equal(state.resource?.environment ?? state.environment, ENVIRONMENT_DEV, `${label} environment`);
|
|
assert.ok(state.ports?.DO1, `${label} requires DO1`);
|
|
assert.ok(state.ports?.DI1, `${label} requires DI1`);
|
|
assert.equal(state.constraints?.crossDevicePropagation, "patch-panel-only", `${label} propagation`);
|
|
assert.equal(state.constraints?.localLoopbackEnabled, false, `${label} local loopback`);
|
|
}
|
|
|
|
function assertGatewayState(state, label) {
|
|
assert.equal(state?.serviceId, "hwlab-gateway-simu", `${label} serviceId`);
|
|
assert.equal(state.live, true, `${label} live`);
|
|
assert.equal(state.session?.status, "connected", `${label} session status`);
|
|
assert.equal(state.session?.environment, ENVIRONMENT_DEV, `${label} environment`);
|
|
assert.ok(Array.isArray(state.boxes) && state.boxes.length >= 1, `${label} boxes`);
|
|
}
|
|
|
|
function assertPatchPanelStatus(state) {
|
|
assert.equal(state?.serviceId, "hwlab-patch-panel", "patch-panel serviceId");
|
|
assert.equal(state.state, "active", "patch-panel state");
|
|
assert.equal(state.environment, ENVIRONMENT_DEV, "patch-panel environment");
|
|
assert.ok(Array.isArray(state.activeConnections), "patch-panel activeConnections");
|
|
}
|
|
|
|
function findM3Connection({ status, wiring, box1, box2 }) {
|
|
const fromResourceId = box1.resource?.resourceId;
|
|
const toResourceId = box2.resource?.resourceId;
|
|
const activeConnections = status.activeConnections ?? [];
|
|
const wiringConnections = wiring.connections ?? [];
|
|
|
|
const active = activeConnections.find(
|
|
(connection) =>
|
|
connection.fromResourceId === fromResourceId &&
|
|
connection.fromPort === "DO1" &&
|
|
connection.toResourceId === toResourceId &&
|
|
connection.toPort === "DI1"
|
|
);
|
|
const configured = wiringConnections.find(
|
|
(connection) =>
|
|
connection.from?.resourceId === fromResourceId &&
|
|
connection.from?.port === "DO1" &&
|
|
connection.to?.resourceId === toResourceId &&
|
|
connection.to?.port === "DI1" &&
|
|
connection.mode === "exclusive"
|
|
);
|
|
|
|
return { active, configured };
|
|
}
|
|
|
|
async function runLiveM3Targets(targets) {
|
|
const observed = {};
|
|
const probes = [];
|
|
|
|
for (const [id, target] of targets.entries()) {
|
|
const health = await probeJson(joinUrl(target.baseUrl, "/health/live"));
|
|
const status = await probeJson(joinUrl(target.baseUrl, target.statusPath));
|
|
probes.push({ id, kind: "health", probe: health }, { id, kind: "status", probe: status });
|
|
assert.ok(health.ok, `${id} health failed`);
|
|
assert.ok(status.ok, `${id} status failed`);
|
|
assert.equal(health.json?.serviceId, target.serviceId, `${id} health serviceId`);
|
|
observed[id] = status.json;
|
|
}
|
|
|
|
assertBoxState(observed["box-simu-1"], "box-simu-1");
|
|
assertBoxState(observed["box-simu-2"], "box-simu-2");
|
|
assertGatewayState(observed["gateway-simu-1"], "gateway-simu-1");
|
|
assertGatewayState(observed["gateway-simu-2"], "gateway-simu-2");
|
|
assertPatchPanelStatus(observed["patch-panel"]);
|
|
|
|
const patchTarget = targets.get("patch-panel");
|
|
const wiring = await probeJson(joinUrl(patchTarget.baseUrl, patchTarget.wiringPath));
|
|
probes.push({ id: "patch-panel", kind: "wiring", probe: wiring });
|
|
assert.ok(wiring.ok, "patch-panel wiring failed");
|
|
assert.equal(wiring.json?.status, "active", "wiring status");
|
|
assert.equal(wiring.json?.constraints?.propagation, "patch-panel-only", "wiring propagation");
|
|
|
|
const connection = findM3Connection({
|
|
status: observed["patch-panel"],
|
|
wiring: wiring.json,
|
|
box1: observed["box-simu-1"],
|
|
box2: observed["box-simu-2"]
|
|
});
|
|
assert.ok(connection.active, "patch-panel active DO1 -> DI1 connection missing");
|
|
assert.ok(connection.configured, "wiring config DO1 -> DI1 connection missing");
|
|
|
|
const operationId = `op_dev_m3_${randomUUID()}`;
|
|
const traceId = `trc_dev_m3_${randomUUID()}`;
|
|
const box1Target = targets.get("box-simu-1");
|
|
const box2Target = targets.get("box-simu-2");
|
|
|
|
const write = await probeJson(joinUrl(box1Target.baseUrl, "/ports/write"), {
|
|
method: "POST",
|
|
body: {
|
|
port: "DO1",
|
|
value: true,
|
|
operationId,
|
|
traceId
|
|
}
|
|
});
|
|
probes.push({ id: "box-simu-1", kind: "do.write", probe: write });
|
|
assert.ok(write.ok, "box-simu-1 DO1 write failed");
|
|
assert.equal(write.json?.accepted, true, "box-simu-1 DO1 write accepted");
|
|
|
|
const route = await probeJson(joinUrl(patchTarget.baseUrl, patchTarget.routePath), {
|
|
method: "POST",
|
|
body: {
|
|
fromResourceId: observed["box-simu-1"].resource.resourceId,
|
|
fromPort: "DO1",
|
|
value: true,
|
|
operationId,
|
|
traceId
|
|
}
|
|
});
|
|
probes.push({ id: "patch-panel", kind: "signals.route", probe: route });
|
|
assert.ok(route.ok, "patch-panel signal route failed");
|
|
assert.equal(route.json?.accepted, true, "patch-panel signal route accepted");
|
|
assert.equal(route.json?.deliveryCount, 1, "patch-panel signal route delivery count");
|
|
|
|
const after = await probeJson(joinUrl(box2Target.baseUrl, "/status"));
|
|
probes.push({ id: "box-simu-2", kind: "di.read", probe: after });
|
|
assert.ok(after.ok, "box-simu-2 DI1 read failed");
|
|
assert.equal(after.json?.ports?.DI1?.value, true, "box-simu-2 DI1 must read true");
|
|
|
|
const auditId = route.json?.auditId ?? write.json?.auditId ?? after.json?.auditId;
|
|
const evidenceId = route.json?.evidenceId ?? write.json?.evidenceId ?? after.json?.evidenceId;
|
|
assert.equal(typeof auditId, "string", "live M3 result must include auditId");
|
|
assert.equal(typeof evidenceId, "string", "live M3 result must include evidenceId");
|
|
|
|
return {
|
|
operationId,
|
|
traceId,
|
|
auditId,
|
|
evidenceId,
|
|
probes,
|
|
observed
|
|
};
|
|
}
|
|
|
|
function baseReport({ commitId, observedAt }) {
|
|
return {
|
|
$schema: "https://hwlab.pikastech.local/schemas/dev-m3-hardware-loop-report.schema.json",
|
|
$id: "https://hwlab.pikastech.local/reports/dev-gate/dev-m3-hardware-loop.json",
|
|
reportVersion: "v1",
|
|
issue,
|
|
taskId: "dev-m3-hardware-loop",
|
|
commitId,
|
|
acceptanceLevel: "dev_m3_hardware_loop",
|
|
devOnly: true,
|
|
prodDisabled: true,
|
|
reportLifecycle: activeReportLifecycle("Current M3 DEV hardware trusted-loop report; DEV-LIVE requires res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 with operation, audit, and evidence ids."),
|
|
sourceContract: {
|
|
status: "pass",
|
|
documents: [
|
|
"docs/dev-acceptance-matrix.md",
|
|
"docs/m3-hardware-loop.md",
|
|
"fixtures/mvp/m3-hardware-loop/topology.json"
|
|
],
|
|
summary: "Source contracts define DEV-only M3 simulator wiring and prohibit fixture output as live evidence."
|
|
},
|
|
validationCommands: [
|
|
"node --check scripts/dev-m3-hardware-loop-smoke.mjs",
|
|
"node --check scripts/validate-dev-m3-cardinality.mjs",
|
|
requiredManifestCardinalityCommand,
|
|
"node scripts/dev-m3-hardware-loop-smoke.mjs --live --confirm-dev --confirmed-non-production",
|
|
"node --check scripts/validate-dev-gate-report.mjs",
|
|
"node scripts/validate-dev-gate-report.mjs"
|
|
],
|
|
runtimeTarget: {
|
|
endpoint: DEV_ENDPOINT,
|
|
frontendEndpoint: DEV_FRONTEND_ENDPOINT,
|
|
namespace,
|
|
environment: ENVIRONMENT_DEV,
|
|
requiredBoxSimulators: 2,
|
|
requiredGatewaySimulators: 2,
|
|
realHardwareAllowed: false,
|
|
prodAllowed: false
|
|
},
|
|
safetyGates: {
|
|
liveFlagRequired: true,
|
|
confirmDevRequired: true,
|
|
confirmedNonProductionRequired: true,
|
|
prodForbidden: true,
|
|
realHardwareForbidden: true,
|
|
secretReadForbidden: true,
|
|
forcePushForbidden: true,
|
|
unideskRuntimeSubstitutionForbidden: true
|
|
},
|
|
liveChecks: [
|
|
{
|
|
id: "static-contract-parse",
|
|
status: "pass",
|
|
summary: "DEV M3 smoke contract files were readable before live probing.",
|
|
evidence: [
|
|
"docs/dev-acceptance-matrix.md",
|
|
"docs/m3-hardware-loop.md"
|
|
]
|
|
},
|
|
{
|
|
id: "endpoint-freeze",
|
|
status: "pass",
|
|
summary: `DEV endpoint is frozen at ${DEV_ENDPOINT}.`,
|
|
evidence: [DEV_ENDPOINT]
|
|
}
|
|
],
|
|
readOnlySupplementalEvidence: [],
|
|
localRuntimeObservations: [],
|
|
d601Observability: {
|
|
runnerKubeconfigReadable: false,
|
|
runnerKubeconfigProbeOk: false,
|
|
runnerKubeconfigProbeExitCode: null,
|
|
runnerKubeconfigProbeStderr: "not_observed",
|
|
d601PublicEndpointsReachable: false,
|
|
d601K3sUnavailable: false,
|
|
classification: "not_observed",
|
|
sourceIssue: "pikasTech/HWLAB#46",
|
|
inferenceRule: "runnerKubeconfigReadable=false must not imply d601K3sUnavailable=true; public 16666/16667 endpoint reachability is tracked separately."
|
|
},
|
|
liveOperation: {
|
|
status: "not_run",
|
|
operationId: "not_observed",
|
|
traceId: "not_observed",
|
|
auditId: "not_observed",
|
|
evidenceId: "not_observed",
|
|
summary: "No live DEV operation has run yet."
|
|
},
|
|
blockers: [],
|
|
summary: {
|
|
status: "not_run",
|
|
observedAt,
|
|
result: "DEV M3 smoke has not completed."
|
|
}
|
|
};
|
|
}
|
|
|
|
function addNotRunM3Checks(report, reason) {
|
|
for (const id of [
|
|
"two-box-simu-online",
|
|
"two-gateway-simu-online",
|
|
"patch-panel-healthy",
|
|
"wiring-do1-di1-applied",
|
|
"direct-call-do-write-di-read",
|
|
"audit-evidence-traceable"
|
|
]) {
|
|
report.liveChecks.push({
|
|
id,
|
|
status: "not_run",
|
|
summary: reason,
|
|
evidence: ["No live DEV evidence collected for this check."]
|
|
});
|
|
}
|
|
}
|
|
|
|
async function ensureContractFiles() {
|
|
await access(path.join(repoRoot, "docs/dev-acceptance-matrix.md"));
|
|
await access(path.join(repoRoot, "docs/m3-hardware-loop.md"));
|
|
await access(path.join(repoRoot, "fixtures/mvp/m3-hardware-loop/topology.json"));
|
|
}
|
|
|
|
async function writeReport(report, reportPath) {
|
|
await mkdir(path.dirname(reportPath), { recursive: true });
|
|
await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
|
}
|
|
|
|
async function main() {
|
|
const { flags, values } = parseArgs(process.argv.slice(2));
|
|
requireSafetyGates(flags);
|
|
await ensureContractFiles();
|
|
|
|
const reportPath = resolveReportPath(values.get("output"));
|
|
const observedAt = isoNow();
|
|
const report = baseReport({
|
|
commitId: currentCommit(),
|
|
observedAt
|
|
});
|
|
report.readOnlySupplementalEvidence = await collectReadOnlySupplementalEvidence();
|
|
|
|
const ingress = await observeDevIngress();
|
|
report.d601Observability = collectD601RunnerObservability(ingress);
|
|
report.localRuntimeObservations = collectLocalRuntimeObservations(report.d601Observability);
|
|
report.liveChecks.push({
|
|
id: "dev-ingress-health",
|
|
status: ingress.accepted ? "pass" : "blocked",
|
|
blockerClass: ingress.accepted ? undefined : "network_blocker",
|
|
summary: ingress.accepted
|
|
? `DEV ingress returned HWLAB identity at ${ingress.acceptedPath}.`
|
|
: `DEV ingress at ${DEV_ENDPOINT} did not return reachable HWLAB DEV health.`,
|
|
evidence: ingress.probes.map((probe) => JSON.stringify(probe))
|
|
});
|
|
|
|
if (!ingress.accepted) {
|
|
addNotRunM3Checks(report, "Stopped at DEV ingress per the DEV smoke matrix.");
|
|
report.liveOperation = {
|
|
status: "not_run",
|
|
operationId: "not_observed",
|
|
traceId: "not_observed",
|
|
auditId: "not_observed",
|
|
evidenceId: "not_observed",
|
|
summary: "No live do.write -> di.read operation was attempted because DEV ingress is blocked."
|
|
};
|
|
report.blockers.push({
|
|
type: "network_blocker",
|
|
scope: "frp",
|
|
status: "open",
|
|
classification: "frp",
|
|
summary: "Blocked at the #33 DEV runtime readiness condition: public DEV ingress does not accept HWLAB health requests, so #36/M3 hardware-loop checks were not reached."
|
|
});
|
|
report.summary = {
|
|
status: "blocked",
|
|
classification: "frp",
|
|
observedAt,
|
|
result: "No live DEV M3 evidence was produced; fixture-backed local evidence was not used as a substitute."
|
|
};
|
|
await writeReport(report, reportPath);
|
|
console.log(`[dev-m3-smoke] status=blocked report=${relativePath(reportPath)}`);
|
|
console.log("[dev-m3-smoke] blocker=network_blocker scope=dev-ingress-health");
|
|
return;
|
|
}
|
|
|
|
const { missing, targets } = targetMapFromEnvironment();
|
|
if (missing.length > 0) {
|
|
addNotRunM3Checks(report, "Stopped before M3 direct checks because service target URLs were not provided.");
|
|
report.blockers.push({
|
|
type: "observability_blocker",
|
|
scope: "m3-service-discovery",
|
|
status: "open",
|
|
classification: "runner permission/mount gap",
|
|
sourceIssue: "pikasTech/HWLAB#46",
|
|
summary: `DEV ingress is reachable on frozen :16667, but this runner cannot use ${runnerK3sKubeconfigPath} as readable kubeconfig to discover direct M3 service URLs (${missing.join(", ")}); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline.`
|
|
});
|
|
report.summary = {
|
|
status: "blocked",
|
|
classification: "runner permission/mount gap",
|
|
observedAt,
|
|
result: "DEV ingress was reachable, but live M3 simulator and patch-panel targets were not discoverable."
|
|
};
|
|
await writeReport(report, reportPath);
|
|
console.log(`[dev-m3-smoke] status=blocked report=${relativePath(reportPath)}`);
|
|
console.log("[dev-m3-smoke] blocker=observability_blocker scope=m3-service-discovery");
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const live = await runLiveM3Targets(targets);
|
|
report.liveChecks.push(
|
|
{
|
|
id: "two-box-simu-online",
|
|
status: "pass",
|
|
summary: "Both DEV box simulators reported live status with DO1 and DI1 ports.",
|
|
evidence: ["box-simu-1 /status", "box-simu-2 /status"]
|
|
},
|
|
{
|
|
id: "two-gateway-simu-online",
|
|
status: "pass",
|
|
summary: "Both DEV gateway simulators reported connected sessions.",
|
|
evidence: ["gateway-simu-1 /status", "gateway-simu-2 /status"]
|
|
},
|
|
{
|
|
id: "patch-panel-healthy",
|
|
status: "pass",
|
|
summary: "DEV patch-panel reported active status.",
|
|
evidence: ["patch-panel /status"]
|
|
},
|
|
{
|
|
id: "wiring-do1-di1-applied",
|
|
status: "pass",
|
|
summary: "Patch-panel active wiring matched box-simu-1 DO1 -> box-simu-2 DI1.",
|
|
evidence: ["patch-panel /wiring"]
|
|
},
|
|
{
|
|
id: "direct-call-do-write-di-read",
|
|
status: "pass",
|
|
summary: "Live DEV direct call observed do.write true followed by di.read true.",
|
|
evidence: [`operationId=${live.operationId}`, `traceId=${live.traceId}`]
|
|
},
|
|
{
|
|
id: "audit-evidence-traceable",
|
|
status: "pass",
|
|
summary: "Live DEV direct call returned audit and evidence identifiers.",
|
|
evidence: [`auditId=${live.auditId}`, `evidenceId=${live.evidenceId}`]
|
|
}
|
|
);
|
|
report.liveOperation = {
|
|
status: "pass",
|
|
operationId: live.operationId,
|
|
traceId: live.traceId,
|
|
auditId: live.auditId,
|
|
evidenceId: live.evidenceId,
|
|
summary: "Live DEV M3 do.write true -> di.read true succeeded with traceable audit/evidence identifiers."
|
|
};
|
|
report.rawLiveProbeCount = live.probes.length;
|
|
report.summary = {
|
|
status: "pass",
|
|
observedAt,
|
|
result: "Live DEV M3 hardware loop passed without fixture substitution."
|
|
};
|
|
await writeReport(report, reportPath);
|
|
console.log(`[dev-m3-smoke] status=pass report=${relativePath(reportPath)}`);
|
|
console.log(
|
|
`[dev-m3-smoke] operationId=${live.operationId} auditId=${live.auditId} evidenceId=${live.evidenceId}`
|
|
);
|
|
} catch (error) {
|
|
addNotRunM3Checks(report, "A live DEV M3 runtime contract check failed before all checks passed.");
|
|
report.blockers.push({
|
|
type: "runtime_blocker",
|
|
scope: "m3-hardware-loop-runtime",
|
|
status: "open",
|
|
classification: "hardware loop",
|
|
summary: error instanceof Error ? error.message : String(error)
|
|
});
|
|
report.summary = {
|
|
status: "blocked",
|
|
classification: "hardware loop",
|
|
observedAt,
|
|
result: "Live DEV M3 runtime was reachable but did not satisfy the M3 hardware-loop contract."
|
|
};
|
|
await writeReport(report, reportPath);
|
|
console.log(`[dev-m3-smoke] status=blocked report=${relativePath(reportPath)}`);
|
|
console.log("[dev-m3-smoke] blocker=runtime_blocker scope=m3-hardware-loop-runtime");
|
|
}
|
|
}
|
|
|
|
try {
|
|
await main();
|
|
} catch (error) {
|
|
console.error(`[dev-m3-smoke] ${error instanceof Error ? error.message : String(error)}`);
|
|
process.exitCode = 1;
|
|
}
|