test: classify DEV M3 direct target blockers
test: classify DEV M3 direct target blockers
This commit is contained in:
Lyon
@@ -20,6 +20,12 @@ const deployWorkloadsPath = "deploy/k8s/base/workloads.yaml";
|
||||
const requiredManifestCardinalityCommand = "node scripts/validate-dev-m3-cardinality.mjs";
|
||||
const requiredM3BoxIds = Object.freeze(["boxsimu_1", "boxsimu_2"]);
|
||||
const requiredM3BoxResources = Object.freeze(["res_boxsimu_1", "res_boxsimu_2"]);
|
||||
const requiredM3Connection = Object.freeze({
|
||||
fromResourceId: "res_boxsimu_1",
|
||||
fromPort: "DO1",
|
||||
toResourceId: "res_boxsimu_2",
|
||||
toPort: "DI1"
|
||||
});
|
||||
const runnerK3sKubeconfigPath = "/etc/rancher/k3s/k3s.yaml";
|
||||
|
||||
const serviceTargets = Object.freeze([
|
||||
@@ -189,6 +195,9 @@ function collectD601RunnerObservability(ingress) {
|
||||
.filter(Boolean);
|
||||
const d601PublicEndpointsReachable = ingress.probes.some((item) => item.ok && isHwlabDevIdentity(item));
|
||||
const runnerKubeconfigReadable = fileReadable(runnerK3sKubeconfigPath);
|
||||
const classification = probe.exitCode === 0
|
||||
? "read_only_kubectl_available_kubeconfig_file_unreadable"
|
||||
: "runner_permission_mount_gap";
|
||||
|
||||
return {
|
||||
runnerKubeconfigReadable,
|
||||
@@ -197,11 +206,13 @@ function collectD601RunnerObservability(ingress) {
|
||||
runnerKubeconfigProbeStderr: stderr,
|
||||
d601PublicEndpointsReachable,
|
||||
d601K3sUnavailable: false,
|
||||
classification: "runner_permission_mount_gap",
|
||||
classification,
|
||||
sourceIssue: "pikasTech/HWLAB#46",
|
||||
command: runnerKubeconfigCommandText(args),
|
||||
stdoutSummary: probe.exitCode === 0 ? `${stdoutLines.length} pod name(s) observed` : undefined,
|
||||
summary: "Runner /etc/rancher/k3s/k3s.yaml is not treated as readable from this container; a kubectl success may come from alternate in-cluster config, so file readability and command exit code stay separate.",
|
||||
summary: probe.exitCode === 0
|
||||
? "Runner /etc/rancher/k3s/k3s.yaml is not readable through fs access, but read-only kubectl observation succeeds; file readability, kubectl reachability, and service discovery are recorded separately."
|
||||
: "Runner /etc/rancher/k3s/k3s.yaml is not treated as readable from this container; kubectl read-only observation did not succeed, so this is a runner permission/mount gap, not proof that D601 is globally offline.",
|
||||
inferenceRule: "runnerKubeconfigReadable=false must not imply d601K3sUnavailable=true; public 16666/16667 endpoint reachability is tracked separately.",
|
||||
secretValuesRead: false,
|
||||
secretResourcesRead: false
|
||||
@@ -443,6 +454,23 @@ async function observeDevIngress() {
|
||||
for (const routePath of paths) {
|
||||
probes.push(await probeJson(joinUrl(DEV_ENDPOINT, routePath)));
|
||||
}
|
||||
probes.push(await probeJson(joinUrl(DEV_FRONTEND_ENDPOINT, "/health/live")));
|
||||
probes.push(await probeJson(joinUrl(DEV_FRONTEND_ENDPOINT, "/health")));
|
||||
probes.push(await probeJson(joinUrl(DEV_ENDPOINT, "/json-rpc"), {
|
||||
method: "POST",
|
||||
body: {
|
||||
jsonrpc: "2.0",
|
||||
id: `req_dev_m3_${randomUUID()}`,
|
||||
method: "system.health",
|
||||
params: {},
|
||||
meta: {
|
||||
traceId: `trc_dev_m3_${randomUUID()}`,
|
||||
actorId: "system_code_queue_d601",
|
||||
serviceId: "hwlab-cloud-api",
|
||||
environment: ENVIRONMENT_DEV
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
const accepted = probes.find(isHwlabDevIdentity);
|
||||
return {
|
||||
@@ -471,6 +499,418 @@ function targetMapFromEnvironment() {
|
||||
return { missing, targets };
|
||||
}
|
||||
|
||||
function serviceDefinitionByServiceId(serviceId) {
|
||||
return serviceTargets.find((target) => target.serviceId === serviceId);
|
||||
}
|
||||
|
||||
function serviceEndpointUrl(address, port) {
|
||||
return `http://${address}:${port}`;
|
||||
}
|
||||
|
||||
function endpointReady(endpoint) {
|
||||
return endpoint.conditions?.ready !== false && endpoint.conditions?.serving !== false;
|
||||
}
|
||||
|
||||
function collectKubernetesDirectTargetDiscovery() {
|
||||
if (!commandExists("kubectl")) {
|
||||
return {
|
||||
status: "unavailable",
|
||||
source: "kubectl",
|
||||
command: "command -v kubectl",
|
||||
services: {},
|
||||
summary: "kubectl is not available in this runner, so Kubernetes endpoint-slice service discovery was not attempted."
|
||||
};
|
||||
}
|
||||
|
||||
const args = ["-n", namespace, "get", "endpointslices.discovery.k8s.io", "-o", "json"];
|
||||
const probe = commandResult("env", [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args], {
|
||||
timeoutMs: 8000,
|
||||
maxChars: 100000
|
||||
});
|
||||
const command = runnerKubeconfigCommandText(args);
|
||||
|
||||
if (probe.exitCode !== 0) {
|
||||
return {
|
||||
status: "blocked",
|
||||
source: "kubernetes-endpointslices",
|
||||
command,
|
||||
exitCode: probe.exitCode,
|
||||
services: {},
|
||||
summary: oneLine(probe.stderr || probe.stdout || "kubectl endpoint-slice discovery failed.")
|
||||
};
|
||||
}
|
||||
|
||||
let document;
|
||||
try {
|
||||
document = JSON.parse(probe.stdout);
|
||||
} catch (error) {
|
||||
return {
|
||||
status: "blocked",
|
||||
source: "kubernetes-endpointslices",
|
||||
command,
|
||||
exitCode: probe.exitCode,
|
||||
services: {},
|
||||
summary: `kubectl endpoint-slice discovery returned non-JSON output: ${error.message}`
|
||||
};
|
||||
}
|
||||
|
||||
const requiredServices = new Set(["hwlab-box-simu", "hwlab-gateway-simu", "hwlab-patch-panel"]);
|
||||
const services = {};
|
||||
|
||||
for (const item of document.items ?? []) {
|
||||
const serviceId = item.metadata?.labels?.["kubernetes.io/service-name"];
|
||||
if (!requiredServices.has(serviceId)) {
|
||||
continue;
|
||||
}
|
||||
const port = item.ports?.find((entry) => entry.name === "http")?.port ?? item.ports?.[0]?.port;
|
||||
if (!Number.isInteger(port)) {
|
||||
continue;
|
||||
}
|
||||
const endpoints = [];
|
||||
for (const endpoint of item.endpoints ?? []) {
|
||||
if (!endpointReady(endpoint)) {
|
||||
continue;
|
||||
}
|
||||
for (const address of endpoint.addresses ?? []) {
|
||||
endpoints.push({
|
||||
url: serviceEndpointUrl(address, port),
|
||||
address,
|
||||
port,
|
||||
targetRef: endpoint.targetRef?.name ?? "unknown"
|
||||
});
|
||||
}
|
||||
}
|
||||
services[serviceId] = {
|
||||
endpointSlice: item.metadata?.name ?? "unknown",
|
||||
port,
|
||||
count: endpoints.length,
|
||||
endpoints
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: "observed",
|
||||
source: "kubernetes-endpointslices",
|
||||
command,
|
||||
exitCode: probe.exitCode,
|
||||
services,
|
||||
summary: `Observed endpoint slices for ${Object.keys(services).join(", ") || "no M3 services"}.`
|
||||
};
|
||||
}
|
||||
|
||||
function candidateTargetsFromKubernetes(discovery) {
|
||||
const candidates = [];
|
||||
for (const [serviceId, service] of Object.entries(discovery.services ?? {})) {
|
||||
const definition = serviceDefinitionByServiceId(serviceId);
|
||||
if (!definition) {
|
||||
continue;
|
||||
}
|
||||
let index = 1;
|
||||
for (const endpoint of service.endpoints ?? []) {
|
||||
candidates.push({
|
||||
...definition,
|
||||
id: `${definition.id}-candidate-${index}`,
|
||||
baseUrl: endpoint.url,
|
||||
discoverySource: discovery.source,
|
||||
targetRef: endpoint.targetRef,
|
||||
address: endpoint.address,
|
||||
port: endpoint.port
|
||||
});
|
||||
index += 1;
|
||||
}
|
||||
}
|
||||
return candidates;
|
||||
}
|
||||
|
||||
function candidateTargetsFromEnvironment(targets) {
|
||||
return [...targets.values()].map((target) => ({
|
||||
...target,
|
||||
discoverySource: "environment"
|
||||
}));
|
||||
}
|
||||
|
||||
function resourceIdOfBoxStatus(status) {
|
||||
return status?.resource?.resourceId ?? status?.resourceId ?? null;
|
||||
}
|
||||
|
||||
function gatewayIdentity(status) {
|
||||
return `${status?.gatewayId ?? "unknown"}:${status?.session?.gatewaySessionId ?? status?.gatewaySessionId ?? "unknown"}`;
|
||||
}
|
||||
|
||||
function summarizeCandidate(candidate) {
|
||||
const body = candidate.status?.json ?? {};
|
||||
return {
|
||||
id: candidate.id,
|
||||
serviceId: candidate.serviceId,
|
||||
baseUrl: candidate.baseUrl,
|
||||
targetRef: candidate.targetRef,
|
||||
ok: candidate.ok,
|
||||
statusCode: candidate.status?.status ?? null,
|
||||
healthCode: candidate.health?.status ?? null,
|
||||
boxId: body.boxId ?? null,
|
||||
resourceId: resourceIdOfBoxStatus(body),
|
||||
gatewayId: body.gatewayId ?? null,
|
||||
gatewaySessionId: body.session?.gatewaySessionId ?? body.gatewaySessionId ?? null,
|
||||
patchPanelState: body.state ?? null
|
||||
};
|
||||
}
|
||||
|
||||
function formatConnection(connection) {
|
||||
if (!connection) return "none";
|
||||
if (connection.from && connection.to) {
|
||||
return `${connection.from.resourceId}:${connection.from.port}->${connection.to.resourceId}:${connection.to.port}`;
|
||||
}
|
||||
return `${connection.fromResourceId}:${connection.fromPort}->${connection.toResourceId}:${connection.toPort}`;
|
||||
}
|
||||
|
||||
function hasRequiredM3Connection({ status, wiring }) {
|
||||
const activeConnections = status?.activeConnections ?? [];
|
||||
const wiringConnections = wiring?.connections ?? [];
|
||||
const active = activeConnections.find(
|
||||
(connection) =>
|
||||
connection.fromResourceId === requiredM3Connection.fromResourceId &&
|
||||
connection.fromPort === requiredM3Connection.fromPort &&
|
||||
connection.toResourceId === requiredM3Connection.toResourceId &&
|
||||
connection.toPort === requiredM3Connection.toPort
|
||||
);
|
||||
const configured = wiringConnections.find(
|
||||
(connection) =>
|
||||
connection.from?.resourceId === requiredM3Connection.fromResourceId &&
|
||||
connection.from?.port === requiredM3Connection.fromPort &&
|
||||
connection.to?.resourceId === requiredM3Connection.toResourceId &&
|
||||
connection.to?.port === requiredM3Connection.toPort &&
|
||||
connection.mode === "exclusive"
|
||||
);
|
||||
return { active, configured };
|
||||
}
|
||||
|
||||
async function probeDirectCandidate(candidate) {
|
||||
const health = await probeJson(joinUrl(candidate.baseUrl, "/health/live"));
|
||||
const status = await probeJson(joinUrl(candidate.baseUrl, candidate.statusPath));
|
||||
return {
|
||||
...candidate,
|
||||
health,
|
||||
status,
|
||||
ok: health.ok && status.ok && health.json?.serviceId === candidate.serviceId
|
||||
};
|
||||
}
|
||||
|
||||
async function resolveDirectM3Targets({ environmentTargets, kubernetesDiscovery }) {
|
||||
const allEnvironmentTargetsAvailable = environmentTargets.missing.length === 0;
|
||||
const candidates = allEnvironmentTargetsAvailable
|
||||
? candidateTargetsFromEnvironment(environmentTargets.targets)
|
||||
: candidateTargetsFromKubernetes(kubernetesDiscovery);
|
||||
const probed = [];
|
||||
|
||||
for (const candidate of candidates) {
|
||||
probed.push(await probeDirectCandidate(candidate));
|
||||
}
|
||||
|
||||
const boxCandidates = probed.filter((candidate) => candidate.serviceId === "hwlab-box-simu" && candidate.ok);
|
||||
const gatewayCandidates = probed.filter((candidate) => candidate.serviceId === "hwlab-gateway-simu" && candidate.ok);
|
||||
const patchCandidates = probed.filter((candidate) => candidate.serviceId === "hwlab-patch-panel" && candidate.ok);
|
||||
const box1 = boxCandidates.find((candidate) => resourceIdOfBoxStatus(candidate.status.json) === "res_boxsimu_1");
|
||||
const box2 = boxCandidates.find(
|
||||
(candidate) =>
|
||||
resourceIdOfBoxStatus(candidate.status.json) === "res_boxsimu_2" &&
|
||||
candidate.baseUrl !== box1?.baseUrl
|
||||
);
|
||||
const distinctBoxResources = new Set(boxCandidates.map((candidate) => resourceIdOfBoxStatus(candidate.status.json)).filter(Boolean));
|
||||
const distinctGatewayIdentities = new Set(gatewayCandidates.map((candidate) => gatewayIdentity(candidate.status.json)));
|
||||
const selectedGateways = [];
|
||||
for (const candidate of gatewayCandidates) {
|
||||
if (!selectedGateways.some((item) => gatewayIdentity(item.status.json) === gatewayIdentity(candidate.status.json))) {
|
||||
selectedGateways.push(candidate);
|
||||
}
|
||||
}
|
||||
const patch = patchCandidates[0] ?? null;
|
||||
let patchWiring = null;
|
||||
let m3Connection = { active: null, configured: null };
|
||||
|
||||
if (patch) {
|
||||
patchWiring = await probeJson(joinUrl(patch.baseUrl, patch.wiringPath));
|
||||
m3Connection = hasRequiredM3Connection({
|
||||
status: patch.status.json,
|
||||
wiring: patchWiring.json
|
||||
});
|
||||
}
|
||||
|
||||
const blockers = [];
|
||||
const serviceDiscoveryBlocked =
|
||||
!allEnvironmentTargetsAvailable && kubernetesDiscovery.status !== "observed";
|
||||
if (serviceDiscoveryBlocked) {
|
||||
blockers.push({
|
||||
type: "observability_blocker",
|
||||
scope: "m3-service-discovery",
|
||||
status: "open",
|
||||
classification: "runner_kubeconfig_readonly_gap",
|
||||
sourceIssue: "pikasTech/HWLAB#46",
|
||||
summary: `DEV ingress is reachable on frozen :16667, but direct M3 target discovery via ${kubernetesDiscovery.source} is ${kubernetesDiscovery.status}; environment URLs missing ${environmentTargets.missing.join(", ")}; discovery detail=${kubernetesDiscovery.summary}.`
|
||||
});
|
||||
} else if (boxCandidates.length < 2 || gatewayCandidates.length < 2 || patchCandidates.length < 1) {
|
||||
blockers.push({
|
||||
type: "observability_blocker",
|
||||
scope: "m3-direct-target-missing",
|
||||
status: "open",
|
||||
classification: "direct_target_missing",
|
||||
sourceIssue: "pikasTech/HWLAB#64",
|
||||
summary: `Read-only service discovery did not expose enough callable DEV M3 direct targets: box-simu=${boxCandidates.length}/2, gateway-simu=${gatewayCandidates.length}/2, patch-panel=${patchCandidates.length}/1.`
|
||||
});
|
||||
}
|
||||
|
||||
if (boxCandidates.length >= 2 && (!box1 || !box2 || distinctBoxResources.size < 2)) {
|
||||
blockers.push({
|
||||
type: "runtime_blocker",
|
||||
scope: "m3-box-simu-identity",
|
||||
status: "open",
|
||||
classification: "direct_target_identity_gap",
|
||||
sourceIssue: "pikasTech/HWLAB#64",
|
||||
summary: `DEV exposes two box-simu endpoints, but live identities are not the required distinct resources res_boxsimu_1 and res_boxsimu_2; observed resources=${[...distinctBoxResources].join(", ") || "none"}.`
|
||||
});
|
||||
}
|
||||
|
||||
if (gatewayCandidates.length >= 2 && distinctGatewayIdentities.size < 2) {
|
||||
blockers.push({
|
||||
type: "runtime_blocker",
|
||||
scope: "m3-gateway-simu-identity",
|
||||
status: "open",
|
||||
classification: "direct_target_identity_gap",
|
||||
sourceIssue: "pikasTech/HWLAB#64",
|
||||
summary: `DEV exposes two gateway-simu endpoints, but live gateway identities are not distinct; observed identities=${[...distinctGatewayIdentities].join(", ") || "none"}.`
|
||||
});
|
||||
}
|
||||
|
||||
if (patch && (!m3Connection.active || !m3Connection.configured)) {
|
||||
const observedActive = (patch.status.json?.activeConnections ?? []).map(formatConnection).join(", ") || "none";
|
||||
const observedConfigured = (patchWiring?.json?.connections ?? []).map(formatConnection).join(", ") || "none";
|
||||
blockers.push({
|
||||
type: "runtime_blocker",
|
||||
scope: "m3-patch-panel-wiring",
|
||||
status: "open",
|
||||
classification: "direct_target_m3_wiring_missing",
|
||||
sourceIssue: "pikasTech/HWLAB#64",
|
||||
summary: `DEV patch-panel is callable, but live wiring does not contain ${requiredM3Connection.fromResourceId}:${requiredM3Connection.fromPort} -> ${requiredM3Connection.toResourceId}:${requiredM3Connection.toPort}; active=${observedActive}; configured=${observedConfigured}.`
|
||||
});
|
||||
}
|
||||
|
||||
const ok =
|
||||
blockers.length === 0 &&
|
||||
box1 &&
|
||||
box2 &&
|
||||
selectedGateways.length >= 2 &&
|
||||
patch &&
|
||||
m3Connection.active &&
|
||||
m3Connection.configured;
|
||||
|
||||
const targets = new Map();
|
||||
if (ok) {
|
||||
targets.set("box-simu-1", { ...serviceTargets.find((item) => item.id === "box-simu-1"), baseUrl: box1.baseUrl });
|
||||
targets.set("box-simu-2", { ...serviceTargets.find((item) => item.id === "box-simu-2"), baseUrl: box2.baseUrl });
|
||||
targets.set("gateway-simu-1", { ...serviceTargets.find((item) => item.id === "gateway-simu-1"), baseUrl: selectedGateways[0].baseUrl });
|
||||
targets.set("gateway-simu-2", { ...serviceTargets.find((item) => item.id === "gateway-simu-2"), baseUrl: selectedGateways[1].baseUrl });
|
||||
targets.set("patch-panel", { ...serviceTargets.find((item) => item.id === "patch-panel"), baseUrl: patch.baseUrl });
|
||||
}
|
||||
|
||||
return {
|
||||
ok: Boolean(ok),
|
||||
source: allEnvironmentTargetsAvailable ? "environment" : kubernetesDiscovery.source,
|
||||
environmentMissing: environmentTargets.missing,
|
||||
candidates: probed.map(summarizeCandidate),
|
||||
counts: {
|
||||
boxSimu: boxCandidates.length,
|
||||
gatewaySimu: gatewayCandidates.length,
|
||||
patchPanel: patchCandidates.length,
|
||||
distinctBoxResources: distinctBoxResources.size,
|
||||
distinctGatewayIdentities: distinctGatewayIdentities.size
|
||||
},
|
||||
selected: {
|
||||
boxSimu1: box1 ? summarizeCandidate(box1) : null,
|
||||
boxSimu2: box2 ? summarizeCandidate(box2) : null,
|
||||
gateways: selectedGateways.slice(0, 2).map(summarizeCandidate),
|
||||
patchPanel: patch ? summarizeCandidate(patch) : null
|
||||
},
|
||||
patchPanelWiring: patch
|
||||
? {
|
||||
url: patchWiring?.url ?? joinUrl(patch.baseUrl, patch.wiringPath),
|
||||
ok: patchWiring?.ok === true,
|
||||
requiredConnection: requiredM3Connection,
|
||||
activeObserved: (patch.status.json?.activeConnections ?? []).map(formatConnection),
|
||||
configuredObserved: (patchWiring?.json?.connections ?? []).map(formatConnection),
|
||||
hasRequiredActiveConnection: Boolean(m3Connection.active),
|
||||
hasRequiredConfiguredConnection: Boolean(m3Connection.configured),
|
||||
configSource: patch.status.json?.metadata?.configSource ?? "unknown",
|
||||
syncableConnectionCount: patch.status.json?.metadata?.syncableConnectionCount ?? null
|
||||
}
|
||||
: null,
|
||||
blockers,
|
||||
targets
|
||||
};
|
||||
}
|
||||
|
||||
function addBlockedM3ChecksFromResolution(report, resolution) {
|
||||
const identityBlocked = resolution.blockers.some((blocker) =>
|
||||
blocker.scope === "m3-box-simu-identity" || blocker.scope === "m3-gateway-simu-identity"
|
||||
);
|
||||
const missingBlocked = resolution.blockers.some((blocker) =>
|
||||
blocker.scope === "m3-direct-target-missing" || blocker.scope === "m3-service-discovery"
|
||||
);
|
||||
const wiringBlocked = resolution.blockers.some((blocker) => blocker.scope === "m3-patch-panel-wiring");
|
||||
const candidatesEvidence = [JSON.stringify({ source: resolution.source, counts: resolution.counts, selected: resolution.selected })];
|
||||
|
||||
report.liveChecks.push(
|
||||
{
|
||||
id: "two-box-simu-online",
|
||||
status: missingBlocked || identityBlocked ? "blocked" : "pass",
|
||||
blockerClass: missingBlocked ? "observability_blocker" : identityBlocked ? "runtime_blocker" : undefined,
|
||||
summary: missingBlocked || identityBlocked
|
||||
? "Read-only direct target probes did not prove two distinct live DEV box-simu resources res_boxsimu_1 and res_boxsimu_2."
|
||||
: "Read-only direct target probes proved both required DEV box-simu resources.",
|
||||
evidence: candidatesEvidence
|
||||
},
|
||||
{
|
||||
id: "two-gateway-simu-online",
|
||||
status: missingBlocked || identityBlocked ? "blocked" : "pass",
|
||||
blockerClass: missingBlocked ? "observability_blocker" : identityBlocked ? "runtime_blocker" : undefined,
|
||||
summary: missingBlocked || identityBlocked
|
||||
? "Read-only direct target probes did not prove two distinct live DEV gateway-simu identities."
|
||||
: "Read-only direct target probes proved two distinct DEV gateway-simu identities.",
|
||||
evidence: candidatesEvidence
|
||||
},
|
||||
{
|
||||
id: "patch-panel-healthy",
|
||||
status: resolution.selected.patchPanel ? "pass" : "blocked",
|
||||
blockerClass: resolution.selected.patchPanel ? undefined : "observability_blocker",
|
||||
summary: resolution.selected.patchPanel
|
||||
? "DEV patch-panel direct target returned live status."
|
||||
: "DEV patch-panel direct target was not callable.",
|
||||
evidence: [JSON.stringify(resolution.selected.patchPanel ?? { status: "not_observed" })]
|
||||
},
|
||||
{
|
||||
id: "wiring-do1-di1-applied",
|
||||
status: wiringBlocked ? "blocked" : resolution.ok ? "pass" : "not_run",
|
||||
blockerClass: wiringBlocked ? "runtime_blocker" : undefined,
|
||||
summary: wiringBlocked
|
||||
? "DEV patch-panel wiring is active but does not contain the required res_boxsimu_1:DO1 -> res_boxsimu_2:DI1 route."
|
||||
: resolution.ok
|
||||
? "DEV patch-panel active wiring matched res_boxsimu_1:DO1 -> res_boxsimu_2:DI1."
|
||||
: "Patch-panel wiring check was not conclusive because earlier direct target checks were blocked.",
|
||||
evidence: [JSON.stringify(resolution.patchPanelWiring ?? { status: "not_observed" })]
|
||||
},
|
||||
{
|
||||
id: "direct-call-do-write-di-read",
|
||||
status: "not_run",
|
||||
summary: "Not attempted because read-only DEV M3 preconditions were blocked; no box-simu loopback or SOURCE/DRY-RUN substitute was used.",
|
||||
evidence: ["No live DEV write was sent."]
|
||||
},
|
||||
{
|
||||
id: "audit-evidence-traceable",
|
||||
status: "not_run",
|
||||
summary: "Not attempted because the patch-panel-bound live operation was not safe to trigger.",
|
||||
evidence: ["operationId=not_observed", "traceId=not_observed", "auditId=not_observed", "evidenceId=not_observed"]
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
function assertBoxState(state, label) {
|
||||
assert.equal(state?.serviceId, "hwlab-box-simu", `${label} serviceId`);
|
||||
assert.equal(state.live, true, `${label} live`);
|
||||
@@ -790,31 +1230,68 @@ async function main() {
|
||||
return;
|
||||
}
|
||||
|
||||
const { missing, targets } = targetMapFromEnvironment();
|
||||
if (missing.length > 0) {
|
||||
addNotRunM3Checks(report, "Stopped before M3 direct checks because service target URLs were not provided.");
|
||||
report.blockers.push({
|
||||
type: "observability_blocker",
|
||||
scope: "m3-service-discovery",
|
||||
status: "open",
|
||||
classification: "runner permission/mount gap",
|
||||
sourceIssue: "pikasTech/HWLAB#46",
|
||||
summary: `DEV ingress is reachable on frozen :16667, but this runner cannot use ${runnerK3sKubeconfigPath} as readable kubeconfig to discover direct M3 service URLs (${missing.join(", ")}); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline.`
|
||||
});
|
||||
const environmentTargets = targetMapFromEnvironment();
|
||||
const kubernetesDiscovery = environmentTargets.missing.length === 0
|
||||
? {
|
||||
status: "not_run",
|
||||
source: "environment",
|
||||
command: "environment direct target variables",
|
||||
services: {},
|
||||
summary: "All direct target URLs were supplied by environment variables."
|
||||
}
|
||||
: collectKubernetesDirectTargetDiscovery();
|
||||
report.directTargetDiscovery = {
|
||||
environmentMissing: environmentTargets.missing,
|
||||
kubernetes: {
|
||||
status: kubernetesDiscovery.status,
|
||||
source: kubernetesDiscovery.source,
|
||||
command: kubernetesDiscovery.command,
|
||||
exitCode: kubernetesDiscovery.exitCode ?? null,
|
||||
services: kubernetesDiscovery.services,
|
||||
summary: kubernetesDiscovery.summary
|
||||
}
|
||||
};
|
||||
const directTargets = await resolveDirectM3Targets({
|
||||
environmentTargets,
|
||||
kubernetesDiscovery
|
||||
});
|
||||
report.directTargetDiscovery = {
|
||||
...report.directTargetDiscovery,
|
||||
source: directTargets.source,
|
||||
counts: directTargets.counts,
|
||||
selected: directTargets.selected,
|
||||
candidates: directTargets.candidates,
|
||||
patchPanelWiring: directTargets.patchPanelWiring,
|
||||
summary: directTargets.ok
|
||||
? "Direct DEV M3 targets and patch-panel M3 wiring were discovered; live operation can be attempted."
|
||||
: "Direct DEV M3 targets were probed read-only, but required identities or patch-panel M3 wiring were not satisfied."
|
||||
};
|
||||
|
||||
if (!directTargets.ok) {
|
||||
addBlockedM3ChecksFromResolution(report, directTargets);
|
||||
report.liveOperation = {
|
||||
status: "not_run",
|
||||
operationId: "not_observed",
|
||||
traceId: "not_observed",
|
||||
auditId: "not_observed",
|
||||
evidenceId: "not_observed",
|
||||
summary: "No live DEV operation was attempted because read-only direct target checks did not prove the required patch-panel-owned M3 route."
|
||||
};
|
||||
report.blockers.push(...directTargets.blockers);
|
||||
report.summary = {
|
||||
status: "blocked",
|
||||
classification: "runner permission/mount gap",
|
||||
classification: directTargets.blockers[0]?.classification ?? "direct_target_blocked",
|
||||
observedAt,
|
||||
result: "DEV ingress was reachable, but live M3 simulator and patch-panel targets were not discoverable."
|
||||
result: "DEV ingress and direct targets were reachable, but M3 DEV-LIVE was not triggered because required identities or patch-panel wiring were missing."
|
||||
};
|
||||
await writeReport(report, reportPath);
|
||||
console.log(`[dev-m3-smoke] status=blocked report=${relativePath(reportPath)}`);
|
||||
console.log("[dev-m3-smoke] blocker=observability_blocker scope=m3-service-discovery");
|
||||
console.log(`[dev-m3-smoke] blocker=${directTargets.blockers[0]?.type ?? "runtime_blocker"} scope=${directTargets.blockers[0]?.scope ?? "m3-direct-targets"}`);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const live = await runLiveM3Targets(targets);
|
||||
const live = await runLiveM3Targets(directTargets.targets);
|
||||
report.liveChecks.push(
|
||||
{
|
||||
id: "two-box-simu-online",
|
||||
|
||||
@@ -268,6 +268,7 @@ function applyPriority(blocker) {
|
||||
blocker.scope === "d601-k3s" ||
|
||||
blocker.scope === "runner-kubeconfig-readonly-gap" ||
|
||||
blocker.scope === "m3-service-discovery" ||
|
||||
blocker.scope === "m3-direct-target-missing" ||
|
||||
blocker.scope.startsWith("d601-")
|
||||
) {
|
||||
return {
|
||||
@@ -319,7 +320,12 @@ function applyPriority(blocker) {
|
||||
};
|
||||
}
|
||||
|
||||
if (blocker.scope === "m3-hardware-loop-runtime") {
|
||||
if (
|
||||
blocker.scope === "m3-hardware-loop-runtime" ||
|
||||
blocker.scope === "m3-patch-panel-wiring" ||
|
||||
blocker.scope === "m3-box-simu-identity" ||
|
||||
blocker.scope === "m3-gateway-simu-identity"
|
||||
) {
|
||||
return {
|
||||
...blocker,
|
||||
priority: "P0",
|
||||
@@ -526,12 +532,16 @@ function m3TrustedLoopSummary(m3Report) {
|
||||
const operationSummary = m3Report.liveOperation?.summary ?? "No live M3 hardware operation was observed.";
|
||||
const blockerSummary = m3Report.blockers?.find((blocker) =>
|
||||
blocker.scope === "m3-hardware-loop-runtime" ||
|
||||
blocker.scope === "m3-service-discovery"
|
||||
blocker.scope === "m3-service-discovery" ||
|
||||
blocker.scope === "m3-direct-target-missing" ||
|
||||
blocker.scope === "m3-box-simu-identity" ||
|
||||
blocker.scope === "m3-gateway-simu-identity" ||
|
||||
blocker.scope === "m3-patch-panel-wiring"
|
||||
)?.summary;
|
||||
if (statusIsPass(m3Report.liveOperation?.status) || !blockerSummary) {
|
||||
return operationSummary;
|
||||
}
|
||||
return `${operationSummary} Blocker: ${blockerSummary}.`;
|
||||
return `${operationSummary} Blocker: ${blockerSummary.replace(/[.。]+$/u, "")}.`;
|
||||
}
|
||||
|
||||
function collectM4Evidence(reports) {
|
||||
@@ -1009,13 +1019,23 @@ function buildMilestoneBlockerClassification(reports) {
|
||||
const m3Live = statusIsPass(reports.devM3Hardware.liveOperation?.status);
|
||||
const m4Live = statusIsPass(reports.devM4Agent.livePreflight?.status);
|
||||
const m3ServiceDiscoveryBlocked = reports.devM3Hardware.blockers?.some((blocker) =>
|
||||
blocker.scope === "m3-service-discovery"
|
||||
blocker.scope === "m3-service-discovery" || blocker.scope === "m3-direct-target-missing"
|
||||
) === true;
|
||||
const m3IdentityBlocked = reports.devM3Hardware.blockers?.some((blocker) =>
|
||||
blocker.scope === "m3-box-simu-identity" || blocker.scope === "m3-gateway-simu-identity"
|
||||
) === true;
|
||||
const m3WiringBlocked = reports.devM3Hardware.blockers?.some((blocker) =>
|
||||
blocker.scope === "m3-patch-panel-wiring"
|
||||
) === true;
|
||||
const m3BlockerClass = m3Live
|
||||
? "cleared"
|
||||
: m3ServiceDiscoveryBlocked
|
||||
? "runner-readonly-observability-gap"
|
||||
: "hardware-loop-runtime";
|
||||
? "direct-target-missing"
|
||||
: m3IdentityBlocked
|
||||
? "direct-target-identity-gap"
|
||||
: m3WiringBlocked
|
||||
? "patch-panel-m3-wiring-missing"
|
||||
: "hardware-loop-runtime";
|
||||
|
||||
return [
|
||||
{
|
||||
@@ -1024,8 +1044,12 @@ function buildMilestoneBlockerClassification(reports) {
|
||||
currentLevel: m3Live ? "DEV-LIVE" : "BLOCKED",
|
||||
blockerClass: m3BlockerClass,
|
||||
dependency: m3ServiceDiscoveryBlocked
|
||||
? "Runner read-only service discovery must be repaired before the real DEV trusted loop can be observed; this does not mean D601/k3s is globally unavailable."
|
||||
: "Real DEV trusted loop through res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1.",
|
||||
? "Direct DEV simulator and patch-panel targets must be discoverable before the real trusted loop can be observed."
|
||||
: m3IdentityBlocked
|
||||
? "Two distinct live box-simu resources and two distinct gateway-simu identities are required before M3 can run."
|
||||
: m3WiringBlocked
|
||||
? "The callable DEV patch-panel must actively carry res_boxsimu_1:DO1 -> res_boxsimu_2:DI1 before the write/read operation can run."
|
||||
: "Real DEV trusted loop through res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1.",
|
||||
evidence: [
|
||||
`operationId=${reports.devM3Hardware.liveOperation?.operationId ?? "not_observed"}`,
|
||||
`traceId=${reports.devM3Hardware.liveOperation?.traceId ?? "not_observed"}`,
|
||||
@@ -1033,11 +1057,18 @@ function buildMilestoneBlockerClassification(reports) {
|
||||
`evidenceId=${reports.devM3Hardware.liveOperation?.evidenceId ?? "not_observed"}`,
|
||||
`runnerKubeconfigReadable=${reports.devM3Hardware.d601Observability?.runnerKubeconfigReadable === true}`,
|
||||
`d601PublicEndpointsReachable=${reports.devM3Hardware.d601Observability?.d601PublicEndpointsReachable === true}`,
|
||||
`d601K3sUnavailable=${reports.devM3Hardware.d601Observability?.d601K3sUnavailable === true}`
|
||||
`d601K3sUnavailable=${reports.devM3Hardware.d601Observability?.d601K3sUnavailable === true}`,
|
||||
`directTargetSource=${reports.devM3Hardware.directTargetDiscovery?.source ?? "not_observed"}`,
|
||||
`directTargetCounts=${JSON.stringify(reports.devM3Hardware.directTargetDiscovery?.counts ?? {})}`,
|
||||
`patchPanelM3Wiring=${reports.devM3Hardware.directTargetDiscovery?.patchPanelWiring?.hasRequiredConfiguredConnection === true}`
|
||||
],
|
||||
nextRequired: m3ServiceDiscoveryBlocked
|
||||
? "Repair the #46 runner readonly kubeconfig/service-discovery gap, then run the bounded DEV M3 live smoke only when direct simulator and patch-panel targets are discoverable."
|
||||
: "Run the bounded DEV M3 live smoke only after patch-panel topology can prove operation, trace, audit, and evidence IDs.",
|
||||
? "Expose or document callable DEV direct targets for both simulators and the patch-panel, then rerun the read-only M3 preflight."
|
||||
: m3IdentityBlocked
|
||||
? "Fix DEV simulator instance identity so two box-simu pods report res_boxsimu_1/res_boxsimu_2 and two gateway-simu pods report distinct gateway identities."
|
||||
: m3WiringBlocked
|
||||
? "Load/apply DEV patch-panel wiring for res_boxsimu_1:DO1 -> res_boxsimu_2:DI1, then rerun the bounded DEV M3 smoke."
|
||||
: "Run the bounded DEV M3 live smoke only after patch-panel topology can prove operation, trace, audit, and evidence IDs.",
|
||||
nonPromotionReason: "Public frontend, route, and artifact evidence do not prove the required hardware loop."
|
||||
},
|
||||
{
|
||||
@@ -1125,8 +1156,11 @@ function fallbackAction(scope) {
|
||||
if (scope.includes("edge") || scope.includes("ingress") || scope.includes("frp")) return "Repair frp/master-edge/D601 router path and rerun read-only DEV edge health.";
|
||||
if (scope.includes("cloud-api-db")) return "Configure DEV cloud-api DB env readiness and rerun health/preflight without exposing secrets.";
|
||||
if (scope === "db-live") return "Repair DEV cloud-api DB live readiness, then rerun the read-only health and M4 preflight reports without exposing secret values.";
|
||||
if (scope === "m3-patch-panel-wiring") return "Load/apply DEV patch-panel wiring for res_boxsimu_1:DO1 -> res_boxsimu_2:DI1, then rerun the bounded DEV M3 smoke.";
|
||||
if (scope === "m3-box-simu-identity") return "Fix DEV box-simu instance identity so direct endpoints expose distinct res_boxsimu_1 and res_boxsimu_2 resources.";
|
||||
if (scope === "m3-gateway-simu-identity") return "Fix DEV gateway-simu instance identity so direct endpoints expose two distinct gateway sessions.";
|
||||
if (scope === "m3-hardware-loop-runtime") return "Prove the real DEV M3 trusted loop res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 with operation, trace, audit, and evidence identifiers.";
|
||||
if (scope === "runner-kubeconfig-readonly-gap" || scope === "m3-service-discovery") return "Repair the #46 runner kubeconfig mount/permission or document the approved alternate read-only KUBECONFIG/service-discovery path; do not classify this as D601 global offline.";
|
||||
if (scope === "runner-kubeconfig-readonly-gap" || scope === "m3-service-discovery" || scope === "m3-direct-target-missing") return "Repair the #46 runner kubeconfig mount/permission or document the approved alternate read-only KUBECONFIG/service-discovery path; do not classify this as D601 global offline.";
|
||||
if (scope.includes("kubectl") || scope.includes("k3s")) return "Provide read-only kubectl/kubeconfig observability for hwlab-dev.";
|
||||
return "Resolve the blocker and attach source/local/dry-run/DEV-live evidence at the correct level.";
|
||||
}
|
||||
@@ -1138,8 +1172,11 @@ function evidenceRequiredFor(scope) {
|
||||
if (scope.includes("kubectl") || scope.includes("k3s")) return "Read-only kubectl/k3s report proving pods/services/configmaps are observable in hwlab-dev without reading Secrets.";
|
||||
if (scope.includes("cloud-api-db")) return "Cloud API health/live output showing DB env ready and redacted secret references, without secret material.";
|
||||
if (scope === "db-live") return "Cloud API /health/live output with ready=true, connected=true, liveDbEvidence=true, and redacted secret references.";
|
||||
if (scope === "m3-patch-panel-wiring") return "Read-only patch-panel /status and /wiring showing active res_boxsimu_1:DO1 -> res_boxsimu_2:DI1 before a bounded write/read smoke records operation, trace, audit, and evidence IDs.";
|
||||
if (scope === "m3-box-simu-identity") return "Read-only direct box-simu /health/live and /status output showing distinct res_boxsimu_1 and res_boxsimu_2 resources.";
|
||||
if (scope === "m3-gateway-simu-identity") return "Read-only direct gateway-simu /health/live and /status output showing two distinct gateway identities/sessions.";
|
||||
if (scope === "m3-hardware-loop-runtime") return "Operation, trace, audit, and evidence IDs from the real DEV DO1 -> patch-panel -> DI1 trusted loop.";
|
||||
if (scope === "runner-kubeconfig-readonly-gap" || scope === "m3-service-discovery") return "Read-only report with runnerKubeconfigReadable, runnerKubeconfigProbeExitCode/stderr, d601PublicEndpointsReachable, and d601K3sUnavailable recorded separately, plus direct M3 service target discovery before any DO write.";
|
||||
if (scope === "runner-kubeconfig-readonly-gap" || scope === "m3-service-discovery" || scope === "m3-direct-target-missing") return "Read-only report with runnerKubeconfigReadable, runnerKubeconfigProbeExitCode/stderr, d601PublicEndpointsReachable, and d601K3sUnavailable recorded separately, plus direct M3 service target discovery before any DO write.";
|
||||
if (scope.includes("edge") || scope.includes("ingress") || scope.includes("frp")) return "Read-only DEV route observation for :16667/frp/edge/router with HWLAB service identity and artifact identity.";
|
||||
return "A committed report with the exact evidence level and command used.";
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user