feat: add dev evidence blocker aggregator
This commit is contained in:
HWLAB Code Queue
@@ -0,0 +1,72 @@
|
||||
# DEV Evidence / Blocker Aggregator
|
||||
|
||||
This is the lightweight report-only entry point for `pikasTech/HWLAB#41`. It
|
||||
helps a commander decide whether the DEV acceptance path is green, blocked, or
|
||||
ready for another repair round across `#33-#39`, `#7`, and `#31`.
|
||||
|
||||
Run from the repository root:
|
||||
|
||||
```sh
|
||||
node scripts/dev-evidence-blocker-aggregator.mjs --pretty
|
||||
```
|
||||
|
||||
For a short validation response:
|
||||
|
||||
```sh
|
||||
node scripts/dev-evidence-blocker-aggregator.mjs --check
|
||||
```
|
||||
|
||||
## Source Levels
|
||||
|
||||
The aggregator keeps evidence levels separate so fixture output cannot be
|
||||
promoted to live DEV evidence:
|
||||
|
||||
| Level | Meaning |
|
||||
| --- | --- |
|
||||
| `SOURCE` | Repository contracts, schemas, docs, manifests, or examples. |
|
||||
| `LOCAL` | Localhost or in-process smoke results only. |
|
||||
| `DRY-RUN` | Fixture-backed or no-mutation dry-run readiness evidence. |
|
||||
| `DEV-LIVE` | Read-only evidence observed from the real DEV route. |
|
||||
| `BLOCKED` | Missing issue reads, missing DEV evidence, missing artifacts, or other open blockers. |
|
||||
|
||||
## Current Contract Map
|
||||
|
||||
| Milestone | Current visible level | Contract entry |
|
||||
| --- | --- | --- |
|
||||
| M0 | `SOURCE` | `docs/m0-contract-audit.md`, protocol schemas, evidence chain examples, M0 validator. |
|
||||
| M1 | `LOCAL` | `docs/m1-local-smoke.md`, `fixtures/mvp/runtime.json`, `scripts/m1-contract-smoke.mjs`. |
|
||||
| M2 | `DRY-RUN` | DEV deploy smoke fixture, artifact catalog, runtime boundary, and M2 dry-run validator. |
|
||||
| M3 | `LOCAL` | Hardware trusted loop topology fixture and local patch-panel smoke. |
|
||||
| M4 | `LOCAL` | Agent automation loop fixture, local runtime skeleton, skills commit, evidence, and cleanup smoke. |
|
||||
| M5 | `DRY-RUN` | MVP e2e dry-run plan and orchestration validator. |
|
||||
|
||||
## Blocker Posture
|
||||
|
||||
If GitHub issues are not readable in the runner, the report marks
|
||||
`issue-read` as blocked and uses only committed repository artifacts plus the
|
||||
task prompt. This keeps #33-#39 status visible without treating issue comments
|
||||
as silently available.
|
||||
|
||||
The report is blocked until real DEV-LIVE evidence is attached for the DEV
|
||||
route and downstream HWLAB runtime services. Current source, local, and dry-run
|
||||
contracts are useful readiness signals, but they are not live DEV proof.
|
||||
|
||||
## Next-Round Buckets
|
||||
|
||||
The JSON output groups recommended next actions into:
|
||||
|
||||
- `artifact/base image`: real publish or provenance evidence for frozen HWLAB
|
||||
service IDs.
|
||||
- `edge/network`: read-only DEV route observations for ingress, edge, frp, and
|
||||
D601 router.
|
||||
- `runtime/health`: DEV health JSON for cloud, gateway/simulator,
|
||||
patch-panel, agent, worker, skills, and CLI surfaces.
|
||||
- `M3/M4 live`: narrow authorized DEV-only loop evidence with trace, audit,
|
||||
operation, session, evidence, and cleanup IDs.
|
||||
|
||||
## Boundaries
|
||||
|
||||
The aggregator does not deploy, call DEV, call PROD, run browser tests, run
|
||||
heavy e2e, read secrets, restart services, or replace HWLAB runtime services
|
||||
with UniDesk services. It is a local report generator and static contract
|
||||
checker only.
|
||||
+3
-1
@@ -5,8 +5,10 @@
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"validate": "node scripts/validate-contract.mjs",
|
||||
"check": "node --check internal/protocol/index.mjs && node --check internal/agent/index.mjs && node --check scripts/validate-contract.mjs && node --check scripts/dev-artifact-publish.mjs && node scripts/validate-contract.mjs && node --test internal/agent/index.test.mjs && sh -n scripts/bootstrap-skills.sh scripts/worker-entrypoint.sh",
|
||||
"check": "node --check internal/protocol/index.mjs && node --check internal/agent/index.mjs && node --check scripts/validate-contract.mjs && node --check scripts/dev-artifact-publish.mjs && node --check scripts/dev-evidence-blocker-aggregator.mjs && node scripts/validate-contract.mjs && node scripts/dev-evidence-blocker-aggregator.mjs --check && node --test internal/agent/index.test.mjs && sh -n scripts/bootstrap-skills.sh scripts/worker-entrypoint.sh",
|
||||
"m1:smoke": "node scripts/m1-contract-smoke.mjs",
|
||||
"dev:evidence": "node scripts/dev-evidence-blocker-aggregator.mjs --pretty",
|
||||
"dev:evidence:check": "node scripts/dev-evidence-blocker-aggregator.mjs --check",
|
||||
"cli:health": "node tools/hwlab-cli/bin/hwlab-cli.mjs health",
|
||||
"cli:dry-run": "node tools/hwlab-cli/bin/hwlab-cli.mjs test e2e --env dev --mvp --dry-run",
|
||||
"cli:projects": "node tools/hwlab-cli/bin/hwlab-cli.mjs project list",
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
#!/usr/bin/env node
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { access, readdir, readFile } from "node:fs/promises";
|
||||
import { constants as fsConstants } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { DEV_ENDPOINT, ENVIRONMENT_DEV, SERVICE_IDS } from "../internal/protocol/index.mjs";
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const levels = ["SOURCE", "LOCAL", "DRY-RUN", "DEV-LIVE", "BLOCKED"];
|
||||
const blockerTypes = ["contract_blocker", "environment_blocker", "network_blocker", "runtime_blocker", "agent_blocker", "observability_blocker", "safety_blocker"];
|
||||
const requiredArtifactFields = ["serviceId", "commitId", "image", "tag", "digest", "buildSource", "deployEnv", "healthTimestamp"];
|
||||
const issue = (id) => `pikasTech/HWLAB${id}`;
|
||||
const allSupportedIssues = ["#33", "#34", "#35", "#36", "#37", "#38", "#39", "#7", "#31"].map(issue);
|
||||
|
||||
const contracts = [
|
||||
contract("M0", "SOURCE", "artifact/base image", ["#7", "#31"], ["docs/m0-contract-audit.md", "protocol/schemas/evidence-record.schema.json", "protocol/evidence-chain.md", "protocol/examples/m0-contract/service-ids.json", "scripts/validate-m0-contract.mjs", "scripts/validate-evidence-chain.mjs"], ["node scripts/validate-m0-contract.mjs", "node scripts/validate-evidence-chain.mjs"], "Source contracts freeze service IDs, JSON-RPC, audit, topology, evidence records, and deploy shape."),
|
||||
contract("M1", "LOCAL", "runtime/health", ["#7", "#31"], ["docs/m1-local-smoke.md", "fixtures/mvp/runtime.json", "scripts/m1-contract-smoke.mjs"], ["node scripts/m1-contract-smoke.mjs"], "Local smoke covers cloud-api, simulators, patch-panel routing, CLI dry-run boundary, and no DEV/PROD mutation."),
|
||||
contract("M2", "DRY-RUN", "edge/network", ["#7", "#31", "#33", "#34", "#35", "#39"], ["docs/m2-dev-deploy-smoke.md", "docs/dev-acceptance-checklist.json", "fixtures/dev-deploy-smoke/dev-deploy-smoke.json", "deploy/artifact-catalog.dev.json", "scripts/m2-dev-deploy-smoke.mjs", "scripts/validate-artifact-catalog.mjs", "scripts/validate-runtime-boundary.mjs"], ["node scripts/m2-dev-deploy-smoke.mjs --dry-run", "node scripts/validate-artifact-catalog.mjs", "node scripts/validate-runtime-boundary.mjs"], "Dry-run contract covers edge proxy, frp, D601 router, cloud surface, agent runtime, sim, and patch-panel route phases."),
|
||||
contract("M3", "LOCAL", "M3/M4 live", ["#7", "#31", "#36", "#39"], ["docs/m3-hardware-loop.md", "fixtures/mvp/m3-hardware-loop/topology.json", "scripts/m3-hardware-loop-smoke.mjs"], ["node scripts/m3-hardware-loop-smoke.mjs"], "Local hardware trusted loop fixture proves patch-panel-owned box-simu-1 DO1 to box-simu-2 DI1 propagation."),
|
||||
contract("M4", "LOCAL", "M3/M4 live", ["#7", "#31", "#37", "#39"], ["docs/m4-agent-loop.md", "fixtures/mvp/m4-agent-loop/runtime.json", "fixtures/mvp/m4-agent-loop/evidence/m4-agent-loop-plan.md", "scripts/m4-agent-loop-smoke.mjs"], ["node scripts/m4-agent-loop-smoke.mjs"], "Local agent loop fixture covers session lifecycle, workspace isolation, skills commit wiring, evidence, and cleanup."),
|
||||
contract("M5", "DRY-RUN", "runtime/health", ["#7", "#31", "#38", "#39"], ["docs/m5-mvp-e2e.md", "protocol/mvp-e2e-contract.md", "fixtures/mvp/m5-e2e/dry-run-plan.json", "scripts/m5-mvp-e2e-dry-run.mjs"], ["node scripts/m5-mvp-e2e-dry-run.mjs"], "Dry-run orchestration checks the M0-M4 dependency graph, MVP sequence, health, trace, audit, evidence, and cleanup contracts.")
|
||||
];
|
||||
|
||||
const visibleParallelResults = [
|
||||
parallel("#33", "BLOCKED", "Prompt says #33-#39 are parallel real DEV workstreams. No issue-specific committed DEV-LIVE result is visible in this checkout."),
|
||||
parallel("#34", "DRY-RUN", "Repository M2 dry-run and artifact contracts can support deploy/artifact review, but this runner cannot read the issue result itself."),
|
||||
parallel("#35", "DRY-RUN", "Repository DEV route fixtures cover edge, frp, and D601 router contracts as DRY-RUN evidence only."),
|
||||
parallel("#36", "LOCAL", "Repository M3 topology fixture and smoke script provide LOCAL evidence only; no live DEV M3 loop output is attached."),
|
||||
parallel("#37", "LOCAL", "Repository M4 runtime fixture and smoke script provide LOCAL evidence only; no live DEV M4 agent run output is attached."),
|
||||
parallel("#38", "DRY-RUN", "Repository M5 dry-run plan validates orchestration readiness, not a live DEV MVP e2e run."),
|
||||
parallel("#39", "BLOCKED", "No repository-local DEV-LIVE gate result is visible to this aggregator.")
|
||||
];
|
||||
|
||||
const nextRoundRecommendations = [
|
||||
next("artifact/base image", "high", ["#33", "#35"], "Publish or attach real DEV artifact provenance for each frozen service ID: commitId, image, immutable tag, digest, buildSource, deployEnv, and healthTimestamp."),
|
||||
next("edge/network", "high", ["#35", "#39"], "Attach read-only DEV route observations for public ingress, master edge proxy, frp, and D601 hwlab-dev/hwlab-router without deploying, restarting, or reading secrets."),
|
||||
next("runtime/health", "high", ["#34", "#35", "#38", "#39"], "Attach current DEV health JSON for cloud API/web, gateway or simulator, box simulator, patch panel, agent manager, worker, skills, and CLI, all tied to HWLAB service IDs."),
|
||||
next("M3/M4 live", "medium", ["#36", "#37", "#38"], "After DEV runtime smoke is green, run the smallest authorized DEV-only M3/M4 loop and attach trace, audit, evidence, operation, session, and cleanup IDs.")
|
||||
];
|
||||
|
||||
function contract(milestone, source, category, supports, paths, commands, summary) {
|
||||
return {
|
||||
milestone,
|
||||
source,
|
||||
status: source === "SOURCE" ? "contract_available" : `${source.toLowerCase().replace("-", "_")}_contract_available`,
|
||||
category,
|
||||
supports: supports.map(issue),
|
||||
paths,
|
||||
commands,
|
||||
summary
|
||||
};
|
||||
}
|
||||
|
||||
function parallel(issue, source, visibleResult) {
|
||||
return {
|
||||
issue: `pikasTech/HWLAB${issue}`,
|
||||
issueRead: "blocked",
|
||||
visibleResult,
|
||||
source
|
||||
};
|
||||
}
|
||||
|
||||
function next(category, priority, owners, recommendation) {
|
||||
return { category, priority, owners, recommendation };
|
||||
}
|
||||
|
||||
async function readJSON(relativePath) {
|
||||
const raw = await readFile(path.join(repoRoot, relativePath), "utf8");
|
||||
return JSON.parse(raw);
|
||||
}
|
||||
|
||||
async function pathExists(relativePath) {
|
||||
try {
|
||||
await access(path.join(repoRoot, relativePath), fsConstants.R_OK);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function gitCommit() {
|
||||
try {
|
||||
return execFileSync("git", ["rev-parse", "--short=12", "HEAD"], {
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "ignore"]
|
||||
}).trim();
|
||||
} catch {
|
||||
return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
async function assertStaticContracts() {
|
||||
const checklist = await readJSON("docs/dev-acceptance-checklist.json");
|
||||
assert.equal(checklist.environment, ENVIRONMENT_DEV, "DEV checklist environment");
|
||||
assert.equal(checklist.endpoint, DEV_ENDPOINT, "DEV checklist endpoint");
|
||||
assert.deepEqual(checklist.artifactObservabilityFields, requiredArtifactFields);
|
||||
|
||||
const catalog = await readJSON("deploy/artifact-catalog.dev.json");
|
||||
assert.equal(catalog.environment, ENVIRONMENT_DEV, "artifact catalog environment");
|
||||
assert.equal(catalog.endpoint, DEV_ENDPOINT, "artifact catalog endpoint");
|
||||
assert.equal(catalog.publish.ciPublished, false, "skeleton catalog must not claim CI publish");
|
||||
assert.equal(catalog.publish.registryVerified, false, "skeleton catalog must not claim registry verification");
|
||||
assert.deepEqual(catalog.services.map((service) => service.serviceId).sort(), [...SERVICE_IDS].sort());
|
||||
|
||||
const m2Fixture = await readJSON("fixtures/dev-deploy-smoke/dev-deploy-smoke.json");
|
||||
assert.equal(m2Fixture.environment, ENVIRONMENT_DEV, "M2 fixture environment");
|
||||
assert.equal(m2Fixture.endpoint, DEV_ENDPOINT, "M2 fixture endpoint");
|
||||
assert.equal(m2Fixture.defaultMode, "dry-run", "M2 fixture default mode");
|
||||
|
||||
const runtime = await readJSON("fixtures/mvp/runtime.json");
|
||||
assert.equal(runtime.endpoints.dev, DEV_ENDPOINT, "MVP runtime endpoint");
|
||||
|
||||
const m5Plan = await readJSON("fixtures/mvp/m5-e2e/dry-run-plan.json");
|
||||
assert.equal(m5Plan.mode, "dry-run", "M5 plan mode");
|
||||
assert.equal(m5Plan.environment, ENVIRONMENT_DEV, "M5 plan environment");
|
||||
assert.equal(m5Plan.endpoint, DEV_ENDPOINT, "M5 plan endpoint");
|
||||
assert.equal(m5Plan.safety.allowNetwork, false, "M5 dry-run network gate");
|
||||
assert.equal(m5Plan.safety.allowDeploy, false, "M5 dry-run deploy gate");
|
||||
assert.equal(m5Plan.safety.allowSecrets, false, "M5 dry-run secret gate");
|
||||
}
|
||||
|
||||
async function collectContracts() {
|
||||
return Promise.all(
|
||||
contracts.map(async (entry) => {
|
||||
const paths = await Promise.all(entry.paths.map(async (item) => ({ path: item, exists: await pathExists(item) })));
|
||||
const missing = paths.filter((item) => !item.exists).map((item) => item.path);
|
||||
return {
|
||||
...entry,
|
||||
status: missing.length === 0 ? entry.status : "blocked_missing_contract_file",
|
||||
paths,
|
||||
missing
|
||||
};
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
async function collectDevGateReports() {
|
||||
const entries = await readdir(path.join(repoRoot, "reports/dev-gate"), { withFileTypes: true });
|
||||
const reports = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile() || !entry.name.endsWith(".json")) continue;
|
||||
const reportPath = path.join("reports/dev-gate", entry.name);
|
||||
const report = await readJSON(reportPath);
|
||||
reports.push({
|
||||
path: reportPath,
|
||||
issue: report.issue,
|
||||
taskId: report.taskId,
|
||||
acceptanceLevel: report.acceptanceLevel,
|
||||
status: "contract_available",
|
||||
blockers: Array.isArray(report.blockers) ? report.blockers.length : 0,
|
||||
notes: report.notes ?? ""
|
||||
});
|
||||
}
|
||||
return reports.sort((a, b) => a.path.localeCompare(b.path));
|
||||
}
|
||||
|
||||
function buildBlockers(collectedContracts) {
|
||||
const blockers = [
|
||||
blocker("observability_blocker", "issue-read", "GitHub issue state is not readable in this runner; visible #33-#39 results are limited to repository artifacts and prompt context."),
|
||||
blocker("network_blocker", "DEV-LIVE", "No repository-local artifact can be treated as live DEV evidence for ingress, edge, frp, D601 router, or downstream HWLAB services."),
|
||||
blocker("observability_blocker", "artifact-provenance", "The checked artifact catalog is a contract skeleton; it explicitly does not claim real CI publish, registry verification, or deployment."),
|
||||
blocker("runtime_blocker", "M3/M4 DEV loops", "M3 and M4 currently have local contract smoke evidence only; live DEV trace, audit, operation, evidence, and cleanup IDs are not attached.")
|
||||
];
|
||||
for (const item of collectedContracts) {
|
||||
if (item.missing.length > 0) {
|
||||
blockers.push(blocker("contract_blocker", item.milestone, `${item.milestone} is missing repository contract file(s): ${item.missing.join(", ")}.`));
|
||||
}
|
||||
}
|
||||
return blockers;
|
||||
}
|
||||
|
||||
function blocker(type, scope, summary) {
|
||||
return { type, scope, status: "open", source: "BLOCKED", summary };
|
||||
}
|
||||
|
||||
function groupBySource(collectedContracts, devLive, blockers) {
|
||||
const grouped = Object.fromEntries(levels.map((level) => [level, []]));
|
||||
for (const item of collectedContracts) {
|
||||
grouped[item.source].push({
|
||||
milestone: item.milestone,
|
||||
supports: item.supports,
|
||||
status: item.status,
|
||||
category: item.category,
|
||||
summary: item.summary
|
||||
});
|
||||
}
|
||||
grouped["DEV-LIVE"].push(...devLive);
|
||||
grouped.BLOCKED.push(...blockers);
|
||||
return grouped;
|
||||
}
|
||||
|
||||
function validateOutput(output) {
|
||||
assert.equal(output.environment, ENVIRONMENT_DEV, "output environment");
|
||||
assert.equal(output.endpoint, DEV_ENDPOINT, "output endpoint");
|
||||
assert.equal(output.devOnly, true, "output devOnly");
|
||||
assert.equal(output.prodDisabled, true, "output prodDisabled");
|
||||
assert.deepEqual(Object.keys(output.levels), levels, "output levels");
|
||||
assert.ok(output.blockers.length > 0, "aggregator must not report green without DEV-LIVE evidence");
|
||||
for (const item of output.blockers) {
|
||||
assert.ok(blockerTypes.includes(item.type), `unknown blocker type ${item.type}`);
|
||||
assert.equal(item.status, "open", "aggregator blockers remain open until live evidence is attached");
|
||||
}
|
||||
}
|
||||
|
||||
async function buildReport() {
|
||||
await assertStaticContracts();
|
||||
const evidenceContracts = await collectContracts();
|
||||
const blockers = buildBlockers(evidenceContracts);
|
||||
const devLive = [{
|
||||
milestone: "M2-M5",
|
||||
issue: "pikasTech/HWLAB#39",
|
||||
status: "not_available",
|
||||
category: "DEV-LIVE",
|
||||
summary: "No live DEV evidence was read or generated by this report-only aggregator."
|
||||
}];
|
||||
const output = {
|
||||
reportVersion: "v1",
|
||||
taskId: "dev-evidence-blocker-aggregator",
|
||||
issue: "pikasTech/HWLAB#41",
|
||||
supports: allSupportedIssues,
|
||||
generatedFromCommit: gitCommit(),
|
||||
environment: ENVIRONMENT_DEV,
|
||||
endpoint: DEV_ENDPOINT,
|
||||
devOnly: true,
|
||||
prodDisabled: true,
|
||||
safety: {
|
||||
reportOnly: true,
|
||||
noLiveOperation: true,
|
||||
noDeploy: true,
|
||||
noProd: true,
|
||||
noSecretRead: true,
|
||||
noHeavyE2E: true,
|
||||
noBrowserTest: true
|
||||
},
|
||||
issueRead: {
|
||||
status: "blocked",
|
||||
reason: "GitHub issue data is not available from this runner; this report uses repository files and prompt context only."
|
||||
},
|
||||
overall: {
|
||||
status: "blocked",
|
||||
green: false,
|
||||
reason: "SOURCE, LOCAL, and DRY-RUN contracts are visible, but DEV-LIVE evidence and issue reads are blocked."
|
||||
},
|
||||
evidenceContracts,
|
||||
devGateReports: await collectDevGateReports(),
|
||||
visibleParallelResults,
|
||||
levels: groupBySource(evidenceContracts, devLive, blockers),
|
||||
blockers,
|
||||
nextRoundRecommendations,
|
||||
validationCommands: [
|
||||
"node --check scripts/dev-evidence-blocker-aggregator.mjs",
|
||||
"node scripts/dev-evidence-blocker-aggregator.mjs --check",
|
||||
"node scripts/dev-evidence-blocker-aggregator.mjs --pretty"
|
||||
]
|
||||
};
|
||||
validateOutput(output);
|
||||
return output;
|
||||
}
|
||||
|
||||
try {
|
||||
const output = await buildReport();
|
||||
if (process.argv.includes("--check")) {
|
||||
process.stdout.write(JSON.stringify({
|
||||
ok: true,
|
||||
taskId: output.taskId,
|
||||
status: output.overall.status,
|
||||
blockers: output.blockers.length,
|
||||
sourceLevels: levels
|
||||
}) + "\n");
|
||||
} else {
|
||||
process.stdout.write(JSON.stringify(output, null, process.argv.includes("--pretty") ? 2 : 0) + "\n");
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
process.stderr.write(JSON.stringify({ ok: false, error: message }) + "\n");
|
||||
process.exitCode = 1;
|
||||
}
|
||||
Reference in New Issue
Block a user