diff --git a/docs/dev-evidence-blocker-aggregator.md b/docs/dev-evidence-blocker-aggregator.md new file mode 100644 index 00000000..019ec851 --- /dev/null +++ b/docs/dev-evidence-blocker-aggregator.md @@ -0,0 +1,72 @@ +# DEV Evidence / Blocker Aggregator + +This is the lightweight report-only entry point for `pikasTech/HWLAB#41`. It +helps a commander decide whether the DEV acceptance path is green, blocked, or +ready for another repair round across `#33-#39`, `#7`, and `#31`. + +Run from the repository root: + +```sh +node scripts/dev-evidence-blocker-aggregator.mjs --pretty +``` + +For a short validation response: + +```sh +node scripts/dev-evidence-blocker-aggregator.mjs --check +``` + +## Source Levels + +The aggregator keeps evidence levels separate so fixture output cannot be +promoted to live DEV evidence: + +| Level | Meaning | +| --- | --- | +| `SOURCE` | Repository contracts, schemas, docs, manifests, or examples. | +| `LOCAL` | Localhost or in-process smoke results only. | +| `DRY-RUN` | Fixture-backed or no-mutation dry-run readiness evidence. | +| `DEV-LIVE` | Read-only evidence observed from the real DEV route. | +| `BLOCKED` | Missing issue reads, missing DEV evidence, missing artifacts, or other open blockers. | + +## Current Contract Map + +| Milestone | Current visible level | Contract entry | +| --- | --- | --- | +| M0 | `SOURCE` | `docs/m0-contract-audit.md`, protocol schemas, evidence chain examples, M0 validator. | +| M1 | `LOCAL` | `docs/m1-local-smoke.md`, `fixtures/mvp/runtime.json`, `scripts/m1-contract-smoke.mjs`. | +| M2 | `DRY-RUN` | DEV deploy smoke fixture, artifact catalog, runtime boundary, and M2 dry-run validator. | +| M3 | `LOCAL` | Hardware trusted loop topology fixture and local patch-panel smoke. | +| M4 | `LOCAL` | Agent automation loop fixture, local runtime skeleton, skills commit, evidence, and cleanup smoke. | +| M5 | `DRY-RUN` | MVP e2e dry-run plan and orchestration validator. | + +## Blocker Posture + +If GitHub issues are not readable in the runner, the report marks +`issue-read` as blocked and uses only committed repository artifacts plus the +task prompt. This keeps #33-#39 status visible without treating issue comments +as silently available. + +The report is blocked until real DEV-LIVE evidence is attached for the DEV +route and downstream HWLAB runtime services. Current source, local, and dry-run +contracts are useful readiness signals, but they are not live DEV proof. + +## Next-Round Buckets + +The JSON output groups recommended next actions into: + +- `artifact/base image`: real publish or provenance evidence for frozen HWLAB + service IDs. +- `edge/network`: read-only DEV route observations for ingress, edge, frp, and + D601 router. +- `runtime/health`: DEV health JSON for cloud, gateway/simulator, + patch-panel, agent, worker, skills, and CLI surfaces. +- `M3/M4 live`: narrow authorized DEV-only loop evidence with trace, audit, + operation, session, evidence, and cleanup IDs. + +## Boundaries + +The aggregator does not deploy, call DEV, call PROD, run browser tests, run +heavy e2e, read secrets, restart services, or replace HWLAB runtime services +with UniDesk services. It is a local report generator and static contract +checker only. diff --git a/package.json b/package.json index 7e59da53..e351a5e8 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,10 @@ "type": "module", "scripts": { "validate": "node scripts/validate-contract.mjs", - "check": "node --check internal/protocol/index.mjs && node --check internal/agent/index.mjs && node --check scripts/validate-contract.mjs && node --check scripts/dev-artifact-publish.mjs && node scripts/validate-contract.mjs && node --test internal/agent/index.test.mjs && sh -n scripts/bootstrap-skills.sh scripts/worker-entrypoint.sh", + "check": "node --check internal/protocol/index.mjs && node --check internal/agent/index.mjs && node --check scripts/validate-contract.mjs && node --check scripts/dev-artifact-publish.mjs && node --check scripts/dev-evidence-blocker-aggregator.mjs && node scripts/validate-contract.mjs && node scripts/dev-evidence-blocker-aggregator.mjs --check && node --test internal/agent/index.test.mjs && sh -n scripts/bootstrap-skills.sh scripts/worker-entrypoint.sh", "m1:smoke": "node scripts/m1-contract-smoke.mjs", + "dev:evidence": "node scripts/dev-evidence-blocker-aggregator.mjs --pretty", + "dev:evidence:check": "node scripts/dev-evidence-blocker-aggregator.mjs --check", "cli:health": "node tools/hwlab-cli/bin/hwlab-cli.mjs health", "cli:dry-run": "node tools/hwlab-cli/bin/hwlab-cli.mjs test e2e --env dev --mvp --dry-run", "cli:projects": "node tools/hwlab-cli/bin/hwlab-cli.mjs project list", diff --git a/scripts/dev-evidence-blocker-aggregator.mjs b/scripts/dev-evidence-blocker-aggregator.mjs new file mode 100644 index 00000000..53c93f0d --- /dev/null +++ b/scripts/dev-evidence-blocker-aggregator.mjs @@ -0,0 +1,281 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { access, readdir, readFile } from "node:fs/promises"; +import { constants as fsConstants } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { DEV_ENDPOINT, ENVIRONMENT_DEV, SERVICE_IDS } from "../internal/protocol/index.mjs"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const levels = ["SOURCE", "LOCAL", "DRY-RUN", "DEV-LIVE", "BLOCKED"]; +const blockerTypes = ["contract_blocker", "environment_blocker", "network_blocker", "runtime_blocker", "agent_blocker", "observability_blocker", "safety_blocker"]; +const requiredArtifactFields = ["serviceId", "commitId", "image", "tag", "digest", "buildSource", "deployEnv", "healthTimestamp"]; +const issue = (id) => `pikasTech/HWLAB${id}`; +const allSupportedIssues = ["#33", "#34", "#35", "#36", "#37", "#38", "#39", "#7", "#31"].map(issue); + +const contracts = [ + contract("M0", "SOURCE", "artifact/base image", ["#7", "#31"], ["docs/m0-contract-audit.md", "protocol/schemas/evidence-record.schema.json", "protocol/evidence-chain.md", "protocol/examples/m0-contract/service-ids.json", "scripts/validate-m0-contract.mjs", "scripts/validate-evidence-chain.mjs"], ["node scripts/validate-m0-contract.mjs", "node scripts/validate-evidence-chain.mjs"], "Source contracts freeze service IDs, JSON-RPC, audit, topology, evidence records, and deploy shape."), + contract("M1", "LOCAL", "runtime/health", ["#7", "#31"], ["docs/m1-local-smoke.md", "fixtures/mvp/runtime.json", "scripts/m1-contract-smoke.mjs"], ["node scripts/m1-contract-smoke.mjs"], "Local smoke covers cloud-api, simulators, patch-panel routing, CLI dry-run boundary, and no DEV/PROD mutation."), + contract("M2", "DRY-RUN", "edge/network", ["#7", "#31", "#33", "#34", "#35", "#39"], ["docs/m2-dev-deploy-smoke.md", "docs/dev-acceptance-checklist.json", "fixtures/dev-deploy-smoke/dev-deploy-smoke.json", "deploy/artifact-catalog.dev.json", "scripts/m2-dev-deploy-smoke.mjs", "scripts/validate-artifact-catalog.mjs", "scripts/validate-runtime-boundary.mjs"], ["node scripts/m2-dev-deploy-smoke.mjs --dry-run", "node scripts/validate-artifact-catalog.mjs", "node scripts/validate-runtime-boundary.mjs"], "Dry-run contract covers edge proxy, frp, D601 router, cloud surface, agent runtime, sim, and patch-panel route phases."), + contract("M3", "LOCAL", "M3/M4 live", ["#7", "#31", "#36", "#39"], ["docs/m3-hardware-loop.md", "fixtures/mvp/m3-hardware-loop/topology.json", "scripts/m3-hardware-loop-smoke.mjs"], ["node scripts/m3-hardware-loop-smoke.mjs"], "Local hardware trusted loop fixture proves patch-panel-owned box-simu-1 DO1 to box-simu-2 DI1 propagation."), + contract("M4", "LOCAL", "M3/M4 live", ["#7", "#31", "#37", "#39"], ["docs/m4-agent-loop.md", "fixtures/mvp/m4-agent-loop/runtime.json", "fixtures/mvp/m4-agent-loop/evidence/m4-agent-loop-plan.md", "scripts/m4-agent-loop-smoke.mjs"], ["node scripts/m4-agent-loop-smoke.mjs"], "Local agent loop fixture covers session lifecycle, workspace isolation, skills commit wiring, evidence, and cleanup."), + contract("M5", "DRY-RUN", "runtime/health", ["#7", "#31", "#38", "#39"], ["docs/m5-mvp-e2e.md", "protocol/mvp-e2e-contract.md", "fixtures/mvp/m5-e2e/dry-run-plan.json", "scripts/m5-mvp-e2e-dry-run.mjs"], ["node scripts/m5-mvp-e2e-dry-run.mjs"], "Dry-run orchestration checks the M0-M4 dependency graph, MVP sequence, health, trace, audit, evidence, and cleanup contracts.") +]; + +const visibleParallelResults = [ + parallel("#33", "BLOCKED", "Prompt says #33-#39 are parallel real DEV workstreams. No issue-specific committed DEV-LIVE result is visible in this checkout."), + parallel("#34", "DRY-RUN", "Repository M2 dry-run and artifact contracts can support deploy/artifact review, but this runner cannot read the issue result itself."), + parallel("#35", "DRY-RUN", "Repository DEV route fixtures cover edge, frp, and D601 router contracts as DRY-RUN evidence only."), + parallel("#36", "LOCAL", "Repository M3 topology fixture and smoke script provide LOCAL evidence only; no live DEV M3 loop output is attached."), + parallel("#37", "LOCAL", "Repository M4 runtime fixture and smoke script provide LOCAL evidence only; no live DEV M4 agent run output is attached."), + parallel("#38", "DRY-RUN", "Repository M5 dry-run plan validates orchestration readiness, not a live DEV MVP e2e run."), + parallel("#39", "BLOCKED", "No repository-local DEV-LIVE gate result is visible to this aggregator.") +]; + +const nextRoundRecommendations = [ + next("artifact/base image", "high", ["#33", "#35"], "Publish or attach real DEV artifact provenance for each frozen service ID: commitId, image, immutable tag, digest, buildSource, deployEnv, and healthTimestamp."), + next("edge/network", "high", ["#35", "#39"], "Attach read-only DEV route observations for public ingress, master edge proxy, frp, and D601 hwlab-dev/hwlab-router without deploying, restarting, or reading secrets."), + next("runtime/health", "high", ["#34", "#35", "#38", "#39"], "Attach current DEV health JSON for cloud API/web, gateway or simulator, box simulator, patch panel, agent manager, worker, skills, and CLI, all tied to HWLAB service IDs."), + next("M3/M4 live", "medium", ["#36", "#37", "#38"], "After DEV runtime smoke is green, run the smallest authorized DEV-only M3/M4 loop and attach trace, audit, evidence, operation, session, and cleanup IDs.") +]; + +function contract(milestone, source, category, supports, paths, commands, summary) { + return { + milestone, + source, + status: source === "SOURCE" ? "contract_available" : `${source.toLowerCase().replace("-", "_")}_contract_available`, + category, + supports: supports.map(issue), + paths, + commands, + summary + }; +} + +function parallel(issue, source, visibleResult) { + return { + issue: `pikasTech/HWLAB${issue}`, + issueRead: "blocked", + visibleResult, + source + }; +} + +function next(category, priority, owners, recommendation) { + return { category, priority, owners, recommendation }; +} + +async function readJSON(relativePath) { + const raw = await readFile(path.join(repoRoot, relativePath), "utf8"); + return JSON.parse(raw); +} + +async function pathExists(relativePath) { + try { + await access(path.join(repoRoot, relativePath), fsConstants.R_OK); + return true; + } catch { + return false; + } +} + +function gitCommit() { + try { + return execFileSync("git", ["rev-parse", "--short=12", "HEAD"], { + cwd: repoRoot, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"] + }).trim(); + } catch { + return "unknown"; + } +} + +async function assertStaticContracts() { + const checklist = await readJSON("docs/dev-acceptance-checklist.json"); + assert.equal(checklist.environment, ENVIRONMENT_DEV, "DEV checklist environment"); + assert.equal(checklist.endpoint, DEV_ENDPOINT, "DEV checklist endpoint"); + assert.deepEqual(checklist.artifactObservabilityFields, requiredArtifactFields); + + const catalog = await readJSON("deploy/artifact-catalog.dev.json"); + assert.equal(catalog.environment, ENVIRONMENT_DEV, "artifact catalog environment"); + assert.equal(catalog.endpoint, DEV_ENDPOINT, "artifact catalog endpoint"); + assert.equal(catalog.publish.ciPublished, false, "skeleton catalog must not claim CI publish"); + assert.equal(catalog.publish.registryVerified, false, "skeleton catalog must not claim registry verification"); + assert.deepEqual(catalog.services.map((service) => service.serviceId).sort(), [...SERVICE_IDS].sort()); + + const m2Fixture = await readJSON("fixtures/dev-deploy-smoke/dev-deploy-smoke.json"); + assert.equal(m2Fixture.environment, ENVIRONMENT_DEV, "M2 fixture environment"); + assert.equal(m2Fixture.endpoint, DEV_ENDPOINT, "M2 fixture endpoint"); + assert.equal(m2Fixture.defaultMode, "dry-run", "M2 fixture default mode"); + + const runtime = await readJSON("fixtures/mvp/runtime.json"); + assert.equal(runtime.endpoints.dev, DEV_ENDPOINT, "MVP runtime endpoint"); + + const m5Plan = await readJSON("fixtures/mvp/m5-e2e/dry-run-plan.json"); + assert.equal(m5Plan.mode, "dry-run", "M5 plan mode"); + assert.equal(m5Plan.environment, ENVIRONMENT_DEV, "M5 plan environment"); + assert.equal(m5Plan.endpoint, DEV_ENDPOINT, "M5 plan endpoint"); + assert.equal(m5Plan.safety.allowNetwork, false, "M5 dry-run network gate"); + assert.equal(m5Plan.safety.allowDeploy, false, "M5 dry-run deploy gate"); + assert.equal(m5Plan.safety.allowSecrets, false, "M5 dry-run secret gate"); +} + +async function collectContracts() { + return Promise.all( + contracts.map(async (entry) => { + const paths = await Promise.all(entry.paths.map(async (item) => ({ path: item, exists: await pathExists(item) }))); + const missing = paths.filter((item) => !item.exists).map((item) => item.path); + return { + ...entry, + status: missing.length === 0 ? entry.status : "blocked_missing_contract_file", + paths, + missing + }; + }) + ); +} + +async function collectDevGateReports() { + const entries = await readdir(path.join(repoRoot, "reports/dev-gate"), { withFileTypes: true }); + const reports = []; + for (const entry of entries) { + if (!entry.isFile() || !entry.name.endsWith(".json")) continue; + const reportPath = path.join("reports/dev-gate", entry.name); + const report = await readJSON(reportPath); + reports.push({ + path: reportPath, + issue: report.issue, + taskId: report.taskId, + acceptanceLevel: report.acceptanceLevel, + status: "contract_available", + blockers: Array.isArray(report.blockers) ? report.blockers.length : 0, + notes: report.notes ?? "" + }); + } + return reports.sort((a, b) => a.path.localeCompare(b.path)); +} + +function buildBlockers(collectedContracts) { + const blockers = [ + blocker("observability_blocker", "issue-read", "GitHub issue state is not readable in this runner; visible #33-#39 results are limited to repository artifacts and prompt context."), + blocker("network_blocker", "DEV-LIVE", "No repository-local artifact can be treated as live DEV evidence for ingress, edge, frp, D601 router, or downstream HWLAB services."), + blocker("observability_blocker", "artifact-provenance", "The checked artifact catalog is a contract skeleton; it explicitly does not claim real CI publish, registry verification, or deployment."), + blocker("runtime_blocker", "M3/M4 DEV loops", "M3 and M4 currently have local contract smoke evidence only; live DEV trace, audit, operation, evidence, and cleanup IDs are not attached.") + ]; + for (const item of collectedContracts) { + if (item.missing.length > 0) { + blockers.push(blocker("contract_blocker", item.milestone, `${item.milestone} is missing repository contract file(s): ${item.missing.join(", ")}.`)); + } + } + return blockers; +} + +function blocker(type, scope, summary) { + return { type, scope, status: "open", source: "BLOCKED", summary }; +} + +function groupBySource(collectedContracts, devLive, blockers) { + const grouped = Object.fromEntries(levels.map((level) => [level, []])); + for (const item of collectedContracts) { + grouped[item.source].push({ + milestone: item.milestone, + supports: item.supports, + status: item.status, + category: item.category, + summary: item.summary + }); + } + grouped["DEV-LIVE"].push(...devLive); + grouped.BLOCKED.push(...blockers); + return grouped; +} + +function validateOutput(output) { + assert.equal(output.environment, ENVIRONMENT_DEV, "output environment"); + assert.equal(output.endpoint, DEV_ENDPOINT, "output endpoint"); + assert.equal(output.devOnly, true, "output devOnly"); + assert.equal(output.prodDisabled, true, "output prodDisabled"); + assert.deepEqual(Object.keys(output.levels), levels, "output levels"); + assert.ok(output.blockers.length > 0, "aggregator must not report green without DEV-LIVE evidence"); + for (const item of output.blockers) { + assert.ok(blockerTypes.includes(item.type), `unknown blocker type ${item.type}`); + assert.equal(item.status, "open", "aggregator blockers remain open until live evidence is attached"); + } +} + +async function buildReport() { + await assertStaticContracts(); + const evidenceContracts = await collectContracts(); + const blockers = buildBlockers(evidenceContracts); + const devLive = [{ + milestone: "M2-M5", + issue: "pikasTech/HWLAB#39", + status: "not_available", + category: "DEV-LIVE", + summary: "No live DEV evidence was read or generated by this report-only aggregator." + }]; + const output = { + reportVersion: "v1", + taskId: "dev-evidence-blocker-aggregator", + issue: "pikasTech/HWLAB#41", + supports: allSupportedIssues, + generatedFromCommit: gitCommit(), + environment: ENVIRONMENT_DEV, + endpoint: DEV_ENDPOINT, + devOnly: true, + prodDisabled: true, + safety: { + reportOnly: true, + noLiveOperation: true, + noDeploy: true, + noProd: true, + noSecretRead: true, + noHeavyE2E: true, + noBrowserTest: true + }, + issueRead: { + status: "blocked", + reason: "GitHub issue data is not available from this runner; this report uses repository files and prompt context only." + }, + overall: { + status: "blocked", + green: false, + reason: "SOURCE, LOCAL, and DRY-RUN contracts are visible, but DEV-LIVE evidence and issue reads are blocked." + }, + evidenceContracts, + devGateReports: await collectDevGateReports(), + visibleParallelResults, + levels: groupBySource(evidenceContracts, devLive, blockers), + blockers, + nextRoundRecommendations, + validationCommands: [ + "node --check scripts/dev-evidence-blocker-aggregator.mjs", + "node scripts/dev-evidence-blocker-aggregator.mjs --check", + "node scripts/dev-evidence-blocker-aggregator.mjs --pretty" + ] + }; + validateOutput(output); + return output; +} + +try { + const output = await buildReport(); + if (process.argv.includes("--check")) { + process.stdout.write(JSON.stringify({ + ok: true, + taskId: output.taskId, + status: output.overall.status, + blockers: output.blockers.length, + sourceLevels: levels + }) + "\n"); + } else { + process.stdout.write(JSON.stringify(output, null, process.argv.includes("--pretty") ? 2 : 0) + "\n"); + } +} catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(JSON.stringify({ ok: false, error: message }) + "\n"); + process.exitCode = 1; +}