From caa9ed0e288c7919c9efb227452d529dadc7ce56 Mon Sep 17 00:00:00 2001 From: HWLAB Code Queue Date: Thu, 21 May 2026 15:01:08 +0000 Subject: [PATCH] docs: add dev acceptance matrix --- docs/dev-acceptance-checklist.json | 456 +++++++++++++++++++++++++++++ docs/dev-acceptance-matrix.md | 100 +++++++ 2 files changed, 556 insertions(+) create mode 100644 docs/dev-acceptance-checklist.json create mode 100644 docs/dev-acceptance-matrix.md diff --git a/docs/dev-acceptance-checklist.json b/docs/dev-acceptance-checklist.json new file mode 100644 index 00000000..87fe5c85 --- /dev/null +++ b/docs/dev-acceptance-checklist.json @@ -0,0 +1,456 @@ +{ + "$schema": "https://hwlab.pikastech.local/docs/dev-acceptance-checklist.schema.json", + "$id": "https://hwlab.pikastech.local/docs/dev-acceptance-checklist.json", + "matrixVersion": "v1", + "issue": "pikasTech/HWLAB#22", + "environment": "dev", + "endpoint": "http://74.48.78.17:6667", + "runtimeSubstitutePolicy": { + "allowedExternalRoles": [ + "scheduling", + "ci", + "cd" + ], + "forbiddenRuntimeSubstitutes": [ + "unidesk-backend", + "unidesk-provider-gateway", + "unidesk-microservice-proxy" + ] + }, + "artifactObservabilityFields": [ + "serviceId", + "commitId", + "image", + "tag", + "digest", + "buildSource", + "deployEnv", + "healthTimestamp" + ], + "artifactDigestPolicy": { + "requiredFor": [ + "container-image", + "digestable-build-output" + ], + "notApplicableValue": "not_applicable", + "reasonRequiredWhenNotApplicable": true + }, + "route": [ + { + "id": "dev-endpoint", + "label": "DEV endpoint", + "serviceId": "hwlab-edge-proxy", + "endpoint": "http://74.48.78.17:6667", + "role": "public-dev-ingress" + }, + { + "id": "master-edge-proxy", + "label": "master edge proxy", + "serviceId": "hwlab-edge-proxy", + "role": "edge-route" + }, + { + "id": "frp", + "label": "frp", + "serviceId": "hwlab-tunnel-client", + "role": "tunnel" + }, + { + "id": "d601-router", + "label": "D601 hwlab-dev/hwlab-router", + "serviceId": "hwlab-router", + "namespace": "hwlab-dev", + "role": "dev-router" + }, + { + "id": "cloud-api-web", + "label": "cloud-api/web", + "serviceIds": [ + "hwlab-cloud-api", + "hwlab-cloud-web" + ], + "role": "cloud-surface" + } + ], + "healthContracts": [ + { + "component": "DEV ingress", + "serviceId": "hwlab-edge-proxy", + "probe": { + "method": "GET", + "url": "http://74.48.78.17:6667/health" + }, + "requiredFields": [ + "serviceId", + "deployEnv", + "healthTimestamp" + ], + "success": "HTTP 2xx/3xx reaches the HWLAB DEV route and reports edge or downstream HWLAB identity.", + "failure": "Timeout, non-HWLAB response, PROD route, or missing artifact fields." + }, + { + "component": "master edge proxy", + "serviceId": "hwlab-edge-proxy", + "probe": { + "type": "route-observation", + "target": "http://74.48.78.17:6667" + }, + "requiredFields": [ + "serviceId", + "commitId", + "image", + "tag", + "digest", + "buildSource", + "deployEnv", + "healthTimestamp" + ], + "success": "DEV route forwards to frp and identifies its artifact.", + "failure": "No DEV route, wrong port, or missing route observability." + }, + { + "component": "frp", + "serviceId": "hwlab-tunnel-client", + "probe": { + "type": "tunnel-status", + "target": "D601 hwlab-dev/hwlab-router" + }, + "requiredFields": [ + "serviceId", + "commitId", + "image", + "tag", + "digest", + "buildSource", + "deployEnv", + "healthTimestamp" + ], + "success": "Tunnel is established from master edge to D601 router for DEV.", + "failure": "Tunnel down, wrong target, or ambiguous service identity." + }, + { + "component": "D601 router", + "serviceId": "hwlab-router", + "namespace": "hwlab-dev", + "probe": { + "type": "router-health", + "target": "hwlab-dev/hwlab-router" + }, + "requiredFields": [ + "serviceId", + "commitId", + "image", + "tag", + "digest", + "buildSource", + "deployEnv", + "healthTimestamp" + ], + "success": "Router is live in dev and forwards only to HWLAB DEV services.", + "failure": "Namespace mismatch, missing route, or stale health timestamp." + }, + { + "component": "Cloud API", + "serviceId": "hwlab-cloud-api", + "probe": { + "methods": [ + "GET /health", + "GET /live", + "POST /rpc" + ] + }, + "requiredFields": [ + "serviceId", + "deployEnv", + "healthTimestamp" + ], + "success": "Health is valid JSON with HWLAB cloud API identity and dev environment.", + "failure": "HTTP error, wrong service ID, wrong environment, or invalid JSON-RPC envelope." + }, + { + "component": "Cloud Web", + "serviceId": "hwlab-cloud-web", + "probe": { + "commands": [ + "npm run web:check", + "npm run web:build" + ] + }, + "requiredFields": [ + "serviceId", + "commitId", + "buildSource", + "deployEnv", + "healthTimestamp" + ], + "success": "Static assets exist, build succeeds, and runtime uses the fixed DEV endpoint.", + "failure": "Endpoint drift, direct hardware control path, or missing asset." + }, + { + "component": "Agent manager", + "serviceId": "hwlab-agent-mgr", + "probe": { + "type": "agent-scheduler-health" + }, + "requiredFields": [ + "serviceId", + "commitId", + "image", + "tag", + "digest", + "buildSource", + "deployEnv", + "healthTimestamp" + ], + "success": "Scheduler is live for dev and does not leave worker sessions open after smoke.", + "failure": "Scheduler unavailable, wrong environment, or unbounded worker session." + }, + { + "component": "Agent worker", + "serviceId": "hwlab-agent-worker", + "probe": { + "type": "scoped-worker-health" + }, + "requiredFields": [ + "serviceId", + "commitId", + "image", + "tag", + "digest", + "buildSource", + "deployEnv", + "healthTimestamp" + ], + "success": "Worker session is scoped to the DEV project and emits trace/audit identifiers.", + "failure": "Missing session identity, unsafe mutation, or cleanup failure." + }, + { + "component": "Agent skills", + "serviceId": "hwlab-agent-skills", + "probe": { + "type": "skill-bundle-version" + }, + "requiredFields": [ + "serviceId", + "commitId", + "buildSource", + "deployEnv", + "healthTimestamp" + ], + "success": "Skill artifact is traceable to commit/build source and compatible with worker.", + "failure": "Unversioned skill bundle or missing build source." + }, + { + "component": "Gateway", + "serviceId": "hwlab-gateway", + "probe": { + "methods": [ + "GET /health/live", + "GET /status" + ] + }, + "requiredFields": [ + "serviceId", + "deployEnv", + "healthTimestamp" + ], + "success": "Gateway boundary is live for dev and does not bypass patch-panel constraints.", + "failure": "Hardware boundary unavailable, non-dev gateway, or direct box mutation." + }, + { + "component": "Gateway simulator", + "serviceId": "hwlab-gateway-simu", + "probe": { + "methods": [ + "GET /health/live", + "GET /status", + "GET /boxes" + ] + }, + "requiredFields": [ + "serviceId", + "deployEnv", + "healthTimestamp" + ], + "success": "Simulator is live, lists expected box resources, and identifies the DEV project.", + "failure": "Simulator down, box list missing, or stale timestamp." + }, + { + "component": "Box simulator", + "serviceId": "hwlab-box-simu", + "probe": { + "methods": [ + "GET /health/live", + "GET /status" + ] + }, + "requiredFields": [ + "serviceId", + "deployEnv", + "healthTimestamp" + ], + "success": "Simulator is live and reports resources, ports, and patch-panel-only propagation.", + "failure": "Cross-device propagation outside patch panel or missing resource state." + }, + { + "component": "Patch panel", + "serviceId": "hwlab-patch-panel", + "probe": { + "methods": [ + "GET /health/live", + "GET /status", + "GET /wiring" + ] + }, + "requiredFields": [ + "serviceId", + "deployEnv", + "healthTimestamp" + ], + "success": "Wiring config is parseable and patch-panel state owns routing decisions.", + "failure": "Invalid topology, direct bypass path, or stale patch state." + }, + { + "component": "CLI", + "serviceId": "hwlab-cli", + "probe": { + "commands": [ + "npm run cli:health", + "npm run cli:dry-run" + ] + }, + "requiredFields": [ + "serviceId", + "commitId", + "buildSource", + "deployEnv", + "healthTimestamp" + ], + "success": "CLI uses the fixed DEV endpoint and dry-run states no DEV/PROD changes were made.", + "failure": "Wrong endpoint, missing dry-run guard, or real mutation attempted." + } + ], + "smokeSteps": [ + { + "id": "static-contract-parse", + "order": 1, + "probe": "Parse docs/dev-acceptance-checklist.json and confirm docs/dev-acceptance-matrix.md exists.", + "successCriteria": "Checklist JSON parses and markdown file is present.", + "failureCriteria": "Missing file or invalid JSON.", + "blockerClass": "contract_blocker" + }, + { + "id": "endpoint-freeze", + "order": 2, + "probe": "Compare documented and fixture DEV endpoints.", + "successCriteria": "All DEV endpoints equal http://74.48.78.17:6667.", + "failureCriteria": "Any alternate DEV endpoint or PROD target.", + "blockerClass": "environment_blocker" + }, + { + "id": "dev-ingress-health", + "order": 3, + "probe": "Observe GET http://74.48.78.17:6667/health or recorded equivalent.", + "successCriteria": "Request reaches HWLAB DEV route and returns HWLAB identity or accepted downstream health.", + "failureCriteria": "Timeout, non-HWLAB target, or wrong port.", + "blockerClass": "network_blocker" + }, + { + "id": "edge-route", + "order": 4, + "probe": "Observe master edge proxy route table for DEV.", + "successCriteria": "Edge route maps DEV endpoint to frp and records artifact identity.", + "failureCriteria": "Missing route, stale route, or missing artifact identity.", + "blockerClass": "network_blocker" + }, + { + "id": "frp-tunnel", + "order": 5, + "probe": "Observe frp tunnel status.", + "successCriteria": "Tunnel links master edge to D601 router for DEV.", + "failureCriteria": "Tunnel down or target mismatch.", + "blockerClass": "network_blocker" + }, + { + "id": "d601-router", + "order": 6, + "probe": "Observe hwlab-dev/hwlab-router health/status.", + "successCriteria": "Router is live and forwards only to HWLAB DEV services.", + "failureCriteria": "Namespace mismatch or route bypass.", + "blockerClass": "runtime_blocker" + }, + { + "id": "cloud-surface", + "order": 7, + "probe": "Check cloud API health and cloud web endpoint configuration.", + "successCriteria": "Cloud API health is valid and web assets point at the fixed DEV endpoint.", + "failureCriteria": "Bad health, endpoint drift, or direct hardware control from web.", + "blockerClass": "runtime_blocker" + }, + { + "id": "gateway-sim-patch-panel", + "order": 8, + "probe": "Check gateway, gateway simulator, box simulator, and patch-panel health/status.", + "successCriteria": "Health/status JSON is parseable and topology keeps routing under patch-panel ownership.", + "failureCriteria": "Missing health, invalid topology, or bypass path.", + "blockerClass": "runtime_blocker" + }, + { + "id": "agent-runtime", + "order": 9, + "probe": "Check agent manager, worker, and skills contracts.", + "successCriteria": "Agent artifacts are traceable; dry-run/scoped smoke emits session, trace, audit, and cleanup evidence.", + "failureCriteria": "Missing traceability, unsafe mutation, or cleanup leak.", + "blockerClass": "agent_blocker" + }, + { + "id": "artifact-observability", + "order": 10, + "probe": "Verify required artifact observability fields for each accepted artifact.", + "successCriteria": "Every accepted artifact has service ID, commit, image/tag/digest or reason, build source, env, and health timestamp.", + "failureCriteria": "Missing required observability field.", + "blockerClass": "observability_blocker" + } + ], + "blockerClasses": [ + { + "id": "contract_blocker", + "description": "Static contract, schema, checklist, or documentation cannot be parsed or contradicts frozen names." + }, + { + "id": "environment_blocker", + "description": "DEV endpoint, environment, namespace, or PROD boundary is wrong." + }, + { + "id": "network_blocker", + "description": "Master edge, frp, D601 route, or public DEV ingress cannot be observed or routes incorrectly." + }, + { + "id": "runtime_blocker", + "description": "HWLAB cloud, router, gateway, simulator, box simulator, or patch-panel runtime contract fails." + }, + { + "id": "agent_blocker", + "description": "Agent manager, worker, skills, trace, audit, or cleanup contract fails." + }, + { + "id": "observability_blocker", + "description": "Artifact identity, commit, image/tag/digest, build source, deploy env, or health timestamp is missing." + }, + { + "id": "safety_blocker", + "description": "A prohibited action was attempted: real deployment, PROD target, heavyweight e2e, secret read, force push, or UniDesk runtime substitution." + } + ], + "prohibitedActions": [ + "real-dev-deploy", + "prod-deploy", + "prod-smoke", + "heavyweight-e2e", + "secret-or-token-read", + "force-push", + "unidesk-runtime-substitution" + ], + "passRule": "All smoke steps must be successful or explicitly not applicable with a non-production reason, and every accepted artifact must satisfy the observability contract.", + "failRule": "Fail on any blocker class, wrong endpoint, wrong environment, missing HWLAB service identity, missing artifact observability, invalid JSON, or prohibited action." +} diff --git a/docs/dev-acceptance-matrix.md b/docs/dev-acceptance-matrix.md new file mode 100644 index 00000000..294f487b --- /dev/null +++ b/docs/dev-acceptance-matrix.md @@ -0,0 +1,100 @@ +# HWLAB DEV Acceptance Matrix + +This matrix is the DEV acceptance contract for the MVP runtime. It is scoped to +static review and lightweight smoke checks only. It does not authorize a real +DEV or PROD deployment. + +## Fixed Boundary + +- DEV endpoint: `http://74.48.78.17:6667` +- Deploy environment: `dev` +- PROD endpoint `:6666` is reserved and must not be targeted by this matrix. +- Runtime route: client, master `hwlab-edge-proxy`, `frp`, D601 + `hwlab-dev/hwlab-router`, then HWLAB cloud/runtime services. +- UniDesk backend, provider-gateway, and microservice proxy are not accepted as + substitutes for HWLAB runtime services in this first version. They may remain + external scheduling, CI, or CD support only. + +## Required Artifact Observability + +Every deployable HWLAB artifact that participates in DEV acceptance must expose +or be joined with these fields: + +| Field | Requirement | +| --- | --- | +| `serviceId` | One of the frozen HWLAB service IDs. | +| `commitId` | Git commit that produced the artifact, short or full SHA. | +| `image` | Image repository/name when containerized, or package artifact name for non-container artifacts. | +| `tag` | Immutable or traceable tag, preferably including the commit. | +| `digest` | Content digest when the artifact is an image or build output that supports digests. | +| `buildSource` | Source repository, branch/ref, and build workflow or local command. | +| `deployEnv` | Must be `dev` for this matrix. | +| `healthTimestamp` | ISO-8601 timestamp from the health observation or smoke collector. | + +Missing `digest` is allowed only for non-container client assets or local dry-run +artifacts, and must be recorded as `not_applicable` with a reason. + +## Health Contract + +All health responses must be JSON, include `serviceId`, identify `dev`, and +provide a current health timestamp. Existing skeleton services may expose +`observedAt`; the acceptance collector maps it to `healthTimestamp`. + +| Component | Service ID | Check | Success | Failure | +| --- | --- | --- | --- | --- | +| DEV ingress | `hwlab-edge-proxy` | `GET http://74.48.78.17:6667/health` or routed service health | HTTP 2xx/3xx reaches HWLAB DEV path and reports edge identity or downstream HWLAB identity | Timeout, non-HWLAB response, PROD route, missing artifact fields | +| master edge proxy | `hwlab-edge-proxy` | Edge proxy health and route table observation | Route for `http://74.48.78.17:6667` forwards to `frp` and identifies commit/image | No DEV route, wrong port, missing route observability | +| frp tunnel | `hwlab-tunnel-client` | Tunnel session/status observation | Tunnel for DEV route is established to D601 router and identifies commit/image | Tunnel down, wrong target, ambiguous service identity | +| D601 router | `hwlab-router` | `hwlab-dev/hwlab-router` health/status | Router is live, environment is `dev`, route target is cloud API/web or runtime service | Namespace mismatch, route missing, stale health timestamp | +| Cloud API | `hwlab-cloud-api` | `GET /health`, `GET /live`, optional JSON-RPC `/rpc` probe | Returns service ID, `dev`, healthy/live status, and accepts a harmless protocol probe | HTTP error, unknown service ID, wrong environment, invalid JSON-RPC envelope | +| Cloud Web | `hwlab-cloud-web` | Static asset check and configured DEV endpoint | Web bundle references `http://74.48.78.17:6667` and does not call hardware directly | Endpoint drift, direct hardware control path, missing asset | +| Agent manager | `hwlab-agent-mgr` | Agent scheduling health/status | Reports live manager, commit/image, and DEV environment; no worker leak after smoke | Scheduler unavailable, wrong environment, unbounded worker session | +| Agent worker | `hwlab-agent-worker` | Worker health/status during dry-run or scoped smoke | Worker session is scoped to DEV project and emits trace/audit identifiers | Missing session identity, unsafe mutation, cleanup failure | +| Agent skills | `hwlab-agent-skills` | Skill bundle version/status | Skill artifact is traceable to commit/build source and compatible with worker | Unversioned skill bundle, missing build source | +| Gateway | `hwlab-gateway` | `GET /health/live` and `GET /status` when present | Reports DEV gateway boundary and does not bypass patch-panel constraints | Hardware boundary unavailable, non-DEV gateway, direct box mutation | +| Gateway simulator | `hwlab-gateway-simu` | `GET /health/live`, `GET /status`, `GET /boxes` | Simulator is live, lists expected box resources, and identifies DEV project | Simulator down, box list missing, stale timestamp | +| Box simulator | `hwlab-box-simu` | `GET /health/live`, `GET /status` | Simulator is live and reports resources, ports, and patch-panel-only propagation | Cross-device propagation outside patch panel, missing resource state | +| Patch panel | `hwlab-patch-panel` | `GET /health/live`, `GET /status`, `GET /wiring` | Wiring config is parseable and patch-panel state owns routing decisions | Invalid topology, direct bypass path, stale patch state | +| CLI | `hwlab-cli` | `npm run cli:health` and dry-run command | CLI uses DEV endpoint and dry-run states no DEV/PROD changes were made | Wrong endpoint, missing dry-run guard, real mutation attempted | + +## DEV Smoke Matrix + +The smoke sequence must run in order and stop on the first critical blocker. +Network checks may be replaced by recorded observations when the runner cannot +reach the DEV host, but replacement evidence must include the artifact fields +listed above. + +| Step | Probe | Success Criteria | Failure Criteria | Blocker Class | +| --- | --- | --- | --- | --- | +| 1 | Confirm repository contract files and JSON checklist parse. | `docs/dev-acceptance-matrix.md` exists and `docs/dev-acceptance-checklist.json` parses. | Missing file or invalid JSON. | `contract_blocker` | +| 2 | Verify frozen DEV endpoint. | Endpoint is exactly `http://74.48.78.17:6667` in docs/checklist/runtime fixtures. | Any alternate DEV endpoint or PROD target. | `environment_blocker` | +| 3 | Observe DEV ingress health. | Request reaches HWLAB DEV route and returns HWLAB identity or accepted downstream health. | Timeout, non-HWLAB target, wrong port. | `network_blocker` | +| 4 | Observe master edge proxy route. | Edge route maps DEV endpoint to `frp` and records artifact identity. | Missing route, stale route, missing artifact identity. | `network_blocker` | +| 5 | Observe `frp` tunnel. | Tunnel links master edge to D601 router for DEV. | Tunnel down or target mismatch. | `network_blocker` | +| 6 | Observe D601 `hwlab-dev/hwlab-router`. | Router is live and forwards only to HWLAB DEV services. | Namespace mismatch or route bypass. | `runtime_blocker` | +| 7 | Check cloud API/web surface. | Cloud API health is valid; web assets point at fixed DEV endpoint. | Bad health, endpoint drift, direct hardware control from web. | `runtime_blocker` | +| 8 | Check gateway, simulator, box simulator, and patch-panel contracts. | Health/status JSON is parseable and topology keeps routing under patch-panel ownership. | Missing health, invalid topology, bypass path. | `runtime_blocker` | +| 9 | Check agent manager, worker, and skills contracts. | Agent artifacts are traceable; dry-run/scoped smoke emits session, trace, audit, and cleanup evidence. | Missing traceability, unsafe mutation, cleanup leak. | `agent_blocker` | +| 10 | Verify evidence and artifact observability. | Every accepted artifact has service ID, commit, image/tag/digest or reason, build source, env, and health timestamp. | Missing required observability field. | `observability_blocker` | + +## Pass And Fail Rules + +The matrix passes only when every smoke step is either observed as successful or +explicitly marked not applicable with a non-production reason. A step is failed +when the observed result contradicts a frozen contract, cannot be parsed, or +cannot be tied to a traceable HWLAB artifact. + +Critical blockers are `contract_blocker`, `environment_blocker`, +`network_blocker`, `runtime_blocker`, `agent_blocker`, +`observability_blocker`, and `safety_blocker`. A `safety_blocker` must be raised +for any attempted PROD deployment, real deployment from this matrix, secret +read, heavyweight e2e run, or UniDesk runtime substitution. + +## First Version Exclusions + +- No PROD deployment or PROD smoke. +- No real DEV deployment from this document. +- No heavyweight e2e or destructive hardware action. +- No secret or token reads. +- No replacement of HWLAB runtime services by UniDesk backend, + provider-gateway, or microservice proxy.