diff --git a/docs/dev-acceptance-checklist.json b/docs/dev-acceptance-checklist.json index f262b100..5745594e 100644 --- a/docs/dev-acceptance-checklist.json +++ b/docs/dev-acceptance-checklist.json @@ -36,6 +36,45 @@ "unidesk-microservice-proxy" ] }, + "manualAcceptance": { + "referenceDoc": "docs/reference/m3-loop-rollout-runbook.md", + "m3MvpPassCondition": "res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 with two distinct box simulators, two distinct gateway simulators, one patch-panel participant, and linked operation, trace, audit, and evidence IDs.", + "mustHaveEvidence": [ + "two distinct box-simu identities", + "two distinct gateway-simu identities", + "patch-panel ownership of the route", + "operationId", + "traceId", + "auditId", + "evidenceId" + ], + "supportOnlySignals": [ + "SOURCE", + "LOCAL", + "DRY-RUN", + "fixture-only", + "edge-only", + "Cloud Web polish", + "generic console work", + "artifact report", + "desired-state plan" + ], + "forbiddenM3PassSubstitutes": [ + "box loopback", + "front-end direct state edit", + "UniDesk runtime substitution", + "UniDesk proxy substitution" + ], + "failureClasses": [ + "contract_blocker", + "environment_blocker", + "network_blocker", + "runtime_blocker", + "agent_blocker", + "observability_blocker", + "safety_blocker" + ] + }, "m3EvidenceClassification": { "p0Scope": "M3 virtual hardware trusted loop", "requiredCardinality": { diff --git a/docs/dev-acceptance-matrix.md b/docs/dev-acceptance-matrix.md index 762e2880..ddeba87b 100644 --- a/docs/dev-acceptance-matrix.md +++ b/docs/dev-acceptance-matrix.md @@ -26,19 +26,21 @@ Current P0 DEV acceptance remains the M3 virtual hardware trusted loop: two `hwlab-box-simu` instances, two `hwlab-gateway-simu` instances, and one `hwlab-patch-panel`. -`M3 live` can be claimed only when a DEV observation proves a real -`DO1 -> hwlab-patch-panel -> DI1` path and records the operation, trace, audit, -and evidence identifiers. The patch panel must own the route decision; direct -box-to-box propagation or fixture output is not M3 live evidence. +`M3 live` can be claimed only when a DEV observation proves the live +`res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1` path and records +the operation, trace, audit, and evidence identifiers. The patch panel must +own the route decision; direct box-to-box propagation or fixture output is not +M3 live evidence. -`M3 support` includes this endpoint freeze, read-only public edge curl evidence, -source contracts, static manifest cardinality, local smoke output, and dry-run -fixtures. These items support release or pre-release decisions, but they are not -P0 live acceptance. +`M3 support` includes this endpoint freeze, read-only public edge curl +evidence, source contracts, static manifest cardinality, local smoke output, +and dry-run fixtures. These items support release or pre-release decisions, but +they are not P0 live acceptance. `Non-P0` includes SOURCE, LOCAL, DRY-RUN, fixture-only, diagnostic, and -edge-only observations that do not prove the full `DO1 -> patch-panel -> DI1` -link. These observations must not be written as `DEV-LIVE`. +edge-only observations that do not prove the full +`res_boxsimu_1 -> hwlab-patch-panel -> res_boxsimu_2` link. These observations +must not be written as `DEV-LIVE`. ## Required Artifact Observability @@ -109,6 +111,11 @@ explicitly marked not applicable with a non-production reason. A step is failed when the observed result contradicts a frozen contract, cannot be parsed, or cannot be tied to a traceable HWLAB artifact. +M3 live cannot be inferred from SOURCE, LOCAL, DRY-RUN, edge-only, or +front-end-only evidence. If the evidence does not show the operation, trace, +audit, and evidence IDs on the `res_boxsimu_1 -> hwlab-patch-panel -> +res_boxsimu_2` chain, it is not a pass. + Critical blockers are `contract_blocker`, `environment_blocker`, `network_blocker`, `runtime_blocker`, `agent_blocker`, `observability_blocker`, and `safety_blocker`. A `safety_blocker` must be raised diff --git a/docs/operator-runbook.md b/docs/operator-runbook.md index 86c05261..33a9df07 100644 --- a/docs/operator-runbook.md +++ b/docs/operator-runbook.md @@ -12,14 +12,37 @@ heavyweight e2e run by itself. - Repository: `pikasTech/HWLAB`. - MVP environment: `dev`. -- DEV endpoint: `http://74.48.78.17:16667`. -- Legacy endpoint `:6666` is deprecated and is not an MVP acceptance target; the browser entry is `:16666`. +- DEV frontend endpoint: `http://74.48.78.17:16666`. +- DEV API/edge endpoint: `http://74.48.78.17:16667`. +- Legacy public endpoints `:6666` and `:6667` are deprecated and are not MVP acceptance targets. +- Internal k3s service ports may still use `6667`; internal `6667` is not a public acceptance endpoint. - Runtime route: client, master `hwlab-edge-proxy`, `frp`, D601 `hwlab-dev/hwlab-router`, then HWLAB cloud/runtime services. - HWLAB runtime services must keep the frozen service IDs in `README.md`. - UniDesk services may support scheduling, CI, or CD only. They are not accepted as substitutes for HWLAB runtime services. +## M3 MVP Acceptance + +M3 MVP PASS is the smallest real DEV evidence chain that proves the hardware +trusted loop. It requires: + +- two box simulators with different observable identities; +- two gateway simulators with different observable identities; +- one active patch-panel participant; +- the traced `res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1` + operation with trace, audit, operation, and evidence IDs; +- no box loopback substitution, no front-end direct state edit, and no + UniDesk runtime or proxy substitution. + +Cloud Web, generic console work, artifact report generation, and desired-state +planning are support surfaces only. They may help operators, but they do not +count as M3 PASS on their own and cannot replace the loop evidence above. + +The detailed step order and misclassification bans live in +`docs/reference/m3-loop-rollout-runbook.md`; this runbook and the acceptance +matrix mirror that contract. + ## Runner Issue Visibility Handoff GitHub issue and PR threads are auxiliary context only. The runner prompt must be self-contained and must include the full task objective, background, constraints, acceptance, and validation commands. @@ -43,7 +66,9 @@ operator, script, or agent attempts one of these actions: - Use UniDesk backend, provider-gateway, or microservice proxy as the HWLAB runtime instead of `hwlab-cloud-api`, `hwlab-router`, `hwlab-gateway`, `hwlab-patch-panel`, agent services, and the other frozen HWLAB services. -- Promote a source-only, local smoke, or dry-run result as real DEV evidence. +- Promote SOURCE, LOCAL, DRY-RUN, or edge-only result as real DEV evidence. +- Treat Cloud Web polish, generic console work, artifact reports, or desired + state plans as M3 PASS evidence. ## Evidence Log @@ -113,7 +138,7 @@ reason. | M0 contract audit | Source-only | `docs/m0-contract-audit.md`, protocol docs, schemas, deploy schema, and M0 examples agree on service IDs, JSON-RPC envelope, errors, audit fields, topology, and DEV-only deploy shape. | | M1 local contract smoke | Local smoke | `npm run m1:smoke` or `node scripts/m1-contract-smoke.mjs` passes on localhost and confirms the local cloud-api runtime, simulator contracts, patch-panel routing, CLI dry-run boundary, and no DEV/PROD changes. | | M2 DEV deploy/runtime smoke | Dry-run, then authorized DEV runtime smoke | Source/manifests and `docs/dev-acceptance-matrix.md` identify the DEV route through edge, `frp`, and D601 router. Real DEV observation is read-only and must not be performed without commander authorization. | -| M3 hardware trusted loop | Local smoke, then MVP e2e | `node scripts/m3-hardware-loop-smoke.mjs` proves the local `box-simu-1 DO1 -> box-simu-2 DI1` patch-panel contract. Real DEV acceptance must preserve patch-panel ownership and produce trace, audit, and evidence records. | +| M3 hardware trusted loop | Local smoke, then MVP e2e | `node scripts/m3-hardware-loop-smoke.mjs` proves the local contract, but M3 MVP PASS in DEV requires the live `res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1` evidence chain with distinct box/gateway identities and operation, trace, audit, and evidence IDs. | | M4 agent automation loop | Source-only or dry-run, then MVP e2e | Agent manager, worker, skills, session lifecycle, scoped workspace, trace events, cleanup, and evidence chain are observable in fixtures or dry-run before any DEV runtime action. | | M5 MVP e2e dry-run and acceptance | Dry-run, then MVP e2e | `npm run cli:dry-run` or equivalent names the MVP route, closed loops, evidence, and cleanup without mutation. Real acceptance is DEV-only, bounded, and requires prior DEV runtime smoke pass. | @@ -139,6 +164,9 @@ gateway sessions, two box resources, the patch-panel status, agent and worker session IDs, evidence records, and a statement that no DEV/PROD changes were made. +The dry-run shape is support evidence only. It cannot be promoted to M3 PASS +unless the real DEV loop above is present and traceable. + The live DEV entry point is intentionally separate: ```sh diff --git a/docs/reference/m3-loop-rollout-runbook.md b/docs/reference/m3-loop-rollout-runbook.md index adb59f3c..7ce91ed4 100644 --- a/docs/reference/m3-loop-rollout-runbook.md +++ b/docs/reference/m3-loop-rollout-runbook.md @@ -1,85 +1,72 @@ # M3 Loop Rollout Runbook -This reference is support / diagnostics / contract for the HWLAB P0 M3 virtual -hardware trusted loop only. It is not P0 live evidence and it must not relabel -SOURCE / LOCAL / DRY-RUN / fixture output as `DEV-LIVE`. +This reference is the human acceptance contract for the M3 virtual hardware +trusted loop. It is the authoritative place for the manual step order, evidence +requirements, failure classes, and misclassification bans used by commanders +and operators. -## M3 Loop Boundary +It does not authorize a real deployment, PROD action, secret read, service +restart, or heavyweight e2e run. -- P0 scope: `2 x hwlab-box-simu`, `2 x hwlab-gateway-simu`, `1 x hwlab-patch-panel`. -- Accepted chain for live M3 proof: `DO1 -> patch-panel -> DI1`. -- Current public endpoints: browser/frontend `http://74.48.78.17:16666/` and - API/edge/live `http://74.48.78.17:16667/`. -- Legacy public ports `:6666` and `:6667` are historical / deprecated only. -- `pikasTech/HWLAB#63` remains a known CD blocker until suspended Job immutable - apply is fixed. +## M3 MVP Pass Condition -## Source Of Truth +M3 MVP PASS requires the live DEV chain: -`deploy/deploy.json` is the deploy intent source. `deploy/artifact-catalog.dev.json` -is the artifact identity source. No shell scrollback, issue comment, or report -fixture is authoritative over those two files. - -| Fact | Read From | Notes | -| --- | --- | --- | -| source commit | `deploy/deploy.json.commitId` and `deploy/artifact-catalog.dev.json.commitId` | They must match. | -| image tag | `deploy/deploy.json.services[].image` and `deploy/artifact-catalog.dev.json.services[].imageTag` | The tag must stay tied to the source commit. | -| digest | `deploy/artifact-catalog.dev.json.services[].digest` | Real digests come only from publish evidence; otherwise keep `not_published`. | -| public endpoint | `deploy/deploy.json.publicEndpoints.*.url` and `deploy/deploy.json.endpoint` | Current endpoints are `16666/16667`; do not backslide to legacy ports. | -| frp | `deploy/deploy.json.frp.proxies[]` | Derive proxy name, local service, local port, and remote port from the manifest. | -| k3s service | `deploy/deploy.json.k3s.serviceMappings[]` | Derive service name, namespace, port, and targetPort from the manifest. | -| health path | `deploy/deploy.json.health.path` and `deploy/deploy.json.services[].healthPath` | Must stay on `/health/live`. | - -## Rollout Support - -Use this order for M3 loop support work: - -1. Render the manifest contract and compare it with the artifact catalog. -2. Verify the M3 inventory cardinality for the two boxes, two gateways, and - one patch-panel. -3. Check that the current public endpoints stay on `16666/16667`. -4. Confirm the health path is `/health/live`. -5. Record any manual hotfix facts back into the issue, PR, and runbook before - closing the task. - -Recommended checks: - -```sh -node scripts/deploy-contract-plan.mjs --pretty -node scripts/deploy-contract-plan.mjs --check -node scripts/validate-dev-m3-cardinality.mjs -npm run docs:validate:m3-rollout +```text +res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 ``` -These checks are support only. They do not apply to DEV, do not touch PROD, do -not restart services, and do not prove live M3 acceptance. +The evidence must show all of the following: -## Rollback Support +- two `hwlab-box-simu` identities that are visibly distinct; +- two `hwlab-gateway-simu` identities that are visibly distinct; +- one `hwlab-patch-panel` participant that owns the route decision; +- one operation ID, one trace ID, one audit ID, and one evidence ID that all + resolve to the same loop; +- no box loopback substitute, no front-end direct state edit, and no UniDesk + runtime or proxy substitute. -If a manual hotfix or rollback is required, record the following before the -task closes: +Cloud Web polish, generic console work, artifact reports, and desired-state +planning are support surfaces only. They can help explain the system, but they +do not count as M3 PASS evidence. -- source commit and branch or PR; -- image reference, image tag, and digest; -- public endpoint and whether it was read-only or routed; -- FRP proxy name and ownership; -- k3s service mapping and ownership; -- health path used for verification; -- any field that diverged from `deploy/deploy.json` or the artifact catalog. +## Manual Acceptance Steps -Rollback support must keep the live chain honest. If the chain was not actually -`DO1 -> patch-panel -> DI1`, do not claim live M3. Only write support, -diagnostics, or contract evidence. +| Step | What to observe | Must have evidence | Failure class | Must not misread | +| --- | --- | --- | --- | --- | +| 1 | Confirm the public DEV boundary and frozen route. | `http://74.48.78.17:16666` for frontend, `http://74.48.78.17:16667` for API/edge, internal `6667` only as a k3s port, and the checked-in docs or checklist that freeze those values. | `environment_blocker` | Do not treat legacy `:6666` or public `:6667` as acceptance targets. | +| 2 | Confirm two box simulators and two gateway simulators are present and distinct. | Two observable `hwlab-box-simu` identities, two observable `hwlab-gateway-simu` identities, their resource/session IDs, and the shared DEV project. | `runtime_blocker` | Do not infer cardinality from one box, one gateway, or fixture-only output. | +| 3 | Confirm one patch panel owns the route. | `hwlab-patch-panel` status, wiring, and the active connection from `res_boxsimu_1:DO1` to `res_boxsimu_2:DI1`. | `runtime_blocker` | Do not accept box loopback, direct box mutation, or a UI state edit as routing proof. | +| 4 | Confirm the live loop operation. | Operation ID, trace ID, audit ID, and evidence ID tied to `res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1`. | `runtime_blocker` | Do not promote SOURCE, LOCAL, DRY-RUN, or edge-only evidence to PASS. | +| 5 | Confirm cleanup and containment. | Logs or records showing no leaked worker, gateway, patch-panel, project, or hardware state after the run. | `agent_blocker` | Do not leave cleanup implicit. | -## Prohibited Actions +## Failure Classes -- Do not treat this runbook as a general console or PROD flow. -- Do not deploy or smoke PROD. -- Do not read, print, or infer secrets, tokens, kubeconfig, or tunnel - credentials. -- Do not restart Code Queue, `backend-core`, or any runtime service from the - runner. -- Do not run heavyweight e2e or destructive hardware tests. -- Do not bypass `pikasTech/HWLAB#63` and then claim CD is green. -- Do not promote SOURCE / LOCAL / DRY-RUN / fixture output to `DEV-LIVE`. +- `contract_blocker`: the doc, checklist, or contract files do not parse or + disagree on frozen names, endpoints, or IDs. +- `environment_blocker`: the public DEV boundary is wrong or confused with a + legacy endpoint. +- `network_blocker`: the route to the DEV edge, frp, or router cannot be + observed. +- `runtime_blocker`: the two box simulators, two gateway simulators, patch + panel, or wiring evidence is missing or contradictory. +- `agent_blocker`: trace, audit, evidence, session, or cleanup records are + missing or leaked. +- `observability_blocker`: artifact identity, commit, image/tag/digest, + build source, environment, or health timestamp is missing. +- `safety_blocker`: a prohibited action was attempted. +## Misclassification Bans + +- SOURCE, LOCAL, DRY-RUN, fixture-only, and edge-only evidence are support + signals only. +- Cloud Web polish does not prove M3. +- Generic console work does not prove M3. +- Artifact reports do not prove M3. +- Desired-state plans do not prove M3. +- UniDesk runtime or proxy substitution does not prove M3. + +## Validation + +This file is validated indirectly by the operator runbook and DEV acceptance +matrix, plus the checklist JSON shape.