test: add workbench p1 reliability inventory
This commit is contained in:
lyon
@@ -9,6 +9,7 @@
|
||||
"check:plan": "node scripts/check-runner.mjs --profile check --list",
|
||||
"check:cloud-api": "node scripts/check-runner.mjs --profile check --group cloud-api",
|
||||
"check:cloud-web": "node scripts/check-runner.mjs --profile check --group cloud-web",
|
||||
"workbench:p1:inventory": "node scripts/workbench-long-reliability-p1-inventory.mjs",
|
||||
"dev-runtime-base:build": "node scripts/dev-runtime-base-image.mjs",
|
||||
"cloud-api:smoke": "node scripts/run-bun.mjs scripts/cloud-api-runtime-smoke.mjs",
|
||||
"l2:smoke": "node scripts/l2-runtime-contract-smoke.mjs",
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env node
|
||||
// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p0-durable-facts-model.
|
||||
// Responsibility: P1 source inventory for HWLAB#1651/#1658. It intentionally exits non-zero while read-side projection assembly remains.
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
const args = new Set(process.argv.slice(2));
|
||||
const allowRed = args.has("--allow-red");
|
||||
const json = args.has("--json");
|
||||
const root = process.cwd();
|
||||
const maxMatchesPerCheck = 8;
|
||||
|
||||
const checks = [
|
||||
{
|
||||
id: "workbench-get-result-for-trace",
|
||||
phase: "P4",
|
||||
paths: ["internal/cloud/server-workbench-http.ts", "internal/cloud/workbench-read-model.ts"],
|
||||
pattern: /resultForTrace\s*[:(]/gu,
|
||||
expectation: "GET/read-model path must not assemble turn facts from mutable Code Agent result cache."
|
||||
},
|
||||
{
|
||||
id: "workbench-get-trace-snapshot",
|
||||
phase: "P4",
|
||||
paths: ["internal/cloud/server-workbench-http.ts", "internal/cloud/workbench-read-model.ts"],
|
||||
pattern: /traceSnapshot\s*[:(]/gu,
|
||||
expectation: "GET/read-model path must read durable Workbench facts, not hydrate trace snapshots on demand."
|
||||
},
|
||||
{
|
||||
id: "compact-session-trace-result",
|
||||
phase: "P4",
|
||||
paths: ["internal/cloud/server-workbench-http.ts"],
|
||||
pattern: /compactSessionTraceResult|compactSessionTraceSnapshot/gu,
|
||||
expectation: "Session list/detail must not reconstruct trace result or snapshot from compact session payloads."
|
||||
},
|
||||
{
|
||||
id: "facts-store-runtime-read-through",
|
||||
phase: "P2",
|
||||
paths: ["internal/cloud/workbench-facts-store.ts"],
|
||||
pattern: /runtimeStore\.queryAgentTraceEvents|durableTraceSnapshot\s*\(|shouldPreferDurableTrace|mergeTraceProjectionDiagnostic/gu,
|
||||
expectation: "WorkbenchFactsStore should expose already persisted facts; runtime trace read-through belongs to projector/finalizer."
|
||||
},
|
||||
{
|
||||
id: "get-handler-projection-assembly",
|
||||
phase: "P4",
|
||||
paths: ["internal/cloud/server-workbench-http.ts"],
|
||||
pattern: /createWorkbenchTurnProjection\s*\(|sessionProjectionOptions\s*\(|sessionListProjectionOptions\s*\(/gu,
|
||||
expectation: "GET handlers should serialize a read model DTO, not run projection assembly in the request path."
|
||||
},
|
||||
{
|
||||
id: "frontend-trace-final-response-fallback",
|
||||
phase: "P5",
|
||||
paths: ["web/hwlab-cloud-web/src/composables/useTraceSubscription.ts", "web/hwlab-cloud-web/src/stores/workbench.ts"],
|
||||
pattern: /finalResponse\s*:\s*[^\n]*\?\?|finalResponseText\(result\.finalResponse\)|traceSnapshotWithTurnStatus|mergeTerminalResultTrace/gu,
|
||||
expectation: "Frontend must render sealed final facts from the server DTO, not infer final response from prior trace/result objects."
|
||||
},
|
||||
{
|
||||
id: "frontend-status-fallback",
|
||||
phase: "P5",
|
||||
paths: ["web/hwlab-cloud-web/src/composables/useTraceSubscription.ts", "web/hwlab-cloud-web/src/stores/workbench.ts"],
|
||||
pattern: /status\s*:\s*[^\n]*\?\?|\.status\s*\?\?\s*[^\n]*/gu,
|
||||
expectation: "Frontend status authority must come from the Workbench DTO/reducer, not status fallback chains."
|
||||
},
|
||||
{
|
||||
id: "probe-session-repair",
|
||||
phase: "P7",
|
||||
paths: ["scripts/web-live-dom-probe.mjs"],
|
||||
pattern: /sessionRepair/gu,
|
||||
expectation: "The final golden probe should not rely on helper-side session repair to mask default-route/read-model gaps."
|
||||
}
|
||||
];
|
||||
|
||||
const findings = [];
|
||||
const missingFiles = [];
|
||||
|
||||
for (const check of checks) {
|
||||
const matches = [];
|
||||
for (const relativePath of check.paths) {
|
||||
const filePath = path.resolve(root, relativePath);
|
||||
if (!fs.existsSync(filePath)) {
|
||||
missingFiles.push(relativePath);
|
||||
continue;
|
||||
}
|
||||
const source = fs.readFileSync(filePath, "utf8");
|
||||
const lineStarts = lineStartOffsets(source);
|
||||
for (const match of source.matchAll(check.pattern)) {
|
||||
const line = lineNumberForOffset(lineStarts, match.index ?? 0);
|
||||
const snippet = source.split("\n")[line - 1]?.trim() ?? "";
|
||||
matches.push({ path: relativePath, line, text: match[0], snippet });
|
||||
if (matches.length >= maxMatchesPerCheck) break;
|
||||
}
|
||||
}
|
||||
if (matches.length > 0) findings.push({ ...check, pattern: String(check.pattern), matches });
|
||||
}
|
||||
|
||||
const report = {
|
||||
ok: findings.length === 0 && missingFiles.length === 0,
|
||||
status: findings.length > 0 ? "red" : missingFiles.length > 0 ? "blocked" : "green",
|
||||
issue: "pikasTech/HWLAB#1658",
|
||||
parentIssue: "pikasTech/HWLAB#1651",
|
||||
checkedAt: new Date().toISOString(),
|
||||
checkedFiles: [...new Set(checks.flatMap((check) => check.paths))],
|
||||
missingFiles,
|
||||
redFindingCount: findings.length,
|
||||
findings,
|
||||
nextPhases: [...new Set(findings.map((finding) => finding.phase))].sort(),
|
||||
valuesPrinted: false
|
||||
};
|
||||
|
||||
if (json) {
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
} else {
|
||||
printTextReport(report);
|
||||
}
|
||||
|
||||
if (!report.ok && !allowRed) process.exit(1);
|
||||
|
||||
function printTextReport(report) {
|
||||
console.log(`Workbench P1 inventory: ${report.status}`);
|
||||
console.log(`redFindingCount=${report.redFindingCount} missingFiles=${report.missingFiles.length}`);
|
||||
if (report.missingFiles.length > 0) console.log(`missingFiles=${report.missingFiles.join(",")}`);
|
||||
for (const finding of report.findings) {
|
||||
console.log(`\n[${finding.phase}] ${finding.id}`);
|
||||
console.log(`expectation: ${finding.expectation}`);
|
||||
for (const match of finding.matches) {
|
||||
console.log(`- ${match.path}:${match.line}: ${match.snippet}`);
|
||||
}
|
||||
}
|
||||
if (report.findings.length > 0) {
|
||||
console.log("\nThis is the expected P1 red inventory. Use --allow-red when collecting evidence without failing the shell.");
|
||||
}
|
||||
}
|
||||
|
||||
function lineStartOffsets(source) {
|
||||
const starts = [0];
|
||||
for (let index = 0; index < source.length; index += 1) {
|
||||
if (source.charCodeAt(index) === 10) starts.push(index + 1);
|
||||
}
|
||||
return starts;
|
||||
}
|
||||
|
||||
function lineNumberForOffset(starts, offset) {
|
||||
let low = 0;
|
||||
let high = starts.length - 1;
|
||||
while (low <= high) {
|
||||
const mid = Math.floor((low + high) / 2);
|
||||
if (starts[mid] <= offset) low = mid + 1;
|
||||
else high = mid - 1;
|
||||
}
|
||||
return Math.max(1, high + 1);
|
||||
}
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
{
|
||||
"metadata": {
|
||||
"captureId": "d601-v03-workbench-p1-long-reliability-redacted-001",
|
||||
"capturedFrom": {
|
||||
"node": "D601",
|
||||
"lane": "HWLAB v0.3",
|
||||
"origin": "https://hwlab.pikapython.com",
|
||||
"source": "UniDesk controlled web-probe plus existing redacted Workbench fake-server captures",
|
||||
"webProbe": {
|
||||
"command": "bun scripts/cli.ts hwlab nodes web-probe script --node D601 --lane v03",
|
||||
"scriptSha256": "sha256:5b0537b26266937b84eeebaef23985e1aa7507019ccd30d11dec78228e0d76ed",
|
||||
"reportPath": "/home/ubuntu/workspace/hwlab-v03/.state/web-probe-script/run.6US9qW/web-probe-script-report.json",
|
||||
"reportSha256": "sha256:00b6543859599fa416328aee509f2c2a9158a286132e30a2e916045ed501fcd9",
|
||||
"valuesPrinted": false
|
||||
}
|
||||
},
|
||||
"capturedAt": "2026-06-20T00:24:00.000Z",
|
||||
"schemaVersion": "workbench-p1-long-reliability-inventory.v1",
|
||||
"redactionVersion": "metadata-only-redaction.v1",
|
||||
"derivedFrom": [
|
||||
"web/hwlab-cloud-web/tests/workbench-e2e/fixtures/real-captures/d601-v03-redacted.json",
|
||||
"web/hwlab-cloud-web/scripts/workbench-e2e-server.ts#sealed-final-response-diagnostics",
|
||||
"web/hwlab-cloud-web/scripts/workbench-e2e-server.ts#trace-hydration-timeout-diagnostic",
|
||||
"web/hwlab-cloud-web/scripts/workbench-e2e-server.ts#projection-degraded-diagnostics",
|
||||
"web/hwlab-cloud-web/scripts/workbench-e2e-server.ts#projector-resume-from-checkpoint"
|
||||
],
|
||||
"valuesPrinted": false,
|
||||
"notes": [
|
||||
"This fixture intentionally stores endpoint shape, scenario IDs, and source references only.",
|
||||
"Session IDs in live evidence are reduced to prefix/shape in issue comments; full values remain in local redacted probe artifacts.",
|
||||
"No cookie, Authorization header, API key, provider token, DSN, raw prompt, or user secret is stored."
|
||||
]
|
||||
},
|
||||
"liveReadModelShape": {
|
||||
"finalUrlKind": "workbench-session-route",
|
||||
"selectedSessionShape": {
|
||||
"sessionIdPrefix": "ses_79e7",
|
||||
"status": "idle",
|
||||
"hasLastTraceId": false
|
||||
},
|
||||
"endpoints": [
|
||||
{
|
||||
"path": "/auth/session",
|
||||
"status": 200,
|
||||
"topLevelKeys": ["authenticated", "authMethod", "identityAuthority", "sessionKind"]
|
||||
},
|
||||
{
|
||||
"path": "/v1/workbench/sessions?limit=5",
|
||||
"status": 200,
|
||||
"topLevelKeys": ["ok", "status", "contractVersion", "sessions"]
|
||||
},
|
||||
{
|
||||
"path": "/v1/workbench/sessions/<selected-session>",
|
||||
"status": 200,
|
||||
"topLevelKeys": ["ok", "status", "contractVersion", "session"]
|
||||
},
|
||||
{
|
||||
"path": "/v1/workbench/sessions/<selected-session>/messages?limit=100",
|
||||
"status": 200,
|
||||
"topLevelKeys": ["ok", "status", "contractVersion", "sessionId"]
|
||||
}
|
||||
],
|
||||
"valuesPrinted": false
|
||||
},
|
||||
"redScenarios": [
|
||||
{
|
||||
"scenarioId": "sealed-final-response-diagnostics",
|
||||
"purpose": "completed sealed final response remains visible while trace hydration diagnostics are shown separately",
|
||||
"covers": ["completed sealed after polling", "trace hydration error does not overwrite main message"]
|
||||
},
|
||||
{
|
||||
"scenarioId": "trace-hydration-timeout-diagnostic",
|
||||
"purpose": "GET lag or trace hydration timeout surfaces diagnostics instead of terminal/status inference",
|
||||
"covers": ["projection lag", "diagnostic visibility", "no terminal inference"]
|
||||
},
|
||||
{
|
||||
"scenarioId": "projection-degraded-diagnostics",
|
||||
"purpose": "AgentRun result lag stays a projection diagnostic and keeps lifecycle running",
|
||||
"covers": ["projection blocker", "running lifecycle authority"]
|
||||
},
|
||||
{
|
||||
"scenarioId": "projector-resume-from-checkpoint",
|
||||
"purpose": "future P2/P3 projector catch-up can turn P1 red inventory green without GET repair",
|
||||
"covers": ["durable checkpoint", "pagination consistency", "no legacy request ledger"]
|
||||
}
|
||||
],
|
||||
"negativeSourceScan": {
|
||||
"script": "scripts/workbench-long-reliability-p1-inventory.mjs",
|
||||
"entrypoint": "npm run workbench:p1:inventory -- --json",
|
||||
"expectedCurrentStatus": "red",
|
||||
"trackedTerms": [
|
||||
"resultForTrace",
|
||||
"traceSnapshot",
|
||||
"compactSessionTraceResult",
|
||||
"compactSessionTraceSnapshot",
|
||||
"status ??",
|
||||
"finalResponse ?? previous",
|
||||
"sessionRepair",
|
||||
"GET handler projection assembly"
|
||||
],
|
||||
"valuesPrinted": false
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user