Merge pull request #255 from pikasTech/fix/deploy-contract-toml-python310-251
fix: remove Python tomllib deploy validation dependency
This commit is contained in:
Lyon
+1
-1
@@ -5,7 +5,7 @@
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"validate": "node scripts/validate-contract.mjs && node scripts/deploy-contract-plan.mjs --check && node scripts/deploy-desired-state-plan.mjs --check && node scripts/dev-runtime-migration.mjs --check && node --test scripts/artifact-runtime-readiness-guard.test.mjs",
|
||||
"check": "node --check internal/protocol/index.mjs && node --check internal/agent/index.mjs && node --check internal/agent/runtime.mjs && node --check internal/audit/index.mjs && node --check internal/db/runtime-store.mjs && node --check internal/db/runtime-store.test.mjs && node --check internal/mvp-gate/summary.mjs && node --check internal/cloud/db-contract.mjs && node --check internal/cloud/code-agent-contract.mjs && node --check internal/cloud/json-rpc.mjs && node --check internal/cloud/code-agent-chat.mjs && node --check internal/cloud/server.mjs && node --check internal/sim/model.mjs && node --check internal/sim/model.test.mjs && node --check internal/sim/http.mjs && node --check internal/sim/l2-runtime.mjs && node --check internal/patchpanel/model.mjs && node --check internal/patchpanel/runtime.mjs && node --check cmd/hwlab-cloud-api/main.mjs && node --check cmd/hwlab-agent-mgr/main.mjs && node --check cmd/hwlab-agent-worker/main.mjs && node --check cmd/hwlab-box-simu/main.mjs && node --check cmd/hwlab-gateway-simu/main.mjs && node --check scripts/cloud-api-runtime-smoke.mjs && node --check scripts/code-agent-chat-smoke.mjs && node --check scripts/dev-edge-health-smoke.mjs && node --check scripts/src/dev-edge-health-smoke-lib.mjs && node --check scripts/validate-contract.mjs && node --check scripts/deploy-desired-state-plan.mjs && node --check scripts/src/deploy-desired-state-plan.mjs && node --check scripts/artifact-runtime-readiness-guard.mjs && node --check scripts/src/artifact-runtime-readiness-guard.mjs && node --check scripts/artifact-runtime-readiness-guard.test.mjs && node --check scripts/report-lifecycle.mjs && node --check scripts/report-lifecycle.test.mjs && node --check scripts/validate-dev-gate-report.mjs && node --check scripts/dev-m3-hardware-loop-smoke.test.mjs && node --check scripts/dev-cloud-workbench-smoke.test.mjs && node --check scripts/validate-dev-m3-cardinality.mjs && node --check scripts/validate-artifact-catalog.mjs && node --check scripts/refresh-artifact-catalog.mjs && node --check scripts/dev-artifact-publish.mjs && node --check scripts/src/dev-artifact-services.mjs && node --check scripts/src/registry-capabilities.mjs && node --check scripts/preflight-dev-base-image.mjs && node --check scripts/src/dev-base-image-preflight.mjs && node --check scripts/dev-evidence-blocker-aggregator.mjs && node --check scripts/src/dev-evidence-blocker-aggregator.mjs && node --check scripts/d601-k3s-readonly-observability.mjs && node --check scripts/src/d601-k3s-readonly-observability.mjs && node --check scripts/dev-runtime-migration.mjs && node --check scripts/src/dev-runtime-migration.mjs && node --check scripts/src/dev-runtime-migration.test.mjs && node --check scripts/l2-runtime-contract-smoke.mjs && node --check scripts/patch-panel-runtime-smoke.mjs && node --check scripts/export-web-gate-summary.mjs && node --check scripts/l6-cli-web-smoke.mjs && node --check tools/hwlab-cli/bin/hwlab-cli.mjs && node --check tools/hwlab-cli/lib/cli.mjs && node --check web/hwlab-cloud-web/app.mjs && node --check web/hwlab-cloud-web/gate-summary.mjs && node --check web/hwlab-cloud-web/scripts/check.mjs && node --check web/hwlab-cloud-web/scripts/build.mjs && node --check web/hwlab-cloud-web/scripts/dist-contract.mjs && node scripts/validate-contract.mjs && node scripts/validate-dev-gate-report.mjs && node scripts/report-lifecycle.test.mjs && node scripts/validate-dev-m3-cardinality.mjs && node scripts/validate-artifact-catalog.mjs && node scripts/deploy-desired-state-plan.mjs --check && node scripts/dev-runtime-migration.mjs --check && node scripts/dev-evidence-blocker-aggregator.mjs --check && node scripts/l2-runtime-contract-smoke.mjs && node scripts/l6-cli-web-smoke.mjs && node web/hwlab-cloud-web/scripts/check.mjs && node scripts/code-agent-chat-smoke.mjs && node --test scripts/artifact-runtime-readiness-guard.test.mjs scripts/dev-m3-hardware-loop-smoke.test.mjs scripts/dev-cloud-workbench-smoke.test.mjs scripts/deploy-desired-state-plan.test.mjs scripts/src/dev-deploy-apply.test.mjs scripts/src/dev-runtime-migration.test.mjs internal/agent/index.test.mjs internal/audit/index.test.mjs internal/db/schema.test.mjs internal/db/runtime-store.test.mjs internal/cloud/json-rpc.test.mjs internal/cloud/server.test.mjs internal/sim/model.test.mjs internal/patchpanel/model.test.mjs internal/patchpanel/runtime.test.mjs && node scripts/cloud-api-runtime-smoke.mjs && sh -n scripts/bootstrap-skills.sh scripts/worker-entrypoint.sh",
|
||||
"check": "node --check internal/protocol/index.mjs && node --check internal/agent/index.mjs && node --check internal/agent/runtime.mjs && node --check internal/audit/index.mjs && node --check internal/db/runtime-store.mjs && node --check internal/db/runtime-store.test.mjs && node --check internal/mvp-gate/summary.mjs && node --check internal/cloud/db-contract.mjs && node --check internal/cloud/code-agent-contract.mjs && node --check internal/cloud/json-rpc.mjs && node --check internal/cloud/code-agent-chat.mjs && node --check internal/cloud/server.mjs && node --check internal/sim/model.mjs && node --check internal/sim/model.test.mjs && node --check internal/sim/http.mjs && node --check internal/sim/l2-runtime.mjs && node --check internal/patchpanel/model.mjs && node --check internal/patchpanel/runtime.mjs && node --check cmd/hwlab-cloud-api/main.mjs && node --check cmd/hwlab-agent-mgr/main.mjs && node --check cmd/hwlab-agent-worker/main.mjs && node --check cmd/hwlab-box-simu/main.mjs && node --check cmd/hwlab-gateway-simu/main.mjs && node --check scripts/cloud-api-runtime-smoke.mjs && node --check scripts/code-agent-chat-smoke.mjs && node --check scripts/dev-edge-health-smoke.mjs && node --check scripts/src/dev-edge-health-smoke-lib.mjs && node --check scripts/validate-contract.mjs && node --check scripts/deploy-contract-plan.test.mjs && node --check scripts/deploy-desired-state-plan.mjs && node --check scripts/src/deploy-desired-state-plan.mjs && node --check scripts/artifact-runtime-readiness-guard.mjs && node --check scripts/src/artifact-runtime-readiness-guard.mjs && node --check scripts/artifact-runtime-readiness-guard.test.mjs && node --check scripts/report-lifecycle.mjs && node --check scripts/report-lifecycle.test.mjs && node --check scripts/validate-dev-gate-report.mjs && node --check scripts/dev-m3-hardware-loop-smoke.test.mjs && node --check scripts/dev-cloud-workbench-smoke.test.mjs && node --check scripts/validate-dev-m3-cardinality.mjs && node --check scripts/validate-artifact-catalog.mjs && node --check scripts/refresh-artifact-catalog.mjs && node --check scripts/dev-artifact-publish.mjs && node --check scripts/src/dev-artifact-services.mjs && node --check scripts/src/registry-capabilities.mjs && node --check scripts/preflight-dev-base-image.mjs && node --check scripts/src/dev-base-image-preflight.mjs && node --check scripts/dev-evidence-blocker-aggregator.mjs && node --check scripts/src/dev-evidence-blocker-aggregator.mjs && node --check scripts/d601-k3s-readonly-observability.mjs && node --check scripts/src/d601-k3s-readonly-observability.mjs && node --check scripts/dev-runtime-migration.mjs && node --check scripts/src/dev-runtime-migration.mjs && node --check scripts/src/dev-runtime-migration.test.mjs && node --check scripts/l2-runtime-contract-smoke.mjs && node --check scripts/patch-panel-runtime-smoke.mjs && node --check scripts/export-web-gate-summary.mjs && node --check scripts/l6-cli-web-smoke.mjs && node --check tools/hwlab-cli/bin/hwlab-cli.mjs && node --check tools/hwlab-cli/lib/cli.mjs && node --check web/hwlab-cloud-web/app.mjs && node --check web/hwlab-cloud-web/gate-summary.mjs && node --check web/hwlab-cloud-web/scripts/check.mjs && node --check web/hwlab-cloud-web/scripts/build.mjs && node --check web/hwlab-cloud-web/scripts/dist-contract.mjs && node scripts/validate-contract.mjs && node scripts/validate-dev-gate-report.mjs && node scripts/report-lifecycle.test.mjs && node scripts/validate-dev-m3-cardinality.mjs && node scripts/validate-artifact-catalog.mjs && node scripts/deploy-desired-state-plan.mjs --check && node scripts/dev-runtime-migration.mjs --check && node scripts/dev-evidence-blocker-aggregator.mjs --check && node scripts/l2-runtime-contract-smoke.mjs && node scripts/l6-cli-web-smoke.mjs && node web/hwlab-cloud-web/scripts/check.mjs && node scripts/code-agent-chat-smoke.mjs && node --test scripts/artifact-runtime-readiness-guard.test.mjs scripts/deploy-contract-plan.test.mjs scripts/dev-m3-hardware-loop-smoke.test.mjs scripts/dev-cloud-workbench-smoke.test.mjs scripts/deploy-desired-state-plan.test.mjs scripts/src/dev-deploy-apply.test.mjs scripts/src/dev-runtime-migration.test.mjs internal/agent/index.test.mjs internal/audit/index.test.mjs internal/db/schema.test.mjs internal/db/runtime-store.test.mjs internal/cloud/json-rpc.test.mjs internal/cloud/server.test.mjs internal/sim/model.test.mjs internal/patchpanel/model.test.mjs internal/patchpanel/runtime.test.mjs && node scripts/cloud-api-runtime-smoke.mjs && sh -n scripts/bootstrap-skills.sh scripts/worker-entrypoint.sh",
|
||||
"dev-base-image:preflight": "node scripts/preflight-dev-base-image.mjs",
|
||||
"cloud-api:smoke": "node scripts/cloud-api-runtime-smoke.mjs",
|
||||
"m1:smoke": "node scripts/m1-contract-smoke.mjs",
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { chmod, mkdtemp, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import { parseFrpTomlText } from "./src/deploy-contract-plan.mjs";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
|
||||
test("parses FRP TOML tables without Python tomllib", () => {
|
||||
const parsed = parseFrpTomlText(`
|
||||
serverAddr = "74.48.78.17"
|
||||
serverPort = 7000
|
||||
loginFailExit = true
|
||||
|
||||
[[proxies]]
|
||||
name = "hwlab-dev-cloud-web"
|
||||
type = "tcp"
|
||||
localIP = "hwlab-cloud-web.hwlab-dev.svc.cluster.local"
|
||||
localPort = 8080
|
||||
remotePort = 16666
|
||||
|
||||
[[proxies]]
|
||||
name = "hwlab-dev-edge-proxy"
|
||||
type = "tcp"
|
||||
localIP = "hwlab-edge-proxy.hwlab-dev.svc.cluster.local"
|
||||
localPort = 6667
|
||||
remotePort = 16667
|
||||
`, "fixture/frpc.dev.toml");
|
||||
|
||||
assert.deepEqual(parsed, {
|
||||
serverAddr: "74.48.78.17",
|
||||
serverPort: 7000,
|
||||
loginFailExit: true,
|
||||
proxies: [
|
||||
{
|
||||
name: "hwlab-dev-cloud-web",
|
||||
type: "tcp",
|
||||
localIP: "hwlab-cloud-web.hwlab-dev.svc.cluster.local",
|
||||
localPort: 8080,
|
||||
remotePort: 16666
|
||||
},
|
||||
{
|
||||
name: "hwlab-dev-edge-proxy",
|
||||
type: "tcp",
|
||||
localIP: "hwlab-edge-proxy.hwlab-dev.svc.cluster.local",
|
||||
localPort: 6667,
|
||||
remotePort: 16667
|
||||
}
|
||||
]
|
||||
});
|
||||
});
|
||||
|
||||
test("deploy contract check does not require python3 tomllib", async () => {
|
||||
const shimDir = await mkdtemp(path.join(os.tmpdir(), "hwlab-python-shim-"));
|
||||
const pythonShim = path.join(shimDir, "python3");
|
||||
await writeFile(
|
||||
pythonShim,
|
||||
[
|
||||
"#!/usr/bin/env bash",
|
||||
"echo 'ModuleNotFoundError: No module named tomllib' >&2",
|
||||
"exit 1",
|
||||
""
|
||||
].join("\n")
|
||||
);
|
||||
await chmod(pythonShim, 0o755);
|
||||
|
||||
const result = await execFileAsync(process.execPath, ["scripts/deploy-contract-plan.mjs", "--check"], {
|
||||
cwd: repoRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
PATH: `${shimDir}${path.delimiter}${process.env.PATH ?? ""}`
|
||||
},
|
||||
maxBuffer: 1024 * 1024
|
||||
});
|
||||
const plan = JSON.parse(result.stdout);
|
||||
|
||||
assert.equal(plan.status, "pass");
|
||||
assert.equal(plan.summary.frpProxies, 3);
|
||||
assert.deepEqual(plan.diagnostics, []);
|
||||
});
|
||||
@@ -1,19 +1,17 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { DEV_DB_ENV_CONTRACT } from "../../internal/cloud/db-contract.mjs";
|
||||
import {
|
||||
DEV_CODE_AGENT_PROVIDER_CONTRACT,
|
||||
codeAgentSecretRefPlaceholder
|
||||
} from "../../internal/cloud/code-agent-contract.mjs";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const expectedPorts = Object.freeze({ frontend: 16666, api: 16667 });
|
||||
const legacyPublicPorts = new Set([6666, 6667]);
|
||||
const duplicateMappingAllowed = new Set(["hwlab-box-simu", "hwlab-gateway-simu"]);
|
||||
const tomlBareKeyPattern = /^[A-Za-z0-9_-]+$/u;
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = { check: false, pretty: false };
|
||||
@@ -33,14 +31,94 @@ async function readJson(relativePath) {
|
||||
return JSON.parse(await readFile(path.join(repoRoot, relativePath), "utf8"));
|
||||
}
|
||||
|
||||
function stripTomlComment(line) {
|
||||
let inString = false;
|
||||
let escaped = false;
|
||||
for (let index = 0; index < line.length; index += 1) {
|
||||
const char = line[index];
|
||||
if (inString) {
|
||||
if (escaped) {
|
||||
escaped = false;
|
||||
} else if (char === "\\") {
|
||||
escaped = true;
|
||||
} else if (char === "\"") {
|
||||
inString = false;
|
||||
}
|
||||
} else if (char === "\"") {
|
||||
inString = true;
|
||||
} else if (char === "#") {
|
||||
return line.slice(0, index);
|
||||
}
|
||||
}
|
||||
return line;
|
||||
}
|
||||
|
||||
function parseTomlValue(value, relativePath, lineNumber) {
|
||||
if (value.startsWith("\"")) {
|
||||
try {
|
||||
const parsed = JSON.parse(value);
|
||||
if (typeof parsed !== "string") {
|
||||
throw new Error("not a string");
|
||||
}
|
||||
return parsed;
|
||||
} catch {
|
||||
throw new Error(`${relativePath}:${lineNumber}: invalid TOML basic string`);
|
||||
}
|
||||
}
|
||||
if (value === "true") return true;
|
||||
if (value === "false") return false;
|
||||
if (/^[+-]?\d(?:_?\d)*$/u.test(value)) {
|
||||
return Number.parseInt(value.replaceAll("_", ""), 10);
|
||||
}
|
||||
throw new Error(`${relativePath}:${lineNumber}: unsupported TOML value ${JSON.stringify(value)}`);
|
||||
}
|
||||
|
||||
// Keep this parser scoped to the FRP TOML subset used in this repo so the dry-run
|
||||
// contract check does not depend on Python 3.11's tomllib being present.
|
||||
export function parseFrpTomlText(text, relativePath = "FRP TOML") {
|
||||
const document = {};
|
||||
let current = document;
|
||||
for (const [lineIndex, rawLine] of text.split(/\r?\n/u).entries()) {
|
||||
const lineNumber = lineIndex + 1;
|
||||
const line = stripTomlComment(rawLine).trim();
|
||||
if (line === "") continue;
|
||||
|
||||
const arrayTableMatch = line.match(/^\[\[([A-Za-z0-9_-]+)\]\]$/u);
|
||||
if (arrayTableMatch) {
|
||||
const name = arrayTableMatch[1];
|
||||
if (!Array.isArray(document[name])) {
|
||||
if (Object.hasOwn(document, name)) {
|
||||
throw new Error(`${relativePath}:${lineNumber}: cannot redefine scalar ${name} as array table`);
|
||||
}
|
||||
document[name] = [];
|
||||
}
|
||||
current = {};
|
||||
document[name].push(current);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line.startsWith("[") || line.includes("]")) {
|
||||
throw new Error(`${relativePath}:${lineNumber}: only root keys and array-of-table headers are supported`);
|
||||
}
|
||||
|
||||
const assignmentMatch = line.match(/^([A-Za-z0-9_-]+)\s*=\s*(.+)$/u);
|
||||
if (!assignmentMatch) {
|
||||
throw new Error(`${relativePath}:${lineNumber}: expected key = value`);
|
||||
}
|
||||
const [, key, valueSource] = assignmentMatch;
|
||||
if (!tomlBareKeyPattern.test(key)) {
|
||||
throw new Error(`${relativePath}:${lineNumber}: unsupported TOML key ${JSON.stringify(key)}`);
|
||||
}
|
||||
if (Object.hasOwn(current, key)) {
|
||||
throw new Error(`${relativePath}:${lineNumber}: duplicate TOML key ${key}`);
|
||||
}
|
||||
current[key] = parseTomlValue(valueSource.trim(), relativePath, lineNumber);
|
||||
}
|
||||
return document;
|
||||
}
|
||||
|
||||
async function parseToml(relativePath) {
|
||||
const code = [
|
||||
"import json,sys,tomllib",
|
||||
"with open(sys.argv[1], 'rb') as handle:",
|
||||
" print(json.dumps(tomllib.load(handle), sort_keys=True))"
|
||||
].join("\n");
|
||||
const result = await execFileAsync("python3", ["-c", code, path.join(repoRoot, relativePath)]);
|
||||
return JSON.parse(result.stdout);
|
||||
return parseFrpTomlText(await readFile(path.join(repoRoot, relativePath), "utf8"), relativePath);
|
||||
}
|
||||
|
||||
function addDiagnostic(ctx, code, jsonPath, message, { expected, actual } = {}) {
|
||||
|
||||
Reference in New Issue
Block a user