Merge pull request #2466 from pikasTech/fix/2464-kafka-projector-v2
Pipelines as Code CI / hwlab-nc01-v03-ci-poll- Success
Pipelines as Code CI / hwlab-nc01-v03-ci-poll- Success
feat: 拉通 Workbench 纯 Kafka 实时链路并独立隔离投影能力 (#2467)
This commit is contained in:
+7
-10
@@ -521,19 +521,9 @@ lanes:
|
||||
TZ: Asia/Shanghai
|
||||
HWLAB_METRICS_NAMESPACE: hwlab-v03
|
||||
HWLAB_OBSERVABILITY_CONFIG_REVISION: PJ2026-01060505-20260619-long-trace-timeout-mitigation
|
||||
HWLAB_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS: "30000"
|
||||
HWLAB_CODE_AGENT_AGENTRUN_NO_RESPONSE_TIMEOUT_MS: "600000"
|
||||
HWLAB_CODE_AGENT_AGENTRUN_DISPATCH_RETRY_MAX: "5"
|
||||
HWLAB_CODE_AGENT_AGENTRUN_DISPATCH_RETRY_BASE_MS: "1000"
|
||||
HWLAB_CODE_AGENT_AGENTRUN_DISPATCH_RETRY_MAX_DELAY_MS: "30000"
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_POLL_INTERVAL_MS: "1000"
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_ENABLED: "1"
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INITIAL_DELAY_MS: "5000"
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INTERVAL_MS: "60000"
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS: "30000"
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE: "100"
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_JITTER_MS: "15000"
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_FAILURE_BACKOFF_MS: "120000"
|
||||
observable: true
|
||||
hwlab-workbench-runtime:
|
||||
runtimeKind: go-service
|
||||
@@ -845,6 +835,13 @@ lanes:
|
||||
HWLAB_KAFKA_EVENT_TOPIC: hwlab.event.v1
|
||||
HWLAB_KAFKA_CLIENT_ID: hwlab-v03-cloud-api
|
||||
HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID: hwlab-v03-agentrun-event-bridge
|
||||
HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS: "2000"
|
||||
HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS: "250"
|
||||
HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE: "100"
|
||||
HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS: "30000"
|
||||
HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS: "10000"
|
||||
HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS: "1000"
|
||||
HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100"
|
||||
HWLAB_WORKBENCH_EMPTY_SESSION_GC_ENABLED: "1"
|
||||
HWLAB_WORKBENCH_EMPTY_SESSION_TTL_MS: "600000"
|
||||
HWLAB_WORKBENCH_EMPTY_SESSION_GC_INTERVAL_MS: "60000"
|
||||
|
||||
@@ -214,20 +214,21 @@ export function createHwlabAgentRunDispatchAssembly(options = {}) {
|
||||
providerProfile: backendProfile,
|
||||
dispatchSource: "hwlab-code-agent"
|
||||
},
|
||||
dispatch: {
|
||||
kind: "kubernetes-job",
|
||||
input: {
|
||||
idempotencyKey: options.runnerJobIdempotencyKey ?? `hwlab:${traceId}:runner-job`,
|
||||
...(options.attemptId ? { attemptId: nonEmptyString(options.attemptId, "attemptId") } : {}),
|
||||
transientEnv
|
||||
}
|
||||
},
|
||||
idempotencyKey: options.commandIdempotencyKey ?? `hwlab:${traceId}:turn`
|
||||
};
|
||||
|
||||
const runnerJobPayload = {
|
||||
idempotencyKey: options.runnerJobIdempotencyKey ?? `hwlab:${traceId}:runner-job`,
|
||||
...(options.attemptId ? { attemptId: nonEmptyString(options.attemptId, "attemptId") } : {}),
|
||||
transientEnv
|
||||
};
|
||||
|
||||
return {
|
||||
managerUrl,
|
||||
runPayload,
|
||||
commandPayload,
|
||||
runnerJobPayload,
|
||||
boundaries: {
|
||||
valuesPrinted: false,
|
||||
githubCredentialSource: includeGithubToolCredential ? "toolCredentials" : null,
|
||||
@@ -249,18 +250,19 @@ export async function dispatchHwlabAgentRun(options = {}) {
|
||||
const run = await postJson(fetchImpl, managerUrl, "/api/v1/runs", assembly.runPayload);
|
||||
const runId = nonEmptyString(run.id, "run.id");
|
||||
const command = await postJson(fetchImpl, managerUrl, `/api/v1/runs/${encodeURIComponent(runId)}/commands`, assembly.commandPayload);
|
||||
const commandId = nonEmptyString(command.id, "command.id");
|
||||
const runnerJobPayload = { ...assembly.runnerJobPayload, commandId };
|
||||
const runnerJob = await postJson(fetchImpl, managerUrl, `/api/v1/runs/${encodeURIComponent(runId)}/runner-jobs`, runnerJobPayload);
|
||||
nonEmptyString(command.id, "command.id");
|
||||
const dispatchIntent = command.dispatchIntent;
|
||||
if (!dispatchIntent || dispatchIntent.durable !== true || !optionalString(dispatchIntent.id)) {
|
||||
throw new Error("AgentRun command response requires a durable dispatchIntent");
|
||||
}
|
||||
return {
|
||||
managerUrl,
|
||||
run,
|
||||
command,
|
||||
runnerJob,
|
||||
dispatchIntent,
|
||||
requests: {
|
||||
runPayload: assembly.runPayload,
|
||||
commandPayload: assembly.commandPayload,
|
||||
runnerJobPayload
|
||||
commandPayload: assembly.commandPayload
|
||||
},
|
||||
boundaries: assembly.boundaries
|
||||
};
|
||||
|
||||
@@ -61,7 +61,7 @@ test("HWLAB AgentRun assembly grants GitHub and UniDesk SSH through toolCredenti
|
||||
assert.equal(tools.some((item) => item.tool === "github" && item.projection.envName === "GH_TOKEN"), true);
|
||||
assert.equal(tools.some((item) => item.tool === "unidesk-ssh" && item.projection.envName === "UNIDESK_SSH_CLIENT_TOKEN"), true);
|
||||
|
||||
const transientEnv = assembly.runnerJobPayload.transientEnv;
|
||||
const transientEnv = assembly.commandPayload.dispatch.input.transientEnv;
|
||||
assert.equal(transientEnv.find((item) => item.name === DEFAULT_UNIDESK_MAIN_SERVER_ENV)?.value, "https://unidesk.example.test");
|
||||
assert.equal(transientEnv.some((item) => item.name === "UNIDESK_SSH_CLIENT_TOKEN"), false);
|
||||
assert.equal(assembly.boundaries.unideskSshCredentialSource, "toolCredentials");
|
||||
@@ -112,7 +112,7 @@ test("HWLAB AgentRun assembly does not duplicate explicit UniDesk main-server en
|
||||
unideskMainServerIp: "https://unidesk.example.test",
|
||||
transientEnv: [{ name: DEFAULT_UNIDESK_MAIN_SERVER_ENV, value: "https://explicit.example.test", sensitive: false }]
|
||||
});
|
||||
const entries = assembly.runnerJobPayload.transientEnv.filter((item) => item.name === DEFAULT_UNIDESK_MAIN_SERVER_ENV);
|
||||
const entries = assembly.commandPayload.dispatch.input.transientEnv.filter((item) => item.name === DEFAULT_UNIDESK_MAIN_SERVER_ENV);
|
||||
assert.equal(entries.length, 1);
|
||||
assert.equal(entries[0].value, "https://explicit.example.test");
|
||||
});
|
||||
@@ -128,14 +128,13 @@ test("HWLAB AgentRun assembly requires full commit and main server when UniDesk
|
||||
);
|
||||
});
|
||||
|
||||
test("dispatchHwlabAgentRun posts run command and runner job with commandId", async () => {
|
||||
test("dispatchHwlabAgentRun atomically requests durable dispatch with the command", async () => {
|
||||
const requests = [];
|
||||
const fetchImpl = async (url, init) => {
|
||||
const request = { url, method: init.method, body: JSON.parse(init.body) };
|
||||
requests.push(request);
|
||||
if (url.endsWith("/api/v1/runs")) return jsonResponse({ id: "run_hwlab_001" });
|
||||
if (url.endsWith("/api/v1/runs/run_hwlab_001/commands")) return jsonResponse({ id: "cmd_hwlab_001" });
|
||||
if (url.endsWith("/api/v1/runs/run_hwlab_001/runner-jobs")) return jsonResponse({ id: "job_hwlab_001", jobName: "agentrun-v01-runner-selftest" });
|
||||
if (url.endsWith("/api/v1/runs/run_hwlab_001/commands")) return jsonResponse({ id: "cmd_hwlab_001", dispatchIntent: { id: "dispatch_hwlab_001", state: "pending", runnerJobId: "rjob_hwlab_001", durable: true } });
|
||||
return jsonResponse({ error: "unexpected path" }, { status: 404 });
|
||||
};
|
||||
const result = await dispatchHwlabAgentRun({
|
||||
@@ -148,21 +147,19 @@ test("dispatchHwlabAgentRun posts run command and runner job with commandId", as
|
||||
|
||||
assert.equal(result.run.id, "run_hwlab_001");
|
||||
assert.equal(result.command.id, "cmd_hwlab_001");
|
||||
assert.equal(result.runnerJob.jobName, "agentrun-v01-runner-selftest");
|
||||
assert.equal(result.dispatchIntent.id, "dispatch_hwlab_001");
|
||||
assert.deepEqual(requests.map((item) => new URL(item.url).pathname), [
|
||||
"/api/v1/runs",
|
||||
"/api/v1/runs/run_hwlab_001/commands",
|
||||
"/api/v1/runs/run_hwlab_001/runner-jobs"
|
||||
"/api/v1/runs/run_hwlab_001/commands"
|
||||
]);
|
||||
assert.equal(requests[2].body.commandId, "cmd_hwlab_001");
|
||||
assert.equal(requests[2].body.transientEnv.some((item) => item.name === "UNIDESK_SSH_CLIENT_TOKEN"), false);
|
||||
assert.equal(requests[1].body.dispatch.kind, "kubernetes-job");
|
||||
assert.equal(requests[1].body.dispatch.input.transientEnv.some((item) => item.name === "UNIDESK_SSH_CLIENT_TOKEN"), false);
|
||||
});
|
||||
|
||||
test("dispatchHwlabAgentRun unwraps AgentRun manager response envelopes", async () => {
|
||||
const fetchImpl = async (url, init) => {
|
||||
if (url.endsWith("/api/v1/runs")) return jsonResponse({ ok: true, data: { id: "run_hwlab_envelope" } });
|
||||
if (url.endsWith("/api/v1/runs/run_hwlab_envelope/commands")) return jsonResponse({ ok: true, data: { id: "cmd_hwlab_envelope" } });
|
||||
if (url.endsWith("/api/v1/runs/run_hwlab_envelope/runner-jobs")) return jsonResponse({ ok: true, data: { id: "job_hwlab_envelope", jobName: "agentrun-v01-runner-envelope" } });
|
||||
if (url.endsWith("/api/v1/runs/run_hwlab_envelope/commands")) return jsonResponse({ ok: true, data: { id: "cmd_hwlab_envelope", dispatchIntent: { id: "dispatch_hwlab_envelope", state: "pending", runnerJobId: "rjob_hwlab_envelope", durable: true } } });
|
||||
return jsonResponse({ ok: false, failureKind: "unexpected-path" }, { status: 404 });
|
||||
};
|
||||
|
||||
@@ -176,7 +173,7 @@ test("dispatchHwlabAgentRun unwraps AgentRun manager response envelopes", async
|
||||
|
||||
assert.equal(result.run.id, "run_hwlab_envelope");
|
||||
assert.equal(result.command.id, "cmd_hwlab_envelope");
|
||||
assert.equal(result.runnerJob.jobName, "agentrun-v01-runner-envelope");
|
||||
assert.equal(result.dispatchIntent.id, "dispatch_hwlab_envelope");
|
||||
});
|
||||
|
||||
function jsonResponse(body, { status = 200 } = {}) {
|
||||
|
||||
@@ -6,18 +6,18 @@ import { createServer as createHttpServer } from "node:http";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { createBackendPerformanceStore, backendPerformanceRouteTemplate, withBackendPerformanceContext } from "./backend-performance.ts";
|
||||
import { submitAgentRunChatTurn, syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts";
|
||||
import { submitAgentRunChatTurn } from "./code-agent-agentrun-adapter.ts";
|
||||
import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
||||
import { createCloudApiServer } from "./server.ts";
|
||||
import { createCodeAgentChatResultStore } from "./server-code-agent-http.ts";
|
||||
|
||||
test("backend performance store templates routes and appends Cloud API metrics to loopback metrics", async () => {
|
||||
assert.equal(backendPerformanceRouteTemplate("/v1/agent/traces/trc_secret_0123456789abcdef?full=1"), "/v1/agent/traces/:id");
|
||||
assert.equal(backendPerformanceRouteTemplate("/v1/agent/traces/trc_secret_0123456789abcdef?full=1"), "/v1/agent/traces/:traceId");
|
||||
assert.equal(backendPerformanceRouteTemplate("/api/v1/runs/run_0123456789abcdef/commands/cmd_0123456789abcdef/result"), "/api/v1/runs/:id/commands/:id/result");
|
||||
assert.equal(backendPerformanceRouteTemplate("/v1/admin/billing/users/12345/credits/adjust"), "/v1/admin/billing/users/:id/credits/adjust");
|
||||
|
||||
const backendPerformanceStore = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } });
|
||||
const server = createCloudApiServer({ backendPerformanceStore, env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } });
|
||||
const accessController = { required: true, async authenticate() { return { ok: true, actor: { id: "usr_backend_metrics", role: "admin" }, session: { id: "web_backend_metrics" } }; } };
|
||||
const server = createCloudApiServer({ backendPerformanceStore, accessController, env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } });
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
try {
|
||||
const address = server.address();
|
||||
@@ -37,7 +37,6 @@ test("backend performance store templates routes and appends Cloud API metrics t
|
||||
await new Promise<void>((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("backend performance store exposes low-cardinality request evidence for RUM alignment", () => {
|
||||
const store = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } });
|
||||
const context = store.createManualContext({ route: "/v1/workbench/sessions/ses_secret/messages", method: "GET" });
|
||||
@@ -79,10 +78,9 @@ test("backend performance records Code Agent trace handler phases without high-c
|
||||
const runId = "run_backend_phase";
|
||||
const commandId = "cmd_backend_phase";
|
||||
const traceStore = createCodeAgentTraceStore();
|
||||
const codeAgentChatResults = createCodeAgentChatResultStore();
|
||||
traceStore.append(traceId, { type: "message", role: "assistant", label: "assistant:start", sourceSeq: 1, commandId });
|
||||
traceStore.append(traceId, { type: "result", status: "completed", terminal: true, label: "result:completed", sourceSeq: 2, commandId });
|
||||
codeAgentChatResults.set(traceId, {
|
||||
const projectedResult = {
|
||||
status: "completed",
|
||||
conversationId: "cnv_backend_phase",
|
||||
sessionId: "ses_backend_phase",
|
||||
@@ -107,9 +105,16 @@ test("backend performance records Code Agent trace handler phases without high-c
|
||||
terminalStatus: "completed",
|
||||
eventCount: 2,
|
||||
updatedAt: "2026-06-18T00:00:00.000Z"
|
||||
}
|
||||
});
|
||||
},
|
||||
runnerTrace: traceStore.snapshot(traceId)
|
||||
};
|
||||
const backendPerformanceStore = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } });
|
||||
const workbenchRuntime = {
|
||||
async queryAgentTraceEvents({ traceId: requestedTraceId }: { traceId: string }) {
|
||||
assert.equal(requestedTraceId, traceId);
|
||||
return { events: traceStore.snapshot(traceId).events };
|
||||
}
|
||||
};
|
||||
const accessController = {
|
||||
required: true,
|
||||
async authenticate() {
|
||||
@@ -123,11 +128,11 @@ test("backend performance records Code Agent trace handler phases without high-c
|
||||
projectId: "prj_hwpod_workbench",
|
||||
ownerUserId: "usr_backend_phase",
|
||||
ownerRole: "user",
|
||||
session: {}
|
||||
session: { traceResults: { [traceId]: projectedResult } }
|
||||
};
|
||||
}
|
||||
};
|
||||
const server = createCloudApiServer({ backendPerformanceStore, traceStore, codeAgentChatResults, accessController, env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } });
|
||||
const server = createCloudApiServer({ backendPerformanceStore, traceStore, workbenchRuntime, accessController, env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } });
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
try {
|
||||
const address = server.address();
|
||||
@@ -145,8 +150,8 @@ test("backend performance records Code Agent trace handler phases without high-c
|
||||
const metrics = await fetch(`${baseUrl}/v1/web-performance/metrics`);
|
||||
const text = await metrics.text();
|
||||
assert.equal(metrics.status, 200);
|
||||
assert.match(text, /hwlab_cloud_api_request_phase_duration_seconds_count\{[^}]*route="\/v1\/agent\/traces\/:id"[^}]*phase="trace_project_check"[^}]*\} 1/u);
|
||||
assert.match(text, /hwlab_cloud_api_request_phase_duration_seconds_count\{[^}]*route="\/v1\/agent\/traces\/:id"[^}]*phase="trace_paginate"[^}]*\} 1/u);
|
||||
assert.match(text, /hwlab_cloud_api_request_phase_duration_seconds_count\{[^}]*route="\/v1\/agent\/traces\/:traceId"[^}]*phase="trace_project_check"[^}]*\} 1/u);
|
||||
assert.match(text, /hwlab_cloud_api_request_phase_duration_seconds_count\{[^}]*route="\/v1\/agent\/traces\/:traceId"[^}]*phase="trace_paginate"[^}]*\} 1/u);
|
||||
assert.doesNotMatch(text, /trc_backend_phase_trace|run_backend_phase|cmd_backend_phase|ses_backend_phase|cnv_backend_phase/u);
|
||||
} finally {
|
||||
await new Promise<void>((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
@@ -167,8 +172,7 @@ test("AgentRun adapter records upstream timing through backend performance conte
|
||||
};
|
||||
if (request.method === "POST" && url.pathname === "/api/v1/sessions") return send({ ok: true });
|
||||
if (request.method === "POST" && url.pathname === "/api/v1/runs") return send({ id: "run_backend_perf", status: "pending", backendProfile: "codex", sessionRef: body.sessionRef, resourceBundleRef: body.resourceBundleRef });
|
||||
if (request.method === "POST" && url.pathname === "/api/v1/runs/run_backend_perf/commands") return send({ id: "cmd_backend_perf", runId: "run_backend_perf", state: "pending", type: "turn", seq: 1 });
|
||||
if (request.method === "POST" && url.pathname === "/api/v1/runs/run_backend_perf/runner-jobs") return send({ action: "create-kubernetes-job", runId: "run_backend_perf", commandId: body.commandId, attemptId: "attempt_backend_perf", runnerId: "runner_backend_perf", namespace: "agentrun-v01", jobName: "agentrun-v01-runner-backend-perf" });
|
||||
if (request.method === "POST" && url.pathname === "/api/v1/runs/run_backend_perf/commands") return send({ id: "cmd_backend_perf", runId: "run_backend_perf", state: "pending", type: "turn", seq: 1, dispatchIntent: { id: "dispatch_backend_perf", state: "pending", runnerJobId: "rjob_backend_perf", attemptCount: 0, durable: true } });
|
||||
response.writeHead(404, { "content-type": "application/json" });
|
||||
response.end(`${JSON.stringify({ ok: false, message: `unexpected ${request.method} ${url.pathname}` })}\n`);
|
||||
});
|
||||
@@ -215,11 +219,11 @@ test("AgentRun adapter records upstream timing through backend performance conte
|
||||
}));
|
||||
|
||||
assert.equal(result.status, "running");
|
||||
assert.ok(calls.some((call) => call.path === "/api/v1/runs/run_backend_perf/runner-jobs"));
|
||||
assert.ok(!calls.some((call) => call.path === "/api/v1/runs/run_backend_perf/runner-jobs"));
|
||||
const text = store.metricsText();
|
||||
assert.match(text, /hwlab_cloud_api_upstream_duration_seconds_count\{[^}]*component="agentrun"[^}]*route="\/api\/v1\/runs"[^}]*\} 1/u);
|
||||
assert.match(text, /hwlab_cloud_api_upstream_duration_seconds_count\{[^}]*component="agentrun"[^}]*route="\/api\/v1\/runs\/:id\/commands"[^}]*\} 1/u);
|
||||
assert.match(text, /hwlab_cloud_api_upstream_duration_seconds_count\{[^}]*component="agentrun"[^}]*route="\/api\/v1\/runs\/:id\/runner-jobs"[^}]*\} 1/u);
|
||||
assert.doesNotMatch(text, /runner-jobs/u);
|
||||
assert.doesNotMatch(text, /run_backend_perf|cmd_backend_perf|ses_backend_perf|cnv_backend_perf/u);
|
||||
} finally {
|
||||
await new Promise<void>((resolve, reject) => agentRunServer.close((error) => error ? reject(error) : resolve()));
|
||||
@@ -289,72 +293,3 @@ test("backend performance reports zero projection lag for caught-up terminal tra
|
||||
assert.ok(stuckLine?.endsWith(" 0"));
|
||||
assert.ok(terminalDelayBucketLine?.endsWith(" 1"));
|
||||
});
|
||||
|
||||
test("AgentRun read-side sync advances event cursor without forcing result aggregation before terminal evidence", async () => {
|
||||
const calls: string[] = [];
|
||||
const agentRunServer = createHttpServer(async (request, response) => {
|
||||
const url = new URL(request.url || "/", "http://127.0.0.1");
|
||||
calls.push(`${request.method} ${url.pathname}`);
|
||||
const send = (data: Record<string, unknown>) => {
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(`${JSON.stringify({ ok: true, data })}\n`);
|
||||
};
|
||||
if (request.method === "GET" && url.pathname === "/api/v1/runs/run_read_side/events") {
|
||||
return send({
|
||||
items: [
|
||||
{ seq: 11, type: "assistant_message", runId: "run_read_side", payload: { commandId: "cmd_read_side", text: "working" }, createdAt: "2026-06-19T02:00:01.000Z" },
|
||||
{ seq: 12, type: "tool_call", runId: "run_read_side", payload: { commandId: "cmd_read_side", toolName: "shell" }, createdAt: "2026-06-19T02:00:02.000Z" },
|
||||
],
|
||||
});
|
||||
}
|
||||
if (request.method === "GET" && url.pathname.endsWith("/result")) return send({ terminalStatus: "completed", text: "should not be fetched" });
|
||||
response.writeHead(404, { "content-type": "application/json" });
|
||||
response.end(`${JSON.stringify({ ok: false, message: `unexpected ${request.method} ${url.pathname}` })}\n`);
|
||||
});
|
||||
await new Promise<void>((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve));
|
||||
try {
|
||||
const address = agentRunServer.address();
|
||||
assert.ok(address && typeof address === "object");
|
||||
const store = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03", HWLAB_RUNTIME_LANE: "v03", HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601" } });
|
||||
const traceStore = createCodeAgentTraceStore();
|
||||
const synced = await syncAgentRunChatResult({
|
||||
traceId: "trc_read_side_cursor",
|
||||
currentResult: {
|
||||
status: "running",
|
||||
traceId: "trc_read_side_cursor",
|
||||
agentRun: {
|
||||
adapter: "agentrun-v01",
|
||||
runId: "run_read_side",
|
||||
commandId: "cmd_read_side",
|
||||
traceId: "trc_read_side_cursor",
|
||||
providerTrace: { traceId: "trc_read_side_cursor" },
|
||||
lastSeq: 10,
|
||||
managerUrl: `http://127.0.0.1:${address.port}`,
|
||||
backendProfile: "codex",
|
||||
},
|
||||
},
|
||||
options: {
|
||||
backendPerformanceStore: store,
|
||||
env: {
|
||||
HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS: "2500",
|
||||
},
|
||||
},
|
||||
traceStore,
|
||||
forceResultSync: false,
|
||||
});
|
||||
|
||||
assert.equal(synced.resultSynced, false);
|
||||
assert.equal(synced.eventsRefreshed, true);
|
||||
assert.equal(synced.result.agentRun.lastSeq, 12);
|
||||
assert.ok(calls.some((call) => call === "GET /api/v1/runs/run_read_side/events"));
|
||||
assert.ok(!calls.some((call) => call.endsWith("/result")));
|
||||
const text = store.metricsText();
|
||||
assert.match(text, /hwlab_workbench_projection_cursor_advance_total\{[^}]*status="advanced"[^}]*\} 2/u);
|
||||
assert.match(text, /hwlab_workbench_projector_events_processed_total\{[^}]*event_kind="assistant_message"[^}]*\} 1/u);
|
||||
assert.match(text, /hwlab_workbench_projector_batch_duration_seconds_count\{[^}]*phase="agentrun_events_fetch"[^}]*status="ok"[^}]*\} 1/u);
|
||||
} finally {
|
||||
await new Promise<void>((resolve, reject) => agentRunServer.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
@@ -582,19 +582,19 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp
|
||||
"# HELP hwlab_agentrun_result_timeout_total AgentRun result timeout count by refresh budget.",
|
||||
"# TYPE hwlab_agentrun_result_timeout_total counter",
|
||||
...renderCounters("hwlab_agentrun_result_timeout_total", agentRunResultTimeoutSeries),
|
||||
"# HELP hwlab_workbench_projector_batch_duration_seconds Workbench projector/finalizer batch phase duration in seconds.",
|
||||
"# HELP hwlab_workbench_projector_batch_duration_seconds Workbench Kafka projector batch phase duration in seconds.",
|
||||
"# TYPE hwlab_workbench_projector_batch_duration_seconds histogram",
|
||||
...renderHistogram("hwlab_workbench_projector_batch_duration_seconds", projectorBatchDurationSeries, PROJECTOR_BATCH_BUCKETS_SECONDS),
|
||||
"# HELP hwlab_workbench_projector_candidates_total Workbench projector/finalizer candidate count.",
|
||||
"# HELP hwlab_workbench_projector_candidates_total Workbench Kafka projector candidate count.",
|
||||
"# TYPE hwlab_workbench_projector_candidates_total counter",
|
||||
...renderCounters("hwlab_workbench_projector_candidates_total", projectorCandidatesSeries),
|
||||
"# HELP hwlab_workbench_projector_events_processed_total Workbench projector/finalizer processed event count.",
|
||||
"# HELP hwlab_workbench_projector_events_processed_total Workbench Kafka projector processed event count.",
|
||||
"# TYPE hwlab_workbench_projector_events_processed_total counter",
|
||||
...renderCounters("hwlab_workbench_projector_events_processed_total", projectorEventsProcessedSeries),
|
||||
"# HELP hwlab_workbench_projector_errors_total Workbench projector/finalizer error count.",
|
||||
"# HELP hwlab_workbench_projector_errors_total Workbench Kafka projector error count.",
|
||||
"# TYPE hwlab_workbench_projector_errors_total counter",
|
||||
...renderCounters("hwlab_workbench_projector_errors_total", projectorErrorsSeries),
|
||||
"# HELP hwlab_workbench_projector_last_success_unixtime Last successful Workbench projector/finalizer sync unix time.",
|
||||
"# HELP hwlab_workbench_projector_last_success_unixtime Last successful Workbench Kafka projector transaction unix time.",
|
||||
"# TYPE hwlab_workbench_projector_last_success_unixtime gauge",
|
||||
...renderGauges("hwlab_workbench_projector_last_success_unixtime", projectorLastSuccessSeries),
|
||||
""
|
||||
|
||||
@@ -11,6 +11,12 @@ export async function createCloudApiBunServer(options: any = {}) {
|
||||
installPostgresTransientUnhandledRejectionBoundary({ env, logger });
|
||||
const hwpodNodeWsRegistry = options.hwpodNodeWsRegistry || createHwpodNodeWsRegistry({ ...options, env });
|
||||
const innerServer = createCloudApiServer({ ...options, env, hwpodNodeWsRegistry });
|
||||
try {
|
||||
await innerServer.hwlabStartupReady;
|
||||
} catch (error) {
|
||||
await innerServer.hwlabAbortStartup?.();
|
||||
throw error;
|
||||
}
|
||||
await new Promise((resolve) => innerServer.listen(0, "127.0.0.1", resolve));
|
||||
const innerAddress = innerServer.address();
|
||||
const innerPort = typeof innerAddress === "object" && innerAddress ? innerAddress.port : 0;
|
||||
@@ -19,7 +25,7 @@ export async function createCloudApiBunServer(options: any = {}) {
|
||||
const host = options.host ?? "0.0.0.0";
|
||||
const port = options.port ?? 0;
|
||||
const idleTimeout = parsePositiveInteger(options.idleTimeout ?? env.HWLAB_CLOUD_API_IDLE_TIMEOUT_SECONDS, 120);
|
||||
const server = Bun.serve({
|
||||
const server = (options.bunServe ?? Bun.serve)({
|
||||
hostname: host,
|
||||
port,
|
||||
idleTimeout,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-17-r0; draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
||||
// Responsibility: AgentRun v0.1 adapter, incremental Workbench projection cursor sync, and upstream timing projection for Cloud API observability.
|
||||
// Responsibility: AgentRun command admission and control adapter; Kafka owns event/result projection.
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
@@ -13,32 +13,12 @@ import {
|
||||
safeTraceId,
|
||||
text
|
||||
} from "./server-http-utils.ts";
|
||||
import { currentBackendPerformanceContext } from "./backend-performance.ts";
|
||||
import {
|
||||
DEFAULT_AGENTRUN_MGR_URL,
|
||||
agentRunJson,
|
||||
isAgentRunManagerInternalServiceHost,
|
||||
resolveAgentRunManagerUrl
|
||||
} from "./code-agent-agentrun-runtime.ts";
|
||||
import {
|
||||
agentRunErrorWithPolling,
|
||||
agentRunEventCommandId,
|
||||
agentRunMetricStatusForError,
|
||||
agentRunPollingRetryableError,
|
||||
agentRunPollingRootCause,
|
||||
agentRunProjectionPollCount,
|
||||
agentRunProjectionPollingState,
|
||||
agentRunTerminalConsistencyGap,
|
||||
agentRunTerminalResultRefreshSatisfied,
|
||||
agentRunTerminalStatusFromEvents,
|
||||
agentRunTraceCursorSeq,
|
||||
emitAgentRunProjectionPollingSpan,
|
||||
fetchAgentRunEventsForTrace,
|
||||
measuredAgentRunEventsFetch,
|
||||
measuredAgentRunResultFetch,
|
||||
nonNegativeNumber,
|
||||
recordAgentRunEventsProjectionMetrics
|
||||
} from "./code-agent-agentrun-projection-polling.ts";
|
||||
import {
|
||||
agentRunAlreadyTerminalCancelPayload,
|
||||
agentRunCancelBlockedPayload,
|
||||
@@ -65,7 +45,6 @@ import {
|
||||
agentRunPromptMetadataFromText,
|
||||
messageAuthorityTextValue
|
||||
} from "./code-agent-agentrun-prompt.ts";
|
||||
import { agentRunResultSyncDeferred, buildAgentRunProjectionEventsFetchPlan, buildWorkbenchProjectionStateUpdate } from "./workbench-projection-cursor.ts";
|
||||
import { codeAgentOtelTraceFields, codeAgentOtelTraceContext, emitCodeAgentOtelSpan } from "./otel-trace.ts";
|
||||
import * as agentRunResult from "./code-agent-agentrun-result.ts";
|
||||
|
||||
@@ -700,333 +679,11 @@ function sleepAgentRunDispatchRetry(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, Math.max(0, Number(ms) || 0)));
|
||||
}
|
||||
|
||||
export async function syncAgentRunChatResult({ traceId, currentResult = null, options = {}, traceStore = defaultCodeAgentTraceStore, appendResultEvent = true, refreshEvents = true, forceResultSync = false, retryParams = null }) {
|
||||
const initial = currentResult ?? await loadPersistedAgentRunResult(traceId, options);
|
||||
if (isAgentRunUserCancelSealed(initial)) {
|
||||
const runnerTrace = initial.runnerTrace ?? traceStore.snapshot(traceId);
|
||||
return { result: initial, runnerTrace, found: true, eventsRefreshed: false, resultSynced: false, terminalRefreshSkipped: true, localCancelSealed: true };
|
||||
}
|
||||
const mapped = initial ? await resolveAgentRunTraceCommandMapping({ traceId, mapped: initial, options }) : initial;
|
||||
if (isAgentRunUserCancelSealed(mapped)) {
|
||||
const runnerTrace = mapped.runnerTrace ?? traceStore.snapshot(traceId);
|
||||
return { result: mapped, runnerTrace, found: true, eventsRefreshed: false, resultSynced: false, terminalRefreshSkipped: true, localCancelSealed: true };
|
||||
}
|
||||
const env = options.env ?? process.env;
|
||||
const performanceStore = options.backendPerformanceStore ?? currentBackendPerformanceContext();
|
||||
if (!forceResultSync && agentRunTerminalResultRefreshSatisfied(mapped, traceId)) {
|
||||
const runnerTrace = mapped.runnerTrace ?? traceStore.snapshot(traceId);
|
||||
const polling = agentRunProjectionPollingState({ env, terminalFromEvents: "completed", rootCause: "terminal_result_sealed", terminalConsistencyGap: 0 });
|
||||
emitAgentRunProjectionPollingSpan({ traceId, env, agentRun: mapped?.agentRun, polling, terminalRefreshSkipped: true });
|
||||
return { result: mapped, runnerTrace, found: true, eventsRefreshed: false, resultSynced: false, terminalRefreshSkipped: true, polling };
|
||||
}
|
||||
if (!mapped?.agentRun?.runId || !mapped?.agentRun?.commandId) return { result: currentResult, runnerTrace: traceStore.snapshot(traceId), found: Boolean(currentResult), eventsRefreshed: false, resultSynced: false };
|
||||
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
||||
const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl);
|
||||
const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000);
|
||||
const projectionState = await loadWorkbenchProjectionStateForAgentRun(options.runtimeStore, { traceId, agentRun: mapped.agentRun });
|
||||
const projectionRetryParams = retryParams ?? projectionState?.retryParams ?? null;
|
||||
const projectionPollCount = agentRunProjectionPollCount(projectionState);
|
||||
const fetchPlan = buildAgentRunProjectionEventsFetchPlan({
|
||||
projectionState,
|
||||
agentRun: mapped.agentRun,
|
||||
pageLimit: env.HWLAB_CODE_AGENT_AGENTRUN_EVENTS_PAGE_LIMIT
|
||||
});
|
||||
const previousLastSeq = Number(fetchPlan.afterSeq ?? mapped.agentRun.lastSeq ?? 0);
|
||||
let eventsResponse = null;
|
||||
try {
|
||||
eventsResponse = refreshEvents ? await measuredAgentRunEventsFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapping: { ...mapped.agentRun, lastSeq: fetchPlan.afterSeq, afterSeqOverride: fetchPlan.afterSeq, eventsPageLimit: fetchPlan.limit, traceSummary: mapped.traceSummary, pollCount: projectionPollCount, backoffMs: nonNegativeNumber(projectionState?.backoffMs), rootCause: projectionState?.rootCause ?? null, terminalConsistencyGap: nonNegativeNumber(projectionState?.terminalConsistencyGap) } }) : null;
|
||||
} catch (error) {
|
||||
const polling = agentRunProjectionPollingState({ projectionState, env, error, rootCause: agentRunPollingRootCause(error), terminalConsistencyGap: nonNegativeNumber(projectionState?.terminalConsistencyGap) });
|
||||
await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, {
|
||||
currentState: projectionState,
|
||||
result: mapped,
|
||||
agentRun: { ...mapped.agentRun, lastSeq: previousLastSeq },
|
||||
eventsResponse: null,
|
||||
resultSyncState: projectionState?.resultSyncState ?? null,
|
||||
projectionStatus: "degraded",
|
||||
projectionHealth: "degraded",
|
||||
retryParams: projectionRetryParams,
|
||||
polling,
|
||||
error,
|
||||
now: options.now
|
||||
});
|
||||
throw agentRunErrorWithPolling(error, polling);
|
||||
}
|
||||
if (eventsResponse) appendAgentRunEventsToTrace(traceStore, traceId, eventsResponse.events, mapped.agentRun);
|
||||
const nextLastSeq = eventsResponse ? agentRunTraceCursorSeq(eventsResponse, mapped.agentRun.lastSeq) : mapped.agentRun.lastSeq;
|
||||
const backendProfile = resolveAgentRunBackendProfile(env, {
|
||||
providerProfile: firstNonEmpty(
|
||||
projectionRetryParams?.providerProfile,
|
||||
projectionRetryParams?.codeAgentProviderProfile,
|
||||
projectionRetryParams?.backendProfile,
|
||||
mapped.agentRun.backendProfile
|
||||
)
|
||||
});
|
||||
recordAgentRunEventsProjectionMetrics(performanceStore, eventsResponse, previousLastSeq, nextLastSeq);
|
||||
const terminalFromEvents = agentRunTerminalStatusFromEvents(eventsResponse?.events);
|
||||
const cursorAdvance = Math.max(0, Number(nextLastSeq ?? 0) - Number(previousLastSeq ?? 0));
|
||||
const terminalConsistencyGap = agentRunTerminalConsistencyGap({ mapped, eventsResponse });
|
||||
const projectionPolling = agentRunProjectionPollingState({
|
||||
projectionState,
|
||||
env,
|
||||
pollCount: projectionPollCount,
|
||||
cursorAdvance,
|
||||
terminalFromEvents,
|
||||
rootCause: terminalFromEvents ? "terminal_from_events" : cursorAdvance > 0 ? "cursor_advanced" : "projection_no_progress",
|
||||
terminalConsistencyGap
|
||||
});
|
||||
await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, {
|
||||
currentState: projectionState,
|
||||
result: mapped,
|
||||
agentRun: { ...mapped.agentRun, lastSeq: nextLastSeq },
|
||||
eventsResponse,
|
||||
resultSyncState: terminalFromEvents ? "pending" : null,
|
||||
projectionStatus: terminalFromEvents ? "terminal" : null,
|
||||
polling: projectionPolling,
|
||||
retryParams: projectionRetryParams,
|
||||
now: options.now
|
||||
});
|
||||
const deferResultSync = terminalFromEvents ? false : agentRunResultSyncDeferred({ forceResultSync, options, env });
|
||||
if (!forceResultSync && (deferResultSync || !agentRunCommandResultSyncRequired(mapped, eventsResponse))) {
|
||||
const nextMapping = withAgentRunRunnerJobCount({
|
||||
...mapped.agentRun,
|
||||
lastSeq: nextLastSeq,
|
||||
status: mapped.agentRun.status ?? "running",
|
||||
runStatus: mapped.agentRun.runStatus ?? null,
|
||||
commandState: mapped.agentRun.commandState ?? null,
|
||||
terminalStatus: null,
|
||||
resultSyncState: terminalFromEvents ? "pending" : mapped.agentRun.resultSyncState ?? null,
|
||||
updatedAt: nowIso(options.now),
|
||||
valuesPrinted: false
|
||||
});
|
||||
const payload = decorateAgentRunRunningResult({ base: { ...mapped, status: "running", agentRun: nextMapping, updatedAt: nowIso(options.now) }, mapping: nextMapping, traceStore, traceId });
|
||||
options.codeAgentChatResults?.set?.(traceId, payload);
|
||||
return { result: payload, runnerTrace: payload.runnerTrace ?? traceStore.snapshot(traceId), found: true, eventsRefreshed: Boolean(eventsResponse), resultSynced: false, polling: projectionPolling };
|
||||
}
|
||||
let result;
|
||||
try {
|
||||
result = await measuredAgentRunResultFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapped, eventsResponse });
|
||||
} catch (error) {
|
||||
const retryable = agentRunPollingRetryableError(error);
|
||||
const polling = agentRunProjectionPollingState({ projectionState, env, pollCount: projectionPollCount, cursorAdvance, terminalFromEvents, error, rootCause: agentRunPollingRootCause(error), terminalConsistencyGap });
|
||||
await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, {
|
||||
currentState: projectionState,
|
||||
result: mapped,
|
||||
agentRun: { ...mapped.agentRun, lastSeq: nextLastSeq },
|
||||
eventsResponse,
|
||||
resultSyncState: retryable ? "pending" : agentRunMetricStatusForError(error) === "timeout" ? "timed_out" : "failed",
|
||||
projectionStatus: retryable ? "degraded" : terminalFromEvents ? "terminal" : null,
|
||||
projectionHealth: "degraded",
|
||||
retryParams: projectionRetryParams,
|
||||
polling,
|
||||
error,
|
||||
now: options.now
|
||||
});
|
||||
throw agentRunErrorWithPolling(error, polling);
|
||||
}
|
||||
const nextMapping = withAgentRunRunnerJobCount({
|
||||
...mapped.agentRun,
|
||||
...agentRunResultRefs(result),
|
||||
lastSeq: nextLastSeq,
|
||||
status: result?.status ?? mapped.agentRun.status ?? "running",
|
||||
runStatus: result?.runStatus ?? mapped.agentRun.runStatus ?? null,
|
||||
commandState: result?.commandState ?? mapped.agentRun.commandState ?? null,
|
||||
terminalStatus: result?.terminalStatus ?? mapped.agentRun.terminalStatus ?? null,
|
||||
updatedAt: nowIso(options.now),
|
||||
valuesPrinted: false
|
||||
});
|
||||
const freshSessionRetry = await retryAgentRunTurnAfterFreshSessionFailure({
|
||||
params: projectionRetryParams,
|
||||
options,
|
||||
traceId,
|
||||
traceStore,
|
||||
mapped,
|
||||
failedResult: result,
|
||||
failedMapping: nextMapping,
|
||||
env,
|
||||
managerUrl,
|
||||
fetchImpl,
|
||||
timeoutMs,
|
||||
backendProfile
|
||||
});
|
||||
if (freshSessionRetry) {
|
||||
options.codeAgentChatResults?.set?.(traceId, freshSessionRetry.payload);
|
||||
await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, {
|
||||
currentState: projectionState,
|
||||
result: freshSessionRetry.payload,
|
||||
agentRun: freshSessionRetry.mapping,
|
||||
eventsResponse: null,
|
||||
resultSyncState: null,
|
||||
projectionStatus: "projecting",
|
||||
retryParams: projectionRetryParams,
|
||||
now: options.now
|
||||
});
|
||||
const retrySync = await syncAgentRunChatResult({
|
||||
traceId,
|
||||
currentResult: freshSessionRetry.payload,
|
||||
options,
|
||||
traceStore,
|
||||
appendResultEvent,
|
||||
refreshEvents: true,
|
||||
forceResultSync,
|
||||
retryParams: projectionRetryParams
|
||||
});
|
||||
if (retrySync?.result) {
|
||||
return {
|
||||
...retrySync,
|
||||
found: true,
|
||||
eventsRefreshed: Boolean(eventsResponse) || Boolean(retrySync.eventsRefreshed),
|
||||
freshSessionRetried: true
|
||||
};
|
||||
}
|
||||
return { result: freshSessionRetry.payload, runnerTrace: freshSessionRetry.payload.runnerTrace ?? traceStore.snapshot(traceId), found: true, eventsRefreshed: Boolean(eventsResponse), resultSynced: false, freshSessionRetried: true };
|
||||
}
|
||||
const base = { ...mapped, agentRun: nextMapping, updatedAt: nowIso(options.now) };
|
||||
const payload = agentRunResultToCodeAgentPayload({ base, result, traceStore, traceId, appendResultEvent });
|
||||
options.codeAgentChatResults?.set?.(traceId, payload);
|
||||
const resultPolling = agentRunProjectionPollingState({
|
||||
projectionState,
|
||||
env,
|
||||
pollCount: projectionPollCount,
|
||||
cursorAdvance,
|
||||
terminalFromEvents: isAgentRunTerminalStatus(payload?.status ?? nextMapping.terminalStatus ?? nextMapping.commandState) ? (nextMapping.terminalStatus ?? payload?.status) : terminalFromEvents,
|
||||
rootCause: isAgentRunTerminalStatus(payload?.status ?? nextMapping.terminalStatus ?? nextMapping.commandState) ? "terminal_result_synced" : "result_synced",
|
||||
terminalConsistencyGap: agentRunTerminalConsistencyGap({ mapped: payload, eventsResponse, result })
|
||||
});
|
||||
await writeWorkbenchProjectionStateForAgentRun(options.runtimeStore, {
|
||||
currentState: projectionState,
|
||||
result: payload,
|
||||
agentRun: nextMapping,
|
||||
eventsResponse,
|
||||
resultSyncState: "synced",
|
||||
projectionStatus: isAgentRunTerminalStatus(payload?.status) ? "terminal" : null,
|
||||
polling: resultPolling,
|
||||
retryParams: projectionRetryParams,
|
||||
now: options.now
|
||||
});
|
||||
return { result: payload, runnerTrace: payload.runnerTrace ?? traceStore.snapshot(traceId), found: true, eventsRefreshed: Boolean(eventsResponse), resultSynced: true, polling: resultPolling };
|
||||
}
|
||||
|
||||
async function loadWorkbenchProjectionStateForAgentRun(runtimeStore, { traceId, agentRun = {} } = {}) {
|
||||
if (!runtimeStore || typeof runtimeStore.getWorkbenchProjectionState !== "function") return null;
|
||||
try {
|
||||
const result = await runtimeStore.getWorkbenchProjectionState({ traceId });
|
||||
const state = result?.projectionState ?? null;
|
||||
if (!state) return null;
|
||||
if (text(state.runId ?? state.sourceRunId) !== text(agentRun.runId)) return null;
|
||||
if (text(state.commandId ?? state.sourceCommandId) !== text(agentRun.commandId)) return null;
|
||||
return state;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function writeWorkbenchProjectionStateForAgentRun(runtimeStore, params = {}) {
|
||||
if (!runtimeStore || typeof runtimeStore.writeWorkbenchProjectionState !== "function") return null;
|
||||
const projectionState = buildWorkbenchProjectionStateUpdate(params);
|
||||
if (!projectionState.traceId || !projectionState.runId || !projectionState.commandId) return null;
|
||||
return await runtimeStore.writeWorkbenchProjectionState({ projectionState });
|
||||
}
|
||||
|
||||
function agentRunCommandResultSyncRequired(mapped = {}, eventsResponse = null) {
|
||||
const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : {};
|
||||
if (agentRunTerminalResultRefreshSatisfied(mapped, mapped?.traceId)) return true;
|
||||
if (isAgentRunTerminalStatus(agentRun.terminalStatus ?? agentRun.commandState)) return true;
|
||||
return Boolean(agentRunTerminalStatusFromEvents(eventsResponse?.events));
|
||||
}
|
||||
|
||||
function isAgentRunTerminalStatus(value) {
|
||||
const status = normalizeAgentRunStatus(value);
|
||||
return status === "timeout" || TERMINAL_RUN_STATUSES.has(status);
|
||||
}
|
||||
|
||||
function normalizeAgentRunStatus(value) {
|
||||
const status = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-");
|
||||
return status === "cancelled" ? "canceled" : status;
|
||||
}
|
||||
|
||||
function isAgentRunUserCancelSealed(payload = null) {
|
||||
if (!payload || typeof payload !== "object") return false;
|
||||
const status = normalizeAgentRunStatus(payload.status);
|
||||
const cancelStatus = normalizeAgentRunStatus(payload.cancelStatus ?? payload.cancelDisposition?.status);
|
||||
const terminalStatus = normalizeAgentRunStatus(payload.agentRun?.terminalStatus ?? payload.agentRun?.commandState ?? payload.agentRun?.status);
|
||||
return payload.canceled === true || status === "canceled" || cancelStatus === "canceled" || terminalStatus === "canceled";
|
||||
}
|
||||
|
||||
async function resolveAgentRunTraceCommandMapping({ traceId, mapped, options = {} }) {
|
||||
const safeId = safeTraceId(traceId);
|
||||
const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : null;
|
||||
if (!safeId || !agentRun?.runId) return mapped;
|
||||
const agentRunTraceId = safeTraceId(agentRun.traceId);
|
||||
const providerTraceId = safeTraceId(agentRun.providerTrace?.traceId);
|
||||
const traceSummaryCommandId = text(mapped?.traceSummary?.agentRun?.commandId);
|
||||
const commandId = text(agentRun.commandId);
|
||||
const traceFieldsMatch = agentRunTraceId === safeId && providerTraceId === safeId && (!traceSummaryCommandId || traceSummaryCommandId === commandId);
|
||||
if (traceFieldsMatch && (mapped.status === "running" || agentRunTerminalResultRefreshSatisfied(mapped, safeId))) return mapped;
|
||||
const env = options.env ?? process.env;
|
||||
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
||||
const managerUrl = resolveAgentRunManagerUrl(env, agentRun.managerUrl);
|
||||
const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000);
|
||||
const command = await findAgentRunCommandForTrace({ fetchImpl, managerUrl, timeoutMs, env, runId: agentRun.runId, traceId: safeId });
|
||||
if (!command?.id) {
|
||||
throw Object.assign(new Error(`AgentRun command registry has no command for ${safeId} in run ${agentRun.runId}`), {
|
||||
code: "agentrun_trace_command_not_found",
|
||||
statusCode: 404,
|
||||
traceId: safeId,
|
||||
runId: agentRun.runId
|
||||
});
|
||||
}
|
||||
if (command.id === commandId) {
|
||||
return {
|
||||
...mapped,
|
||||
finalResponse: null,
|
||||
traceSummary: null,
|
||||
agentRun: {
|
||||
...agentRun,
|
||||
traceId: safeId,
|
||||
lastSeq: 0,
|
||||
providerTrace: null,
|
||||
commandState: command.state ?? agentRun.commandState ?? null,
|
||||
updatedAt: nowIso(options.now),
|
||||
valuesPrinted: false
|
||||
}
|
||||
};
|
||||
}
|
||||
return {
|
||||
...mapped,
|
||||
finalResponse: null,
|
||||
traceSummary: null,
|
||||
agentRun: {
|
||||
...agentRun,
|
||||
commandId: command.id,
|
||||
traceId: safeId,
|
||||
lastSeq: 0,
|
||||
providerTrace: null,
|
||||
commandState: command.state ?? agentRun.commandState ?? null,
|
||||
updatedAt: nowIso(options.now),
|
||||
valuesPrinted: false
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export async function refreshAgentRunTrace({ traceId, result = null, options = {}, traceStore = defaultCodeAgentTraceStore }) {
|
||||
const initial = result ?? await loadPersistedAgentRunResult(traceId, options);
|
||||
const mapped = initial ? await resolveAgentRunTraceCommandMapping({ traceId, mapped: initial, options }) : initial;
|
||||
if (!mapped?.agentRun?.runId) return traceStore.snapshot(traceId);
|
||||
const env = options.env ?? process.env;
|
||||
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
||||
const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl);
|
||||
const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000);
|
||||
const eventsResponse = await fetchAgentRunEventsForTrace({ fetchImpl, managerUrl, timeoutMs, env, mapping: { ...mapped.agentRun, traceSummary: mapped.traceSummary } });
|
||||
const events = eventsResponse.events;
|
||||
appendAgentRunEventsToTrace(traceStore, traceId, events, mapped.agentRun);
|
||||
const lastSeq = agentRunTraceCursorSeq(eventsResponse, mapped.agentRun.lastSeq);
|
||||
if (lastSeq !== Number(mapped.agentRun.lastSeq ?? 0)) {
|
||||
options.codeAgentChatResults?.set?.(traceId, { ...mapped, agentRun: { ...mapped.agentRun, lastSeq, updatedAt: nowIso(options.now) } });
|
||||
}
|
||||
return traceStore.snapshot(traceId);
|
||||
}
|
||||
|
||||
export async function cancelAgentRunChatTurn({ traceId, currentResult = null, options = {}, traceStore = defaultCodeAgentTraceStore }) {
|
||||
let mapped = currentResult ?? await loadPersistedAgentRunResult(traceId, options);
|
||||
if (!mapped?.agentRun?.commandId) return null;
|
||||
@@ -1043,12 +700,6 @@ export async function cancelAgentRunChatTurn({ traceId, currentResult = null, op
|
||||
};
|
||||
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
||||
const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl);
|
||||
try {
|
||||
const synced = await syncAgentRunChatResult({ traceId, currentResult: mapped, options: cancelOptions, traceStore, forceResultSync: true });
|
||||
mapped = synced?.result ?? synced ?? mapped;
|
||||
} catch {
|
||||
// Cancel must still expose the command cancel disposition; result refresh failures are reported by the normal trace/result paths.
|
||||
}
|
||||
const terminalStatus = agentRunCancelTerminalStatus(mapped);
|
||||
if (terminalStatus) return agentRunAlreadyTerminalCancelPayload({ traceId, payload: mapped, terminalStatus, options: cancelOptions, traceStore });
|
||||
const cancelPath = `/api/v1/commands/${encodeURIComponent(mapped.agentRun.commandId)}/cancel`;
|
||||
@@ -1155,7 +806,7 @@ export async function steerAgentRunChatTurn({ traceId, currentResult = null, par
|
||||
accepted: true,
|
||||
status: "running",
|
||||
shortConnection: true,
|
||||
controlSemantics: "steer-active-turn-and-poll-target-trace",
|
||||
controlSemantics: "steer-active-turn-and-project-sse",
|
||||
route: "/v1/agent/chat/steer",
|
||||
traceId,
|
||||
targetTraceId: traceId,
|
||||
@@ -1163,8 +814,8 @@ export async function steerAgentRunChatTurn({ traceId, currentResult = null, par
|
||||
conversationId: safeConversationId(mapped.conversationId ?? params.conversationId) || null,
|
||||
sessionId: safeSessionId(mapped.sessionId ?? params.sessionId) || null,
|
||||
threadId: safeOpaqueId(mapped.threadId ?? params.threadId) || null,
|
||||
resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`,
|
||||
traceUrl: `/v1/agent/traces/${encodeURIComponent(traceId)}`,
|
||||
workbenchEventsUrl: `/v1/workbench/events?sessionId=${encodeURIComponent(safeSessionId(mapped.sessionId ?? params.sessionId) || "")}&traceId=${encodeURIComponent(traceId)}`,
|
||||
agentRun: {
|
||||
...agentRun,
|
||||
steerCommandId,
|
||||
@@ -1243,18 +894,6 @@ function agentRunSeedFromSession(session, traceId) {
|
||||
return topLevelAgentRun?.runId ? topLevelAgentRun : null;
|
||||
}
|
||||
|
||||
async function findAgentRunCommandForTrace({ fetchImpl, managerUrl, timeoutMs, env, runId, traceId }) {
|
||||
if (!runId || !safeTraceId(traceId)) return null;
|
||||
const commands = await agentRunJson(fetchImpl, managerUrl, `/api/v1/runs/${encodeURIComponent(runId)}/commands?afterSeq=0&limit=100`, { method: "GET", timeoutMs, env });
|
||||
return (Array.isArray(commands?.items) ? commands.items : []).find((item) => agentRunCommandMatchesTrace(item, traceId)) ?? null;
|
||||
}
|
||||
|
||||
function agentRunCommandMatchesTrace(command, traceId) {
|
||||
const payload = command?.payload && typeof command.payload === "object" ? command.payload : {};
|
||||
return command?.idempotencyKey === traceId || payload.traceId === traceId;
|
||||
}
|
||||
|
||||
|
||||
function withoutUserBillingReservation(value = {}) {
|
||||
if (!value || typeof value !== "object") return value;
|
||||
const { userBillingReservation, ...rest } = value;
|
||||
@@ -1353,126 +992,6 @@ function newSessionIdAfterEviction(baseSessionId, traceId) {
|
||||
return baseSessionId + "-reset-" + profile;
|
||||
}
|
||||
|
||||
async function retryAgentRunTurnAfterFreshSessionFailure({
|
||||
params = null,
|
||||
options = {},
|
||||
traceId,
|
||||
traceStore = defaultCodeAgentTraceStore,
|
||||
mapped = {},
|
||||
failedResult = {},
|
||||
failedMapping = {},
|
||||
env = process.env,
|
||||
managerUrl,
|
||||
fetchImpl,
|
||||
timeoutMs,
|
||||
backendProfile
|
||||
} = {}) {
|
||||
const failure = agentRunFreshSessionFailure(failedResult);
|
||||
if (!failure.requiresFreshSession) return null;
|
||||
if (mapped?.agentRun?.freshSessionRetryOf || failedMapping?.freshSessionRetryOf) return null;
|
||||
const promptText = messageAuthorityTextValue(params?.message ?? params?.prompt ?? params?.text ?? "");
|
||||
if (!promptText) {
|
||||
traceStore.append(traceId, agentRunTraceEvent({
|
||||
type: "backend",
|
||||
status: "failed",
|
||||
label: "agentrun:session-reset:skipped",
|
||||
errorCode: "agentrun_retry_prompt_missing",
|
||||
message: "AgentRun session storage was evicted, but hwlab-cloud-api cannot retry the current turn because the original prompt text is unavailable in this sync path.",
|
||||
runId: failedMapping?.runId ?? mapped?.agentRun?.runId ?? null,
|
||||
commandId: failedMapping?.commandId ?? mapped?.agentRun?.commandId ?? null,
|
||||
terminal: true,
|
||||
valuesPrinted: false
|
||||
}, failedMapping || mapped?.agentRun || backendProfile));
|
||||
return null;
|
||||
}
|
||||
const baseSessionId = scopedAgentRunSessionIdForParams(params, traceId);
|
||||
const sessionId = newSessionIdAfterEviction(baseSessionId, traceId);
|
||||
const retryParams = paramsWithAgentRunSessionThread(params, null);
|
||||
const startedAt = nowIso(options.now);
|
||||
const previous = {
|
||||
runId: failedMapping?.runId ?? mapped?.agentRun?.runId ?? null,
|
||||
commandId: failedMapping?.commandId ?? mapped?.agentRun?.commandId ?? null,
|
||||
sessionId: failedMapping?.sessionId ?? mapped?.agentRun?.sessionId ?? baseSessionId,
|
||||
threadId: failedMapping?.threadId ?? mapped?.agentRun?.threadId ?? mapped?.threadId ?? null,
|
||||
failureKind: failure.code,
|
||||
failureMessage: failure.message,
|
||||
valuesPrinted: false
|
||||
};
|
||||
traceStore.append(traceId, agentRunTraceEvent({
|
||||
type: "backend",
|
||||
status: "running",
|
||||
label: "agentrun:session-reset:retry-current-turn",
|
||||
message: "AgentRun session storage/thread resume failed; hwlab-cloud-api is retrying the same user turn with a fresh AgentRun session and threadId=null.",
|
||||
runId: previous.runId,
|
||||
commandId: previous.commandId,
|
||||
sessionId,
|
||||
previousSessionId: previous.sessionId,
|
||||
previousThreadId: previous.threadId,
|
||||
errorCode: failure.code,
|
||||
backendProfile,
|
||||
waitingFor: "agentrun-run-create",
|
||||
valuesPrinted: false
|
||||
}, failedMapping || mapped?.agentRun || backendProfile));
|
||||
await ensureAgentRunSessionPersistent({ fetchImpl, managerUrl, sessionId, env, traceId, backendProfile, traceStore });
|
||||
const toolCapabilities = await resolveToolCapabilities({ params: retryParams, options });
|
||||
const runInput = buildAgentRunCreateRunInput({ params: retryParams, env, traceId, backendProfile, sessionId, toolCapabilities });
|
||||
const run = await agentRunDispatchJson({ fetchImpl, managerUrl, path: "/api/v1/runs", method: "POST", body: runInput, timeoutMs, env, traceStore, traceId, backendProfile, stage: "run-create-after-session-reset" });
|
||||
const runId = requiredString(run?.id, "run.id");
|
||||
traceStore.append(traceId, agentRunTraceEvent({
|
||||
type: "backend",
|
||||
status: "running",
|
||||
label: "agentrun:run:created-after-session-reset",
|
||||
message: "AgentRun fresh-session retry run " + runId + " created for the current user turn.",
|
||||
runId,
|
||||
backendProfile,
|
||||
waitingFor: "agentrun-command-create",
|
||||
valuesPrinted: false
|
||||
}, backendProfile));
|
||||
const dispatchInput = buildAgentRunDurableDispatchInput({ env, traceId, backendProfile });
|
||||
const commandInput = buildAgentRunCommandInput({ params: retryParams, traceId, backendProfile, sessionId, dispatchInput });
|
||||
const command = await agentRunDispatchJson({ fetchImpl, managerUrl, path: "/api/v1/runs/" + encodeURIComponent(runId) + "/commands", method: "POST", body: commandInput, timeoutMs, env, traceStore, traceId, backendProfile, runId, stage: "command-create-after-session-reset" });
|
||||
const commandId = requiredString(command?.id, "command.id");
|
||||
const dispatchIntent = requireAgentRunDurableDispatchIntent(command?.dispatchIntent);
|
||||
traceStore.append(traceId, agentRunTraceEvent({
|
||||
type: "backend",
|
||||
status: "running",
|
||||
label: "agentrun:command:created-after-session-reset",
|
||||
message: "AgentRun fresh-session retry command " + commandId + " and durable dispatch intent " + dispatchIntent.id + " were admitted atomically.",
|
||||
runId,
|
||||
commandId,
|
||||
backendProfile,
|
||||
dispatchIntentId: dispatchIntent.id,
|
||||
runnerJobId: dispatchIntent.runnerJobId,
|
||||
waitingFor: "agentrun-result",
|
||||
valuesPrinted: false
|
||||
}, backendProfile));
|
||||
const mapping = {
|
||||
...agentRunMapping({ env, managerUrl, backendProfile, run, command, dispatchIntent, traceId, startedAt, params: retryParams }),
|
||||
freshSessionRetryAttempt: 1,
|
||||
freshSessionRetryOf: previous,
|
||||
status: `dispatch-intent-${dispatchIntent.state}`,
|
||||
valuesPrinted: false
|
||||
};
|
||||
traceStore.append(traceId, agentRunTraceEvent({
|
||||
type: "backend",
|
||||
status: "running",
|
||||
label: "agentrun:dispatch-intent:admitted-after-session-reset",
|
||||
message: "AgentRun owns the durable fresh-session runner dispatch; HWLAB has no process-local runner Job creation window.",
|
||||
runId: mapping.runId,
|
||||
commandId: mapping.commandId,
|
||||
backendProfile,
|
||||
dispatchIntentId: dispatchIntent.id,
|
||||
dispatchIntentState: dispatchIntent.state,
|
||||
runnerJobId: dispatchIntent.runnerJobId,
|
||||
waitingFor: "agentrun-result",
|
||||
valuesPrinted: false
|
||||
}, mapping));
|
||||
const base = initialAgentRunChatResult({ params: retryParams, options, traceId });
|
||||
const payload = decorateAgentRunRunningResult({ base: { ...base, freshSessionRetryOf: previous }, mapping, traceStore, traceId });
|
||||
return { payload, mapping };
|
||||
}
|
||||
|
||||
|
||||
function buildAgentRunCreateRunInput({ params, env, traceId, backendProfile, sessionId, toolCapabilities = null }) {
|
||||
const commitId = requireAgentRunSourceCommit(env);
|
||||
const runtimeAuthority = requireAgentRunRuntimeAuthority(env);
|
||||
@@ -1536,7 +1055,6 @@ function buildAgentRunCreateRunInput({ params, env, traceId, backendProfile, ses
|
||||
kind: "hwlab-cloud-api",
|
||||
traceId,
|
||||
...codeAgentOtelTraceFields(traceId, env),
|
||||
resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`,
|
||||
traceUrl: `/v1/agent/traces/${encodeURIComponent(traceId)}`,
|
||||
valuesPrinted: false
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { cancelAgentRunChatTurn, syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts";
|
||||
import { cancelAgentRunChatTurn } from "./code-agent-agentrun-adapter.ts";
|
||||
import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
||||
|
||||
test("AgentRun cancel downstream failure returns visible blocked disposition instead of throwing", async () => {
|
||||
@@ -54,44 +54,3 @@ test("AgentRun cancel downstream failure returns visible blocked disposition ins
|
||||
assert.equal(codeAgentChatResults.get(traceId)?.cancelDisposition?.code, "command_not_cancelable");
|
||||
assert.ok(traceStore.snapshot(traceId).events.some((event) => event.label === "agentrun:cancel:blocked" && event.status === "blocked"));
|
||||
});
|
||||
|
||||
test("AgentRun result sync preserves a user-canceled payload instead of accepting late completed result", async () => {
|
||||
const traceId = "trc_cancel_sealed_result";
|
||||
const traceStore = createCodeAgentTraceStore();
|
||||
const currentResult = {
|
||||
traceId,
|
||||
status: "canceled",
|
||||
canceled: true,
|
||||
cancelStatus: "canceled",
|
||||
cancelDisposition: { status: "canceled", forwarded: true, terminal: true, traceId, valuesPrinted: false },
|
||||
finalResponse: { text: "hwlab-user-cancel", status: "canceled", traceId, valuesPrinted: false },
|
||||
agentRun: {
|
||||
runId: "run_cancel_sealed_result",
|
||||
commandId: "cmd_cancel_sealed_result",
|
||||
managerUrl: "http://127.0.0.1:65535",
|
||||
status: "cancelled",
|
||||
commandState: "cancelled",
|
||||
terminalStatus: "cancelled",
|
||||
traceId,
|
||||
valuesPrinted: false
|
||||
},
|
||||
valuesPrinted: false
|
||||
};
|
||||
|
||||
const synced = await syncAgentRunChatResult({
|
||||
traceId,
|
||||
currentResult,
|
||||
traceStore,
|
||||
options: {
|
||||
env: { HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1" },
|
||||
fetchImpl() {
|
||||
throw new Error("late AgentRun result must not be fetched after HWLAB user cancel is sealed");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
assert.equal(synced.localCancelSealed, true);
|
||||
assert.equal(synced.resultSynced, false);
|
||||
assert.equal(synced.result.status, "canceled");
|
||||
assert.equal(synced.result.finalResponse.text, "hwlab-user-cancel");
|
||||
});
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { submitAgentRunChatTurn, syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts";
|
||||
import { submitAgentRunChatTurn } from "./code-agent-agentrun-adapter.ts";
|
||||
import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
||||
|
||||
const sourceCommit = "0123456789abcdef0123456789abcdef01234567";
|
||||
@@ -154,70 +154,6 @@ for (const terminal of [
|
||||
});
|
||||
}
|
||||
|
||||
test("gpt.pika session-eviction retry uses the same durable dispatch admission", async () => {
|
||||
const traceId = "trc_gpt_pika_durable_retry";
|
||||
const calls = [];
|
||||
const traceStore = createCodeAgentTraceStore();
|
||||
const synced = await syncAgentRunChatResult({
|
||||
traceId,
|
||||
traceStore,
|
||||
refreshEvents: false,
|
||||
forceResultSync: true,
|
||||
retryParams: {
|
||||
ownerUserId: "usr_gpt_pika_retry",
|
||||
projectId: "pikasTech/HWLAB",
|
||||
conversationId: "cnv_gpt_pika_retry",
|
||||
sessionId: "ses_gpt_pika_retry",
|
||||
threadId: "thread_evicted",
|
||||
message: "retry the evicted gpt.pika turn",
|
||||
providerProfile: "gpt.pika"
|
||||
},
|
||||
currentResult: {
|
||||
traceId,
|
||||
status: "running",
|
||||
conversationId: "cnv_gpt_pika_retry",
|
||||
sessionId: "ses_gpt_pika_retry",
|
||||
threadId: "thread_evicted",
|
||||
providerProfile: "gpt.pika",
|
||||
agentRun: {
|
||||
adapter: "agentrun-v01",
|
||||
runId: "run_gpt_pika_evicted",
|
||||
commandId: "cmd_gpt_pika_evicted",
|
||||
sessionId: "ses_agentrun_gpt_pika_retry",
|
||||
threadId: "thread_evicted",
|
||||
backendProfile: "gpt.pika",
|
||||
managerUrl: "http://127.0.0.1:65535",
|
||||
status: "running",
|
||||
traceId,
|
||||
lastSeq: 0,
|
||||
providerTrace: { traceId, valuesPrinted: false },
|
||||
valuesPrinted: false
|
||||
},
|
||||
valuesPrinted: false
|
||||
},
|
||||
options: {
|
||||
env: agentRunTestEnv(),
|
||||
fetchImpl: createAgentRunRetryFetch(calls, traceId)
|
||||
}
|
||||
});
|
||||
|
||||
assert.equal(synced.freshSessionRetried, true);
|
||||
assert.equal(synced.result.status, "completed");
|
||||
assert.equal(synced.result.agentRun.runId, "run_gpt_pika_retry");
|
||||
assert.equal(synced.result.agentRun.commandId, "cmd_gpt_pika_retry");
|
||||
assert.equal(synced.result.agentRun.freshSessionRetryOf.runId, "run_gpt_pika_evicted");
|
||||
const retryCommand = calls.find((call) => call.path === "/api/v1/runs/run_gpt_pika_retry/commands");
|
||||
assert.ok(retryCommand);
|
||||
assert.equal(retryCommand.body.idempotencyKey, traceId);
|
||||
assert.equal(retryCommand.body.payload.threadId, null);
|
||||
assert.equal(retryCommand.body.dispatch.kind, "kubernetes-job");
|
||||
assert.equal(Object.hasOwn(retryCommand.body.dispatch.input, "commandId"), false);
|
||||
assert.equal(retryCommand.body.dispatch.input.transientEnv.every((entry) => entry.sensitive === false), true);
|
||||
assert.equal(JSON.stringify(retryCommand.body).includes("HWLAB_API_KEY"), false);
|
||||
assert.equal(calls.some((call) => call.path.endsWith("/runner-jobs")), false);
|
||||
assert.ok(traceStore.snapshot(traceId).events.some((event) => event.label === "agentrun:dispatch-intent:admitted-after-session-reset"));
|
||||
});
|
||||
|
||||
function agentRunTestEnv() {
|
||||
return {
|
||||
AGENTRUN_MGR_URL: "http://127.0.0.1:65535",
|
||||
@@ -273,89 +209,3 @@ function createAgentRunFetch(calls, { dispatchIntent }) {
|
||||
throw new Error(`unexpected AgentRun call ${method} ${parsed.pathname}`);
|
||||
};
|
||||
}
|
||||
|
||||
function createAgentRunRetryFetch(calls, traceId) {
|
||||
return async (url, init = {}) => {
|
||||
const parsed = new URL(url);
|
||||
const method = init.method ?? "GET";
|
||||
const body = init.body ? JSON.parse(String(init.body)) : null;
|
||||
calls.push({ method, path: parsed.pathname, body });
|
||||
const send = (data) => new Response(JSON.stringify({ ok: true, data }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" }
|
||||
});
|
||||
if (method === "GET" && parsed.pathname === "/api/v1/runs/run_gpt_pika_evicted/commands/cmd_gpt_pika_evicted/result") {
|
||||
return send({
|
||||
runId: "run_gpt_pika_evicted",
|
||||
commandId: "cmd_gpt_pika_evicted",
|
||||
status: "failed",
|
||||
runStatus: "failed",
|
||||
commandState: "failed",
|
||||
terminalStatus: "failed",
|
||||
failureKind: "session-store-evicted",
|
||||
failureMessage: "session storage was evicted while resuming the thread",
|
||||
completed: false,
|
||||
lastSeq: 1
|
||||
});
|
||||
}
|
||||
if (method === "POST" && parsed.pathname === "/api/v1/sessions") {
|
||||
return send({ session: { id: body.sessionId, metadata: {} } });
|
||||
}
|
||||
if (method === "POST" && parsed.pathname === "/api/v1/runs") {
|
||||
return send({
|
||||
id: "run_gpt_pika_retry",
|
||||
status: "pending",
|
||||
sessionRef: body.sessionRef,
|
||||
backendProfile: body.backendProfile
|
||||
});
|
||||
}
|
||||
if (method === "POST" && parsed.pathname === "/api/v1/runs/run_gpt_pika_retry/commands") {
|
||||
return send({
|
||||
id: "cmd_gpt_pika_retry",
|
||||
runId: "run_gpt_pika_retry",
|
||||
state: "pending",
|
||||
type: "turn",
|
||||
seq: 1,
|
||||
dispatchIntent: {
|
||||
id: "dispatch_gpt_pika_retry",
|
||||
state: "pending",
|
||||
runnerJobId: "rjob_gpt_pika_retry",
|
||||
attemptCount: 0,
|
||||
durable: true
|
||||
}
|
||||
});
|
||||
}
|
||||
if (method === "GET" && parsed.pathname === "/api/v1/runs/run_gpt_pika_retry/events") {
|
||||
return send({ items: [] });
|
||||
}
|
||||
if (method === "GET" && parsed.pathname === "/api/v1/runs/run_gpt_pika_retry/commands") {
|
||||
return send({ items: [{
|
||||
id: "cmd_gpt_pika_retry",
|
||||
runId: "run_gpt_pika_retry",
|
||||
state: "completed",
|
||||
idempotencyKey: traceId,
|
||||
payload: { traceId }
|
||||
}] });
|
||||
}
|
||||
if (method === "GET" && parsed.pathname === "/api/v1/runs/run_gpt_pika_retry/commands/cmd_gpt_pika_retry/result") {
|
||||
return send({
|
||||
runId: "run_gpt_pika_retry",
|
||||
commandId: "cmd_gpt_pika_retry",
|
||||
status: "completed",
|
||||
runStatus: "completed",
|
||||
commandState: "completed",
|
||||
terminalStatus: "completed",
|
||||
completed: true,
|
||||
reply: "GPT_PIKA_RETRY_OK",
|
||||
lastSeq: 2,
|
||||
sessionRef: {
|
||||
sessionId: "ses_agentrun_gpt_pika_retry-reset-trcgptpikadu",
|
||||
conversationId: "cnv_gpt_pika_retry",
|
||||
threadId: "thread_gpt_pika_retry"
|
||||
},
|
||||
traceId
|
||||
});
|
||||
}
|
||||
throw new Error(`unexpected AgentRun retry call ${method} ${parsed.pathname}`);
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,350 +0,0 @@
|
||||
// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
||||
// Responsibility: AgentRun projection polling/backoff I/O, durable polling state, and projection_sync observability attributes.
|
||||
|
||||
import { agentRunJson } from "./code-agent-agentrun-runtime.ts";
|
||||
import { emitCodeAgentOtelSpan } from "./otel-trace.ts";
|
||||
import { parsePositiveInteger, safeTraceId, text } from "./server-http-utils.ts";
|
||||
|
||||
const DEFAULT_AGENTRUN_PROJECTION_SYNC_BACKOFF_INITIAL_MS = 1_000;
|
||||
const DEFAULT_AGENTRUN_PROJECTION_SYNC_BACKOFF_MAX_MS = 30_000;
|
||||
const AGENTRUN_TERMINAL_STATUSES = new Set(["completed", "failed", "blocked", "cancelled", "canceled", "timeout"]);
|
||||
|
||||
export function agentRunProjectionPollCount(projectionState = null) {
|
||||
return nonNegativeNumber(projectionState?.pollCount) + 1;
|
||||
}
|
||||
|
||||
export function agentRunProjectionPollingState({ projectionState = null, env = process.env, pollCount = null, cursorAdvance = 0, terminalFromEvents = null, error = null, rootCause = null, terminalConsistencyGap = 0 } = {}) {
|
||||
const retryable = agentRunPollingRetryableError(error);
|
||||
const noProgress = !error && !terminalFromEvents && Number(cursorAdvance ?? 0) <= 0;
|
||||
const previousNoProgress = nonNegativeNumber(projectionState?.noProgressPollCount);
|
||||
const noProgressPollCount = retryable || noProgress ? previousNoProgress + 1 : 0;
|
||||
const backoffReason = retryable ? "provider_retry_or_rate_limit" : noProgress ? "projection_no_progress" : null;
|
||||
const backoffMs = backoffReason ? agentRunProjectionBackoffMs(noProgressPollCount, env) : 0;
|
||||
return {
|
||||
pollCount: Number.isInteger(Number(pollCount)) && Number(pollCount) > 0 ? Math.floor(Number(pollCount)) : agentRunProjectionPollCount(projectionState),
|
||||
noProgressPollCount,
|
||||
backoffMs,
|
||||
backoffReason,
|
||||
rootCause: boundedAgentRunRootCause(rootCause ?? agentRunPollingRootCause(error) ?? backoffReason ?? "projection_sync"),
|
||||
terminalConsistencyGap: nonNegativeNumber(terminalConsistencyGap),
|
||||
nextRetryAt: backoffMs > 0 ? new Date(Date.now() + backoffMs).toISOString() : null,
|
||||
valuesPrinted: false
|
||||
};
|
||||
}
|
||||
|
||||
export function agentRunPollingRetryableError(error = null) {
|
||||
const statusCode = Number(error?.statusCode ?? error?.agentRunError?.statusCode ?? 0);
|
||||
if ([408, 425, 429, 500, 502, 503, 504].includes(statusCode)) return true;
|
||||
const code = String(error?.code ?? error?.agentRunError?.failureKind ?? "").toLowerCase();
|
||||
return error?.retryable === true || code === "agentrun_timeout" || /rate.?limit|retry|timeout|temporar|unavailable|overload/u.test(code);
|
||||
}
|
||||
|
||||
export function agentRunPollingRootCause(error = null) {
|
||||
if (!error) return null;
|
||||
const statusCode = Number(error?.statusCode ?? error?.agentRunError?.statusCode ?? 0);
|
||||
if (statusCode === 429) return "provider_rate_limited";
|
||||
if ([408, 425, 500, 502, 503, 504].includes(statusCode)) return `provider_retry_http_${statusCode}`;
|
||||
return error?.code ?? error?.agentRunError?.failureKind ?? error?.name ?? "agentrun_sync_failed";
|
||||
}
|
||||
|
||||
export function agentRunErrorWithPolling(error, polling = null) {
|
||||
if (!error || !polling) return error;
|
||||
error.polling = polling;
|
||||
error.backoffMs = polling.backoffMs;
|
||||
error.nextRetryAt = polling.nextRetryAt;
|
||||
error.rootCause = polling.rootCause;
|
||||
return error;
|
||||
}
|
||||
|
||||
export function agentRunTerminalConsistencyGap({ mapped = {}, eventsResponse = null, result = null } = {}) {
|
||||
const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : {};
|
||||
const terminal = agentRunPollingTerminalStatus(result?.terminalStatus ?? result?.commandState ?? result?.status ?? mapped?.status ?? agentRun.terminalStatus ?? agentRun.commandState) || Boolean(agentRunTerminalStatusFromEvents(eventsResponse?.events));
|
||||
if (!terminal) return 0;
|
||||
const latest = Math.max(
|
||||
nonNegativeNumber(eventsResponse?.maxSeq),
|
||||
nonNegativeNumber(result?.eventCount),
|
||||
nonNegativeNumber(result?.lastSeq),
|
||||
nonNegativeNumber(agentRun.lastSeq)
|
||||
);
|
||||
const observed = Math.max(
|
||||
nonNegativeNumber(eventsResponse?.traceLastSeq),
|
||||
nonNegativeNumber(result?.lastSeq),
|
||||
nonNegativeNumber(agentRun.lastSeq)
|
||||
);
|
||||
return Math.max(0, latest - observed);
|
||||
}
|
||||
|
||||
export function agentRunTerminalResultRefreshSatisfied(mapped = {}, traceId = null) {
|
||||
const safeId = safeTraceId(traceId ?? mapped?.traceId);
|
||||
const agentRun = mapped?.agentRun && typeof mapped.agentRun === "object" ? mapped.agentRun : null;
|
||||
const traceSummary = mapped?.traceSummary && typeof mapped.traceSummary === "object" ? mapped.traceSummary : null;
|
||||
const finalResponse = mapped?.finalResponse && typeof mapped.finalResponse === "object" ? mapped.finalResponse : null;
|
||||
const providerTrace = agentRun?.providerTrace && typeof agentRun.providerTrace === "object"
|
||||
? agentRun.providerTrace
|
||||
: mapped?.providerTrace && typeof mapped.providerTrace === "object"
|
||||
? mapped.providerTrace
|
||||
: null;
|
||||
const commandId = text(agentRun?.commandId);
|
||||
const runId = text(agentRun?.runId);
|
||||
if (!safeId || !runId || !commandId) return false;
|
||||
if (!agentRunPollingTerminalStatus(mapped?.status ?? agentRun?.terminalStatus ?? agentRun?.commandState ?? agentRun?.status)) return false;
|
||||
if (safeTraceId(mapped?.traceId ?? agentRun?.traceId) !== safeId) return false;
|
||||
if (safeTraceId(providerTrace?.traceId) !== safeId) return false;
|
||||
const finalTraceId = safeTraceId(finalResponse?.traceId ?? safeId);
|
||||
if (finalTraceId && finalTraceId !== safeId) return false;
|
||||
if (!String(finalResponse?.text ?? "").trim()) return false;
|
||||
if (traceSummary?.source !== "agentrun-command-result") return false;
|
||||
if (safeTraceId(traceSummary.traceId ?? safeId) !== safeId) return false;
|
||||
const summaryAgentRun = traceSummary.agentRun && typeof traceSummary.agentRun === "object" ? traceSummary.agentRun : null;
|
||||
if (text(summaryAgentRun?.runId) !== runId) return false;
|
||||
if (text(summaryAgentRun?.commandId) !== commandId) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
export function emitAgentRunProjectionPollingSpan({ traceId, env = process.env, agentRun = {}, polling = {}, terminalRefreshSkipped = false } = {}) {
|
||||
void emitCodeAgentOtelSpan("projection_sync", traceId, env, {
|
||||
attributes: {
|
||||
runId: agentRun?.runId ?? null,
|
||||
commandId: agentRun?.commandId ?? null,
|
||||
terminalRefreshSkipped: terminalRefreshSkipped === true,
|
||||
pollCount: nonNegativeNumber(polling.pollCount),
|
||||
backoffMs: nonNegativeNumber(polling.backoffMs),
|
||||
rootCause: boundedAgentRunRootCause(polling.rootCause),
|
||||
terminalConsistencyGap: nonNegativeNumber(polling.terminalConsistencyGap)
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export async function fetchAgentRunEventsForTrace({ fetchImpl, managerUrl, timeoutMs, env, mapping = {} }) {
|
||||
const runId = requiredString(mapping.runId, "runId");
|
||||
const currentCommandId = typeof mapping.commandId === "string" ? mapping.commandId : "";
|
||||
const replayWindow = agentRunTraceReplayWindow(mapping);
|
||||
const afterSeqOverride = Number(mapping.afterSeqOverride);
|
||||
const afterSeq = Number.isFinite(afterSeqOverride) && afterSeqOverride >= 0 ? Math.floor(afterSeqOverride) : replayWindow.afterSeq;
|
||||
const endSeq = Number.isFinite(afterSeqOverride) && afterSeqOverride >= 0 ? 0 : replayWindow.endSeq;
|
||||
const limit = parsePositiveInteger(mapping.eventsPageLimit ?? env?.HWLAB_CODE_AGENT_AGENTRUN_EVENTS_PAGE_LIMIT, 500);
|
||||
const path = `/api/v1/runs/${encodeURIComponent(runId)}/events?afterSeq=${encodeURIComponent(String(afterSeq))}&limit=${encodeURIComponent(String(limit))}`;
|
||||
const response = await agentRunJson(fetchImpl, managerUrl, path, { method: "GET", timeoutMs, env });
|
||||
const rawEvents = Array.isArray(response?.items) ? response.items : [];
|
||||
const events = rawEvents.filter((event) => agentRunEventBelongsToTrace(event, { currentCommandId, afterSeq, endSeq }));
|
||||
const maxSeq = Math.max(afterSeq, ...rawEvents.map((event) => Number(event?.seq ?? 0)).filter(Number.isFinite));
|
||||
const traceLastSeq = Math.max(afterSeq, ...events.map((event) => Number(event?.seq ?? 0)).filter(Number.isFinite));
|
||||
void emitCodeAgentOtelSpan("projection_sync", mapping.traceId ?? mapping.hwlabTraceId ?? mapping.businessTraceId, env, {
|
||||
attributes: {
|
||||
runId,
|
||||
commandId: currentCommandId || null,
|
||||
afterSeq,
|
||||
endSeq,
|
||||
limit,
|
||||
rawEventCount: rawEvents.length,
|
||||
eventCount: events.length,
|
||||
commandFiltered: Boolean(currentCommandId),
|
||||
maxSeq,
|
||||
traceLastSeq,
|
||||
terminalFromRawEvents: agentRunTerminalStatusFromEvents(rawEvents),
|
||||
terminalFromEvents: agentRunTerminalStatusFromEvents(events),
|
||||
pollCount: nonNegativeNumber(mapping.pollCount),
|
||||
backoffMs: nonNegativeNumber(mapping.backoffMs),
|
||||
rootCause: boundedAgentRunRootCause(mapping.rootCause),
|
||||
terminalConsistencyGap: nonNegativeNumber(mapping.terminalConsistencyGap)
|
||||
}
|
||||
});
|
||||
return {
|
||||
events,
|
||||
afterSeq,
|
||||
endSeq,
|
||||
rawEventCount: rawEvents.length,
|
||||
commandFiltered: Boolean(currentCommandId),
|
||||
maxSeq,
|
||||
traceLastSeq
|
||||
};
|
||||
}
|
||||
|
||||
export async function measuredAgentRunEventsFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapping = {} }) {
|
||||
const startedAt = nowMs();
|
||||
try {
|
||||
const response = await fetchAgentRunEventsForTrace({ fetchImpl, managerUrl, timeoutMs, env, mapping });
|
||||
performanceStore?.recordWorkbenchProjectorBatch?.({ phase: "agentrun_events_fetch", status: "ok", durationMs: nowMs() - startedAt });
|
||||
return response;
|
||||
} catch (error) {
|
||||
performanceStore?.recordWorkbenchProjectorBatch?.({ phase: "agentrun_events_fetch", status: agentRunMetricStatusForError(error), durationMs: nowMs() - startedAt });
|
||||
performanceStore?.recordWorkbenchProjectorError?.({ reason: error?.code ?? "agentrun_events_fetch_failed" });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function measuredAgentRunResultFetch({ performanceStore, fetchImpl, managerUrl, timeoutMs, env, mapped = {}, eventsResponse = null }) {
|
||||
const startedAt = nowMs();
|
||||
const path = `/api/v1/runs/${encodeURIComponent(mapped.agentRun.runId)}/commands/${encodeURIComponent(mapped.agentRun.commandId)}/result`;
|
||||
try {
|
||||
const result = await agentRunJson(fetchImpl, managerUrl, path, { method: "GET", timeoutMs, env });
|
||||
const eventCount = agentRunResultEventCount(result, eventsResponse);
|
||||
performanceStore?.recordAgentRunResult?.({
|
||||
durationMs: nowMs() - startedAt,
|
||||
eventCount,
|
||||
eventsScanned: eventCount,
|
||||
pagesScanned: estimateAgentRunResultPages(eventCount),
|
||||
status: result?.terminalStatus ?? result?.status ?? "ok",
|
||||
budget: timeoutMs
|
||||
});
|
||||
return result;
|
||||
} catch (error) {
|
||||
const status = agentRunMetricStatusForError(error);
|
||||
const eventCount = agentRunResultEventCount(null, eventsResponse);
|
||||
performanceStore?.recordAgentRunResult?.({
|
||||
durationMs: nowMs() - startedAt,
|
||||
eventCount,
|
||||
eventsScanned: eventCount,
|
||||
pagesScanned: estimateAgentRunResultPages(eventCount),
|
||||
status,
|
||||
budget: timeoutMs,
|
||||
timedOut: status === "timeout"
|
||||
});
|
||||
performanceStore?.recordWorkbenchProjectorError?.({ reason: error?.code ?? "agentrun_result_failed" });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function recordAgentRunEventsProjectionMetrics(performanceStore, eventsResponse = null, previousLastSeq = 0, nextLastSeq = 0) {
|
||||
if (!performanceStore || !eventsResponse) return;
|
||||
const cursorAdvance = Math.max(0, Number(nextLastSeq ?? 0) - Number(previousLastSeq ?? 0));
|
||||
if (cursorAdvance > 0) {
|
||||
performanceStore.recordWorkbenchProjectionCursorAdvance?.({
|
||||
status: "advanced",
|
||||
reason: eventsResponse.commandFiltered ? "command_filtered_events" : "run_events",
|
||||
projectionStatus: "projecting",
|
||||
source: "agentrun_v01",
|
||||
count: cursorAdvance
|
||||
});
|
||||
}
|
||||
const counts = new Map();
|
||||
for (const event of eventsResponse.events ?? []) {
|
||||
const kind = String(event?.type ?? event?.payload?.phase ?? "unknown").trim() || "unknown";
|
||||
counts.set(kind, (counts.get(kind) ?? 0) + 1);
|
||||
}
|
||||
if (counts.size === 0 && Number(eventsResponse.rawEventCount ?? 0) > 0) counts.set("filtered", Number(eventsResponse.rawEventCount));
|
||||
for (const [eventKind, count] of counts.entries()) {
|
||||
performanceStore.recordWorkbenchProjectorEvents?.({ eventKind, status: "ok", count });
|
||||
}
|
||||
}
|
||||
|
||||
export function agentRunMetricStatusForError(error) {
|
||||
return error?.code === "agentrun_timeout" || error?.name === "AbortError" || Number(error?.statusCode) === 504 ? "timeout" : "error";
|
||||
}
|
||||
|
||||
export function agentRunTraceCursorSeq(eventsResponse = {}, previousLastSeq = 0) {
|
||||
const afterSeq = Number(eventsResponse.afterSeq ?? 0);
|
||||
const endSeq = Number(eventsResponse.endSeq ?? 0);
|
||||
const traceLastSeq = Number(eventsResponse.traceLastSeq ?? 0);
|
||||
if (Number.isFinite(traceLastSeq) && traceLastSeq > afterSeq) return Math.floor(traceLastSeq);
|
||||
if (Number.isFinite(endSeq) && endSeq > 0) return Math.floor(endSeq);
|
||||
if (eventsResponse.commandFiltered === true) return Math.max(Number(previousLastSeq ?? 0), afterSeq);
|
||||
return Math.max(Number(previousLastSeq ?? 0), Number(eventsResponse.maxSeq ?? 0));
|
||||
}
|
||||
|
||||
export function agentRunTerminalStatusFromEvents(events = []) {
|
||||
if (!Array.isArray(events)) return null;
|
||||
for (const event of [...events].reverse()) {
|
||||
const payload = event?.payload && typeof event.payload === "object" ? event.payload : {};
|
||||
const phase = payload.phase ?? event?.phase;
|
||||
const candidates = [
|
||||
event?.type === "terminal_status" ? payload.terminalStatus ?? event?.terminalStatus ?? payload.status ?? event?.status : null,
|
||||
phase === "command-terminal" ? payload.terminalStatus ?? event?.terminalStatus ?? payload.status ?? event?.status : null,
|
||||
phase === "turn:completed" || phase === "turn/steer:completed" ? "completed" : null,
|
||||
event?.terminal === true ? payload.terminalStatus ?? event?.terminalStatus ?? payload.status ?? event?.status ?? "completed" : null,
|
||||
payload.terminal === true ? payload.terminalStatus ?? event?.terminalStatus ?? payload.status ?? event?.status ?? "completed" : null
|
||||
];
|
||||
for (const value of candidates) {
|
||||
const status = normalizeAgentRunStatus(value);
|
||||
if (agentRunPollingTerminalStatus(status)) return status;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function agentRunEventCommandId(event) {
|
||||
const payload = event?.payload && typeof event.payload === "object" ? event.payload : {};
|
||||
if (typeof event?.commandId === "string") return event.commandId;
|
||||
return typeof payload.commandId === "string" ? payload.commandId : "";
|
||||
}
|
||||
|
||||
export function nonNegativeNumber(value) {
|
||||
const number = Number(value ?? 0);
|
||||
return Number.isFinite(number) && number > 0 ? Math.floor(number) : 0;
|
||||
}
|
||||
|
||||
function agentRunProjectionBackoffMs(attempt, env = process.env) {
|
||||
const initialMs = parsePositiveInteger(env?.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_BACKOFF_INITIAL_MS, DEFAULT_AGENTRUN_PROJECTION_SYNC_BACKOFF_INITIAL_MS);
|
||||
const maxMs = parsePositiveInteger(env?.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_BACKOFF_MAX_MS, DEFAULT_AGENTRUN_PROJECTION_SYNC_BACKOFF_MAX_MS);
|
||||
const exponent = Math.max(0, Math.min(10, nonNegativeNumber(attempt) - 1));
|
||||
return Math.min(maxMs, initialMs * 2 ** exponent);
|
||||
}
|
||||
|
||||
function boundedAgentRunRootCause(value) {
|
||||
const raw = String(value ?? "").trim().replace(/[^a-z0-9_.:-]+/giu, "_").replace(/^_+|_+$/gu, "").toLowerCase();
|
||||
return raw ? raw.slice(0, 120) : null;
|
||||
}
|
||||
|
||||
function agentRunResultEventCount(result = null, eventsResponse = null) {
|
||||
const candidates = [
|
||||
result?.traceSummary?.sourceEventCount,
|
||||
result?.traceSummary?.eventCount,
|
||||
result?.sourceEventCount,
|
||||
result?.eventCount,
|
||||
result?.providerTrace?.eventCount,
|
||||
eventsResponse?.maxSeq,
|
||||
eventsResponse?.rawEventCount,
|
||||
eventsResponse?.events?.length
|
||||
];
|
||||
for (const value of candidates) {
|
||||
const count = Number(value);
|
||||
if (Number.isFinite(count) && count >= 0) return Math.floor(count);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function estimateAgentRunResultPages(eventCount) {
|
||||
const count = Number(eventCount);
|
||||
if (!Number.isFinite(count) || count <= 0) return 0;
|
||||
return Math.max(1, Math.ceil(count / 500));
|
||||
}
|
||||
|
||||
function agentRunTraceReplayWindow(mapping = {}) {
|
||||
const eventStartSeq = Number(mapping.eventStartSeq ?? mapping.commandStartSeq ?? mapping.startSeq ?? 0);
|
||||
const summary = mapping.traceSummary && typeof mapping.traceSummary === "object" ? mapping.traceSummary : null;
|
||||
const summaryAgentRun = summary?.agentRun && typeof summary.agentRun === "object" ? summary.agentRun : null;
|
||||
const summaryLastSeq = Number(summaryAgentRun?.lastSeq ?? summary?.lastSeq ?? 0);
|
||||
const currentLastSeq = Number(mapping.lastSeq ?? 0);
|
||||
const endSeq = Number.isFinite(summaryLastSeq) && summaryLastSeq > 0 ? Math.floor(summaryLastSeq) : 0;
|
||||
if (Number.isFinite(eventStartSeq) && eventStartSeq > 0) return { afterSeq: Math.max(0, Math.floor(eventStartSeq) - 1), endSeq };
|
||||
if (endSeq > 0) return { afterSeq: Math.max(0, endSeq - 500), endSeq };
|
||||
if (Number.isFinite(currentLastSeq) && currentLastSeq > 0) return { afterSeq: Math.floor(currentLastSeq), endSeq: 0 };
|
||||
return { afterSeq: 0, endSeq: 0 };
|
||||
}
|
||||
|
||||
function agentRunEventBelongsToTrace(event, { currentCommandId = "", afterSeq = 0, endSeq = 0 } = {}) {
|
||||
const eventCommandId = agentRunEventCommandId(event);
|
||||
const seq = Number(event?.seq ?? 0);
|
||||
if (currentCommandId && eventCommandId && eventCommandId !== currentCommandId) return false;
|
||||
if (endSeq > 0 && Number.isFinite(seq)) return seq > afterSeq && seq <= endSeq;
|
||||
return true;
|
||||
}
|
||||
|
||||
function agentRunPollingTerminalStatus(value) {
|
||||
return AGENTRUN_TERMINAL_STATUSES.has(normalizeAgentRunStatus(value));
|
||||
}
|
||||
|
||||
function normalizeAgentRunStatus(value) {
|
||||
const status = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-");
|
||||
return status === "cancelled" ? "canceled" : status;
|
||||
}
|
||||
|
||||
function requiredString(value, fieldName) {
|
||||
if (typeof value === "string" && value.trim()) return value.trim();
|
||||
throw Object.assign(new Error(`AgentRun response missing ${fieldName}`), { code: "agentrun_response_invalid", statusCode: 502 });
|
||||
}
|
||||
|
||||
function nowMs() {
|
||||
if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now();
|
||||
return Date.now();
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-17-r0; draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
||||
// Responsibility: AgentRun command-result/event mapping, terminal projection, and trace observability helpers.
|
||||
// Responsibility: AgentRun command-result/event mapping and trace observability helpers; Kafka owns persistence.
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
@@ -20,25 +20,6 @@ import {
|
||||
isAgentRunManagerInternalServiceHost,
|
||||
resolveAgentRunManagerUrl
|
||||
} from "./code-agent-agentrun-runtime.ts";
|
||||
import {
|
||||
agentRunErrorWithPolling,
|
||||
agentRunEventCommandId,
|
||||
agentRunMetricStatusForError,
|
||||
agentRunPollingRetryableError,
|
||||
agentRunPollingRootCause,
|
||||
agentRunProjectionPollCount,
|
||||
agentRunProjectionPollingState,
|
||||
agentRunTerminalConsistencyGap,
|
||||
agentRunTerminalResultRefreshSatisfied,
|
||||
agentRunTerminalStatusFromEvents,
|
||||
agentRunTraceCursorSeq,
|
||||
emitAgentRunProjectionPollingSpan,
|
||||
fetchAgentRunEventsForTrace,
|
||||
measuredAgentRunEventsFetch,
|
||||
measuredAgentRunResultFetch,
|
||||
nonNegativeNumber,
|
||||
recordAgentRunEventsProjectionMetrics
|
||||
} from "./code-agent-agentrun-projection-polling.ts";
|
||||
import {
|
||||
agentRunAlreadyTerminalCancelPayload,
|
||||
agentRunCancelBlockedPayload,
|
||||
@@ -65,7 +46,6 @@ import {
|
||||
agentRunPromptMetadataFromText,
|
||||
messageAuthorityTextValue
|
||||
} from "./code-agent-agentrun-prompt.ts";
|
||||
import { agentRunResultSyncDeferred, buildAgentRunProjectionEventsFetchPlan, buildWorkbenchProjectionStateUpdate } from "./workbench-projection-cursor.ts";
|
||||
import { codeAgentOtelTraceFields, codeAgentOtelTraceContext, emitCodeAgentOtelSpan } from "./otel-trace.ts";
|
||||
|
||||
const ADAPTER_ID = "agentrun-v01";
|
||||
@@ -313,7 +293,7 @@ export function agentRunResultToCodeAgentPayload({ base, result, traceStore, tra
|
||||
status: "completed",
|
||||
label: "agentrun:result:completed",
|
||||
createdAt: terminalEventCreatedAt,
|
||||
message: "AgentRun result is ready for HWLAB short-connection polling.",
|
||||
message: "AgentRun result is committed to the Workbench projection event stream.",
|
||||
runId: base.agentRun.runId,
|
||||
commandId: base.agentRun.commandId,
|
||||
terminal: true,
|
||||
@@ -574,6 +554,11 @@ export function appendAgentRunEventsToTrace(traceStore, traceId, events, mapping
|
||||
if (normalized) traceStore.append(traceId, normalized, agentRunTraceMeta({}, {}));
|
||||
}
|
||||
}
|
||||
function agentRunEventCommandId(event) {
|
||||
const payload = event?.payload && typeof event.payload === "object" ? event.payload : {};
|
||||
if (typeof event?.commandId === "string") return event.commandId;
|
||||
return typeof payload.commandId === "string" ? payload.commandId : "";
|
||||
}
|
||||
|
||||
export function isForeignAgentRunCommandEvent(event, mapping = {}) {
|
||||
const currentCommandId = typeof mapping.commandId === "string" ? mapping.commandId : "";
|
||||
|
||||
@@ -45,9 +45,8 @@ test("code agent trace store dedupes repeated AgentRun source events", () => {
|
||||
assert.deepEqual(snapshot.events.map((item) => item.label), ["agentrun:backend:resource-bundle-materialized", "agentrun:assistant:message"]);
|
||||
});
|
||||
|
||||
test("durable code agent trace store projects appended events without counting assistant deltas", async () => {
|
||||
test("durable code agent trace store persists diagnostics without counting assistant deltas", async () => {
|
||||
const writes = [];
|
||||
const workbenchWrites = [];
|
||||
const baseStore = createCodeAgentTraceStore({ maxEvents: 20 });
|
||||
const traceStore = createDurableCodeAgentTraceStore({
|
||||
traceStore: baseStore,
|
||||
@@ -56,10 +55,6 @@ test("durable code agent trace store projects appended events without counting a
|
||||
writes.push({ params, requestMeta });
|
||||
return { written: true };
|
||||
}
|
||||
},
|
||||
workbenchEventWriter(event, requestMeta) {
|
||||
workbenchWrites.push({ event, requestMeta });
|
||||
return { written: true };
|
||||
}
|
||||
});
|
||||
const traceId = "trc_trace-store-durable-projection";
|
||||
@@ -76,10 +71,6 @@ test("durable code agent trace store projects appended events without counting a
|
||||
assert.equal(writes[0].params.event.traceId, traceId);
|
||||
assert.equal(writes[0].params.event.seq, 1);
|
||||
assert.equal(writes[0].requestMeta.agentSessionId, "ses_trace_store_durable");
|
||||
assert.equal(workbenchWrites.length, 1);
|
||||
assert.equal(workbenchWrites[0].event.traceId, traceId);
|
||||
assert.equal(workbenchWrites[0].requestMeta.sessionId, "ses_trace_store_durable");
|
||||
assert.equal(workbenchWrites[0].requestMeta.agentSessionId, "ses_trace_store_durable");
|
||||
});
|
||||
|
||||
test("code agent trace store retains six thousand regular events", () => {
|
||||
|
||||
@@ -228,12 +228,12 @@ export function createCodeAgentTraceStore(options = {}) {
|
||||
|
||||
export const defaultCodeAgentTraceStore = createCodeAgentTraceStore();
|
||||
|
||||
export function createDurableCodeAgentTraceStore({ traceStore = defaultCodeAgentTraceStore, traceEventStore = null, workbenchEventWriter = null } = {}) {
|
||||
export function createDurableCodeAgentTraceStore({ traceStore = defaultCodeAgentTraceStore, traceEventStore = null } = {}) {
|
||||
if (!traceEventStore || typeof traceEventStore.writeAgentTraceEvent !== "function") return traceStore;
|
||||
|
||||
function append(traceId, event = {}, meta = {}) {
|
||||
const normalized = traceStore.append(traceId, event, meta);
|
||||
if (normalized?.traceId) persistTraceEvent(traceEventStore, normalized, meta, workbenchEventWriter);
|
||||
if (normalized?.traceId) persistTraceEvent(traceEventStore, normalized, meta);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
@@ -247,11 +247,11 @@ export function createDurableCodeAgentTraceStore({ traceStore = defaultCodeAgent
|
||||
};
|
||||
}
|
||||
|
||||
function persistTraceEvent(traceEventStore, event, meta = {}, workbenchEventWriter = null) {
|
||||
void persistTraceEventAsync(traceEventStore, event, meta, workbenchEventWriter);
|
||||
function persistTraceEvent(traceEventStore, event, meta = {}) {
|
||||
void persistTraceEventAsync(traceEventStore, event, meta);
|
||||
}
|
||||
|
||||
async function persistTraceEventAsync(traceEventStore, event, meta = {}, workbenchEventWriter = null) {
|
||||
async function persistTraceEventAsync(traceEventStore, event, meta = {}) {
|
||||
try {
|
||||
const resolvedAgentSessionId = event.agentSessionId ?? event.sessionId ?? meta.agentSessionId ?? meta.sessionId;
|
||||
const resolvedSessionId = event.sessionId ?? meta.sessionId ?? resolvedAgentSessionId;
|
||||
@@ -265,13 +265,6 @@ async function persistTraceEventAsync(traceEventStore, event, meta = {}, workben
|
||||
} catch (error) {
|
||||
console.error(`[workbench-projection] durable trace event write failed: traceId=${event.traceId} sourceEventId=${event.sourceEventId ?? "none"} error=${error?.message ?? error}`);
|
||||
}
|
||||
if (typeof workbenchEventWriter === "function") {
|
||||
try {
|
||||
await workbenchEventWriter(event, projectionContext);
|
||||
} catch (error) {
|
||||
console.error(`[workbench-projection] workbench event writer failed: traceId=${event.traceId} error=${error?.message ?? error}`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// Trace capture must not fail the user turn, but projection write errors must be visible for diagnostics.
|
||||
console.error(`[workbench-projection] persistTraceEvent sync error: traceId=${event.traceId} error=${error?.message ?? error}`);
|
||||
|
||||
@@ -13,10 +13,11 @@ import {
|
||||
|
||||
export const CLOUD_API_HEALTH_CONTRACT_VERSION = "v3";
|
||||
|
||||
export function buildCloudApiReadiness({ db, codeAgent, runtime } = {}) {
|
||||
export function buildCloudApiReadiness({ db, codeAgent, runtime, kafkaProjector } = {}) {
|
||||
const dbReady = isDbReady(db);
|
||||
const codeAgentReady = isCodeAgentReady(codeAgent);
|
||||
const runtimeReady = isRuntimeReady(runtime);
|
||||
const kafkaProjectorReady = !kafkaProjector?.started || kafkaProjector.status === "ready";
|
||||
const blockers = [];
|
||||
let runtimeReadinessBlocker = null;
|
||||
|
||||
@@ -32,6 +33,7 @@ export function buildCloudApiReadiness({ db, codeAgent, runtime } = {}) {
|
||||
runtimeReadinessBlocker = runtimeBlocker(runtime);
|
||||
blockers.push(runtimeReadinessBlocker);
|
||||
}
|
||||
if (!kafkaProjectorReady) blockers.push(kafkaProjectorBlocker(kafkaProjector));
|
||||
const provider = buildProviderReadiness(codeAgent);
|
||||
const dbDurable = buildRuntimeDurabilityReadiness({
|
||||
db,
|
||||
@@ -59,6 +61,7 @@ export function buildCloudApiReadiness({ db, codeAgent, runtime } = {}) {
|
||||
db: dbReady ? "ready" : "blocked",
|
||||
codeAgent: codeAgentReady ? "ready" : "blocked",
|
||||
runtime: runtimeReady ? "ready" : dbReady ? "blocked" : "waiting_for_db",
|
||||
kafkaProjector: kafkaProjectorReady ? (kafkaProjector?.started ? "ready" : "disabled") : "blocked",
|
||||
provider: provider.status,
|
||||
dbDurable: dbDurable.status,
|
||||
sessionRunner: sessionRunner.status,
|
||||
@@ -73,6 +76,24 @@ export function buildCloudApiReadiness({ db, codeAgent, runtime } = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
function kafkaProjectorBlocker(projector) {
|
||||
return {
|
||||
code: projector?.errorCode ?? "hwlab_kafka_projector_not_ready",
|
||||
type: "runtime_blocker",
|
||||
scope: "agentrun-kafka-projector",
|
||||
status: "open",
|
||||
sourceIssue: "pikasTech/HWLAB#2464",
|
||||
summary: projector?.message ?? `AgentRun Kafka projector is ${projector?.status ?? "not ready"}`,
|
||||
evidence: {
|
||||
status: projector?.status ?? "unknown",
|
||||
groupId: projector?.groupId ?? null,
|
||||
agentRunTopic: projector?.agentRunTopic ?? null,
|
||||
hwlabTopic: projector?.hwlabTopic ?? null,
|
||||
valuesRedacted: true
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function isDbReady(db) {
|
||||
return Boolean(db?.ready && db?.connected && db?.liveConnected !== false && db?.liveDbEvidence);
|
||||
}
|
||||
|
||||
@@ -4,7 +4,47 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { projectAgentRunKafkaEventToHwlabEvent } from "./kafka-event-bridge.ts";
|
||||
import { createCloudApiBunServer } from "./bun-server.ts";
|
||||
import { buildCloudApiReadiness } from "./health-contract.ts";
|
||||
import { decodeCanonicalAgentRunKafkaMessage, kafkaEventBridgeConfig, liveKafkaOtelSpanAttributes, projectAgentRunKafkaEventToHwlabEvent, publishAgentRunKafkaMessageLive, relayHwlabKafkaOutboxOnce, shouldEmitLiveKafkaOtelSpan, startHwlabKafkaEventBridge } from "./kafka-event-bridge.ts";
|
||||
|
||||
const PROJECTOR_ENV = Object.freeze({
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false",
|
||||
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "true",
|
||||
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true",
|
||||
HWLAB_KAFKA_ENABLED: "true",
|
||||
HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED: "true",
|
||||
HWLAB_KAFKA_BOOTSTRAP_SERVERS: "kafka.test:9092",
|
||||
HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC: "agentrun.event.v1",
|
||||
HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1",
|
||||
HWLAB_KAFKA_CLIENT_ID: "hwlab-projector-test",
|
||||
HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID: "hwlab-projector-test-v1",
|
||||
HWLAB_KAFKA_PROJECTOR_GROUP_ID: "hwlab-projector-test-v1",
|
||||
HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS: "10",
|
||||
HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS: "60000",
|
||||
HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE: "1",
|
||||
HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS: "30000",
|
||||
HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS: "10000",
|
||||
HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS: "1000"
|
||||
});
|
||||
|
||||
const LIVE_ENV = Object.freeze({
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
|
||||
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
|
||||
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false",
|
||||
HWLAB_KAFKA_ENABLED: "true",
|
||||
HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED: "true",
|
||||
HWLAB_KAFKA_BOOTSTRAP_SERVERS: "kafka.test:9092",
|
||||
HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC: "agentrun.event.v1",
|
||||
HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1",
|
||||
HWLAB_KAFKA_CLIENT_ID: "hwlab-live-test",
|
||||
HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID: "hwlab-live-bridge-fixed",
|
||||
HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID: "hwlab-live-fanout-fixed"
|
||||
});
|
||||
|
||||
test("projects AgentRun assistant_message Kafka event into HWLAB trace event", () => {
|
||||
const projected = projectAgentRunKafkaEventToHwlabEvent({
|
||||
@@ -39,7 +79,10 @@ test("projects AgentRun assistant_message Kafka event into HWLAB trace event", (
|
||||
assert.equal(projected.eventType, "hwlab.trace.event.projected");
|
||||
assert.equal(projected.source, "hwlab-test");
|
||||
assert.equal(projected.traceId, "trc_kafka_projection");
|
||||
assert.equal(projected.hwlabSessionId, "ses_kafka_projection");
|
||||
assert.equal(projected.sessionId, "ses_kafka_projection");
|
||||
assert.equal(projected.runId, "run_kafka_projection");
|
||||
assert.equal(projected.commandId, "cmd_kafka_projection");
|
||||
assert.equal(projected.context.runId, "run_kafka_projection");
|
||||
assert.equal(projected.context.commandId, "cmd_kafka_projection");
|
||||
assert.equal(projected.context.sourceSeq, 7);
|
||||
@@ -48,12 +91,30 @@ test("projects AgentRun assistant_message Kafka event into HWLAB trace event", (
|
||||
assert.equal(projected.event.label, "agentrun:assistant:message");
|
||||
assert.equal(projected.event.status, "running");
|
||||
assert.equal(projected.event.text, "partial answer");
|
||||
assert.equal(projected.event.assistantText, "partial answer");
|
||||
assert.equal(projected.event.finalResponse, null);
|
||||
assert.equal(projected.event.terminal, false);
|
||||
assert.equal(projected.sourceEvent.topic, "agentrun.event.v1");
|
||||
assert.equal(projected.sourceEvent.offset, "42");
|
||||
assert.equal(projected.valuesPrinted, false);
|
||||
});
|
||||
|
||||
test("projects final AgentRun assistant text without sealing before terminal_status", () => {
|
||||
const projected = projectAgentRunKafkaEventToHwlabEvent({
|
||||
schema: "agentrun.event.v1",
|
||||
eventType: "agentrun.run.event",
|
||||
run: { runId: "run_final", sessionId: "ses_final" },
|
||||
command: { commandId: "cmd_final" },
|
||||
event: { seq: 9, type: "assistant_message", payload: { traceId: "trc_final", text: "durable final answer", final: true, replyAuthority: true } }
|
||||
}, { source: "hwlab-test" });
|
||||
|
||||
assert.equal(projected.event.assistantText, "durable final answer");
|
||||
assert.equal(projected.event.status, "running");
|
||||
assert.equal(projected.event.terminal, false);
|
||||
assert.equal(projected.event.finalResponse.text, "durable final answer");
|
||||
assert.equal(projected.event.finalResponse.traceId, "trc_final");
|
||||
});
|
||||
|
||||
test("projects AgentRun terminal_status Kafka event into terminal HWLAB event", () => {
|
||||
const projected = projectAgentRunKafkaEventToHwlabEvent({
|
||||
schema: "agentrun.event.v1",
|
||||
@@ -80,6 +141,202 @@ test("projects AgentRun terminal_status Kafka event into terminal HWLAB event",
|
||||
assert.equal(projected.event.message, "AgentRun completed");
|
||||
});
|
||||
|
||||
test("live Kafka mode publishes canonical AgentRun input directly and fans one broker event to all SSE subscribers", async () => {
|
||||
const consumerConfigs = [];
|
||||
const subscriptions = [];
|
||||
const runOptions = [];
|
||||
const runOrder = [];
|
||||
const produced = [];
|
||||
const otelSpans = [];
|
||||
const consumers = [0, 1].map((index) => ({
|
||||
async connect() {},
|
||||
async subscribe(input) { subscriptions[index] = input; },
|
||||
async run(input) { runOptions[index] = input; runOrder.push(index); },
|
||||
async stop() {},
|
||||
async disconnect() {}
|
||||
}));
|
||||
const producer = {
|
||||
async connect() {},
|
||||
async send(input) { produced.push(input); return [{ topicName: input.topic, partition: 4, baseOffset: "91" }]; },
|
||||
async disconnect() {}
|
||||
};
|
||||
let consumerIndex = 0;
|
||||
const kafkaFactory = () => ({
|
||||
consumer(config) {
|
||||
consumerConfigs.push(config);
|
||||
return consumers[consumerIndex++];
|
||||
},
|
||||
producer() { return producer; }
|
||||
});
|
||||
const forbiddenRuntimeStore = new Proxy({}, {
|
||||
get() { throw new Error("live mode must not touch the runtime store"); }
|
||||
});
|
||||
const bridge = startHwlabKafkaEventBridge({
|
||||
env: LIVE_ENV,
|
||||
runtimeStore: forbiddenRuntimeStore,
|
||||
kafkaFactory,
|
||||
logger: null,
|
||||
otelSpanEmitter(name, traceId, env, options) { otelSpans.push({ name, traceId, env, options }); }
|
||||
});
|
||||
await bridge.ready;
|
||||
|
||||
assert.deepEqual(consumerConfigs.map((item) => item.groupId), ["hwlab-live-bridge-fixed", "hwlab-live-fanout-fixed"]);
|
||||
assert.deepEqual(runOrder, [1, 0], "fanout group must be ready before direct publish can create a new HWLAB event");
|
||||
assert.deepEqual(subscriptions, [
|
||||
{ topic: "agentrun.event.v1", fromBeginning: false },
|
||||
{ topic: "hwlab.event.v1", fromBeginning: false }
|
||||
]);
|
||||
const message = canonicalMessage({ offset: "70" });
|
||||
await runOptions[0].eachMessage({ topic: "agentrun.event.v1", partition: 0, message });
|
||||
assert.equal(produced.length, 1);
|
||||
assert.equal(produced[0].topic, "hwlab.event.v1");
|
||||
const envelope = JSON.parse(produced[0].messages[0].value);
|
||||
assert.equal(envelope.schema, "hwlab.event.v1");
|
||||
assert.equal(envelope.traceId, "trc_projector_test");
|
||||
assert.equal(envelope.hwlabSessionId, "ses_projector_test");
|
||||
assert.equal(envelope.runId, "run_projector_test");
|
||||
assert.equal(envelope.commandId, "cmd_projector_test");
|
||||
assert.equal(envelope.event.assistantText, "projected answer");
|
||||
assert.equal(otelSpans[0].name, "hwlab.kafka.live.direct_publish");
|
||||
assert.equal(otelSpans[0].traceId, "trc_projector_test");
|
||||
assert.deepEqual(otelSpans[0].options.attributes, {
|
||||
businessTraceId: "trc_projector_test",
|
||||
hwlabSessionId: "ses_projector_test",
|
||||
runId: "run_projector_test",
|
||||
commandId: "cmd_projector_test",
|
||||
topic: "hwlab.event.v1",
|
||||
partition: 4,
|
||||
offset: "91",
|
||||
sourceTopic: "agentrun.event.v1",
|
||||
sourcePartition: 0,
|
||||
sourceOffset: "70",
|
||||
eventType: "assistant_message",
|
||||
terminal: false,
|
||||
valuesRedacted: true
|
||||
});
|
||||
|
||||
const first = [];
|
||||
const second = [];
|
||||
const firstTransports = [];
|
||||
bridge.subscribeLiveHwlabEvents((value, transport) => { first.push(value); firstTransports.push(transport); });
|
||||
bridge.subscribeLiveHwlabEvents((value) => second.push(value));
|
||||
await runOptions[1].eachBatch({
|
||||
batch: { topic: "hwlab.event.v1", partition: 0, highWatermark: "1", messages: [{ offset: "0", value: Buffer.from(JSON.stringify(envelope)) }] },
|
||||
resolveOffset() {},
|
||||
async heartbeat() {},
|
||||
isRunning: () => true,
|
||||
isStale: () => false
|
||||
});
|
||||
assert.equal(bridge.liveSubscriberCount(), 2);
|
||||
assert.deepEqual(first, [envelope]);
|
||||
assert.deepEqual(second, [envelope]);
|
||||
assert.deepEqual(firstTransports, [{ topic: "hwlab.event.v1", partition: 0, offset: "0" }]);
|
||||
assert.equal(otelSpans[1].name, "hwlab.kafka.live.fanout_receive");
|
||||
assert.equal(otelSpans[1].options.attributes.topic, "hwlab.event.v1");
|
||||
assert.equal(otelSpans[1].options.attributes.partition, 0);
|
||||
assert.equal(otelSpans[1].options.attributes.offset, "0");
|
||||
assert.equal(consumerConfigs.length, 2, "browser subscribers must not allocate Kafka consumer groups");
|
||||
assert.equal((await bridge.status()).consumerLag.total, 0);
|
||||
await bridge.stop();
|
||||
});
|
||||
|
||||
test("live Kafka publisher has no PG inbox, facts, checkpoint, or outbox dependency", async () => {
|
||||
const sent = [];
|
||||
const runtimeStore = new Proxy({}, { get() { throw new Error("unexpected DB access"); } });
|
||||
void runtimeStore;
|
||||
const result = await publishAgentRunKafkaMessageLive({ topic: "agentrun.event.v1", partition: 3, message: canonicalMessage({ offset: "71" }) }, {
|
||||
producer: { async send(input) { sent.push(input); } },
|
||||
config: { clientId: "hwlab-live-test", hwlabTopic: "hwlab.event.v1" },
|
||||
logger: null
|
||||
});
|
||||
assert.equal(result.published, true);
|
||||
assert.equal(sent.length, 1);
|
||||
assert.equal(JSON.parse(sent[0].messages[0].value).sourceEvent.offset, "71");
|
||||
});
|
||||
|
||||
test("live Kafka OTel sampling is deterministic and bounded without payload attributes", () => {
|
||||
const envelope = (agentRunEventType, sourceSeq, event = {}) => ({
|
||||
schema: "hwlab.event.v1",
|
||||
eventId: `hwlab:evt_otel_${sourceSeq}`,
|
||||
sourceEventId: `evt_otel_${sourceSeq}`,
|
||||
traceId: "trc_live_otel",
|
||||
hwlabSessionId: "ses_live_otel",
|
||||
runId: "run_live_otel",
|
||||
commandId: "cmd_live_otel",
|
||||
context: { agentRunEventType, sourceSeq, valuesRedacted: true },
|
||||
event: { type: "backend", eventType: "backend", terminal: false, message: "must-not-leak", payload: { password: "must-not-leak" }, ...event }
|
||||
});
|
||||
|
||||
assert.equal(shouldEmitLiveKafkaOtelSpan(envelope("assistant_message", 1, { type: "assistant" })), true);
|
||||
assert.equal(shouldEmitLiveKafkaOtelSpan(envelope("tool_call", 2, { type: "tool" })), true);
|
||||
assert.equal(shouldEmitLiveKafkaOtelSpan(envelope("terminal_status", 3, { type: "result", terminal: true })), true);
|
||||
const decisions = Array.from({ length: 64 }, (_, index) => shouldEmitLiveKafkaOtelSpan(envelope("command_output", index + 1, { type: "output" })));
|
||||
assert.deepEqual(decisions, Array.from({ length: 64 }, (_, index) => (index + 1) % 32 === 0));
|
||||
assert.deepEqual(decisions, Array.from({ length: 64 }, (_, index) => shouldEmitLiveKafkaOtelSpan(envelope("command_output", index + 1, { type: "output" }))));
|
||||
|
||||
const attributes = liveKafkaOtelSpanAttributes(envelope("assistant_message", 1, { type: "assistant" }), {
|
||||
topic: "hwlab.event.v1",
|
||||
partition: 2,
|
||||
offset: "14"
|
||||
});
|
||||
assert.deepEqual(attributes, {
|
||||
businessTraceId: "trc_live_otel",
|
||||
hwlabSessionId: "ses_live_otel",
|
||||
runId: "run_live_otel",
|
||||
commandId: "cmd_live_otel",
|
||||
topic: "hwlab.event.v1",
|
||||
partition: 2,
|
||||
offset: "14",
|
||||
sourceTopic: null,
|
||||
sourcePartition: null,
|
||||
sourceOffset: null,
|
||||
eventType: "assistant_message",
|
||||
terminal: false,
|
||||
valuesRedacted: true
|
||||
});
|
||||
assert.equal(JSON.stringify(attributes).includes("must-not-leak"), false);
|
||||
});
|
||||
|
||||
test("composable Kafka capabilities reject consumer group collisions", () => {
|
||||
assert.throws(
|
||||
() => kafkaEventBridgeConfig({
|
||||
...LIVE_ENV,
|
||||
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "true",
|
||||
HWLAB_KAFKA_PROJECTOR_GROUP_ID: LIVE_ENV.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID,
|
||||
HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS: "1000"
|
||||
}),
|
||||
(error) => error?.code === "hwlab_kafka_consumer_groups_not_distinct"
|
||||
);
|
||||
});
|
||||
|
||||
test("projection realtime subscribes to projection commits without enabling Kafka ingest or relay", async () => {
|
||||
let subscribeCount = 0;
|
||||
let unsubscribeCount = 0;
|
||||
const bridge = startHwlabKafkaEventBridge({
|
||||
env: {
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false",
|
||||
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
|
||||
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true"
|
||||
},
|
||||
runtimeStore: {
|
||||
async assertWorkbenchTransactionalRealtimeReady() { return { ready: true }; },
|
||||
async subscribeWorkbenchProjectionCommits() {
|
||||
subscribeCount += 1;
|
||||
return () => { unsubscribeCount += 1; };
|
||||
}
|
||||
},
|
||||
kafkaFactory() { throw new Error("projection realtime must not create a Kafka client"); },
|
||||
logger: null
|
||||
});
|
||||
|
||||
await bridge.ready;
|
||||
assert.equal(subscribeCount, 1);
|
||||
await bridge.stop();
|
||||
assert.equal(unsubscribeCount, 1);
|
||||
});
|
||||
|
||||
test("projects AgentRun run-created envelope even when no source event is present", () => {
|
||||
const projected = projectAgentRunKafkaEventToHwlabEvent({
|
||||
schema: "agentrun.event.v1",
|
||||
@@ -99,3 +356,308 @@ test("projects AgentRun run-created envelope even when no source event is presen
|
||||
assert.equal(projected.event.label, "agentrun:event:agentrun.run.created");
|
||||
assert.equal(projected.event.status, "running");
|
||||
});
|
||||
|
||||
test("accepts canonical runner-job-created payload with redacted tool credential references", () => {
|
||||
const message = canonicalMessage({ offset: "60" });
|
||||
const envelope = JSON.parse(message.value.toString("utf8"));
|
||||
envelope.event.type = "backend_status";
|
||||
envelope.event.payload = {
|
||||
traceId: envelope.traceId,
|
||||
hwlabSessionId: envelope.hwlabSessionId,
|
||||
commandId: envelope.commandId,
|
||||
phase: "runner-job-created",
|
||||
toolCredentials: [{
|
||||
tool: "unidesk-ssh",
|
||||
purpose: "ssh-passthrough",
|
||||
name: "agentrun-v02-tool-unidesk-ssh",
|
||||
namespace: "agentrun-v02",
|
||||
keys: ["UNIDESK_SSH_CLIENT_TOKEN"],
|
||||
projection: { kind: "env", envName: "UNIDESK_SSH_CLIENT_TOKEN" },
|
||||
valuesPrinted: false
|
||||
}],
|
||||
valuesPrinted: false
|
||||
};
|
||||
message.value = Buffer.from(JSON.stringify(envelope));
|
||||
|
||||
const decoded = decodeCanonicalAgentRunKafkaMessage({ topic: "agentrun.event.v1", partition: 0, message });
|
||||
assert.equal(decoded.ok, true);
|
||||
const projected = projectAgentRunKafkaEventToHwlabEvent(decoded.event, { source: "hwlab-test" });
|
||||
assert.equal(projected.event.label, "agentrun:backend:runner-job-created");
|
||||
});
|
||||
|
||||
test("Kafka projector commits next offset only after durable projection", async () => {
|
||||
const calls = [];
|
||||
let commitInput = null;
|
||||
const harness = kafkaProjectorHarness({
|
||||
async commitAgentRunKafkaProjection(input) {
|
||||
commitInput = input;
|
||||
calls.push("db");
|
||||
return { projectedSeq: 1 };
|
||||
}
|
||||
}, calls);
|
||||
const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null });
|
||||
await bridge.ready;
|
||||
await harness.runBatch(canonicalMessage({ offset: "41" }));
|
||||
|
||||
assert.ok(calls.indexOf("heartbeat") < calls.indexOf("db"));
|
||||
assert.ok(calls.indexOf("db") < calls.indexOf("resolve:41"));
|
||||
assert.equal(commitInput.partitionKey, "ses_projector_test");
|
||||
assert.deepEqual(calls.slice(-3), ["resolve:41", "commit:42", "heartbeat"]);
|
||||
await bridge.stop();
|
||||
});
|
||||
|
||||
test("Kafka projector leaves offset uncommitted when durable projection throws", async () => {
|
||||
const calls = [];
|
||||
const expected = Object.assign(new Error("database unavailable"), { code: "db_unavailable" });
|
||||
const harness = kafkaProjectorHarness({
|
||||
async commitAgentRunKafkaProjection() {
|
||||
calls.push("db");
|
||||
throw expected;
|
||||
}
|
||||
}, calls);
|
||||
const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null });
|
||||
await bridge.ready;
|
||||
|
||||
await assert.rejects(harness.runBatch(canonicalMessage({ offset: "42" })), expected);
|
||||
assert.equal(calls.some((entry) => entry.startsWith("commit:")), false);
|
||||
assert.equal(calls.some((entry) => entry.startsWith("resolve:")), false);
|
||||
await bridge.stop();
|
||||
});
|
||||
|
||||
test("Kafka projector signals committed projection but does not commit a stale generation offset", async () => {
|
||||
const calls = [];
|
||||
let signal = null;
|
||||
let harness;
|
||||
harness = kafkaProjectorHarness({
|
||||
async commitAgentRunKafkaProjection() {
|
||||
calls.push("db");
|
||||
harness.setStale(true);
|
||||
return { projectedSeq: 1 };
|
||||
}
|
||||
}, calls);
|
||||
const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null });
|
||||
bridge.subscribeProjectionCommits((value) => { signal = value; });
|
||||
await bridge.ready;
|
||||
|
||||
await harness.runBatch(canonicalMessage({ offset: "43" }));
|
||||
|
||||
assert.equal(calls.includes("resolve:43"), false);
|
||||
assert.equal(calls.includes("commit:44"), false);
|
||||
assert.equal(signal.sessionId, "ses_projector_test");
|
||||
assert.equal(signal.traceId, "trc_projector_test");
|
||||
await bridge.stop();
|
||||
});
|
||||
|
||||
test("Kafka projector commits a suppressed post-seal offset without publishing a projection signal", async () => {
|
||||
const calls = [];
|
||||
let signal = null;
|
||||
const harness = kafkaProjectorHarness({
|
||||
async commitAgentRunKafkaProjection() {
|
||||
calls.push("db");
|
||||
return { projectedSeq: 9, projectionWritten: false, suppressedAfterSeal: true };
|
||||
}
|
||||
}, calls);
|
||||
const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null });
|
||||
bridge.subscribeProjectionCommits((value) => { signal = value; });
|
||||
await bridge.ready;
|
||||
|
||||
await harness.runBatch(canonicalMessage({ offset: "44" }));
|
||||
|
||||
assert.equal(signal, null);
|
||||
assert.deepEqual(calls.slice(-3), ["resolve:44", "commit:45", "heartbeat"]);
|
||||
await bridge.stop();
|
||||
});
|
||||
|
||||
test("Kafka projector persists malformed and non-HWLAB events before committing offsets", async () => {
|
||||
const calls = [];
|
||||
const harness = kafkaProjectorHarness({}, calls);
|
||||
const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null });
|
||||
await bridge.ready;
|
||||
await harness.runBatch({ offset: "50", value: Buffer.from("{}") });
|
||||
await harness.runBatch(canonicalMessage({ offset: "51", traceId: null, hwlabSessionId: null }));
|
||||
|
||||
assert.deepEqual(calls.filter((entry) => ["dlq", "ignored", "commit:51", "commit:52"].includes(entry)), ["dlq", "commit:51", "ignored", "commit:52"]);
|
||||
await bridge.stop();
|
||||
});
|
||||
|
||||
test("Kafka projector exposes startup failure as rejected readiness and blocked status", async () => {
|
||||
const expected = Object.assign(new Error("broker unavailable"), { code: "kafka_unavailable" });
|
||||
const harness = kafkaProjectorHarness({}, []);
|
||||
harness.kafkaFactory = () => ({
|
||||
consumer: () => harness.consumer,
|
||||
producer: () => ({ ...harness.producer, connect: async () => { throw expected; } })
|
||||
});
|
||||
const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null });
|
||||
|
||||
await assert.rejects(bridge.ready, expected);
|
||||
assert.equal((await bridge.status()).status, "blocked");
|
||||
assert.equal((await bridge.status()).errorCode, "kafka_unavailable");
|
||||
await bridge.stop();
|
||||
});
|
||||
|
||||
test("cloud API Bun entry does not listen when Kafka projector startup fails", async () => {
|
||||
const expected = Object.assign(new Error("broker unavailable before listen"), { code: "kafka_startup_blocked" });
|
||||
const harness = kafkaProjectorHarness({}, []);
|
||||
let bunServeCalls = 0;
|
||||
const kafkaFactory = () => ({
|
||||
consumer: () => harness.consumer,
|
||||
producer: () => ({ ...harness.producer, connect: async () => { throw expected; } })
|
||||
});
|
||||
|
||||
await assert.rejects(createCloudApiBunServer({
|
||||
env: { ...PROJECTOR_ENV },
|
||||
runtimeStore: harness.runtimeStore,
|
||||
kafkaFactory,
|
||||
accessController: { async ensureBootstrap() {}, async authenticate() { return { ok: false }; } },
|
||||
bunServe() { bunServeCalls += 1; throw new Error("Bun.serve must not run"); },
|
||||
installSignalHandlers: false,
|
||||
logger: { log() {}, info() {}, warn() {}, error() {} }
|
||||
}), expected);
|
||||
|
||||
assert.equal(bunServeCalls, 0);
|
||||
});
|
||||
|
||||
test("cloud readiness is blocked while the Kafka projector is blocked", () => {
|
||||
const readiness = buildCloudApiReadiness({
|
||||
db: {},
|
||||
codeAgent: {},
|
||||
runtime: {},
|
||||
kafkaProjector: { started: true, status: "blocked", errorCode: "kafka_startup_blocked", message: "broker unavailable" }
|
||||
});
|
||||
|
||||
assert.equal(readiness.components.kafkaProjector, "blocked");
|
||||
assert.equal(readiness.blockerCodes.includes("kafka_startup_blocked"), true);
|
||||
assert.equal(readiness.ready, false);
|
||||
});
|
||||
|
||||
test("HWLAB Kafka relay claims just in time and forwards fencing token", async () => {
|
||||
const calls = [];
|
||||
const pending = [
|
||||
{ outboxSeq: 1, eventId: "evt-relay-1", topic: "hwlab.event.v1", partitionKey: "trc-relay", payload: { eventId: "evt-relay-1" }, headers: {}, attemptCount: 2, leaseOwner: "relay-owner", leaseExpiresAt: "2026-07-10T10:02:30.000Z" },
|
||||
{ outboxSeq: 2, eventId: "evt-relay-2", topic: "hwlab.event.v1", partitionKey: "trc-relay", payload: { eventId: "evt-relay-2" }, headers: {}, attemptCount: 1, leaseOwner: "relay-owner", leaseExpiresAt: "2026-07-10T10:02:30.000Z" }
|
||||
];
|
||||
const runtimeStore = {
|
||||
async claimHwlabKafkaOutbox(params) { calls.push(`claim:${params.limit}`); return pending.length ? [pending.shift()] : []; },
|
||||
async completeHwlabKafkaOutbox(item) { calls.push(`complete:${item.outboxSeq}:${item.attemptCount}:${item.leaseOwner}`); }
|
||||
};
|
||||
const producer = { async send(record) { calls.push(`send:${record.messages[0].value.includes("evt-relay-1") ? 1 : 2}`); } };
|
||||
const config = { relay: { batchSize: 2, leaseMs: 30000, retryBackoffMs: 1000 } };
|
||||
|
||||
const result = await relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner: "relay-owner", logger: null });
|
||||
|
||||
assert.equal(result.deliveredCount, 2);
|
||||
assert.deepEqual(calls, ["claim:1", "send:1", "complete:1:2:relay-owner", "claim:1", "send:2", "complete:2:1:relay-owner"]);
|
||||
});
|
||||
|
||||
test("HWLAB Kafka relay retries a failed send and clears backlog on the next cycle", async () => {
|
||||
const calls = [];
|
||||
let pending = true;
|
||||
let attemptCount = 0;
|
||||
const runtimeStore = {
|
||||
async claimHwlabKafkaOutbox() {
|
||||
if (!pending) return [];
|
||||
attemptCount += 1;
|
||||
return [{ outboxSeq: 7, eventId: "evt-relay-retry", topic: "hwlab.event.v1", partitionKey: "trc-relay-retry", payload: { eventId: "evt-relay-retry" }, headers: {}, attemptCount, leaseOwner: "relay-retry-owner", leaseExpiresAt: `2026-07-10T10:0${attemptCount}:30.000Z` }];
|
||||
},
|
||||
async completeHwlabKafkaOutbox(item) {
|
||||
calls.push(`complete:${item.attemptCount}`);
|
||||
pending = false;
|
||||
},
|
||||
async retryHwlabKafkaOutbox(item, error, retryAt) {
|
||||
calls.push(`retry:${item.attemptCount}:${error.message}:${retryAt}`);
|
||||
},
|
||||
async hwlabKafkaProjectorStatus() {
|
||||
return { backlogCount: pending ? 1 : 0, retryCount: pending && attemptCount > 0 ? 1 : 0, failedInboxCount: 0, dlqCount: 0 };
|
||||
}
|
||||
};
|
||||
let sendCount = 0;
|
||||
const producer = {
|
||||
async send() {
|
||||
sendCount += 1;
|
||||
calls.push(`send:${sendCount}`);
|
||||
if (sendCount === 1) throw new Error("broker unavailable");
|
||||
}
|
||||
};
|
||||
const config = { relay: { batchSize: 1, leaseMs: 30000, sendTimeoutMs: 10000, retryBackoffMs: 1000 } };
|
||||
|
||||
const failedCycle = await relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner: "relay-retry-owner", now: () => "2026-07-10T10:00:00.000Z", logger: null });
|
||||
assert.deepEqual(failedCycle, { claimedCount: 1, deliveredCount: 0, retryCount: 1, valuesPrinted: false });
|
||||
assert.deepEqual(await runtimeStore.hwlabKafkaProjectorStatus(), { backlogCount: 1, retryCount: 1, failedInboxCount: 0, dlqCount: 0 });
|
||||
|
||||
const recoveredCycle = await relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner: "relay-retry-owner", now: () => "2026-07-10T10:00:02.000Z", logger: null });
|
||||
assert.deepEqual(recoveredCycle, { claimedCount: 1, deliveredCount: 1, retryCount: 0, valuesPrinted: false });
|
||||
assert.deepEqual(await runtimeStore.hwlabKafkaProjectorStatus(), { backlogCount: 0, retryCount: 0, failedInboxCount: 0, dlqCount: 0 });
|
||||
assert.deepEqual(calls, [
|
||||
"send:1",
|
||||
"retry:1:broker unavailable:2026-07-10T10:00:01.000Z",
|
||||
"send:2",
|
||||
"complete:2"
|
||||
]);
|
||||
});
|
||||
|
||||
function kafkaProjectorHarness(overrides = {}, calls = []) {
|
||||
let runOptions = null;
|
||||
let stale = false;
|
||||
const consumer = {
|
||||
async connect() {},
|
||||
async subscribe() {},
|
||||
async run(options) { runOptions = options; },
|
||||
async commitOffsets(offsets) { calls.push(`commit:${offsets[0].offset}`); },
|
||||
async stop() {},
|
||||
async disconnect() {}
|
||||
};
|
||||
const producer = { async connect() {}, async send() {}, async disconnect() {} };
|
||||
const runtimeStore = {
|
||||
async assertWorkbenchTransactionalRealtimeReady() { return { ready: true }; },
|
||||
async subscribeWorkbenchProjectionCommits() { return () => {}; },
|
||||
async commitAgentRunKafkaProjection() { return { projectedSeq: 1 }; },
|
||||
async recordFailedAgentRunKafkaMessage() { calls.push("dlq"); },
|
||||
async recordIgnoredAgentRunKafkaMessage() { calls.push("ignored"); return { ignored: true }; },
|
||||
async claimHwlabKafkaOutbox() { return []; },
|
||||
async completeHwlabKafkaOutbox() {},
|
||||
async retryHwlabKafkaOutbox() {},
|
||||
async hwlabKafkaProjectorStatus() { return { backlogCount: 0 }; },
|
||||
...overrides
|
||||
};
|
||||
return {
|
||||
consumer,
|
||||
producer,
|
||||
runtimeStore,
|
||||
kafkaFactory: () => ({ consumer: () => consumer, producer: () => producer }),
|
||||
setStale(value) { stale = value === true; },
|
||||
async runBatch(message) {
|
||||
assert.ok(runOptions?.eachBatch);
|
||||
await runOptions.eachBatch({
|
||||
batch: { topic: "agentrun.event.v1", partition: 2, messages: [message] },
|
||||
resolveOffset(offset) { calls.push(`resolve:${offset}`); },
|
||||
async heartbeat() { calls.push("heartbeat"); },
|
||||
isRunning: () => true,
|
||||
isStale: () => stale
|
||||
});
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function canonicalMessage({ offset, traceId = "trc_projector_test", hwlabSessionId = "ses_projector_test" }) {
|
||||
const eventId = `evt_projector_${offset}`;
|
||||
const runId = "run_projector_test";
|
||||
return {
|
||||
offset,
|
||||
key: Buffer.from(runId),
|
||||
value: Buffer.from(JSON.stringify({
|
||||
schema: "agentrun.event.v1",
|
||||
eventType: "agentrun.event.committed",
|
||||
source: "agentrun-v02",
|
||||
eventId,
|
||||
sourceSeq: Number(offset) + 1,
|
||||
traceId,
|
||||
hwlabSessionId,
|
||||
runId,
|
||||
commandId: "cmd_projector_test",
|
||||
run: { runId, status: "running", valuesPrinted: false },
|
||||
command: { commandId: "cmd_projector_test", runId, seq: 1, type: "prompt", state: "running", valuesPrinted: false },
|
||||
event: { id: eventId, runId, seq: Number(offset) + 1, type: "assistant_message", payload: { traceId, hwlabSessionId, commandId: "cmd_projector_test", text: "projected answer" }, createdAt: "2026-07-10T10:00:00.000Z" },
|
||||
valuesPrinted: false
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
@@ -4,64 +4,191 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
|
||||
import { Kafka, logLevel } from "kafkajs";
|
||||
import { emitCodeAgentOtelSpan } from "./otel-trace.ts";
|
||||
import { buildWorkbenchProjectionEventFacts } from "./workbench-projection-writer.ts";
|
||||
import {
|
||||
WORKBENCH_REALTIME_CAPABILITY_ENVS,
|
||||
workbenchRealtimeCapabilities
|
||||
} from "./workbench-realtime-capabilities.ts";
|
||||
|
||||
const TRUE_VALUES = new Set(["1", "true", "yes", "on"]);
|
||||
const DEFAULT_AGENTRUN_EVENT_TOPIC = "agentrun.event.v1";
|
||||
const DEFAULT_HWLAB_EVENT_TOPIC = "hwlab.event.v1";
|
||||
const DEFAULT_STDIO_TOPIC = "codex-stdio.raw.v1";
|
||||
const DEFAULT_CLIENT_ID = "hwlab-v03-cloud-api";
|
||||
const DEFAULT_GROUP_ID = "hwlab-v03-agentrun-event-bridge";
|
||||
const DEFAULT_QUERY_TIMEOUT_MS = 5000;
|
||||
const DEFAULT_QUERY_LIMIT = 50;
|
||||
const LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS = 32;
|
||||
const REQUIRED_KAFKA_ENV = Object.freeze([
|
||||
"HWLAB_KAFKA_BOOTSTRAP_SERVERS",
|
||||
"HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC",
|
||||
"HWLAB_KAFKA_EVENT_TOPIC",
|
||||
"HWLAB_KAFKA_CLIENT_ID"
|
||||
]);
|
||||
|
||||
export function kafkaEventBridgeConfig(env = process.env) {
|
||||
if (!truthy(env.HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED ?? env.HWLAB_KAFKA_AGENTRUN_CONSUME_ENABLED ?? env.HWLAB_KAFKA_ENABLED)) return null;
|
||||
const featureEnvPresent = Object.values(WORKBENCH_REALTIME_CAPABILITY_ENVS).some((name) => stringValue(env[name]));
|
||||
const legacyKafkaEnabled = truthy(env.HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED ?? env.HWLAB_KAFKA_AGENTRUN_CONSUME_ENABLED ?? env.HWLAB_KAFKA_ENABLED);
|
||||
if (!featureEnvPresent && !legacyKafkaEnabled) return null;
|
||||
const capabilities = workbenchRealtimeCapabilities(env);
|
||||
const kafkaEnabled = capabilities.directPublish || capabilities.liveKafkaSse || capabilities.transactionalProjector || capabilities.projectionOutboxRelay;
|
||||
const transactionalEnabled = capabilities.transactionalProjector || capabilities.projectionOutboxRelay || capabilities.projectionRealtime;
|
||||
if (!kafkaEnabled && !transactionalEnabled) return null;
|
||||
const required = [...REQUIRED_KAFKA_ENV];
|
||||
if (!kafkaEnabled) required.length = 0;
|
||||
if (capabilities.directPublish) required.push("HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID");
|
||||
if (capabilities.liveKafkaSse) required.push("HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID");
|
||||
if (capabilities.transactionalProjector) required.push("HWLAB_KAFKA_PROJECTOR_GROUP_ID", "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS");
|
||||
if (capabilities.projectionOutboxRelay) required.push(
|
||||
"HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS",
|
||||
"HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE",
|
||||
"HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS",
|
||||
"HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS",
|
||||
"HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS"
|
||||
);
|
||||
const missing = required.filter((name) => !stringValue(env[name]));
|
||||
if (missing.length > 0) {
|
||||
const error = new Error(`HWLAB Kafka capability configuration is incomplete: ${missing.join(", ")}`);
|
||||
error.code = "hwlab_kafka_realtime_config_incomplete";
|
||||
error.missing = missing;
|
||||
throw error;
|
||||
}
|
||||
const brokers = csv(env.HWLAB_KAFKA_BOOTSTRAP_SERVERS);
|
||||
if (brokers.length === 0) return null;
|
||||
const agentRunTopic = stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC) || DEFAULT_AGENTRUN_EVENT_TOPIC;
|
||||
const hwlabTopic = stringValue(env.HWLAB_KAFKA_EVENT_TOPIC) || DEFAULT_HWLAB_EVENT_TOPIC;
|
||||
const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID) || DEFAULT_CLIENT_ID;
|
||||
const groupId = stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID ?? env.HWLAB_KAFKA_AGENTRUN_CONSUMER_GROUP_ID) || DEFAULT_GROUP_ID;
|
||||
return { brokers, agentRunTopic, hwlabTopic, clientId, groupId };
|
||||
const agentRunTopic = stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC);
|
||||
const hwlabTopic = stringValue(env.HWLAB_KAFKA_EVENT_TOPIC);
|
||||
const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID);
|
||||
const relay = capabilities.projectionOutboxRelay ? {
|
||||
intervalMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS, "HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS"),
|
||||
batchSize: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE, "HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE"),
|
||||
leaseMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS, "HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS"),
|
||||
sendTimeoutMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS, "HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS"),
|
||||
retryBackoffMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS, "HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS")
|
||||
} : null;
|
||||
if (relay && relay.sendTimeoutMs + 5000 >= relay.leaseMs) {
|
||||
const error = new Error("HWLAB Kafka relay lease must exceed producer send timeout by at least 5000ms.");
|
||||
error.code = "hwlab_kafka_relay_lease_invalid";
|
||||
throw error;
|
||||
}
|
||||
const enabledConsumerGroups = [
|
||||
capabilities.directPublish ? stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID) : null,
|
||||
capabilities.transactionalProjector ? stringValue(env.HWLAB_KAFKA_PROJECTOR_GROUP_ID) : null,
|
||||
capabilities.liveKafkaSse ? stringValue(env.HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID) : null
|
||||
].filter(Boolean);
|
||||
if (new Set(enabledConsumerGroups).size !== enabledConsumerGroups.length) {
|
||||
const error = new Error("HWLAB Kafka direct publisher, projector, and live SSE consumer groups must be distinct when enabled together.");
|
||||
error.code = "hwlab_kafka_consumer_groups_not_distinct";
|
||||
error.groupIds = enabledConsumerGroups;
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
capabilities,
|
||||
brokers,
|
||||
agentRunTopic,
|
||||
hwlabTopic,
|
||||
clientId,
|
||||
directPublishGroupId: stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID),
|
||||
projectorGroupId: stringValue(env.HWLAB_KAFKA_PROJECTOR_GROUP_ID),
|
||||
hwlabEventGroupId: stringValue(env.HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID),
|
||||
projectorHeartbeatIntervalMs: capabilities.transactionalProjector ? strictPositiveInteger(env.HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS, "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS") : null,
|
||||
relay
|
||||
};
|
||||
}
|
||||
|
||||
export function startHwlabKafkaEventBridge({ env = process.env, logger = console, kafkaFactory = defaultKafkaFactory } = {}) {
|
||||
export function startHwlabKafkaEventBridge({ env = process.env, logger = console, kafkaFactory = defaultKafkaFactory, runtimeStore = null, now = () => new Date().toISOString(), otelSpanEmitter = emitCodeAgentOtelSpan } = {}) {
|
||||
const config = kafkaEventBridgeConfig(env);
|
||||
if (!config) return { started: false, reason: "disabled_or_unconfigured", stop() {}, valuesPrinted: false };
|
||||
if (!config) return { started: false, reason: "disabled_or_unconfigured", stop() {}, subscribeProjectionCommits() { return () => {}; }, valuesPrinted: false };
|
||||
const components = [];
|
||||
if (config.capabilities.directPublish || config.capabilities.liveKafkaSse) {
|
||||
components.push(startLiveHwlabKafkaEventBridge({ config, env, logger, kafkaFactory, otelSpanEmitter }));
|
||||
}
|
||||
if (config.capabilities.transactionalProjector || config.capabilities.projectionOutboxRelay || config.capabilities.projectionRealtime) {
|
||||
components.push(startTransactionalHwlabKafkaEventBridge({ config, logger, kafkaFactory, runtimeStore, now }));
|
||||
}
|
||||
return components.length === 1 ? components[0] : combineKafkaEventBridgeComponents(config, components);
|
||||
}
|
||||
|
||||
function startTransactionalHwlabKafkaEventBridge({ config, logger = console, kafkaFactory = defaultKafkaFactory, runtimeStore = null, now = () => new Date().toISOString() } = {}) {
|
||||
requireKafkaProjectorStore(runtimeStore, config.capabilities);
|
||||
|
||||
let stopped = false;
|
||||
let consumer = null;
|
||||
let producer = null;
|
||||
let relayTimer = null;
|
||||
let relayRunning = false;
|
||||
let stopProjectionNotifications = null;
|
||||
const projectionCommitListeners = new Set();
|
||||
const publishProjectionCommit = (signal = {}) => {
|
||||
for (const listener of [...projectionCommitListeners]) {
|
||||
try { listener({ sessionId: signal.sessionId ?? null, traceId: signal.traceId ?? null, outboxSeq: signal.outboxSeq ?? null, recovery: signal.recovery === true, valuesRedacted: true }); } catch {}
|
||||
}
|
||||
};
|
||||
const relayOwner = `${config.clientId}:outbox-relay:${randomUUID()}`;
|
||||
let startupError = null;
|
||||
let startupReady = false;
|
||||
const ready = (async () => {
|
||||
const kafka = kafkaFactory(config);
|
||||
consumer = kafka.consumer({ groupId: config.groupId, allowAutoTopicCreation: false });
|
||||
producer = kafka.producer({ allowAutoTopicCreation: false });
|
||||
await producer.connect();
|
||||
await consumer.connect();
|
||||
await consumer.subscribe({ topic: config.agentRunTopic, fromBeginning: false });
|
||||
await consumer.run({
|
||||
eachMessage: async ({ topic, partition, message }) => {
|
||||
if (stopped) return;
|
||||
const projected = projectAgentRunKafkaMessageToHwlabEvent({ topic, partition, message }, { source: config.clientId });
|
||||
if (!projected) return;
|
||||
await producer.send({
|
||||
topic: config.hwlabTopic,
|
||||
messages: [{
|
||||
key: kafkaMessageKey(projected),
|
||||
value: JSON.stringify(projected),
|
||||
headers: kafkaHeaders(projected)
|
||||
}]
|
||||
});
|
||||
await runtimeStore.assertWorkbenchTransactionalRealtimeReady();
|
||||
const kafka = config.capabilities.transactionalProjector || config.capabilities.projectionOutboxRelay ? kafkaFactory(config) : null;
|
||||
if (config.capabilities.transactionalProjector) consumer = kafka.consumer({ groupId: config.projectorGroupId, allowAutoTopicCreation: false });
|
||||
if (config.capabilities.projectionOutboxRelay) producer = kafka.producer({ allowAutoTopicCreation: false });
|
||||
if (producer) await producer.connect();
|
||||
if (consumer) await consumer.connect();
|
||||
if (config.capabilities.projectionRealtime && typeof runtimeStore.subscribeWorkbenchProjectionCommits === "function") {
|
||||
stopProjectionNotifications = await runtimeStore.subscribeWorkbenchProjectionCommits(publishProjectionCommit);
|
||||
}
|
||||
if (consumer) await consumer.subscribe({ topic: config.agentRunTopic, fromBeginning: false });
|
||||
if (consumer) await consumer.run({
|
||||
autoCommit: false,
|
||||
eachBatchAutoResolve: false,
|
||||
eachBatch: async ({ batch, resolveOffset, heartbeat, isRunning, isStale }) => {
|
||||
for (const message of batch.messages) {
|
||||
if (stopped || !isRunning() || isStale()) break;
|
||||
await heartbeat();
|
||||
const heartbeatLoop = startKafkaHeartbeatLoop(heartbeat, config.projectorHeartbeatIntervalMs);
|
||||
let projection;
|
||||
try {
|
||||
projection = await projectAgentRunKafkaMessage({ topic: batch.topic, partition: batch.partition, message }, { runtimeStore, config, logger });
|
||||
} finally {
|
||||
await heartbeatLoop.stop();
|
||||
}
|
||||
if (projection?.projected) publishProjectionCommit(projection);
|
||||
if (heartbeatLoop.error) throw heartbeatLoop.error;
|
||||
if (stopped || !isRunning() || isStale()) {
|
||||
logWarn(logger, "hwlab-kafka-offset-commit-skipped", { topic: batch.topic, partition: batch.partition, offset: message.offset, reason: isStale() ? "stale_generation" : "consumer_stopped", valuesPrinted: false });
|
||||
break;
|
||||
}
|
||||
resolveOffset(message.offset);
|
||||
await consumer.commitOffsets([{ topic: batch.topic, partition: batch.partition, offset: nextKafkaOffset(message.offset) }]);
|
||||
await heartbeat();
|
||||
}
|
||||
}
|
||||
});
|
||||
const runRelay = async () => {
|
||||
if (!config.capabilities.projectionOutboxRelay) return;
|
||||
if (stopped || relayRunning) return;
|
||||
relayRunning = true;
|
||||
try {
|
||||
await relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner: relayOwner, now, logger });
|
||||
} finally {
|
||||
relayRunning = false;
|
||||
}
|
||||
};
|
||||
if (config.capabilities.projectionOutboxRelay) {
|
||||
relayTimer = setInterval(() => { void runRelay().catch((error) => logWarn(logger, "hwlab-kafka-outbox-relay-failed", { message: errorMessage(error), valuesPrinted: false })); }, config.relay.intervalMs);
|
||||
relayTimer.unref?.();
|
||||
await runRelay();
|
||||
}
|
||||
startupReady = true;
|
||||
logInfo(logger, "hwlab-kafka-event-bridge-started", {
|
||||
agentRunTopic: config.agentRunTopic,
|
||||
hwlabTopic: config.hwlabTopic,
|
||||
clientId: config.clientId,
|
||||
groupId: config.groupId,
|
||||
projectorGroupId: config.projectorGroupId,
|
||||
capabilities: config.capabilities,
|
||||
valuesPrinted: false
|
||||
});
|
||||
})().catch((error) => {
|
||||
})();
|
||||
void ready.catch((error) => {
|
||||
startupError = error;
|
||||
logWarn(logger, "hwlab-kafka-event-bridge-start-failed", {
|
||||
message: errorMessage(error),
|
||||
agentRunTopic: config.agentRunTopic,
|
||||
@@ -76,12 +203,386 @@ export function startHwlabKafkaEventBridge({ env = process.env, logger = console
|
||||
ready,
|
||||
async stop() {
|
||||
stopped = true;
|
||||
if (relayTimer) clearInterval(relayTimer);
|
||||
if (typeof stopProjectionNotifications === "function") {
|
||||
try { await stopProjectionNotifications(); } catch {}
|
||||
}
|
||||
if (consumer?.stop) await consumer.stop().catch(() => undefined);
|
||||
await Promise.allSettled([consumer?.disconnect?.(), producer?.disconnect?.()]);
|
||||
},
|
||||
async status() {
|
||||
if (startupError) return { status: "blocked", errorCode: startupError.code ?? "hwlab_kafka_projector_start_failed", message: errorMessage(startupError), valuesRedacted: true };
|
||||
if (!startupReady) return { status: "initializing", valuesRedacted: true };
|
||||
const transactionalReadiness = typeof runtimeStore.workbenchTransactionalRealtimeReadiness === "function"
|
||||
? await runtimeStore.workbenchTransactionalRealtimeReadiness()
|
||||
: { ready: true, valuesRedacted: true };
|
||||
const projectorStatus = config.capabilities.transactionalProjector || config.capabilities.projectionOutboxRelay
|
||||
? await runtimeStore.hwlabKafkaProjectorStatus()
|
||||
: {};
|
||||
return { status: transactionalReadiness.ready === false ? "blocked" : "ready", capabilities: config.capabilities, transactionalReadiness, ...projectorStatus, valuesRedacted: true };
|
||||
},
|
||||
startupStatus() {
|
||||
if (startupError) return { status: "blocked", errorCode: startupError.code ?? "hwlab_kafka_projector_start_failed", message: errorMessage(startupError), valuesRedacted: true };
|
||||
return { status: startupReady ? "ready" : "initializing", valuesRedacted: true };
|
||||
},
|
||||
subscribeProjectionCommits(listener) {
|
||||
if (typeof listener !== "function") return () => {};
|
||||
projectionCommitListeners.add(listener);
|
||||
return () => projectionCommitListeners.delete(listener);
|
||||
},
|
||||
valuesPrinted: false
|
||||
};
|
||||
}
|
||||
|
||||
function startLiveHwlabKafkaEventBridge({ config, env = process.env, logger = console, kafkaFactory = defaultKafkaFactory, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) {
|
||||
let stopped = false;
|
||||
let bridgeConsumer = null;
|
||||
let fanoutConsumer = null;
|
||||
let producer = null;
|
||||
let startupError = null;
|
||||
let startupReady = false;
|
||||
const subscribers = new Set();
|
||||
const lagByPartition = new Map();
|
||||
const publishLiveEnvelope = (envelope, transport) => {
|
||||
for (const listener of [...subscribers]) {
|
||||
try {
|
||||
listener(envelope, transport);
|
||||
} catch (error) {
|
||||
logWarn(logger, "hwlab-live-kafka-subscriber-failed", { message: errorMessage(error), valuesPrinted: false });
|
||||
}
|
||||
}
|
||||
};
|
||||
const ready = (async () => {
|
||||
const kafka = kafkaFactory(config);
|
||||
if (config.capabilities.directPublish) {
|
||||
bridgeConsumer = kafka.consumer({ groupId: config.directPublishGroupId, allowAutoTopicCreation: false });
|
||||
producer = kafka.producer({ allowAutoTopicCreation: false });
|
||||
}
|
||||
if (config.capabilities.liveKafkaSse) fanoutConsumer = kafka.consumer({ groupId: config.hwlabEventGroupId, allowAutoTopicCreation: false });
|
||||
if (producer) await producer.connect();
|
||||
if (bridgeConsumer) await bridgeConsumer.connect();
|
||||
if (fanoutConsumer) await fanoutConsumer.connect();
|
||||
if (bridgeConsumer) await bridgeConsumer.subscribe({ topic: config.agentRunTopic, fromBeginning: false });
|
||||
if (fanoutConsumer) await fanoutConsumer.subscribe({ topic: config.hwlabTopic, fromBeginning: false });
|
||||
if (fanoutConsumer) await fanoutConsumer.run({
|
||||
eachBatchAutoResolve: false,
|
||||
eachBatch: async ({ batch, resolveOffset, heartbeat, isRunning, isStale }) => {
|
||||
let lastOffset = null;
|
||||
for (const message of batch.messages) {
|
||||
if (stopped || !isRunning() || isStale()) break;
|
||||
const envelope = parseJson(message?.value ? Buffer.from(message.value).toString("utf8") : "");
|
||||
if (!envelope || envelope.schema !== "hwlab.event.v1") {
|
||||
logWarn(logger, "hwlab-live-kafka-envelope-ignored", { reason: "schema-invalid", valuesPrinted: false });
|
||||
} else {
|
||||
const transport = { topic: batch.topic, partition: batch.partition, offset: message.offset };
|
||||
emitLiveKafkaOtelSpan("hwlab.kafka.live.fanout_receive", envelope, transport, { env, otelSpanEmitter });
|
||||
publishLiveEnvelope(envelope, transport);
|
||||
}
|
||||
resolveOffset(message.offset);
|
||||
lastOffset = message.offset;
|
||||
await heartbeat();
|
||||
}
|
||||
if (lastOffset !== null) lagByPartition.set(batch.partition, kafkaPartitionLag(batch.highWatermark, lastOffset));
|
||||
}
|
||||
});
|
||||
if (bridgeConsumer) await bridgeConsumer.run({
|
||||
eachMessage: async ({ topic, partition, message }) => {
|
||||
if (stopped) return;
|
||||
await publishAgentRunKafkaMessageLive({ topic, partition, message }, { producer, config, env, logger, otelSpanEmitter });
|
||||
}
|
||||
});
|
||||
startupReady = true;
|
||||
logInfo(logger, "hwlab-live-kafka-event-bridge-started", {
|
||||
capabilities: config.capabilities,
|
||||
agentRunTopic: config.agentRunTopic,
|
||||
hwlabTopic: config.hwlabTopic,
|
||||
clientId: config.clientId,
|
||||
directPublishGroupId: config.directPublishGroupId,
|
||||
hwlabEventGroupId: config.hwlabEventGroupId,
|
||||
valuesPrinted: false
|
||||
});
|
||||
})();
|
||||
void ready.catch((error) => {
|
||||
startupError = error;
|
||||
logWarn(logger, "hwlab-live-kafka-event-bridge-start-failed", {
|
||||
message: errorMessage(error),
|
||||
agentRunTopic: config.agentRunTopic,
|
||||
hwlabTopic: config.hwlabTopic,
|
||||
valuesPrinted: false
|
||||
});
|
||||
});
|
||||
return {
|
||||
started: true,
|
||||
...config,
|
||||
ready,
|
||||
async stop() {
|
||||
stopped = true;
|
||||
subscribers.clear();
|
||||
await Promise.allSettled([bridgeConsumer?.stop?.(), fanoutConsumer?.stop?.()]);
|
||||
await Promise.allSettled([bridgeConsumer?.disconnect?.(), fanoutConsumer?.disconnect?.(), producer?.disconnect?.()]);
|
||||
},
|
||||
async status() {
|
||||
if (startupError) return { status: "blocked", capabilities: config.capabilities, errorCode: startupError.code ?? "hwlab_live_kafka_start_failed", message: errorMessage(startupError), valuesRedacted: true };
|
||||
return { status: startupReady ? "ready" : "initializing", capabilities: config.capabilities, subscriberCount: subscribers.size, consumerLag: liveKafkaLagSummary(lagByPartition), lossPossible: true, valuesRedacted: true };
|
||||
},
|
||||
startupStatus() {
|
||||
if (startupError) return { status: "blocked", capabilities: config.capabilities, errorCode: startupError.code ?? "hwlab_live_kafka_start_failed", message: errorMessage(startupError), valuesRedacted: true };
|
||||
return { status: startupReady ? "ready" : "initializing", capabilities: config.capabilities, valuesRedacted: true };
|
||||
},
|
||||
subscribeProjectionCommits() { return () => {}; },
|
||||
subscribeLiveHwlabEvents(listener) {
|
||||
if (typeof listener !== "function") return () => {};
|
||||
subscribers.add(listener);
|
||||
return () => subscribers.delete(listener);
|
||||
},
|
||||
liveSubscriberCount() { return subscribers.size; },
|
||||
valuesPrinted: false
|
||||
};
|
||||
}
|
||||
|
||||
function kafkaPartitionLag(highWatermark, lastOffset) {
|
||||
try {
|
||||
const lag = BigInt(String(highWatermark ?? "0")) - (BigInt(String(lastOffset ?? "0")) + 1n);
|
||||
return Number(lag > 0n ? lag : 0n);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function liveKafkaLagSummary(lagByPartition) {
|
||||
const partitions = [...lagByPartition.entries()].map(([partition, lag]) => ({ partition, lag })).sort((left, right) => left.partition - right.partition);
|
||||
const known = partitions.map((item) => item.lag).filter((lag) => Number.isFinite(lag));
|
||||
return {
|
||||
known: known.length > 0,
|
||||
total: known.length > 0 ? known.reduce((sum, lag) => sum + lag, 0) : null,
|
||||
partitions,
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function combineKafkaEventBridgeComponents(config, components) {
|
||||
const ready = Promise.all(components.map((component) => component.ready));
|
||||
const componentStartup = () => components.map((component) => component.startupStatus?.() ?? { status: "initializing" });
|
||||
return {
|
||||
started: true,
|
||||
...config,
|
||||
ready,
|
||||
async stop() { await Promise.allSettled(components.map((component) => component.stop())); },
|
||||
async status() {
|
||||
const statuses = await Promise.all(components.map((component) => component.status()));
|
||||
const blocked = statuses.find((status) => status.status === "blocked");
|
||||
const initializing = statuses.some((status) => status.status !== "ready");
|
||||
return {
|
||||
status: blocked ? "blocked" : initializing ? "initializing" : "ready",
|
||||
capabilities: config.capabilities,
|
||||
components: statuses,
|
||||
errorCode: blocked?.errorCode ?? null,
|
||||
message: blocked?.message ?? null,
|
||||
valuesRedacted: true
|
||||
};
|
||||
},
|
||||
startupStatus() {
|
||||
const statuses = componentStartup();
|
||||
const blocked = statuses.find((status) => status.status === "blocked");
|
||||
return {
|
||||
status: blocked ? "blocked" : statuses.every((status) => status.status === "ready") ? "ready" : "initializing",
|
||||
capabilities: config.capabilities,
|
||||
components: statuses,
|
||||
errorCode: blocked?.errorCode ?? null,
|
||||
message: blocked?.message ?? null,
|
||||
valuesRedacted: true
|
||||
};
|
||||
},
|
||||
subscribeProjectionCommits(listener) {
|
||||
const stops = components.map((component) => component.subscribeProjectionCommits?.(listener)).filter((stop) => typeof stop === "function");
|
||||
return () => stops.forEach((stop) => stop());
|
||||
},
|
||||
subscribeLiveHwlabEvents(listener) {
|
||||
const owner = components.find((component) => typeof component.subscribeLiveHwlabEvents === "function");
|
||||
return owner?.subscribeLiveHwlabEvents(listener) ?? (() => {});
|
||||
},
|
||||
liveSubscriberCount() {
|
||||
const owner = components.find((component) => typeof component.liveSubscriberCount === "function");
|
||||
return owner?.liveSubscriberCount() ?? 0;
|
||||
},
|
||||
valuesPrinted: false
|
||||
};
|
||||
}
|
||||
|
||||
export async function publishAgentRunKafkaMessageLive(kafkaMessage = {}, { producer, config, env = process.env, logger = console, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) {
|
||||
const transport = kafkaTransport(kafkaMessage);
|
||||
const decoded = decodeCanonicalAgentRunKafkaMessage(kafkaMessage);
|
||||
if (!decoded.ok) {
|
||||
logWarn(logger, "hwlab-live-kafka-message-invalid", { topic: transport.sourceTopic, partition: transport.sourcePartition, offset: transport.sourceOffset, errorCode: decoded.error.code, valuesPrinted: false });
|
||||
return { handled: true, invalid: true, published: false, valuesPrinted: false };
|
||||
}
|
||||
if (!stringValue(decoded.event.traceId) || !stringValue(decoded.event.hwlabSessionId)) {
|
||||
return { handled: true, ignored: true, published: false, valuesPrinted: false };
|
||||
}
|
||||
const projected = projectAgentRunKafkaEventToHwlabEvent(decoded.event, {
|
||||
source: config.clientId,
|
||||
sourceTopic: transport.sourceTopic,
|
||||
sourcePartition: transport.sourcePartition,
|
||||
sourceOffset: transport.sourceOffset,
|
||||
sourceKey: transport.sourceKey,
|
||||
inputSha256: transport.inputSha256
|
||||
});
|
||||
const publishMetadata = await producer.send({
|
||||
topic: config.hwlabTopic,
|
||||
messages: [{ key: kafkaMessageKey(projected), value: JSON.stringify(projected), headers: kafkaHeaders(projected) }]
|
||||
});
|
||||
const publishedRecord = Array.isArray(publishMetadata) ? publishMetadata[0] : null;
|
||||
emitLiveKafkaOtelSpan("hwlab.kafka.live.direct_publish", projected, {
|
||||
topic: config.hwlabTopic,
|
||||
partition: publishedRecord?.partition,
|
||||
offset: publishedRecord?.baseOffset ?? publishedRecord?.offset,
|
||||
sourceTopic: transport.sourceTopic,
|
||||
sourcePartition: transport.sourcePartition,
|
||||
sourceOffset: transport.sourceOffset
|
||||
}, { env, otelSpanEmitter });
|
||||
return { handled: true, published: true, envelope: projected, sessionId: projected.sessionId, traceId: projected.traceId, valuesPrinted: false };
|
||||
}
|
||||
|
||||
export function shouldEmitLiveKafkaOtelSpan(envelope = {}) {
|
||||
const event = objectValue(envelope.event);
|
||||
const context = objectValue(envelope.context);
|
||||
const eventType = firstText(context.agentRunEventType, event.eventType, event.type, envelope.eventType)?.toLowerCase();
|
||||
if (event.terminal === true || eventType === "terminal" || eventType === "terminal_status") return true;
|
||||
if (["assistant", "assistant_message", "tool", "tool_call"].includes(eventType)) return true;
|
||||
if (eventType !== "command_output" && event.type !== "output") return true;
|
||||
const sourceSeq = integerValue(context.sourceSeq ?? event.sourceSeq);
|
||||
if (Number.isInteger(sourceSeq) && sourceSeq > 0) return sourceSeq % LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS === 0;
|
||||
const identity = firstText(envelope.sourceEventId, envelope.eventId, envelope.traceId, envelope.hwlabSessionId, envelope.sessionId);
|
||||
if (!identity) return false;
|
||||
return createHash("sha256").update(identity).digest().readUInt32BE(0) % LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS === 0;
|
||||
}
|
||||
|
||||
export function liveKafkaOtelSpanAttributes(envelope = {}, transport = {}) {
|
||||
const event = objectValue(envelope.event);
|
||||
const context = objectValue(envelope.context);
|
||||
const sourceEvent = objectValue(envelope.sourceEvent);
|
||||
return {
|
||||
businessTraceId: firstText(envelope.traceId, event.traceId),
|
||||
hwlabSessionId: firstText(envelope.hwlabSessionId, envelope.sessionId, event.sessionId),
|
||||
runId: firstText(envelope.runId, context.runId, event.runId),
|
||||
commandId: firstText(envelope.commandId, context.commandId, event.commandId),
|
||||
topic: firstText(transport.topic, sourceEvent.topic),
|
||||
partition: integerValue(transport.partition ?? sourceEvent.partition),
|
||||
offset: firstText(transport.offset, sourceEvent.offset),
|
||||
sourceTopic: firstText(transport.sourceTopic),
|
||||
sourcePartition: integerValue(transport.sourcePartition),
|
||||
sourceOffset: firstText(transport.sourceOffset),
|
||||
eventType: firstText(context.agentRunEventType, event.eventType, event.type, envelope.eventType),
|
||||
terminal: event.terminal === true,
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
export function emitLiveKafkaOtelSpan(name, envelope = {}, transport = {}, { env = process.env, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) {
|
||||
if (typeof otelSpanEmitter !== "function" || !shouldEmitLiveKafkaOtelSpan(envelope)) return { emitted: false, valuesRedacted: true };
|
||||
const attributes = liveKafkaOtelSpanAttributes(envelope, transport);
|
||||
if (!attributes.businessTraceId) return { emitted: false, valuesRedacted: true };
|
||||
try {
|
||||
const pending = otelSpanEmitter(name, attributes.businessTraceId, env, { attributes });
|
||||
void Promise.resolve(pending).catch(() => undefined);
|
||||
} catch {
|
||||
// OTel is best effort and must never change the live Kafka delivery path.
|
||||
}
|
||||
return { emitted: true, valuesRedacted: true };
|
||||
}
|
||||
|
||||
export async function projectAgentRunKafkaMessage(kafkaMessage = {}, { runtimeStore, config, logger = console } = {}) {
|
||||
const transport = kafkaTransport(kafkaMessage);
|
||||
const decoded = decodeCanonicalAgentRunKafkaMessage(kafkaMessage);
|
||||
if (!decoded.ok) {
|
||||
await runtimeStore.recordFailedAgentRunKafkaMessage({ transport, sourceEventId: decoded.sourceEventId, envelope: decoded.envelope, error: decoded.error });
|
||||
logWarn(logger, "hwlab-kafka-message-dlq", { topic: transport.sourceTopic, partition: transport.sourcePartition, offset: transport.sourceOffset, errorCode: decoded.error.code, valuesPrinted: false });
|
||||
return { handled: true, invalid: true, valuesPrinted: false };
|
||||
}
|
||||
if (!stringValue(decoded.event.traceId) || !stringValue(decoded.event.hwlabSessionId)) {
|
||||
const ignored = await runtimeStore.recordIgnoredAgentRunKafkaMessage({ transport, envelope: decoded.event, reason: "hwlab-correlation-absent" });
|
||||
return { handled: true, ignored: true, duplicate: ignored?.duplicate === true, valuesPrinted: false };
|
||||
}
|
||||
const projected = projectAgentRunKafkaEventToHwlabEvent(decoded.event, {
|
||||
source: config.clientId,
|
||||
sourceTopic: transport.sourceTopic,
|
||||
sourcePartition: transport.sourcePartition,
|
||||
sourceOffset: transport.sourceOffset,
|
||||
sourceKey: transport.sourceKey,
|
||||
inputSha256: transport.inputSha256
|
||||
});
|
||||
const result = await runtimeStore.commitAgentRunKafkaProjection({
|
||||
transport,
|
||||
canonicalEvent: decoded.event,
|
||||
projectedEvent: projected.event,
|
||||
requestMeta: { traceId: projected.traceId, sessionId: projected.sessionId, agentSessionId: projected.sessionId, valuesPrinted: false },
|
||||
factsFactory: ({ projectedSeq, previousCheckpoint, projectedAt }) => buildWorkbenchProjectionEventFacts({ event: projected.event, requestMeta: { traceId: projected.traceId, sessionId: projected.sessionId }, previousCheckpoint, projectedSeq, projectedAt }),
|
||||
hwlabEvent: projected,
|
||||
hwlabTopic: config.hwlabTopic,
|
||||
partitionKey: kafkaMessageKey(projected),
|
||||
headers: kafkaHeaders(projected)
|
||||
});
|
||||
if (result.conflict) logWarn(logger, "hwlab-kafka-message-conflict", { topic: transport.sourceTopic, partition: transport.sourcePartition, offset: transport.sourceOffset, errorCode: result.diagnostic?.code, valuesPrinted: false });
|
||||
return {
|
||||
handled: true,
|
||||
projected: !result.duplicate && !result.conflict && result.projectionWritten !== false && result.suppressedAfterSeal !== true,
|
||||
duplicate: result.duplicate === true,
|
||||
conflict: result.conflict === true,
|
||||
suppressedAfterSeal: result.suppressedAfterSeal === true,
|
||||
projectedSeq: result.projectedSeq,
|
||||
sessionId: projected.sessionId,
|
||||
traceId: projected.traceId,
|
||||
valuesPrinted: false
|
||||
};
|
||||
}
|
||||
|
||||
export async function relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner, now = () => new Date().toISOString(), logger = console } = {}) {
|
||||
let claimedCount = 0;
|
||||
let deliveredCount = 0;
|
||||
let retryCount = 0;
|
||||
for (let index = 0; index < config.relay.batchSize; index += 1) {
|
||||
const item = (await runtimeStore.claimHwlabKafkaOutbox({ owner, leaseMs: config.relay.leaseMs, limit: 1 }))[0];
|
||||
if (!item) break;
|
||||
claimedCount += 1;
|
||||
try {
|
||||
await producer.send({ topic: item.topic, timeout: config.relay.sendTimeoutMs, messages: [{ key: item.partitionKey, value: JSON.stringify(item.payload), headers: item.headers }] });
|
||||
await runtimeStore.completeHwlabKafkaOutbox(item);
|
||||
deliveredCount += 1;
|
||||
} catch (error) {
|
||||
const retryAt = new Date(Date.parse(now()) + config.relay.retryBackoffMs).toISOString();
|
||||
await runtimeStore.retryHwlabKafkaOutbox(item, error, retryAt);
|
||||
retryCount += 1;
|
||||
logWarn(logger, "hwlab-kafka-outbox-publish-retry", { outboxSeq: item.outboxSeq, eventId: item.eventId, retryAt, message: errorMessage(error), valuesPrinted: false });
|
||||
break;
|
||||
}
|
||||
}
|
||||
return { claimedCount, deliveredCount, retryCount, valuesPrinted: false };
|
||||
}
|
||||
|
||||
export function startKafkaHeartbeatLoop(heartbeat, intervalMs) {
|
||||
let stopped = false;
|
||||
let running = null;
|
||||
let heartbeatError = null;
|
||||
const pulse = () => {
|
||||
if (stopped || running || typeof heartbeat !== "function") return;
|
||||
running = Promise.resolve()
|
||||
.then(() => heartbeat())
|
||||
.catch((error) => { heartbeatError ??= error; })
|
||||
.finally(() => { running = null; });
|
||||
};
|
||||
pulse();
|
||||
const timer = setInterval(pulse, intervalMs);
|
||||
timer.unref?.();
|
||||
return {
|
||||
get error() { return heartbeatError; },
|
||||
async stop() {
|
||||
stopped = true;
|
||||
clearInterval(timer);
|
||||
if (running) await running;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function projectAgentRunKafkaMessageToHwlabEvent(kafkaMessage = {}, options = {}) {
|
||||
const valueText = kafkaMessage?.message?.value ? Buffer.from(kafkaMessage.message.value).toString("utf8") : "";
|
||||
const input = parseJson(valueText);
|
||||
@@ -106,7 +607,8 @@ export function projectAgentRunKafkaEventToHwlabEvent(input = {}, options = {})
|
||||
const context = objectValue(input.context ?? payload.context);
|
||||
const sourceSeq = integerValue(sourceEvent.seq ?? input.seq ?? input.sourceSeq);
|
||||
const traceId = firstText(input.traceId, hwlab.traceId, context.traceId, sourceEvent.traceId, payload.traceId, payload.hwlabTraceId, payload.businessTraceId, payload.metadata?.traceId, command.traceId, run.traceId);
|
||||
const sessionId = firstText(input.sessionId, hwlab.sessionId, context.sessionId, run.sessionId, payload.sessionId, payload.hwlabSessionId);
|
||||
const sourceEventId = firstText(input.eventId, sourceEvent.id);
|
||||
const sessionId = firstText(input.hwlabSessionId, payload.hwlabSessionId, hwlab.sessionId, context.hwlabSessionId, input.sessionId, context.sessionId, run.hwlabSessionId, run.sessionId, payload.sessionId);
|
||||
const runId = firstText(run.runId, sourceEvent.runId, payload.runId, input.runId);
|
||||
const commandId = firstText(command.commandId, sourceEvent.commandId, payload.commandId, input.commandId);
|
||||
const event = mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run, command, traceId, sessionId, sourceSeq, runId, commandId });
|
||||
@@ -114,10 +616,15 @@ export function projectAgentRunKafkaEventToHwlabEvent(input = {}, options = {})
|
||||
return {
|
||||
schema: "hwlab.event.v1",
|
||||
eventType: "hwlab.trace.event.projected",
|
||||
eventId: sourceEventId ? `hwlab:${sourceEventId}` : null,
|
||||
sourceEventId,
|
||||
source: stringValue(options.source) || DEFAULT_CLIENT_ID,
|
||||
producedAt,
|
||||
traceId,
|
||||
hwlabSessionId: sessionId,
|
||||
sessionId,
|
||||
runId,
|
||||
commandId,
|
||||
context: {
|
||||
conversationId: firstText(input.conversationId, hwlab.conversationId, context.conversationId, run.conversationId, payload.conversationId),
|
||||
threadId: firstText(input.threadId, hwlab.threadId, context.threadId, run.threadId, payload.threadId),
|
||||
@@ -129,7 +636,7 @@ export function projectAgentRunKafkaEventToHwlabEvent(input = {}, options = {})
|
||||
agentRunEventType: firstText(sourceEvent.type, payload.type),
|
||||
valuesRedacted: true
|
||||
},
|
||||
event,
|
||||
event: { ...event, sourceEventId },
|
||||
sourceEvent: {
|
||||
schema: firstText(input.schema),
|
||||
topic: stringValue(options.sourceTopic),
|
||||
@@ -147,6 +654,86 @@ export function projectAgentRunKafkaEventToHwlabEvent(input = {}, options = {})
|
||||
};
|
||||
}
|
||||
|
||||
export function decodeCanonicalAgentRunKafkaMessage(kafkaMessage = {}) {
|
||||
const valueText = kafkaMessage?.message?.value ? Buffer.from(kafkaMessage.message.value).toString("utf8") : "";
|
||||
const envelope = parseJson(valueText);
|
||||
const sourceEventId = firstText(envelope?.eventId, envelope?.event?.id);
|
||||
try {
|
||||
assertCanonicalAgentRunEvent(envelope);
|
||||
return { ok: true, event: envelope, sourceEventId, inputSha256: sha256(valueText), valuesPrinted: false };
|
||||
} catch (error) {
|
||||
return { ok: false, envelope, sourceEventId, inputSha256: sha256(valueText), error: { code: error?.code ?? "workbench_kafka_message_invalid", message: errorMessage(error), valuesRedacted: true }, valuesPrinted: false };
|
||||
}
|
||||
}
|
||||
|
||||
function assertCanonicalAgentRunEvent(value) {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) throw contractError("workbench_kafka_json_invalid", "Kafka value must be a JSON object.");
|
||||
assertClosedObject(value, ["schema", "eventType", "source", "eventId", "outboxSeq", "sourceSeq", "committedAt", "traceId", "sessionId", "hwlabSessionId", "runId", "commandId", "run", "command", "event", "valuesPrinted"], "AgentRun envelope");
|
||||
if (value.schema !== "agentrun.event.v1" || value.eventType !== "agentrun.event.committed") throw contractError("workbench_kafka_schema_invalid", "Kafka value is not a canonical AgentRun committed event.");
|
||||
for (const [name, field] of [["eventId", value.eventId], ["runId", value.runId]]) {
|
||||
if (!stringValue(field)) throw contractError("workbench_kafka_schema_invalid", `Canonical AgentRun ${name} is required.`);
|
||||
}
|
||||
for (const [name, field] of [["traceId", value.traceId], ["hwlabSessionId", value.hwlabSessionId]]) {
|
||||
if (field !== null && field !== undefined && !stringValue(field)) throw contractError("workbench_kafka_schema_invalid", `Canonical AgentRun ${name} must be a string or null.`);
|
||||
}
|
||||
if (!Number.isInteger(Number(value.sourceSeq)) || Number(value.sourceSeq) <= 0) throw contractError("workbench_kafka_schema_invalid", "Canonical AgentRun sourceSeq must be a positive integer.");
|
||||
if (value.valuesPrinted !== false) throw contractError("workbench_kafka_redaction_invalid", "Canonical AgentRun event must declare valuesPrinted=false.");
|
||||
assertClosedObject(value.run, ["runId", "status", "terminalStatus", "failureKind", "tenantId", "projectId", "providerId", "backendProfile", "sessionId", "hwlabSessionId", "conversationId", "threadId", "valuesPrinted"], "AgentRun run");
|
||||
if (value.command !== null && value.command !== undefined) assertClosedObject(value.command, ["commandId", "runId", "seq", "type", "state", "payloadHash", "valuesPrinted"], "AgentRun command");
|
||||
assertClosedObject(value.event, ["id", "runId", "seq", "type", "payload", "createdAt"], "AgentRun event");
|
||||
if (value.event.id !== value.eventId || Number(value.event.seq) !== Number(value.sourceSeq)) throw contractError("workbench_kafka_identity_invalid", "Canonical AgentRun event identity does not match the envelope.");
|
||||
if (!value.event.payload || typeof value.event.payload !== "object" || Array.isArray(value.event.payload)) throw contractError("workbench_kafka_schema_invalid", "Canonical AgentRun event payload must be an object.");
|
||||
if (containsSecretLikeKey(value)) throw contractError("workbench_kafka_redaction_invalid", "Canonical AgentRun event contains a secret-like field.");
|
||||
return value;
|
||||
}
|
||||
|
||||
function assertClosedObject(value, allowed, label) {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) throw contractError("workbench_kafka_schema_invalid", `${label} must be an object.`);
|
||||
const unknown = Object.keys(value).filter((key) => !allowed.includes(key));
|
||||
if (unknown.length > 0) throw contractError("workbench_kafka_schema_invalid", `${label} contains unsupported fields: ${unknown.join(", ")}`);
|
||||
}
|
||||
|
||||
function containsSecretLikeKey(value, depth = 0) {
|
||||
if (depth > 12 || value === null || value === undefined) return false;
|
||||
if (Array.isArray(value)) return value.some((item) => containsSecretLikeKey(item, depth + 1));
|
||||
if (typeof value !== "object") return false;
|
||||
return Object.entries(value).some(([key, entry]) => {
|
||||
const normalized = key.replace(/[^a-z0-9]/giu, "").toLowerCase();
|
||||
if (/^(apikey|accesstoken|refreshtoken|password|authorization|credentialvalue|secretvalue|privatekey|clientsecret|bearertoken)$/u.test(normalized)) return !redactedSecretMarker(entry);
|
||||
if (typeof entry === "string" && /^(?:https?|postgres(?:ql)?):\/\/[^/@\s]+:[^/@\s]+@/iu.test(entry)) return true;
|
||||
return containsSecretLikeKey(entry, depth + 1);
|
||||
});
|
||||
}
|
||||
|
||||
function redactedSecretMarker(value) {
|
||||
if (value === null || value === undefined) return true;
|
||||
const normalized = stringValue(value)?.toLowerCase();
|
||||
return normalized === "redacted" || normalized === "[redacted]" || normalized === "present" || normalized === "missing";
|
||||
}
|
||||
|
||||
function kafkaTransport(kafkaMessage = {}) {
|
||||
const valueText = kafkaMessage?.message?.value ? Buffer.from(kafkaMessage.message.value).toString("utf8") : "";
|
||||
return {
|
||||
sourceTopic: stringValue(kafkaMessage.topic),
|
||||
sourcePartition: integerValue(kafkaMessage.partition),
|
||||
sourceOffset: stringValue(kafkaMessage.message?.offset),
|
||||
sourceKey: kafkaMessage.message?.key ? Buffer.from(kafkaMessage.message.key).toString("utf8") : null,
|
||||
inputSha256: sha256(valueText),
|
||||
valuesPrinted: false
|
||||
};
|
||||
}
|
||||
|
||||
function requireKafkaProjectorStore(runtimeStore, capabilities = {}) {
|
||||
const required = [];
|
||||
if (capabilities.transactionalProjector) required.push("commitAgentRunKafkaProjection", "recordFailedAgentRunKafkaMessage", "recordIgnoredAgentRunKafkaMessage");
|
||||
if (capabilities.projectionOutboxRelay) required.push("claimHwlabKafkaOutbox", "completeHwlabKafkaOutbox", "retryHwlabKafkaOutbox");
|
||||
if (capabilities.projectionRealtime) required.push("subscribeWorkbenchProjectionCommits");
|
||||
if (capabilities.transactionalProjector || capabilities.projectionOutboxRelay) required.push("hwlabKafkaProjectorStatus");
|
||||
if (capabilities.transactionalProjector || capabilities.projectionOutboxRelay || capabilities.projectionRealtime) required.push("assertWorkbenchTransactionalRealtimeReady");
|
||||
const missing = required.filter((name) => typeof runtimeStore?.[name] !== "function");
|
||||
if (missing.length > 0) throw contractError("hwlab_kafka_projector_store_invalid", `Kafka durable capabilities require a runtime store: ${missing.join(", ")}`);
|
||||
}
|
||||
|
||||
export async function queryKafkaEventStream({ env = process.env, stream = "hwlab", topic = null, traceId = null, sessionId = null, runId = null, commandId = null, limit = DEFAULT_QUERY_LIMIT, timeoutMs = DEFAULT_QUERY_TIMEOUT_MS, fromBeginning = true, kafkaFactory = defaultKafkaFactory } = {}) {
|
||||
const brokers = csv(env.HWLAB_KAFKA_BOOTSTRAP_SERVERS);
|
||||
if (brokers.length === 0) throw new Error("HWLAB_KAFKA_BOOTSTRAP_SERVERS is required for Kafka event queries.");
|
||||
@@ -214,35 +801,43 @@ export async function openKafkaEventStream({ env = process.env, stream = "hwlab"
|
||||
const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID) || DEFAULT_CLIENT_ID;
|
||||
const resolvedTopic = stringValue(topic) || kafkaTopicForStream(stream, env);
|
||||
const filters = compactObject({ traceId, sessionId, runId, commandId });
|
||||
const kafka = kafkaFactory({ brokers, clientId: `${clientId}-sse` });
|
||||
const consumer = kafka.consumer({ groupId: `${clientId}-sse-${Date.now()}-${randomUUID().slice(0, 8)}`, allowAutoTopicCreation: false });
|
||||
const kafka = kafkaFactory({ brokers, clientId: `${clientId}-debug-sse` });
|
||||
const groupId = `${clientId}-debug-sse-${Date.now()}-${randomUUID().slice(0, 8)}`;
|
||||
const consumer = kafka.consumer({ groupId, allowAutoTopicCreation: false });
|
||||
let running = false;
|
||||
let stopped = false;
|
||||
await consumer.connect();
|
||||
await consumer.subscribe({ topic: resolvedTopic, fromBeginning: fromBeginning === true });
|
||||
running = true;
|
||||
consumer.run({
|
||||
eachMessage: async ({ topic: messageTopic, partition, message }) => {
|
||||
if (stopped) return;
|
||||
const valueText = message.value ? Buffer.from(message.value).toString("utf8") : "";
|
||||
const value = parseJson(valueText);
|
||||
if (!value || !eventMatchesFilters(value, filters)) return;
|
||||
await onEvent({
|
||||
topic: messageTopic,
|
||||
partition,
|
||||
offset: message.offset,
|
||||
key: message.key ? Buffer.from(message.key).toString("utf8") : null,
|
||||
timestamp: message.timestamp ?? null,
|
||||
value
|
||||
});
|
||||
}
|
||||
}).catch((error) => {
|
||||
if (!stopped && typeof onError === "function") onError(error);
|
||||
});
|
||||
try {
|
||||
await consumer.run({
|
||||
eachMessage: async ({ topic: messageTopic, partition, message }) => {
|
||||
if (stopped) return;
|
||||
const valueText = message.value ? Buffer.from(message.value).toString("utf8") : "";
|
||||
const value = parseJson(valueText);
|
||||
if (!value || !eventMatchesFilters(value, filters)) return;
|
||||
await onEvent({
|
||||
topic: messageTopic,
|
||||
partition,
|
||||
offset: message.offset,
|
||||
key: message.key ? Buffer.from(message.key).toString("utf8") : null,
|
||||
timestamp: message.timestamp ?? null,
|
||||
value
|
||||
});
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
if (typeof onError === "function") onError(error);
|
||||
stopped = true;
|
||||
if (running) await consumer.stop().catch(() => undefined);
|
||||
await boundedDisconnect(consumer);
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
stream,
|
||||
topic: resolvedTopic,
|
||||
groupId,
|
||||
filters,
|
||||
async stop() {
|
||||
stopped = true;
|
||||
@@ -269,8 +864,23 @@ function mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run,
|
||||
};
|
||||
const type = firstText(sourceEvent.type, payload.type, input.eventType) || "event";
|
||||
if (type === "assistant_message") {
|
||||
const terminal = payload.replyAuthority === true || payload.final === true;
|
||||
return { ...base, type: "assistant", eventType: "assistant", status: terminal ? "completed" : "running", label: "agentrun:assistant:message", message: textPayload(payload, "assistant message"), text: textPayload(payload, ""), itemId: firstText(payload.itemId), replyAuthority: payload.replyAuthority === true, final: payload.final === true, terminal };
|
||||
const authoritative = payload.replyAuthority === true || payload.final === true;
|
||||
const assistantText = textPayload(payload, "assistant message");
|
||||
return {
|
||||
...base,
|
||||
type: "assistant",
|
||||
eventType: "assistant",
|
||||
status: "running",
|
||||
label: "agentrun:assistant:message",
|
||||
message: assistantText,
|
||||
text: assistantText,
|
||||
assistantText,
|
||||
finalResponse: authoritative ? { text: assistantText, status: "completed", traceId, source: "agentrun.kafka", valuesPrinted: false } : null,
|
||||
itemId: firstText(payload.itemId),
|
||||
replyAuthority: payload.replyAuthority === true,
|
||||
final: payload.final === true,
|
||||
terminal: false
|
||||
};
|
||||
}
|
||||
if (type === "backend_status") {
|
||||
const phase = firstText(payload.phase) || "status";
|
||||
@@ -281,6 +891,24 @@ function mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run,
|
||||
const normalized = terminalStatus === "cancelled" ? "canceled" : terminalStatus;
|
||||
return { ...base, type: "result", eventType: "terminal", status: normalized, label: `agentrun:terminal:${terminalStatus}`, errorCode: firstText(payload.failureKind, payload.errorCode, sourceEvent.failureKind, sourceEvent.errorCode), failureKind: firstText(payload.failureKind, sourceEvent.failureKind), message: textPayload({ ...sourceEvent, ...payload }, `AgentRun terminal status ${terminalStatus}`), terminal: true };
|
||||
}
|
||||
if (type === "tool_call") {
|
||||
const toolName = firstText(payload.toolName, payload.name, payload.method) || "tool";
|
||||
return {
|
||||
...base,
|
||||
type: "tool",
|
||||
eventType: "tool",
|
||||
status: firstText(payload.status) || "running",
|
||||
label: `agentrun:tool:${toolName}`,
|
||||
message: textPayload(payload, `AgentRun tool ${toolName}`),
|
||||
toolName,
|
||||
method: firstText(payload.method),
|
||||
itemId: firstText(payload.itemId),
|
||||
exitCode: integerValue(payload.exitCode),
|
||||
durationMs: integerValue(payload.durationMs),
|
||||
outputSummary: firstText(payload.outputSummary, payload.summary?.text),
|
||||
terminal: false
|
||||
};
|
||||
}
|
||||
if (type === "command_output") {
|
||||
const stream = payload.stream === "stderr" ? "stderr" : "stdout";
|
||||
return { ...base, type: "output", eventType: "status", status: "running", label: `agentrun:output:${stream}`, stream, message: textPayload(payload, ""), outputTruncated: payload.outputTruncated === true };
|
||||
@@ -349,7 +977,7 @@ async function boundedDisconnect(consumer, timeoutMs = 2000) {
|
||||
}
|
||||
|
||||
function kafkaMessageKey(projected) {
|
||||
return firstText(projected.traceId, projected.sessionId, projected.context?.commandId, projected.context?.runId) || "hwlab-event";
|
||||
return firstText(projected.sessionId, projected.traceId, projected.context?.commandId, projected.context?.runId) || "hwlab-event";
|
||||
}
|
||||
|
||||
function kafkaHeaders(projected) {
|
||||
@@ -408,6 +1036,26 @@ function integerValue(value) {
|
||||
return Number.isFinite(parsed) ? Math.floor(parsed) : null;
|
||||
}
|
||||
|
||||
function nextKafkaOffset(value) {
|
||||
try {
|
||||
return (BigInt(String(value)) + 1n).toString();
|
||||
} catch {
|
||||
throw contractError("workbench_kafka_offset_invalid", "Kafka source offset must be an integer string.");
|
||||
}
|
||||
}
|
||||
|
||||
function strictPositiveInteger(value, name) {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) throw contractError("hwlab_kafka_projector_config_invalid", `${name} must be a positive integer.`);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function contractError(code, message) {
|
||||
const error = new Error(message);
|
||||
error.code = code;
|
||||
return error;
|
||||
}
|
||||
|
||||
function timestampValue(value) {
|
||||
const text = stringValue(value);
|
||||
if (!text) return null;
|
||||
|
||||
@@ -1,185 +0,0 @@
|
||||
// SPEC: PJ2026-010205 HWLAB接入 draft-2026-06-25-p0-session-warm-runner-contract.
|
||||
// Responsibility: best-effort Kafka shadow production for HWLAB -> AgentRun command admission.
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import { Kafka, logLevel } from "kafkajs";
|
||||
|
||||
const TRUE_VALUES = new Set(["1", "true", "yes", "on"]);
|
||||
const warnedCodes = new Set();
|
||||
let producerState = null;
|
||||
|
||||
export function publishHwlabAgentRunCommandShadow({ params = {}, payload = {}, traceId, lifecycle = {}, env = process.env } = {}) {
|
||||
const config = kafkaShadowConfig(env);
|
||||
if (!config) return;
|
||||
const message = buildHwlabCommandShadowMessage({ params, payload, traceId, lifecycle, config });
|
||||
void producerForConfig(config)
|
||||
.then((producer) => producer.send({ topic: config.topic, messages: [message] }))
|
||||
.catch((error) => warnShadowProducer("hwlab-kafka-shadow-produce-failed", {
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
topic: config.topic,
|
||||
clientId: config.clientId
|
||||
}));
|
||||
}
|
||||
|
||||
function kafkaShadowConfig(env) {
|
||||
if (!truthy(env.HWLAB_KAFKA_SHADOW_PRODUCE_ENABLED)) return null;
|
||||
if (truthy(env.HWLAB_KAFKA_SHADOW_CONSUME_ENABLED)) {
|
||||
warnOnce("hwlab-kafka-shadow-consume-ignored", {
|
||||
message: "HWLAB Kafka consumer cutover is disabled in this stage; producer continues in shadow mode.",
|
||||
valuesPrinted: false
|
||||
});
|
||||
}
|
||||
const brokers = csv(env.HWLAB_KAFKA_BOOTSTRAP_SERVERS);
|
||||
const topic = stringValue(env.HWLAB_KAFKA_COMMAND_TOPIC);
|
||||
const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID);
|
||||
const missing = [];
|
||||
if (brokers.length === 0) missing.push("HWLAB_KAFKA_BOOTSTRAP_SERVERS");
|
||||
if (!topic) missing.push("HWLAB_KAFKA_COMMAND_TOPIC");
|
||||
if (!clientId) missing.push("HWLAB_KAFKA_CLIENT_ID");
|
||||
if (missing.length > 0) {
|
||||
warnOnce("hwlab-kafka-shadow-config-missing", { missing, valuesPrinted: false });
|
||||
return null;
|
||||
}
|
||||
return { brokers, topic, clientId, key: `${clientId}|${topic}|${brokers.join(",")}` };
|
||||
}
|
||||
|
||||
function buildHwlabCommandShadowMessage({ params, payload, traceId, lifecycle, config }) {
|
||||
const agentRun = objectValue(payload.agentRun);
|
||||
const prompt = promptEvidence(params);
|
||||
const event = {
|
||||
schema: "hwlab.agentrun.command.shadow.v1",
|
||||
eventType: "hwlab.command.admitted",
|
||||
source: config.clientId,
|
||||
producedAt: new Date().toISOString(),
|
||||
mode: "shadow-produce-only",
|
||||
traceId: safeText(traceId ?? payload.traceId ?? params.traceId),
|
||||
hwlab: {
|
||||
sessionId: safeText(params.sessionId ?? payload.sessionId ?? agentRun.hwlabSessionId),
|
||||
conversationId: safeText(params.conversationId ?? payload.conversationId ?? agentRun.conversationId),
|
||||
turnId: safeText(lifecycle.turnId ?? payload.turnId),
|
||||
userMessageId: safeText(lifecycle.userMessageId ?? payload.userMessageId),
|
||||
assistantMessageId: safeText(lifecycle.assistantMessageId ?? payload.assistantMessageId),
|
||||
projectId: safeText(params.projectId ?? payload.projectId),
|
||||
providerProfile: safeText(params.providerProfile ?? agentRun.backendProfile)
|
||||
},
|
||||
agentRun: {
|
||||
adapter: safeText(agentRun.adapter),
|
||||
runId: safeText(agentRun.runId),
|
||||
commandId: safeText(agentRun.commandId),
|
||||
attemptId: safeText(agentRun.attemptId),
|
||||
runnerId: safeText(agentRun.runnerId),
|
||||
jobName: safeText(agentRun.jobName),
|
||||
namespace: safeText(agentRun.namespace),
|
||||
backendProfile: safeText(agentRun.backendProfile),
|
||||
status: safeText(agentRun.status),
|
||||
commandState: safeText(agentRun.commandState),
|
||||
terminalStatus: safeText(agentRun.terminalStatus)
|
||||
},
|
||||
prompt,
|
||||
diagnostics: {
|
||||
payloadSha256: jsonSha256({ traceId: traceId ?? null, agentRun: eventKeyParts(agentRun), prompt }),
|
||||
payloadBytes: jsonBytes({ traceId: traceId ?? null, agentRun: eventKeyParts(agentRun), prompt }),
|
||||
valuesPrinted: false,
|
||||
shadowConsumeEnabled: false
|
||||
},
|
||||
valuesPrinted: false
|
||||
};
|
||||
return {
|
||||
key: safeText(agentRun.commandId ?? traceId ?? params.sessionId) || config.clientId,
|
||||
value: JSON.stringify(event),
|
||||
headers: {
|
||||
"x-shadow-mode": "produce-only",
|
||||
"x-values-printed": "false",
|
||||
"x-trace-id": event.traceId ?? ""
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function producerForConfig(config) {
|
||||
if (producerState?.key === config.key && producerState.producer) return producerState.producer;
|
||||
if (producerState?.key === config.key && producerState.connecting) return producerState.connecting;
|
||||
const kafka = new Kafka({ clientId: config.clientId, brokers: config.brokers, logLevel: logLevel.NOTHING });
|
||||
const producer = kafka.producer({ allowAutoTopicCreation: false });
|
||||
const state = {
|
||||
key: config.key,
|
||||
producer,
|
||||
connecting: producer.connect().then(() => {
|
||||
state.connecting = null;
|
||||
return producer;
|
||||
}).catch((error) => {
|
||||
if (producerState === state) producerState = null;
|
||||
throw error;
|
||||
})
|
||||
};
|
||||
producerState = state;
|
||||
return state.connecting;
|
||||
}
|
||||
|
||||
function promptEvidence(params) {
|
||||
const text = firstString(params.prompt, params.message, params.input, objectValue(params.userMessage).text, objectValue(params.userMessage).content);
|
||||
if (!text) return { present: false, bytes: 0, sha256: null, valuesPrinted: false };
|
||||
return {
|
||||
present: true,
|
||||
bytes: Buffer.byteLength(text, "utf8"),
|
||||
sha256: createHash("sha256").update(text).digest("hex"),
|
||||
valuesPrinted: false
|
||||
};
|
||||
}
|
||||
|
||||
function eventKeyParts(agentRun) {
|
||||
return {
|
||||
runId: safeText(agentRun.runId),
|
||||
commandId: safeText(agentRun.commandId),
|
||||
attemptId: safeText(agentRun.attemptId),
|
||||
backendProfile: safeText(agentRun.backendProfile)
|
||||
};
|
||||
}
|
||||
|
||||
function truthy(value) {
|
||||
return TRUE_VALUES.has(String(value ?? "").trim().toLowerCase());
|
||||
}
|
||||
|
||||
function csv(value) {
|
||||
return String(value ?? "").split(",").map((entry) => entry.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
function stringValue(value) {
|
||||
const text = String(value ?? "").trim();
|
||||
return text.length > 0 ? text : null;
|
||||
}
|
||||
|
||||
function safeText(value, max = 240) {
|
||||
const text = stringValue(value);
|
||||
if (!text) return null;
|
||||
return text.length > max ? `${text.slice(0, max)}...` : text;
|
||||
}
|
||||
|
||||
function firstString(...values) {
|
||||
for (const value of values) {
|
||||
if (typeof value === "string" && value.trim().length > 0) return value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function objectValue(value) {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
||||
}
|
||||
|
||||
function jsonSha256(value) {
|
||||
return createHash("sha256").update(JSON.stringify(value)).digest("hex");
|
||||
}
|
||||
|
||||
function jsonBytes(value) {
|
||||
return Buffer.byteLength(JSON.stringify(value), "utf8");
|
||||
}
|
||||
|
||||
function warnOnce(code, details) {
|
||||
if (warnedCodes.has(code)) return;
|
||||
warnedCodes.add(code);
|
||||
warnShadowProducer(code, details);
|
||||
}
|
||||
|
||||
function warnShadowProducer(code, details = {}) {
|
||||
console.warn(JSON.stringify({ code, component: "hwlab-kafka-shadow-producer", ...details, valuesPrinted: false }));
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
||||
// Responsibility: Focused AgentRun projection polling/backoff regressions for Cloud API chat result sync.
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer } from "node:http";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts";
|
||||
import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
||||
|
||||
test("AgentRun sync stops upstream polling when terminal command result is already sealed (#1445)", async () => {
|
||||
const traceId = "trc_1445_sealed_terminal";
|
||||
const runId = "run_1445_sealed_terminal";
|
||||
const commandId = "cmd_1445_sealed_terminal";
|
||||
const traceStore = createCodeAgentTraceStore();
|
||||
let fetched = false;
|
||||
const synced = await syncAgentRunChatResult({
|
||||
traceId,
|
||||
currentResult: {
|
||||
ok: true,
|
||||
status: "completed",
|
||||
traceId,
|
||||
finalResponse: { text: "sealed final response", traceId },
|
||||
traceSummary: { source: "agentrun-command-result", traceId, agentRun: { runId, commandId, lastSeq: 17 } },
|
||||
agentRun: {
|
||||
adapter: "agentrun-v01",
|
||||
runId,
|
||||
commandId,
|
||||
traceId,
|
||||
status: "completed",
|
||||
runStatus: "completed",
|
||||
commandState: "completed",
|
||||
terminalStatus: "completed",
|
||||
providerTrace: { traceId, runId, commandId, valuesPrinted: false },
|
||||
lastSeq: 17,
|
||||
managerUrl: "http://127.0.0.1:1",
|
||||
valuesPrinted: false
|
||||
},
|
||||
valuesPrinted: false
|
||||
},
|
||||
options: {
|
||||
fetchImpl: async () => {
|
||||
fetched = true;
|
||||
throw new Error("sealed terminal result should not fetch AgentRun");
|
||||
},
|
||||
env: { HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1" }
|
||||
},
|
||||
traceStore
|
||||
});
|
||||
|
||||
assert.equal(fetched, false);
|
||||
assert.equal(synced.terminalRefreshSkipped, true);
|
||||
assert.equal(synced.resultSynced, false);
|
||||
assert.equal(synced.polling.rootCause, "terminal_result_sealed");
|
||||
assert.equal(synced.polling.backoffMs, 0);
|
||||
assert.equal(synced.polling.terminalConsistencyGap, 0);
|
||||
});
|
||||
|
||||
test("AgentRun events rate limit writes durable projection backoff (#1445)", async () => {
|
||||
const traceId = "trc_1445_events_rate_limit";
|
||||
const runId = "run_1445_events_rate_limit";
|
||||
const commandId = "cmd_1445_events_rate_limit";
|
||||
const calls = [];
|
||||
const writes = [];
|
||||
const runtimeStore = {
|
||||
async getWorkbenchProjectionState() {
|
||||
return { projectionState: { traceId, sourceRunId: runId, sourceCommandId: commandId, lastSourceSeq: 40, pollCount: 2, noProgressPollCount: 1, resultSyncState: "not_started", createdAt: "2026-07-02T00:00:00.000Z" } };
|
||||
},
|
||||
async writeWorkbenchProjectionState(input = {}) {
|
||||
writes.push(input.projectionState);
|
||||
return { written: true, projectionState: input.projectionState };
|
||||
}
|
||||
};
|
||||
const agentRunServer = createHttpServer(async (request, response) => {
|
||||
const url = new URL(request.url || "/", "http://127.0.0.1");
|
||||
calls.push(`${request.method} ${url.pathname}${url.search}`);
|
||||
if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/events`) {
|
||||
response.writeHead(429, { "content-type": "application/json" });
|
||||
response.end(`${JSON.stringify({ ok: false, failureKind: "provider_rate_limited", message: "provider rate limited; retry later" })}\n`);
|
||||
return;
|
||||
}
|
||||
response.writeHead(404, { "content-type": "application/json" });
|
||||
response.end(`${JSON.stringify({ ok: false, message: `unexpected ${request.method} ${url.pathname}` })}\n`);
|
||||
});
|
||||
await new Promise((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve));
|
||||
const agentRunPort = agentRunServer.address().port;
|
||||
const traceStore = createCodeAgentTraceStore();
|
||||
try {
|
||||
await assert.rejects(
|
||||
syncAgentRunChatResult({
|
||||
traceId,
|
||||
currentResult: {
|
||||
status: "running",
|
||||
traceId,
|
||||
agentRun: { adapter: "agentrun-v01", runId, commandId, traceId, providerTrace: { traceId }, lastSeq: 40, managerUrl: `http://127.0.0.1:${agentRunPort}`, valuesPrinted: false },
|
||||
valuesPrinted: false
|
||||
},
|
||||
options: {
|
||||
runtimeStore,
|
||||
env: {
|
||||
HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_BACKOFF_INITIAL_MS: "1000",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS: "2500"
|
||||
}
|
||||
},
|
||||
traceStore
|
||||
}),
|
||||
(error) => {
|
||||
assert.equal(error.statusCode, 429);
|
||||
assert.equal(error.polling.rootCause, "provider_rate_limited");
|
||||
assert.equal(error.polling.backoffReason, "provider_retry_or_rate_limit");
|
||||
assert.equal(error.polling.backoffMs, 2000);
|
||||
assert.ok(error.polling.nextRetryAt);
|
||||
return true;
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(calls.length, 1);
|
||||
assert.match(calls[0], /afterSeq=40/u);
|
||||
assert.equal(writes.length, 1);
|
||||
assert.equal(writes[0].projectionStatus, "degraded");
|
||||
assert.equal(writes[0].projectionHealth, "degraded");
|
||||
assert.equal(writes[0].backoffMs, 2000);
|
||||
assert.equal(writes[0].backoffReason, "provider_retry_or_rate_limit");
|
||||
assert.equal(writes[0].rootCause, "provider_rate_limited");
|
||||
assert.ok(writes[0].nextRetryAt);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => agentRunServer.close((error) => (error ? reject(error) : resolve())));
|
||||
}
|
||||
});
|
||||
@@ -13,7 +13,7 @@ import { createCloudRuntimeStore } from "../db/runtime-store.ts";
|
||||
import { validateCodeAgentChatSchema } from "./code-agent-chat.ts";
|
||||
import { createCodexStdioSessionManager } from "./codex-stdio-session.ts";
|
||||
import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
||||
import { submitAgentRunChatTurn, syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts";
|
||||
import { submitAgentRunChatTurn } from "./code-agent-agentrun-adapter.ts";
|
||||
import {
|
||||
codexStdioChatFixture,
|
||||
codexStdioReadyFixture,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
||||
// Responsibility: Cloud API AgentRun session/profile continuity and durable projection resume regression tests.
|
||||
// Responsibility: Cloud API AgentRun session/profile continuity regression tests.
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer } from "node:http";
|
||||
@@ -13,7 +13,7 @@ import { createCloudRuntimeStore } from "../db/runtime-store.ts";
|
||||
import { validateCodeAgentChatSchema } from "./code-agent-chat.ts";
|
||||
import { createCodexStdioSessionManager } from "./codex-stdio-session.ts";
|
||||
import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
||||
import { submitAgentRunChatTurn, syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts";
|
||||
import { submitAgentRunChatTurn } from "./code-agent-agentrun-adapter.ts";
|
||||
import {
|
||||
codexStdioChatFixture,
|
||||
codexStdioReadyFixture,
|
||||
@@ -1356,391 +1356,3 @@ test("cloud api turn status reuses request session lookup without loading persis
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test("cloud api resumes AgentRun projection from durable state after restart", async () => {
|
||||
const calls = [];
|
||||
const durableEvents = [];
|
||||
const projectionStates = new Map();
|
||||
const ownerWrites = [];
|
||||
const traceId = "trc_projection_resume_restart";
|
||||
const runId = "run_projection_resume_restart";
|
||||
const commandId = "cmd_projection_resume_restart";
|
||||
const sessionId = "ses_projection_resume_restart";
|
||||
const conversationId = "cnv_projection_resume_restart";
|
||||
const finalText = "OK from resumed AgentRun projection.";
|
||||
|
||||
const agentRunServer = createHttpServer(async (request, response) => {
|
||||
const url = new URL(request.url || "/", "http://127.0.0.1");
|
||||
calls.push({ method: request.method, path: url.pathname, search: url.search, afterSeq: url.searchParams.get("afterSeq") });
|
||||
const send = (body, status = 200) => {
|
||||
response.writeHead(status, { "content-type": "application/json" });
|
||||
response.end(`${JSON.stringify(body)}\n`);
|
||||
};
|
||||
if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/events`) {
|
||||
assert.equal(url.searchParams.get("afterSeq"), "21");
|
||||
return send({ items: [
|
||||
{ id: "evt_projection_resume_22", runId, seq: 22, type: "assistant_message", payload: { commandId, text: finalText }, createdAt: "2026-06-18T16:31:11.000Z" },
|
||||
{ id: "evt_projection_resume_23", runId, seq: 23, type: "terminal_status", payload: { commandId, terminalStatus: "completed" }, createdAt: "2026-06-18T16:31:12.000Z" }
|
||||
] });
|
||||
}
|
||||
if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/commands/${commandId}/result`) {
|
||||
return send({
|
||||
runId,
|
||||
commandId,
|
||||
status: "completed",
|
||||
runStatus: "completed",
|
||||
commandState: "completed",
|
||||
terminalStatus: "completed",
|
||||
completed: true,
|
||||
reply: finalText,
|
||||
lastSeq: 23,
|
||||
eventCount: 23,
|
||||
sessionRef: { sessionId, conversationId, threadId: "thread-projection-resume-restart" }
|
||||
});
|
||||
}
|
||||
return send({ ok: false, message: `unexpected ${request.method} ${url.pathname}` }, 404);
|
||||
});
|
||||
await new Promise((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve));
|
||||
const agentRunPort = agentRunServer.address().port;
|
||||
|
||||
const session = testAgentSessionRecord({
|
||||
sessionId,
|
||||
conversationId,
|
||||
projectId: "prj_hwpod_workbench",
|
||||
status: "running",
|
||||
traceId,
|
||||
updatedAt: "2026-06-18T16:31:04.000Z",
|
||||
session: {
|
||||
traceResults: {
|
||||
[traceId]: {
|
||||
traceId,
|
||||
status: "running",
|
||||
sessionId,
|
||||
conversationId,
|
||||
updatedAt: "2026-06-18T16:31:04.000Z",
|
||||
agentRun: {
|
||||
adapter: "agentrun-v01",
|
||||
managerUrl: `http://127.0.0.1:${agentRunPort}`,
|
||||
runId,
|
||||
commandId,
|
||||
traceId,
|
||||
lastSeq: 0,
|
||||
status: "running",
|
||||
commandState: "running",
|
||||
backendProfile: "codex",
|
||||
valuesPrinted: false
|
||||
},
|
||||
valuesRedacted: true
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
durableEvents.push({
|
||||
traceId,
|
||||
seq: 27,
|
||||
source: "agentrun",
|
||||
sourceSeq: 21,
|
||||
type: "backend",
|
||||
status: "running",
|
||||
label: "agentrun:backend:mcpServer/startupStatus/updated",
|
||||
commandId,
|
||||
runId,
|
||||
createdAt: "2026-06-18T16:31:04.200Z",
|
||||
valuesPrinted: false
|
||||
});
|
||||
projectionStates.set(traceId, {
|
||||
traceId,
|
||||
sessionId,
|
||||
conversationId,
|
||||
threadId: "thread-projection-resume-restart",
|
||||
ownerUserId: TEST_AGENT_ACTOR.id,
|
||||
ownerRole: TEST_AGENT_ACTOR.role,
|
||||
runId,
|
||||
commandId,
|
||||
sourceRunId: runId,
|
||||
sourceCommandId: commandId,
|
||||
managerUrl: `http://127.0.0.1:${agentRunPort}`,
|
||||
backendProfile: "codex",
|
||||
lastSourceSeq: 21,
|
||||
lastAgentRunSeq: 21,
|
||||
lastProjectedSeq: 27,
|
||||
sourceLatestSeq: 21,
|
||||
upstreamLatestSeq: 21,
|
||||
projectionStatus: "projecting",
|
||||
projectionHealth: "healthy",
|
||||
resultSyncState: "not_started",
|
||||
createdAt: "2026-06-18T16:31:04.000Z",
|
||||
updatedAt: "2026-06-18T16:31:04.000Z",
|
||||
valuesPrinted: false
|
||||
});
|
||||
|
||||
const runtimeStore = {
|
||||
async queryAgentTraceEvents(params = {}) {
|
||||
return { events: durableEvents.filter((event) => event.traceId === params.traceId), count: durableEvents.length };
|
||||
},
|
||||
async writeAgentTraceEvent(params = {}) {
|
||||
const event = params.event ?? params.traceEvent ?? params;
|
||||
durableEvents.push(event);
|
||||
return { written: true, traceEvent: event };
|
||||
},
|
||||
async getWorkbenchProjectionState(params = {}) {
|
||||
const projectionState = projectionStates.get(params.traceId) ?? null;
|
||||
return { projectionState, found: Boolean(projectionState) };
|
||||
},
|
||||
async queryWorkbenchProjectionStates() {
|
||||
return { states: [...projectionStates.values()], count: projectionStates.size };
|
||||
},
|
||||
async writeWorkbenchProjectionState(params = {}) {
|
||||
const projectionState = params.projectionState ?? params.state ?? params;
|
||||
projectionStates.set(projectionState.traceId, projectionState);
|
||||
return { written: true, projectionState };
|
||||
}
|
||||
};
|
||||
const accessController = {
|
||||
required: false,
|
||||
store: {
|
||||
async listAgentSessionsForUser() { return [session]; },
|
||||
async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; }
|
||||
},
|
||||
async authenticate() { return { ok: true, actor: TEST_AGENT_ACTOR, session: TEST_AUTH_SESSION }; },
|
||||
async recordAgentSessionOwner(input = {}) {
|
||||
ownerWrites.push(input);
|
||||
return { ...session, status: input.status ?? session.status, session: input.session ?? session.session };
|
||||
}
|
||||
};
|
||||
|
||||
const server = createCloudApiServer({
|
||||
accessController,
|
||||
runtimeStore,
|
||||
codeAgentChatResults: new Map(),
|
||||
env: {
|
||||
HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_ENABLED: "1",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INITIAL_DELAY_MS: "1",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INTERVAL_MS: "0",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS: "0",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE: "10"
|
||||
}
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
for (let index = 0; index < 80 && projectionStates.get(traceId)?.resultSyncState !== "synced"; index += 1) await delay(10);
|
||||
assert.ok(calls.some((call) => call.path === `/api/v1/runs/${runId}/events` && call.afterSeq === "21"));
|
||||
assert.equal(calls.some((call) => call.path === `/api/v1/runs/${runId}/commands/${commandId}/result`), true);
|
||||
assert.equal(projectionStates.get(traceId)?.lastSourceSeq, 23);
|
||||
assert.equal(projectionStates.get(traceId)?.resultSyncState, "synced");
|
||||
assert.equal(ownerWrites.some((write) => write.status === "completed"), true);
|
||||
assert.ok(durableEvents.some((event) => event.sourceSeq === 22));
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
});
|
||||
await new Promise((resolve, reject) => {
|
||||
agentRunServer.close((error) => (error ? reject(error) : resolve()));
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test("cloud api projection resume retries evicted AgentRun session with stored prompt", async () => {
|
||||
const calls = [];
|
||||
const durableEvents = [];
|
||||
const projectionStates = new Map();
|
||||
const traceId = "trc_projection_resume_evicted_retry";
|
||||
const runId = "run_projection_resume_evicted";
|
||||
const commandId = "cmd_projection_resume_evicted";
|
||||
const retryRunId = "run_projection_resume_evicted_retry";
|
||||
const retryCommandId = "cmd_projection_resume_evicted_retry";
|
||||
const sessionId = "ses_projection_resume_evicted";
|
||||
const conversationId = "cnv_projection_resume_evicted";
|
||||
const promptText = "resume projection evicted session with original prompt";
|
||||
|
||||
const agentRunServer = createHttpServer(async (request, response) => {
|
||||
const url = new URL(request.url || "/", "http://127.0.0.1");
|
||||
const chunks = [];
|
||||
for await (const chunk of request) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
const body = chunks.length ? JSON.parse(Buffer.concat(chunks).toString("utf8")) : null;
|
||||
calls.push({ method: request.method, path: url.pathname, search: url.search, afterSeq: url.searchParams.get("afterSeq"), body });
|
||||
const send = (data, status = 200) => {
|
||||
response.writeHead(status, { "content-type": "application/json" });
|
||||
response.end(`${JSON.stringify({ ok: status < 400, data })}\n`);
|
||||
};
|
||||
if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/events`) {
|
||||
return send({ items: [
|
||||
{ id: "evt_projection_evicted_1", runId, seq: 1, type: "error", payload: { commandId, failureKind: "session-store-evicted", message: "codex app-server thread/resume reported no rollout found for PVC-backed session; session storage was likely evicted" }, createdAt: "2026-06-19T12:01:00.000Z" },
|
||||
{ id: "evt_projection_evicted_2", runId, seq: 2, type: "terminal_status", payload: { commandId, terminalStatus: "failed", failureKind: "session-store-evicted" }, createdAt: "2026-06-19T12:01:01.000Z" }
|
||||
] });
|
||||
}
|
||||
if (request.method === "GET" && url.pathname === `/api/v1/runs/${runId}/commands/${commandId}/result`) {
|
||||
return send({
|
||||
runId,
|
||||
commandId,
|
||||
status: "failed",
|
||||
runStatus: "failed",
|
||||
commandState: "failed",
|
||||
terminalStatus: "failed",
|
||||
failureKind: "session-store-evicted",
|
||||
failureMessage: "codex app-server thread/resume reported no rollout found for PVC-backed session; session storage was likely evicted",
|
||||
completed: false,
|
||||
lastSeq: 2,
|
||||
eventCount: 2,
|
||||
sessionRef: { sessionId: "ses_agentrun_projection_resume_evicted", conversationId, threadId: "thread_projection_evicted" }
|
||||
});
|
||||
}
|
||||
if (request.method === "POST" && url.pathname === "/api/v1/sessions") {
|
||||
assert.match(body.sessionId, /-reset-/u);
|
||||
return send({ sessionId: body.sessionId, backendProfile: body.backendProfile });
|
||||
}
|
||||
if (request.method === "POST" && url.pathname === "/api/v1/runs") {
|
||||
assert.match(body.sessionRef?.sessionId, /-reset-/u);
|
||||
assert.equal(body.sessionRef?.threadId, undefined);
|
||||
return send({ id: retryRunId, status: "pending", backendProfile: "dsflash-go", sessionRef: body.sessionRef, resourceBundleRef: body.resourceBundleRef });
|
||||
}
|
||||
if (request.method === "POST" && url.pathname === `/api/v1/runs/${retryRunId}/commands`) {
|
||||
assert.equal(body.type, "turn");
|
||||
assert.equal(body.payload.prompt, promptText);
|
||||
assert.equal(body.payload.message, promptText);
|
||||
assert.equal(body.payload.threadId, null);
|
||||
assert.match(body.payload.sessionId, /-reset-/u);
|
||||
assert.equal(body.payload.providerProfile, "dsflash-go");
|
||||
assert.equal(body.dispatch?.kind, "kubernetes-job");
|
||||
assert.equal(body.dispatch?.input?.commandId, undefined);
|
||||
return send({
|
||||
id: retryCommandId,
|
||||
runId: retryRunId,
|
||||
state: "queued",
|
||||
dispatchIntent: {
|
||||
id: "dispatch_projection_resume_evicted_retry",
|
||||
state: "pending",
|
||||
runnerJobId: "rjob_projection_resume_evicted_retry",
|
||||
attemptCount: 0,
|
||||
durable: true
|
||||
}
|
||||
});
|
||||
}
|
||||
return send({ message: `unexpected ${request.method} ${url.pathname}` }, 500);
|
||||
});
|
||||
await new Promise((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve));
|
||||
const agentRunPort = agentRunServer.address().port;
|
||||
|
||||
projectionStates.set(traceId, {
|
||||
traceId,
|
||||
sessionId,
|
||||
conversationId,
|
||||
threadId: "thread_projection_evicted",
|
||||
ownerUserId: TEST_AGENT_ACTOR.id,
|
||||
ownerRole: TEST_AGENT_ACTOR.role,
|
||||
runId,
|
||||
commandId,
|
||||
sourceRunId: runId,
|
||||
sourceCommandId: commandId,
|
||||
managerUrl: `http://127.0.0.1:${agentRunPort}`,
|
||||
backendProfile: "dsflash-go",
|
||||
providerId: "JD01",
|
||||
lastSourceSeq: 0,
|
||||
lastAgentRunSeq: 0,
|
||||
lastProjectedSeq: 0,
|
||||
sourceLatestSeq: 0,
|
||||
upstreamLatestSeq: 0,
|
||||
projectionStatus: "projecting",
|
||||
projectionHealth: "healthy",
|
||||
resultSyncState: "not_started",
|
||||
retryParams: {
|
||||
message: promptText,
|
||||
prompt: promptText,
|
||||
text: promptText,
|
||||
sessionId,
|
||||
conversationId,
|
||||
threadId: "thread_projection_evicted",
|
||||
ownerUserId: TEST_AGENT_ACTOR.id,
|
||||
ownerRole: TEST_AGENT_ACTOR.role,
|
||||
providerProfile: "dsflash-go",
|
||||
codeAgentProviderProfile: "dsflash-go",
|
||||
backendProfile: "dsflash-go",
|
||||
valuesPrinted: false
|
||||
},
|
||||
createdAt: "2026-06-19T12:00:00.000Z",
|
||||
updatedAt: "2026-06-19T12:00:00.000Z",
|
||||
valuesPrinted: false
|
||||
});
|
||||
|
||||
const runtimeStore = {
|
||||
async queryAgentTraceEvents(params = {}) {
|
||||
return { events: durableEvents.filter((event) => event.traceId === params.traceId), count: durableEvents.length };
|
||||
},
|
||||
async writeAgentTraceEvent(params = {}) {
|
||||
const event = params.event ?? params.traceEvent ?? params;
|
||||
durableEvents.push(event);
|
||||
return { written: true, traceEvent: event };
|
||||
},
|
||||
async getWorkbenchProjectionState(params = {}) {
|
||||
const projectionState = projectionStates.get(params.traceId) ?? null;
|
||||
return { projectionState, found: Boolean(projectionState) };
|
||||
},
|
||||
async queryWorkbenchProjectionStates() {
|
||||
return { states: [...projectionStates.values()], count: projectionStates.size };
|
||||
},
|
||||
async writeWorkbenchProjectionState(params = {}) {
|
||||
const projectionState = params.projectionState ?? params.state ?? params;
|
||||
projectionStates.set(projectionState.traceId, projectionState);
|
||||
return { written: true, projectionState };
|
||||
}
|
||||
};
|
||||
const accessController = {
|
||||
required: false,
|
||||
store: {
|
||||
async listAgentSessionsForUser() { return []; },
|
||||
async getAgentSessionByTraceId() { return null; }
|
||||
},
|
||||
async authenticate() { return { ok: true, actor: TEST_AGENT_ACTOR, session: TEST_AUTH_SESSION }; }
|
||||
};
|
||||
|
||||
const server = createCloudApiServer({
|
||||
accessController,
|
||||
runtimeStore,
|
||||
codeAgentChatResults: new Map(),
|
||||
env: {
|
||||
HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_SOURCE_COMMIT: "a".repeat(40),
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "JD01",
|
||||
HWLAB_RUNTIME_NAMESPACE: "hwlab-v03",
|
||||
HWLAB_RUNTIME_LANE: "v03",
|
||||
HWLAB_RUNTIME_API_URL: "http://hwlab-cloud-api.hwlab-v03.svc.cluster.local:6667",
|
||||
HWLAB_RUNTIME_WEB_URL: "http://hwlab-cloud-web.hwlab-v03.svc.cluster.local:8080",
|
||||
HWLAB_RUNTIME_ENDPOINT_LOCKED: "1",
|
||||
HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME: "1",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_BACKEND_PROFILE: "dsflash-go",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_ENABLED: "1",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INITIAL_DELAY_MS: "1",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_INTERVAL_MS: "0",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS: "0",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE: "10"
|
||||
}
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
for (let index = 0; index < 100 && projectionStates.get(traceId)?.runId !== retryRunId; index += 1) await delay(10);
|
||||
assert.equal(calls.some((call) => call.path === `/api/v1/runs/${runId}/commands/${commandId}/result`), true);
|
||||
assert.equal(calls.some((call) => call.path === `/api/v1/runs/${retryRunId}/commands`), true);
|
||||
const state = projectionStates.get(traceId);
|
||||
assert.equal(state.runId, retryRunId);
|
||||
assert.equal(state.commandId, retryCommandId);
|
||||
assert.equal(state.sourceRunId, retryRunId);
|
||||
assert.equal(state.sourceCommandId, retryCommandId);
|
||||
assert.equal(state.projectionStatus, "projecting");
|
||||
assert.equal(state.retryParams.message, promptText);
|
||||
assert.equal(state.retryParams.providerProfile, "dsflash-go");
|
||||
assert.equal(calls.some((call) => call.path.endsWith("/runner-jobs")), false);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
});
|
||||
await new Promise((resolve, reject) => {
|
||||
agentRunServer.close((error) => (error ? reject(error) : resolve()));
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,520 @@
|
||||
// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-0102 Agent编排 draft-2026-06-17-r0; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
||||
// Responsibility: Code Agent HTTP steer and trace resources.
|
||||
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
|
||||
import { M3_IO_CONTROL_ROUTE, M3_STATUS_ROUTE, describeM3StatusLive, handleM3IoControl } from "./m3-io-control.ts";
|
||||
import {
|
||||
agentRunSessionEvidence,
|
||||
cancelAgentRunChatTurn,
|
||||
codeAgentAgentRunAdapterEnabled,
|
||||
initialAgentRunChatResult,
|
||||
loadPersistedAgentRunResult,
|
||||
steerAgentRunChatTurn,
|
||||
submitAgentRunChatTurn
|
||||
} from "./code-agent-agentrun-adapter.ts";
|
||||
import { createCodeAgentErrorPayload, handleCodeAgentChat } from "./code-agent-chat.ts";
|
||||
import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
||||
import { codeAgentSessionLifecycleSummary } from "./code-agent-session-lifecycle.ts";
|
||||
import { messageAuthorityTextValue } from "./code-agent-agentrun-prompt.ts";
|
||||
import { createWorkbenchReadModel } from "./workbench-read-model.ts";
|
||||
import { codeAgentOtelTraceFields, emitCodeAgentOtelSpan } from "./otel-trace.ts";
|
||||
import {
|
||||
firstHeaderValue,
|
||||
getHeader,
|
||||
parsePositiveInteger,
|
||||
positiveInteger,
|
||||
readBody,
|
||||
safeConversationId,
|
||||
safeOpaqueId,
|
||||
safeSessionId,
|
||||
safeTraceId,
|
||||
sendJson,
|
||||
truthyFlag
|
||||
} from "./server-http-utils.ts";
|
||||
|
||||
import {
|
||||
normalizeCodeAgentProviderProfile,
|
||||
firstNonEmptyValue,
|
||||
recordCodeAgentSessionInputFact,
|
||||
stableCodeAgentInputId,
|
||||
codeAgentInputDelivery,
|
||||
codeAgentInputStatus,
|
||||
codeAgentResultTimingFields,
|
||||
firstTimestampIso,
|
||||
firstLatestTimestampIso,
|
||||
codeAgentChatShortConnectionRequested,
|
||||
preflightCodeAgentBilling,
|
||||
recordCodeAgentBillingUsage,
|
||||
finalizeCodeAgentBillingUsage,
|
||||
releaseCodeAgentBillingReservation,
|
||||
withCodeAgentBillingReservation,
|
||||
codeAgentBillingEnabled,
|
||||
codeAgentBillingMetadata,
|
||||
sanitizeBillingReservation,
|
||||
sanitizeBillingRecord,
|
||||
sanitizeBillingRelease,
|
||||
codeAgentUsedTokens,
|
||||
normalizeTurnStatus,
|
||||
recordCodeAgentConversationFact,
|
||||
recordCodeAgentSessionOwner,
|
||||
codeAgentOwnerStatusForResult,
|
||||
textValue,
|
||||
timestampIsoOrNow,
|
||||
timestampIsoOrNull,
|
||||
codeAgentSessionOwnerEvidence,
|
||||
codeAgentContinuationEvidence,
|
||||
codeAgentFreshContinuationFailure,
|
||||
normalizeCodeAgentFailureKind,
|
||||
codeAgentFailureRequiresFreshContinuation,
|
||||
codeAgentProviderProfileEvidence,
|
||||
codeAgentTraceResultEvidence,
|
||||
codeAgentPromptMetadata,
|
||||
codeAgentPromptMetadataFromText,
|
||||
codeAgentPromptMetadataFromParts,
|
||||
codeAgentPromptFields,
|
||||
codeAgentPromptId,
|
||||
clippedPromptText,
|
||||
sha256Text,
|
||||
compactCodeAgentObject,
|
||||
codeAgentBillingReservationEvidence,
|
||||
codeAgentConversationMessagesEvidence,
|
||||
boundedConversationMessageText,
|
||||
conversationText,
|
||||
conversationTextRaw,
|
||||
codeAgentMessageTraceSuffix,
|
||||
codeAgentTurnLifecycleFields,
|
||||
codeAgentLifecycleMessageId,
|
||||
safeTurnId,
|
||||
safeMessageId,
|
||||
codeAgentConversationAgentMessageStatus,
|
||||
codeAgentPayloadHasSealedFinalResponse,
|
||||
codeAgentConversationRunnerTraceEvidence,
|
||||
codeAgentFinalResponseEvidence,
|
||||
codeAgentFinalResponseText,
|
||||
codeAgentTraceSummaryEvidence,
|
||||
annotateOwner,
|
||||
canAccessOwnedResult,
|
||||
isCodeAgentResultCanceled,
|
||||
codeAgentCancelScopeMismatch,
|
||||
cancelExpectedValues,
|
||||
cancelScopeFieldMismatch,
|
||||
cancelBlockedPayload,
|
||||
isTraceCommandTerminalStatus,
|
||||
traceSnapshotWithTerminalEvidence,
|
||||
traceSnapshotWithAttachedTerminalEvidence,
|
||||
agentRunTerminalTraceEvidence,
|
||||
agentRunTerminalFinalResponse,
|
||||
traceRetentionSummary,
|
||||
numberOrNull,
|
||||
createCodeAgentChatResultStore,
|
||||
compactCodeAgentChatResultPayload,
|
||||
terminalEvidencePayload,
|
||||
compactRunnerTraceForResult,
|
||||
traceEventLabel,
|
||||
resultTraceEventLimit,
|
||||
createCodeAgentM3HwlabApiRequestJson,
|
||||
recordCodeAgentProjectionMetrics,
|
||||
recordCodeAgentTurnReadMetric,
|
||||
turnReadDegradedReason,
|
||||
responseBodyBytes,
|
||||
statusClassLabel,
|
||||
sourceLatestSeqFromResult,
|
||||
sourceLatestAtFromResult,
|
||||
terminalSourceAtFromResult,
|
||||
projectionReason,
|
||||
nowMs,
|
||||
uniqueStrings
|
||||
} from "./server-code-agent-http-support.ts";
|
||||
|
||||
import {
|
||||
codeAgentCompatProjectionPayload,
|
||||
codeAgentRequestScopedOptions,
|
||||
measureCodeAgentHttpPhase,
|
||||
readCodeAgentCompatProjection,
|
||||
traceProjectMismatchSnapshot
|
||||
} from "./server-code-agent-turn-http.ts";
|
||||
|
||||
const DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT = 120;
|
||||
const DEFAULT_CODE_AGENT_TRACE_PAGE_LIMIT = 100;
|
||||
const MAX_CODE_AGENT_TRACE_PAGE_LIMIT = 100;
|
||||
const DEFAULT_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS = 2500;
|
||||
const DEFAULT_CODE_AGENT_PROJECT_ID = "prj_hwpod_workbench";
|
||||
const DEFAULT_CODE_AGENT_PROVIDER_PROFILE = "deepseek";
|
||||
const CODE_AGENT_TERMINAL_STATUSES = new Set(["completed", "failed", "blocked", "timeout", "cancelled", "canceled"]);
|
||||
const CODE_AGENT_PROVIDER_PROFILE_ALIASES = Object.freeze({ codex: "codex-api" });
|
||||
const CODE_AGENT_PROVIDER_PROFILE_ID_PATTERN = /^[a-z0-9][a-z0-9.-]{0,63}$/u;
|
||||
const CODE_AGENT_PROVIDER_PROFILE_LABELS = Object.freeze({
|
||||
"deepseek": "DeepSeek",
|
||||
"codex-api": "Codex API",
|
||||
"gpt.pika": "gpt.pika",
|
||||
"minimax-m3": "MiniMax-M3 via AgentRun",
|
||||
"runtime-default": "运行默认"
|
||||
});
|
||||
const DEFAULT_CODE_AGENT_CODEX_API_MODEL = "gpt-5.5";
|
||||
const DEFAULT_CODE_AGENT_CODEX_API_BASE_URL = "http://127.0.0.1:49280/responses";
|
||||
const DEFAULT_CODE_AGENT_DEEPSEEK_MODEL = "deepseek-chat";
|
||||
const DEFAULT_CODE_AGENT_MINIMAX_M3_MODEL = "MiniMax-M3";
|
||||
|
||||
export async function handleCodeAgentSteerHttp(request, response, options) {
|
||||
const body = await readBody(request, options.bodyLimitBytes);
|
||||
let params = {};
|
||||
|
||||
try {
|
||||
params = body ? JSON.parse(body) : {};
|
||||
} catch (error) {
|
||||
sendJson(response, 400, {
|
||||
...createCodeAgentErrorPayload({
|
||||
code: "parse_error",
|
||||
message: "Invalid JSON body",
|
||||
reason: error.message,
|
||||
traceId: getHeader(request, "x-trace-id") || "trc_unassigned",
|
||||
layer: "api",
|
||||
retryable: true,
|
||||
route: "/v1/agent/chat/steer"
|
||||
})
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (!params || typeof params !== "object" || Array.isArray(params)) {
|
||||
sendJson(response, 400, {
|
||||
...createCodeAgentErrorPayload({
|
||||
code: "invalid_params",
|
||||
message: "Code Agent steer body must be a JSON object",
|
||||
traceId: getHeader(request, "x-trace-id") || "trc_unassigned",
|
||||
layer: "api",
|
||||
retryable: true,
|
||||
route: "/v1/agent/chat/steer"
|
||||
})
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const traceId = safeTraceId(getHeader(request, "x-trace-id") || params.traceId || params.targetTraceId);
|
||||
const message = firstNonEmptyValue(params.message, params.prompt, params.text);
|
||||
if (!traceId || !message) {
|
||||
sendJson(response, 400, {
|
||||
...createCodeAgentErrorPayload({
|
||||
code: !traceId ? "steer_trace_missing" : "steer_message_missing",
|
||||
message: !traceId ? "traceId is required to steer the current Code Agent turn." : "message is required to steer the current Code Agent turn.",
|
||||
traceId: traceId || "trc_unassigned",
|
||||
layer: "api",
|
||||
retryable: true,
|
||||
route: "/v1/agent/chat/steer"
|
||||
})
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const traceStore = options.traceStore ?? defaultCodeAgentTraceStore;
|
||||
const durableResult = await loadPersistedAgentRunResult(traceId, options);
|
||||
const currentResult = durableResult ?? options.codeAgentChatResults?.get?.(traceId);
|
||||
const projectionContext = await readCodeAgentCompatProjection(traceId, options);
|
||||
if (!canAccessOwnedResult(currentResult, options.actor)) {
|
||||
sendJson(response, 403, {
|
||||
ok: false,
|
||||
status: "forbidden",
|
||||
traceId,
|
||||
error: { code: "code_agent_trace_forbidden", message: "Code Agent trace is not visible to the current actor" }
|
||||
});
|
||||
return;
|
||||
}
|
||||
const projectionStoreBlocker = codeAgentProjectionStoreBlocker(projectionContext);
|
||||
if (projectionStoreBlocker) {
|
||||
sendJson(response, 503, {
|
||||
ok: false,
|
||||
accepted: false,
|
||||
status: "blocked",
|
||||
traceId,
|
||||
route: "/v1/agent/chat/steer",
|
||||
error: {
|
||||
...projectionStoreBlocker,
|
||||
code: "projection_store_unavailable",
|
||||
layer: projectionStoreBlocker.layer ?? "runtime-store",
|
||||
category: projectionStoreBlocker.category ?? "projection-store-unavailable",
|
||||
retryable: true,
|
||||
message: projectionStoreBlocker.message ?? "Workbench projection store is unavailable; steer is blocked until terminal authority can be checked.",
|
||||
traceId,
|
||||
route: "/v1/agent/chat/steer"
|
||||
},
|
||||
valuesPrinted: false
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (!currentResult?.agentRun?.runId || !currentResult?.agentRun?.commandId) {
|
||||
sendJson(response, 404, {
|
||||
ok: false,
|
||||
status: "not_found",
|
||||
traceId,
|
||||
error: { code: "steer_trace_not_found", message: "No AgentRun-backed Code Agent turn was found for the requested traceId." },
|
||||
route: "/v1/agent/chat/steer",
|
||||
runnerTrace: traceStore.snapshot(traceId)
|
||||
});
|
||||
return;
|
||||
}
|
||||
const projectedTerminalStatus = normalizeTurnStatus(
|
||||
projectionContext.trace?.terminalEvidence?.status,
|
||||
projectionContext.trace?.status
|
||||
);
|
||||
if (isTraceCommandTerminalStatus(projectedTerminalStatus)) {
|
||||
sendJson(response, 409, {
|
||||
ok: false,
|
||||
accepted: false,
|
||||
status: "blocked",
|
||||
traceId,
|
||||
route: "/v1/agent/chat/steer",
|
||||
sessionId: safeSessionId(currentResult.sessionId ?? params.sessionId) || null,
|
||||
threadId: safeOpaqueId(currentResult.threadId ?? params.threadId) || null,
|
||||
agentRun: currentResult.agentRun,
|
||||
runnerTrace: traceStore.snapshot(traceId),
|
||||
error: {
|
||||
code: "steer_trace_terminal",
|
||||
layer: "api",
|
||||
category: "not_in_flight",
|
||||
retryable: false,
|
||||
message: `Trace ${traceId} is already terminal and cannot accept a steer command.`,
|
||||
traceId,
|
||||
route: "/v1/agent/chat/steer",
|
||||
toolName: "agentrun.command.steer"
|
||||
},
|
||||
valuesPrinted: false
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await recordCodeAgentSessionInputFact({
|
||||
params: {
|
||||
...params,
|
||||
sessionId: currentResult.sessionId ?? currentResult.session?.sessionId ?? params.sessionId,
|
||||
ownerUserId: options.actor?.id,
|
||||
ownerRole: options.actor?.role
|
||||
},
|
||||
options,
|
||||
traceId,
|
||||
delivery: "steer",
|
||||
status: "promoted",
|
||||
commandId: currentResult.agentRun.commandId
|
||||
});
|
||||
const payload = await steerAgentRunChatTurn({
|
||||
traceId,
|
||||
currentResult,
|
||||
params: {
|
||||
...params,
|
||||
message,
|
||||
ownerUserId: options.actor?.id ?? params.ownerUserId,
|
||||
ownerRole: options.actor?.role ?? params.ownerRole
|
||||
},
|
||||
options,
|
||||
traceStore
|
||||
});
|
||||
sendJson(response, 202, payload);
|
||||
} catch (error) {
|
||||
traceStore.append(traceId, {
|
||||
type: "backend",
|
||||
status: "failed",
|
||||
label: "agentrun:steer:failed",
|
||||
errorCode: error?.code ?? "agentrun_steer_failed",
|
||||
message: error?.message ?? "AgentRun steer command failed.",
|
||||
terminal: false,
|
||||
valuesPrinted: false
|
||||
});
|
||||
sendJson(response, error?.statusCode === 404 ? 404 : 409, {
|
||||
ok: false,
|
||||
accepted: false,
|
||||
status: "failed",
|
||||
traceId,
|
||||
route: "/v1/agent/chat/steer",
|
||||
sessionId: safeSessionId(currentResult.sessionId ?? params.sessionId) || null,
|
||||
threadId: safeOpaqueId(currentResult.threadId ?? params.threadId) || null,
|
||||
agentRun: currentResult.agentRun,
|
||||
runnerTrace: traceStore.snapshot(traceId),
|
||||
error: {
|
||||
code: error?.code ?? "agentrun_steer_failed",
|
||||
layer: "agentrun",
|
||||
category: "steer_failed",
|
||||
retryable: true,
|
||||
message: error?.message ?? "AgentRun steer command failed.",
|
||||
traceId,
|
||||
route: "/v1/agent/chat/steer",
|
||||
toolName: "agentrun.command.steer"
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleCodeAgentTraceHttp(request, response, url, options) {
|
||||
const startedAt = nowMs();
|
||||
const parts = url.pathname.split("/").filter(Boolean);
|
||||
const traceId = decodeURIComponent(parts[3] ?? "");
|
||||
const streamSegment = parts[4];
|
||||
const pageOptions = codeAgentTracePageOptions(url);
|
||||
if (!/^trc_[A-Za-z0-9_.:-]+$/u.test(traceId)) {
|
||||
sendJson(response, 400, {
|
||||
error: {
|
||||
code: "invalid_trace_id",
|
||||
message: "traceId must start with trc_ and contain only safe identifier characters"
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
const requestOptions = codeAgentRequestScopedOptions(options);
|
||||
const projectMismatch = await measureCodeAgentHttpPhase(requestOptions, "trace_project_check", () => traceProjectMismatchSnapshot(traceId, url, requestOptions, "trace"));
|
||||
if (projectMismatch) {
|
||||
sendJson(response, projectMismatch.statusCode, projectMismatch.body);
|
||||
return;
|
||||
}
|
||||
const context = await measureCodeAgentHttpPhase(requestOptions, "trace_projection_read", () => readCodeAgentCompatProjection(traceId, requestOptions));
|
||||
if (context.statusCode) {
|
||||
sendJson(response, context.statusCode, context.body);
|
||||
return;
|
||||
}
|
||||
|
||||
if (streamSegment === "stream") {
|
||||
sendJson(response, 410, {
|
||||
ok: false,
|
||||
status: "removed",
|
||||
traceId,
|
||||
error: {
|
||||
code: "trace_store_sse_removed",
|
||||
message: "Trace-store SSE was removed; use the committed Workbench projection event stream.",
|
||||
workbenchEventsUrl: `/v1/workbench/events?traceId=${encodeURIComponent(traceId)}`
|
||||
},
|
||||
valuesRedacted: true
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const diagnosticTrace = (requestOptions.traceStore ?? defaultCodeAgentTraceStore).snapshot(traceId);
|
||||
const trace = mergeCodeAgentDiagnosticTrace(context.trace, diagnosticTrace, traceId);
|
||||
const diagnosticFound = trace.events.length > 0;
|
||||
const body = await measureCodeAgentHttpPhase(requestOptions, "trace_paginate", () => {
|
||||
return paginateTraceSnapshot(trace, pageOptions);
|
||||
});
|
||||
const statusCode = context.found || diagnosticFound ? 200 : 404;
|
||||
void emitCodeAgentOtelSpan("trace_events_read", traceId, requestOptions.env ?? process.env, {
|
||||
startTimeMs: startedAt,
|
||||
attributes: {
|
||||
"http.method": "GET",
|
||||
"http.route": "/v1/agent/traces/:traceId",
|
||||
"http.status_code": statusCode,
|
||||
found: Boolean(context.found),
|
||||
returnedEvents: Array.isArray(body?.events) ? body.events.length : 0,
|
||||
sinceSeq: pageOptions.sinceSeq,
|
||||
limit: pageOptions.limit,
|
||||
fromSeq: body?.range?.fromSeq ?? null,
|
||||
toSeq: body?.range?.toSeq ?? null,
|
||||
totalEvents: body?.range?.total ?? body?.eventCount ?? null,
|
||||
hasMore: Boolean(body?.hasMore),
|
||||
fullTraceLoaded: Boolean(body?.fullTraceLoaded)
|
||||
}
|
||||
});
|
||||
sendJson(response, statusCode, codeAgentCompatProjectionPayload(body, context));
|
||||
}
|
||||
|
||||
function mergeCodeAgentDiagnosticTrace(projectionTrace, diagnosticTrace, traceId) {
|
||||
const projected = projectionTrace && typeof projectionTrace === "object" ? projectionTrace : {};
|
||||
const diagnostics = diagnosticTrace && typeof diagnosticTrace === "object" ? diagnosticTrace : {};
|
||||
const merged = new Map();
|
||||
for (const [source, events] of [["diagnostic", diagnostics.events], ["projection", projected.events]]) {
|
||||
for (const [index, event] of (Array.isArray(events) ? events : []).entries()) {
|
||||
const originalSeq = traceEventSequence(event, index);
|
||||
const key = textValue(event?.id ?? event?.sourceEventId) || `${originalSeq}:${textValue(event?.label ?? event?.type)}:${textValue(event?.createdAt ?? event?.occurredAt)}`;
|
||||
const normalized = source === "projection"
|
||||
? {
|
||||
...event,
|
||||
projectedSeq: Number.isFinite(Number(event?.projectedSeq)) ? Number(event.projectedSeq) : originalSeq,
|
||||
authority: "workbench-projection"
|
||||
}
|
||||
: {
|
||||
...event,
|
||||
sourceSeq: Number.isFinite(Number(event?.sourceSeq)) ? Number(event.sourceSeq) : originalSeq,
|
||||
terminal: false,
|
||||
sealed: false,
|
||||
diagnosticOnly: true,
|
||||
authority: "trace-store-diagnostics"
|
||||
};
|
||||
merged.set(key, { event: normalized, source, originalSeq, sourceIndex: index, key });
|
||||
}
|
||||
}
|
||||
const events = [...merged.values()]
|
||||
.sort((left, right) => left.originalSeq - right.originalSeq
|
||||
|| (left.source === right.source ? 0 : left.source === "projection" ? -1 : 1)
|
||||
|| left.sourceIndex - right.sourceIndex
|
||||
|| left.key.localeCompare(right.key))
|
||||
.map(({ event }, index) => ({ ...event, seq: index + 1, paginationSeq: index + 1 }));
|
||||
const projectionFound = projected.status && projected.status !== "missing";
|
||||
return {
|
||||
...projected,
|
||||
traceId,
|
||||
status: projectionFound ? projected.status : events.length > 0 ? "running" : "missing",
|
||||
terminal: projected.terminal === true,
|
||||
sealed: projected.sealed === true,
|
||||
events,
|
||||
eventCount: events.length,
|
||||
diagnosticEventCount: events.filter((event) => event.diagnosticOnly === true).length,
|
||||
terminalAuthority: "workbench-projection",
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function codeAgentProjectionStoreBlocker(context = {}) {
|
||||
const candidates = [
|
||||
context?.projection?.blocker,
|
||||
context?.trace?.projection?.blocker,
|
||||
context?.trace?.blocker,
|
||||
context?.result?.projection?.blocker,
|
||||
context?.result?.blocker
|
||||
];
|
||||
return candidates.find((blocker) => blocker?.code === "projection_store_unavailable") ?? null;
|
||||
}
|
||||
|
||||
function codeAgentTracePageOptions(url) {
|
||||
const sinceSeq = nonNegativeInteger(url.searchParams.get("sinceSeq") ?? url.searchParams.get("since"), 0);
|
||||
const requestedLimit = parsePositiveInteger(url.searchParams.get("limit"), DEFAULT_CODE_AGENT_TRACE_PAGE_LIMIT);
|
||||
return {
|
||||
sinceSeq,
|
||||
limit: Math.min(requestedLimit, MAX_CODE_AGENT_TRACE_PAGE_LIMIT)
|
||||
};
|
||||
}
|
||||
|
||||
function paginateTraceSnapshot(snapshot, { sinceSeq, limit }) {
|
||||
const sourceEvents = Array.isArray(snapshot?.events) ? snapshot.events : [];
|
||||
const indexedEvents = sourceEvents.map((event, index) => ({ event, seq: traceEventSequence(event, index) }));
|
||||
const firstIndex = indexedEvents.findIndex((item) => item.seq > sinceSeq);
|
||||
const startIndex = firstIndex >= 0 ? firstIndex : indexedEvents.length;
|
||||
const pageItems = indexedEvents.slice(startIndex, startIndex + limit);
|
||||
const events = pageItems.map((item) => item.event);
|
||||
const fromSeq = pageItems.length > 0 ? pageItems[0].seq : null;
|
||||
const toSeq = pageItems.length > 0 ? pageItems[pageItems.length - 1].seq : null;
|
||||
const hasMore = startIndex + pageItems.length < indexedEvents.length;
|
||||
const totalEvents = snapshot?.eventCount ?? indexedEvents.length;
|
||||
return {
|
||||
...snapshot,
|
||||
events,
|
||||
eventCount: totalEvents,
|
||||
range: {
|
||||
sinceSeq,
|
||||
fromSeq,
|
||||
toSeq,
|
||||
limit,
|
||||
returned: events.length,
|
||||
total: totalEvents
|
||||
},
|
||||
truncated: hasMore,
|
||||
hasMore,
|
||||
nextSinceSeq: pageItems.length > 0 ? toSeq : sinceSeq,
|
||||
fullTraceLoaded: !hasMore
|
||||
};
|
||||
}
|
||||
|
||||
function traceEventSequence(event, index) {
|
||||
const seq = Number(event?.seq);
|
||||
return Number.isFinite(seq) && seq > 0 ? Math.trunc(seq) : index + 1;
|
||||
}
|
||||
|
||||
function nonNegativeInteger(value, fallback) {
|
||||
const parsed = Number.parseInt(value ?? "", 10);
|
||||
return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback;
|
||||
}
|
||||
@@ -0,0 +1,921 @@
|
||||
// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010205 HWLAB接入 draft-2026-06-25-p0-session-warm-runner-contract; PJ2026-0102 Agent编排 draft-2026-06-17-r0; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
||||
// Responsibility: Code Agent HTTP result, turn, inspect, and cancel resources.
|
||||
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
|
||||
import { M3_IO_CONTROL_ROUTE, M3_STATUS_ROUTE, describeM3StatusLive, handleM3IoControl } from "./m3-io-control.ts";
|
||||
import {
|
||||
agentRunSessionEvidence,
|
||||
cancelAgentRunChatTurn,
|
||||
codeAgentAgentRunAdapterEnabled,
|
||||
initialAgentRunChatResult,
|
||||
loadPersistedAgentRunResult,
|
||||
steerAgentRunChatTurn,
|
||||
submitAgentRunChatTurn
|
||||
} from "./code-agent-agentrun-adapter.ts";
|
||||
import { createCodeAgentErrorPayload, handleCodeAgentChat } from "./code-agent-chat.ts";
|
||||
import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
||||
import { codeAgentSessionLifecycleSummary } from "./code-agent-session-lifecycle.ts";
|
||||
import { messageAuthorityTextValue } from "./code-agent-agentrun-prompt.ts";
|
||||
import { createWorkbenchReadModel } from "./workbench-read-model.ts";
|
||||
import { codeAgentOtelTraceFields, emitCodeAgentOtelSpan } from "./otel-trace.ts";
|
||||
import {
|
||||
firstHeaderValue,
|
||||
getHeader,
|
||||
parsePositiveInteger,
|
||||
positiveInteger,
|
||||
readBody,
|
||||
safeConversationId,
|
||||
safeOpaqueId,
|
||||
safeSessionId,
|
||||
safeTraceId,
|
||||
sendJson,
|
||||
truthyFlag
|
||||
} from "./server-http-utils.ts";
|
||||
|
||||
import {
|
||||
normalizeCodeAgentProviderProfile,
|
||||
firstNonEmptyValue,
|
||||
recordCodeAgentSessionInputFact,
|
||||
stableCodeAgentInputId,
|
||||
codeAgentInputDelivery,
|
||||
codeAgentInputStatus,
|
||||
codeAgentResultTimingFields,
|
||||
firstTimestampIso,
|
||||
firstLatestTimestampIso,
|
||||
codeAgentChatShortConnectionRequested,
|
||||
preflightCodeAgentBilling,
|
||||
recordCodeAgentBillingUsage,
|
||||
finalizeCodeAgentBillingUsage,
|
||||
releaseCodeAgentBillingReservation,
|
||||
withCodeAgentBillingReservation,
|
||||
codeAgentBillingEnabled,
|
||||
codeAgentBillingMetadata,
|
||||
sanitizeBillingReservation,
|
||||
sanitizeBillingRecord,
|
||||
sanitizeBillingRelease,
|
||||
codeAgentUsedTokens,
|
||||
normalizeTurnStatus,
|
||||
recordCodeAgentConversationFact,
|
||||
recordCodeAgentSessionOwner,
|
||||
codeAgentOwnerStatusForResult,
|
||||
textValue,
|
||||
timestampIsoOrNow,
|
||||
timestampIsoOrNull,
|
||||
codeAgentSessionOwnerEvidence,
|
||||
codeAgentContinuationEvidence,
|
||||
codeAgentFreshContinuationFailure,
|
||||
normalizeCodeAgentFailureKind,
|
||||
codeAgentFailureRequiresFreshContinuation,
|
||||
codeAgentProviderProfileEvidence,
|
||||
codeAgentTraceResultEvidence,
|
||||
codeAgentPromptMetadata,
|
||||
codeAgentPromptMetadataFromText,
|
||||
codeAgentPromptMetadataFromParts,
|
||||
codeAgentPromptFields,
|
||||
codeAgentPromptId,
|
||||
clippedPromptText,
|
||||
sha256Text,
|
||||
compactCodeAgentObject,
|
||||
codeAgentBillingReservationEvidence,
|
||||
codeAgentConversationMessagesEvidence,
|
||||
boundedConversationMessageText,
|
||||
conversationText,
|
||||
conversationTextRaw,
|
||||
codeAgentMessageTraceSuffix,
|
||||
codeAgentTurnLifecycleFields,
|
||||
codeAgentLifecycleMessageId,
|
||||
safeTurnId,
|
||||
safeMessageId,
|
||||
codeAgentConversationAgentMessageStatus,
|
||||
codeAgentPayloadHasSealedFinalResponse,
|
||||
codeAgentConversationRunnerTraceEvidence,
|
||||
codeAgentFinalResponseEvidence,
|
||||
codeAgentFinalResponseText,
|
||||
codeAgentTraceSummaryEvidence,
|
||||
annotateOwner,
|
||||
canAccessOwnedResult,
|
||||
isCodeAgentResultCanceled,
|
||||
codeAgentCancelScopeMismatch,
|
||||
cancelExpectedValues,
|
||||
cancelScopeFieldMismatch,
|
||||
cancelBlockedPayload,
|
||||
isTraceCommandTerminalStatus,
|
||||
traceSnapshotWithTerminalEvidence,
|
||||
traceSnapshotWithAttachedTerminalEvidence,
|
||||
agentRunTerminalTraceEvidence,
|
||||
agentRunTerminalFinalResponse,
|
||||
traceRetentionSummary,
|
||||
numberOrNull,
|
||||
createCodeAgentChatResultStore,
|
||||
compactCodeAgentChatResultPayload,
|
||||
terminalEvidencePayload,
|
||||
compactRunnerTraceForResult,
|
||||
traceEventLabel,
|
||||
resultTraceEventLimit,
|
||||
createCodeAgentM3HwlabApiRequestJson,
|
||||
recordCodeAgentProjectionMetrics,
|
||||
recordCodeAgentTurnReadMetric,
|
||||
turnReadDegradedReason,
|
||||
responseBodyBytes,
|
||||
statusClassLabel,
|
||||
sourceLatestSeqFromResult,
|
||||
sourceLatestAtFromResult,
|
||||
terminalSourceAtFromResult,
|
||||
projectionReason,
|
||||
nowMs,
|
||||
uniqueStrings
|
||||
} from "./server-code-agent-http-support.ts";
|
||||
|
||||
const DEFAULT_CODE_AGENT_RESULT_TRACE_EVENT_LIMIT = 120;
|
||||
const DEFAULT_CODE_AGENT_TRACE_PAGE_LIMIT = 100;
|
||||
const MAX_CODE_AGENT_TRACE_PAGE_LIMIT = 100;
|
||||
const DEFAULT_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS = 2500;
|
||||
const DEFAULT_CODE_AGENT_PROJECT_ID = "prj_hwpod_workbench";
|
||||
const DEFAULT_CODE_AGENT_PROVIDER_PROFILE = "deepseek";
|
||||
const CODE_AGENT_TERMINAL_STATUSES = new Set(["completed", "failed", "blocked", "timeout", "cancelled", "canceled"]);
|
||||
const CODE_AGENT_PROVIDER_PROFILE_ALIASES = Object.freeze({ codex: "codex-api" });
|
||||
const CODE_AGENT_PROVIDER_PROFILE_ID_PATTERN = /^[a-z0-9][a-z0-9.-]{0,63}$/u;
|
||||
const CODE_AGENT_PROVIDER_PROFILE_LABELS = Object.freeze({
|
||||
"deepseek": "DeepSeek",
|
||||
"codex-api": "Codex API",
|
||||
"gpt.pika": "gpt.pika",
|
||||
"minimax-m3": "MiniMax-M3 via AgentRun",
|
||||
"runtime-default": "运行默认"
|
||||
});
|
||||
const DEFAULT_CODE_AGENT_CODEX_API_MODEL = "gpt-5.5";
|
||||
const DEFAULT_CODE_AGENT_CODEX_API_BASE_URL = "http://127.0.0.1:49280/responses";
|
||||
const DEFAULT_CODE_AGENT_DEEPSEEK_MODEL = "deepseek-chat";
|
||||
const DEFAULT_CODE_AGENT_MINIMAX_M3_MODEL = "MiniMax-M3";
|
||||
|
||||
export async function handleCodeAgentChatResultHttp(request, response, url, options) {
|
||||
const parts = url.pathname.split("/").filter(Boolean);
|
||||
const traceId = decodeURIComponent(parts[4] ?? "");
|
||||
if (!safeTraceId(traceId)) {
|
||||
sendJson(response, 400, {
|
||||
error: {
|
||||
code: "invalid_trace_id",
|
||||
message: "traceId must start with trc_ and contain only safe identifier characters"
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
const context = await measureCodeAgentHttpPhase(options, "result_projection_read", () => readCodeAgentCompatProjection(traceId, options));
|
||||
if (context.statusCode) {
|
||||
sendJson(response, context.statusCode, context.body);
|
||||
return;
|
||||
}
|
||||
const result = context.result;
|
||||
if (result && result.status !== "running" && !codeAgentResultUsesProjectionTerminalAuthority(result)) {
|
||||
sendJson(response, 200, codeAgentCompatProjectionPayload(compactCodeAgentChatResultPayload(result, options), context));
|
||||
return;
|
||||
}
|
||||
if (context.found) {
|
||||
const body = codeAgentTurnStatusPayload({ traceId, result, snapshot: context.trace, resultPollError: null, refreshError: context.refreshError ?? null, options });
|
||||
const resultPollTerminalReady = codeAgentResultPollTerminalReady(body);
|
||||
sendJson(response, resultPollTerminalReady ? 200 : 202, codeAgentCompatProjectionPayload({
|
||||
accepted: body.ok,
|
||||
shortConnection: true,
|
||||
...body,
|
||||
waitingFor: resultPollTerminalReady ? body.waitingFor ?? context.trace?.waitingFor ?? "workbench-projection" : "agentrun-result",
|
||||
...(body.terminal === true && !resultPollTerminalReady ? { status: "running", running: true, terminal: false } : {})
|
||||
}, context));
|
||||
return;
|
||||
}
|
||||
sendJson(response, 404, {
|
||||
...codeAgentCompatProjectionPayload({ traceId, status: "unknown" }, context),
|
||||
error: {
|
||||
code: "code_agent_result_not_found",
|
||||
message: `No Code Agent result is registered for ${traceId}`
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export async function handleCodeAgentTurnHttp(request, response, url, options) {
|
||||
const startedAt = nowMs();
|
||||
const parts = url.pathname.split("/").filter(Boolean);
|
||||
const traceId = decodeURIComponent(parts[3] ?? "");
|
||||
if (!safeTraceId(traceId)) {
|
||||
const body = {
|
||||
ok: false,
|
||||
status: "unknown",
|
||||
running: false,
|
||||
terminal: false,
|
||||
error: {
|
||||
code: "invalid_trace_id",
|
||||
message: "traceId must start with trc_ and contain only safe identifier characters"
|
||||
}
|
||||
};
|
||||
recordCodeAgentTurnReadMetric(options, url, 400, body, startedAt);
|
||||
sendJson(response, 400, body);
|
||||
return;
|
||||
}
|
||||
const requestOptions = codeAgentRequestScopedOptions(options);
|
||||
let completed = false;
|
||||
const emitTurnStatusReadSpan = (name, attributes = {}, error = null) => {
|
||||
void emitCodeAgentOtelSpan(name, traceId, requestOptions.env ?? process.env, {
|
||||
startTimeMs: startedAt,
|
||||
status: error ? "error" : undefined,
|
||||
error,
|
||||
attributes: {
|
||||
"http.method": "GET",
|
||||
"http.route": "/v1/agent/turns/:traceId",
|
||||
durationMs: nowMs() - startedAt,
|
||||
...attributes
|
||||
}
|
||||
});
|
||||
};
|
||||
response.once?.("close", () => {
|
||||
if (completed) return;
|
||||
emitTurnStatusReadSpan("turn_status_read.client_closed", {
|
||||
status: "client_closed",
|
||||
phase: "response_close",
|
||||
terminal: false
|
||||
}, new Error("Code Agent turn status client connection closed before response completed."));
|
||||
});
|
||||
try {
|
||||
const projectMismatch = await measureCodeAgentHttpPhase(requestOptions, "turn_project_check", () => traceProjectMismatchSnapshot(traceId, url, requestOptions, "turn"));
|
||||
if (projectMismatch) {
|
||||
completed = true;
|
||||
recordCodeAgentTurnReadMetric(options, url, projectMismatch.statusCode, projectMismatch.body, startedAt);
|
||||
emitTurnStatusReadSpan("turn_status_read", { "http.status_code": projectMismatch.statusCode, status: projectMismatch.body?.status ?? null, terminal: projectMismatch.body?.terminal ?? null, phase: "project_check" });
|
||||
sendJson(response, projectMismatch.statusCode, projectMismatch.body);
|
||||
return;
|
||||
}
|
||||
const resolved = await measureCodeAgentHttpPhase(requestOptions, "turn_resolve_status", () => resolveCodeAgentTurnStatusSnapshot(traceId, requestOptions));
|
||||
completed = true;
|
||||
recordCodeAgentTurnReadMetric(options, url, resolved.statusCode, resolved.body, startedAt);
|
||||
emitTurnStatusReadSpan("turn_status_read", { "http.status_code": resolved.statusCode, status: resolved.body?.status ?? null, terminal: resolved.body?.terminal ?? null, phase: "resolved" });
|
||||
sendJson(response, resolved.statusCode, resolved.body);
|
||||
} catch (error) {
|
||||
completed = true;
|
||||
const body = {
|
||||
ok: false,
|
||||
status: "unknown",
|
||||
running: false,
|
||||
terminal: false,
|
||||
traceId,
|
||||
error: {
|
||||
code: error?.code ?? "turn_status_read_failed",
|
||||
message: error?.message ?? "Code Agent turn status read failed."
|
||||
},
|
||||
valuesRedacted: true
|
||||
};
|
||||
recordCodeAgentTurnReadMetric(options, url, 500, body, startedAt);
|
||||
emitTurnStatusReadSpan("turn_status_read", { "http.status_code": 500, status: body.status, terminal: false, phase: "failed", errorCode: body.error.code }, error);
|
||||
sendJson(response, 500, body);
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveCodeAgentTurnStatusSnapshot(traceId, options) {
|
||||
const context = await readCodeAgentCompatProjection(traceId, options);
|
||||
if (context.statusCode) return context;
|
||||
const body = await measureCodeAgentHttpPhase(options, "turn_payload", () => codeAgentTurnStatusPayload({
|
||||
traceId,
|
||||
result: context.result,
|
||||
snapshot: context.trace,
|
||||
resultPollError: null,
|
||||
refreshError: context.refreshError ?? null,
|
||||
options
|
||||
}));
|
||||
return { statusCode: body.ok ? 200 : 404, body: codeAgentCompatProjectionPayload(body, context) };
|
||||
}
|
||||
|
||||
function codeAgentRequestScopedOptions(options = {}) {
|
||||
return {
|
||||
...options,
|
||||
codeAgentTraceSessionCache: options.codeAgentTraceSessionCache instanceof Map ? options.codeAgentTraceSessionCache : new Map()
|
||||
};
|
||||
}
|
||||
|
||||
async function measureCodeAgentHttpPhase(options, phase, callback) {
|
||||
const perf = options?.backendPerformance;
|
||||
return perf && typeof perf.measure === "function" ? await perf.measure(phase, callback) : await callback();
|
||||
}
|
||||
|
||||
function forbiddenTurnSnapshot(traceId) {
|
||||
return {
|
||||
statusCode: 403,
|
||||
body: {
|
||||
ok: false,
|
||||
status: "unknown",
|
||||
running: false,
|
||||
terminal: false,
|
||||
traceId,
|
||||
error: {
|
||||
code: "agent_session_owner_required",
|
||||
message: "Only the session owner or admin can read this Code Agent turn status"
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function readCodeAgentCompatProjection(traceId, options = {}) {
|
||||
const readModel = createWorkbenchReadModel(options, options.actor ?? null);
|
||||
const session = await readModel.getSessionByTraceId(traceId);
|
||||
let result = readModel.resultForTrace(traceId) ?? projectedSessionTraceResult(session, traceId);
|
||||
if (result && !canAccessOwnedResult(result, options.actor)) return forbiddenTurnSnapshot(traceId);
|
||||
if (result?.ownerUserId && !readModel.canReadOwner(result.ownerUserId)) return forbiddenTurnSnapshot(traceId);
|
||||
const projectionTrace = await readModel.traceSnapshot(traceId);
|
||||
let trace = traceSnapshotWithTerminalEvidence(projectionTrace, result, traceId, null);
|
||||
const projection = readModel.projectionDiagnostics({ traceId, result, trace, refreshError: null });
|
||||
recordCodeAgentProjectionMetrics(options, { result, trace, projection, refreshError: null });
|
||||
const found = Boolean(result || session || (trace && trace.status !== "missing") || trace?.persisted === true);
|
||||
return { result, session, trace, projection, found, refreshError: null };
|
||||
}
|
||||
|
||||
function projectedSessionTraceResult(session, traceId) {
|
||||
const safeId = safeTraceId(traceId);
|
||||
if (!session || !safeId) return null;
|
||||
const snapshot = session.session && typeof session.session === "object" ? session.session : null;
|
||||
const traceResults = snapshot?.traceResults && typeof snapshot.traceResults === "object" ? snapshot.traceResults : null;
|
||||
const stored = traceResults?.[safeId] && typeof traceResults[safeId] === "object" ? traceResults[safeId] : null;
|
||||
if (!stored) return null;
|
||||
return {
|
||||
...stored,
|
||||
traceId: safeId,
|
||||
conversationId: safeConversationId(stored.conversationId ?? session.conversationId) || null,
|
||||
sessionId: safeSessionId(stored.sessionId ?? session.id) || null,
|
||||
threadId: safeOpaqueId(stored.threadId ?? session.threadId) || null,
|
||||
ownerUserId: session.ownerUserId ?? stored.ownerUserId ?? null,
|
||||
ownerRole: session.ownerRole ?? stored.ownerRole ?? null,
|
||||
status: stored.status ?? session.status ?? null,
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function codeAgentCompatProjectionPayload(payload = {}, context = {}) {
|
||||
const traceId = safeTraceId(payload.traceId ?? context.trace?.traceId) ?? null;
|
||||
const projection = context.projection && typeof context.projection === "object" ? context.projection : {};
|
||||
return {
|
||||
...payload,
|
||||
projection,
|
||||
projectionStatus: projection.projectionStatus ?? null,
|
||||
projectionHealth: projection.projectionHealth ?? null,
|
||||
lastProjectedSeq: projection.lastProjectedSeq ?? null,
|
||||
sourceRunId: projection.sourceRunId ?? null,
|
||||
sourceCommandId: projection.sourceCommandId ?? null,
|
||||
staleMs: projection.staleMs ?? null,
|
||||
blocker: projection.blocker ?? null,
|
||||
workbench: traceId ? {
|
||||
detailOnly: true,
|
||||
turnDetailUrl: `/v1/workbench/turns/${encodeURIComponent(traceId)}`,
|
||||
traceEventsDetailUrl: `/v1/workbench/traces/${encodeURIComponent(traceId)}/events`
|
||||
} : null,
|
||||
valuesRedacted: true,
|
||||
secretMaterialStored: false
|
||||
};
|
||||
}
|
||||
|
||||
function codeAgentResultPollTerminalReady(body = {}) {
|
||||
if (body?.terminal !== true) return false;
|
||||
return Boolean(messageAuthorityTextValue(body.finalResponse ?? body.reply ?? body.terminalEvidence?.finalResponse));
|
||||
}
|
||||
|
||||
async function traceProjectMismatchSnapshot(traceId, url, options, resourceKind) {
|
||||
const requestedProjectId = textValue(url.searchParams.get("projectId"));
|
||||
if (!requestedProjectId || !options.accessController?.getAgentSessionByTraceId) return null;
|
||||
const session = await getCodeAgentSessionByTraceId(traceId, options);
|
||||
if (!session) return null;
|
||||
if (session.ownerUserId && options.actor?.role !== "admin" && session.ownerUserId !== options.actor?.id) return forbiddenTurnSnapshot(traceId);
|
||||
const actualProjectId = textValue(session.projectId);
|
||||
if (!actualProjectId || actualProjectId === requestedProjectId) return null;
|
||||
return {
|
||||
statusCode: 404,
|
||||
body: {
|
||||
ok: false,
|
||||
status: "not_found",
|
||||
running: false,
|
||||
terminal: false,
|
||||
traceId,
|
||||
error: {
|
||||
code: "trace_project_mismatch",
|
||||
message: `${resourceKind === "turn" ? "Agent turn" : "Agent trace"} is not visible in the requested project`
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function getCodeAgentSessionByTraceId(traceId, options = {}) {
|
||||
const safeId = safeTraceId(traceId);
|
||||
if (!safeId || typeof options.accessController?.getAgentSessionByTraceId !== "function") return null;
|
||||
const cache = options.codeAgentTraceSessionCache;
|
||||
if (cache instanceof Map) {
|
||||
if (cache.has(safeId)) return cache.get(safeId);
|
||||
const session = await options.accessController.getAgentSessionByTraceId(safeId);
|
||||
cache.set(safeId, session ?? null);
|
||||
return session ?? null;
|
||||
}
|
||||
return await options.accessController.getAgentSessionByTraceId(safeId);
|
||||
}
|
||||
|
||||
export function codeAgentTurnStatusPayload({ traceId, result, snapshot, resultPollError, refreshError, options }) {
|
||||
const resultObject = result && typeof result === "object" ? result : null;
|
||||
const snapshotObject = snapshot && typeof snapshot === "object" ? snapshot : null;
|
||||
const projectionTerminalAuthority = codeAgentResultUsesProjectionTerminalAuthority(resultObject);
|
||||
const lifecycle = codeAgentTurnLifecycleFields(traceId, resultObject ?? snapshotObject ?? {});
|
||||
const events = Array.isArray(snapshotObject?.events) ? snapshotObject.events : Array.isArray(resultObject?.runnerTrace?.events) ? resultObject.runnerTrace.events : [];
|
||||
const lastEvent = events.at(-1) ?? null;
|
||||
const finalResponse = projectionTerminalAuthority
|
||||
? snapshotObject?.finalResponse ?? snapshotObject?.terminalEvidence?.finalResponse ?? codeAgentFinalResponseEvidence(snapshotObject ?? {}, traceId)
|
||||
: resultObject?.finalResponse ?? snapshotObject?.finalResponse ?? snapshotObject?.terminalEvidence?.finalResponse ?? codeAgentFinalResponseEvidence(resultObject ?? snapshotObject ?? {}, traceId);
|
||||
const terminalStatus = projectionTerminalAuthority
|
||||
? codeAgentProjectionTerminalStatus(snapshotObject)
|
||||
: codeAgentAuthoritativeTerminalStatus(resultObject, snapshotObject, traceId);
|
||||
const terminalSealBlocked = Boolean(terminalStatus && isTurnTerminalStatus(terminalStatus) && !codeAgentTerminalFinalText(finalResponse, projectionTerminalAuthority ? null : resultObject, snapshotObject));
|
||||
const rawStatus = projectionTerminalAuthority
|
||||
? normalizeTurnStatus(
|
||||
terminalStatus,
|
||||
codeAgentRunningStatus(snapshotObject?.status),
|
||||
codeAgentRunningStatus(snapshotObject?.traceStatus),
|
||||
codeAgentRunningStatus(snapshotObject?.runnerTrace?.status),
|
||||
codeAgentRunningStatus(resultObject?.status),
|
||||
"running"
|
||||
)
|
||||
: normalizeTurnStatus(
|
||||
terminalStatus,
|
||||
resultObject?.agentRun?.commandState,
|
||||
resultObject?.agentRun?.status,
|
||||
resultObject?.agentRun?.runStatus,
|
||||
codeAgentRunningStatus(snapshotObject?.status),
|
||||
codeAgentRunningStatus(snapshotObject?.traceStatus),
|
||||
codeAgentRunningStatus(snapshotObject?.runnerTrace?.status),
|
||||
codeAgentRunningStatus(resultObject?.status),
|
||||
resultObject?.agentRun?.terminalStatus,
|
||||
snapshotObject?.terminalEvidence?.traceSummary?.terminalStatus
|
||||
);
|
||||
const status = terminalSealBlocked ? "running" : rawStatus;
|
||||
const found = Boolean(resultObject || (snapshotObject && snapshotObject.status !== "missing") || snapshotObject?.persisted === true);
|
||||
const running = isTurnRunningStatus(status);
|
||||
const terminal = isTurnTerminalStatus(status);
|
||||
const runnerTrace = snapshotObject && snapshotObject.status !== "missing" ? snapshotObject : resultObject?.runnerTrace ?? null;
|
||||
const timingFields = codeAgentResultTimingFields(resultObject, snapshotObject, runnerTrace);
|
||||
return {
|
||||
ok: found,
|
||||
action: "code-agent.turn.status",
|
||||
contractVersion: "code-agent-turn-status-v1",
|
||||
status: found ? status ?? "unknown" : "unknown",
|
||||
running: found ? running : false,
|
||||
terminal: found ? terminal : false,
|
||||
traceId,
|
||||
...lifecycle,
|
||||
conversationId: safeConversationId(resultObject?.conversationId ?? snapshotObject?.conversationId) || null,
|
||||
sessionId: safeSessionId(resultObject?.sessionId ?? resultObject?.session?.sessionId ?? snapshotObject?.sessionId) || null,
|
||||
threadId: safeOpaqueId(resultObject?.threadId ?? resultObject?.session?.threadId ?? snapshotObject?.threadId) || null,
|
||||
updatedAt: textValue(resultObject?.updatedAt ?? resultObject?.agentRun?.updatedAt ?? snapshotObject?.updatedAt ?? timingFields.lastEventAt ?? timingFields.startedAt) || null,
|
||||
...timingFields,
|
||||
lastEventLabel: textValue(snapshotObject?.lastEventLabel ?? runnerTrace?.lastEventLabel ?? lastEvent?.label ?? lastEvent?.type) || null,
|
||||
waitingFor: terminalSealBlocked ? "final_response" : textValue(snapshotObject?.waitingFor ?? runnerTrace?.waitingFor) || null,
|
||||
terminalObserved: Boolean(terminalStatus),
|
||||
terminalObservedStatus: terminalStatus ?? null,
|
||||
terminalSealBlocked,
|
||||
resultUrl: `/v1/agent/chat/result/${encodeURIComponent(traceId)}`,
|
||||
traceUrl: `/v1/agent/traces/${encodeURIComponent(traceId)}`,
|
||||
turnUrl: `/v1/agent/turns/${encodeURIComponent(traceId)}`,
|
||||
runnerTrace: runnerTrace ? compactRunnerTraceForResult(runnerTrace, resultTraceEventLimit(options)) : null,
|
||||
agentRun: resultObject?.agentRun ?? snapshotObject?.agentRun ?? null,
|
||||
terminalEvidence: snapshotObject?.terminalEvidence ?? null,
|
||||
finalResponse,
|
||||
traceSummary: resultObject?.traceSummary ?? snapshotObject?.traceSummary ?? snapshotObject?.terminalEvidence?.traceSummary ?? null,
|
||||
retention: snapshotObject?.retention ?? null,
|
||||
eventCount: numberOrNull(snapshotObject?.eventCount ?? runnerTrace?.eventCount ?? events.length),
|
||||
error: resultPollError || refreshError
|
||||
? codeAgentRefreshErrorPayload(resultPollError ?? refreshError, traceId, resultObject?.agentRun ?? snapshotObject?.agentRun, "turn_status_degraded")
|
||||
: resultObject?.error ?? snapshotObject?.error ?? null,
|
||||
valuesRedacted: true,
|
||||
secretMaterialStored: false
|
||||
};
|
||||
}
|
||||
|
||||
function codeAgentResultUsesProjectionTerminalAuthority(result = null) {
|
||||
const agentRun = result?.agentRun && typeof result.agentRun === "object" ? result.agentRun : null;
|
||||
if (!agentRun || agentRun.adapter !== "agentrun-v01") return false;
|
||||
return agentRun.durableDispatch === true
|
||||
|| Boolean(textValue(agentRun.dispatchIntentId ?? agentRun.commandId));
|
||||
}
|
||||
|
||||
function codeAgentProjectionTerminalStatus(snapshotObject = null) {
|
||||
const status = normalizeTurnStatus(
|
||||
snapshotObject?.terminalEvidence?.status,
|
||||
snapshotObject?.terminalEvidence?.traceSummary?.terminalStatus,
|
||||
snapshotObject?.status
|
||||
);
|
||||
return status && isTurnTerminalStatus(status) && codeAgentSnapshotHasTerminalAuthority(snapshotObject) ? status : null;
|
||||
}
|
||||
|
||||
function codeAgentTerminalFinalText(finalResponse = null, resultObject = null, snapshotObject = null) {
|
||||
const candidates = [
|
||||
finalResponse,
|
||||
resultObject?.finalResponse,
|
||||
snapshotObject?.finalResponse,
|
||||
snapshotObject?.terminalEvidence?.finalResponse,
|
||||
resultObject?.terminalEvidence?.finalResponse
|
||||
];
|
||||
for (const value of candidates) {
|
||||
const text = conversationText(value);
|
||||
if (text) return text;
|
||||
}
|
||||
return codeAgentFinalResponseText(resultObject ?? {}) || codeAgentFinalResponseText(snapshotObject ?? {});
|
||||
}
|
||||
|
||||
function codeAgentAuthoritativeTerminalStatus(resultObject, snapshotObject, traceId) {
|
||||
const sealedPayload = codeAgentPayloadHasSealedFinalResponse(resultObject ?? {}) ? resultObject : codeAgentPayloadHasSealedFinalResponse(snapshotObject ?? {}) ? snapshotObject : null;
|
||||
if (sealedPayload) {
|
||||
const sealedStatus = normalizeTurnStatus(sealedPayload?.finalResponse?.status, sealedPayload?.terminalEvidence?.finalResponse?.status, sealedPayload?.agentRun?.terminalStatus, sealedPayload?.agentRun?.status, sealedPayload?.status);
|
||||
return sealedStatus && isTurnTerminalStatus(sealedStatus) ? sealedStatus : "completed";
|
||||
}
|
||||
const snapshotStatus = normalizeTurnStatus(snapshotObject?.terminalEvidence?.traceSummary?.terminalStatus, snapshotObject?.terminalEvidence?.agentRun?.terminalStatus, snapshotObject?.terminalEvidence?.status, snapshotObject?.status);
|
||||
if (snapshotStatus && isTurnTerminalStatus(snapshotStatus) && codeAgentSnapshotHasTerminalAuthority(snapshotObject)) return snapshotStatus;
|
||||
const agentRunStatus = normalizeTurnStatus(resultObject?.agentRun?.terminalStatus, resultObject?.agentRun?.commandState, resultObject?.agentRun?.status, resultObject?.agentRun?.runStatus);
|
||||
if (agentRunStatus && isTurnTerminalStatus(agentRunStatus)) return agentRunStatus;
|
||||
const resultStatus = normalizeTurnStatus(resultObject?.status);
|
||||
if (resultStatus && isTurnTerminalStatus(resultStatus) && codeAgentResultHasTerminalAuthority(resultObject, traceId)) return resultStatus;
|
||||
return null;
|
||||
}
|
||||
|
||||
function codeAgentSnapshotHasTerminalAuthority(snapshot = null) {
|
||||
if (!snapshot || typeof snapshot !== "object") return false;
|
||||
if (snapshot.terminal === true || snapshot.sealed === true) return true;
|
||||
if (snapshot.terminalEvidence?.available === true || snapshot.terminalEvidence?.source) return true;
|
||||
const events = Array.isArray(snapshot.events) ? snapshot.events : [];
|
||||
return events.some((event) => event?.terminal === true);
|
||||
}
|
||||
|
||||
function codeAgentResultHasTerminalAuthority(result = null, traceId = null) {
|
||||
if (!result || typeof result !== "object") return false;
|
||||
if (result.terminal === true || result.sealed === true) return true;
|
||||
if (result.error || result.blocker) return true;
|
||||
if (codeAgentPayloadHasSealedFinalResponse(result)) return true;
|
||||
if (agentRunTerminalTraceEvidence(result, traceId)) return true;
|
||||
return Boolean(textValue(result.finishedAt ?? result.completedAt ?? result.endedAt));
|
||||
}
|
||||
|
||||
function codeAgentRefreshErrorPayload(error, traceId, agentRun, fallbackCode) {
|
||||
return {
|
||||
code: error?.code ?? fallbackCode,
|
||||
layer: error?.layer ?? "agentrun",
|
||||
category: error?.category ?? (error?.code === "agentrun_timeout" ? "upstream-timeout" : "upstream-refresh-failed"),
|
||||
retryable: error?.retryable !== false,
|
||||
message: error?.message ?? "Code Agent turn status refresh degraded",
|
||||
traceId,
|
||||
runId: agentRun?.runId ?? error?.runId ?? null,
|
||||
commandId: agentRun?.commandId ?? error?.commandId ?? null,
|
||||
method: error?.method ?? null,
|
||||
route: error?.route ?? error?.path ?? null,
|
||||
timeoutMs: numberOrNull(error?.timeoutMs),
|
||||
timeoutStage: error?.timeoutStage ?? null,
|
||||
upstreamStatus: numberOrNull(error?.statusCode),
|
||||
managerHost: error?.managerHost ?? null,
|
||||
valuesPrinted: false
|
||||
};
|
||||
}
|
||||
|
||||
function isTurnRunningStatus(status) {
|
||||
return status === "running";
|
||||
}
|
||||
|
||||
function codeAgentRunningStatus(value) {
|
||||
const status = normalizeTurnStatus(value);
|
||||
return isTurnRunningStatus(status) ? status : null;
|
||||
}
|
||||
|
||||
function isTurnTerminalStatus(status) {
|
||||
return CODE_AGENT_TERMINAL_STATUSES.has(String(status ?? "").trim().toLowerCase().replace(/_/gu, "-"));
|
||||
}
|
||||
|
||||
export async function handleCodeAgentInspectHttp(request, response, url, options) {
|
||||
const query = {
|
||||
conversationId: safeConversationId(url.searchParams.get("conversationId")),
|
||||
sessionId: safeSessionId(url.searchParams.get("sessionId")),
|
||||
threadId: safeOpaqueId(url.searchParams.get("threadId")),
|
||||
traceId: safeTraceId(url.searchParams.get("traceId"))
|
||||
};
|
||||
const sessionRegistry = options.sessionRegistry;
|
||||
const traceStore = options.traceStore ?? defaultCodeAgentTraceStore;
|
||||
const manager = options.codexStdioManager;
|
||||
const managerDescription = manager && typeof manager.describe === "function" ? manager.describe() : null;
|
||||
const managerRecentSessions = Array.isArray(managerDescription?.recentSessions) ? managerDescription.recentSessions : [];
|
||||
const directManagerSession = query.sessionId && manager && typeof manager.get === "function"
|
||||
? manager.get(query.sessionId, { conversationId: query.conversationId })
|
||||
: null;
|
||||
const matchedManagerSession = directManagerSession ?? managerRecentSessions.find((session) => sessionMatchesInspectQuery(session, query)) ?? null;
|
||||
const registryInspect = sessionRegistry && typeof sessionRegistry.inspect === "function"
|
||||
? sessionRegistry.inspect(query)
|
||||
: { ok: false, status: "unavailable", traceIds: [], session: null, conversationFacts: null };
|
||||
const resultEvidence = await codeAgentInspectResultEvidence(query.traceId, options);
|
||||
const session = matchedManagerSession ?? registryInspect.session ?? resultEvidence.session ?? null;
|
||||
const conversationFacts = registryInspect.conversationFacts ?? resultEvidence.conversationFacts ?? null;
|
||||
const requestedRunnerTrace = query.traceId ? traceStore.snapshot(query.traceId) : null;
|
||||
const requestedTraceFound = Boolean(requestedRunnerTrace && requestedRunnerTrace.status !== "missing");
|
||||
const traceIds = uniqueStrings([
|
||||
requestedTraceFound ? query.traceId : null,
|
||||
session?.currentTraceId,
|
||||
session?.lastTraceId,
|
||||
conversationFacts?.latestTraceId,
|
||||
resultEvidence.latestTraceId,
|
||||
...(Array.isArray(conversationFacts?.traceIds) ? conversationFacts.traceIds : []),
|
||||
...(Array.isArray(registryInspect.traceIds) ? registryInspect.traceIds : [])
|
||||
]);
|
||||
const latestTraceId = traceIds[0] ?? null;
|
||||
const runnerTrace = latestTraceId
|
||||
? latestTraceId === query.traceId && requestedRunnerTrace
|
||||
? requestedRunnerTrace
|
||||
: traceStore.snapshot(latestTraceId)
|
||||
: null;
|
||||
const found = Boolean(registryInspect.ok || session || latestTraceId);
|
||||
sendJson(response, found ? 200 : 404, {
|
||||
ok: found,
|
||||
action: "code-agent.chat.inspect",
|
||||
status: found ? "found" : "not_found",
|
||||
query,
|
||||
session,
|
||||
conversationFacts,
|
||||
traceIds,
|
||||
latestTraceId,
|
||||
traceUrl: latestTraceId ? `/v1/agent/traces/${encodeURIComponent(latestTraceId)}` : null,
|
||||
resultUrl: latestTraceId ? `/v1/agent/chat/result/${encodeURIComponent(latestTraceId)}` : null,
|
||||
runnerTrace,
|
||||
valuesRedacted: true,
|
||||
secretMaterialStored: false
|
||||
});
|
||||
}
|
||||
|
||||
async function codeAgentInspectResultEvidence(traceId, options = {}) {
|
||||
if (!safeTraceId(traceId)) return { session: null, conversationFacts: null, latestTraceId: null };
|
||||
const cached = options.codeAgentChatResults?.get?.(traceId) ?? null;
|
||||
const persisted = cached ? null : await loadPersistedAgentRunResult(traceId, options);
|
||||
const result = cached ?? persisted ?? null;
|
||||
if (!result || typeof result !== "object") return { session: null, conversationFacts: null, latestTraceId: null };
|
||||
const agentRun = result.agentRun && typeof result.agentRun === "object" ? result.agentRun : null;
|
||||
const resultSession = result.session && typeof result.session === "object" ? result.session : null;
|
||||
const sessionReuse = result.sessionReuse && typeof result.sessionReuse === "object" ? result.sessionReuse : null;
|
||||
const providerTrace = result.providerTrace && typeof result.providerTrace === "object" ? result.providerTrace : null;
|
||||
const sessionId = safeSessionId(result.sessionId ?? resultSession?.sessionId ?? sessionReuse?.sessionId ?? agentRun?.sessionId) || null;
|
||||
const conversationId = safeConversationId(result.conversationId ?? resultSession?.conversationId ?? sessionReuse?.conversationId ?? agentRun?.conversationId) || null;
|
||||
const threadId = safeOpaqueId(result.threadId ?? resultSession?.threadId ?? sessionReuse?.threadId ?? providerTrace?.threadId ?? agentRun?.threadId) || null;
|
||||
const status = textValue(resultSession?.status ?? result.status ?? agentRun?.status) || null;
|
||||
const updatedAt = textValue(result.updatedAt ?? agentRun?.updatedAt ?? resultSession?.updatedAt) || null;
|
||||
const session = sessionId || conversationId || threadId ? {
|
||||
sessionId,
|
||||
conversationId,
|
||||
threadId,
|
||||
status,
|
||||
lastTraceId: traceId,
|
||||
source: "code-agent-result",
|
||||
agentRun: agentRun ? agentRunSessionEvidence(result).agentRun : null,
|
||||
valuesRedacted: true,
|
||||
secretMaterialStored: false
|
||||
} : null;
|
||||
const conversationFacts = conversationId ? {
|
||||
conversationId,
|
||||
sessionId,
|
||||
threadId,
|
||||
latestTraceId: traceId,
|
||||
traceIds: [traceId],
|
||||
turnCount: 1,
|
||||
source: "code-agent-result",
|
||||
latestStatus: status,
|
||||
updatedAt,
|
||||
valuesRedacted: true,
|
||||
secretMaterialStored: false
|
||||
} : null;
|
||||
return { session, conversationFacts, latestTraceId: traceId };
|
||||
}
|
||||
|
||||
function sessionMatchesInspectQuery(session, query) {
|
||||
if (!session || typeof session !== "object") return false;
|
||||
if (query.sessionId && session.sessionId === query.sessionId) return true;
|
||||
if (query.threadId && session.threadId === query.threadId) return true;
|
||||
if (query.conversationId && session.conversationId === query.conversationId) return true;
|
||||
if (query.traceId && (session.currentTraceId === query.traceId || session.lastTraceId === query.traceId)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function handleCodeAgentCancelHttp(request, response, options) {
|
||||
const body = await readBody(request, options.bodyLimitBytes);
|
||||
let params = {};
|
||||
|
||||
try {
|
||||
params = body ? JSON.parse(body) : {};
|
||||
} catch (error) {
|
||||
sendJson(response, 400, {
|
||||
...createCodeAgentErrorPayload({
|
||||
code: "parse_error",
|
||||
message: "Invalid JSON body",
|
||||
reason: error.message,
|
||||
traceId: getHeader(request, "x-trace-id") || "trc_unassigned",
|
||||
layer: "api",
|
||||
retryable: true
|
||||
})
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (!params || typeof params !== "object" || Array.isArray(params)) {
|
||||
sendJson(response, 400, {
|
||||
...createCodeAgentErrorPayload({
|
||||
code: "invalid_params",
|
||||
message: "Code Agent cancel body must be a JSON object",
|
||||
traceId: getHeader(request, "x-trace-id") || "trc_unassigned",
|
||||
layer: "api",
|
||||
retryable: true
|
||||
})
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const traceId = safeTraceId(getHeader(request, "x-trace-id") || params.traceId);
|
||||
const traceStore = options.traceStore ?? defaultCodeAgentTraceStore;
|
||||
const snapshot = traceId ? traceStore.snapshot(traceId) : null;
|
||||
const durableResult = traceId ? await loadPersistedAgentRunResult(traceId, options) : null;
|
||||
const currentResult = durableResult ?? (traceId ? options.codeAgentChatResults?.get(traceId) : null);
|
||||
if (currentResult?.agentRun?.commandId) {
|
||||
const mismatch = codeAgentCancelScopeMismatch(params, currentResult);
|
||||
if (mismatch) {
|
||||
traceStore.append(traceId, {
|
||||
type: "cancel",
|
||||
status: "blocked",
|
||||
label: "agentrun:cancel:scope_mismatch",
|
||||
errorCode: "cancel_scope_mismatch",
|
||||
message: `Cancel request ${mismatch.field} does not match the trace owner; AgentRun cancel was not forwarded.`,
|
||||
requested: mismatch.requested,
|
||||
expected: mismatch.expected,
|
||||
waitingFor: "cancel-scope",
|
||||
valuesPrinted: false
|
||||
});
|
||||
sendJson(response, 409, cancelBlockedPayload({
|
||||
code: "cancel_scope_mismatch",
|
||||
message: `取消请求的 ${mismatch.field} 与 trace 归属不一致,已拒绝转发 AgentRun cancel,避免误取消其他 session。`,
|
||||
traceId,
|
||||
conversationId: safeConversationId(params.conversationId) || safeConversationId(currentResult.conversationId) || null,
|
||||
sessionId: safeSessionId(params.sessionId) || safeSessionId(currentResult.sessionId) || null,
|
||||
runnerTrace: traceStore.snapshot(traceId),
|
||||
status: "blocked"
|
||||
}));
|
||||
return;
|
||||
}
|
||||
await recordCodeAgentSessionInputFact({
|
||||
params: {
|
||||
...params,
|
||||
sessionId: currentResult.sessionId ?? currentResult.session?.sessionId ?? params.sessionId,
|
||||
messageId: codeAgentTurnLifecycleFields(traceId, currentResult).userMessageId,
|
||||
ownerUserId: options.actor?.id,
|
||||
ownerRole: options.actor?.role
|
||||
},
|
||||
options,
|
||||
traceId,
|
||||
delivery: "cancel",
|
||||
status: "promoted",
|
||||
commandId: currentResult.agentRun.commandId
|
||||
});
|
||||
const payload = await cancelAgentRunChatTurn({ traceId, currentResult, options, traceStore });
|
||||
if (payload) {
|
||||
sendJson(response, 200, payload);
|
||||
return;
|
||||
}
|
||||
}
|
||||
const sessionId = safeSessionId(params.sessionId) || safeSessionId(snapshot?.sessionId);
|
||||
const conversationId = safeConversationId(params.conversationId);
|
||||
const manager = options.codexStdioManager;
|
||||
|
||||
if (!traceId) {
|
||||
sendJson(response, 400, cancelBlockedPayload({
|
||||
code: "cancel_trace_missing",
|
||||
message: "traceId is required to cancel the current Code Agent request.",
|
||||
traceId: "trc_unassigned",
|
||||
conversationId,
|
||||
sessionId
|
||||
}));
|
||||
return;
|
||||
}
|
||||
|
||||
if (!manager || typeof manager.get !== "function" || typeof manager.cancel !== "function") {
|
||||
traceStore.append(traceId, {
|
||||
type: "cancel",
|
||||
status: "unsupported",
|
||||
label: "cancel:unsupported",
|
||||
errorCode: "cancel_unsupported",
|
||||
message: "Codex stdio cancel/interrupt backend is not available on this runtime.",
|
||||
waitingFor: "session-control"
|
||||
});
|
||||
const runnerTrace = traceStore.snapshot(traceId);
|
||||
sendJson(response, 501, cancelBlockedPayload({
|
||||
code: "cancel_unsupported",
|
||||
message: "当前后端不支持 Code Agent interrupt/cancel;本次按 unsupported/degraded 返回,不会静默假装已中断。",
|
||||
traceId,
|
||||
conversationId,
|
||||
sessionId,
|
||||
runnerTrace,
|
||||
status: "degraded",
|
||||
unsupported: true,
|
||||
degraded: true
|
||||
}));
|
||||
return;
|
||||
}
|
||||
|
||||
if (!sessionId) {
|
||||
traceStore.append(traceId, {
|
||||
type: "cancel",
|
||||
status: "blocked",
|
||||
label: "cancel:not_cancelable",
|
||||
errorCode: "cancel_session_missing",
|
||||
message: "Cancel request did not include a bound Codex stdio sessionId.",
|
||||
waitingFor: "session-binding"
|
||||
});
|
||||
sendJson(response, 409, cancelBlockedPayload({
|
||||
code: "cancel_session_missing",
|
||||
message: "当前请求尚未暴露可取消的 Codex stdio sessionId;页面不能只隐藏 UI,已保留输入和 trace。",
|
||||
traceId,
|
||||
conversationId,
|
||||
sessionId: null,
|
||||
runnerTrace: traceStore.snapshot(traceId)
|
||||
}));
|
||||
return;
|
||||
}
|
||||
|
||||
const currentSession = manager.get(sessionId, { conversationId }) ?? null;
|
||||
if (!currentSession || !["busy", "creating"].includes(currentSession.status)) {
|
||||
traceStore.append(traceId, {
|
||||
type: "cancel",
|
||||
status: "blocked",
|
||||
label: "cancel:not_in_flight",
|
||||
errorCode: "cancel_not_in_flight",
|
||||
message: `Session ${sessionId} is not an in-flight Codex stdio request.`,
|
||||
sessionId,
|
||||
sessionStatus: currentSession?.status ?? "missing"
|
||||
});
|
||||
sendJson(response, 409, cancelBlockedPayload({
|
||||
code: currentSession ? "cancel_not_in_flight" : "cancel_session_not_found",
|
||||
message: currentSession
|
||||
? `当前 session 状态为 ${currentSession.status},没有可取消的 in-flight Codex stdio 请求。`
|
||||
: `没有找到 sessionId=${sessionId} 的 Codex stdio session。`,
|
||||
traceId,
|
||||
conversationId,
|
||||
sessionId,
|
||||
session: currentSession,
|
||||
runnerTrace: traceStore.snapshot(traceId)
|
||||
}));
|
||||
return;
|
||||
}
|
||||
|
||||
const canceledSession = manager.cancel(sessionId, {
|
||||
traceId,
|
||||
conversationId,
|
||||
reason: "user_cancel"
|
||||
});
|
||||
traceStore.append(traceId, {
|
||||
type: "cancel",
|
||||
status: "canceled",
|
||||
label: "cancel:canceled",
|
||||
message: "User canceled the current Codex stdio request.",
|
||||
sessionId,
|
||||
sessionStatus: canceledSession?.status ?? "canceled",
|
||||
sessionLifecycleStatus: codeAgentSessionLifecycleSummary({ session: canceledSession, status: "canceled" }).status,
|
||||
waitingFor: "user-retry",
|
||||
terminal: true
|
||||
});
|
||||
const runnerTraceSnapshot = traceStore.snapshot(traceId);
|
||||
const sessionSummary = codeAgentSessionLifecycleSummary({
|
||||
session: canceledSession,
|
||||
runnerTrace: runnerTraceSnapshot,
|
||||
status: "canceled"
|
||||
});
|
||||
const runnerTrace = {
|
||||
...runnerTraceSnapshot,
|
||||
sessionLifecycleStatus: sessionSummary.status
|
||||
};
|
||||
const payload = {
|
||||
accepted: true,
|
||||
canceled: true,
|
||||
status: "canceled",
|
||||
conversationId: conversationId ?? canceledSession?.conversationId ?? null,
|
||||
sessionId,
|
||||
traceId,
|
||||
session: canceledSession,
|
||||
sessionLifecycleStatus: sessionSummary.status,
|
||||
sessionLifecycle: sessionSummary,
|
||||
sessionSummary,
|
||||
runnerTrace,
|
||||
lastTraceEvent: runnerTrace.lastEvent,
|
||||
retryable: true,
|
||||
error: {
|
||||
code: "codex_stdio_canceled",
|
||||
layer: "session",
|
||||
category: "canceled",
|
||||
retryable: true,
|
||||
message: "user canceled current Code Agent request",
|
||||
userMessage: "当前 Codex stdio 请求已取消;输入、sessionId、traceId 和最后 trace event 已保留,可重试上一条消息。",
|
||||
traceId,
|
||||
route: "/v1/agent/chat/cancel",
|
||||
toolName: "codex-stdio.cancel"
|
||||
},
|
||||
userMessage: "当前 Codex stdio 请求已取消;输入、sessionId、traceId 和最后 trace event 已保留,可重试上一条消息。",
|
||||
updatedAt: new Date().toISOString()
|
||||
};
|
||||
await recordCodeAgentSessionOwner({ payload, params: { ...params, traceId, ownerUserId: options.actor?.id, ownerRole: options.actor?.role }, options, status: "canceled" });
|
||||
recordCodeAgentConversationFact(payload, options);
|
||||
options.codeAgentChatResults?.set(traceId, annotateOwner(payload, { ownerUserId: options.actor?.id, ownerRole: options.actor?.role }));
|
||||
sendJson(response, 200, payload);
|
||||
}
|
||||
|
||||
export { codeAgentCompatProjectionPayload, codeAgentRequestScopedOptions, measureCodeAgentHttpPhase, readCodeAgentCompatProjection, traceProjectMismatchSnapshot };
|
||||
@@ -348,7 +348,7 @@ test("cloud api health separates DB connected from durable runtime schema readin
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const response = await fetch(`http://127.0.0.1:${port}/health/live`);
|
||||
const response = await fetch(`http://127.0.0.1:${port}/health`);
|
||||
const payload = await response.json();
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(payload.status, "degraded");
|
||||
@@ -449,7 +449,7 @@ test("cloud api health keeps DB live evidence separate when durable adapter quer
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const response = await fetch(`http://127.0.0.1:${port}/health/live`);
|
||||
const response = await fetch(`http://127.0.0.1:${port}/health`);
|
||||
const payload = await response.json();
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(payload.status, "degraded");
|
||||
@@ -593,7 +593,7 @@ test("cloud api health contract distinguishes durable SSL, auth, schema, migrati
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const response = await fetch(`http://127.0.0.1:${port}/health/live`);
|
||||
const response = await fetch(`http://127.0.0.1:${port}/health`);
|
||||
const payload = await response.json();
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(payload.status, "degraded");
|
||||
@@ -653,7 +653,7 @@ test("cloud api health does not treat DB env presence-only as live readiness", a
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const response = await fetch(`http://127.0.0.1:${port}/health/live`);
|
||||
const response = await fetch(`http://127.0.0.1:${port}/health`);
|
||||
const payload = await response.json();
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(payload.db.configReady, true);
|
||||
@@ -782,7 +782,7 @@ test("cloud api health reports provider, durable DB, and codex stdio ready when
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const response = await fetch(`http://127.0.0.1:${port}/health/live`);
|
||||
const response = await fetch(`http://127.0.0.1:${port}/health`);
|
||||
assert.equal(response.status, 200);
|
||||
const payload = await response.json();
|
||||
assert.equal(payload.status, "ok");
|
||||
@@ -868,7 +868,7 @@ test("cloud api health reports AgentRun adapter readiness without repo-owned cod
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const response = await fetch(`http://127.0.0.1:${port}/health/live`);
|
||||
const response = await fetch(`http://127.0.0.1:${port}/health`);
|
||||
assert.equal(response.status, 200);
|
||||
const payload = await response.json();
|
||||
assert.equal(payload.status, "ok");
|
||||
|
||||
@@ -0,0 +1,528 @@
|
||||
/*
|
||||
* Responsibility: Cloud API HTTP handlers for HWLAB node updates, HWPOD discovery, and node-ops dispatch.
|
||||
*/
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
import { ENVIRONMENT_DEV } from "../protocol/index.mjs";
|
||||
import { CLOUD_API_SERVICE_ID } from "../audit/index.mjs";
|
||||
import {
|
||||
discoverHwpodSpecs,
|
||||
hwpodSpecDiscoveryPayload,
|
||||
hwpodSpecWorkspaceProbePlan
|
||||
} from "./hwpod-spec-discovery.ts";
|
||||
import {
|
||||
getHeader,
|
||||
parsePositiveInteger,
|
||||
readBody,
|
||||
safeOpaqueId,
|
||||
sendJson,
|
||||
truthyFlag
|
||||
} from "./server-http-utils.ts";
|
||||
import { HWPOD_NODE_OPS, HWPOD_NODE_OPS_CONTRACT_VERSION } from "../../tools/src/hwpod-node-ops-contract.ts";
|
||||
|
||||
const DEFAULT_BODY_LIMIT_BYTES = 1024 * 1024;
|
||||
|
||||
export async function handleHwpodNodeOpsHttp(request, response, options) {
|
||||
if (request.method === "GET") {
|
||||
sendJson(response, 200, {
|
||||
ok: true,
|
||||
status: "ready",
|
||||
contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION,
|
||||
route: "/v1/hwpod-node-ops",
|
||||
specAuthority: "code-agent-workspace",
|
||||
compiler: "hwpod-compiler-cli",
|
||||
apiRole: "node-ops-forwarder",
|
||||
nodeRole: "thin-hwpod-node-executor",
|
||||
supportedOps: Array.from(HWPOD_NODE_OPS),
|
||||
websocket: options.hwpodNodeWsRegistry.describe()
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (request.method !== "POST") {
|
||||
sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "hwpod-node-ops only supports GET and POST" } });
|
||||
return;
|
||||
}
|
||||
|
||||
const body = await readJsonObject(request, options.bodyLimitBytes);
|
||||
if (!body.ok) {
|
||||
sendJson(response, 400, body.error);
|
||||
return;
|
||||
}
|
||||
const validation = validateHwpodNodeOpsPlan(body.value);
|
||||
if (!validation.ok) {
|
||||
sendJson(response, 400, {
|
||||
ok: false,
|
||||
status: "rejected",
|
||||
contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION,
|
||||
error: validation.error
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const plan = validation.plan;
|
||||
const requestMeta = hwpodNodeOpsRequestMeta(request, options, "hwpod");
|
||||
try {
|
||||
const handled = await dispatchHwpodNodeOpsPlan(plan, requestMeta, options, { request });
|
||||
sendJson(response, handled.httpStatus, handled.payload);
|
||||
} catch (error) {
|
||||
const summary = error?.message ?? "hwpod-node-ops handler failed";
|
||||
recordHwpodNodeOpsBlocked(request, options, plan, requestMeta, "hwpod_node_handler_failed", summary, { dispatchMode: "handler-exception" });
|
||||
sendJson(response, 200, hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, { dispatchMode: "handler-exception" }));
|
||||
}
|
||||
}
|
||||
|
||||
export function handleHwlabNodeUpdateHttp(request, response, url, options) {
|
||||
if (request.method !== "GET") {
|
||||
sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "GET required." } });
|
||||
return;
|
||||
}
|
||||
const env = options.env ?? process.env;
|
||||
const currentVersion = cleanText(url.searchParams.get("current") || url.searchParams.get("version"));
|
||||
const channel = cleanText(url.searchParams.get("channel")) || "stable";
|
||||
const platform = cleanText(url.searchParams.get("platform")) || "unknown";
|
||||
const bundled = hwlabNodeBundledMetadata(env);
|
||||
const latestVersion = cleanText(env.HWLAB_NODE_PY_LATEST_VERSION) || bundled.version || "0.1.0";
|
||||
const releaseNotesUrl = cleanText(env.HWLAB_NODE_PY_RELEASE_NOTES_URL) || null;
|
||||
const downloadUrl = hwlabNodeDownloadUrl(env);
|
||||
const sha256 = cleanText(env.HWLAB_NODE_PY_SHA256) || bundled.sha256 || null;
|
||||
const newer = currentVersion ? compareVersionStrings(latestVersion, currentVersion) > 0 : Boolean(latestVersion);
|
||||
const updateAvailable = Boolean(downloadUrl && newer);
|
||||
sendJson(response, 200, {
|
||||
ok: true,
|
||||
contractVersion: "hwlab-node-update-v1",
|
||||
serviceId: "hwlab-node",
|
||||
route: "/v1/hwlab-node/update",
|
||||
channel,
|
||||
platform,
|
||||
currentVersion: currentVersion || null,
|
||||
latestVersion,
|
||||
updateAvailable,
|
||||
downloadUrl: updateAvailable ? downloadUrl : null,
|
||||
sha256: updateAvailable ? sha256 : null,
|
||||
releaseNotesUrl,
|
||||
manualDefault: true,
|
||||
autoApplyDefault: false,
|
||||
checkIntervalSeconds: parsePositiveInteger(env.HWLAB_NODE_PY_UPDATE_INTERVAL_SECONDS, 300),
|
||||
observedAt: new Date().toISOString()
|
||||
});
|
||||
}
|
||||
|
||||
export function handleHwlabNodeDownloadHttp(request, response, options) {
|
||||
if (request.method !== "GET") {
|
||||
sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "GET required." } });
|
||||
return;
|
||||
}
|
||||
const bundled = hwlabNodeBundledMetadata(options.env ?? process.env);
|
||||
if (!bundled.content) {
|
||||
sendJson(response, 404, { ok: false, error: { code: "hwlab_node_bundle_missing", message: "bundled hwlab-node.py is not available" } });
|
||||
return;
|
||||
}
|
||||
const bytes = Buffer.from(bundled.content, "utf8");
|
||||
response.writeHead(200, {
|
||||
"content-type": "text/x-python; charset=utf-8",
|
||||
"cache-control": "no-store",
|
||||
"x-hwlab-node-version": bundled.version || "unknown",
|
||||
"x-hwlab-node-sha256": bundled.sha256 || "",
|
||||
"content-length": String(bytes.length)
|
||||
});
|
||||
response.end(bytes);
|
||||
}
|
||||
|
||||
export async function handleHwpodSpecDiscoveryHttp(request, response, url, options) {
|
||||
const probe = truthyFlag(url.searchParams.get("probe"));
|
||||
const observedAt = new Date().toISOString();
|
||||
let specs = await discoverHwpodSpecs(options);
|
||||
if (probe) {
|
||||
specs = await Promise.all(specs.map((spec) => probeDiscoveredHwpodSpec(spec, { request, options, observedAt })));
|
||||
}
|
||||
sendJson(response, 200, hwpodSpecDiscoveryPayload(specs, { observedAt }));
|
||||
}
|
||||
|
||||
function runtimeEnvironment(env = process.env) {
|
||||
const value = env.HWLAB_GITOPS_PROFILE || env.HWLAB_ENVIRONMENT || ENVIRONMENT_DEV;
|
||||
return typeof value === "string" && value.trim() ? value.trim() : ENVIRONMENT_DEV;
|
||||
}
|
||||
|
||||
function hwpodNodeOpsRequestMeta(request, options, label) {
|
||||
const httpContext = request?.hwlabHttpRequestContext ?? {};
|
||||
return {
|
||||
requestId: getHeader(request, "x-request-id") || httpContext.requestId || `req_${label}_${randomUUID()}`,
|
||||
traceId: getHeader(request, "x-trace-id") || `trc_${label}_${randomUUID()}`,
|
||||
serviceId: getHeader(request, "x-source-service-id") || CLOUD_API_SERVICE_ID,
|
||||
environment: runtimeEnvironment(options.env ?? process.env),
|
||||
otelTraceId: httpContext.traceId ?? null,
|
||||
traceparent: httpContext.traceparent ?? null,
|
||||
valuesPrinted: false
|
||||
};
|
||||
}
|
||||
|
||||
function hwlabNodeBundledMetadata(env) {
|
||||
const bundlePath = cleanText(env.HWLAB_NODE_PY_BUNDLE_PATH) || path.resolve(process.cwd(), "tools/hwlab-node.py");
|
||||
try {
|
||||
const content = readFileSync(bundlePath, "utf8");
|
||||
const version = cleanText(content.match(/APP_VERSION\s*=\s*["']([^"']+)["']/u)?.[1]);
|
||||
const sha256 = createHash("sha256").update(content, "utf8").digest("hex");
|
||||
return { path: bundlePath, content, version, sha256 };
|
||||
} catch {
|
||||
return { path: bundlePath, content: "", version: "", sha256: "" };
|
||||
}
|
||||
}
|
||||
|
||||
function hwlabNodeDownloadUrl(env) {
|
||||
const explicit = cleanText(env.HWLAB_NODE_PY_DOWNLOAD_URL);
|
||||
if (explicit) return explicit;
|
||||
const downloadPath = cleanText(env.HWLAB_NODE_PY_DOWNLOAD_PATH) || "/v1/hwlab-node/download/hwlab-node.py";
|
||||
if (/^https?:\/\//iu.test(downloadPath)) return downloadPath;
|
||||
const publicEndpoint = cleanText(env.HWLAB_PUBLIC_ENDPOINT) || "https://hwlab.pikapython.com";
|
||||
try {
|
||||
return new URL(downloadPath, publicEndpoint.endsWith("/") ? publicEndpoint : `${publicEndpoint}/`).toString();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function compareVersionStrings(left, right) {
|
||||
const a = versionParts(left);
|
||||
const b = versionParts(right);
|
||||
for (let index = 0; index < Math.max(a.length, b.length, 3); index += 1) {
|
||||
const delta = (a[index] ?? 0) - (b[index] ?? 0);
|
||||
if (delta !== 0) return delta > 0 ? 1 : -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function versionParts(value) {
|
||||
return String(value ?? "")
|
||||
.trim()
|
||||
.replace(/^v/iu, "")
|
||||
.split(/[.+-]/u)
|
||||
.slice(0, 4)
|
||||
.map((part) => parseInt(part, 10))
|
||||
.map((part) => (Number.isFinite(part) && part >= 0 ? part : 0));
|
||||
}
|
||||
|
||||
function cleanText(value) {
|
||||
const text = String(value ?? "").trim();
|
||||
return text || "";
|
||||
}
|
||||
|
||||
async function probeDiscoveredHwpodSpec(spec, { request, options, observedAt }) {
|
||||
if (spec.ok === false) return spec;
|
||||
const plan = hwpodSpecWorkspaceProbePlan(spec);
|
||||
const validation = validateHwpodNodeOpsPlan(plan);
|
||||
if (!validation.ok) {
|
||||
return {
|
||||
...spec,
|
||||
availability: {
|
||||
ok: false,
|
||||
status: "invalid_probe_plan",
|
||||
checkedAt: observedAt,
|
||||
blocker: validation.error
|
||||
}
|
||||
};
|
||||
}
|
||||
const requestMeta = hwpodNodeOpsRequestMeta(request, options, "hwpod_spec");
|
||||
const handled = await dispatchHwpodNodeOpsPlan(validation.plan, requestMeta, options, { request });
|
||||
const payload = handled.payload;
|
||||
return {
|
||||
...spec,
|
||||
availability: {
|
||||
ok: payload.ok === true,
|
||||
status: payload.ok === true ? "available" : "blocked",
|
||||
checkedAt: observedAt,
|
||||
probePlanId: payload.planId,
|
||||
nodeOpsRoute: "/v1/hwpod-node-ops",
|
||||
results: payload.results ?? [],
|
||||
blocker: payload.blocker ?? null,
|
||||
httpStatus: handled.httpStatus
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function dispatchHwpodNodeOpsPlan(plan, requestMeta, options, context = {}) {
|
||||
const hwpodNodeOpsUrl = normalizedHwpodNodeOpsUrl(options.env ?? process.env);
|
||||
const hwpodNodeWsRegistry = options.hwpodNodeWsRegistry;
|
||||
const hasWsNode = hwpodNodeWsRegistry?.hasNode?.(plan.nodeId);
|
||||
if (typeof options.hwpodNodeOpsHandler !== "function" && !hasWsNode && !hwpodNodeOpsUrl) {
|
||||
const summary = "no outbound WebSocket hwpod-node is connected and HWLAB_HWPOD_NODE_OPS_URL is not configured; cloud-api is only validating the hwpod-node-ops contract";
|
||||
const details = {
|
||||
dispatchMode: "none",
|
||||
websocketRegistryConfigured: Boolean(hwpodNodeWsRegistry),
|
||||
websocketConnected: false,
|
||||
directUrlConfigured: false
|
||||
};
|
||||
recordHwpodNodeOpsBlocked(context.request, options, plan, requestMeta, "hwpod_node_unavailable", summary, details);
|
||||
return {
|
||||
httpStatus: 200,
|
||||
payload: hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, details)
|
||||
};
|
||||
}
|
||||
const handled = typeof options.hwpodNodeOpsHandler === "function"
|
||||
? await options.hwpodNodeOpsHandler(plan, { request: context.request, requestMeta, env: options.env ?? process.env })
|
||||
: hasWsNode
|
||||
? await hwpodNodeWsRegistry.dispatch(plan, requestMeta, { timeoutMs: parsePositiveInteger(options.env?.HWLAB_HWPOD_NODE_OPS_TIMEOUT_MS, 30000) })
|
||||
: await forwardHwpodNodeOpsPlan(hwpodNodeOpsUrl, plan, requestMeta, options);
|
||||
const results = Array.isArray(handled?.results) ? handled.results : [];
|
||||
const failed = results.some((item) => item?.ok === false);
|
||||
const payload = attachHwpodNodeOpsDiagnostics({
|
||||
ok: handled?.ok ?? !failed,
|
||||
status: handled?.status ?? (failed ? "failed" : "completed"),
|
||||
contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION,
|
||||
planId: plan.planId,
|
||||
hwpodId: plan.hwpodId,
|
||||
nodeId: plan.nodeId,
|
||||
acceptedOps: plan.ops.length,
|
||||
results,
|
||||
blocker: handled?.blocker ?? null,
|
||||
requestMeta
|
||||
}, requestMeta, { dispatchMode: hasWsNode ? "websocket" : hwpodNodeOpsUrl ? "direct-url" : "handler" });
|
||||
if (payload.ok === false && payload.status === "blocked" && payload.blocker?.code) {
|
||||
recordHwpodNodeOpsBlocked(context.request, options, plan, requestMeta, payload.blocker.code, payload.blocker.summary ?? "hwpod-node-ops blocked", payload.blocker.details ?? {});
|
||||
}
|
||||
return { httpStatus: handled?.httpStatus ?? (failed ? 409 : 200), payload };
|
||||
}
|
||||
|
||||
async function forwardHwpodNodeOpsPlan(targetUrl, plan, requestMeta, options) {
|
||||
const target = await describeDirectHwpodNode(targetUrl, options);
|
||||
if (!target.ok || !target.nodeId) {
|
||||
return directHwpodNodeBlocked(plan, "hwpod_node_identity_unverified", "configured hwpod-node URL did not expose a verifiable nodeId", {
|
||||
requestedNodeId: plan.nodeId,
|
||||
targetUrl: redactNodeOpsUrl(targetUrl),
|
||||
dispatchMode: "direct-url",
|
||||
targetStatus: target.status ?? null,
|
||||
error: target.error ?? null
|
||||
}, requestMeta);
|
||||
}
|
||||
if (target.ok && target.nodeId && target.nodeId !== plan.nodeId) {
|
||||
return directHwpodNodeBlocked(plan, "hwpod_node_id_mismatch", `HWLAB_HWPOD_NODE_OPS_URL points to ${target.nodeId}, not requested node ${plan.nodeId}`, {
|
||||
requestedNodeId: plan.nodeId,
|
||||
targetNodeId: target.nodeId,
|
||||
targetUrl: redactNodeOpsUrl(targetUrl),
|
||||
dispatchMode: "direct-url"
|
||||
}, requestMeta);
|
||||
}
|
||||
const response = await fetch(targetUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"x-request-id": requestMeta.requestId,
|
||||
"x-trace-id": requestMeta.traceId,
|
||||
"x-source-service-id": CLOUD_API_SERVICE_ID,
|
||||
...(requestMeta.otelTraceId ? { "x-hwlab-otel-trace-id": requestMeta.otelTraceId } : {}),
|
||||
...(requestMeta.traceparent ? { traceparent: requestMeta.traceparent } : {})
|
||||
},
|
||||
body: JSON.stringify(plan),
|
||||
signal: AbortSignal.timeout(parsePositiveInteger(options.env?.HWLAB_HWPOD_NODE_OPS_TIMEOUT_MS, 30000))
|
||||
});
|
||||
const body = await response.json().catch(() => null);
|
||||
if (!body || typeof body !== "object") {
|
||||
return directHwpodNodeBlocked(plan, "hwpod_node_response_invalid", `hwpod-node returned HTTP ${response.status} without a JSON object payload`, {
|
||||
targetUrl: redactNodeOpsUrl(targetUrl),
|
||||
dispatchMode: "direct-url",
|
||||
targetStatus: response.status
|
||||
}, requestMeta);
|
||||
}
|
||||
return {
|
||||
ok: body.ok,
|
||||
status: body.status,
|
||||
httpStatus: response.status,
|
||||
results: body.results,
|
||||
blocker: body.blocker ?? null
|
||||
};
|
||||
}
|
||||
|
||||
function directHwpodNodeBlocked(plan, code, summary, details, requestMeta = {}) {
|
||||
const blocker = hwpodNodeOpsBlocker({ code, layer: "hwpod-node", retryable: true, summary, details }, requestMeta, details);
|
||||
return {
|
||||
ok: false,
|
||||
status: "blocked",
|
||||
httpStatus: 200,
|
||||
results: plan.ops.map((op) => ({ opId: op.opId, op: op.op, ok: false, status: "blocked", blocker })),
|
||||
blocker,
|
||||
otelTraceId: blocker.otelTraceId ?? null,
|
||||
diagnostic: blocker.diagnostic ?? null
|
||||
};
|
||||
}
|
||||
|
||||
async function describeDirectHwpodNode(targetUrl, options) {
|
||||
try {
|
||||
const response = await fetch(targetUrl, {
|
||||
method: "GET",
|
||||
signal: AbortSignal.timeout(parsePositiveInteger(options.env?.HWLAB_HWPOD_NODE_OPS_TIMEOUT_MS, 30000))
|
||||
});
|
||||
const body = await response.json().catch(() => null);
|
||||
return { ok: response.ok && body && typeof body === "object", status: response.status, nodeId: safeOpaqueId(body?.nodeId), body };
|
||||
} catch (error) {
|
||||
return { ok: false, error: error instanceof Error ? error.message : String(error), nodeId: "" };
|
||||
}
|
||||
}
|
||||
|
||||
function redactNodeOpsUrl(value) {
|
||||
try {
|
||||
const parsed = new URL(value);
|
||||
parsed.username = "";
|
||||
parsed.password = "";
|
||||
parsed.search = parsed.search ? "?redacted=1" : "";
|
||||
return parsed.toString();
|
||||
} catch {
|
||||
return "<invalid-url>";
|
||||
}
|
||||
}
|
||||
|
||||
function normalizedHwpodNodeOpsUrl(env = process.env) {
|
||||
const direct = String(env.HWLAB_HWPOD_NODE_OPS_URL ?? "").trim();
|
||||
if (direct) return direct;
|
||||
const base = String(env.HWLAB_HWPOD_NODE_URL ?? "").trim().replace(/\/+$/u, "");
|
||||
return base ? `${base}/v1/hwpod-node-ops` : "";
|
||||
}
|
||||
|
||||
function validateHwpodNodeOpsPlan(value) {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return { ok: false, error: { code: "invalid_hwpod_node_ops_plan", message: "hwpod-node-ops body must be a JSON object" } };
|
||||
}
|
||||
if (value.contractVersion !== HWPOD_NODE_OPS_CONTRACT_VERSION) {
|
||||
return { ok: false, error: { code: "invalid_hwpod_node_ops_contract", message: `contractVersion must be ${HWPOD_NODE_OPS_CONTRACT_VERSION}`, actual: value.contractVersion ?? null } };
|
||||
}
|
||||
const planId = safeOpaqueId(value.planId) || `hwpod_plan_${randomUUID()}`;
|
||||
const nodeId = safeOpaqueId(value.nodeId);
|
||||
const hwpodId = safeOpaqueId(value.hwpodId);
|
||||
if (!nodeId) return { ok: false, error: { code: "invalid_hwpod_node_id", message: "nodeId is required" } };
|
||||
if (!hwpodId) return { ok: false, error: { code: "invalid_hwpod_id", message: "hwpodId is required" } };
|
||||
if (!Array.isArray(value.ops) || value.ops.length === 0) {
|
||||
return { ok: false, error: { code: "invalid_hwpod_node_ops", message: "ops must be a non-empty array" } };
|
||||
}
|
||||
const ops = [];
|
||||
for (const [index, item] of value.ops.entries()) {
|
||||
if (!item || typeof item !== "object" || Array.isArray(item)) {
|
||||
return { ok: false, error: { code: "invalid_hwpod_node_op", message: `ops[${index}] must be a JSON object` } };
|
||||
}
|
||||
const op = typeof item.op === "string" ? item.op.trim() : "";
|
||||
if (!HWPOD_NODE_OPS.has(op)) {
|
||||
return { ok: false, error: { code: "unsupported_hwpod_node_op", message: `unsupported hwpod-node op: ${op || "<empty>"}`, supportedOps: Array.from(HWPOD_NODE_OPS) } };
|
||||
}
|
||||
ops.push({
|
||||
opId: safeOpaqueId(item.opId) || `op_${index + 1}`,
|
||||
op,
|
||||
args: item.args && typeof item.args === "object" && !Array.isArray(item.args) ? item.args : {}
|
||||
});
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
plan: {
|
||||
...value,
|
||||
planId,
|
||||
hwpodId,
|
||||
nodeId,
|
||||
ops
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, details = {}) {
|
||||
const blocker = hwpodNodeOpsBlocker({
|
||||
code: "hwpod_node_unavailable",
|
||||
layer: "hwpod-node",
|
||||
retryable: true,
|
||||
summary,
|
||||
details,
|
||||
userMessage: "hwpod-node 尚未接入执行面;cloud-api 已完成 hwpod-node-ops 合同校验。"
|
||||
}, requestMeta, details);
|
||||
return {
|
||||
ok: false,
|
||||
status: "blocked",
|
||||
contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION,
|
||||
planId: plan.planId,
|
||||
hwpodId: plan.hwpodId,
|
||||
nodeId: plan.nodeId,
|
||||
acceptedOps: plan.ops.length,
|
||||
results: plan.ops.map((op) => ({
|
||||
opId: op.opId,
|
||||
op: op.op,
|
||||
ok: false,
|
||||
status: "blocked",
|
||||
blocker
|
||||
})),
|
||||
blocker,
|
||||
otelTraceId: blocker.otelTraceId ?? null,
|
||||
diagnostic: blocker.diagnostic ?? null,
|
||||
requestMeta
|
||||
};
|
||||
}
|
||||
|
||||
function hwpodNodeOpsBlocker(blocker, requestMeta = {}, details = {}) {
|
||||
const code = cleanText(blocker?.code) || "hwpod_node_ops_blocked";
|
||||
const summary = cleanText(blocker?.summary ?? blocker?.message) || "hwpod-node-ops blocked";
|
||||
const otelTraceId = cleanText(blocker?.otelTraceId ?? requestMeta?.otelTraceId) || null;
|
||||
const diagnostic = {
|
||||
code,
|
||||
rootCauseCode: code,
|
||||
summary,
|
||||
otelTraceId,
|
||||
traceLine: otelTraceId ? `OTel traceId: ${otelTraceId}` : null,
|
||||
details: details && typeof details === "object" ? details : {},
|
||||
valuesPrinted: false
|
||||
};
|
||||
const userMessage = cleanText(blocker?.userMessage);
|
||||
return {
|
||||
...blocker,
|
||||
code,
|
||||
layer: blocker?.layer ?? "hwpod-node",
|
||||
retryable: blocker?.retryable ?? true,
|
||||
summary,
|
||||
...(otelTraceId ? { otelTraceId } : {}),
|
||||
diagnostic,
|
||||
...(userMessage ? { userMessage: `${userMessage}\n${diagnostic.traceLine ?? "OTel traceId: unavailable"}` } : {})
|
||||
};
|
||||
}
|
||||
|
||||
function attachHwpodNodeOpsDiagnostics(payload, requestMeta, details = {}) {
|
||||
if (!payload || typeof payload !== "object" || payload.ok !== false) return payload;
|
||||
const topBlocker = payload.blocker ? hwpodNodeOpsBlocker(payload.blocker, requestMeta, payload.blocker.details ?? details) : null;
|
||||
const results = Array.isArray(payload.results)
|
||||
? payload.results.map((result) => result?.blocker ? { ...result, blocker: hwpodNodeOpsBlocker(result.blocker, requestMeta, result.blocker.details ?? details) } : result)
|
||||
: payload.results;
|
||||
const diagnostic = topBlocker?.diagnostic ?? results?.find?.((result) => result?.blocker?.diagnostic)?.blocker?.diagnostic ?? null;
|
||||
return {
|
||||
...payload,
|
||||
results,
|
||||
blocker: topBlocker,
|
||||
...(diagnostic?.otelTraceId ? { otelTraceId: diagnostic.otelTraceId } : {}),
|
||||
...(diagnostic ? { diagnostic } : {})
|
||||
};
|
||||
}
|
||||
|
||||
function recordHwpodNodeOpsBlocked(request, options, plan, requestMeta, code, summary, details = {}) {
|
||||
const httpContext = request?.hwlabHttpRequestContext;
|
||||
const attributes = {
|
||||
"hwlab.hwpod.plan_id": cleanText(plan?.planId) || null,
|
||||
"hwlab.hwpod.hwpod_id": cleanText(plan?.hwpodId) || null,
|
||||
"hwlab.hwpod.node_id": cleanText(plan?.nodeId) || null,
|
||||
"hwlab.hwpod.accepted_ops": Array.isArray(plan?.ops) ? plan.ops.length : 0,
|
||||
"hwlab.hwpod.ops": Array.isArray(plan?.ops) ? plan.ops.map((op) => cleanText(op?.op)).filter(Boolean).join(",").slice(0, 240) : null,
|
||||
"hwlab.hwpod.blocker.code": code,
|
||||
"hwlab.hwpod.blocker.summary": String(summary ?? "").slice(0, 500),
|
||||
"hwlab.hwpod.dispatch_mode": cleanText(details?.dispatchMode) || "unknown",
|
||||
"hwlab.hwpod.websocket_connected": Boolean(details?.websocketConnected),
|
||||
"hwlab.hwpod.direct_url_configured": Boolean(details?.directUrlConfigured),
|
||||
"hwlab.hwpod.otel_trace_id": cleanText(requestMeta?.otelTraceId) || null,
|
||||
traceId: cleanText(requestMeta?.traceId) || null
|
||||
};
|
||||
if (httpContext) httpContext.otelAttributes = { ...(httpContext.otelAttributes ?? {}), ...attributes };
|
||||
const error = Object.assign(new Error(summary), { code });
|
||||
options.emitHttpRoutePhaseSpan?.(request, options, "hwpod-node-ops.blocked", Date.now(), Date.now(), "error", error, attributes);
|
||||
}
|
||||
|
||||
async function readJsonObject(request, limitBytes) {
|
||||
const body = await readBody(request, limitBytes ?? DEFAULT_BODY_LIMIT_BYTES);
|
||||
try {
|
||||
const value = body ? JSON.parse(body) : {};
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return { ok: false, error: { accepted: false, error: { code: "invalid_params", message: "body must be a JSON object" } } };
|
||||
}
|
||||
return { ok: true, value };
|
||||
} catch (error) {
|
||||
return { ok: false, error: { accepted: false, error: { code: "parse_error", message: "Invalid JSON body", reason: error.message } } };
|
||||
}
|
||||
}
|
||||
@@ -149,6 +149,10 @@ test("cloud api aggregates live HWLAB build times from health and controlled dep
|
||||
};
|
||||
|
||||
const server = createCloudApiServer({
|
||||
accessController: {
|
||||
required: false,
|
||||
async authenticate() { return { ok: true, actor: { id: "usr_live_builds", role: "admin" }, session: { id: "uss_live_builds" } }; }
|
||||
},
|
||||
env: {
|
||||
PATH: "",
|
||||
HWLAB_COMMIT_ID: "apiabcdef123456",
|
||||
@@ -285,6 +289,10 @@ test("cloud api ignores old repository reports and keeps missing buildCreatedAt
|
||||
|
||||
const observe = async () => {
|
||||
const server = createCloudApiServer({
|
||||
accessController: {
|
||||
required: false,
|
||||
async authenticate() { return { ok: true, actor: { id: "usr_live_builds", role: "admin" }, session: { id: "uss_live_builds" } }; }
|
||||
},
|
||||
env: {
|
||||
PATH: "",
|
||||
HWLAB_GATEWAY_URL: "http://live.test/hwlab-gateway"
|
||||
|
||||
@@ -146,6 +146,10 @@ test("cloud api /v1/diagnostics/gate exposes Chinese single-table rows from live
|
||||
now: () => "2026-05-23T00:00:00.000Z"
|
||||
});
|
||||
const server = createCloudApiServer({
|
||||
accessController: {
|
||||
required: false,
|
||||
async authenticate() { return { ok: true, actor: { id: "usr_m3_diagnostics", role: "admin" }, session: { id: "uss_m3_diagnostics" } }; }
|
||||
},
|
||||
runtimeStore,
|
||||
env: {
|
||||
HWLAB_M3_GATEWAY_SIMU_1_URL: "http://gateway-1",
|
||||
|
||||
@@ -683,15 +683,16 @@ function sessionCookieFromResponse(response) {
|
||||
return value.split(";")[0] || "";
|
||||
}
|
||||
|
||||
export async function pollAgentResult(port, traceId) {
|
||||
export async function pollAgentResult(port, traceId, init = {}) {
|
||||
let last = null;
|
||||
for (let attempt = 0; attempt < 20; attempt += 1) {
|
||||
const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat/result/${encodeURIComponent(traceId)}`);
|
||||
const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat/result/${encodeURIComponent(traceId)}`, init);
|
||||
last = {
|
||||
status: response.status,
|
||||
body: await response.json()
|
||||
};
|
||||
if (response.status === 200) return last.body;
|
||||
if (response.status !== 202) throw new Error(`Code Agent result polling failed: ${JSON.stringify(last)}`);
|
||||
await delay(10);
|
||||
}
|
||||
throw new Error(`Code Agent result did not complete: ${JSON.stringify(last)}`);
|
||||
|
||||
@@ -17,7 +17,6 @@ import { createWorkbenchReadModel } from "./workbench-read-model.ts";
|
||||
import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts";
|
||||
import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts";
|
||||
import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts";
|
||||
import { openKafkaEventStream } from "./kafka-event-bridge.ts";
|
||||
import { durableTraceStatus, RUNNING_STATUSES, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts";
|
||||
import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts";
|
||||
|
||||
|
||||
@@ -52,9 +52,9 @@ test("workbench launch records OTel stage for Project Management link write 404"
|
||||
}
|
||||
},
|
||||
runtimeStore: {
|
||||
async writeWorkbenchFacts(params, requestMeta) {
|
||||
async writeWorkbenchSessionAdmissionFact(params, requestMeta) {
|
||||
factWrites.push({ params, requestMeta });
|
||||
return { ok: true, facts: params.facts };
|
||||
return { ok: true, fact: params.fact, admissionOnly: true };
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -86,7 +86,7 @@ test("workbench launch records OTel stage for Project Management link write 404"
|
||||
assert.match(context.otelAttributes["workbench.launch.task_ref_hash"], /^[0-9a-f]{24}$/u);
|
||||
assert.equal(context.otelAttributes["workbench.launch.values_redacted"], true);
|
||||
assert.equal(factWrites.length, 1);
|
||||
const sessionJson = factWrites[0].params.facts.sessions[0].sessionJson;
|
||||
const sessionJson = factWrites[0].params.fact.sessionJson;
|
||||
assert.equal(sessionJson.launchContext.sourceId, "hwlab-v03-mdtodo");
|
||||
assert.equal(sessionJson.launchContext.hwpodId, "constart-71freq-c");
|
||||
assert.equal(sessionJson.launchContext.contextFingerprint, "ctx_launch_otel");
|
||||
|
||||
@@ -17,7 +17,7 @@ import { createWorkbenchReadModel } from "./workbench-read-model.ts";
|
||||
import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts";
|
||||
import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts";
|
||||
import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts";
|
||||
import { openKafkaEventStream } from "./kafka-event-bridge.ts";
|
||||
import { workbenchRealtimeCapabilities } from "./workbench-realtime-capabilities.ts";
|
||||
import { durableTraceStatus, RUNNING_STATUSES, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts";
|
||||
import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts";
|
||||
import * as workbenchFacts from "./server-workbench-facts.ts";
|
||||
@@ -157,6 +157,19 @@ export async function handleWorkbenchReadModelHttp(request, response, url, optio
|
||||
if (!auth) return;
|
||||
|
||||
if (url.pathname === "/v1/workbench/sync") {
|
||||
const capabilities = workbenchRealtimeCapabilities(options.env ?? process.env);
|
||||
if (!capabilities.projectionRealtime) {
|
||||
sendJson(response, 503, {
|
||||
ok: false,
|
||||
error: {
|
||||
code: "workbench_projection_realtime_disabled",
|
||||
message: "Workbench projection sync/replay capability is disabled.",
|
||||
capabilities,
|
||||
valuesRedacted: true
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
await (perf ? perf.measure("workbench_sync", () => handleWorkbenchSyncHttp(request, response, url, options, auth.actor)) : handleWorkbenchSyncHttp(request, response, url, options, auth.actor));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -2,12 +2,14 @@
|
||||
// Responsibility: Workbench realtime and projection blocker regression tests.
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { get as httpGet } from "node:http";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { createCloudApiServer } from "./server.ts";
|
||||
import { createBackendPerformanceStore } from "./backend-performance.ts";
|
||||
import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
||||
import { classifyWorkbenchReadModelFailure } from "./server-workbench-http.ts";
|
||||
import { projectionOutboxRealtimeEvents, workbenchRealtimeAfterSeq } from "./server-workbench-realtime-http.ts";
|
||||
import { codeAgentTurnStatusPayload, createCodeAgentChatResultStore } from "./server-code-agent-http.ts";
|
||||
import { createWorkbenchTurnProjection, durableTraceStatus, projectionDiagnostics, traceTerminalEvidence } from "./workbench-turn-projection.ts";
|
||||
|
||||
@@ -32,11 +34,660 @@ import {
|
||||
emptyFacts,
|
||||
normalizeTestStatus,
|
||||
getSseEvents,
|
||||
parseSseBlock,
|
||||
createFakeKafkaFactory,
|
||||
waitFor
|
||||
parseSseBlock
|
||||
} from "./server-workbench-http-test-helpers.ts";
|
||||
|
||||
const TRANSACTIONAL_REALTIME_ENV = Object.freeze({
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false",
|
||||
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
|
||||
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true"
|
||||
});
|
||||
|
||||
test("projection outbox emits immutable assistant versions in row order", () => {
|
||||
const traceId = "trc_outbox_versions";
|
||||
const sessionId = "ses_outbox_versions";
|
||||
const messageId = "msg_outbox_versions_agent";
|
||||
const events = projectionOutboxRealtimeEvents({
|
||||
facts: { messages: [{ messageId, traceId, sessionId, text: "current state must not replace history", projectedSeq: 99 }] },
|
||||
events: [
|
||||
{ outboxSeq: 10, outboxEventId: "outbox-progress-1", entityFamily: "messages", entityId: messageId, projectedSeq: 1, projectionRevision: 1, traceId, sessionId, commitType: "message", payload: { family: "messages", fact: { messageId, traceId, sessionId, text: "first progress", projectedSeq: 1, status: "running" } } },
|
||||
{ outboxSeq: 11, outboxEventId: "outbox-progress-2", entityFamily: "messages", entityId: messageId, projectedSeq: 2, projectionRevision: 2, traceId, sessionId, commitType: "message", payload: { family: "messages", fact: { messageId, traceId, sessionId, text: "second progress", projectedSeq: 2, status: "running" } } }
|
||||
]
|
||||
});
|
||||
|
||||
assert.deepEqual(events.map((item) => item.payload.message.text), ["first progress", "second progress"]);
|
||||
assert.deepEqual(events.map((item) => item.payload.cursor.outboxSeq), [10, 11]);
|
||||
});
|
||||
|
||||
test("live Kafka SSE transparently fans out one envelope without DB, snapshot, cursor, replay, or SSE id", async () => {
|
||||
const sessionId = "ses_live_kafka_sse";
|
||||
const traceId = "trc_live_kafka_sse";
|
||||
const subscribers = new Set();
|
||||
const otelSpans = [];
|
||||
let releaseBridgeReady;
|
||||
const bridgeReady = new Promise((resolve) => { releaseBridgeReady = resolve; });
|
||||
const bridge = {
|
||||
capabilities: { directPublish: true, liveKafkaSse: true, transactionalProjector: false, projectionOutboxRelay: false, projectionRealtime: false },
|
||||
ready: bridgeReady,
|
||||
subscribeLiveHwlabEvents(listener) {
|
||||
subscribers.add(listener);
|
||||
return () => subscribers.delete(listener);
|
||||
},
|
||||
async stop() {}
|
||||
};
|
||||
let dbCalls = 0;
|
||||
const workbenchRuntime = new Proxy({}, {
|
||||
get() {
|
||||
dbCalls += 1;
|
||||
throw new Error("live SSE must not access Workbench runtime facts");
|
||||
}
|
||||
});
|
||||
const server = createCloudApiServer({
|
||||
accessController: realtimeAccessController({
|
||||
sessions: [{ id: sessionId, ownerUserId: ACTOR.id, lastTraceId: traceId }]
|
||||
}),
|
||||
workbenchRuntime,
|
||||
kafkaEventBridge: bridge,
|
||||
otelSpanEmitter(name, emittedTraceId, env, spanOptions) { otelSpans.push({ name, emittedTraceId, env, spanOptions }); },
|
||||
env: {
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
|
||||
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
|
||||
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false"
|
||||
}
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
const envelope = {
|
||||
schema: "hwlab.event.v1",
|
||||
eventType: "hwlab.trace.event.projected",
|
||||
eventId: "hwlab:evt_live_kafka_sse",
|
||||
traceId,
|
||||
hwlabSessionId: sessionId,
|
||||
sessionId,
|
||||
runId: "run_live_kafka_sse",
|
||||
commandId: "cmd_live_kafka_sse",
|
||||
context: { runId: "run_live_kafka_sse", commandId: "cmd_live_kafka_sse", sourceSeq: 4, valuesRedacted: true },
|
||||
event: { type: "assistant", eventType: "assistant", status: "running", traceId, sessionId, text: "live increment", sourceSeq: 4, terminal: false, valuesPrinted: false },
|
||||
valuesPrinted: false
|
||||
};
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const firstPromise = getSseEvents(port, `/v1/workbench/events?sessionId=${sessionId}&afterSeq=999`, 2);
|
||||
const secondPromise = getSseEvents(port, `/v1/workbench/events?sessionId=${sessionId}&traceId=${traceId}&afterSeq=888`, 2);
|
||||
await waitForCondition(() => subscribers.size === 2);
|
||||
for (const listener of subscribers) listener(envelope, { topic: "hwlab.event.v1", partition: 6, offset: "101" });
|
||||
releaseBridgeReady();
|
||||
const [first, second] = await Promise.all([firstPromise, secondPromise]);
|
||||
for (const events of [first, second]) {
|
||||
assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "hwlab.event.v1"]);
|
||||
assert.deepEqual(events[0].data.capabilities, bridge.capabilities);
|
||||
assert.equal(events[0].data.liveOnly, true);
|
||||
assert.equal(events[0].data.replay, false);
|
||||
assert.equal(events[0].data.lossPossible, true);
|
||||
assert.equal(events[0].data.cursor, undefined);
|
||||
assert.equal(events[1].id, null);
|
||||
assert.deepEqual(events[1].data, envelope);
|
||||
}
|
||||
assert.equal(otelSpans.length, 2);
|
||||
for (const span of otelSpans) {
|
||||
assert.equal(span.name, "hwlab.workbench.live_sse.business_event_write");
|
||||
assert.equal(span.emittedTraceId, traceId);
|
||||
assert.deepEqual(span.spanOptions.attributes, {
|
||||
businessTraceId: traceId,
|
||||
hwlabSessionId: sessionId,
|
||||
runId: "run_live_kafka_sse",
|
||||
commandId: "cmd_live_kafka_sse",
|
||||
topic: "hwlab.event.v1",
|
||||
partition: 6,
|
||||
offset: "101",
|
||||
sourceTopic: null,
|
||||
sourcePartition: null,
|
||||
sourceOffset: null,
|
||||
eventType: "assistant",
|
||||
terminal: false,
|
||||
valuesRedacted: true
|
||||
});
|
||||
assert.equal(JSON.stringify(span.spanOptions.attributes).includes("live increment"), false);
|
||||
}
|
||||
assert.equal(dbCalls, 0);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("live Kafka SSE heartbeat keeps the transport open without DB, cursor, snapshot, or replay", async () => {
|
||||
const sessionId = "ses_live_kafka_heartbeat";
|
||||
let dbCalls = 0;
|
||||
const server = createCloudApiServer({
|
||||
accessController: realtimeAccessController({ sessions: [{ id: sessionId, ownerUserId: ACTOR.id }] }),
|
||||
workbenchRuntime: new Proxy({}, {
|
||||
get() {
|
||||
dbCalls += 1;
|
||||
throw new Error("live heartbeat must not access Workbench projection state");
|
||||
}
|
||||
}),
|
||||
kafkaEventBridge: {
|
||||
capabilities: { liveKafkaSse: true },
|
||||
ready: Promise.resolve(),
|
||||
subscribeLiveHwlabEvents() { return () => {}; },
|
||||
async stop() {}
|
||||
},
|
||||
env: {
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
|
||||
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
|
||||
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "5"
|
||||
}
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const events = await getSseEvents(server.address().port, `/v1/workbench/events?sessionId=${sessionId}`, 2);
|
||||
assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.heartbeat"]);
|
||||
assert.equal(events[1].id, null);
|
||||
assert.equal(events[1].data.liveOnly, true);
|
||||
assert.equal(events[1].data.replay, false);
|
||||
assert.equal(events[1].data.cursor, undefined);
|
||||
assert.equal(events[1].data.snapshot, undefined);
|
||||
assert.equal(dbCalls, 0);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("live Kafka SSE rejects foreign or inconsistent ownership scopes before fanout subscription", async () => {
|
||||
const ownedSession = { id: "ses_live_kafka_owned", ownerUserId: ACTOR.id, lastTraceId: "trc_live_kafka_owned" };
|
||||
const foreignSession = { id: "ses_live_kafka_foreign", ownerUserId: "usr_live_kafka_foreign", lastTraceId: "trc_live_kafka_foreign" };
|
||||
let subscriptions = 0;
|
||||
let projectionReads = 0;
|
||||
const server = createCloudApiServer({
|
||||
accessController: realtimeAccessController({ sessions: [ownedSession, foreignSession] }),
|
||||
workbenchRuntime: new Proxy({}, {
|
||||
get() {
|
||||
projectionReads += 1;
|
||||
throw new Error("live SSE authorization must not access Workbench projection state");
|
||||
}
|
||||
}),
|
||||
kafkaEventBridge: {
|
||||
capabilities: { liveKafkaSse: true },
|
||||
ready: Promise.resolve(),
|
||||
subscribeLiveHwlabEvents() {
|
||||
subscriptions += 1;
|
||||
return () => {};
|
||||
},
|
||||
async stop() {}
|
||||
},
|
||||
env: {
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
|
||||
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
|
||||
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false"
|
||||
}
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const foreignSessionResponse = await getJson(port, `/v1/workbench/events?sessionId=${foreignSession.id}`);
|
||||
const foreignTraceResponse = await getJson(port, `/v1/workbench/events?traceId=${foreignSession.lastTraceId}`);
|
||||
const inconsistentResponse = await getJson(port, `/v1/workbench/events?sessionId=${ownedSession.id}&traceId=${foreignSession.lastTraceId}`);
|
||||
const missingResponse = await getJson(port, "/v1/workbench/events?sessionId=ses_live_kafka_missing");
|
||||
|
||||
for (const response of [foreignSessionResponse, foreignTraceResponse, inconsistentResponse, missingResponse]) {
|
||||
assert.equal(response.status, 404);
|
||||
assert.equal(response.body.error.code, "workbench_realtime_scope_not_found");
|
||||
}
|
||||
assert.equal(subscriptions, 0);
|
||||
assert.equal(projectionReads, 0);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("live Kafka SSE fails closed when ownership lookup is not configured", async () => {
|
||||
let subscriptions = 0;
|
||||
const server = createCloudApiServer({
|
||||
accessController: {
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_realtime_unconfigured" } }; }
|
||||
},
|
||||
kafkaEventBridge: {
|
||||
capabilities: { liveKafkaSse: true },
|
||||
ready: Promise.resolve(),
|
||||
subscribeLiveHwlabEvents() {
|
||||
subscriptions += 1;
|
||||
return () => {};
|
||||
},
|
||||
async stop() {}
|
||||
},
|
||||
env: {
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
|
||||
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
|
||||
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false"
|
||||
}
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const response = await getJson(server.address().port, "/v1/workbench/events?sessionId=ses_live_kafka_unconfigured");
|
||||
assert.equal(response.status, 503);
|
||||
assert.equal(response.body.error.code, "workbench_realtime_authorization_unconfigured");
|
||||
assert.equal(subscriptions, 0);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("live Kafka SSE lets admin subscribe to another owner's consistent session and trace", async () => {
|
||||
const session = { id: "ses_live_kafka_admin", ownerUserId: "usr_live_kafka_owner", lastTraceId: "trc_live_kafka_admin" };
|
||||
const admin = { ...ACTOR, id: "usr_live_kafka_admin", role: "admin" };
|
||||
let subscriptions = 0;
|
||||
const server = createCloudApiServer({
|
||||
accessController: realtimeAccessController({ actor: admin, sessions: [session] }),
|
||||
kafkaEventBridge: {
|
||||
capabilities: { liveKafkaSse: true },
|
||||
ready: Promise.resolve(),
|
||||
subscribeLiveHwlabEvents() {
|
||||
subscriptions += 1;
|
||||
return () => {};
|
||||
},
|
||||
async stop() {}
|
||||
},
|
||||
env: {
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
|
||||
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
|
||||
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false"
|
||||
}
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const events = await getSseEvents(server.address().port, `/v1/workbench/events?sessionId=${session.id}&traceId=${session.lastTraceId}`, 1);
|
||||
assert.equal(events[0].event, "workbench.connected");
|
||||
assert.deepEqual(events[0].data.filters, { sessionId: session.id, traceId: session.lastTraceId });
|
||||
assert.equal(subscriptions, 1);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench realtime initial connection emits current snapshot without replaying historical outbox", async () => {
|
||||
const sessionId = "ses_realtime_snapshot_only";
|
||||
const traceId = "trc_realtime_snapshot_only";
|
||||
const calls = [];
|
||||
const runtime = {
|
||||
async readAtomicWorkbenchProjectionSync(params = {}) {
|
||||
calls.push({ ...params });
|
||||
return {
|
||||
facts: {
|
||||
sessions: [{ sessionId, lastTraceId: traceId }],
|
||||
messages: [{ messageId: "msg_snapshot_only", sessionId, traceId, role: "agent", text: "current", projectedSeq: 9 }],
|
||||
turns: [{ turnId: traceId, sessionId, traceId, status: "running", projectedSeq: 9 }]
|
||||
},
|
||||
events: [{ outboxSeq: 1, entityFamily: "messages", entityId: "msg_old", payload: { family: "messages", fact: { messageId: "msg_old", sessionId, traceId, text: "historical" } } }],
|
||||
cutoffOutboxSeq: 90,
|
||||
cursorOutboxSeq: 90,
|
||||
hasMore: true
|
||||
};
|
||||
}
|
||||
};
|
||||
const server = createCloudApiServer({
|
||||
accessController: realtimeAccessController(),
|
||||
workbenchRuntime: runtime,
|
||||
env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" }
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
try {
|
||||
const events = await getSseEvents(server.address().port, `/v1/workbench/events?sessionId=${sessionId}`, 3);
|
||||
assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.message.snapshot", "workbench.turn.snapshot"]);
|
||||
assert.equal(JSON.stringify(events).includes("historical"), false);
|
||||
assert.equal(events[1].id, "90");
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal(calls[0].snapshotOnly, true);
|
||||
assert.equal(calls[0].deltaOnly, false);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("live and projection realtime remain independently reachable when both capabilities are enabled", async () => {
|
||||
const sessionId = "ses_composable_realtime";
|
||||
const traceId = "trc_composable_realtime";
|
||||
let projectionReads = 0;
|
||||
const runtime = {
|
||||
async readAtomicWorkbenchProjectionSync() {
|
||||
projectionReads += 1;
|
||||
return {
|
||||
facts: {
|
||||
sessions: [{ sessionId, lastTraceId: traceId }],
|
||||
messages: [{ messageId: "msg_composable_realtime", sessionId, traceId, role: "agent", text: "projection snapshot", projectedSeq: 1 }],
|
||||
turns: []
|
||||
},
|
||||
events: [],
|
||||
cutoffOutboxSeq: 1,
|
||||
cursorOutboxSeq: 1,
|
||||
hasMore: false
|
||||
};
|
||||
}
|
||||
};
|
||||
const server = createCloudApiServer({
|
||||
accessController: realtimeAccessController(),
|
||||
workbenchRuntime: runtime,
|
||||
env: {
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
|
||||
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
|
||||
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000",
|
||||
HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100"
|
||||
}
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
try {
|
||||
const events = await getSseEvents(server.address().port, `/v1/workbench/projection-events?sessionId=${sessionId}`, 2);
|
||||
assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.message.snapshot"]);
|
||||
assert.equal(events[0].data.realtimeSource, "projection-outbox");
|
||||
assert.equal(projectionReads, 1);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("synchronous projection notification is buffered until the unique initial snapshot completes", async () => {
|
||||
const sessionId = "ses_realtime_initial_barrier";
|
||||
const traceId = "trc_realtime_initial_barrier";
|
||||
const calls = [];
|
||||
let releaseInitial;
|
||||
let unsubscribeCount = 0;
|
||||
const initialGate = new Promise((resolve) => { releaseInitial = resolve; });
|
||||
const kafkaEventBridge = {
|
||||
subscribeProjectionCommits(listener) {
|
||||
listener({ sessionId, traceId });
|
||||
return () => { unsubscribeCount += 1; };
|
||||
},
|
||||
async stop() {}
|
||||
};
|
||||
const runtime = {
|
||||
async readAtomicWorkbenchProjectionSync(params = {}) {
|
||||
calls.push({ ...params });
|
||||
if (calls.length === 1) {
|
||||
await initialGate;
|
||||
return {
|
||||
facts: {
|
||||
sessions: [{ sessionId, lastTraceId: traceId }],
|
||||
messages: [{ messageId: "msg_initial_barrier", sessionId, traceId, role: "agent", text: "snapshot before delta", projectedSeq: 10 }],
|
||||
turns: []
|
||||
},
|
||||
events: [],
|
||||
cutoffOutboxSeq: 10,
|
||||
cursorOutboxSeq: 10,
|
||||
hasMore: false
|
||||
};
|
||||
}
|
||||
const event = { id: "wte_initial_barrier_delta", sessionId, traceId, projectedSeq: 11, message: "delta after snapshot" };
|
||||
return {
|
||||
facts: {},
|
||||
events: [{ outboxSeq: 11, entityFamily: "traceEvents", entityId: event.id, projectedSeq: 11, projectionRevision: 11, sessionId, traceId, commitType: "event", payload: { family: "traceEvents", fact: event } }],
|
||||
cutoffOutboxSeq: 11,
|
||||
cursorOutboxSeq: 11,
|
||||
hasMore: false
|
||||
};
|
||||
}
|
||||
};
|
||||
const server = createCloudApiServer({
|
||||
accessController: realtimeAccessController(),
|
||||
workbenchRuntime: runtime,
|
||||
kafkaEventBridge,
|
||||
env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" }
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
try {
|
||||
const eventsPromise = getSseEvents(server.address().port, `/v1/workbench/events?sessionId=${sessionId}`, 3);
|
||||
await waitForCondition(() => calls.length === 1);
|
||||
assert.equal(calls[0].snapshotOnly, true);
|
||||
assert.equal(calls[0].deltaOnly, false);
|
||||
assert.equal(calls.length, 1);
|
||||
releaseInitial();
|
||||
const events = await eventsPromise;
|
||||
assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.message.snapshot", "workbench.trace.event"]);
|
||||
assert.equal(events[1].data.message.text, "snapshot before delta");
|
||||
assert.equal(events[2].data.event.message, "delta after snapshot");
|
||||
assert.equal(calls.length, 2);
|
||||
assert.equal(calls[1].snapshotOnly, false);
|
||||
assert.equal(calls[1].deltaOnly, true);
|
||||
assert.equal(calls[1].afterOutboxSeq, 10);
|
||||
} finally {
|
||||
releaseInitial?.();
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
assert.equal(unsubscribeCount, 1);
|
||||
});
|
||||
|
||||
test("initial atomic snapshot failure emits an error and closes SSE for reconnect", async () => {
|
||||
const sessionId = "ses_realtime_initial_failure";
|
||||
let unsubscribeCount = 0;
|
||||
const server = createCloudApiServer({
|
||||
accessController: realtimeAccessController(),
|
||||
workbenchRuntime: {
|
||||
async readAtomicWorkbenchProjectionSync() {
|
||||
const error = new Error("initial snapshot failed");
|
||||
error.code = "workbench_initial_snapshot_failed";
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
kafkaEventBridge: {
|
||||
subscribeProjectionCommits() { return () => { unsubscribeCount += 1; }; },
|
||||
async stop() {}
|
||||
},
|
||||
env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" }
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
const controller = new AbortController();
|
||||
let timeout = null;
|
||||
try {
|
||||
const response = await fetch(`http://127.0.0.1:${server.address().port}/v1/workbench/events?sessionId=${sessionId}`, { signal: controller.signal });
|
||||
assert.equal(response.status, 200);
|
||||
const body = await Promise.race([
|
||||
response.text(),
|
||||
new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error("initial snapshot failure left SSE open")), 500); })
|
||||
]);
|
||||
const events = body.trim().split("\n\n").filter(Boolean).map(parseSseBlock);
|
||||
assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.error"]);
|
||||
assert.equal(events[1].data.error.code, "workbench_initial_snapshot_failed");
|
||||
assert.equal(body.includes("workbench.heartbeat"), false);
|
||||
await waitForCondition(() => unsubscribeCount === 1);
|
||||
} finally {
|
||||
if (timeout) clearTimeout(timeout);
|
||||
controller.abort();
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("delta scan failure after a successful snapshot closes SSE for cursor reconnect", async () => {
|
||||
const sessionId = "ses_realtime_delta_failure";
|
||||
const traceId = "trc_realtime_delta_failure";
|
||||
let unsubscribeCount = 0;
|
||||
let calls = 0;
|
||||
const server = createCloudApiServer({
|
||||
accessController: realtimeAccessController(),
|
||||
workbenchRuntime: {
|
||||
async readAtomicWorkbenchProjectionSync() {
|
||||
calls += 1;
|
||||
if (calls === 1) {
|
||||
return {
|
||||
facts: {
|
||||
sessions: [{ sessionId, lastTraceId: traceId }],
|
||||
messages: [{ messageId: "msg_delta_failure", sessionId, traceId, role: "agent", text: "durable snapshot", projectedSeq: 5 }],
|
||||
turns: []
|
||||
},
|
||||
events: [],
|
||||
cutoffOutboxSeq: 5,
|
||||
cursorOutboxSeq: 5,
|
||||
hasMore: false
|
||||
};
|
||||
}
|
||||
const error = new Error("delta scan failed");
|
||||
error.code = "workbench_delta_scan_failed";
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
kafkaEventBridge: {
|
||||
subscribeProjectionCommits(listener) {
|
||||
listener({ sessionId, traceId });
|
||||
return () => { unsubscribeCount += 1; };
|
||||
},
|
||||
async stop() {}
|
||||
},
|
||||
env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" }
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
const controller = new AbortController();
|
||||
let timeout = null;
|
||||
try {
|
||||
const response = await fetch(`http://127.0.0.1:${server.address().port}/v1/workbench/events?sessionId=${sessionId}`, { signal: controller.signal });
|
||||
assert.equal(response.status, 200);
|
||||
const body = await Promise.race([
|
||||
response.text(),
|
||||
new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error("delta failure left SSE open")), 500); })
|
||||
]);
|
||||
const events = body.trim().split("\n\n").filter(Boolean).map(parseSseBlock);
|
||||
assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.message.snapshot", "workbench.error"]);
|
||||
assert.equal(events[2].data.error.code, "workbench_delta_scan_failed");
|
||||
assert.equal(body.includes("workbench.heartbeat"), false);
|
||||
assert.equal(calls, 2);
|
||||
await waitForCondition(() => unsubscribeCount === 1);
|
||||
} finally {
|
||||
if (timeout) clearTimeout(timeout);
|
||||
controller.abort();
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("projector commit during initial snapshot wakes one coalesced delta scan without idle polling", async () => {
|
||||
const sessionId = "ses_realtime_commit_wakeup";
|
||||
const traceId = "trc_realtime_commit_wakeup";
|
||||
const calls = [];
|
||||
let releaseInitial;
|
||||
let notify = null;
|
||||
let unsubscribeCount = 0;
|
||||
const initialGate = new Promise((resolve) => { releaseInitial = resolve; });
|
||||
const kafkaEventBridge = {
|
||||
subscribeProjectionCommits(listener) { notify = listener; return () => { unsubscribeCount += 1; }; },
|
||||
async stop() {}
|
||||
};
|
||||
const runtime = {
|
||||
async readAtomicWorkbenchProjectionSync(params = {}) {
|
||||
calls.push({ ...params });
|
||||
if (calls.length === 1) {
|
||||
await initialGate;
|
||||
return { facts: { sessions: [{ sessionId, lastTraceId: traceId }], messages: [], turns: [] }, events: [], cutoffOutboxSeq: 10, cursorOutboxSeq: 10, hasMore: false };
|
||||
}
|
||||
const event = { id: "wte_commit_wakeup", sessionId, traceId, projectedSeq: 11, message: "commit wakeup" };
|
||||
return { facts: {}, events: [{ outboxSeq: 11, entityFamily: "traceEvents", entityId: event.id, projectedSeq: 11, projectionRevision: 11, sessionId, traceId, commitType: "event", payload: { family: "traceEvents", fact: event } }], cutoffOutboxSeq: 11, cursorOutboxSeq: 11, hasMore: false };
|
||||
}
|
||||
};
|
||||
const server = createCloudApiServer({
|
||||
accessController: realtimeAccessController(),
|
||||
workbenchRuntime: runtime,
|
||||
kafkaEventBridge,
|
||||
env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" }
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
try {
|
||||
const eventsPromise = getSseEvents(server.address().port, `/v1/workbench/events?sessionId=${sessionId}`, 2);
|
||||
await waitForCondition(() => calls.length === 1 && typeof notify === "function");
|
||||
notify({ sessionId, traceId });
|
||||
notify({ sessionId, traceId });
|
||||
releaseInitial();
|
||||
const events = await eventsPromise;
|
||||
assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.trace.event"]);
|
||||
assert.equal(events[1].data.event.message, "commit wakeup");
|
||||
assert.equal(calls.length, 2);
|
||||
assert.equal(calls[0].snapshotOnly, true);
|
||||
assert.equal(calls[1].deltaOnly, true);
|
||||
await new Promise((resolve) => setTimeout(resolve, 30));
|
||||
assert.equal(calls.length, 2);
|
||||
} finally {
|
||||
releaseInitial?.();
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
assert.equal(unsubscribeCount, 1);
|
||||
});
|
||||
|
||||
test("workbench realtime early disconnect releases commit subscription during initial scan", async () => {
|
||||
const sessionId = "ses_realtime_early_disconnect";
|
||||
let releaseInitial;
|
||||
let subscribed = 0;
|
||||
let unsubscribed = 0;
|
||||
const gate = new Promise((resolve) => { releaseInitial = resolve; });
|
||||
const server = createCloudApiServer({
|
||||
accessController: realtimeAccessController(),
|
||||
workbenchRuntime: { async readAtomicWorkbenchProjectionSync() { await gate; return { facts: {}, events: [], cutoffOutboxSeq: 0, cursorOutboxSeq: 0, hasMore: false }; } },
|
||||
kafkaEventBridge: { subscribeProjectionCommits() { subscribed += 1; return () => { unsubscribed += 1; }; }, async stop() {} },
|
||||
env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" }
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
let clientRequest = null;
|
||||
try {
|
||||
const response = await new Promise((resolve, reject) => {
|
||||
clientRequest = httpGet(`http://127.0.0.1:${server.address().port}/v1/workbench/events?sessionId=${sessionId}`, resolve);
|
||||
clientRequest.once("error", reject);
|
||||
});
|
||||
assert.equal(response.statusCode, 200);
|
||||
await waitForCondition(() => subscribed === 1);
|
||||
response.destroy();
|
||||
clientRequest.destroy();
|
||||
await waitForCondition(() => unsubscribed === 1);
|
||||
releaseInitial();
|
||||
} finally {
|
||||
clientRequest?.destroy();
|
||||
releaseInitial?.();
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench realtime prefers automatic Last-Event-ID over stale URL cursor", () => {
|
||||
const url = new URL("http://localhost/v1/workbench/events?afterSeq=10&afterOutboxSeq=11");
|
||||
assert.equal(workbenchRealtimeAfterSeq({ headers: { "last-event-id": "42" } }, url), 42);
|
||||
});
|
||||
|
||||
test("workbench realtime resets a cursor ahead of scoped cutoff with one fresh snapshot", async () => {
|
||||
const sessionId = "ses_realtime_cursor_reset";
|
||||
const traceId = "trc_realtime_cursor_reset";
|
||||
const calls = [];
|
||||
const runtime = {
|
||||
async readAtomicWorkbenchProjectionSync(params = {}) {
|
||||
calls.push({ ...params });
|
||||
if (params.snapshotOnly === true) {
|
||||
return { facts: { sessions: [{ sessionId, lastTraceId: traceId }], messages: [{ messageId: "msg_cursor_reset", sessionId, traceId, text: "fresh", projectedSeq: 5 }], turns: [] }, events: [], cutoffOutboxSeq: 5, cursorOutboxSeq: 5, hasMore: false };
|
||||
}
|
||||
return { facts: {}, events: [], cutoffOutboxSeq: 5, cursorOutboxSeq: 5, hasMore: false };
|
||||
}
|
||||
};
|
||||
const server = createCloudApiServer({ accessController: realtimeAccessController(), workbenchRuntime: runtime, env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } });
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
try {
|
||||
const events = await getSseEvents(server.address().port, `/v1/workbench/events?sessionId=${sessionId}&afterSeq=42`, 2);
|
||||
assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.message.snapshot"]);
|
||||
assert.equal(events[1].id, "5");
|
||||
assert.deepEqual(calls.map((call) => [call.deltaOnly, call.snapshotOnly]), [[true, false], [false, true]]);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench realtime stream surfaces facts blocker instead of legacy trace fallback", async () => {
|
||||
const traceStore = createCodeAgentTraceStore();
|
||||
const results = createCodeAgentChatResultStore();
|
||||
@@ -63,36 +714,44 @@ test("workbench realtime stream surfaces facts blocker instead of legacy trace f
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
|
||||
};
|
||||
const server = createCloudApiServer({ accessController, traceStore, codeAgentChatResults: results, env: { HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000" } });
|
||||
const workbenchRuntime = {
|
||||
async readAtomicWorkbenchProjectionSync() {
|
||||
const error = new Error("atomic projection unavailable");
|
||||
error.code = "TEST_ATOMIC_PROJECTION_UNAVAILABLE";
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
const server = createCloudApiServer({
|
||||
accessController,
|
||||
traceStore,
|
||||
codeAgentChatResults: results,
|
||||
workbenchRuntime,
|
||||
env: {
|
||||
...TRANSACTIONAL_REALTIME_ENV,
|
||||
HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000",
|
||||
HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100"
|
||||
}
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const eventsPromise = getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}&traceId=${encodeURIComponent(traceId)}`, 4);
|
||||
const eventsPromise = getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}&traceId=${encodeURIComponent(traceId)}`, 2);
|
||||
const events = await eventsPromise;
|
||||
assert.deepEqual(events.map((event) => event.event), [
|
||||
"workbench.connected",
|
||||
"workbench.error",
|
||||
"workbench.trace.snapshot",
|
||||
"workbench.turn.snapshot"
|
||||
"workbench.error"
|
||||
]);
|
||||
assert.equal(events[0].data.filters.sessionId, session.id);
|
||||
assert.equal(events[1].data.error.code, "workbench_facts_trace_missing");
|
||||
assert.equal(events[2].data.traceId, traceId);
|
||||
assert.equal(events[2].data.snapshot.status, "unknown");
|
||||
assert.equal(events[2].data.snapshot.eventCount, 0);
|
||||
assert.equal(events[3].data.turn.traceId, traceId);
|
||||
assert.equal(events[3].data.turn.status, "unknown");
|
||||
assert.equal(events[3].data.turn.terminal, false);
|
||||
assert.equal(events[3].data.turn.finalResponse, null);
|
||||
assert.equal(events[1].data.error.code, "TEST_ATOMIC_PROJECTION_UNAVAILABLE");
|
||||
assert.equal(JSON.stringify(events).includes("legacy result answer"), false);
|
||||
assert.equal(JSON.stringify(events).includes("legacy trace answer"), false);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
test("workbench realtime stream forwards HWLAB Kafka events after initial connection", async () => {
|
||||
const fakeKafka = createFakeKafkaFactory();
|
||||
test("workbench session realtime follows durable outbox beyond stale lastTraceId", async () => {
|
||||
const staleTraceId = "trc_workbench_realtime_stale";
|
||||
const traceId = "trc_workbench_realtime_after_seq";
|
||||
const session = {
|
||||
id: "ses_workbench_realtime_after_seq",
|
||||
@@ -102,17 +761,25 @@ test("workbench realtime stream forwards HWLAB Kafka events after initial connec
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: "cnv_workbench_realtime_after_seq",
|
||||
threadId: "thread-workbench-realtime-after-seq",
|
||||
lastTraceId: traceId,
|
||||
lastTraceId: staleTraceId,
|
||||
updatedAt: "2026-06-24T14:00:00.000Z",
|
||||
session: { sessionStatus: "running", lastTraceId: traceId }
|
||||
session: { sessionStatus: "running", lastTraceId: staleTraceId }
|
||||
};
|
||||
const outboxQueries = [];
|
||||
const workbenchRuntime = {
|
||||
async readWorkbenchProjectionOutbox(params = {}) {
|
||||
async readAtomicWorkbenchProjectionSync(params = {}) {
|
||||
outboxQueries.push({ ...params });
|
||||
return [
|
||||
{ outboxSeq: 11, projectedSeq: 7, traceId, sessionId: session.id, turnId: traceId, commitType: "event", terminal: false, sealed: false, createdAt: "2026-06-24T14:00:01.000Z" }
|
||||
];
|
||||
const after = Number(params.afterOutboxSeq ?? 0);
|
||||
const outboxSeq = after + 1;
|
||||
const projectedSeq = outboxSeq === 11 ? 7 : 8;
|
||||
const event = { id: `wte_workbench_realtime_after_seq_${outboxSeq}`, sourceEventId: `src_workbench_realtime_after_seq_${outboxSeq}`, projectedSeq, sourceSeq: projectedSeq, traceId, sessionId: session.id, turnId: traceId, eventType: "backend", status: "running", message: outboxSeq === 11 ? "durable outbox realtime event" : "second outbox page", terminal: false, sealed: false, updatedAt: "2026-06-24T14:00:01.000Z" };
|
||||
return {
|
||||
facts: { sessions: [{ sessionId: session.id, ownerUserId: ACTOR.id, lastTraceId: staleTraceId, status: "running" }], messages: [], parts: [], turns: [], traceEvents: [event], checkpoints: [] },
|
||||
events: [{ outboxSeq, outboxEventId: `outbox-workbench-realtime-after-seq-${outboxSeq}`, entityFamily: "traceEvents", entityId: event.id, projectionRevision: projectedSeq, projectedSeq, traceId, sessionId: session.id, turnId: traceId, commitType: "event", terminal: false, sealed: false, payload: { family: "traceEvents", fact: event }, createdAt: event.updatedAt }],
|
||||
cutoffOutboxSeq: 12,
|
||||
cursorOutboxSeq: outboxSeq,
|
||||
hasMore: outboxSeq < 12
|
||||
};
|
||||
}
|
||||
};
|
||||
const accessController = {
|
||||
@@ -126,110 +793,34 @@ test("workbench realtime stream forwards HWLAB Kafka events after initial connec
|
||||
const serverWithKafka = createCloudApiServer({
|
||||
accessController,
|
||||
workbenchRuntime,
|
||||
kafkaFactory: fakeKafka.factory,
|
||||
env: {
|
||||
...TRANSACTIONAL_REALTIME_ENV,
|
||||
HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000",
|
||||
HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092",
|
||||
HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api",
|
||||
HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1"
|
||||
HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100"
|
||||
}
|
||||
});
|
||||
await new Promise((resolve) => serverWithKafka.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = serverWithKafka.address();
|
||||
setTimeout(() => { void fakeKafka.emit({
|
||||
eventType: "hwlab.trace.event.projected",
|
||||
sessionId: "ses_agentrun_workbench_realtime_after_seq",
|
||||
traceId,
|
||||
context: { sourceSeq: 7, runId: "run_workbench_realtime_after_seq", commandId: "cmd_workbench_realtime_after_seq" },
|
||||
event: { type: "backend", eventType: "backend", status: "running", label: "agentrun:event:test", message: "Kafka realtime event", sourceSeq: 7 }
|
||||
}); }, 25);
|
||||
const events = await getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}&traceId=${encodeURIComponent(traceId)}&afterSeq=10`, 2);
|
||||
assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.trace.event"]);
|
||||
const events = await getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}&afterSeq=10`, 3);
|
||||
assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.trace.event", "workbench.trace.event"]);
|
||||
assert.equal(events[0].id, "10");
|
||||
assert.equal(events[1].id, "hwlab.event.v1:0:0");
|
||||
assert.equal(events[1].data.realtimeSource, "kafka");
|
||||
assert.equal(events[1].id, "11");
|
||||
assert.equal(events[1].data.realtimeSource, "projection-outbox");
|
||||
assert.equal(events[1].data.realtimeAuthority, "workbench-realtime-authority-v2");
|
||||
assert.equal(events[1].data.entity.family, "traceEvents");
|
||||
assert.equal(events[1].data.entity.version, 7);
|
||||
assert.equal(events[1].data.entity.projectionRevision, "kafka:hwlab.event.v1:0:0");
|
||||
assert.equal(events[1].data.event.message, "Kafka realtime event");
|
||||
assert.equal(events[1].data.kafka.topic, "hwlab.event.v1");
|
||||
assert.equal(events[1].data.entity.projectionRevision, "7");
|
||||
assert.equal(events[1].data.event.message, "durable outbox realtime event");
|
||||
assert.equal(events[1].data.traceId, traceId);
|
||||
assert.equal(events[1].data.cursor.traceSeq, 7);
|
||||
assert.deepEqual(outboxQueries, []);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => serverWithKafka.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench session realtime Kafka stream does not pin subscription to stale lastTraceId", async () => {
|
||||
const fakeKafka = createFakeKafkaFactory();
|
||||
const staleTraceId = "trc_workbench_realtime_stale_trace";
|
||||
const liveTraceId = "trc_workbench_realtime_live_trace";
|
||||
const session = {
|
||||
id: "ses_workbench_realtime_session_wide",
|
||||
projectId: "prj_hwpod_workbench",
|
||||
agentId: "hwlab-code-agent",
|
||||
status: "running",
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: "cnv_workbench_realtime_session_wide",
|
||||
threadId: "thread-workbench-realtime-session-wide",
|
||||
lastTraceId: staleTraceId,
|
||||
updatedAt: "2026-06-24T14:00:00.000Z",
|
||||
session: { sessionStatus: "running", lastTraceId: staleTraceId }
|
||||
};
|
||||
const accessController = {
|
||||
store: {
|
||||
async getAgentSession(sessionId) { return sessionId === session.id ? session : null; },
|
||||
async getAgentSessionByTraceId() { return null; }
|
||||
},
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
|
||||
};
|
||||
const serverWithKafka = createCloudApiServer({
|
||||
accessController,
|
||||
workbenchRuntime: {
|
||||
async queryWorkbenchFacts(params = {}) {
|
||||
return {
|
||||
facts: {
|
||||
sessions: [{ sessionId: session.id, ownerUserId: ACTOR.id, threadId: session.threadId, lastTraceId: staleTraceId, status: "running", valuesRedacted: true }],
|
||||
messages: [],
|
||||
parts: [],
|
||||
turns: [],
|
||||
checkpoints: []
|
||||
},
|
||||
count: 1,
|
||||
persistence: { adapter: "test-session-wide-kafka", durable: true },
|
||||
params
|
||||
};
|
||||
}
|
||||
},
|
||||
kafkaFactory: fakeKafka.factory,
|
||||
env: {
|
||||
HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000",
|
||||
HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092",
|
||||
HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api",
|
||||
HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1"
|
||||
}
|
||||
});
|
||||
await new Promise((resolve) => serverWithKafka.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = serverWithKafka.address();
|
||||
setTimeout(() => { void fakeKafka.emit({
|
||||
eventType: "hwlab.trace.event.projected",
|
||||
sessionId: "ses_agentrun_workbench_realtime_session_wide",
|
||||
traceId: liveTraceId,
|
||||
context: { runId: "run_workbench_realtime_session_wide", commandId: "cmd_workbench_realtime_session_wide" },
|
||||
event: { type: "backend", eventType: "backend", status: "running", label: "agentrun:event:live", message: "Kafka live trace event" }
|
||||
}); }, 100);
|
||||
const events = await getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}`, 4);
|
||||
assert.equal(events[0].event, "workbench.connected");
|
||||
const liveEvent = events.find((event) => event.event === "workbench.trace.event" && event.data?.traceId === liveTraceId);
|
||||
assert.ok(liveEvent, JSON.stringify(events.map((event) => ({ event: event.event, traceId: event.data?.traceId, message: event.data?.event?.message }))));
|
||||
assert.equal(liveEvent.data.event.message, "Kafka live trace event");
|
||||
assert.equal(liveEvent.data.realtimeAuthority, "workbench-realtime-authority-v2");
|
||||
assert.equal(events[2].id, "12");
|
||||
assert.equal(events[2].data.event.message, "second outbox page");
|
||||
assert.equal(outboxQueries.length, 2);
|
||||
assert.deepEqual(outboxQueries.map((query) => query.afterOutboxSeq), [10, 11]);
|
||||
assert.equal(outboxQueries[0].sessionId, session.id);
|
||||
assert.deepEqual(outboxQueries[0].actor, { id: ACTOR.id, role: ACTOR.role });
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => serverWithKafka.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
@@ -277,7 +868,7 @@ test("workbench read model exposes runtime trace projection query failures as pr
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: results });
|
||||
const server = createCloudApiServer({ accessController, traceStore, workbenchRuntime: runtimeStore, codeAgentChatResults: results });
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
@@ -355,7 +946,7 @@ test("workbench trace events remain visible after session lastTraceId moves", as
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
|
||||
};
|
||||
const server = createCloudApiServer({ accessController, runtimeStore });
|
||||
const server = createCloudApiServer({ accessController, workbenchRuntime: runtimeStore });
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
@@ -446,3 +1037,14 @@ test("workbench trace event page keeps per-trace terminal status after later ses
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
function realtimeAccessController({ actor = ACTOR, sessions = [] } = {}) {
|
||||
const bySessionId = new Map(sessions.map((session) => [session.id, session]));
|
||||
const byTraceId = new Map(sessions.filter((session) => session.lastTraceId).map((session) => [session.lastTraceId, session]));
|
||||
return {
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor, session: { id: "uss_realtime_test" } }; },
|
||||
async getAgentSession(sessionId) { return bySessionId.get(sessionId) ?? null; },
|
||||
async getAgentSessionByTraceId(traceId) { return byTraceId.get(traceId) ?? null; }
|
||||
};
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
||||
import {
|
||||
parsePositiveInteger,
|
||||
safeConversationId,
|
||||
@@ -15,11 +14,15 @@ import {
|
||||
} from "./server-http-utils.ts";
|
||||
import { createWorkbenchReadModel } from "./workbench-read-model.ts";
|
||||
import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts";
|
||||
import { emitLiveKafkaOtelSpan } from "./kafka-event-bridge.ts";
|
||||
import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts";
|
||||
import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts";
|
||||
import { openKafkaEventStream } from "./kafka-event-bridge.ts";
|
||||
import { durableTraceStatus, RUNNING_STATUSES, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts";
|
||||
import { projectionOutboxRealtimeEvents } from "./workbench-projection-outbox-events.ts";
|
||||
import { durableTraceStatus, RUNNING_STATUSES, TERMINAL_STATUSES } from "./workbench-turn-projection.ts";
|
||||
import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts";
|
||||
import {
|
||||
workbenchRealtimeCapabilities
|
||||
} from "./workbench-realtime-capabilities.ts";
|
||||
import * as workbenchFacts from "./server-workbench-facts.ts";
|
||||
|
||||
const DEFAULT_PAGE_LIMIT = 50;
|
||||
@@ -27,7 +30,6 @@ const DEFAULT_SESSION_LIST_LIMIT = 20;
|
||||
const MAX_PAGE_LIMIT = 100;
|
||||
const DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS = 15000;
|
||||
const WORKBENCH_REALTIME_DRAIN_TIMEOUT_MS = 2500;
|
||||
const WORKBENCH_REALTIME_AUTHORITY_VERSION = "workbench-realtime-authority-v2";
|
||||
const WORKBENCH_SESSION_LIST_PAGE_FAMILIES = Object.freeze(["sessions"]);
|
||||
const WORKBENCH_SESSION_DETAIL_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]);
|
||||
const WORKBENCH_SESSION_MESSAGE_PAGE_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]);
|
||||
@@ -180,14 +182,25 @@ export async function drainWorkbenchRealtimeConnections(options = {}) {
|
||||
export async function handleWorkbenchRealtimeHttp(request, response, url, options = {}) {
|
||||
try {
|
||||
if (request.method !== "GET") return methodNotAllowed(response, "GET");
|
||||
const realtimeCapabilities = workbenchRealtimeCapabilities(options.env ?? process.env);
|
||||
const projectionOnly = url.pathname === "/v1/workbench/projection-events";
|
||||
if (!projectionOnly && realtimeCapabilities.liveKafkaSse) {
|
||||
await handleLiveKafkaWorkbenchRealtimeHttp(request, response, url, options);
|
||||
return;
|
||||
}
|
||||
if (!realtimeCapabilities.projectionRealtime) {
|
||||
sendJson(response, 503, workbenchError("workbench_realtime_disabled", "No Workbench realtime SSE capability is enabled."));
|
||||
return;
|
||||
}
|
||||
const requestedSessionId = safeSessionId(url.searchParams.get("sessionId") ?? url.searchParams.get("includeSessionId"));
|
||||
const requestedTraceId = safeTraceId(url.searchParams.get("traceId"));
|
||||
const heartbeatMs = parsePositiveInteger(options.env?.HWLAB_WORKBENCH_SSE_HEARTBEAT_MS, DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS);
|
||||
attachWorkbenchRealtimeOtelContext(request, {
|
||||
route: projectionOnly ? "/v1/workbench/projection-events" : "/v1/workbench/events",
|
||||
sessionId: requestedSessionId,
|
||||
traceId: requestedTraceId,
|
||||
heartbeatMs,
|
||||
realtimeSource: "kafka"
|
||||
realtimeSource: "projection-outbox"
|
||||
});
|
||||
const perf = options.backendPerformance;
|
||||
const auth = perf ? await perf.measure("workbench_auth", () => authenticateWorkbenchRead(request, response, options)) : await authenticateWorkbenchRead(request, response, options);
|
||||
@@ -198,9 +211,17 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option
|
||||
return;
|
||||
}
|
||||
|
||||
const traceStore = options.traceStore ?? defaultCodeAgentTraceStore;
|
||||
const readModel = createWorkbenchReadModel(options, auth.actor);
|
||||
if (!requestedSessionId && !requestedTraceId) {
|
||||
sendJson(response, 400, workbenchError("workbench_realtime_scope_required", "Workbench realtime requires sessionId or traceId."));
|
||||
return;
|
||||
}
|
||||
const runtime = options.workbenchRuntime ?? createWorkbenchRuntimeClient({ env: options.env ?? process.env, fetch: options.fetch, traceparent: request?.hwlabHttpRequestContext?.traceparent });
|
||||
if (typeof runtime?.readAtomicWorkbenchProjectionSync !== "function") {
|
||||
sendJson(response, 503, workbenchError("workbench_realtime_runtime_unconfigured", "Workbench realtime requires an atomic projection snapshot reader."));
|
||||
return;
|
||||
}
|
||||
const requestedAfterSeq = workbenchRealtimeAfterSeq(request, url);
|
||||
const outboxTailBatchSize = requiredPositiveRuntimeSetting(options.env ?? process.env, "HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE");
|
||||
let closed = false;
|
||||
let realtimeCloseReason = "client_close";
|
||||
let realtimeCloseSignal = null;
|
||||
@@ -210,6 +231,315 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option
|
||||
const realtimeStartedAtMs = Date.now();
|
||||
const cleanup = [];
|
||||
|
||||
let streamSessionId = requestedSessionId ?? null;
|
||||
let streamThreadId = null;
|
||||
let activeTraceId = requestedTraceId ?? null;
|
||||
let outboxCursor = requestedAfterSeq;
|
||||
let scanRunning = false;
|
||||
let scanRequested = false;
|
||||
let initialScanState = "pending";
|
||||
let initialScanWakeRequested = false;
|
||||
let realtimeConnection = null;
|
||||
|
||||
const isActive = () => !closed && !response.destroyed && !response.writableEnded;
|
||||
const closeConnection = () => {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
emitWorkbenchRealtimeClosedOtelSpan(request, options.env, {
|
||||
reason: realtimeCloseReason,
|
||||
signal: realtimeCloseSignal,
|
||||
startedAtMs: realtimeStartedAtMs,
|
||||
sessionId: realtimeSessionId,
|
||||
threadId: realtimeThreadId,
|
||||
traceId: realtimeTraceId,
|
||||
activeConnectionCount: activeWorkbenchRealtimeConnections.size
|
||||
});
|
||||
for (const item of cleanup.splice(0)) item();
|
||||
};
|
||||
|
||||
response.writeHead(200, {
|
||||
"content-type": "text/event-stream; charset=utf-8",
|
||||
"cache-control": "no-store, no-transform",
|
||||
connection: "keep-alive",
|
||||
"x-accel-buffering": "no",
|
||||
"x-content-type-options": "nosniff"
|
||||
});
|
||||
response.once("close", closeConnection);
|
||||
request.once?.("aborted", closeConnection);
|
||||
request.socket?.once?.("close", closeConnection);
|
||||
cleanup.push(() => request.off?.("aborted", closeConnection));
|
||||
cleanup.push(() => request.socket?.off?.("close", closeConnection));
|
||||
if (typeof response.flushHeaders === "function") response.flushHeaders();
|
||||
|
||||
const writeEvent = async (name, payload = {}) => {
|
||||
if (!isActive()) return false;
|
||||
const startedAt = nowMs();
|
||||
const eventCreatedAt = realtimeEventCreatedAt(payload);
|
||||
const traceSeq = realtimeTraceSeq(payload);
|
||||
const eventId = realtimeEventId(payload);
|
||||
const block = `event: ${name}\n${eventId ? `id: ${eventId}\n` : ""}data: ${JSON.stringify({ contractVersion: "workbench-events-v1", serverSentAt: new Date().toISOString(), eventCreatedAt, traceSeq, ...payload })}\n\n`;
|
||||
const writable = response.write(block);
|
||||
if (!writable) await waitForResponseDrain(response, isActive);
|
||||
perf?.recordPhase({ phase: "sse_write", durationMs: nowMs() - startedAt, outcome: isActive() ? "ok" : "closed" });
|
||||
return isActive();
|
||||
};
|
||||
|
||||
realtimeConnection = {
|
||||
close(fields = {}) {
|
||||
if (!isActive()) return false;
|
||||
const reason = textValue(fields.reason) || "server_shutdown";
|
||||
realtimeCloseReason = reason;
|
||||
realtimeCloseSignal = textValue(fields.signal) || null;
|
||||
void writeEvent("workbench.server_draining", {
|
||||
type: "server.draining",
|
||||
status: "closing",
|
||||
reason,
|
||||
signal: realtimeCloseSignal,
|
||||
sessionId: realtimeSessionId,
|
||||
threadId: realtimeThreadId,
|
||||
traceId: realtimeTraceId,
|
||||
observedAt: new Date().toISOString()
|
||||
}).finally(() => {
|
||||
if (!response.writableEnded) response.end();
|
||||
});
|
||||
return true;
|
||||
},
|
||||
isActive
|
||||
};
|
||||
activeWorkbenchRealtimeConnections.add(realtimeConnection);
|
||||
cleanup.push(() => activeWorkbenchRealtimeConnections.delete(realtimeConnection));
|
||||
|
||||
await writeEvent("workbench.connected", {
|
||||
type: "connected",
|
||||
status: "connected",
|
||||
realtimeSource: "projection-outbox",
|
||||
snapshotSource: "atomic-projection-sync",
|
||||
heartbeatMs,
|
||||
cursor: { outboxSeq: requestedAfterSeq },
|
||||
filters: { sessionId: requestedSessionId, traceId: requestedTraceId }
|
||||
});
|
||||
|
||||
realtimeSessionId = streamSessionId ?? requestedSessionId ?? null;
|
||||
realtimeTraceId = activeTraceId ?? requestedTraceId ?? null;
|
||||
realtimeThreadId = streamThreadId ?? null;
|
||||
attachWorkbenchRealtimeOtelContext(request, {
|
||||
sessionId: streamSessionId ?? requestedSessionId,
|
||||
traceId: activeTraceId,
|
||||
threadId: streamThreadId,
|
||||
heartbeatMs,
|
||||
realtimeSource: "projection-outbox"
|
||||
});
|
||||
emitWorkbenchRealtimeAcceptedOtelSpan(request, options.env);
|
||||
|
||||
const syncParams = (extra = {}) => ({
|
||||
sessionId: requestedSessionId,
|
||||
traceId: requestedTraceId,
|
||||
afterOutboxSeq: outboxCursor,
|
||||
afterSeq: outboxCursor,
|
||||
limit: outboxTailBatchSize,
|
||||
actor: { id: auth.actor?.id, role: auth.actor?.role ?? "user" },
|
||||
...extra
|
||||
});
|
||||
const updateStreamScope = (snapshot) => {
|
||||
const session = factArray(snapshot?.facts?.sessions)[0] ?? null;
|
||||
streamSessionId = textValue(session?.sessionId) || requestedSessionId || streamSessionId;
|
||||
streamThreadId = safeOpaqueId(session?.threadId) || textValue(session?.threadId) || streamThreadId;
|
||||
activeTraceId = requestedTraceId || textValue(session?.lastTraceId) || activeTraceId;
|
||||
realtimeSessionId = streamSessionId;
|
||||
realtimeThreadId = streamThreadId;
|
||||
realtimeTraceId = activeTraceId;
|
||||
};
|
||||
const emitProjectionSnapshot = async (snapshot) => {
|
||||
updateStreamScope(snapshot);
|
||||
for (const item of projectionOutboxRealtimeEvents(snapshot, { includeSnapshot: true, includeEvents: false })) {
|
||||
if (!await writeEvent(item.name, item.payload)) break;
|
||||
}
|
||||
outboxCursor = nonNegativeInteger(snapshot?.cutoffOutboxSeq ?? snapshot?.cursorOutboxSeq);
|
||||
};
|
||||
const recoverCursor = async () => {
|
||||
const snapshot = await runtime.readAtomicWorkbenchProjectionSync(syncParams({ afterOutboxSeq: 0, afterSeq: 0, snapshotOnly: true, deltaOnly: false }));
|
||||
await emitProjectionSnapshot(snapshot);
|
||||
};
|
||||
const runProjectionScan = async ({ initial = false } = {}) => {
|
||||
const snapshotOnly = initial && requestedAfterSeq <= 0;
|
||||
if (snapshotOnly) {
|
||||
await recoverCursor();
|
||||
return;
|
||||
}
|
||||
let hasMore = true;
|
||||
while (hasMore && isActive()) {
|
||||
const snapshot = await runtime.readAtomicWorkbenchProjectionSync(syncParams({ snapshotOnly: false, deltaOnly: true }));
|
||||
const nextCursor = nonNegativeInteger(snapshot?.cursorOutboxSeq);
|
||||
if (nextCursor < outboxCursor) {
|
||||
await recoverCursor();
|
||||
return;
|
||||
}
|
||||
for (const item of projectionOutboxRealtimeEvents(snapshot)) {
|
||||
if (!await writeEvent(item.name, item.payload)) return;
|
||||
}
|
||||
outboxCursor = nextCursor;
|
||||
hasMore = snapshot?.hasMore === true;
|
||||
}
|
||||
};
|
||||
const emitProjectionScanError = async (error) => {
|
||||
await writeEvent("workbench.error", { type: "error", realtimeSource: "projection-outbox", sessionId: streamSessionId, traceId: activeTraceId, error: { code: error?.code ?? "workbench_outbox_scan_failed", message: error?.message ?? "Workbench projection outbox scan failed.", valuesRedacted: true } });
|
||||
};
|
||||
const scanProjectionOutbox = async () => {
|
||||
if (!isActive()) return;
|
||||
if (initialScanState !== "complete") {
|
||||
if (initialScanState === "pending") initialScanWakeRequested = true;
|
||||
return;
|
||||
}
|
||||
if (scanRunning) {
|
||||
scanRequested = true;
|
||||
return;
|
||||
}
|
||||
scanRunning = true;
|
||||
try {
|
||||
do {
|
||||
scanRequested = false;
|
||||
await runProjectionScan({ initial: false });
|
||||
} while (scanRequested && isActive());
|
||||
} catch (error) {
|
||||
await emitProjectionScanError(error);
|
||||
if (!response.writableEnded) response.end();
|
||||
} finally {
|
||||
scanRunning = false;
|
||||
}
|
||||
};
|
||||
const requestProjectionScan = () => {
|
||||
if (!isActive()) return;
|
||||
if (initialScanState !== "complete") {
|
||||
if (initialScanState === "pending") initialScanWakeRequested = true;
|
||||
return;
|
||||
}
|
||||
scanRequested = true;
|
||||
void scanProjectionOutbox();
|
||||
};
|
||||
const unsubscribeProjectionCommits = options.kafkaEventBridge?.subscribeProjectionCommits?.((signal = {}) => {
|
||||
if (workbenchProjectionSignalMatches(signal, requestedSessionId, requestedTraceId)) requestProjectionScan();
|
||||
});
|
||||
if (typeof unsubscribeProjectionCommits === "function") cleanup.push(unsubscribeProjectionCommits);
|
||||
|
||||
try {
|
||||
await runProjectionScan({ initial: true });
|
||||
initialScanState = "complete";
|
||||
} catch (error) {
|
||||
initialScanState = "failed";
|
||||
await emitProjectionScanError(error);
|
||||
if (!response.writableEnded) response.end();
|
||||
return;
|
||||
}
|
||||
if (initialScanState === "complete" && initialScanWakeRequested && isActive()) {
|
||||
initialScanWakeRequested = false;
|
||||
scanRequested = true;
|
||||
await scanProjectionOutbox();
|
||||
}
|
||||
if (!isActive()) return;
|
||||
|
||||
const heartbeat = setInterval(() => {
|
||||
void writeEvent("workbench.heartbeat", {
|
||||
type: "heartbeat",
|
||||
sessionId: requestedSessionId ?? null,
|
||||
traceId: activeTraceId ?? null,
|
||||
observedAt: new Date().toISOString()
|
||||
});
|
||||
}, Math.max(1000, heartbeatMs));
|
||||
heartbeat.unref?.();
|
||||
cleanup.push(() => clearInterval(heartbeat));
|
||||
} catch (error) {
|
||||
handleWorkbenchRealtimeFailure(request, response, url, options, error);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleLiveKafkaWorkbenchRealtimeHttp(request, response, url, options = {}) {
|
||||
const requestedSessionId = safeSessionId(url.searchParams.get("sessionId") ?? url.searchParams.get("includeSessionId"));
|
||||
const requestedTraceId = safeTraceId(url.searchParams.get("traceId"));
|
||||
const realtimeCapabilities = workbenchRealtimeCapabilities(options.env ?? process.env);
|
||||
const heartbeatMs = parsePositiveInteger(options.env?.HWLAB_WORKBENCH_SSE_HEARTBEAT_MS, DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS);
|
||||
attachWorkbenchRealtimeOtelContext(request, {
|
||||
sessionId: requestedSessionId,
|
||||
traceId: requestedTraceId,
|
||||
heartbeatMs,
|
||||
realtimeSource: "live-kafka-sse",
|
||||
realtimeCapabilities
|
||||
});
|
||||
const perf = options.backendPerformance;
|
||||
const auth = perf ? await perf.measure("workbench_auth", () => authenticateWorkbenchRead(request, response, options)) : await authenticateWorkbenchRead(request, response, options);
|
||||
if (!auth) return;
|
||||
if (url.searchParams.has("projectId") || url.searchParams.has("workspaceId")) {
|
||||
sendJson(response, 400, workbenchError("workbench_authority_removed", "Workbench realtime is keyed by sessionId/traceId only."));
|
||||
return;
|
||||
}
|
||||
if (!requestedSessionId && !requestedTraceId) {
|
||||
sendJson(response, 400, workbenchError("workbench_realtime_scope_required", "Workbench realtime requires sessionId or traceId."));
|
||||
return;
|
||||
}
|
||||
const authorization = await authorizeLiveKafkaWorkbenchRealtimeScope(options, auth.actor, {
|
||||
sessionId: requestedSessionId,
|
||||
traceId: requestedTraceId
|
||||
});
|
||||
if (!authorization.ok) {
|
||||
sendJson(response, authorization.status, authorization.body);
|
||||
return;
|
||||
}
|
||||
const bridge = options.kafkaEventBridge;
|
||||
if (bridge?.capabilities?.liveKafkaSse !== true || typeof bridge?.subscribeLiveHwlabEvents !== "function") {
|
||||
sendJson(response, 503, workbenchError("workbench_live_kafka_unconfigured", "Workbench live Kafka fanout is not configured."));
|
||||
return;
|
||||
}
|
||||
|
||||
let closed = false;
|
||||
let realtimeCloseReason = "client_close";
|
||||
let realtimeCloseSignal = null;
|
||||
const realtimeStartedAtMs = Date.now();
|
||||
const cleanup = [];
|
||||
const isActive = () => !closed && !response.destroyed && !response.writableEnded;
|
||||
const closeConnection = () => {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
emitWorkbenchRealtimeClosedOtelSpan(request, options.env, {
|
||||
reason: realtimeCloseReason,
|
||||
signal: realtimeCloseSignal,
|
||||
startedAtMs: realtimeStartedAtMs,
|
||||
sessionId: requestedSessionId,
|
||||
threadId: null,
|
||||
traceId: requestedTraceId,
|
||||
activeConnectionCount: activeWorkbenchRealtimeConnections.size
|
||||
});
|
||||
for (const item of cleanup.splice(0)) item();
|
||||
};
|
||||
|
||||
response.once("close", closeConnection);
|
||||
request.once?.("aborted", closeConnection);
|
||||
request.socket?.once?.("close", closeConnection);
|
||||
cleanup.push(() => request.off?.("aborted", closeConnection));
|
||||
cleanup.push(() => request.socket?.off?.("close", closeConnection));
|
||||
|
||||
const bufferedEnvelopes = [];
|
||||
let liveDeliveryStarted = false;
|
||||
let enqueueEvent = null;
|
||||
const unsubscribe = bridge.subscribeLiveHwlabEvents((envelope, transport) => {
|
||||
if (!liveKafkaEnvelopeMatches(envelope, requestedSessionId, requestedTraceId)) return;
|
||||
if (!liveDeliveryStarted || typeof enqueueEvent !== "function") {
|
||||
bufferedEnvelopes.push({ envelope, transport });
|
||||
return;
|
||||
}
|
||||
void enqueueEvent("hwlab.event.v1", envelope, transport);
|
||||
});
|
||||
if (typeof unsubscribe === "function") cleanup.push(unsubscribe);
|
||||
|
||||
try {
|
||||
await bridge.ready;
|
||||
} catch (error) {
|
||||
closeConnection();
|
||||
throw error;
|
||||
}
|
||||
if (!isActive()) {
|
||||
closeConnection();
|
||||
return;
|
||||
}
|
||||
|
||||
response.writeHead(200, {
|
||||
"content-type": "text/event-stream; charset=utf-8",
|
||||
"cache-control": "no-store, no-transform",
|
||||
@@ -219,174 +549,129 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option
|
||||
});
|
||||
if (typeof response.flushHeaders === "function") response.flushHeaders();
|
||||
|
||||
const writeEvent = (name, payload = {}) => {
|
||||
if (closed || response.destroyed) return;
|
||||
const startedAt = nowMs();
|
||||
const eventCreatedAt = realtimeEventCreatedAt(payload);
|
||||
const traceSeq = realtimeTraceSeq(payload);
|
||||
const eventId = realtimeEventId(payload);
|
||||
response.write(`event: ${name}\n`);
|
||||
if (eventId) response.write(`id: ${eventId}\n`);
|
||||
response.write(`data: ${JSON.stringify({ contractVersion: "workbench-events-v1", serverSentAt: new Date().toISOString(), eventCreatedAt, traceSeq, ...payload })}\n\n`);
|
||||
perf?.recordPhase({ phase: "sse_write", durationMs: nowMs() - startedAt, outcome: "ok" });
|
||||
const writeEvent = async (name, payload, transport = null) => {
|
||||
if (!isActive()) return false;
|
||||
const block = `event: ${name}\ndata: ${JSON.stringify(payload)}\n\n`;
|
||||
const writable = response.write(block);
|
||||
if (!writable) await waitForResponseDrain(response, isActive);
|
||||
const active = isActive();
|
||||
if (active && name === "hwlab.event.v1") {
|
||||
emitLiveKafkaOtelSpan("hwlab.workbench.live_sse.business_event_write", payload, transport, {
|
||||
env: options.env,
|
||||
otelSpanEmitter: options.otelSpanEmitter ?? emitCodeAgentOtelSpan
|
||||
});
|
||||
}
|
||||
return active;
|
||||
};
|
||||
let writeChain = Promise.resolve(true);
|
||||
enqueueEvent = (name, payload, transport = null) => {
|
||||
writeChain = writeChain.then(() => writeEvent(name, payload, transport));
|
||||
return writeChain;
|
||||
};
|
||||
|
||||
const realtimeConnection = {
|
||||
close(fields = {}) {
|
||||
if (closed || response.destroyed || response.writableEnded) return false;
|
||||
const reason = textValue(fields.reason) || "server_shutdown";
|
||||
realtimeCloseReason = reason;
|
||||
if (!isActive()) return false;
|
||||
realtimeCloseReason = textValue(fields.reason) || "server_shutdown";
|
||||
realtimeCloseSignal = textValue(fields.signal) || null;
|
||||
try {
|
||||
writeEvent("workbench.server_draining", {
|
||||
type: "server.draining",
|
||||
status: "closing",
|
||||
reason,
|
||||
signal: realtimeCloseSignal,
|
||||
sessionId: realtimeSessionId,
|
||||
threadId: realtimeThreadId,
|
||||
traceId: realtimeTraceId,
|
||||
observedAt: new Date().toISOString()
|
||||
});
|
||||
response.end();
|
||||
return true;
|
||||
} catch {
|
||||
try { response.destroy(); } catch {}
|
||||
return false;
|
||||
}
|
||||
void enqueueEvent("workbench.server_draining", {
|
||||
type: "server.draining",
|
||||
status: "closing",
|
||||
reason: realtimeCloseReason,
|
||||
signal: realtimeCloseSignal,
|
||||
capabilities: realtimeCapabilities,
|
||||
lossPossible: true
|
||||
}).finally(() => {
|
||||
if (!response.writableEnded) response.end();
|
||||
});
|
||||
return true;
|
||||
},
|
||||
isActive() {
|
||||
return !closed && !response.destroyed && !response.writableEnded;
|
||||
}
|
||||
isActive
|
||||
};
|
||||
activeWorkbenchRealtimeConnections.add(realtimeConnection);
|
||||
cleanup.push(() => activeWorkbenchRealtimeConnections.delete(realtimeConnection));
|
||||
|
||||
writeEvent("workbench.connected", {
|
||||
await enqueueEvent("workbench.connected", {
|
||||
type: "connected",
|
||||
status: "connected",
|
||||
realtimeSource: "kafka",
|
||||
snapshotSource: "read-model",
|
||||
heartbeatMs,
|
||||
cursor: { outboxSeq: requestedAfterSeq },
|
||||
filters: {
|
||||
sessionId: requestedSessionId,
|
||||
traceId: requestedTraceId
|
||||
}
|
||||
capabilities: realtimeCapabilities,
|
||||
realtimeSource: "hwlab.event.v1",
|
||||
deliverySemantics: "live-only",
|
||||
liveOnly: true,
|
||||
replay: false,
|
||||
replaySupported: false,
|
||||
lossPossible: true,
|
||||
filters: { sessionId: requestedSessionId, traceId: requestedTraceId }
|
||||
});
|
||||
|
||||
const resolveInitialSession = requestedSessionId && requestedAfterSeq <= 0;
|
||||
const initialContext = resolveInitialSession
|
||||
? await realtimeInitialSessionContext(readModel, auth.actor, requestedSessionId)
|
||||
: { facts: emptyFactSet(), session: null, blocker: null };
|
||||
if (initialContext.blocker) {
|
||||
writeEvent("workbench.error", {
|
||||
type: "error",
|
||||
reason: "initial-session",
|
||||
sessionId: requestedSessionId,
|
||||
traceId: requestedTraceId,
|
||||
error: initialContext.blocker
|
||||
});
|
||||
liveDeliveryStarted = true;
|
||||
const readyBufferedEnvelopes = bufferedEnvelopes.splice(0);
|
||||
for (const { envelope, transport } of readyBufferedEnvelopes) void enqueueEvent("hwlab.event.v1", envelope, transport);
|
||||
await writeChain;
|
||||
if (!isActive()) {
|
||||
closeConnection();
|
||||
return;
|
||||
}
|
||||
const initialFactSession = initialContext.session;
|
||||
const streamSessionId = factSessionId(initialFactSession) ?? requestedSessionId ?? null;
|
||||
const streamThreadId = safeOpaqueId(initialFactSession?.threadId) ?? (textValue(initialFactSession?.threadId) || null);
|
||||
const activeTraceId = requestedTraceId
|
||||
?? factLastTraceId(initialFactSession)
|
||||
?? factLatestTraceIdForSession(initialContext.facts, streamSessionId)
|
||||
?? null;
|
||||
realtimeSessionId = streamSessionId ?? requestedSessionId ?? null;
|
||||
realtimeTraceId = activeTraceId ?? requestedTraceId ?? null;
|
||||
realtimeThreadId = streamThreadId ?? null;
|
||||
attachWorkbenchRealtimeOtelContext(request, {
|
||||
sessionId: streamSessionId ?? requestedSessionId,
|
||||
traceId: activeTraceId,
|
||||
threadId: streamThreadId,
|
||||
heartbeatMs,
|
||||
realtimeSource: "kafka"
|
||||
});
|
||||
const heartbeatTimer = setInterval(() => {
|
||||
void enqueueEvent("workbench.heartbeat", {
|
||||
type: "heartbeat",
|
||||
status: "connected",
|
||||
realtimeSource: "hwlab.event.v1",
|
||||
liveOnly: true,
|
||||
replay: false,
|
||||
lossPossible: true,
|
||||
serverSentAt: new Date().toISOString(),
|
||||
valuesPrinted: false
|
||||
});
|
||||
}, heartbeatMs);
|
||||
heartbeatTimer.unref?.();
|
||||
cleanup.push(() => clearInterval(heartbeatTimer));
|
||||
emitWorkbenchRealtimeAcceptedOtelSpan(request, options.env);
|
||||
if (activeTraceId && requestedAfterSeq <= 0) {
|
||||
await (perf ? perf.measure("workbench_initial_trace", () => writeTraceRealtimeSnapshotSafe({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" })) : writeTraceRealtimeSnapshotSafe({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" }));
|
||||
}
|
||||
const kafkaFilters = resolveWorkbenchRealtimeKafkaFilters({ traceId: requestedTraceId, sessionId: streamSessionId, traceStore });
|
||||
try {
|
||||
const kafkaStream = await openKafkaEventStream({
|
||||
env: options.env ?? process.env,
|
||||
stream: "hwlab",
|
||||
fromBeginning: false,
|
||||
...kafkaFilters,
|
||||
kafkaFactory: options.kafkaFactory,
|
||||
onEvent: async (record) => {
|
||||
if (closed || response.destroyed) return;
|
||||
const realtime = workbenchRealtimeEventFromKafka(record, { sessionId: streamSessionId, threadId: streamThreadId, traceId: activeTraceId });
|
||||
if (!realtime) return;
|
||||
writeEvent("workbench.trace.event", realtime.traceEvent);
|
||||
if (realtime.turnSnapshot) writeEvent("workbench.turn.snapshot", realtime.turnSnapshot);
|
||||
},
|
||||
onError: (error) => {
|
||||
writeEvent("workbench.error", {
|
||||
type: "error",
|
||||
realtimeSource: "kafka",
|
||||
sessionId: streamSessionId,
|
||||
threadId: streamThreadId,
|
||||
traceId: activeTraceId,
|
||||
error: { code: "workbench_kafka_stream_failed", message: error?.message ?? "Workbench Kafka realtime stream failed." }
|
||||
});
|
||||
}
|
||||
});
|
||||
cleanup.push(() => { void kafkaStream.stop?.(); });
|
||||
} catch (kafkaError) {
|
||||
writeEvent("workbench.error", {
|
||||
type: "error",
|
||||
realtimeSource: "kafka",
|
||||
sessionId: streamSessionId,
|
||||
threadId: streamThreadId,
|
||||
traceId: activeTraceId,
|
||||
error: { code: "workbench_kafka_stream_unavailable", message: kafkaError?.message ?? "Workbench Kafka realtime stream is unavailable." }
|
||||
});
|
||||
}
|
||||
|
||||
async function authorizeLiveKafkaWorkbenchRealtimeScope(options, actor, { sessionId, traceId }) {
|
||||
const access = options.accessController;
|
||||
if ((sessionId && typeof access?.getAgentSession !== "function") || (traceId && typeof access?.getAgentSessionByTraceId !== "function")) {
|
||||
return {
|
||||
ok: false,
|
||||
status: 503,
|
||||
body: workbenchError("workbench_realtime_authorization_unconfigured", "Workbench live realtime ownership authorization is not configured.")
|
||||
};
|
||||
}
|
||||
|
||||
const heartbeat = setInterval(async () => {
|
||||
try {
|
||||
writeEvent("workbench.heartbeat", {
|
||||
type: "heartbeat",
|
||||
sessionId: requestedSessionId ?? null,
|
||||
traceId: activeTraceId ?? null,
|
||||
observedAt: new Date().toISOString()
|
||||
});
|
||||
} catch (error) {
|
||||
writeEvent("workbench.error", {
|
||||
type: "error",
|
||||
error: { code: "workbench_realtime_heartbeat_failed", message: error?.message ?? "Workbench realtime heartbeat failed." }
|
||||
});
|
||||
}
|
||||
}, Math.max(1000, heartbeatMs));
|
||||
cleanup.push(() => clearInterval(heartbeat));
|
||||
|
||||
response.on("close", () => {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
emitWorkbenchRealtimeClosedOtelSpan(request, options.env, {
|
||||
reason: realtimeCloseReason,
|
||||
signal: realtimeCloseSignal,
|
||||
startedAtMs: realtimeStartedAtMs,
|
||||
sessionId: realtimeSessionId,
|
||||
threadId: realtimeThreadId,
|
||||
traceId: realtimeTraceId,
|
||||
activeConnectionCount: activeWorkbenchRealtimeConnections.size
|
||||
});
|
||||
for (const item of cleanup.splice(0)) item();
|
||||
});
|
||||
} catch (error) {
|
||||
handleWorkbenchRealtimeFailure(request, response, url, options, error);
|
||||
const [sessionById, sessionByTrace] = await Promise.all([
|
||||
sessionId ? access.getAgentSession(sessionId) : null,
|
||||
traceId ? access.getAgentSessionByTraceId(traceId) : null
|
||||
]);
|
||||
const sessionByIdId = sessionById ? safeSessionId(sessionById.id ?? sessionById.sessionId) : null;
|
||||
const sessionByTraceId = sessionByTrace ? safeSessionId(sessionByTrace.id ?? sessionByTrace.sessionId) : null;
|
||||
const scopeMissing = (sessionId && sessionByIdId !== sessionId)
|
||||
|| (traceId && !sessionByTraceId)
|
||||
|| (sessionId && traceId && sessionByIdId !== sessionByTraceId);
|
||||
const resolvedSessions = [sessionById, sessionByTrace].filter(Boolean);
|
||||
const ownedByActor = actor?.role === "admin" || (Boolean(actor?.id)
|
||||
&& resolvedSessions.length > 0
|
||||
&& resolvedSessions.every((session) => textValue(session.ownerUserId) === actor.id));
|
||||
if (scopeMissing || !ownedByActor) {
|
||||
return {
|
||||
ok: false,
|
||||
status: 404,
|
||||
body: workbenchError("workbench_realtime_scope_not_found", "Workbench realtime scope is not visible to the current actor.")
|
||||
};
|
||||
}
|
||||
return { ok: true, sessionId: sessionByIdId ?? sessionByTraceId };
|
||||
}
|
||||
|
||||
function liveKafkaEnvelopeMatches(envelope, sessionId, traceId) {
|
||||
if (!envelope || envelope.schema !== "hwlab.event.v1") return false;
|
||||
if (sessionId && safeSessionId(envelope.hwlabSessionId ?? envelope.sessionId) !== sessionId) return false;
|
||||
if (traceId && safeTraceId(envelope.traceId) !== traceId) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
export function attachWorkbenchRealtimeOtelContext(request, fields = {}) {
|
||||
const context = request?.hwlabHttpRequestContext;
|
||||
if (!context) return;
|
||||
context.route = "/v1/workbench/events";
|
||||
context.route = textValue(fields.route) || "/v1/workbench/events";
|
||||
context.otelAttributes = {
|
||||
...(context.otelAttributes ?? {}),
|
||||
"workbench.route": "events",
|
||||
@@ -396,151 +681,13 @@ export function attachWorkbenchRealtimeOtelContext(request, fields = {}) {
|
||||
"workbench.thread_id": fields.threadId ?? null,
|
||||
"workbench.sse.heartbeat_ms": fields.heartbeatMs ?? null,
|
||||
"workbench.sse.realtime_source": fields.realtimeSource ?? null,
|
||||
"workbench.sse.outbox_mode": false
|
||||
"workbench.sse.live_kafka_enabled": fields.realtimeCapabilities?.liveKafkaSse ?? null,
|
||||
"workbench.sse.projection_realtime_enabled": fields.realtimeCapabilities?.projectionRealtime ?? null,
|
||||
"workbench.sse.outbox_mode": fields.realtimeCapabilities?.liveKafkaSse === true ? false : true
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveWorkbenchRealtimeKafkaFilters({ traceId = null, sessionId = null, traceStore = null } = {}) {
|
||||
const resolved = traceId && typeof traceStore?.snapshot === "function" ? collectTraceLinkedIds(traceStore.snapshot(traceId)) : {};
|
||||
const hasAgentRunKey = Boolean(resolved.runId || resolved.commandId);
|
||||
const fallbackSessionId = resolved.sessionId || agentRunScopedSessionIdFromWorkbenchSession(sessionId) || sessionId;
|
||||
return compactObject({
|
||||
traceId: hasAgentRunKey ? null : traceId,
|
||||
sessionId: hasAgentRunKey ? null : fallbackSessionId,
|
||||
runId: resolved.runId,
|
||||
commandId: resolved.commandId
|
||||
});
|
||||
}
|
||||
|
||||
export function agentRunScopedSessionIdFromWorkbenchSession(sessionId) {
|
||||
const safeId = safeSessionId(sessionId);
|
||||
if (!safeId || isAgentRunAliasSessionId(safeId)) return safeId;
|
||||
const base = safeId.replace(/^ses_/u, "").replace(/[^A-Za-z0-9_]+/gu, "_").replace(/^_+|_+$/gu, "") || "session";
|
||||
return safeSessionId(`ses_agentrun_${base}`);
|
||||
}
|
||||
|
||||
export function collectTraceLinkedIds(snapshot) {
|
||||
const out = { sessionId: null, runId: null, commandId: null };
|
||||
const events = Array.isArray(snapshot?.events) ? snapshot.events : [];
|
||||
for (const event of events) collectTraceLinkedIdsFromRecord(event, out);
|
||||
collectTraceLinkedIdsFromRecord(snapshot?.lastEvent, out);
|
||||
return out;
|
||||
}
|
||||
|
||||
export function collectTraceLinkedIdsFromRecord(record, out) {
|
||||
const value = record && typeof record === "object" && !Array.isArray(record) ? record : {};
|
||||
const agentRun = value.agentRun && typeof value.agentRun === "object" && !Array.isArray(value.agentRun) ? value.agentRun : {};
|
||||
const payload = value.payload && typeof value.payload === "object" && !Array.isArray(value.payload) ? value.payload : {};
|
||||
out.sessionId ||= safeSessionId(value.sessionId ?? value.sourceSessionId ?? agentRun.sessionId ?? payload.sessionId);
|
||||
out.runId ||= textValue(value.runId ?? value.sourceRunId ?? agentRun.runId ?? payload.runId);
|
||||
out.commandId ||= textValue(value.commandId ?? value.sourceCommandId ?? agentRun.commandId ?? payload.commandId);
|
||||
}
|
||||
|
||||
export function workbenchRealtimeEventFromKafka(record, context = {}) {
|
||||
const value = record?.value && typeof record.value === "object" && !Array.isArray(record.value) ? record.value : null;
|
||||
if (!value) return null;
|
||||
const hwlabEvent = value.event && typeof value.event === "object" && !Array.isArray(value.event) ? value.event : {};
|
||||
const sourceContext = value.context && typeof value.context === "object" && !Array.isArray(value.context) ? value.context : {};
|
||||
const traceId = safeTraceId(value.traceId ?? hwlabEvent.traceId ?? context.traceId) ?? textValue(value.traceId ?? hwlabEvent.traceId ?? context.traceId);
|
||||
const sessionId = safeSessionId(context.sessionId ?? value.sessionId ?? hwlabEvent.sessionId) ?? textValue(context.sessionId ?? value.sessionId ?? hwlabEvent.sessionId);
|
||||
const threadId = safeOpaqueId(context.threadId ?? sourceContext.threadId) ?? textValue(context.threadId ?? sourceContext.threadId);
|
||||
if (!traceId && !sessionId) return null;
|
||||
const sourceSeq = integerValue(hwlabEvent.sourceSeq ?? sourceContext.sourceSeq);
|
||||
const kafka = {
|
||||
topic: textValue(record.topic),
|
||||
partition: integerValue(record.partition),
|
||||
offset: textValue(record.offset),
|
||||
key: textValue(record.key),
|
||||
timestamp: textValue(record.timestamp),
|
||||
valuesRedacted: true
|
||||
};
|
||||
const event = {
|
||||
...hwlabEvent,
|
||||
traceId: traceId ?? hwlabEvent.traceId ?? null,
|
||||
sessionId: sessionId ?? hwlabEvent.sessionId ?? null,
|
||||
threadId: threadId ?? sourceContext.threadId ?? null,
|
||||
runId: textValue(hwlabEvent.runId ?? sourceContext.runId),
|
||||
commandId: textValue(hwlabEvent.commandId ?? sourceContext.commandId),
|
||||
source: textValue(hwlabEvent.source) || "hwlab.kafka",
|
||||
sourceSeq,
|
||||
kafka,
|
||||
valuesPrinted: false
|
||||
};
|
||||
const status = normalizeStatus(event.status);
|
||||
const terminal = event.terminal === true || TERMINAL_STATUSES.has(status);
|
||||
const cursor = {
|
||||
traceSeq: sourceSeq ?? integerValue(record.offset),
|
||||
kafkaOffset: textValue(record.offset),
|
||||
kafkaPartition: integerValue(record.partition)
|
||||
};
|
||||
const entityVersion = sourceSeq ?? integerValue(record.offset) ?? 0;
|
||||
const projectionRevision = ["kafka", kafka.topic, kafka.partition, kafka.offset].filter((part) => textValue(part)).join(":");
|
||||
const traceEntity = {
|
||||
family: "traceEvents",
|
||||
id: [traceId ?? "trace", event.runId, event.commandId, kafka.partition, kafka.offset].filter((part) => textValue(part)).join(":"),
|
||||
version: entityVersion,
|
||||
traceSeq: cursor.traceSeq,
|
||||
projectionRevision,
|
||||
authority: WORKBENCH_REALTIME_AUTHORITY_VERSION
|
||||
};
|
||||
const traceEvent = {
|
||||
id: kafka.topic && kafka.partition !== null && kafka.offset ? `${kafka.topic}:${kafka.partition}:${kafka.offset}` : null,
|
||||
type: "trace.event",
|
||||
contractVersion: "workbench-sync-v1",
|
||||
realtimeAuthority: WORKBENCH_REALTIME_AUTHORITY_VERSION,
|
||||
realtimeSource: "kafka",
|
||||
sessionId,
|
||||
threadId,
|
||||
traceId,
|
||||
event,
|
||||
snapshot: { traceId, sessionId, threadId, status: status || event.status || null, events: [event], eventCount: 1 },
|
||||
cursor,
|
||||
entity: traceEntity,
|
||||
projectionRevision,
|
||||
kafka,
|
||||
valuesPrinted: false
|
||||
};
|
||||
const turnSnapshot = terminal ? {
|
||||
type: "turn.snapshot",
|
||||
contractVersion: "workbench-sync-v1",
|
||||
realtimeAuthority: WORKBENCH_REALTIME_AUTHORITY_VERSION,
|
||||
realtimeSource: "kafka",
|
||||
sessionId,
|
||||
threadId,
|
||||
traceId,
|
||||
reason: "kafka-terminal",
|
||||
cursor,
|
||||
turn: {
|
||||
traceId,
|
||||
sessionId,
|
||||
threadId,
|
||||
status: status || event.status || "completed",
|
||||
running: false,
|
||||
terminal: true,
|
||||
finalResponse: terminalFinalResponse(status || event.status || "completed", {
|
||||
traceId,
|
||||
status: status || event.status || "completed",
|
||||
finalResponse: event.text ? { text: event.text, status: status || event.status || "completed", traceId, source: "kafka-terminal-event", valuesPrinted: false } : null,
|
||||
finalText: event.text ?? event.message ?? null,
|
||||
error: event.errorCode ? { message: event.message ?? event.errorCode } : null
|
||||
}),
|
||||
agentRun: compactObject({ runId: event.runId, commandId: event.commandId }),
|
||||
valuesPrinted: false
|
||||
},
|
||||
entity: {
|
||||
family: "turns",
|
||||
id: traceId ?? event.runId ?? event.commandId ?? traceEntity.id,
|
||||
version: entityVersion,
|
||||
traceSeq: cursor.traceSeq,
|
||||
projectionRevision,
|
||||
authority: WORKBENCH_REALTIME_AUTHORITY_VERSION
|
||||
},
|
||||
projectionRevision,
|
||||
kafka,
|
||||
valuesPrinted: false
|
||||
} : null;
|
||||
return { traceEvent, turnSnapshot };
|
||||
}
|
||||
export { projectionOutboxRealtimeEvents } from "./workbench-projection-outbox-events.ts";
|
||||
|
||||
export function compactObject(value) {
|
||||
return Object.fromEntries(Object.entries(value || {}).filter(([, entry]) => textValue(entry)));
|
||||
@@ -587,7 +734,7 @@ export function emitWorkbenchRealtimeAcceptedOtelSpan(request, env = process.env
|
||||
endTimeMs: endedAtMs,
|
||||
statusCode: 200,
|
||||
method: "GET",
|
||||
route: "/v1/workbench/events",
|
||||
route: context.route ?? "/v1/workbench/events",
|
||||
attributes: {
|
||||
...(context.otelAttributes ?? {}),
|
||||
"http.closed_by": "sse-accepted",
|
||||
@@ -608,7 +755,7 @@ export function emitWorkbenchRealtimeClosedOtelSpan(request, env = process.env,
|
||||
endTimeMs: endedAtMs,
|
||||
statusCode: 200,
|
||||
method: "GET",
|
||||
route: "/v1/workbench/events",
|
||||
route: context.route ?? "/v1/workbench/events",
|
||||
attributes: {
|
||||
...(context.otelAttributes ?? {}),
|
||||
"http.closed_by": reason,
|
||||
@@ -713,13 +860,49 @@ export function realtimeEventId(payload) {
|
||||
}
|
||||
|
||||
export function workbenchRealtimeAfterSeq(request, url) {
|
||||
for (const value of [url.searchParams.get("afterSeq"), url.searchParams.get("afterOutboxSeq"), request?.headers?.["last-event-id"]]) {
|
||||
for (const value of [request?.headers?.["last-event-id"], url.searchParams.get("afterSeq"), url.searchParams.get("afterOutboxSeq")]) {
|
||||
const parsed = nonNegativeInteger(value, 0);
|
||||
if (parsed > 0) return parsed;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function requiredPositiveRuntimeSetting(env, name) {
|
||||
const value = Number(env?.[name]);
|
||||
if (!Number.isInteger(value) || value <= 0) {
|
||||
const error = new Error(`${name} must be explicitly configured as a positive integer.`);
|
||||
error.code = "workbench_outbox_tail_config_invalid";
|
||||
throw error;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function workbenchProjectionSignalMatches(signal = {}, sessionId = null, traceId = null) {
|
||||
if (signal.recovery === true) return true;
|
||||
const signalSessionId = textValue(signal.sessionId);
|
||||
const signalTraceId = textValue(signal.traceId);
|
||||
if (traceId && signalTraceId !== traceId) return false;
|
||||
if (sessionId && signalSessionId !== sessionId) return false;
|
||||
return Boolean(signalSessionId || signalTraceId);
|
||||
}
|
||||
|
||||
export function waitForResponseDrain(response, isActive = () => true) {
|
||||
if (!isActive()) return Promise.resolve(false);
|
||||
return new Promise((resolve) => {
|
||||
const settle = (result) => {
|
||||
response.off?.("drain", onDrain);
|
||||
response.off?.("close", onClose);
|
||||
response.off?.("error", onClose);
|
||||
resolve(result);
|
||||
};
|
||||
const onDrain = () => settle(isActive());
|
||||
const onClose = () => settle(false);
|
||||
response.once("drain", onDrain);
|
||||
response.once("close", onClose);
|
||||
response.once("error", onClose);
|
||||
});
|
||||
}
|
||||
|
||||
export function nowMs() {
|
||||
if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now();
|
||||
return Date.now();
|
||||
|
||||
+40
-511
@@ -3,8 +3,8 @@
|
||||
* 职责: Cloud API REST/SSE route dispatcher。Workbench 新资源路由应委托 read model / compat wrapper,不在 dispatcher 内写业务事实。
|
||||
*/
|
||||
import { createServer } from "node:http";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { copyFileSync, existsSync, lstatSync, mkdirSync, readFileSync, symlinkSync } from "node:fs";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { copyFileSync, existsSync, lstatSync, mkdirSync, symlinkSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
import {
|
||||
@@ -28,7 +28,6 @@ import {
|
||||
handleCodeAgentChat
|
||||
} from "./code-agent-chat.ts";
|
||||
import { createDurableCodeAgentTraceStore, defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
||||
import { writeWorkbenchProjectionEvent } from "./workbench-projection-writer.ts";
|
||||
import { createCodeAgentSessionRegistry } from "./code-agent-session-registry.ts";
|
||||
import { codeAgentSessionLifecycleSummary } from "./code-agent-session-lifecycle.ts";
|
||||
import { createCodexStdioSessionManager } from "./codex-stdio-session.ts";
|
||||
@@ -66,8 +65,7 @@ import {
|
||||
handleCodeAgentSessionsHttp,
|
||||
handleCodeAgentSteerHttp,
|
||||
handleCodeAgentTurnHttp,
|
||||
handleCodeAgentTraceHttp,
|
||||
startAgentRunProjectionResume
|
||||
handleCodeAgentTraceHttp
|
||||
} from "./server-code-agent-http.ts";
|
||||
import { drainWorkbenchRealtimeConnections, handleWorkbenchReadModelHttp, handleWorkbenchRealtimeHttp } from "./server-workbench-http.ts";
|
||||
import { handleWorkbenchDebugFakeSseHttp } from "./workbench-debug-fake-sse.ts";
|
||||
@@ -87,10 +85,11 @@ import {
|
||||
} from "./skills-store.ts";
|
||||
import { createHwpodNodeWsRegistry } from "./hwpod-node-ws-registry.ts";
|
||||
import {
|
||||
discoverHwpodSpecs,
|
||||
hwpodSpecDiscoveryPayload,
|
||||
hwpodSpecWorkspaceProbePlan
|
||||
} from "./hwpod-spec-discovery.ts";
|
||||
handleHwlabNodeDownloadHttp,
|
||||
handleHwlabNodeUpdateHttp,
|
||||
handleHwpodNodeOpsHttp,
|
||||
handleHwpodSpecDiscoveryHttp
|
||||
} from "./server-hwpod-http.ts";
|
||||
import { handleCaseRunHttp } from "./server-caserun-http.ts";
|
||||
import { handleProjectManagementProxyHttp } from "./project-management-proxy.ts";
|
||||
import { HWPOD_NODE_OPS, HWPOD_NODE_OPS_CONTRACT_VERSION } from "../../tools/src/hwpod-node-ops-contract.ts";
|
||||
@@ -155,8 +154,7 @@ export function createCloudApiServer(options = {}) {
|
||||
const runtimeStore = options.runtimeStore || createConfiguredCloudRuntimeStore({ ...options, env });
|
||||
const traceStore = createDurableCodeAgentTraceStore({
|
||||
traceStore: options.traceStore || defaultCodeAgentTraceStore,
|
||||
traceEventStore: runtimeStore,
|
||||
workbenchEventWriter: (event, meta) => writeWorkbenchProjectionEvent({ runtimeStore, event, requestMeta: meta })
|
||||
traceEventStore: runtimeStore
|
||||
});
|
||||
const codexStdioManager = options.codexStdioManager || createCodexStdioSessionManager({ traceStore });
|
||||
const codeAgentChatResults = options.codeAgentChatResults || createCodeAgentChatResultStore({
|
||||
@@ -173,13 +171,8 @@ export function createCloudApiServer(options = {}) {
|
||||
if (typeof accessController.configureCodeAgentWorkspaceContext === "function") {
|
||||
accessController.configureCodeAgentWorkspaceContext({ env, fetchImpl: options.fetchImpl, traceStore, codeAgentChatResults });
|
||||
}
|
||||
const workbenchEmptySessionGc = startWorkbenchEmptySessionGc({ env, accessController, logger: console });
|
||||
const agentRunProjectionResume = startAgentRunProjectionResume({
|
||||
options: { ...options, env, runtimeStore, accessController, traceStore, codeAgentChatResults },
|
||||
traceStore,
|
||||
logger: console
|
||||
});
|
||||
const kafkaEventBridge = startHwlabKafkaEventBridge({ env, logger: console });
|
||||
const workbenchEmptySessionGc = startWorkbenchEmptySessionGc({ env, accessController, logger: options.logger ?? console });
|
||||
const kafkaEventBridge = options.kafkaEventBridge ?? startHwlabKafkaEventBridge({ env, logger: options.logger ?? console, runtimeStore, kafkaFactory: options.kafkaFactory });
|
||||
const server = createServer(async (request, response) => {
|
||||
const requestContext = buildHttpRequestContext(request);
|
||||
attachHttpRequestContext(request, response, requestContext);
|
||||
@@ -214,7 +207,7 @@ export function createCloudApiServer(options = {}) {
|
||||
response.once("close", () => emitHttpSpan("close"));
|
||||
await withBackendPerformanceContext(backendPerformance, async () => {
|
||||
try {
|
||||
await routeRequest(request, response, { ...options, env, runtimeStore, gatewayRegistry, hwpodNodeWsRegistry, accessController, sessionRegistry, traceStore, codexStdioManager, codeAgentChatResults, webPerformanceStore, backendPerformanceStore, backendPerformance });
|
||||
await routeRequest(request, response, { ...options, env, runtimeStore, kafkaEventBridge, gatewayRegistry, hwpodNodeWsRegistry, accessController, sessionRegistry, traceStore, codexStdioManager, codeAgentChatResults, webPerformanceStore, backendPerformanceStore, backendPerformance, emitHttpRoutePhaseSpan });
|
||||
} catch (error) {
|
||||
if (error?.alreadySent || response.headersSent || response.writableEnded) return;
|
||||
backendPerformance.recordPhase({ phase: "route_exception", durationMs: 0, outcome: "error" });
|
||||
@@ -230,9 +223,10 @@ export function createCloudApiServer(options = {}) {
|
||||
});
|
||||
server.on("close", () => {
|
||||
workbenchEmptySessionGc.stop();
|
||||
agentRunProjectionResume.stop();
|
||||
void kafkaEventBridge.stop();
|
||||
});
|
||||
server.hwlabStartupReady = kafkaEventBridge?.ready ?? Promise.resolve();
|
||||
server.hwlabAbortStartup = async () => { workbenchEmptySessionGc.stop(); await kafkaEventBridge.stop(); };
|
||||
return server;
|
||||
}
|
||||
|
||||
@@ -339,8 +333,9 @@ export async function buildHealthPayload(options = {}) {
|
||||
const dbProbe = await buildDbRuntimeReadiness(env, liveProbe ? { ...(options.dbProbe ?? {}), probe: false } : options.dbProbe);
|
||||
const codeAgent = await describeCodeAgentAvailability(env, options);
|
||||
const runtime = liveProbe ? runtimeReadinessSnapshot(runtimeStore) : await runtimeReadiness(runtimeStore);
|
||||
const kafkaProjector = await kafkaProjectorHealth(options.kafkaEventBridge, { liveProbe });
|
||||
const db = applyRuntimeDbReadinessLayers(dbProbe, runtime);
|
||||
const readiness = buildCloudApiReadiness({ db, codeAgent, runtime });
|
||||
const readiness = buildCloudApiReadiness({ db, codeAgent, runtime, kafkaProjector });
|
||||
|
||||
return {
|
||||
serviceId,
|
||||
@@ -366,10 +361,29 @@ export async function buildHealthPayload(options = {}) {
|
||||
observedAt: new Date().toISOString(),
|
||||
db,
|
||||
codeAgent,
|
||||
runtime
|
||||
runtime,
|
||||
kafkaProjector
|
||||
};
|
||||
}
|
||||
|
||||
async function kafkaProjectorHealth(projector, { liveProbe = false } = {}) {
|
||||
if (!projector?.started) return { started: false, reason: projector?.reason ?? "disabled", valuesRedacted: true };
|
||||
const identity = {
|
||||
capabilities: projector.capabilities ?? null,
|
||||
directPublishGroupId: projector.directPublishGroupId ?? null,
|
||||
projectorGroupId: projector.projectorGroupId ?? null,
|
||||
hwlabEventGroupId: projector.hwlabEventGroupId ?? null,
|
||||
agentRunTopic: projector.agentRunTopic,
|
||||
hwlabTopic: projector.hwlabTopic
|
||||
};
|
||||
if (liveProbe) return { started: true, ...identity, ...(projector.startupStatus?.() ?? { status: "initializing" }), statusSource: "startup-state", valuesRedacted: true };
|
||||
try {
|
||||
return { started: true, ...identity, ...(await projector.status()), valuesRedacted: true };
|
||||
} catch (error) {
|
||||
return { started: true, status: "blocked", errorCode: error?.code ?? "hwlab_kafka_projector_status_failed", message: error?.message ?? "Kafka projector status query failed.", valuesRedacted: true };
|
||||
}
|
||||
}
|
||||
|
||||
function runtimeReadinessSnapshot(runtimeStore) {
|
||||
if (runtimeStore && typeof runtimeStore.summary === "function") {
|
||||
try {
|
||||
@@ -610,7 +624,8 @@ async function handleRestAdapter(request, response, url, options) {
|
||||
observeHttpRoutePhase(request, options, "v1.runtime_readiness", () => runtimeReadinessSnapshot(options.runtimeStore))
|
||||
]);
|
||||
const db = applyRuntimeDbReadinessLayers(dbProbe, runtime);
|
||||
const readiness = buildCloudApiReadiness({ db, codeAgent, runtime });
|
||||
const kafkaProjector = await kafkaProjectorHealth(options.kafkaEventBridge, { liveProbe: false });
|
||||
const readiness = buildCloudApiReadiness({ db, codeAgent, runtime, kafkaProjector });
|
||||
sendJson(response, 200, {
|
||||
serviceId: CLOUD_API_SERVICE_ID,
|
||||
adapter: "rest",
|
||||
@@ -728,7 +743,7 @@ async function handleRestAdapter(request, response, url, options) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/v1/workbench/events") {
|
||||
if (url.pathname === "/v1/workbench/events" || url.pathname === "/v1/workbench/projection-events") {
|
||||
await handleWorkbenchRealtimeHttp(request, response, url, options);
|
||||
return;
|
||||
}
|
||||
@@ -980,7 +995,7 @@ function navIdForRestPath(pathname, method = "GET") {
|
||||
if (pathname === "/v1/api-keys" || pathname === "/v1/api-keys/default" || pathname.startsWith("/v1/api-keys/")) return "user.apiKeys";
|
||||
if (pathname === "/v1/users/me/profile" || pathname === "/v1/users/me/password") return "system.settings";
|
||||
if (pathname === "/v1/workbench/debug/fake-sse" || pathname.startsWith("/v1/workbench/debug/fake-sse/") || pathname === "/v1/workbench/debug/kafka-sse" || pathname.startsWith("/v1/workbench/debug/kafka-sse/")) return "workbench.debug";
|
||||
if (pathname === "/v1/workbench/events" || pathname === "/v1/workbench/sync" || pathname === "/v1/workbench/launches" || pathname === "/v1/workbench/sessions" || pathname.startsWith("/v1/workbench/sessions/") || pathname.startsWith("/v1/workbench/turns/") || pathname.startsWith("/v1/workbench/traces/")) return "workbench.code";
|
||||
if (pathname === "/v1/workbench/events" || pathname === "/v1/workbench/projection-events" || pathname === "/v1/workbench/sync" || pathname === "/v1/workbench/launches" || pathname === "/v1/workbench/sessions" || pathname.startsWith("/v1/workbench/sessions/") || pathname.startsWith("/v1/workbench/turns/") || pathname.startsWith("/v1/workbench/traces/")) return "workbench.code";
|
||||
if (pathname === "/v1/agent/chat" || pathname === "/v1/agent/sessions" || pathname.startsWith("/v1/agent/sessions/") || pathname === "/v1/agent/chat/inspect" || pathname.startsWith("/v1/agent/chat/result/") || pathname.startsWith("/v1/agent/turns/") || pathname.startsWith("/v1/agent/traces/") || pathname === "/v1/agent/chat/cancel" || pathname === "/v1/agent/chat/steer") return "workbench.code";
|
||||
if (pathname === "/v1/admin/provider-profiles" || pathname.startsWith("/v1/admin/provider-profiles/")) return "admin.providerProfiles";
|
||||
if (pathname === "/v1/admin/secrets" || pathname.startsWith("/v1/admin/secrets/")) return "admin.secrets";
|
||||
@@ -994,492 +1009,6 @@ function navIdForRestPath(pathname, method = "GET") {
|
||||
return "";
|
||||
}
|
||||
|
||||
async function handleHwpodNodeOpsHttp(request, response, options) {
|
||||
if (request.method === "GET") {
|
||||
sendJson(response, 200, {
|
||||
ok: true,
|
||||
status: "ready",
|
||||
contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION,
|
||||
route: "/v1/hwpod-node-ops",
|
||||
specAuthority: "code-agent-workspace",
|
||||
compiler: "hwpod-compiler-cli",
|
||||
apiRole: "node-ops-forwarder",
|
||||
nodeRole: "thin-hwpod-node-executor",
|
||||
supportedOps: Array.from(HWPOD_NODE_OPS),
|
||||
websocket: options.hwpodNodeWsRegistry.describe()
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (request.method !== "POST") {
|
||||
sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "hwpod-node-ops only supports GET and POST" } });
|
||||
return;
|
||||
}
|
||||
|
||||
const body = await readJsonObject(request, options.bodyLimitBytes);
|
||||
if (!body.ok) {
|
||||
sendJson(response, 400, body.error);
|
||||
return;
|
||||
}
|
||||
const validation = validateHwpodNodeOpsPlan(body.value);
|
||||
if (!validation.ok) {
|
||||
sendJson(response, 400, {
|
||||
ok: false,
|
||||
status: "rejected",
|
||||
contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION,
|
||||
error: validation.error
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const plan = validation.plan;
|
||||
const requestMeta = hwpodNodeOpsRequestMeta(request, options, "hwpod");
|
||||
try {
|
||||
const handled = await dispatchHwpodNodeOpsPlan(plan, requestMeta, options, { request });
|
||||
sendJson(response, handled.httpStatus, handled.payload);
|
||||
} catch (error) {
|
||||
const summary = error?.message ?? "hwpod-node-ops handler failed";
|
||||
recordHwpodNodeOpsBlocked(request, options, plan, requestMeta, "hwpod_node_handler_failed", summary, { dispatchMode: "handler-exception" });
|
||||
sendJson(response, 200, hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, { dispatchMode: "handler-exception" }));
|
||||
}
|
||||
}
|
||||
|
||||
function hwpodNodeOpsRequestMeta(request, options, label) {
|
||||
const httpContext = request?.hwlabHttpRequestContext ?? {};
|
||||
return {
|
||||
requestId: getHeader(request, "x-request-id") || httpContext.requestId || `req_${label}_${randomUUID()}`,
|
||||
traceId: getHeader(request, "x-trace-id") || `trc_${label}_${randomUUID()}`,
|
||||
serviceId: getHeader(request, "x-source-service-id") || CLOUD_API_SERVICE_ID,
|
||||
environment: runtimeEnvironment(options.env ?? process.env),
|
||||
otelTraceId: httpContext.traceId ?? null,
|
||||
traceparent: httpContext.traceparent ?? null,
|
||||
valuesPrinted: false
|
||||
};
|
||||
}
|
||||
|
||||
function handleHwlabNodeUpdateHttp(request, response, url, options) {
|
||||
if (request.method !== "GET") {
|
||||
sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "GET required." } });
|
||||
return;
|
||||
}
|
||||
const env = options.env ?? process.env;
|
||||
const currentVersion = cleanText(url.searchParams.get("current") || url.searchParams.get("version"));
|
||||
const channel = cleanText(url.searchParams.get("channel")) || "stable";
|
||||
const platform = cleanText(url.searchParams.get("platform")) || "unknown";
|
||||
const bundled = hwlabNodeBundledMetadata(env);
|
||||
const latestVersion = cleanText(env.HWLAB_NODE_PY_LATEST_VERSION) || bundled.version || "0.1.0";
|
||||
const releaseNotesUrl = cleanText(env.HWLAB_NODE_PY_RELEASE_NOTES_URL) || null;
|
||||
const downloadUrl = hwlabNodeDownloadUrl(env);
|
||||
const sha256 = cleanText(env.HWLAB_NODE_PY_SHA256) || bundled.sha256 || null;
|
||||
const newer = currentVersion ? compareVersionStrings(latestVersion, currentVersion) > 0 : Boolean(latestVersion);
|
||||
const updateAvailable = Boolean(downloadUrl && newer);
|
||||
sendJson(response, 200, {
|
||||
ok: true,
|
||||
contractVersion: "hwlab-node-update-v1",
|
||||
serviceId: "hwlab-node",
|
||||
route: "/v1/hwlab-node/update",
|
||||
channel,
|
||||
platform,
|
||||
currentVersion: currentVersion || null,
|
||||
latestVersion,
|
||||
updateAvailable,
|
||||
downloadUrl: updateAvailable ? downloadUrl : null,
|
||||
sha256: updateAvailable ? sha256 : null,
|
||||
releaseNotesUrl,
|
||||
manualDefault: true,
|
||||
autoApplyDefault: false,
|
||||
checkIntervalSeconds: parsePositiveInteger(env.HWLAB_NODE_PY_UPDATE_INTERVAL_SECONDS, 300),
|
||||
observedAt: new Date().toISOString()
|
||||
});
|
||||
}
|
||||
|
||||
function handleHwlabNodeDownloadHttp(request, response, options) {
|
||||
if (request.method !== "GET") {
|
||||
sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "GET required." } });
|
||||
return;
|
||||
}
|
||||
const bundled = hwlabNodeBundledMetadata(options.env ?? process.env);
|
||||
if (!bundled.content) {
|
||||
sendJson(response, 404, { ok: false, error: { code: "hwlab_node_bundle_missing", message: "bundled hwlab-node.py is not available" } });
|
||||
return;
|
||||
}
|
||||
const bytes = Buffer.from(bundled.content, "utf8");
|
||||
response.writeHead(200, {
|
||||
"content-type": "text/x-python; charset=utf-8",
|
||||
"cache-control": "no-store",
|
||||
"x-hwlab-node-version": bundled.version || "unknown",
|
||||
"x-hwlab-node-sha256": bundled.sha256 || "",
|
||||
"content-length": String(bytes.length)
|
||||
});
|
||||
response.end(bytes);
|
||||
}
|
||||
|
||||
function hwlabNodeBundledMetadata(env) {
|
||||
const bundlePath = cleanText(env.HWLAB_NODE_PY_BUNDLE_PATH) || path.resolve(process.cwd(), "tools/hwlab-node.py");
|
||||
try {
|
||||
const content = readFileSync(bundlePath, "utf8");
|
||||
const version = cleanText(content.match(/APP_VERSION\s*=\s*["']([^"']+)["']/u)?.[1]);
|
||||
const sha256 = createHash("sha256").update(content, "utf8").digest("hex");
|
||||
return { path: bundlePath, content, version, sha256 };
|
||||
} catch {
|
||||
return { path: bundlePath, content: "", version: "", sha256: "" };
|
||||
}
|
||||
}
|
||||
|
||||
function hwlabNodeDownloadUrl(env) {
|
||||
const explicit = cleanText(env.HWLAB_NODE_PY_DOWNLOAD_URL);
|
||||
if (explicit) return explicit;
|
||||
const downloadPath = cleanText(env.HWLAB_NODE_PY_DOWNLOAD_PATH) || "/v1/hwlab-node/download/hwlab-node.py";
|
||||
if (/^https?:\/\//iu.test(downloadPath)) return downloadPath;
|
||||
const publicEndpoint = cleanText(env.HWLAB_PUBLIC_ENDPOINT) || "https://hwlab.pikapython.com";
|
||||
try {
|
||||
return new URL(downloadPath, publicEndpoint.endsWith("/") ? publicEndpoint : `${publicEndpoint}/`).toString();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function compareVersionStrings(left, right) {
|
||||
const a = versionParts(left);
|
||||
const b = versionParts(right);
|
||||
for (let index = 0; index < Math.max(a.length, b.length, 3); index += 1) {
|
||||
const delta = (a[index] ?? 0) - (b[index] ?? 0);
|
||||
if (delta !== 0) return delta > 0 ? 1 : -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function versionParts(value) {
|
||||
return String(value ?? "")
|
||||
.trim()
|
||||
.replace(/^v/iu, "")
|
||||
.split(/[.+-]/u)
|
||||
.slice(0, 4)
|
||||
.map((part) => parseInt(part, 10))
|
||||
.map((part) => (Number.isFinite(part) && part >= 0 ? part : 0));
|
||||
}
|
||||
|
||||
function cleanText(value) {
|
||||
const text = String(value ?? "").trim();
|
||||
return text || "";
|
||||
}
|
||||
|
||||
async function handleHwpodSpecDiscoveryHttp(request, response, url, options) {
|
||||
const probe = truthyFlag(url.searchParams.get("probe"));
|
||||
const observedAt = new Date().toISOString();
|
||||
let specs = await discoverHwpodSpecs(options);
|
||||
if (probe) {
|
||||
specs = await Promise.all(specs.map((spec) => probeDiscoveredHwpodSpec(spec, { request, options, observedAt })));
|
||||
}
|
||||
sendJson(response, 200, hwpodSpecDiscoveryPayload(specs, { observedAt }));
|
||||
}
|
||||
|
||||
async function probeDiscoveredHwpodSpec(spec, { request, options, observedAt }) {
|
||||
if (spec.ok === false) return spec;
|
||||
const plan = hwpodSpecWorkspaceProbePlan(spec);
|
||||
const validation = validateHwpodNodeOpsPlan(plan);
|
||||
if (!validation.ok) {
|
||||
return {
|
||||
...spec,
|
||||
availability: {
|
||||
ok: false,
|
||||
status: "invalid_probe_plan",
|
||||
checkedAt: observedAt,
|
||||
blocker: validation.error
|
||||
}
|
||||
};
|
||||
}
|
||||
const requestMeta = hwpodNodeOpsRequestMeta(request, options, "hwpod_spec");
|
||||
const handled = await dispatchHwpodNodeOpsPlan(validation.plan, requestMeta, options, { request });
|
||||
const payload = handled.payload;
|
||||
return {
|
||||
...spec,
|
||||
availability: {
|
||||
ok: payload.ok === true,
|
||||
status: payload.ok === true ? "available" : "blocked",
|
||||
checkedAt: observedAt,
|
||||
probePlanId: payload.planId,
|
||||
nodeOpsRoute: "/v1/hwpod-node-ops",
|
||||
results: payload.results ?? [],
|
||||
blocker: payload.blocker ?? null,
|
||||
httpStatus: handled.httpStatus
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function dispatchHwpodNodeOpsPlan(plan, requestMeta, options, context = {}) {
|
||||
const hwpodNodeOpsUrl = normalizedHwpodNodeOpsUrl(options.env ?? process.env);
|
||||
const hwpodNodeWsRegistry = options.hwpodNodeWsRegistry;
|
||||
const hasWsNode = hwpodNodeWsRegistry?.hasNode?.(plan.nodeId);
|
||||
if (typeof options.hwpodNodeOpsHandler !== "function" && !hasWsNode && !hwpodNodeOpsUrl) {
|
||||
const summary = "no outbound WebSocket hwpod-node is connected and HWLAB_HWPOD_NODE_OPS_URL is not configured; cloud-api is only validating the hwpod-node-ops contract";
|
||||
const details = {
|
||||
dispatchMode: "none",
|
||||
websocketRegistryConfigured: Boolean(hwpodNodeWsRegistry),
|
||||
websocketConnected: false,
|
||||
directUrlConfigured: false
|
||||
};
|
||||
recordHwpodNodeOpsBlocked(context.request, options, plan, requestMeta, "hwpod_node_unavailable", summary, details);
|
||||
return {
|
||||
httpStatus: 200,
|
||||
payload: hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, details)
|
||||
};
|
||||
}
|
||||
const handled = typeof options.hwpodNodeOpsHandler === "function"
|
||||
? await options.hwpodNodeOpsHandler(plan, { request: context.request, requestMeta, env: options.env ?? process.env })
|
||||
: hasWsNode
|
||||
? await hwpodNodeWsRegistry.dispatch(plan, requestMeta, { timeoutMs: parsePositiveInteger(options.env?.HWLAB_HWPOD_NODE_OPS_TIMEOUT_MS, 30000) })
|
||||
: await forwardHwpodNodeOpsPlan(hwpodNodeOpsUrl, plan, requestMeta, options);
|
||||
const results = Array.isArray(handled?.results) ? handled.results : [];
|
||||
const failed = results.some((item) => item?.ok === false);
|
||||
const payload = attachHwpodNodeOpsDiagnostics({
|
||||
ok: handled?.ok ?? !failed,
|
||||
status: handled?.status ?? (failed ? "failed" : "completed"),
|
||||
contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION,
|
||||
planId: plan.planId,
|
||||
hwpodId: plan.hwpodId,
|
||||
nodeId: plan.nodeId,
|
||||
acceptedOps: plan.ops.length,
|
||||
results,
|
||||
blocker: handled?.blocker ?? null,
|
||||
requestMeta
|
||||
}, requestMeta, { dispatchMode: hasWsNode ? "websocket" : hwpodNodeOpsUrl ? "direct-url" : "handler" });
|
||||
if (payload.ok === false && payload.status === "blocked" && payload.blocker?.code) {
|
||||
recordHwpodNodeOpsBlocked(context.request, options, plan, requestMeta, payload.blocker.code, payload.blocker.summary ?? "hwpod-node-ops blocked", payload.blocker.details ?? {});
|
||||
}
|
||||
return { httpStatus: handled?.httpStatus ?? (failed ? 409 : 200), payload };
|
||||
}
|
||||
|
||||
async function forwardHwpodNodeOpsPlan(targetUrl, plan, requestMeta, options) {
|
||||
const target = await describeDirectHwpodNode(targetUrl, options);
|
||||
if (!target.ok || !target.nodeId) {
|
||||
return directHwpodNodeBlocked(plan, "hwpod_node_identity_unverified", "configured hwpod-node URL did not expose a verifiable nodeId", {
|
||||
requestedNodeId: plan.nodeId,
|
||||
targetUrl: redactNodeOpsUrl(targetUrl),
|
||||
dispatchMode: "direct-url",
|
||||
targetStatus: target.status ?? null,
|
||||
error: target.error ?? null
|
||||
}, requestMeta);
|
||||
}
|
||||
if (target.ok && target.nodeId && target.nodeId !== plan.nodeId) {
|
||||
return directHwpodNodeBlocked(plan, "hwpod_node_id_mismatch", `HWLAB_HWPOD_NODE_OPS_URL points to ${target.nodeId}, not requested node ${plan.nodeId}`, {
|
||||
requestedNodeId: plan.nodeId,
|
||||
targetNodeId: target.nodeId,
|
||||
targetUrl: redactNodeOpsUrl(targetUrl),
|
||||
dispatchMode: "direct-url"
|
||||
}, requestMeta);
|
||||
}
|
||||
const response = await fetch(targetUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"x-request-id": requestMeta.requestId,
|
||||
"x-trace-id": requestMeta.traceId,
|
||||
"x-source-service-id": CLOUD_API_SERVICE_ID,
|
||||
...(requestMeta.otelTraceId ? { "x-hwlab-otel-trace-id": requestMeta.otelTraceId } : {}),
|
||||
...(requestMeta.traceparent ? { traceparent: requestMeta.traceparent } : {})
|
||||
},
|
||||
body: JSON.stringify(plan),
|
||||
signal: AbortSignal.timeout(parsePositiveInteger(options.env?.HWLAB_HWPOD_NODE_OPS_TIMEOUT_MS, 30000))
|
||||
});
|
||||
const body = await response.json().catch(() => null);
|
||||
if (!body || typeof body !== "object") {
|
||||
return directHwpodNodeBlocked(plan, "hwpod_node_response_invalid", `hwpod-node returned HTTP ${response.status} without a JSON object payload`, {
|
||||
targetUrl: redactNodeOpsUrl(targetUrl),
|
||||
dispatchMode: "direct-url",
|
||||
targetStatus: response.status
|
||||
}, requestMeta);
|
||||
}
|
||||
return {
|
||||
ok: body.ok,
|
||||
status: body.status,
|
||||
httpStatus: response.status,
|
||||
results: body.results,
|
||||
blocker: body.blocker ?? null
|
||||
};
|
||||
}
|
||||
|
||||
function directHwpodNodeBlocked(plan, code, summary, details, requestMeta = {}) {
|
||||
const blocker = hwpodNodeOpsBlocker({ code, layer: "hwpod-node", retryable: true, summary, details }, requestMeta, details);
|
||||
return {
|
||||
ok: false,
|
||||
status: "blocked",
|
||||
httpStatus: 200,
|
||||
results: plan.ops.map((op) => ({ opId: op.opId, op: op.op, ok: false, status: "blocked", blocker })),
|
||||
blocker,
|
||||
otelTraceId: blocker.otelTraceId ?? null,
|
||||
diagnostic: blocker.diagnostic ?? null
|
||||
};
|
||||
}
|
||||
|
||||
async function describeDirectHwpodNode(targetUrl, options) {
|
||||
try {
|
||||
const response = await fetch(targetUrl, {
|
||||
method: "GET",
|
||||
signal: AbortSignal.timeout(parsePositiveInteger(options.env?.HWLAB_HWPOD_NODE_OPS_TIMEOUT_MS, 30000))
|
||||
});
|
||||
const body = await response.json().catch(() => null);
|
||||
return { ok: response.ok && body && typeof body === "object", status: response.status, nodeId: safeOpaqueId(body?.nodeId), body };
|
||||
} catch (error) {
|
||||
return { ok: false, error: error instanceof Error ? error.message : String(error), nodeId: "" };
|
||||
}
|
||||
}
|
||||
|
||||
function redactNodeOpsUrl(value) {
|
||||
try {
|
||||
const parsed = new URL(value);
|
||||
parsed.username = "";
|
||||
parsed.password = "";
|
||||
parsed.search = parsed.search ? "?redacted=1" : "";
|
||||
return parsed.toString();
|
||||
} catch {
|
||||
return "<invalid-url>";
|
||||
}
|
||||
}
|
||||
|
||||
function normalizedHwpodNodeOpsUrl(env = process.env) {
|
||||
const direct = String(env.HWLAB_HWPOD_NODE_OPS_URL ?? "").trim();
|
||||
if (direct) return direct;
|
||||
const base = String(env.HWLAB_HWPOD_NODE_URL ?? "").trim().replace(/\/+$/u, "");
|
||||
return base ? `${base}/v1/hwpod-node-ops` : "";
|
||||
}
|
||||
|
||||
|
||||
function validateHwpodNodeOpsPlan(value) {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return { ok: false, error: { code: "invalid_hwpod_node_ops_plan", message: "hwpod-node-ops body must be a JSON object" } };
|
||||
}
|
||||
if (value.contractVersion !== HWPOD_NODE_OPS_CONTRACT_VERSION) {
|
||||
return { ok: false, error: { code: "invalid_hwpod_node_ops_contract", message: `contractVersion must be ${HWPOD_NODE_OPS_CONTRACT_VERSION}`, actual: value.contractVersion ?? null } };
|
||||
}
|
||||
const planId = safeOpaqueId(value.planId) || `hwpod_plan_${randomUUID()}`;
|
||||
const nodeId = safeOpaqueId(value.nodeId);
|
||||
const hwpodId = safeOpaqueId(value.hwpodId);
|
||||
if (!nodeId) return { ok: false, error: { code: "invalid_hwpod_node_id", message: "nodeId is required" } };
|
||||
if (!hwpodId) return { ok: false, error: { code: "invalid_hwpod_id", message: "hwpodId is required" } };
|
||||
if (!Array.isArray(value.ops) || value.ops.length === 0) {
|
||||
return { ok: false, error: { code: "invalid_hwpod_node_ops", message: "ops must be a non-empty array" } };
|
||||
}
|
||||
const ops = [];
|
||||
for (const [index, item] of value.ops.entries()) {
|
||||
if (!item || typeof item !== "object" || Array.isArray(item)) {
|
||||
return { ok: false, error: { code: "invalid_hwpod_node_op", message: `ops[${index}] must be a JSON object` } };
|
||||
}
|
||||
const op = typeof item.op === "string" ? item.op.trim() : "";
|
||||
if (!HWPOD_NODE_OPS.has(op)) {
|
||||
return { ok: false, error: { code: "unsupported_hwpod_node_op", message: `unsupported hwpod-node op: ${op || "<empty>"}`, supportedOps: Array.from(HWPOD_NODE_OPS) } };
|
||||
}
|
||||
ops.push({
|
||||
opId: safeOpaqueId(item.opId) || `op_${index + 1}`,
|
||||
op,
|
||||
args: item.args && typeof item.args === "object" && !Array.isArray(item.args) ? item.args : {}
|
||||
});
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
plan: {
|
||||
...value,
|
||||
planId,
|
||||
hwpodId,
|
||||
nodeId,
|
||||
ops
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function hwpodNodeOpsBlockedPayload(plan, requestMeta, summary, details = {}) {
|
||||
const blocker = hwpodNodeOpsBlocker({
|
||||
code: "hwpod_node_unavailable",
|
||||
layer: "hwpod-node",
|
||||
retryable: true,
|
||||
summary,
|
||||
details,
|
||||
userMessage: "hwpod-node 尚未接入执行面;cloud-api 已完成 hwpod-node-ops 合同校验。"
|
||||
}, requestMeta, details);
|
||||
return {
|
||||
ok: false,
|
||||
status: "blocked",
|
||||
contractVersion: HWPOD_NODE_OPS_CONTRACT_VERSION,
|
||||
planId: plan.planId,
|
||||
hwpodId: plan.hwpodId,
|
||||
nodeId: plan.nodeId,
|
||||
acceptedOps: plan.ops.length,
|
||||
results: plan.ops.map((op) => ({
|
||||
opId: op.opId,
|
||||
op: op.op,
|
||||
ok: false,
|
||||
status: "blocked",
|
||||
blocker
|
||||
})),
|
||||
blocker,
|
||||
otelTraceId: blocker.otelTraceId ?? null,
|
||||
diagnostic: blocker.diagnostic ?? null,
|
||||
requestMeta
|
||||
};
|
||||
}
|
||||
|
||||
function hwpodNodeOpsBlocker(blocker, requestMeta = {}, details = {}) {
|
||||
const code = cleanText(blocker?.code) || "hwpod_node_ops_blocked";
|
||||
const summary = cleanText(blocker?.summary ?? blocker?.message) || "hwpod-node-ops blocked";
|
||||
const otelTraceId = cleanText(blocker?.otelTraceId ?? requestMeta?.otelTraceId) || null;
|
||||
const diagnostic = {
|
||||
code,
|
||||
rootCauseCode: code,
|
||||
summary,
|
||||
otelTraceId,
|
||||
traceLine: otelTraceId ? `OTel traceId: ${otelTraceId}` : null,
|
||||
details: details && typeof details === "object" ? details : {},
|
||||
valuesPrinted: false
|
||||
};
|
||||
const userMessage = cleanText(blocker?.userMessage);
|
||||
return {
|
||||
...blocker,
|
||||
code,
|
||||
layer: blocker?.layer ?? "hwpod-node",
|
||||
retryable: blocker?.retryable ?? true,
|
||||
summary,
|
||||
...(otelTraceId ? { otelTraceId } : {}),
|
||||
diagnostic,
|
||||
...(userMessage ? { userMessage: `${userMessage}\n${diagnostic.traceLine ?? "OTel traceId: unavailable"}` } : {})
|
||||
};
|
||||
}
|
||||
|
||||
function attachHwpodNodeOpsDiagnostics(payload, requestMeta, details = {}) {
|
||||
if (!payload || typeof payload !== "object" || payload.ok !== false) return payload;
|
||||
const topBlocker = payload.blocker ? hwpodNodeOpsBlocker(payload.blocker, requestMeta, payload.blocker.details ?? details) : null;
|
||||
const results = Array.isArray(payload.results)
|
||||
? payload.results.map((result) => result?.blocker ? { ...result, blocker: hwpodNodeOpsBlocker(result.blocker, requestMeta, result.blocker.details ?? details) } : result)
|
||||
: payload.results;
|
||||
const diagnostic = topBlocker?.diagnostic ?? results?.find?.((result) => result?.blocker?.diagnostic)?.blocker?.diagnostic ?? null;
|
||||
return {
|
||||
...payload,
|
||||
results,
|
||||
blocker: topBlocker,
|
||||
...(diagnostic?.otelTraceId ? { otelTraceId: diagnostic.otelTraceId } : {}),
|
||||
...(diagnostic ? { diagnostic } : {})
|
||||
};
|
||||
}
|
||||
|
||||
function recordHwpodNodeOpsBlocked(request, options, plan, requestMeta, code, summary, details = {}) {
|
||||
const httpContext = request?.hwlabHttpRequestContext;
|
||||
const attributes = {
|
||||
"hwlab.hwpod.plan_id": cleanText(plan?.planId) || null,
|
||||
"hwlab.hwpod.hwpod_id": cleanText(plan?.hwpodId) || null,
|
||||
"hwlab.hwpod.node_id": cleanText(plan?.nodeId) || null,
|
||||
"hwlab.hwpod.accepted_ops": Array.isArray(plan?.ops) ? plan.ops.length : 0,
|
||||
"hwlab.hwpod.ops": Array.isArray(plan?.ops) ? plan.ops.map((op) => cleanText(op?.op)).filter(Boolean).join(",").slice(0, 240) : null,
|
||||
"hwlab.hwpod.blocker.code": code,
|
||||
"hwlab.hwpod.blocker.summary": String(summary ?? "").slice(0, 500),
|
||||
"hwlab.hwpod.dispatch_mode": cleanText(details?.dispatchMode) || "unknown",
|
||||
"hwlab.hwpod.websocket_connected": Boolean(details?.websocketConnected),
|
||||
"hwlab.hwpod.direct_url_configured": Boolean(details?.directUrlConfigured),
|
||||
"hwlab.hwpod.otel_trace_id": cleanText(requestMeta?.otelTraceId) || null,
|
||||
traceId: cleanText(requestMeta?.traceId) || null
|
||||
};
|
||||
if (httpContext) httpContext.otelAttributes = { ...(httpContext.otelAttributes ?? {}), ...attributes };
|
||||
const error = Object.assign(new Error(summary), { code });
|
||||
emitHttpRoutePhaseSpan(request, options, "hwpod-node-ops.blocked", Date.now(), Date.now(), "error", error, attributes);
|
||||
}
|
||||
|
||||
async function codeAgentOptions(request, response, options, authOptions = {}) {
|
||||
const auth = await options.accessController.authenticate(request, { required: authOptions.required ?? options.accessController.required });
|
||||
if (!auth.ok) {
|
||||
|
||||
@@ -204,158 +204,6 @@ test("cloud api accepts user-billing API keys and records Code Agent billing usa
|
||||
}
|
||||
});
|
||||
|
||||
test("cloud api defers AgentRun Code Agent billing record until terminal result", async () => {
|
||||
const calls = [];
|
||||
const agentRunCalls = [];
|
||||
let eventPolls = 0;
|
||||
let resultPolls = 0;
|
||||
const agentRunServer = createHttpServer(async (request, response) => {
|
||||
const url = new URL(request.url || "/", "http://127.0.0.1");
|
||||
const chunks = [];
|
||||
for await (const chunk of request) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
const body = chunks.length ? JSON.parse(Buffer.concat(chunks).toString("utf8")) : null;
|
||||
agentRunCalls.push({ method: request.method, path: url.pathname, search: url.search, body });
|
||||
const send = (data) => {
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(`${JSON.stringify({ ok: true, data, traceId: "trc_fake_agentrun_billing" })}\n`);
|
||||
};
|
||||
if (request.method === "POST" && url.pathname === "/api/v1/runs") {
|
||||
assert.equal(body.backendProfile, "deepseek");
|
||||
return send({ id: "run_billing_deferred", status: "pending", backendProfile: "deepseek", sessionRef: body.sessionRef, resourceBundleRef: body.resourceBundleRef });
|
||||
}
|
||||
if (request.method === "POST" && url.pathname === "/api/v1/runs/run_billing_deferred/commands") {
|
||||
assert.equal(body.type, "turn");
|
||||
assert.equal(body.idempotencyKey, "trc_user_billing_agentrun_terminal");
|
||||
return send({ id: "cmd_billing_deferred", runId: "run_billing_deferred", state: "pending", type: "turn", seq: 1 });
|
||||
}
|
||||
if (request.method === "GET" && url.pathname === "/api/v1/runs/run_billing_deferred/commands") {
|
||||
return send({ items: [{ id: "cmd_billing_deferred", runId: "run_billing_deferred", state: "running", type: "turn", seq: 1, idempotencyKey: "trc_user_billing_agentrun_terminal", payload: { traceId: "trc_user_billing_agentrun_terminal", conversationId: "cnv_user_billing_agentrun", hwlabSessionId: "ses_user_billing_agentrun", providerProfile: "deepseek" } }] });
|
||||
}
|
||||
if (request.method === "POST" && url.pathname === "/api/v1/runs/run_billing_deferred/runner-jobs") {
|
||||
assert.equal(body.commandId, "cmd_billing_deferred");
|
||||
return send({
|
||||
action: "create-kubernetes-job",
|
||||
runId: "run_billing_deferred",
|
||||
commandId: "cmd_billing_deferred",
|
||||
attemptId: "attempt_billing_deferred",
|
||||
runnerId: "runner_billing_deferred",
|
||||
namespace: "agentrun-v02",
|
||||
jobName: "agentrun-v01-runner-billing-deferred"
|
||||
});
|
||||
}
|
||||
if (request.method === "GET" && url.pathname === "/api/v1/runs/run_billing_deferred/events") {
|
||||
eventPolls += 1;
|
||||
return send({ items: eventPolls > 1 ? [{ id: "evt_done", runId: "run_billing_deferred", seq: 1, type: "terminal_status", payload: { commandId: "cmd_billing_deferred", terminalStatus: "completed" }, createdAt: "2026-06-14T04:20:00.000Z" }] : [] });
|
||||
}
|
||||
if (request.method === "GET" && url.pathname === "/api/v1/runs/run_billing_deferred/commands/cmd_billing_deferred/result") {
|
||||
resultPolls += 1;
|
||||
if (resultPolls === 1) {
|
||||
return send({ runId: "run_billing_deferred", commandId: "cmd_billing_deferred", status: "running", runStatus: "claimed", commandState: "running", terminalStatus: null });
|
||||
}
|
||||
return send({
|
||||
runId: "run_billing_deferred",
|
||||
commandId: "cmd_billing_deferred",
|
||||
attemptId: "attempt_billing_deferred",
|
||||
runnerId: "runner_billing_deferred",
|
||||
jobName: "agentrun-v01-runner-billing-deferred",
|
||||
namespace: "agentrun-v02",
|
||||
status: "completed",
|
||||
runStatus: "completed",
|
||||
commandState: "completed",
|
||||
terminalStatus: "completed",
|
||||
completed: true,
|
||||
reply: "AgentRun billing completed.",
|
||||
lastSeq: 1,
|
||||
eventCount: 1,
|
||||
sessionRef: { sessionId: "ses_agentrun_deepseek_billing", conversationId: "cnv_user_billing_agentrun", threadId: "thr_billing" }
|
||||
});
|
||||
}
|
||||
response.writeHead(404, { "content-type": "application/json" });
|
||||
response.end(`${JSON.stringify({ ok: false, failureKind: "unexpected", message: `${request.method} ${url.pathname}` })}\n`);
|
||||
});
|
||||
await new Promise((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
const userBillingClient = {
|
||||
configured: true,
|
||||
async introspect(token) {
|
||||
calls.push({ op: "introspect", tokenPrefix: token.slice(0, 8) });
|
||||
assert.equal(token, "hwl_user_billing_agentrun_secret");
|
||||
return { ok: true, status: 200, body: { active: true, principal: { userId: "usr_user_billing_agentrun", email: "agentrun@hwlab.local", username: "billing-agentrun", role: "user", scopes: ["api"], authType: "api-key", keyId: "key_user_billing_agentrun" } } };
|
||||
},
|
||||
async billingPreflight(body) {
|
||||
calls.push({ op: "preflight", body });
|
||||
assert.equal(body.apiKey, "hwl_user_billing_agentrun_secret");
|
||||
assert.equal(body.idempotencyKey, "code-agent:trc_user_billing_agentrun_terminal:preflight");
|
||||
return { ok: true, status: 200, body: { allowed: true, reservationId: "res_user_billing_agentrun", estimatedCredits: 1, expiresAt: "2026-06-14T05:00:00.000Z" } };
|
||||
},
|
||||
async billingRecord(body) {
|
||||
calls.push({ op: "record", body });
|
||||
assert.equal(body.reservationId, "res_user_billing_agentrun");
|
||||
assert.equal(body.idempotencyKey, "code-agent:trc_user_billing_agentrun_terminal:record");
|
||||
assert.equal(body.metadata.status, "completed");
|
||||
return { ok: true, status: 200, body: { recordId: "use_user_billing_agentrun", credits: 1, balance: 9 } };
|
||||
},
|
||||
async billingRelease() {
|
||||
throw new Error("completed AgentRun billing should record usage instead of releasing reservation");
|
||||
}
|
||||
};
|
||||
const { port: agentRunPort } = agentRunServer.address();
|
||||
const server = createCloudApiServer({
|
||||
traceStore: createCodeAgentTraceStore(),
|
||||
env: {
|
||||
HWLAB_ACCESS_CONTROL_REQUIRED: "1",
|
||||
HWLAB_USER_BILLING_CODE_AGENT_ENABLED: "1",
|
||||
HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01",
|
||||
AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`,
|
||||
AGENTRUN_API_KEY: "test-agentrun-key",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_SOURCE_COMMIT: "0123456789abcdef0123456789abcdef01234567",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_POLL_INTERVAL_MS: "10",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_SESSION_STORAGE: "metadata-only",
|
||||
HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE: "deepseek"
|
||||
},
|
||||
userBillingClient
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const authHeader = { authorization: "Bearer hwl_user_billing_agentrun_secret" };
|
||||
const sessionResponse = await fetch(`http://127.0.0.1:${port}/v1/agent/sessions`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", ...authHeader },
|
||||
body: JSON.stringify({ conversationId: "cnv_user_billing_agentrun", sessionId: "ses_user_billing_agentrun", providerProfile: "deepseek" })
|
||||
});
|
||||
assert.equal(sessionResponse.status, 201);
|
||||
|
||||
const submit = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", authorization: "Bearer hwl_user_billing_agentrun_secret", "x-trace-id": "trc_user_billing_agentrun_terminal", prefer: "respond-async" },
|
||||
body: JSON.stringify({ conversationId: "cnv_user_billing_agentrun", sessionId: "ses_user_billing_agentrun", shortConnection: true, message: "billing AgentRun terminal smoke" })
|
||||
});
|
||||
assert.equal(submit.status, 202);
|
||||
await waitForUserBillingCondition(() => agentRunCalls.some((call) => call.path === "/api/v1/runs/run_billing_deferred/runner-jobs"));
|
||||
assert.deepEqual(calls.map((call) => call.op), ["introspect", "introspect", "preflight"]);
|
||||
|
||||
const running = await fetch(`http://127.0.0.1:${port}/v1/agent/chat/result/trc_user_billing_agentrun_terminal`, { headers: authHeader });
|
||||
assert.equal(running.status === 202 || running.status === 200, true);
|
||||
assert.equal(calls.some((call) => call.op === "record"), false);
|
||||
|
||||
const payload = running.status === 200 ? await running.json() : await pollUserBillingAgentResult(port, "trc_user_billing_agentrun_terminal", authHeader);
|
||||
assert.equal(payload.status, "completed");
|
||||
assert.equal(payload.billing.recorded, true);
|
||||
assert.equal(payload.billing.reservationId, "res_user_billing_agentrun");
|
||||
assert.equal(JSON.stringify(payload).includes("userBillingReservation"), false);
|
||||
assert.equal(calls.filter((call) => call.op === "preflight").length, 1);
|
||||
assert.equal(calls.filter((call) => call.op === "record").length, 1);
|
||||
assert.equal(calls.some((call) => call.op === "release"), false);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())));
|
||||
await new Promise((resolve, reject) => agentRunServer.close((error) => (error ? reject(error) : resolve())));
|
||||
}
|
||||
});
|
||||
|
||||
test("AgentRun persisted trace evidence restores billing reservation without exposing it on agentRun", async () => {
|
||||
const restored = await loadPersistedAgentRunResult("trc_user_billing_agentrun_restored", {
|
||||
env: { HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01" },
|
||||
|
||||
@@ -491,7 +491,13 @@ test("web performance route templates keep metrics low-cardinality", () => {
|
||||
});
|
||||
|
||||
test("cloud api ingests WebUI performance samples and exposes metrics only on loopback host", async () => {
|
||||
const server = createCloudApiServer({ env: { HWLAB_METRICS_NAMESPACE: "hwlab-v02", HWLAB_GITOPS_TARGET: "v02" } });
|
||||
const server = createCloudApiServer({
|
||||
accessController: {
|
||||
required: false,
|
||||
async authenticate() { return { ok: true, actor: { id: "usr_web_performance", role: "admin" }, session: { id: "uss_web_performance" } }; }
|
||||
},
|
||||
env: { HWLAB_METRICS_NAMESPACE: "hwlab-v02", HWLAB_GITOPS_TARGET: "v02" }
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
try {
|
||||
const address = server.address();
|
||||
|
||||
@@ -2,11 +2,10 @@
|
||||
* SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-18-p0-unique-projection; draft-2026-06-20-p1-zero-split-durable-realtime; PJ2026-010403 API契约 draft-2026-06-18-r1.
|
||||
* 职责: Workbench durable facts store adapter。统一读取 session/message/turn/trace projection facts,不在 GET 中推进 AgentRun facts。
|
||||
*/
|
||||
import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
||||
import { emitCodeAgentOtelSpan } from "./otel-trace.ts";
|
||||
import { safeConversationId, safeSessionId, safeTraceId } from "./server-http-utils.ts";
|
||||
import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts";
|
||||
import { durableTraceStatus, normalizeWorkbenchStatus, TERMINAL_STATUSES, traceTerminalEvidence } from "./workbench-turn-projection.ts";
|
||||
import { durableTraceStatus, TERMINAL_STATUSES, traceTerminalEvidence } from "./workbench-turn-projection.ts";
|
||||
|
||||
const TERMINAL_DURABLE_TRACE_CACHE_MAX = 256;
|
||||
const DEFAULT_WORKBENCH_FACTS_QUERY_MAX_ATTEMPTS = 5;
|
||||
@@ -17,9 +16,7 @@ const terminalDurableTraceCache = new Map();
|
||||
export function createWorkbenchFactsStore(options = {}, actor = null) {
|
||||
const accessStore = options.accessController?.store ?? options.accessController ?? null;
|
||||
const traceSessionCache = options.codeAgentTraceSessionCache instanceof Map ? options.codeAgentTraceSessionCache : null;
|
||||
const traceStore = options.traceStore ?? defaultCodeAgentTraceStore;
|
||||
const runtimeStore = options.runtimeStore ?? null;
|
||||
const runtimeReader = options.workbenchRuntime ?? createWorkbenchRuntimeClient({ env: options.env ?? process.env, fetch: options.fetch, logger: options.logger }) ?? runtimeStore;
|
||||
const runtimeReader = options.workbenchRuntime ?? createWorkbenchRuntimeClient({ env: options.env ?? process.env, fetch: options.fetch, logger: options.logger });
|
||||
const results = options.codeAgentChatResults ?? null;
|
||||
|
||||
async function queryFacts(params = {}) {
|
||||
@@ -135,18 +132,8 @@ export function createWorkbenchFactsStore(options = {}, actor = null) {
|
||||
return safeId ? results?.get?.(safeId) ?? null : null;
|
||||
}
|
||||
|
||||
function traceSnapshotSync(traceId) {
|
||||
return traceStore.snapshot(traceId);
|
||||
}
|
||||
|
||||
async function traceSnapshot(traceId) {
|
||||
const memory = traceSnapshotSync(traceId);
|
||||
if (hasTraceProjection(memory) && TERMINAL_STATUSES.has(normalizeStatus(memory.status))) return memory;
|
||||
const durable = await durableTraceSnapshot(runtimeReader, traceId);
|
||||
if (isProjectionDiagnosticTrace(durable) && hasTraceProjection(memory)) return mergeTraceProjectionDiagnostic(memory, durable);
|
||||
if (durable && shouldPreferDurableTrace(memory, durable)) return durable;
|
||||
if (hasTraceProjection(memory)) return memory;
|
||||
return durable ?? memory;
|
||||
return durableTraceSnapshot(runtimeReader, traceId);
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -157,7 +144,6 @@ export function createWorkbenchFactsStore(options = {}, actor = null) {
|
||||
listSessions,
|
||||
resultForTrace,
|
||||
traceSnapshot,
|
||||
traceSnapshotSync,
|
||||
canReadOwner: (ownerUserId) => canActorReadOwner(ownerUserId, actor)
|
||||
};
|
||||
}
|
||||
@@ -220,7 +206,7 @@ function objectValue(value) {
|
||||
}
|
||||
|
||||
async function durableTraceSnapshot(runtimeStore, traceId) {
|
||||
if (!runtimeStore || typeof runtimeStore.queryAgentTraceEvents !== "function") return null;
|
||||
if (!runtimeStore || typeof runtimeStore.queryWorkbenchFacts !== "function") return null;
|
||||
const safeId = safeTraceId(traceId);
|
||||
if (!safeId) return null;
|
||||
const cached = terminalDurableTraceCache.get(safeId);
|
||||
@@ -231,11 +217,11 @@ async function durableTraceSnapshot(runtimeStore, traceId) {
|
||||
}
|
||||
let result;
|
||||
try {
|
||||
result = await runtimeStore.queryAgentTraceEvents({ traceId: safeId });
|
||||
result = await runtimeStore.queryWorkbenchFacts({ traceId: safeId, families: ["traceEvents"] });
|
||||
} catch (error) {
|
||||
return projectionStoreUnavailableTrace(safeId, error);
|
||||
}
|
||||
const events = Array.isArray(result?.events) ? result.events : [];
|
||||
const events = Array.isArray(result?.facts?.traceEvents) ? result.facts.traceEvents : [];
|
||||
if (events.length === 0) return null;
|
||||
const normalizedEvents = events.map((event, index) => ({ ...event, traceId: safeId, seq: eventSeq(event, index) }));
|
||||
const firstEvent = normalizedEvents[0] ?? null;
|
||||
@@ -391,41 +377,6 @@ function finiteNonNegativeNumber(value) {
|
||||
return Number.isFinite(parsed) && parsed >= 0 ? Math.trunc(parsed) : null;
|
||||
}
|
||||
|
||||
function isProjectionDiagnosticTrace(snapshot) {
|
||||
return Boolean(snapshot?.projection?.blocker || snapshot?.projectionHealth || snapshot?.blocker);
|
||||
}
|
||||
|
||||
function mergeTraceProjectionDiagnostic(memory, diagnostic) {
|
||||
return {
|
||||
...memory,
|
||||
projection: diagnostic.projection ?? memory?.projection ?? null,
|
||||
projectionStatus: diagnostic.projectionStatus ?? diagnostic.projection?.projectionStatus ?? memory?.projectionStatus ?? null,
|
||||
projectionHealth: diagnostic.projectionHealth ?? diagnostic.projection?.projectionHealth ?? memory?.projectionHealth ?? null,
|
||||
blocker: diagnostic.blocker ?? diagnostic.projection?.blocker ?? memory?.blocker ?? null,
|
||||
valuesPrinted: false
|
||||
};
|
||||
}
|
||||
|
||||
function hasTraceProjection(snapshot) {
|
||||
return Boolean(snapshot?.status !== "missing" && (Number(snapshot?.eventCount ?? 0) > 0 || snapshot?.lastEvent));
|
||||
}
|
||||
|
||||
function shouldPreferDurableTrace(memory, durable) {
|
||||
if (!durable) return false;
|
||||
if (!hasTraceProjection(memory)) return true;
|
||||
const memoryStatus = normalizeWorkbenchStatus(memory?.status);
|
||||
const durableStatus = normalizeWorkbenchStatus(durable?.status);
|
||||
if (TERMINAL_STATUSES.has(durableStatus) && !TERMINAL_STATUSES.has(memoryStatus)) return true;
|
||||
return traceLastSeq(durable) > traceLastSeq(memory);
|
||||
}
|
||||
|
||||
function traceLastSeq(snapshot) {
|
||||
const events = Array.isArray(snapshot?.events) ? snapshot.events : [];
|
||||
const lastEvent = snapshot?.lastEvent ?? events.at(-1) ?? null;
|
||||
const indexedMax = events.reduce((max, event, index) => Math.max(max, eventSeq(event, index)), 0);
|
||||
return Math.max(indexedMax, lastEvent ? eventSeq(lastEvent, Math.max(0, events.length - 1)) : 0);
|
||||
}
|
||||
|
||||
function canActorReadSession(session, actor) {
|
||||
if (!session || !actor) return false;
|
||||
if (normalizeStatus(session.status) === "archived") return false;
|
||||
|
||||
@@ -13,7 +13,7 @@ test("workbench Kafka SSE debug endpoint streams filtered raw HWLAB Kafka events
|
||||
};
|
||||
const server = createServer((request, response) => {
|
||||
const url = new URL(request.url ?? "/", "http://127.0.0.1");
|
||||
void handleWorkbenchKafkaSseDebugHttp(request, response, url, { env, kafkaFactory: fakeKafka.factory, logger: null });
|
||||
void handleWorkbenchKafkaSseDebugHttp(request, response, url, { env, kafkaFactory: fakeKafka.factory, accessController: fakeAccessController(), logger: null });
|
||||
});
|
||||
await listen(server);
|
||||
const abort = new AbortController();
|
||||
@@ -25,6 +25,8 @@ test("workbench Kafka SSE debug endpoint streams filtered raw HWLAB Kafka events
|
||||
assert.ok(reader, "SSE response must expose a readable stream");
|
||||
const connected = await readUntil(reader, "hwlab.kafka.connected");
|
||||
assert.match(connected, /hwlab\.event\.v1/u);
|
||||
assert.match(connected, /"consumerReady":true/u);
|
||||
assert.match(connected, new RegExp(String(fakeKafka.groupId).replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"), "u"));
|
||||
|
||||
await fakeKafka.emit({ eventType: "hwlab.trace.event.projected", sessionId: "ses_other", event: { label: "ignored" } });
|
||||
await fakeKafka.emit({ eventType: "hwlab.trace.event.projected", sessionId: "ses_kafka_sse_debug", traceId: "trc_kafka_sse_debug", context: { runId: "run_kafka_sse_debug", commandId: "cmd_kafka_sse_debug" }, event: { label: "agentrun:event:debug", status: "running" } });
|
||||
@@ -47,7 +49,7 @@ test("workbench Kafka SSE debug endpoint streams filtered codex stdio Kafka even
|
||||
};
|
||||
const server = createServer((request, response) => {
|
||||
const url = new URL(request.url ?? "/", "http://127.0.0.1");
|
||||
void handleWorkbenchKafkaSseDebugHttp(request, response, url, { env, kafkaFactory: fakeKafka.factory, logger: null });
|
||||
void handleWorkbenchKafkaSseDebugHttp(request, response, url, { env, kafkaFactory: fakeKafka.factory, accessController: fakeAccessController(), logger: null });
|
||||
});
|
||||
await listen(server);
|
||||
const abort = new AbortController();
|
||||
@@ -90,7 +92,7 @@ test("workbench Kafka SSE debug endpoint resolves traceId to AgentRun Kafka ids"
|
||||
};
|
||||
const server = createServer((request, response) => {
|
||||
const url = new URL(request.url ?? "/", "http://127.0.0.1");
|
||||
void handleWorkbenchKafkaSseDebugHttp(request, response, url, { env, kafkaFactory: fakeKafka.factory, traceStore, logger: null });
|
||||
void handleWorkbenchKafkaSseDebugHttp(request, response, url, { env, kafkaFactory: fakeKafka.factory, traceStore, accessController: fakeAccessController(), logger: null });
|
||||
});
|
||||
await listen(server);
|
||||
const abort = new AbortController();
|
||||
@@ -115,18 +117,131 @@ test("workbench Kafka SSE debug endpoint resolves traceId to AgentRun Kafka ids"
|
||||
}
|
||||
});
|
||||
|
||||
function createFakeKafkaFactory() {
|
||||
test("workbench Kafka SSE debug endpoint is admin-only", async () => {
|
||||
const fakeKafka = createFakeKafkaFactory();
|
||||
const server = createServer((request, response) => {
|
||||
const url = new URL(request.url ?? "/", "http://127.0.0.1");
|
||||
void handleWorkbenchKafkaSseDebugHttp(request, response, url, {
|
||||
env: { HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092" },
|
||||
kafkaFactory: fakeKafka.factory,
|
||||
accessController: fakeAccessController("user"),
|
||||
logger: null
|
||||
});
|
||||
});
|
||||
await listen(server);
|
||||
try {
|
||||
const response = await fetch(`${serverUrl(server)}/v1/workbench/debug/kafka-sse/events?stream=hwlab`);
|
||||
assert.equal(response.status, 403);
|
||||
assert.match(await response.text(), /admin_required/u);
|
||||
assert.equal(fakeKafka.consumerCreated, false);
|
||||
} finally {
|
||||
await close(server);
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench Kafka SSE debug endpoint reports connected only after consumer run is ready", async () => {
|
||||
const fakeKafka = createFakeKafkaFactory({ deferRun: true });
|
||||
const server = createServer((request, response) => {
|
||||
const url = new URL(request.url ?? "/", "http://127.0.0.1");
|
||||
void handleWorkbenchKafkaSseDebugHttp(request, response, url, {
|
||||
env: {
|
||||
HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092",
|
||||
HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api",
|
||||
HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1"
|
||||
},
|
||||
kafkaFactory: fakeKafka.factory,
|
||||
accessController: fakeAccessController(),
|
||||
logger: null
|
||||
});
|
||||
});
|
||||
await listen(server);
|
||||
const abort = new AbortController();
|
||||
try {
|
||||
const response = await fetch(`${serverUrl(server)}/v1/workbench/debug/kafka-sse/events?stream=hwlab`, { signal: abort.signal });
|
||||
const reader = response.body?.getReader();
|
||||
assert.ok(reader);
|
||||
const firstRead = reader.read();
|
||||
const early = await Promise.race([firstRead.then(() => "event"), delay(25).then(() => "pending")]);
|
||||
assert.equal(early, "pending");
|
||||
fakeKafka.releaseRun();
|
||||
const first = await firstRead;
|
||||
assert.equal(first.done, false);
|
||||
const connected = new TextDecoder().decode(first.value);
|
||||
assert.match(connected, /hwlab\.kafka\.connected/u);
|
||||
assert.match(connected, /"consumerReady":true/u);
|
||||
assert.match(connected, /"groupId":"test-hwlab-cloud-api-debug-sse-/u);
|
||||
} finally {
|
||||
abort.abort();
|
||||
fakeKafka.releaseRun();
|
||||
await close(server);
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench Kafka SSE debug endpoint stops a consumer that becomes ready after client close", async () => {
|
||||
const fakeKafka = createFakeKafkaFactory({ deferRun: true });
|
||||
const server = createServer((request, response) => {
|
||||
const url = new URL(request.url ?? "/", "http://127.0.0.1");
|
||||
void handleWorkbenchKafkaSseDebugHttp(request, response, url, {
|
||||
env: { HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1" },
|
||||
kafkaFactory: fakeKafka.factory,
|
||||
accessController: fakeAccessController(),
|
||||
logger: null
|
||||
});
|
||||
});
|
||||
await listen(server);
|
||||
const abort = new AbortController();
|
||||
try {
|
||||
const response = await fetch(`${serverUrl(server)}/v1/workbench/debug/kafka-sse/events?stream=hwlab`, { signal: abort.signal });
|
||||
const reader = response.body?.getReader();
|
||||
assert.ok(reader);
|
||||
const firstRead = reader.read().catch(() => null);
|
||||
await delay(20);
|
||||
abort.abort();
|
||||
fakeKafka.releaseRun();
|
||||
await waitUntil(() => fakeKafka.stopCalls > 0);
|
||||
const first = await firstRead;
|
||||
const text = first && !first.done && first.value ? new TextDecoder().decode(first.value) : "";
|
||||
assert.doesNotMatch(text, /hwlab\.kafka\.connected/u);
|
||||
assert.equal(fakeKafka.disconnectCalls > 0, true);
|
||||
} finally {
|
||||
abort.abort();
|
||||
fakeKafka.releaseRun();
|
||||
await close(server);
|
||||
}
|
||||
});
|
||||
|
||||
function createFakeKafkaFactory(options: { deferRun?: boolean } = {}) {
|
||||
let eachMessage: ((input: any) => Promise<void>) | null = null;
|
||||
let subscribedTopic = "hwlab.event.v1";
|
||||
let releaseRun: (() => void) | null = null;
|
||||
const runGate = options.deferRun ? new Promise<void>((resolve) => { releaseRun = resolve; }) : null;
|
||||
let groupId: string | null = null;
|
||||
let stopCalls = 0;
|
||||
let disconnectCalls = 0;
|
||||
let consumerCreated = false;
|
||||
const consumer = {
|
||||
connect: async () => undefined,
|
||||
subscribe: async (input: { topic?: string }) => { subscribedTopic = String(input.topic ?? subscribedTopic); },
|
||||
run: async (input: any) => { eachMessage = input.eachMessage; },
|
||||
stop: async () => undefined,
|
||||
disconnect: async () => undefined
|
||||
run: async (input: any) => {
|
||||
if (runGate) await runGate;
|
||||
eachMessage = input.eachMessage;
|
||||
},
|
||||
stop: async () => { stopCalls += 1; },
|
||||
disconnect: async () => { disconnectCalls += 1; }
|
||||
};
|
||||
return {
|
||||
factory: () => ({ consumer: () => consumer }),
|
||||
factory: () => ({
|
||||
consumer: (input: { groupId?: string }) => {
|
||||
consumerCreated = true;
|
||||
groupId = String(input.groupId ?? "");
|
||||
return consumer;
|
||||
}
|
||||
}),
|
||||
get groupId() { return groupId; },
|
||||
get stopCalls() { return stopCalls; },
|
||||
get disconnectCalls() { return disconnectCalls; },
|
||||
get consumerCreated() { return consumerCreated; },
|
||||
releaseRun: () => releaseRun?.(),
|
||||
emit: async (value: Record<string, unknown>) => {
|
||||
assert.ok(eachMessage, "consumer.run must be called before emitting fake Kafka events");
|
||||
await eachMessage({
|
||||
@@ -143,6 +258,27 @@ function createFakeKafkaFactory() {
|
||||
};
|
||||
}
|
||||
|
||||
function fakeAccessController(role: "admin" | "user" = "admin") {
|
||||
return {
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() {
|
||||
return { ok: true, status: 200, actor: { id: role === "admin" ? "usr_debug_admin" : "usr_debug_user", role } };
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function delay(ms: number) {
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function waitUntil(predicate: () => boolean, timeoutMs = 1000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (!predicate()) {
|
||||
if (Date.now() >= deadline) throw new Error("condition did not become true before timeout");
|
||||
await delay(10);
|
||||
}
|
||||
}
|
||||
|
||||
async function readUntil(reader: ReadableStreamDefaultReader<Uint8Array>, pattern: string): Promise<string> {
|
||||
const decoder = new TextDecoder();
|
||||
let text = "";
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
import { sendJson } from "./server-http-utils.ts";
|
||||
import { openKafkaEventStream } from "./kafka-event-bridge.ts";
|
||||
import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
||||
import { authenticateWorkbenchRead } from "./server-workbench-read-http.ts";
|
||||
|
||||
const CONTRACT_VERSION = "workbench-debug-kafka-sse-v1";
|
||||
const DEFAULT_STREAM = "hwlab";
|
||||
@@ -13,6 +14,11 @@ const ALLOWED_STREAMS = new Set(["stdio", "agentrun", "hwlab"]);
|
||||
export async function handleWorkbenchKafkaSseDebugHttp(request, response, url, options = {}) {
|
||||
const route = routeSuffix(url.pathname);
|
||||
try {
|
||||
const auth = await authenticateWorkbenchRead(request, response, options);
|
||||
if (!auth) return;
|
||||
if (auth.actor?.role !== "admin") {
|
||||
return sendJson(response, 403, debugError("admin_required", "Only admin users can access raw Workbench Kafka debug streams."));
|
||||
}
|
||||
if (route === "") {
|
||||
if (request.method !== "GET") return methodNotAllowed(response, "GET");
|
||||
return sendJson(response, 200, describeKafkaSseDebug(url, options.env ?? process.env));
|
||||
@@ -64,16 +70,6 @@ async function openKafkaDebugSse(request, response, url, options) {
|
||||
"x-content-type-options": "nosniff"
|
||||
});
|
||||
if (typeof response.flushHeaders === "function") response.flushHeaders();
|
||||
writeSse(response, "hwlab.kafka.connected", {
|
||||
ok: true,
|
||||
contractVersion: CONTRACT_VERSION,
|
||||
stream,
|
||||
topic: topicForStream(stream, env),
|
||||
filters,
|
||||
resolvedFilters,
|
||||
serverSentAt: new Date().toISOString(),
|
||||
valuesPrinted: false
|
||||
});
|
||||
let kafkaStream = null;
|
||||
let closed = false;
|
||||
const close = () => {
|
||||
@@ -81,7 +77,9 @@ async function openKafkaDebugSse(request, response, url, options) {
|
||||
closed = true;
|
||||
void kafkaStream?.stop?.();
|
||||
};
|
||||
request.on("close", close);
|
||||
response.once("close", close);
|
||||
request.once?.("aborted", close);
|
||||
request.socket?.once?.("close", close);
|
||||
try {
|
||||
kafkaStream = await openKafkaEventStream({
|
||||
env,
|
||||
@@ -103,8 +101,23 @@ async function openKafkaDebugSse(request, response, url, options) {
|
||||
serverSentAt: new Date().toISOString(),
|
||||
valuesPrinted: false
|
||||
});
|
||||
},
|
||||
onError: (error) => writeSse(response, "hwlab.kafka.error", { ok: false, error: errorMessagePayload(error), valuesPrinted: false })
|
||||
}
|
||||
});
|
||||
if (closed) {
|
||||
await kafkaStream.stop?.();
|
||||
return;
|
||||
}
|
||||
writeSse(response, "hwlab.kafka.connected", {
|
||||
ok: true,
|
||||
contractVersion: CONTRACT_VERSION,
|
||||
consumerReady: true,
|
||||
groupId: kafkaStream.groupId,
|
||||
stream,
|
||||
topic: kafkaStream.topic ?? topicForStream(stream, env),
|
||||
filters,
|
||||
resolvedFilters,
|
||||
serverSentAt: new Date().toISOString(),
|
||||
valuesPrinted: false
|
||||
});
|
||||
} catch (error) {
|
||||
writeSse(response, "hwlab.kafka.error", { ok: false, error: errorMessagePayload(error), valuesPrinted: false });
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor.
|
||||
// Responsibility: Independent backend unit tests for AgentRun projection cursor planning.
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { buildAgentRunProjectionEventsFetchPlan, buildWorkbenchProjectionStateUpdate } from "./workbench-projection-cursor.ts";
|
||||
|
||||
test("AgentRun projection fetch plan uses durable cursor without scanning long trace", () => {
|
||||
const poisonTrace = {};
|
||||
Object.defineProperty(poisonTrace, "events", {
|
||||
get() {
|
||||
throw new Error("trace events must not be read to compute AgentRun afterSeq");
|
||||
}
|
||||
});
|
||||
const poisonResult = {};
|
||||
Object.defineProperty(poisonResult, "events", {
|
||||
get() {
|
||||
throw new Error("result events must not be read to compute AgentRun afterSeq");
|
||||
}
|
||||
});
|
||||
|
||||
const plan = buildAgentRunProjectionEventsFetchPlan({
|
||||
projectionState: {
|
||||
traceId: "trc_cursor_o1",
|
||||
sourceRunId: "run_cursor_o1",
|
||||
sourceCommandId: "cmd_cursor_o1",
|
||||
lastSourceSeq: 3700,
|
||||
lastProjectedSeq: 129
|
||||
},
|
||||
agentRun: {
|
||||
runId: "run_cursor_o1",
|
||||
commandId: "cmd_cursor_o1",
|
||||
lastSeq: 12
|
||||
},
|
||||
trace: poisonTrace,
|
||||
result: poisonResult,
|
||||
pageLimit: 500
|
||||
});
|
||||
|
||||
assert.equal(plan.runId, "run_cursor_o1");
|
||||
assert.equal(plan.commandId, "cmd_cursor_o1");
|
||||
assert.equal(plan.afterSeq, 3700);
|
||||
assert.equal(plan.limit, 500);
|
||||
assert.equal(plan.cursorSource, "projection-state");
|
||||
});
|
||||
|
||||
test("AgentRun projection state advances from incremental events response", () => {
|
||||
const state = buildWorkbenchProjectionStateUpdate({
|
||||
currentState: {
|
||||
traceId: "trc_cursor_update",
|
||||
sourceRunId: "run_cursor_update",
|
||||
sourceCommandId: "cmd_cursor_update",
|
||||
lastSourceSeq: 3700,
|
||||
lastProjectedSeq: 3700,
|
||||
sourceLatestSeq: 3700,
|
||||
projectionStatus: "projecting",
|
||||
resultSyncState: "not_started",
|
||||
createdAt: "2026-06-19T12:00:00.000Z"
|
||||
},
|
||||
agentRun: {
|
||||
runId: "run_cursor_update",
|
||||
commandId: "cmd_cursor_update",
|
||||
lastSeq: 3700
|
||||
},
|
||||
eventsResponse: {
|
||||
traceLastSeq: 3720,
|
||||
maxSeq: 3721,
|
||||
rawEventCount: 20
|
||||
},
|
||||
now: () => "2026-06-19T12:01:00.000Z"
|
||||
});
|
||||
|
||||
assert.equal(state.lastSourceSeq, 3720);
|
||||
assert.equal(state.lastAgentRunSeq, 3720);
|
||||
assert.equal(state.lastProjectedSeq, 3720);
|
||||
assert.equal(state.sourceLatestSeq, 3721);
|
||||
assert.equal(state.projectionStatus, "projecting");
|
||||
assert.equal(state.resultSyncState, "not_started");
|
||||
assert.equal(state.createdAt, "2026-06-19T12:00:00.000Z");
|
||||
assert.equal(state.updatedAt, "2026-06-19T12:01:00.000Z");
|
||||
});
|
||||
|
||||
test("AgentRun projection state preserves retry params for background resume", () => {
|
||||
const state = buildWorkbenchProjectionStateUpdate({
|
||||
currentState: {
|
||||
traceId: "trc_cursor_retry_params",
|
||||
sourceRunId: "run_cursor_retry_params",
|
||||
sourceCommandId: "cmd_cursor_retry_params",
|
||||
retryParams: {
|
||||
message: "preserve this prompt for fresh-session retry",
|
||||
sessionId: "ses_cursor_retry",
|
||||
providerProfile: "dsflash-go",
|
||||
valuesPrinted: false
|
||||
},
|
||||
createdAt: "2026-06-19T12:00:00.000Z"
|
||||
},
|
||||
agentRun: {
|
||||
runId: "run_cursor_retry_params",
|
||||
commandId: "cmd_cursor_retry_params",
|
||||
lastSeq: 3
|
||||
},
|
||||
now: () => "2026-06-19T12:01:00.000Z"
|
||||
});
|
||||
|
||||
assert.equal(state.retryParams.message, "preserve this prompt for fresh-session retry");
|
||||
assert.equal(state.retryParams.prompt, "preserve this prompt for fresh-session retry");
|
||||
assert.equal(state.retryParams.sessionId, "ses_cursor_retry");
|
||||
assert.equal(state.retryParams.providerProfile, "dsflash-go");
|
||||
assert.equal(state.retryParams.valuesPrinted, false);
|
||||
});
|
||||
|
||||
test("AgentRun projection state resets source cursor when fresh retry switches run", () => {
|
||||
const state = buildWorkbenchProjectionStateUpdate({
|
||||
currentState: {
|
||||
traceId: "trc_cursor_fresh_retry",
|
||||
sourceRunId: "run_cursor_failed",
|
||||
sourceCommandId: "cmd_cursor_failed",
|
||||
lastSourceSeq: 3700,
|
||||
lastAgentRunSeq: 3700,
|
||||
lastProjectedSeq: 3700,
|
||||
sourceLatestSeq: 3700,
|
||||
projectionStatus: "terminal",
|
||||
resultSyncState: "pending",
|
||||
retryParams: {
|
||||
message: "retry on a fresh AgentRun session",
|
||||
sessionId: "ses_cursor_fresh_retry",
|
||||
providerProfile: "dsflash-go",
|
||||
valuesPrinted: false
|
||||
},
|
||||
createdAt: "2026-06-19T12:00:00.000Z"
|
||||
},
|
||||
agentRun: {
|
||||
runId: "run_cursor_fresh_retry",
|
||||
commandId: "cmd_cursor_fresh_retry",
|
||||
lastSeq: 0
|
||||
},
|
||||
projectionStatus: "projecting",
|
||||
resultSyncState: null,
|
||||
now: () => "2026-06-19T12:02:00.000Z"
|
||||
});
|
||||
|
||||
assert.equal(state.sourceRunId, "run_cursor_fresh_retry");
|
||||
assert.equal(state.sourceCommandId, "cmd_cursor_fresh_retry");
|
||||
assert.equal(state.lastSourceSeq, 0);
|
||||
assert.equal(state.lastAgentRunSeq, 0);
|
||||
assert.equal(state.lastProjectedSeq, 0);
|
||||
assert.equal(state.sourceLatestSeq, 0);
|
||||
assert.equal(state.projectionStatus, "projecting");
|
||||
assert.equal(state.retryParams.message, "retry on a fresh AgentRun session");
|
||||
const plan = buildAgentRunProjectionEventsFetchPlan({ projectionState: state, agentRun: { runId: "run_cursor_fresh_retry", commandId: "cmd_cursor_fresh_retry", lastSeq: 0 } });
|
||||
assert.equal(plan.afterSeq, 0);
|
||||
});
|
||||
@@ -1,205 +0,0 @@
|
||||
/*
|
||||
* SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-19-p1-agentrun-incremental-cursor; PJ2026-010205 HWLAB接入 draft-2026-06-17-r0.
|
||||
* Responsibility: Pure cursor planning for AgentRun events projection; never derives cursor by scanning trace events or result payloads.
|
||||
*/
|
||||
|
||||
const DEFAULT_AGENTRUN_EVENTS_PAGE_LIMIT = 500;
|
||||
const MAX_AGENTRUN_EVENTS_PAGE_LIMIT = 1000;
|
||||
|
||||
export function buildAgentRunProjectionEventsFetchPlan({ projectionState = null, agentRun = null, pageLimit = DEFAULT_AGENTRUN_EVENTS_PAGE_LIMIT } = {}) {
|
||||
const state = normalizeProjectionState(projectionState);
|
||||
const mapping = normalizeAgentRunMapping(agentRun);
|
||||
const runId = textValue(state?.sourceRunId ?? state?.runId ?? mapping.runId);
|
||||
const commandId = textValue(state?.sourceCommandId ?? state?.commandId ?? mapping.commandId);
|
||||
const stateSeq = maxNonNegativeInteger(state?.lastSourceSeq, state?.lastAgentRunSeq);
|
||||
const mappingSeq = maxNonNegativeInteger(mapping.lastSeq);
|
||||
return {
|
||||
runId,
|
||||
commandId,
|
||||
afterSeq: Math.max(stateSeq, mappingSeq),
|
||||
limit: boundedPageLimit(pageLimit),
|
||||
cursorSource: state ? "projection-state" : "agent-run-result",
|
||||
valuesPrinted: false
|
||||
};
|
||||
}
|
||||
|
||||
export function buildWorkbenchProjectionStateUpdate({
|
||||
currentState = null,
|
||||
result = null,
|
||||
agentRun = null,
|
||||
eventsResponse = null,
|
||||
resultSyncState = null,
|
||||
projectionStatus = null,
|
||||
projectionHealth = null,
|
||||
polling = null,
|
||||
retryParams = null,
|
||||
error = null,
|
||||
now = () => new Date().toISOString()
|
||||
} = {}) {
|
||||
const previous = normalizeProjectionState(currentState) ?? {};
|
||||
const mapping = normalizeAgentRunMapping(agentRun ?? result?.agentRun);
|
||||
const timestamp = timestampValue(now);
|
||||
const traceId = textValue(previous.traceId ?? result?.traceId ?? mapping.traceId);
|
||||
const runId = textValue(mapping.runId) || textValue(previous.sourceRunId ?? previous.runId);
|
||||
const commandId = textValue(mapping.commandId) || textValue(previous.sourceCommandId ?? previous.commandId);
|
||||
const previousRunId = textValue(previous.sourceRunId ?? previous.runId);
|
||||
const previousCommandId = textValue(previous.sourceCommandId ?? previous.commandId);
|
||||
const sourceChanged = Boolean((previousRunId && runId && previousRunId !== runId) || (previousCommandId && commandId && previousCommandId !== commandId));
|
||||
const previousCursorSeq = sourceChanged ? 0 : maxNonNegativeInteger(previous.lastSourceSeq, previous.lastAgentRunSeq);
|
||||
const lastSourceSeq = Math.max(
|
||||
previousCursorSeq,
|
||||
maxNonNegativeInteger(mapping.lastSeq),
|
||||
maxNonNegativeInteger(eventsResponse?.traceLastSeq)
|
||||
);
|
||||
const sourceLatestSeq = Math.max(
|
||||
sourceChanged ? 0 : maxNonNegativeInteger(previous.sourceLatestSeq, previous.upstreamLatestSeq),
|
||||
maxNonNegativeInteger(eventsResponse?.maxSeq),
|
||||
lastSourceSeq
|
||||
);
|
||||
const nextRetryParams = normalizeProjectionRetryParams(retryParams) ?? normalizeProjectionRetryParams(previous.retryParams);
|
||||
const nextPolling = normalizeProjectionPolling(polling);
|
||||
const nextResultSyncState = textValue(resultSyncState ?? previous.resultSyncState) || "not_started";
|
||||
const nextProjectionStatus = textValue(projectionStatus) || inferProjectionStatus({ lastSourceSeq, sourceLatestSeq, resultSyncState: nextResultSyncState, previous });
|
||||
const nextProjectionHealth = textValue(projectionHealth) || (error ? "degraded" : "healthy");
|
||||
const nextRetryAt = nextPolling?.nextRetryAt ?? (error ? previous.nextRetryAt ?? null : null);
|
||||
return {
|
||||
...previous,
|
||||
traceId,
|
||||
sessionId: textValue(previous.sessionId ?? result?.sessionId ?? mapping.sessionId) || null,
|
||||
conversationId: textValue(previous.conversationId ?? result?.conversationId ?? mapping.conversationId) || null,
|
||||
threadId: textValue(previous.threadId ?? result?.threadId ?? mapping.threadId) || null,
|
||||
ownerUserId: textValue(previous.ownerUserId ?? result?.ownerUserId) || null,
|
||||
ownerRole: textValue(previous.ownerRole ?? result?.ownerRole) || null,
|
||||
managerUrl: textValue(previous.managerUrl ?? mapping.managerUrl) || null,
|
||||
backendProfile: textValue(previous.backendProfile ?? mapping.backendProfile) || null,
|
||||
providerId: textValue(previous.providerId ?? mapping.providerId) || null,
|
||||
sourceRunId: runId,
|
||||
sourceCommandId: commandId,
|
||||
runId,
|
||||
commandId,
|
||||
lastSourceSeq,
|
||||
lastAgentRunSeq: lastSourceSeq,
|
||||
lastProjectedSeq: sourceChanged ? lastSourceSeq : Math.max(maxNonNegativeInteger(previous.lastProjectedSeq), lastSourceSeq),
|
||||
sourceLatestSeq,
|
||||
upstreamLatestSeq: sourceLatestSeq,
|
||||
projectionStatus: nextProjectionStatus,
|
||||
projectionHealth: nextProjectionHealth,
|
||||
resultSyncState: nextResultSyncState,
|
||||
lastProjectedAt: sourceChanged || lastSourceSeq > maxNonNegativeInteger(previous.lastSourceSeq, previous.lastAgentRunSeq) ? timestamp : previous.lastProjectedAt ?? timestamp,
|
||||
lastResultSyncAt: nextResultSyncState === "synced" ? timestamp : previous.lastResultSyncAt ?? null,
|
||||
lastErrorCode: error ? textValue(error.code ?? error.errorCode ?? "projection_sync_failed") : null,
|
||||
lastErrorMessage: error ? textValue(error.message ?? "Projection sync failed.") : null,
|
||||
failureCount: error ? maxNonNegativeInteger(previous.failureCount) + 1 : 0,
|
||||
nextRetryAt,
|
||||
pollCount: nextPolling?.pollCount ?? maxNonNegativeInteger(previous.pollCount),
|
||||
noProgressPollCount: nextPolling?.noProgressPollCount ?? (error ? maxNonNegativeInteger(previous.noProgressPollCount) : 0),
|
||||
backoffMs: nextPolling?.backoffMs ?? 0,
|
||||
backoffReason: nextPolling?.backoffReason ?? null,
|
||||
rootCause: nextPolling?.rootCause ?? null,
|
||||
terminalConsistencyGap: nextPolling?.terminalConsistencyGap ?? null,
|
||||
retryParams: nextRetryParams,
|
||||
createdAt: previous.createdAt ?? timestamp,
|
||||
updatedAt: timestamp,
|
||||
valuesPrinted: false
|
||||
};
|
||||
}
|
||||
|
||||
export function agentRunResultSyncDeferred({ forceResultSync = false, options = {}, env = process.env } = {}) {
|
||||
if (forceResultSync) return false;
|
||||
if (options.deferAgentRunResultSync === true) return true;
|
||||
const configured = String(env?.HWLAB_CODE_AGENT_AGENTRUN_RESULT_SYNC_IN_RUNNING ?? "").trim().toLowerCase();
|
||||
return ["0", "false", "off", "no"].includes(configured);
|
||||
}
|
||||
|
||||
function normalizeProjectionState(value) {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
||||
return value;
|
||||
}
|
||||
|
||||
function normalizeAgentRunMapping(value) {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
||||
}
|
||||
|
||||
function normalizeProjectionRetryParams(value) {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
||||
const message = textValue(value.message ?? value.prompt ?? value.text);
|
||||
if (!message) return null;
|
||||
const result = {
|
||||
message,
|
||||
prompt: message,
|
||||
text: message,
|
||||
sessionId: textValue(value.sessionId) || null,
|
||||
conversationId: textValue(value.conversationId) || null,
|
||||
threadId: textValue(value.threadId) || null,
|
||||
projectId: textValue(value.projectId) || null,
|
||||
ownerUserId: textValue(value.ownerUserId) || null,
|
||||
ownerRole: textValue(value.ownerRole) || null,
|
||||
providerProfile: textValue(value.providerProfile ?? value.codeAgentProviderProfile ?? value.backendProfile) || null,
|
||||
codeAgentProviderProfile: textValue(value.codeAgentProviderProfile ?? value.providerProfile ?? value.backendProfile) || null,
|
||||
backendProfile: textValue(value.backendProfile ?? value.providerProfile ?? value.codeAgentProviderProfile) || null,
|
||||
providerId: textValue(value.providerId) || null,
|
||||
valuesPrinted: false
|
||||
};
|
||||
for (const [key, item] of Object.entries(result)) {
|
||||
if (item === null || item === "") delete result[key];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function normalizeProjectionPolling(value) {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
||||
const result = {
|
||||
pollCount: maxNonNegativeInteger(value.pollCount),
|
||||
noProgressPollCount: maxNonNegativeInteger(value.noProgressPollCount),
|
||||
backoffMs: maxNonNegativeInteger(value.backoffMs),
|
||||
backoffReason: clippedText(value.backoffReason, 80),
|
||||
rootCause: clippedText(value.rootCause, 120),
|
||||
terminalConsistencyGap: maxNonNegativeInteger(value.terminalConsistencyGap),
|
||||
nextRetryAt: timestampText(value.nextRetryAt)
|
||||
};
|
||||
return result;
|
||||
}
|
||||
|
||||
function inferProjectionStatus({ lastSourceSeq, sourceLatestSeq, resultSyncState, previous }) {
|
||||
if (["pending", "timed_out", "failed"].includes(resultSyncState)) return "terminal";
|
||||
if (sourceLatestSeq > lastSourceSeq) return "projecting";
|
||||
return textValue(previous?.projectionStatus) || "projecting";
|
||||
}
|
||||
|
||||
function boundedPageLimit(value) {
|
||||
const parsed = Number.parseInt(String(value ?? ""), 10);
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) return DEFAULT_AGENTRUN_EVENTS_PAGE_LIMIT;
|
||||
return Math.min(parsed, MAX_AGENTRUN_EVENTS_PAGE_LIMIT);
|
||||
}
|
||||
|
||||
function maxNonNegativeInteger(...values) {
|
||||
let max = 0;
|
||||
for (const value of values) {
|
||||
const parsed = Number.parseInt(String(value ?? ""), 10);
|
||||
if (Number.isInteger(parsed) && parsed >= 0 && parsed > max) max = parsed;
|
||||
}
|
||||
return max;
|
||||
}
|
||||
|
||||
function timestampValue(now) {
|
||||
const value = typeof now === "function" ? now() : now;
|
||||
const parsed = Date.parse(String(value ?? ""));
|
||||
return Number.isFinite(parsed) ? new Date(parsed).toISOString() : new Date().toISOString();
|
||||
}
|
||||
|
||||
function textValue(value) {
|
||||
return String(value ?? "").trim();
|
||||
}
|
||||
|
||||
function clippedText(value, limit) {
|
||||
const text = textValue(value);
|
||||
if (!text) return null;
|
||||
return text.length > limit ? text.slice(0, limit) : text;
|
||||
}
|
||||
|
||||
function timestampText(value) {
|
||||
const text = textValue(value);
|
||||
if (!text) return null;
|
||||
const parsed = Date.parse(text);
|
||||
return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null;
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
/*
|
||||
* SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-18-p0-unique-projection; PJ2026-010205 HWLAB接入 draft-2026-06-17-r0.
|
||||
* 职责: WorkbenchProjectionFinalizer 组件入口。以 checkpoint/轮询驱动 AgentRun facts 追平,不让 GET path 承担 finalize/repair。
|
||||
*/
|
||||
import { appendProjectionDiagnostic } from "./workbench-projection-writer.ts";
|
||||
|
||||
export function scheduleWorkbenchProjectionFinalizer({ traceId, currentResult = null, traceStore, getCachedResult, isCanceled, isTerminal, syncResult, onTerminal, activitySignature, noResponseTimeoutMs = null, pollIntervalMs = 1000, sleep = defaultSleep, performanceStore = null } = {}) {
|
||||
if (!traceId || !currentResult?.agentRun?.runId || !currentResult?.agentRun?.commandId || typeof syncResult !== "function") return null;
|
||||
let lastActivityAt = Date.now();
|
||||
let lastActivitySignature = activitySignature?.(currentResult, traceStore?.snapshot?.(traceId)) ?? null;
|
||||
let idleDegraded = false;
|
||||
performanceStore?.recordWorkbenchProjectorCandidate?.({ status: "scheduled", reason: "finalizer" });
|
||||
|
||||
setImmediate(() => {
|
||||
void (async () => {
|
||||
let result = currentResult;
|
||||
let lastError = null;
|
||||
let nextDelayMs = pollIntervalMs;
|
||||
while (true) {
|
||||
const cached = getCachedResult?.(traceId);
|
||||
if (isCanceled?.(cached)) return;
|
||||
if (cached && isTerminal?.(cached, traceStore?.snapshot?.(traceId))) {
|
||||
performanceStore?.recordWorkbenchProjectorCandidate?.({ status: "completed", reason: "cached_terminal" });
|
||||
onTerminal?.(cached, { preserveLastTraceId: true });
|
||||
return;
|
||||
}
|
||||
const syncStartedAt = Date.now();
|
||||
try {
|
||||
const synced = await syncResult({ result });
|
||||
performanceStore?.recordWorkbenchProjectorBatch?.({ phase: "candidate_scan", status: "ok", durationMs: Date.now() - syncStartedAt });
|
||||
result = synced?.result ?? result;
|
||||
nextDelayMs = projectionFinalizerDelayMs(synced?.polling, pollIntervalMs);
|
||||
const runnerTrace = synced?.runnerTrace ?? traceStore?.snapshot?.(traceId);
|
||||
if (result && isTerminal?.(result, runnerTrace)) {
|
||||
performanceStore?.recordWorkbenchProjectorCandidate?.({ status: "completed", reason: "synced_terminal" });
|
||||
onTerminal?.(result, { preserveLastTraceId: true });
|
||||
return;
|
||||
}
|
||||
const signature = activitySignature?.(result, runnerTrace);
|
||||
if (signature && signature !== lastActivitySignature) {
|
||||
lastActivitySignature = signature;
|
||||
lastActivityAt = Date.now();
|
||||
idleDegraded = false;
|
||||
}
|
||||
lastError = null;
|
||||
} catch (error) {
|
||||
performanceStore?.recordWorkbenchProjectorBatch?.({ phase: "candidate_scan", status: error?.code === "agentrun_timeout" ? "timeout" : "error", durationMs: Date.now() - syncStartedAt });
|
||||
performanceStore?.recordWorkbenchProjectorError?.({ reason: error?.code ?? "finalizer_sync_failed" });
|
||||
nextDelayMs = projectionFinalizerDelayMs(error?.polling, pollIntervalMs);
|
||||
lastError = error;
|
||||
}
|
||||
const idleMs = Date.now() - lastActivityAt;
|
||||
if (noResponseTimeoutMs && idleMs >= noResponseTimeoutMs && !idleDegraded) {
|
||||
performanceStore?.recordWorkbenchProjectorCandidate?.({ status: "degraded", reason: "no_response_idle_timeout" });
|
||||
performanceStore?.recordWorkbenchProjectorError?.({ reason: lastError?.code ?? "no_response_idle_timeout" });
|
||||
appendProjectionDiagnostic(traceStore, traceId, {
|
||||
type: "turn-status",
|
||||
status: "degraded",
|
||||
label: "projection-finalizer:no-response-idle-timeout",
|
||||
errorCode: lastError?.code ?? "no_response_idle_timeout",
|
||||
degradedReason: lastError ? "projection_activity_stale" : "no_response_idle_timeout",
|
||||
message: lastError?.message ?? `AgentRun projection has no new activity for ${idleMs}ms; idle threshold ${noResponseTimeoutMs}ms.`,
|
||||
runId: result?.agentRun?.runId ?? currentResult.agentRun.runId,
|
||||
commandId: result?.agentRun?.commandId ?? currentResult.agentRun.commandId,
|
||||
timeoutMs: noResponseTimeoutMs,
|
||||
idleMs,
|
||||
lastActivityAt: new Date(lastActivityAt).toISOString(),
|
||||
waitingFor: "agentrun-result-activity",
|
||||
terminal: false
|
||||
});
|
||||
idleDegraded = true;
|
||||
}
|
||||
await sleep(nextDelayMs);
|
||||
}
|
||||
})().catch((error) => {
|
||||
performanceStore?.recordWorkbenchProjectorError?.({ reason: error?.code ?? "workbench_projection_finalizer_crashed" });
|
||||
appendProjectionDiagnostic(traceStore, traceId, {
|
||||
type: "turn-status",
|
||||
status: "degraded",
|
||||
label: "projection-finalizer:sync-crashed",
|
||||
errorCode: error?.code ?? "workbench_projection_finalizer_crashed",
|
||||
message: error?.message ?? "Workbench projection finalizer crashed.",
|
||||
runId: currentResult.agentRun.runId,
|
||||
commandId: currentResult.agentRun.commandId,
|
||||
waitingFor: "agentrun-result",
|
||||
terminal: false
|
||||
});
|
||||
});
|
||||
});
|
||||
return { scheduled: true, traceId, noResponseTimeoutMs, pollIntervalMs, valuesPrinted: false };
|
||||
}
|
||||
|
||||
function defaultSleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, Math.max(0, Number(ms) || 0)));
|
||||
}
|
||||
|
||||
function projectionFinalizerDelayMs(polling = null, fallbackMs = 1000) {
|
||||
const fallback = positiveInteger(fallbackMs, 1000);
|
||||
const backoffMs = positiveInteger(polling?.backoffMs, 0);
|
||||
return backoffMs > 0 ? Math.max(fallback, backoffMs) : fallback;
|
||||
}
|
||||
|
||||
function positiveInteger(value, fallback) {
|
||||
const parsed = Number.parseInt(String(value ?? ""), 10);
|
||||
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
* Immutable Workbench projection outbox rows are the common SSE and sync-replay authority.
|
||||
*/
|
||||
|
||||
export const WORKBENCH_REALTIME_AUTHORITY_VERSION = "workbench-realtime-authority-v2";
|
||||
|
||||
export function projectionOutboxRealtimeEvents(snapshot = {}, { includeSnapshot = false, includeEvents = true } = {}) {
|
||||
const facts = objectValue(snapshot.facts);
|
||||
const rows = includeEvents ? factArray(snapshot.events) : [];
|
||||
const emitted = new Set();
|
||||
const output = [];
|
||||
for (const row of rows) {
|
||||
const family = textValue(row.entityFamily) || textValue(row.payload?.family) || projectionOutboxFamily(row);
|
||||
const fact = projectionOutboxFact(row);
|
||||
if (!fact || !SUPPORTED_FAMILIES.has(family)) continue;
|
||||
const item = projectionRealtimeItem(row, fact, family);
|
||||
if (!item) continue;
|
||||
emitted.add(`${family}:${item.payload.entity.id}`);
|
||||
output.push(item);
|
||||
}
|
||||
if (includeSnapshot) {
|
||||
const cutoff = nonNegativeInteger(snapshot.cutoffOutboxSeq);
|
||||
for (const [family, name] of [["messages", "workbench.message.snapshot"], ["turns", "workbench.turn.snapshot"]]) {
|
||||
for (const fact of factArray(facts[family])) {
|
||||
const id = projectionFactId(fact, family);
|
||||
if (!id || emitted.has(`${family}:${id}`)) continue;
|
||||
const item = projectionRealtimeItem({ outboxSeq: cutoff, entityFamily: family, entityId: id, projectedSeq: fact.projectedSeq, projectionRevision: fact.projectedSeq, createdAt: fact.updatedAt }, fact, family, name);
|
||||
if (item) output.push(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
return output.sort((left, right) => nonNegativeInteger(left.payload.cursor?.outboxSeq) - nonNegativeInteger(right.payload.cursor?.outboxSeq));
|
||||
}
|
||||
|
||||
function projectionRealtimeItem(row, fact, family, forcedName = null) {
|
||||
const id = textValue(row.entityId) || projectionFactId(fact, family);
|
||||
if (!id) return null;
|
||||
const outboxSeq = nonNegativeInteger(row.outboxSeq);
|
||||
const version = nonNegativeInteger(row.projectionRevision ?? row.projectedSeq ?? fact.projectedSeq ?? outboxSeq);
|
||||
const projectionRevision = String(version);
|
||||
const base = {
|
||||
contractVersion: "workbench-sync-v1",
|
||||
realtimeAuthority: WORKBENCH_REALTIME_AUTHORITY_VERSION,
|
||||
realtimeSource: "projection-outbox",
|
||||
sessionId: textValue(fact.sessionId ?? row.sessionId),
|
||||
traceId: textValue(fact.traceId ?? row.traceId),
|
||||
cursor: { outboxSeq, traceSeq: nonNegativeInteger(row.projectedSeq ?? fact.projectedSeq) },
|
||||
entity: { family, id, version, entityVersion: version, outboxSeq, traceSeq: nonNegativeInteger(row.projectedSeq ?? fact.projectedSeq), projectionRevision, authority: WORKBENCH_REALTIME_AUTHORITY_VERSION },
|
||||
family,
|
||||
id,
|
||||
commitType: textValue(row.commitType),
|
||||
terminal: row.terminal === true || fact.terminal === true,
|
||||
sealed: row.sealed === true || fact.sealed === true,
|
||||
projectionRevision,
|
||||
outboxEventId: textValue(row.outboxEventId),
|
||||
eventCreatedAt: textValue(row.createdAt ?? fact.updatedAt ?? fact.createdAt),
|
||||
valuesRedacted: true
|
||||
};
|
||||
if (family === "messages") return { name: forcedName ?? "workbench.message.snapshot", payload: { ...base, type: "message.snapshot", reason: "projection-outbox", message: fact } };
|
||||
if (family === "turns") return { name: forcedName ?? "workbench.turn.snapshot", payload: { ...base, type: "turn.snapshot", reason: "projection-outbox", turn: fact } };
|
||||
return { name: forcedName ?? "workbench.trace.event", payload: { ...base, type: "trace.event", event: fact, snapshot: { traceId: base.traceId, sessionId: base.sessionId, status: fact.status ?? null, events: [fact], eventCount: 1 } } };
|
||||
}
|
||||
|
||||
function projectionOutboxFact(row) {
|
||||
const fact = row?.payload?.fact;
|
||||
return fact && typeof fact === "object" && !Array.isArray(fact) ? fact : null;
|
||||
}
|
||||
|
||||
function projectionOutboxFamily(row) {
|
||||
if (row.commitType === "message") return "messages";
|
||||
if (row.commitType === "terminal" && row.entityFamily !== "traceEvents") return "turns";
|
||||
return "traceEvents";
|
||||
}
|
||||
|
||||
function projectionFactId(fact, family) {
|
||||
if (family === "messages") return textValue(fact.messageId ?? fact.id);
|
||||
if (family === "turns") return textValue(fact.turnId ?? fact.traceId);
|
||||
return textValue(fact.id ?? fact.sourceEventId);
|
||||
}
|
||||
|
||||
function factArray(value) {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function objectValue(value) {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
||||
}
|
||||
|
||||
function textValue(value) {
|
||||
const text = typeof value === "string" ? value.trim() : value === null || value === undefined ? "" : String(value).trim();
|
||||
return text || "";
|
||||
}
|
||||
|
||||
function nonNegativeInteger(value) {
|
||||
const parsed = Number.parseInt(String(value ?? ""), 10);
|
||||
return Number.isInteger(parsed) && parsed >= 0 ? parsed : 0;
|
||||
}
|
||||
|
||||
const SUPPORTED_FAMILIES = new Set(["messages", "turns", "traceEvents"]);
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -16,7 +16,6 @@ export function createWorkbenchReadModel(options = {}, actor = null) {
|
||||
getSessionByTraceId: (traceId) => facts.getSessionByTraceId(traceId),
|
||||
resultForTrace: (traceId) => facts.resultForTrace(traceId),
|
||||
traceSnapshot: (traceId) => facts.traceSnapshot(traceId),
|
||||
traceSnapshotSync: (traceId) => facts.traceSnapshotSync(traceId),
|
||||
canReadOwner: (ownerUserId) => facts.canReadOwner(ownerUserId),
|
||||
projectionDiagnostics: ({ traceId, result = null, trace = null, projection = null, refreshError = null } = {}) => workbenchProjectionDiagnostics({ traceId, result, trace, projection, refreshError })
|
||||
};
|
||||
|
||||
@@ -7,8 +7,61 @@ import { test } from "bun:test";
|
||||
import { createCloudApiServer } from "./server.ts";
|
||||
|
||||
const ACTOR = { id: "usr_workbench_realtime_authority", username: "reader", displayName: "Reader", role: "user", status: "active" };
|
||||
const PROJECTION_REALTIME_ENV = Object.freeze({
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false",
|
||||
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
|
||||
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true"
|
||||
});
|
||||
const LIVE_REALTIME_ENV = Object.freeze({
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
|
||||
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
|
||||
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false"
|
||||
});
|
||||
|
||||
test("workbench sync delta stays durable while SSE realtime forwards Kafka events", async () => {
|
||||
test("workbench sync fails closed before projection storage in live Kafka capability", async () => {
|
||||
let syncReads = 0;
|
||||
const server = createCloudApiServer({
|
||||
accessController: createAccessController(),
|
||||
workbenchRuntime: {
|
||||
async readAtomicWorkbenchProjectionSync() {
|
||||
syncReads += 1;
|
||||
throw new Error("live Kafka sync must not reach projection storage");
|
||||
}
|
||||
},
|
||||
kafkaEventBridge: {
|
||||
started: true,
|
||||
capabilities: {
|
||||
directPublish: true,
|
||||
liveKafkaSse: true,
|
||||
transactionalProjector: false,
|
||||
projectionOutboxRelay: false,
|
||||
projectionRealtime: false
|
||||
},
|
||||
ready: Promise.resolve(),
|
||||
subscribeLiveHwlabEvents() { return () => {}; },
|
||||
async stop() {}
|
||||
},
|
||||
env: { ...LIVE_REALTIME_ENV }
|
||||
});
|
||||
await listen(server);
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const response = await getJson(port, "/v1/workbench/sync?sessionId=ses_live_only&since=0");
|
||||
assert.equal(response.status, 503);
|
||||
assert.equal(response.body.error.code, "workbench_projection_realtime_disabled");
|
||||
assert.equal(response.body.error.capabilities.liveKafkaSse, true);
|
||||
assert.equal(syncReads, 0);
|
||||
} finally {
|
||||
await close(server);
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench sync and SSE read the same atomic projection outbox", async () => {
|
||||
const sessionId = "ses_realtime_authority_p1";
|
||||
const traceId = "trc_realtime_authority_p1";
|
||||
const outboxRows = [{
|
||||
@@ -28,20 +81,33 @@ test("workbench sync delta stays durable while SSE realtime forwards Kafka event
|
||||
terminal: true,
|
||||
sealed: true,
|
||||
createdAt: "2026-07-08T12:20:00.000Z",
|
||||
payload: { type: "terminal", status: "completed", text: "redacted final" }
|
||||
entityFamily: "turns",
|
||||
entityId: traceId,
|
||||
payload: {
|
||||
family: "turns",
|
||||
fact: {
|
||||
turnId: traceId,
|
||||
sessionId,
|
||||
traceId,
|
||||
messageId: "msg_realtime_authority_agent",
|
||||
status: "completed",
|
||||
projectedSeq: 8,
|
||||
terminal: true,
|
||||
sealed: true,
|
||||
finalResponse: { text: "redacted final", status: "completed", traceId }
|
||||
},
|
||||
valuesRedacted: true
|
||||
}
|
||||
}];
|
||||
const outboxQueries = [];
|
||||
const runtime = createRuntime({ facts: durableFacts({ sessionId, traceId }), outboxRows, outboxQueries });
|
||||
const fakeKafka = createFakeKafkaFactory();
|
||||
const server = createCloudApiServer({
|
||||
accessController: createAccessController(),
|
||||
workbenchRuntime: runtime,
|
||||
kafkaFactory: fakeKafka.factory,
|
||||
env: {
|
||||
...PROJECTION_REALTIME_ENV,
|
||||
HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000",
|
||||
HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092",
|
||||
HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api",
|
||||
HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1"
|
||||
HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100"
|
||||
}
|
||||
});
|
||||
await listen(server);
|
||||
@@ -59,23 +125,16 @@ test("workbench sync delta stays durable while SSE realtime forwards Kafka event
|
||||
assert.equal(sync.body.events[0].terminal, true);
|
||||
assert.equal(sync.body.events[0].sealed, true);
|
||||
|
||||
setTimeout(() => { void fakeKafka.emit({
|
||||
eventType: "hwlab.trace.event.projected",
|
||||
sessionId: "ses_agentrun_realtime_authority_p1",
|
||||
traceId,
|
||||
context: { sourceSeq: 8, runId: "run_realtime_authority_p1", commandId: "cmd_realtime_authority_p1" },
|
||||
event: { type: "result", eventType: "terminal", status: "completed", label: "agentrun:terminal:completed", message: "redacted final", terminal: true, sourceSeq: 8 }
|
||||
}); }, 0);
|
||||
const events = await getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(sessionId)}&traceId=${encodeURIComponent(traceId)}&afterSeq=10`, 2);
|
||||
assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.trace.event"]);
|
||||
assert.equal(events[1].data.realtimeSource, "kafka");
|
||||
assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.turn.snapshot"]);
|
||||
assert.equal(events[1].data.realtimeSource, "projection-outbox");
|
||||
assert.equal(events[1].data.realtimeAuthority, "workbench-realtime-authority-v2");
|
||||
assert.equal(events[1].data.entity.family, "traceEvents");
|
||||
assert.equal(events[1].data.entity.version, 8);
|
||||
assert.equal(events[1].data.entity.projectionRevision, "kafka:hwlab.event.v1:0:0");
|
||||
assert.equal(events[1].data.event.message, "redacted final");
|
||||
assert.deepEqual(outboxQueries.map((query) => query.afterSeq), [10]);
|
||||
assert.equal(events[1].data.entity.family, "turns");
|
||||
assert.equal(events[1].data.entity.version, 42);
|
||||
assert.equal(events[1].data.turn.finalResponse.text, "redacted final");
|
||||
assert.deepEqual(outboxQueries.map((query) => query.afterOutboxSeq), [10, 10]);
|
||||
assert.equal(outboxQueries.every((query) => query.sessionId === sessionId), true);
|
||||
assert.deepEqual(outboxQueries.at(-1).actor, { id: ACTOR.id, role: ACTOR.role });
|
||||
} finally {
|
||||
await close(server);
|
||||
}
|
||||
@@ -93,7 +152,7 @@ test("workbench sync delta includes all authority families and marks trace rows
|
||||
}),
|
||||
outboxRows: []
|
||||
});
|
||||
const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: runtime });
|
||||
const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: runtime, env: { ...PROJECTION_REALTIME_ENV } });
|
||||
await listen(server);
|
||||
|
||||
try {
|
||||
@@ -118,6 +177,27 @@ test("workbench sync delta includes all authority families and marks trace rows
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench sync preserves session and trace scope for narrow replay", async () => {
|
||||
const sessionId = "ses_realtime_authority_narrow";
|
||||
const traceId = "trc_realtime_authority_narrow";
|
||||
const outboxQueries = [];
|
||||
const runtime = createRuntime({ facts: durableFacts({ sessionId, traceId }), outboxQueries });
|
||||
const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: runtime, env: { ...PROJECTION_REALTIME_ENV } });
|
||||
await listen(server);
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const sync = await getJson(port, `/v1/workbench/sync?sessionId=${encodeURIComponent(sessionId)}&traceId=${encodeURIComponent(traceId)}&since=7`);
|
||||
assert.equal(sync.status, 200);
|
||||
assert.deepEqual(sync.body.scope, { kind: "trace", sessionId, traceId, since: 7 });
|
||||
assert.equal(outboxQueries.length, 1);
|
||||
assert.equal(outboxQueries[0].sessionId, sessionId);
|
||||
assert.equal(outboxQueries[0].traceId, traceId);
|
||||
} finally {
|
||||
await close(server);
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench trace event detail route declares detail-only authority", async () => {
|
||||
const sessionId = "ses_realtime_authority_detail";
|
||||
const traceId = "trc_realtime_authority_detail";
|
||||
@@ -125,7 +205,7 @@ test("workbench trace event detail route declares detail-only authority", async
|
||||
facts: durableFacts({ sessionId, traceId, finalText: "sealed final", traceEventText: "detail row" }),
|
||||
outboxRows: []
|
||||
});
|
||||
const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: runtime });
|
||||
const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: runtime, env: { ...PROJECTION_REALTIME_ENV } });
|
||||
await listen(server);
|
||||
|
||||
try {
|
||||
@@ -144,7 +224,7 @@ test("workbench trace event detail route declares detail-only authority", async
|
||||
});
|
||||
|
||||
test("workbench sync rejects unscoped automatic repair requests", async () => {
|
||||
const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: createRuntime({ facts: durableFacts({}) }) });
|
||||
const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: createRuntime({ facts: durableFacts({}) }), env: { ...PROJECTION_REALTIME_ENV } });
|
||||
await listen(server);
|
||||
|
||||
try {
|
||||
@@ -168,9 +248,19 @@ function createAccessController() {
|
||||
|
||||
function createRuntime({ facts, outboxRows = [], outboxQueries = [] }) {
|
||||
return {
|
||||
async readWorkbenchProjectionOutbox(params = {}) {
|
||||
async readAtomicWorkbenchProjectionSync(params = {}) {
|
||||
outboxQueries.push({ ...params });
|
||||
return outboxRows.filter((row) => Number(row.outboxSeq) > Number(params.afterSeq ?? 0));
|
||||
const after = Number(params.afterOutboxSeq ?? params.afterSeq ?? 0);
|
||||
const events = outboxRows.filter((row) => Number(row.outboxSeq) > after);
|
||||
const cutoffOutboxSeq = outboxRows.reduce((max, row) => Math.max(max, Number(row.outboxSeq) || 0), 0);
|
||||
return {
|
||||
facts: filterFacts(facts, params),
|
||||
events,
|
||||
cutoffOutboxSeq,
|
||||
cursorOutboxSeq: cutoffOutboxSeq,
|
||||
hasMore: false,
|
||||
valuesRedacted: true
|
||||
};
|
||||
},
|
||||
async queryWorkbenchFacts(params = {}) {
|
||||
return {
|
||||
@@ -310,44 +400,6 @@ function parseSseBlock(block) {
|
||||
return { event, id, data: JSON.parse(dataLine) };
|
||||
}
|
||||
|
||||
function createFakeKafkaFactory() {
|
||||
let eachMessage = null;
|
||||
let offset = 0;
|
||||
let subscribedTopic = "hwlab.event.v1";
|
||||
const consumer = {
|
||||
connect: async () => undefined,
|
||||
subscribe: async (input = {}) => { subscribedTopic = String(input.topic ?? subscribedTopic); },
|
||||
run: async (input = {}) => { eachMessage = input.eachMessage; },
|
||||
stop: async () => undefined,
|
||||
disconnect: async () => undefined
|
||||
};
|
||||
return {
|
||||
factory: () => ({ consumer: () => consumer }),
|
||||
emit: async (value = {}) => {
|
||||
await waitFor(() => typeof eachMessage === "function");
|
||||
await eachMessage({
|
||||
topic: subscribedTopic,
|
||||
partition: 0,
|
||||
message: {
|
||||
offset: String(offset++),
|
||||
key: Buffer.from(value.traceId ?? value.sessionId ?? "fake"),
|
||||
timestamp: String(Date.now()),
|
||||
value: Buffer.from(JSON.stringify(value))
|
||||
}
|
||||
});
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function waitFor(predicate, timeoutMs = 1000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (predicate()) return;
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
throw new Error("waitFor timed out");
|
||||
}
|
||||
|
||||
function listen(server) {
|
||||
return new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
}
|
||||
|
||||
@@ -4,18 +4,31 @@
|
||||
*/
|
||||
import { safeSessionId, safeTraceId, sendJson } from "./server-http-utils.ts";
|
||||
import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts";
|
||||
import { projectionOutboxRealtimeEvents } from "./workbench-projection-outbox-events.ts";
|
||||
import { workbenchRealtimeCapabilities } from "./workbench-realtime-capabilities.ts";
|
||||
|
||||
const SYNC_CONTRACT_VERSION = "workbench-sync-v1";
|
||||
const REALTIME_AUTHORITY_VERSION = "workbench-realtime-authority-v2";
|
||||
const DEFAULT_SYNC_LIMIT = 100;
|
||||
const MAX_SYNC_LIMIT = 500;
|
||||
const SYNC_FACT_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "traceEvents", "checkpoints"]);
|
||||
|
||||
export async function handleWorkbenchSyncHttp(request, response, url, options = {}, actor = null) {
|
||||
if (request.method !== "GET") {
|
||||
sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "GET required." } });
|
||||
return;
|
||||
}
|
||||
const capabilities = workbenchRealtimeCapabilities(options.env ?? process.env);
|
||||
if (!capabilities.projectionRealtime) {
|
||||
sendJson(response, 503, {
|
||||
ok: false,
|
||||
error: {
|
||||
code: "workbench_projection_realtime_disabled",
|
||||
message: "Workbench projection sync/replay capability is disabled.",
|
||||
valuesRedacted: true
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
const sessionId = safeSessionId(url.searchParams.get("sessionId") ?? url.searchParams.get("includeSessionId"));
|
||||
const traceId = safeTraceId(url.searchParams.get("traceId"));
|
||||
if (!sessionId && !traceId) {
|
||||
@@ -30,12 +43,12 @@ export async function handleWorkbenchSyncHttp(request, response, url, options =
|
||||
return;
|
||||
}
|
||||
const runtime = workbenchRealtimeRuntime(request, options);
|
||||
if (typeof runtime?.queryWorkbenchFacts !== "function" || typeof runtime?.readWorkbenchProjectionOutbox !== "function") {
|
||||
if (typeof runtime?.readAtomicWorkbenchProjectionSync !== "function") {
|
||||
sendJson(response, 503, {
|
||||
ok: false,
|
||||
error: {
|
||||
code: "workbench_sync_runtime_unconfigured",
|
||||
message: "Workbench sync requires durable facts and outbox readers.",
|
||||
message: "Workbench sync requires one atomic durable projection snapshot reader.",
|
||||
valuesRedacted: true
|
||||
}
|
||||
});
|
||||
@@ -45,27 +58,25 @@ export async function handleWorkbenchSyncHttp(request, response, url, options =
|
||||
try {
|
||||
const since = realtimeSinceCursor(request, url);
|
||||
const limit = boundedSyncLimit(url.searchParams.get("limit"));
|
||||
const outboxQuery = { afterSeq: since, limit };
|
||||
if (sessionId) outboxQuery.sessionId = sessionId;
|
||||
else outboxQuery.traceId = traceId;
|
||||
const rows = await runtime.readWorkbenchProjectionOutbox(outboxQuery);
|
||||
const events = rows.map(workbenchRealtimeEntityDelta);
|
||||
const latestOutboxSeq = events.reduce((max, event) => Math.max(max, Number(event.cursor?.outboxSeq ?? 0)), since);
|
||||
const facts = await runtime.queryWorkbenchFacts({
|
||||
families: SYNC_FACT_FAMILIES,
|
||||
...(sessionId ? { sessionId } : { traceId }),
|
||||
const snapshot = await runtime.readAtomicWorkbenchProjectionSync({
|
||||
afterOutboxSeq: since,
|
||||
afterSeq: since,
|
||||
limit,
|
||||
actor: actor ? { id: actor.id, role: actor.role ?? "user", valuesRedacted: true } : undefined
|
||||
...(sessionId ? { sessionId } : {}),
|
||||
...(traceId ? { traceId } : {}),
|
||||
actor: actor ? { id: actor.id, role: actor.role ?? "user" } : undefined
|
||||
});
|
||||
if (facts?.error) throw facts.error;
|
||||
const delta = normalizeSyncFacts(facts?.facts);
|
||||
const rows = Array.isArray(snapshot?.events) ? snapshot.events : [];
|
||||
const events = projectionOutboxRealtimeEvents({ events: rows, facts: {} }).map((item) => item.payload);
|
||||
const latestOutboxSeq = nonNegativeInteger(snapshot?.cursorOutboxSeq);
|
||||
const delta = normalizeSyncFacts(snapshot?.facts);
|
||||
sendJson(response, 200, {
|
||||
ok: true,
|
||||
status: "succeeded",
|
||||
contractVersion: SYNC_CONTRACT_VERSION,
|
||||
realtimeAuthority: REALTIME_AUTHORITY_VERSION,
|
||||
scope: {
|
||||
kind: sessionId ? "session" : "trace",
|
||||
kind: traceId ? "trace" : "session",
|
||||
sessionId,
|
||||
traceId,
|
||||
since
|
||||
@@ -73,7 +84,8 @@ export async function handleWorkbenchSyncHttp(request, response, url, options =
|
||||
cursor: {
|
||||
outboxSeq: latestOutboxSeq,
|
||||
since,
|
||||
hasMore: rows.length >= limit
|
||||
cutoffOutboxSeq: nonNegativeInteger(snapshot?.cutoffOutboxSeq),
|
||||
hasMore: snapshot?.hasMore === true
|
||||
},
|
||||
events,
|
||||
delta,
|
||||
@@ -101,48 +113,8 @@ export async function handleWorkbenchSyncHttp(request, response, url, options =
|
||||
}
|
||||
}
|
||||
|
||||
export function workbenchRealtimeEntityDelta(row = {}) {
|
||||
const commitType = textValue(row.commitType) || "event";
|
||||
const projectionRevision = nonNegativeInteger(row.projectionRevision ?? row.projectedSeq ?? row.outboxSeq);
|
||||
const family = workbenchRealtimeEntityFamily(row, commitType);
|
||||
const id = workbenchRealtimeEntityId(row, family);
|
||||
const outboxSeq = nonNegativeInteger(row.outboxSeq);
|
||||
const projectedSeq = nonNegativeInteger(row.projectedSeq);
|
||||
return {
|
||||
type: "workbench.entity.delta",
|
||||
family,
|
||||
id,
|
||||
entity: {
|
||||
family,
|
||||
id,
|
||||
version: projectionRevision,
|
||||
entityVersion: projectionRevision
|
||||
},
|
||||
cursor: {
|
||||
outboxSeq,
|
||||
traceSeq: projectedSeq
|
||||
},
|
||||
sessionId: safeSessionId(row.sessionId) ?? null,
|
||||
turnId: textValue(row.turnId) || null,
|
||||
traceId: safeTraceId(row.traceId) ?? null,
|
||||
messageId: textValue(row.messageId) || null,
|
||||
commitType,
|
||||
eventSeq: row.eventSeq === null || row.eventSeq === undefined ? null : nonNegativeInteger(row.eventSeq),
|
||||
aggregateId: textValue(row.aggregateId) || null,
|
||||
aggregateSeq: nonNegativeInteger(row.aggregateSeq),
|
||||
projectionRevision,
|
||||
entityVersion: projectionRevision,
|
||||
terminal: row.terminal === true,
|
||||
sealed: row.sealed === true,
|
||||
serverCommittedAt: textValue(row.createdAt) || null,
|
||||
payload: redactedPayload(row.payload),
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function workbenchRealtimeRuntime(request, options = {}) {
|
||||
return options.workbenchRuntime
|
||||
?? options.runtimeStore
|
||||
?? createWorkbenchRuntimeClient({ env: options.env ?? process.env, fetch: options.fetch, logger: options.logger, traceparent: request?.hwlabHttpRequestContext?.traceparent });
|
||||
}
|
||||
|
||||
@@ -175,31 +147,6 @@ function normalizeSyncFacts(facts = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
function workbenchRealtimeEntityFamily(row = {}, commitType = "event") {
|
||||
if (commitType === "terminal" || row.terminal === true || row.sealed === true) return "turns";
|
||||
if (commitType === "message" || row.messageId) return "messages";
|
||||
if (row.traceId) return "traceEvents";
|
||||
if (row.sessionId) return "sessions";
|
||||
return "diagnostics";
|
||||
}
|
||||
|
||||
function workbenchRealtimeEntityId(row = {}, family = "diagnostics") {
|
||||
if (family === "messages") return textValue(row.messageId) || textValue(row.turnId) || textValue(row.traceId) || `outbox:${nonNegativeInteger(row.outboxSeq)}`;
|
||||
if (family === "turns") return textValue(row.turnId) || textValue(row.traceId) || `outbox:${nonNegativeInteger(row.outboxSeq)}`;
|
||||
if (family === "traceEvents") return textValue(row.sourceEventId) || [textValue(row.traceId), nonNegativeInteger(row.projectedSeq)].filter(Boolean).join(":") || `outbox:${nonNegativeInteger(row.outboxSeq)}`;
|
||||
if (family === "sessions") return textValue(row.sessionId) || `outbox:${nonNegativeInteger(row.outboxSeq)}`;
|
||||
return textValue(row.aggregateId) || `outbox:${nonNegativeInteger(row.outboxSeq)}`;
|
||||
}
|
||||
|
||||
function redactedPayload(payload) {
|
||||
if (!payload || typeof payload !== "object" || Array.isArray(payload)) return null;
|
||||
return {
|
||||
kind: textValue(payload.kind ?? payload.type ?? payload.eventType) || null,
|
||||
status: textValue(payload.status) || null,
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function factArray(value) {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
// SPEC: PJ2026-0104010803 Workbench composable realtime capabilities.
|
||||
// Responsibility: validate YAML-owned capability switches without code defaults or implied mode selection.
|
||||
|
||||
export const WORKBENCH_REALTIME_CAPABILITY_ENVS = Object.freeze({
|
||||
directPublish: "HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED",
|
||||
liveKafkaSse: "HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED",
|
||||
transactionalProjector: "HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED",
|
||||
projectionOutboxRelay: "HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED",
|
||||
projectionRealtime: "HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED"
|
||||
});
|
||||
|
||||
export function workbenchRealtimeCapabilities(env = process.env) {
|
||||
return Object.fromEntries(Object.entries(WORKBENCH_REALTIME_CAPABILITY_ENVS).map(([key, name]) => [key, requiredBooleanEnv(env, name)]));
|
||||
}
|
||||
|
||||
export function requiredBooleanEnv(env, name) {
|
||||
const value = String(env?.[name] ?? "").trim().toLowerCase();
|
||||
if (value === "true" || value === "1") return true;
|
||||
if (value === "false" || value === "0") return false;
|
||||
throw capabilityError(
|
||||
"hwlab_workbench_realtime_capability_invalid",
|
||||
`${name} is required and must be explicitly true or false in the owning YAML.`,
|
||||
name
|
||||
);
|
||||
}
|
||||
|
||||
function capabilityError(code, message, envName) {
|
||||
return Object.assign(new Error(message), {
|
||||
code,
|
||||
envName,
|
||||
statusCode: 503,
|
||||
valuesRedacted: true
|
||||
});
|
||||
}
|
||||
@@ -17,6 +17,7 @@ export function createWorkbenchRuntimeClient(options = {}) {
|
||||
queryWorkbenchFacts: (params = {}) => postRuntime({ fetchFn, baseUrl, timeoutMs, traceparent, path: "/v1/workbench-runtime/facts", body: params }),
|
||||
queryAgentTraceEvents: (params = {}) => postRuntime({ fetchFn, baseUrl, timeoutMs, traceparent, path: "/v1/workbench-runtime/trace-events", body: params }),
|
||||
listWorkbenchSessions: (params = {}) => postRuntime({ fetchFn, baseUrl, timeoutMs, traceparent, path: "/v1/workbench-runtime/sessions", body: params }),
|
||||
readAtomicWorkbenchProjectionSync: (params = {}) => postRuntime({ fetchFn, baseUrl, timeoutMs, traceparent, path: "/v1/workbench-runtime/sync", body: params }),
|
||||
readWorkbenchProjectionOutbox: (params = {}) => postRuntime({ fetchFn, baseUrl, timeoutMs, traceparent, path: "/v1/workbench-runtime/projection-outbox", body: params }).then((result) => Array.isArray(result?.rows) ? result.rows : [])
|
||||
};
|
||||
}
|
||||
|
||||
@@ -39,8 +39,8 @@ export function createWorkbenchTurnProjection({ turnId = null, traceId = null, r
|
||||
finalResponse: finalText ? { text: finalText, status, traceId: projectionTraceId, valuesPrinted: false } : null,
|
||||
assistantText: finalText,
|
||||
lastProjectedSeq: traceLastSeq(trace),
|
||||
sourceRunId: agentRun?.runId ?? lastEvent?.runId ?? lastEvent?.payload?.runId ?? null,
|
||||
sourceCommandId: agentRun?.commandId ?? lastEvent?.commandId ?? lastEvent?.payload?.commandId ?? null,
|
||||
sourceRunId: lastEvent?.runId ?? lastEvent?.payload?.runId ?? agentRun?.runId ?? null,
|
||||
sourceCommandId: lastEvent?.commandId ?? lastEvent?.payload?.commandId ?? agentRun?.commandId ?? null,
|
||||
eventCount: normalizedEventCount(trace),
|
||||
updatedAt: trace?.updatedAt ?? result?.updatedAt ?? session?.updatedAt ?? null,
|
||||
timing,
|
||||
@@ -164,7 +164,13 @@ export function traceTerminalEvidence(trace = null) {
|
||||
if (direct) {
|
||||
const status = terminalStatusFromValue(direct.status ?? direct.terminalStatus ?? trace?.status) ?? "completed";
|
||||
if (status !== "completed" && retryableProviderInterruptionEvidence(direct, trace)) return null;
|
||||
return { source: "trace-terminal-evidence", status, evidence: direct, valuesRedacted: true };
|
||||
return {
|
||||
source: "trace-terminal-evidence",
|
||||
status,
|
||||
finalResponse: direct.finalResponse ?? null,
|
||||
evidence: direct,
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
const events = Array.isArray(trace?.events) ? trace.events : [];
|
||||
return terminalTraceEventEvidence(events);
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
-- Versioned v8 upgrade for immutable Workbench replay and the AgentRun Kafka projector.
|
||||
|
||||
ALTER TABLE workbench_projection_outbox ADD COLUMN IF NOT EXISTS outbox_event_id TEXT;
|
||||
ALTER TABLE workbench_projection_outbox ADD COLUMN IF NOT EXISTS entity_family TEXT;
|
||||
ALTER TABLE workbench_projection_outbox ADD COLUMN IF NOT EXISTS entity_id TEXT;
|
||||
|
||||
-- Pre-v8 rows did not store immutable fact payloads. Mark them as cutover rows so
|
||||
-- recovery advances its cursor without fabricating message/turn/trace entities.
|
||||
UPDATE workbench_projection_outbox
|
||||
SET outbox_event_id = COALESCE(outbox_event_id, 'legacy:' || outbox_seq::text),
|
||||
entity_family = COALESCE(entity_family, 'legacy'),
|
||||
entity_id = COALESCE(entity_id, 'legacy:' || outbox_seq::text)
|
||||
WHERE outbox_event_id IS NULL OR entity_family IS NULL OR entity_id IS NULL;
|
||||
|
||||
ALTER TABLE workbench_projection_outbox ALTER COLUMN outbox_event_id SET NOT NULL;
|
||||
ALTER TABLE workbench_projection_outbox ALTER COLUMN entity_family SET NOT NULL;
|
||||
ALTER TABLE workbench_projection_outbox ALTER COLUMN entity_id SET NOT NULL;
|
||||
DROP INDEX IF EXISTS idx_workbench_projection_outbox_event_id;
|
||||
CREATE UNIQUE INDEX idx_workbench_projection_outbox_event_id ON workbench_projection_outbox(outbox_event_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS workbench_kafka_inbox (
|
||||
source_topic TEXT NOT NULL,
|
||||
source_partition INTEGER NOT NULL,
|
||||
source_offset BIGINT NOT NULL,
|
||||
source_key TEXT,
|
||||
source_event_id TEXT,
|
||||
input_sha256 TEXT NOT NULL,
|
||||
trace_id TEXT,
|
||||
session_id TEXT,
|
||||
run_id TEXT,
|
||||
command_id TEXT,
|
||||
source_seq INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL DEFAULT 'processing',
|
||||
projected_seq INTEGER,
|
||||
envelope_json TEXT NOT NULL DEFAULT '{}',
|
||||
error_json TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
projected_at TEXT,
|
||||
PRIMARY KEY (source_topic, source_partition, source_offset)
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_workbench_kafka_inbox_source_event ON workbench_kafka_inbox(source_topic, source_event_id) WHERE source_event_id IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_workbench_kafka_inbox_status_updated ON workbench_kafka_inbox(status, updated_at, source_topic, source_partition, source_offset);
|
||||
CREATE INDEX IF NOT EXISTS idx_workbench_kafka_inbox_trace ON workbench_kafka_inbox(trace_id, source_seq);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS workbench_kafka_dlq (
|
||||
dlq_seq BIGSERIAL PRIMARY KEY,
|
||||
source_topic TEXT NOT NULL,
|
||||
source_partition INTEGER NOT NULL,
|
||||
source_offset BIGINT NOT NULL,
|
||||
source_event_id TEXT,
|
||||
input_sha256 TEXT NOT NULL,
|
||||
error_code TEXT NOT NULL,
|
||||
error_message TEXT NOT NULL,
|
||||
envelope_json TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_workbench_kafka_dlq_transport ON workbench_kafka_dlq(source_topic, source_partition, source_offset);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS hwlab_kafka_outbox (
|
||||
outbox_seq BIGSERIAL PRIMARY KEY,
|
||||
event_id TEXT NOT NULL UNIQUE,
|
||||
topic TEXT NOT NULL,
|
||||
partition_key TEXT NOT NULL,
|
||||
payload_json TEXT NOT NULL,
|
||||
headers_json TEXT NOT NULL DEFAULT '{}',
|
||||
attempt_count INTEGER NOT NULL DEFAULT 0,
|
||||
next_attempt_at TEXT NOT NULL,
|
||||
lease_owner TEXT,
|
||||
lease_expires_at TEXT,
|
||||
published_at TEXT,
|
||||
last_error TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_hwlab_kafka_outbox_due ON hwlab_kafka_outbox(published_at, next_attempt_at, lease_expires_at, outbox_seq);
|
||||
|
||||
CREATE OR REPLACE FUNCTION notify_hwlab_workbench_projection() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
PERFORM pg_notify(
|
||||
'hwlab_workbench_projection',
|
||||
json_build_object(
|
||||
'sessionId', NEW.session_id,
|
||||
'traceId', NEW.trace_id,
|
||||
'outboxSeq', NEW.outbox_seq
|
||||
)::text
|
||||
);
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_notify_hwlab_workbench_projection ON workbench_projection_outbox;
|
||||
CREATE TRIGGER trg_notify_hwlab_workbench_projection
|
||||
AFTER INSERT ON workbench_projection_outbox
|
||||
FOR EACH ROW EXECUTE FUNCTION notify_hwlab_workbench_projection();
|
||||
|
||||
INSERT INTO hwlab_schema_migrations (id, schema_version, applied_at, migration_json)
|
||||
VALUES (
|
||||
'0008_workbench_kafka_realtime',
|
||||
'runtime-durable-postgres-v8',
|
||||
CURRENT_TIMESTAMP,
|
||||
'{"path":"internal/db/migrations/0008_workbench_kafka_realtime.sql","runtime":"cloud-api"}'
|
||||
)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
schema_version = EXCLUDED.schema_version,
|
||||
migration_json = EXCLUDED.migration_json;
|
||||
@@ -421,7 +421,7 @@ export function workbenchInputDelivery(value) {
|
||||
|
||||
export function workbenchInputStatus(value) {
|
||||
const text = textOr(value, "admitted").toLowerCase().replace(/-/gu, "_");
|
||||
return ["admitted", "promoted", "blocked", "failed", "canceled", "completed"].includes(text) ? text : "admitted";
|
||||
return ["admitting", "admitted", "promoted", "blocked", "failed", "canceled", "completed"].includes(text) ? text : "admitted";
|
||||
}
|
||||
|
||||
export function indexWorkbenchAggregateEvents(events = []) {
|
||||
@@ -1301,7 +1301,7 @@ export function isRuntimeDbSslError(error) {
|
||||
].some((pattern) => message.includes(pattern));
|
||||
}
|
||||
|
||||
export function summarizeRuntimeSchema(rows) {
|
||||
export function summarizeRuntimeSchema(rows, requiredSchema = requiredPostgresSchema) {
|
||||
const columnsByTable = new Map();
|
||||
for (const row of rows) {
|
||||
const table = row.table_name;
|
||||
@@ -1315,7 +1315,7 @@ export function summarizeRuntimeSchema(rows) {
|
||||
|
||||
const missingTables = [];
|
||||
const missingColumns = [];
|
||||
for (const [table, columns] of Object.entries(requiredPostgresSchema)) {
|
||||
for (const [table, columns] of Object.entries(requiredSchema)) {
|
||||
const observed = columnsByTable.get(table);
|
||||
if (!observed) {
|
||||
missingTables.push(table);
|
||||
@@ -1332,7 +1332,7 @@ export function summarizeRuntimeSchema(rows) {
|
||||
return {
|
||||
ready: missingTables.length === 0 && missingColumns.length === 0,
|
||||
checked: true,
|
||||
requiredTables: Object.keys(requiredPostgresSchema),
|
||||
requiredTables: Object.keys(requiredSchema),
|
||||
missingTables,
|
||||
missingColumns
|
||||
};
|
||||
|
||||
@@ -16,7 +16,6 @@ import {
|
||||
deriveProtocolActorFromMeta
|
||||
} from "../audit/index.mjs";
|
||||
import {
|
||||
CLOUD_CORE_MIGRATION_ID,
|
||||
CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION,
|
||||
CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE,
|
||||
CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS
|
||||
@@ -214,6 +213,20 @@ export class CloudRuntimeStore {
|
||||
});
|
||||
}
|
||||
|
||||
async workbenchTransactionalRealtimeReadiness() {
|
||||
return {
|
||||
ready: true,
|
||||
adapter: RUNTIME_STORE_KIND,
|
||||
durable: false,
|
||||
testRuntime: true,
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
async assertWorkbenchTransactionalRealtimeReady() {
|
||||
return this.workbenchTransactionalRealtimeReadiness();
|
||||
}
|
||||
|
||||
registerGatewaySession(params = {}, requestMeta = {}) {
|
||||
const now = this.now();
|
||||
const input = asObject(params.gatewaySession ?? params, "gatewaySession");
|
||||
@@ -665,9 +678,15 @@ export class CloudRuntimeStore {
|
||||
const inputFacts = facts.inputs.map((fact) => memoryWorkbenchSessionInputWithSeq(fact, this.workbenchSessionInputs));
|
||||
for (const fact of inputFacts) this.workbenchSessionInputs.set(fact.inputId, fact);
|
||||
for (const fact of facts.sessions) this.workbenchSessions.set(fact.sessionId, fact);
|
||||
for (const fact of facts.messages) this.workbenchMessages.set(fact.messageId, fact);
|
||||
for (const fact of facts.messages) {
|
||||
const previous = this.workbenchMessages.get(fact.messageId) ?? null;
|
||||
this.workbenchMessages.set(fact.messageId, previous && (nonNegativeInteger(previous.projectedSeq) > 0 || previous.sealed === true) ? previous : fact);
|
||||
}
|
||||
for (const fact of facts.parts) this.workbenchParts.set(fact.partId, fact);
|
||||
for (const fact of facts.turns) this.workbenchTurns.set(fact.turnId, fact);
|
||||
for (const fact of facts.turns) {
|
||||
const previous = this.workbenchTurns.get(fact.turnId) ?? null;
|
||||
this.workbenchTurns.set(fact.turnId, previous && (nonNegativeInteger(previous.projectedSeq) > 0 || previous.sealed === true) ? previous : fact);
|
||||
}
|
||||
for (const fact of facts.traceEvents) this.workbenchTraceEvents.set(fact.id, fact);
|
||||
for (const fact of facts.checkpoints) {
|
||||
const previous = this.workbenchProjectionCheckpoints.get(fact.traceId) ?? null;
|
||||
@@ -682,6 +701,57 @@ export class CloudRuntimeStore {
|
||||
};
|
||||
}
|
||||
|
||||
writeWorkbenchSessionAdmissionFact(params = {}, requestMeta = {}) {
|
||||
const facts = normalizeWorkbenchFacts({ sessions: [params.fact ?? params.session ?? params] }, requestMeta, this.now());
|
||||
const fact = facts.sessions[0];
|
||||
if (!fact?.sessionId) throw new Error("Workbench session admission fact requires sessionId.");
|
||||
const previous = this.workbenchSessions.get(fact.sessionId) ?? null;
|
||||
const projected = previous && (nonNegativeInteger(previous.projectedSeq) > 0 || previous.sealed === true);
|
||||
const ownership = {
|
||||
ownerUserId: previous?.ownerUserId ?? fact.ownerUserId ?? null,
|
||||
ownerRole: previous?.ownerRole ?? fact.ownerRole ?? null,
|
||||
projectId: previous?.projectId ?? fact.projectId ?? null,
|
||||
conversationId: previous?.conversationId ?? fact.conversationId ?? null,
|
||||
threadId: previous?.threadId ?? fact.threadId ?? null
|
||||
};
|
||||
const stored = projected
|
||||
? { ...previous, ...ownership }
|
||||
: { ...previous, ...fact, ...ownership, createdAt: previous?.createdAt ?? fact.createdAt };
|
||||
this.workbenchSessions.set(fact.sessionId, stored);
|
||||
return { written: true, admissionOnly: true, fact: stored, persistence: this.summary(), valuesPrinted: false };
|
||||
}
|
||||
|
||||
promoteWorkbenchTurnAdmission(params = {}, requestMeta = {}) {
|
||||
const facts = normalizeWorkbenchFacts(params.facts ?? params, requestMeta, this.now());
|
||||
const sessionFact = facts.sessions[0];
|
||||
const inputFact = facts.inputs[0];
|
||||
const turnFact = facts.turns[0];
|
||||
if (!sessionFact?.sessionId || !inputFact?.inputId || !turnFact?.turnId) throw new Error("Workbench turn admission promotion requires session, input, and turn facts.");
|
||||
const storedSession = this.writeWorkbenchSessionAdmissionFact({ fact: sessionFact }, requestMeta).fact;
|
||||
const storedInput = memoryWorkbenchSessionInputWithSeq(inputFact, this.workbenchSessionInputs);
|
||||
this.workbenchSessionInputs.set(storedInput.inputId, storedInput);
|
||||
for (const fact of facts.messages) {
|
||||
const previous = this.workbenchMessages.get(fact.messageId) ?? null;
|
||||
if (!previous || (nonNegativeInteger(previous.projectedSeq) <= 0 && previous.sealed !== true)) this.workbenchMessages.set(fact.messageId, fact);
|
||||
}
|
||||
for (const fact of facts.parts) {
|
||||
const previous = this.workbenchParts.get(fact.partId) ?? null;
|
||||
if (!previous || (nonNegativeInteger(previous.projectedSeq) <= 0 && previous.sealed !== true)) this.workbenchParts.set(fact.partId, fact);
|
||||
}
|
||||
for (const fact of facts.turns) {
|
||||
const previous = this.workbenchTurns.get(fact.turnId) ?? null;
|
||||
if (!previous || (nonNegativeInteger(previous.projectedSeq) <= 0 && previous.sealed !== true)) this.workbenchTurns.set(fact.turnId, fact);
|
||||
}
|
||||
return {
|
||||
written: true,
|
||||
promoted: true,
|
||||
outboxCommitted: true,
|
||||
facts: { ...facts, sessions: [storedSession], inputs: [storedInput] },
|
||||
persistence: this.summary(),
|
||||
valuesPrinted: false
|
||||
};
|
||||
}
|
||||
|
||||
queryWorkbenchFacts(params = {}) {
|
||||
const families = workbenchFactFamilySet(params);
|
||||
const sessions = [...this.workbenchSessions.values()].filter((record) => matchesWorkbenchSessionFactQuery(record, params));
|
||||
|
||||
@@ -0,0 +1,307 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { test } from "bun:test";
|
||||
import pg from "pg";
|
||||
|
||||
import { buildWorkbenchProjectionEventFacts } from "../cloud/workbench-projection-writer.ts";
|
||||
import { PostgresCloudRuntimeStore } from "./runtime-store-postgres.ts";
|
||||
import { commitAgentRunKafkaProjection as commitProjection } from "./runtime-store-postgres-kafka.ts";
|
||||
|
||||
const dbUrl = String(process.env.HWLAB_KAFKA_PROJECTOR_INTEGRATION_DB_URL ?? "").trim();
|
||||
const enabled = Boolean(dbUrl) && process.env.HWLAB_KAFKA_PROJECTOR_INTEGRATION_CONFIRM_NON_PRODUCTION === "1";
|
||||
|
||||
if (!enabled) {
|
||||
test.skip("Kafka projector real PostgreSQL transaction and fault-injection contract", () => {});
|
||||
} else {
|
||||
test("Kafka projector real PostgreSQL transaction and fault-injection contract", async () => {
|
||||
const { Pool } = pg;
|
||||
const pool = new Pool({ connectionString: dbUrl, ssl: false, max: 4 });
|
||||
const suffix = randomUUID().replaceAll("-", "");
|
||||
const traceId = `trc_it_${suffix}`;
|
||||
const sessionId = `ses_it_${suffix}`;
|
||||
const ownerUserId = `usr_it_${suffix}`;
|
||||
const now = "2026-07-10T12:00:00.000Z";
|
||||
const store = new PostgresCloudRuntimeStore({
|
||||
dbUrl,
|
||||
sslMode: "disable",
|
||||
env: { ...process.env, HWLAB_CLOUD_DB_SSL_MODE: "disable" },
|
||||
queryClient: pool,
|
||||
now: () => now,
|
||||
logger: { error() {}, warn() {}, info() {} }
|
||||
});
|
||||
|
||||
try {
|
||||
const readiness = await store.readiness();
|
||||
assert.equal(readiness.ready, true);
|
||||
assert.equal(readiness.durable, true);
|
||||
assert.equal(readiness.migration?.ready, true);
|
||||
|
||||
await pool.query(
|
||||
"INSERT INTO users (id, username, display_name, role, status, created_at, updated_at) VALUES ($1,$2,$2,'user','active',$3,$3)",
|
||||
[ownerUserId, `integration-${suffix}`, now]
|
||||
);
|
||||
|
||||
await store.writeWorkbenchSessionAdmissionFact({
|
||||
fact: {
|
||||
sessionId,
|
||||
ownerUserId,
|
||||
projectId: `prj_it_${suffix}`,
|
||||
conversationId: `cnv_it_${suffix}`,
|
||||
threadId: `thread_it_${suffix}`,
|
||||
status: "queued",
|
||||
lastTraceId: traceId,
|
||||
projectedSeq: 0,
|
||||
sourceSeq: 0,
|
||||
terminal: false,
|
||||
sealed: false,
|
||||
sessionJson: { sessionId, launchContext: { source: "integration-test" }, valuesRedacted: true },
|
||||
createdAt: now,
|
||||
updatedAt: now
|
||||
}
|
||||
});
|
||||
|
||||
const running = await project(store, {
|
||||
traceId,
|
||||
sessionId,
|
||||
sourceEventId: `evt_running_${suffix}`,
|
||||
sourceSeq: 1,
|
||||
sourceOffset: "100",
|
||||
inputSha256: "1".repeat(64),
|
||||
event: { eventType: "assistant_message", status: "running", assistantText: "partial", createdAt: now }
|
||||
});
|
||||
assert.equal(running.duplicate, false);
|
||||
assert.equal(running.projectedSeq, 1);
|
||||
assert.equal(await count(pool, "workbench_kafka_inbox", "trace_id = $1", [traceId]), 1);
|
||||
assert.equal(await count(pool, "workbench_trace_events", "trace_id = $1", [traceId]), 1);
|
||||
assert.equal(await count(pool, "workbench_projection_outbox", "trace_id = $1", [traceId]), 2);
|
||||
assert.equal(await count(pool, "hwlab_kafka_outbox", "partition_key = $1", [traceId]), 1);
|
||||
|
||||
const sameTransport = await project(store, {
|
||||
traceId,
|
||||
sessionId,
|
||||
sourceEventId: `evt_running_${suffix}`,
|
||||
sourceSeq: 1,
|
||||
sourceOffset: "100",
|
||||
inputSha256: "1".repeat(64),
|
||||
event: { eventType: "assistant_message", status: "running", assistantText: "partial", createdAt: now }
|
||||
});
|
||||
const newOffsetSameEvent = await project(store, {
|
||||
traceId,
|
||||
sessionId,
|
||||
sourceEventId: `evt_running_${suffix}`,
|
||||
sourceSeq: 1,
|
||||
sourceOffset: "101",
|
||||
inputSha256: "1".repeat(64),
|
||||
event: { eventType: "assistant_message", status: "running", assistantText: "partial", createdAt: now }
|
||||
});
|
||||
assert.equal(sameTransport.duplicate, true);
|
||||
assert.equal(newOffsetSameEvent.duplicate, true);
|
||||
assert.equal(await count(pool, "workbench_kafka_inbox", "trace_id = $1", [traceId]), 1);
|
||||
assert.equal(await count(pool, "workbench_projection_outbox", "trace_id = $1", [traceId]), 2);
|
||||
|
||||
const conflict = await project(store, {
|
||||
traceId,
|
||||
sessionId,
|
||||
sourceEventId: `evt_running_${suffix}`,
|
||||
sourceSeq: 1,
|
||||
sourceOffset: "102",
|
||||
inputSha256: "2".repeat(64),
|
||||
event: { eventType: "assistant_message", status: "running", assistantText: "mutated", createdAt: now }
|
||||
});
|
||||
assert.equal(conflict.conflict, true);
|
||||
assert.equal(await count(pool, "workbench_kafka_dlq", "source_event_id = $1", [`evt_running_${suffix}`]), 1);
|
||||
const projectedInbox = await pool.query(
|
||||
"SELECT status, projected_seq FROM workbench_kafka_inbox WHERE source_topic = $1 AND source_event_id = $2",
|
||||
["agentrun.event.v1", `evt_running_${suffix}`]
|
||||
);
|
||||
assert.deepEqual(projectedInbox.rows[0], { status: "projected", projected_seq: 1 });
|
||||
|
||||
await pool.query(`
|
||||
CREATE OR REPLACE FUNCTION issue_2464_delay_inbox_insert() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
IF NEW.source_topic = 'agentrun.event.v1' AND NEW.source_partition = 0 AND NEW.source_offset = 300 THEN
|
||||
PERFORM pg_sleep(0.25);
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
DROP TRIGGER IF EXISTS issue_2464_delay_inbox_insert ON workbench_kafka_inbox;
|
||||
CREATE TRIGGER issue_2464_delay_inbox_insert BEFORE INSERT ON workbench_kafka_inbox
|
||||
FOR EACH ROW EXECUTE FUNCTION issue_2464_delay_inbox_insert();
|
||||
`);
|
||||
try {
|
||||
const concurrent = await Promise.allSettled([
|
||||
project(store, {
|
||||
traceId: `trc_transport_a_${suffix}`,
|
||||
sessionId: `ses_transport_a_${suffix}`,
|
||||
sourceEventId: `evt_transport_a_${suffix}`,
|
||||
sourceSeq: 1,
|
||||
sourceOffset: "300",
|
||||
inputSha256: "6".repeat(64),
|
||||
event: { eventType: "backend_status", status: "running", createdAt: now }
|
||||
}),
|
||||
project(store, {
|
||||
traceId: `trc_transport_b_${suffix}`,
|
||||
sessionId: `ses_transport_b_${suffix}`,
|
||||
sourceEventId: `evt_transport_b_${suffix}`,
|
||||
sourceSeq: 1,
|
||||
sourceOffset: "300",
|
||||
inputSha256: "7".repeat(64),
|
||||
event: { eventType: "backend_status", status: "running", createdAt: now }
|
||||
})
|
||||
]);
|
||||
assert.deepEqual(concurrent.map((result) => result.status), ["fulfilled", "fulfilled"]);
|
||||
const values = concurrent.map((result) => {
|
||||
if (result.status !== "fulfilled") throw result.reason;
|
||||
return result.value;
|
||||
});
|
||||
assert.equal(values.filter((result) => result.conflict === true).length, 1);
|
||||
assert.equal(values.filter((result) => result.conflict !== true && result.duplicate !== true).length, 1);
|
||||
assert.equal(await count(pool, "workbench_kafka_inbox", "source_topic = $1 AND source_partition = 0 AND source_offset = 300", ["agentrun.event.v1"]), 1);
|
||||
assert.equal(await count(pool, "workbench_kafka_dlq", "source_topic = $1 AND source_partition = 0 AND source_offset = 300", ["agentrun.event.v1"]), 1);
|
||||
} finally {
|
||||
await pool.query("DROP TRIGGER IF EXISTS issue_2464_delay_inbox_insert ON workbench_kafka_inbox; DROP FUNCTION IF EXISTS issue_2464_delay_inbox_insert()");
|
||||
}
|
||||
|
||||
const faultTraceId = `trc_fault_${suffix}`;
|
||||
const faultSessionId = `ses_fault_${suffix}`;
|
||||
const faultStore = new PostgresCloudRuntimeStore({
|
||||
dbUrl,
|
||||
sslMode: "disable",
|
||||
env: { ...process.env, HWLAB_CLOUD_DB_SSL_MODE: "disable" },
|
||||
queryClient: faultInjectingPool(pool, /^INSERT INTO workbench_projection_outbox/u),
|
||||
now: () => now,
|
||||
logger: { error() {}, warn() {}, info() {} }
|
||||
});
|
||||
await assert.rejects(project(faultStore, {
|
||||
traceId: faultTraceId,
|
||||
sessionId: faultSessionId,
|
||||
sourceEventId: `evt_fault_${suffix}`,
|
||||
sourceSeq: 1,
|
||||
sourceOffset: "200",
|
||||
inputSha256: "3".repeat(64),
|
||||
event: { eventType: "assistant_message", status: "running", assistantText: "must roll back", createdAt: now }
|
||||
}), /injected projection outbox failure/u);
|
||||
assert.equal(await count(pool, "workbench_kafka_inbox", "trace_id = $1", [faultTraceId]), 0);
|
||||
assert.equal(await count(pool, "workbench_trace_events", "trace_id = $1", [faultTraceId]), 0);
|
||||
assert.equal(await count(pool, "workbench_projection_outbox", "trace_id = $1", [faultTraceId]), 0);
|
||||
assert.equal(await count(pool, "hwlab_kafka_outbox", "partition_key = $1", [faultTraceId]), 0);
|
||||
|
||||
const terminal = await project(store, {
|
||||
traceId,
|
||||
sessionId,
|
||||
sourceEventId: `evt_terminal_${suffix}`,
|
||||
sourceSeq: 2,
|
||||
sourceOffset: "103",
|
||||
inputSha256: "4".repeat(64),
|
||||
event: {
|
||||
eventType: "terminal_status",
|
||||
status: "completed",
|
||||
terminal: true,
|
||||
assistantText: "authoritative final",
|
||||
finalResponse: { text: "authoritative final", status: "completed" },
|
||||
startedAt: now,
|
||||
finishedAt: "2026-07-10T12:00:01.000Z",
|
||||
durationMs: 1000,
|
||||
createdAt: "2026-07-10T12:00:01.000Z"
|
||||
}
|
||||
});
|
||||
assert.equal(terminal.projectedSeq, 2);
|
||||
const sealed = await pool.query(
|
||||
"SELECT projected_seq, terminal, sealed, checkpoint_json FROM workbench_projection_checkpoints WHERE trace_id = $1",
|
||||
[traceId]
|
||||
);
|
||||
assert.equal(sealed.rows[0].projected_seq, 2);
|
||||
assert.equal(sealed.rows[0].terminal, true);
|
||||
assert.equal(sealed.rows[0].sealed, true);
|
||||
assert.equal(JSON.parse(sealed.rows[0].checkpoint_json).finalResponse.text, "authoritative final");
|
||||
assert.equal(await count(pool, "workbench_projection_outbox", "trace_id = $1", [traceId]), 5);
|
||||
|
||||
const late = await project(store, {
|
||||
traceId,
|
||||
sessionId,
|
||||
sourceEventId: `evt_late_${suffix}`,
|
||||
sourceSeq: 3,
|
||||
sourceOffset: "104",
|
||||
inputSha256: "5".repeat(64),
|
||||
event: { eventType: "backend_status", status: "running", terminal: false, createdAt: "2026-07-10T12:00:02.000Z" }
|
||||
});
|
||||
assert.equal(late.suppressedAfterSeal, true);
|
||||
assert.equal(late.projectedSeq, 2);
|
||||
assert.equal(await count(pool, "workbench_trace_events", "trace_id = $1", [traceId]), 2);
|
||||
assert.equal(await count(pool, "workbench_projection_outbox", "trace_id = $1", [traceId]), 5);
|
||||
const lateInbox = await pool.query(
|
||||
"SELECT status, projected_seq FROM workbench_kafka_inbox WHERE source_topic = $1 AND source_event_id = $2",
|
||||
["agentrun.event.v1", `evt_late_${suffix}`]
|
||||
);
|
||||
assert.deepEqual(lateInbox.rows[0], { status: "projected", projected_seq: 2 });
|
||||
|
||||
const firstSync = await store.readAtomicWorkbenchProjectionSync({ traceId, afterOutboxSeq: 0, limit: 100 });
|
||||
assert.equal(firstSync.events.length, 5);
|
||||
assert.equal(firstSync.hasMore, false);
|
||||
assert.equal(firstSync.cursorOutboxSeq, firstSync.cutoffOutboxSeq);
|
||||
assert.equal(new Set(firstSync.events.map((event) => event.outboxSeq)).size, firstSync.events.length);
|
||||
assert.equal(firstSync.facts.checkpoints[0].projectedSeq, 2);
|
||||
assert.equal(firstSync.facts.turns[0].finalResponse.text, "authoritative final");
|
||||
const caughtUp = await store.readAtomicWorkbenchProjectionSync({ traceId, afterOutboxSeq: firstSync.cursorOutboxSeq, limit: 100, deltaOnly: true });
|
||||
assert.equal(caughtUp.events.length, 0);
|
||||
assert.equal(caughtUp.cursorOutboxSeq, firstSync.cursorOutboxSeq);
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
}, 30_000);
|
||||
}
|
||||
|
||||
async function project(store, { traceId, sessionId, sourceEventId, sourceSeq, sourceOffset, inputSha256, event }) {
|
||||
const canonicalEvent = {
|
||||
schema: "agentrun.event.v1",
|
||||
eventId: sourceEventId,
|
||||
sourceSeq,
|
||||
traceId,
|
||||
hwlabSessionId: sessionId,
|
||||
runId: `run_${traceId}`,
|
||||
commandId: `cmd_${traceId}`,
|
||||
event
|
||||
};
|
||||
const projectedEvent = { ...event, traceId, sessionId, sourceEventId, sourceSeq, runId: canonicalEvent.runId, commandId: canonicalEvent.commandId };
|
||||
return commitProjection(store, {
|
||||
transport: { sourceTopic: "agentrun.event.v1", sourcePartition: 0, sourceOffset, sourceKey: canonicalEvent.runId, inputSha256 },
|
||||
canonicalEvent,
|
||||
projectedEvent,
|
||||
requestMeta: { traceId, sessionId, valuesPrinted: false },
|
||||
factsFactory: ({ projectedSeq, previousCheckpoint, projectedAt }) => buildWorkbenchProjectionEventFacts({ projectedSeq, previousCheckpoint, projectedAt, event: projectedEvent }),
|
||||
hwlabEvent: { schema: "hwlab.event.v1", eventId: `hwlab:${sourceEventId}`, traceId, sessionId, sourceEventId, sourceSeq, valuesPrinted: false },
|
||||
hwlabTopic: "hwlab.event.v1",
|
||||
partitionKey: traceId,
|
||||
headers: { sourceEventId }
|
||||
});
|
||||
}
|
||||
|
||||
function faultInjectingPool(pool, pattern) {
|
||||
return {
|
||||
async connect() {
|
||||
const client = await pool.connect();
|
||||
let injected = false;
|
||||
return {
|
||||
async query(sql, params) {
|
||||
if (!injected && pattern.test(String(sql))) {
|
||||
injected = true;
|
||||
const error = new Error("injected projection outbox failure");
|
||||
error.code = "XX2464";
|
||||
throw error;
|
||||
}
|
||||
return client.query(sql, params);
|
||||
},
|
||||
release() {
|
||||
client.release();
|
||||
}
|
||||
};
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function count(pool, table, where, params) {
|
||||
if (!/^[a-z_]+$/u.test(table)) throw new Error("unsafe test table");
|
||||
const result = await pool.query(`SELECT COUNT(*)::int AS count FROM ${table} WHERE ${where}`, params);
|
||||
return Number(result.rows[0].count);
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { buildWorkbenchProjectionEventFacts } from "../cloud/workbench-projection-writer.ts";
|
||||
import { projectionOutboxRealtimeEvents } from "../cloud/server-workbench-realtime-http.ts";
|
||||
import {
|
||||
claimHwlabKafkaOutbox,
|
||||
commitAgentRunKafkaProjection,
|
||||
completeHwlabKafkaOutbox,
|
||||
projectionOutboxEventId,
|
||||
recordFailedAgentRunKafkaMessage,
|
||||
recordIgnoredAgentRunKafkaMessage,
|
||||
retryHwlabKafkaOutbox
|
||||
} from "./runtime-store-postgres-kafka.ts";
|
||||
|
||||
test("malformed duplicate source event at a new offset is durably DLQed without mutating projected inbox", async () => {
|
||||
const queries = [];
|
||||
const client = {
|
||||
async query(sql, params) {
|
||||
queries.push({ sql, params });
|
||||
if (sql.startsWith("SELECT pg_advisory_xact_lock")) return { rows: [] };
|
||||
if (sql.startsWith("SELECT source_topic")) return { rows: [{ source_topic: "agentrun.event.v1", source_partition: 0, source_offset: "10", input_sha256: "prior-hash", status: "projected", projected_seq: 7 }] };
|
||||
if (sql.startsWith("INSERT INTO workbench_kafka_dlq")) return { rows: [] };
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
}
|
||||
};
|
||||
const store = transactionStore(client);
|
||||
const result = await recordFailedAgentRunKafkaMessage(store, {
|
||||
transport: { sourceTopic: "agentrun.event.v1", sourcePartition: 0, sourceOffset: "11", sourceKey: "run-1", inputSha256: "new-hash" },
|
||||
sourceEventId: "evt-duplicate",
|
||||
envelope: { eventId: "evt-duplicate", valuesPrinted: false },
|
||||
error: { code: "workbench_kafka_schema_invalid", message: "malformed" }
|
||||
});
|
||||
|
||||
assert.equal(result.recorded, true);
|
||||
assert.equal(result.conflict, true);
|
||||
assert.equal(result.priorStatus, "projected");
|
||||
assert.equal(queries[0].params[0], "workbench-kafka-inbox:agentrun.event.v1:transport:0:11");
|
||||
assert.equal(queries[1].params[0], "workbench-kafka-inbox:agentrun.event.v1:event:evt-duplicate");
|
||||
assert.equal(queries.some(({ sql }) => sql.startsWith("UPDATE workbench_kafka_inbox")), false);
|
||||
assert.equal(queries.some(({ sql }) => sql.startsWith("INSERT INTO workbench_kafka_dlq")), true);
|
||||
});
|
||||
|
||||
test("ignored Kafka messages acquire transport then event identity locks", async () => {
|
||||
const queries = [];
|
||||
const client = {
|
||||
async query(sql, params) {
|
||||
queries.push({ sql, params });
|
||||
if (sql.startsWith("SELECT pg_advisory_xact_lock")) return { rows: [] };
|
||||
if (sql.startsWith("SELECT input_sha256")) return { rows: [] };
|
||||
if (sql.startsWith("INSERT INTO workbench_kafka_inbox")) return { rows: [] };
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
}
|
||||
};
|
||||
const result = await recordIgnoredAgentRunKafkaMessage(transactionStore(client), {
|
||||
transport: { sourceTopic: "agentrun.event.v1", sourcePartition: 2, sourceOffset: "31", sourceKey: "run-ignored", inputSha256: "hash-ignored" },
|
||||
envelope: { eventId: "evt-ignored", sourceSeq: 1, runId: "run-ignored" },
|
||||
reason: "hwlab-correlation-absent"
|
||||
});
|
||||
|
||||
assert.equal(result.ignored, true);
|
||||
assert.equal(result.duplicate, false);
|
||||
assert.equal(queries[0].params[0], "workbench-kafka-inbox:agentrun.event.v1:transport:2:31");
|
||||
assert.equal(queries[1].params[0], "workbench-kafka-inbox:agentrun.event.v1:event:evt-ignored");
|
||||
});
|
||||
|
||||
test("projection outbox derives a stable non-null identity when a fact has no source event", () => {
|
||||
const input = {
|
||||
fact: { traceId: "trc-derived", projectedSeq: 3, sourceSeq: 2 },
|
||||
event: { aggregateId: "trc-derived", aggregateSeq: 3, projectionRevision: 3 },
|
||||
family: "traceEvents",
|
||||
entityId: "wte-derived",
|
||||
commitType: "event"
|
||||
};
|
||||
const first = projectionOutboxEventId(input);
|
||||
assert.match(first, /^projection:[a-f0-9]{64}$/u);
|
||||
assert.equal(projectionOutboxEventId(input), first);
|
||||
assert.notEqual(projectionOutboxEventId({ ...input, event: { ...input.event, projectionRevision: 4 } }), first);
|
||||
});
|
||||
|
||||
test("Kafka projection preserves admission owner and scope fields", async () => {
|
||||
let persistedSession = null;
|
||||
const client = {
|
||||
async query(sql) {
|
||||
if (sql.startsWith("SELECT pg_advisory_xact_lock")) return { rows: [] };
|
||||
if (sql.startsWith("SELECT source_topic")) return { rows: [] };
|
||||
if (sql.startsWith("INSERT INTO workbench_kafka_inbox")) return { rows: [] };
|
||||
if (sql.startsWith("SELECT checkpoint_json")) return { rows: [] };
|
||||
if (sql.startsWith("SELECT owner_user_id")) return { rows: [{ owner_user_id: "usr-owner", project_id: "prj-owner", conversation_id: "cnv-owner", thread_id: "thread-owner", session_json: { sessionId: "ses-owner", ownerUserId: "usr-owner", ownerRole: "user", launchContext: { source: "workbench" } } }] };
|
||||
if (sql.startsWith("SELECT GREATEST")) return { rows: [{ max_projected_seq: 0 }] };
|
||||
if (sql.startsWith("INSERT INTO hwlab_kafka_outbox")) return { rows: [] };
|
||||
if (sql.startsWith("UPDATE workbench_kafka_inbox")) return { rows: [] };
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
}
|
||||
};
|
||||
const store = {
|
||||
...transactionStore(client),
|
||||
memory: { writeWorkbenchFacts() {} },
|
||||
async persistWorkbenchAggregateEvent(event) { return { ...event, eventSeq: 1, aggregateSeq: 1, projectionRevision: 1 }; },
|
||||
async persistWorkbenchSessionInputFact() {},
|
||||
async persistWorkbenchSessionFact(fact) { persistedSession = fact; },
|
||||
async persistWorkbenchMessageFact() {},
|
||||
async persistWorkbenchPartFact() {},
|
||||
async persistWorkbenchTurnFact() {},
|
||||
async persistWorkbenchTraceEventFact() {},
|
||||
async persistWorkbenchProjectionCheckpoint() {},
|
||||
async persistWorkbenchProjectionOutbox() {}
|
||||
};
|
||||
await commitAgentRunKafkaProjection(store, {
|
||||
transport: { sourceTopic: "agentrun.event.v1", sourcePartition: 0, sourceOffset: "12", sourceKey: "run-owner", inputSha256: "hash-owner" },
|
||||
canonicalEvent: { eventId: "evt-owner", sourceSeq: 1, traceId: "trc-owner", hwlabSessionId: "ses-owner", runId: "run-owner", commandId: "cmd-owner" },
|
||||
projectedEvent: { traceId: "trc-owner", sessionId: "ses-owner" },
|
||||
factsFactory: ({ projectedSeq, projectedAt }) => ({
|
||||
written: true,
|
||||
facts: {
|
||||
sessions: [{ sessionId: "ses-owner", status: "running", lastTraceId: "trc-owner", projectedSeq, sourceSeq: 1, sourceEventId: "evt-owner", terminal: false, sealed: false, updatedAt: projectedAt }],
|
||||
messages: [], parts: [], turns: [], traceEvents: [], checkpoints: []
|
||||
}
|
||||
}),
|
||||
hwlabEvent: { eventId: "hwlab:evt-owner" },
|
||||
hwlabTopic: "hwlab.event.v1",
|
||||
partitionKey: "trc-owner",
|
||||
headers: {}
|
||||
});
|
||||
|
||||
assert.equal(persistedSession.ownerUserId, "usr-owner");
|
||||
assert.equal(persistedSession.ownerRole, "user");
|
||||
assert.equal(persistedSession.projectId, "prj-owner");
|
||||
assert.equal(persistedSession.conversationId, "cnv-owner");
|
||||
assert.equal(persistedSession.threadId, "thread-owner");
|
||||
assert.deepEqual(persistedSession.launchContext, { source: "workbench" });
|
||||
});
|
||||
|
||||
test("terminal Kafka transaction persists sealed turn and immutable SSE snapshot", async () => {
|
||||
let persistedTurn = null;
|
||||
const outbox = [];
|
||||
const client = {
|
||||
async query(sql) {
|
||||
if (sql.startsWith("SELECT pg_advisory_xact_lock")) return { rows: [] };
|
||||
if (sql.startsWith("SELECT source_topic")) return { rows: [] };
|
||||
if (sql.startsWith("INSERT INTO workbench_kafka_inbox")) return { rows: [] };
|
||||
if (sql.startsWith("SELECT checkpoint_json")) return { rows: [{ checkpoint_json: { traceId: "trc_terminal", sessionId: "ses_terminal", assistantText: "final body", finalResponse: { text: "final body", status: "running" }, projectedSeq: 2, sourceSeq: 2, terminal: false, sealed: false } }] };
|
||||
if (sql.startsWith("SELECT owner_user_id")) return { rows: [{ owner_user_id: "usr-owner", session_json: { sessionId: "ses_terminal", ownerUserId: "usr-owner" } }] };
|
||||
if (sql.startsWith("SELECT GREATEST")) return { rows: [{ max_projected_seq: 2 }] };
|
||||
if (sql.startsWith("INSERT INTO hwlab_kafka_outbox") || sql.startsWith("UPDATE workbench_kafka_inbox")) return { rows: [] };
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
}
|
||||
};
|
||||
const store = {
|
||||
...transactionStore(client),
|
||||
memory: { writeWorkbenchFacts() {} },
|
||||
async persistWorkbenchAggregateEvent(event) { return { ...event, eventSeq: 3, aggregateSeq: 3, projectionRevision: 3 }; },
|
||||
async persistWorkbenchSessionInputFact() {}, async persistWorkbenchSessionFact() {}, async persistWorkbenchMessageFact() {}, async persistWorkbenchPartFact() {},
|
||||
async persistWorkbenchTurnFact(fact) { persistedTurn = fact; }, async persistWorkbenchTraceEventFact() {}, async persistWorkbenchProjectionCheckpoint() {},
|
||||
async persistWorkbenchProjectionOutbox(record) { outbox.push(record); }
|
||||
};
|
||||
await commitAgentRunKafkaProjection(store, {
|
||||
transport: { sourceTopic: "agentrun.event.v1", sourcePartition: 0, sourceOffset: "20", inputSha256: "hash-terminal" },
|
||||
canonicalEvent: { eventId: "evt-terminal", sourceSeq: 3, traceId: "trc_terminal", hwlabSessionId: "ses_terminal", runId: "run-terminal", commandId: "cmd-terminal" },
|
||||
projectedEvent: { traceId: "trc_terminal", sessionId: "ses_terminal", sourceEventId: "evt-terminal", sourceSeq: 3, type: "result", eventType: "terminal", status: "completed", terminal: true, createdAt: "2026-07-10T10:00:03.000Z" },
|
||||
factsFactory: ({ projectedSeq, previousCheckpoint, projectedAt }) => buildWorkbenchProjectionEventFacts({ projectedSeq, previousCheckpoint, projectedAt, event: { traceId: "trc_terminal", sessionId: "ses_terminal", sourceEventId: "evt-terminal", sourceSeq: 3, type: "result", eventType: "terminal", status: "completed", terminal: true, createdAt: "2026-07-10T10:00:03.000Z" } }),
|
||||
hwlabEvent: { eventId: "hwlab:evt-terminal" }, hwlabTopic: "hwlab.event.v1", partitionKey: "trc_terminal", headers: {}
|
||||
});
|
||||
|
||||
assert.equal(persistedTurn.terminal, true);
|
||||
assert.equal(persistedTurn.sealed, true);
|
||||
assert.equal(persistedTurn.finalResponse.text, "final body");
|
||||
const turnRow = outbox.find((row) => row.entityFamily === "turns");
|
||||
assert.equal(turnRow.payload.fact.assistantText, "final body");
|
||||
const [sse] = projectionOutboxRealtimeEvents({ events: [{ ...turnRow, outboxSeq: 9 }], facts: {} });
|
||||
assert.equal(sse.name, "workbench.turn.snapshot");
|
||||
assert.equal(sse.payload.turn.finalResponse.text, "final body");
|
||||
});
|
||||
|
||||
test("late second terminal is acknowledged without facts or either projection outbox", async () => {
|
||||
const queries = [];
|
||||
const client = {
|
||||
async query(sql, params) {
|
||||
queries.push({ sql, params });
|
||||
if (sql.startsWith("SELECT pg_advisory_xact_lock")) return { rows: [] };
|
||||
if (sql.startsWith("SELECT source_topic")) return { rows: [] };
|
||||
if (sql.startsWith("INSERT INTO workbench_kafka_inbox")) return { rows: [] };
|
||||
if (sql.startsWith("SELECT checkpoint_json")) return { rows: [{ checkpoint_json: { traceId: "trc_sealed", projectedSeq: 9, terminal: true, sealed: true } }] };
|
||||
if (sql.startsWith("SELECT owner_user_id")) return { rows: [] };
|
||||
if (sql.startsWith("SELECT GREATEST")) return { rows: [{ max_projected_seq: 9 }] };
|
||||
if (sql.startsWith("INSERT INTO hwlab_kafka_outbox") || sql.startsWith("UPDATE workbench_kafka_inbox")) return { rows: [] };
|
||||
throw new Error(`unexpected SQL: ${sql}`);
|
||||
}
|
||||
};
|
||||
const store = { ...transactionStore(client), memory: { writeWorkbenchFacts() { throw new Error("suppressed facts must not reach memory"); } } };
|
||||
const result = await commitAgentRunKafkaProjection(store, {
|
||||
transport: { sourceTopic: "agentrun.event.v1", sourcePartition: 0, sourceOffset: "21", inputSha256: "hash-late" },
|
||||
canonicalEvent: { eventId: "evt-late", sourceSeq: 10, traceId: "trc_sealed", hwlabSessionId: "ses_sealed" },
|
||||
projectedEvent: { traceId: "trc_sealed", sessionId: "ses_sealed" },
|
||||
factsFactory: ({ projectedSeq, previousCheckpoint, projectedAt }) => buildWorkbenchProjectionEventFacts({
|
||||
projectedSeq,
|
||||
previousCheckpoint,
|
||||
projectedAt,
|
||||
event: { traceId: "trc_sealed", sessionId: "ses_sealed", sourceEventId: "evt-late", sourceSeq: 10, type: "result", eventType: "terminal", status: "failed", terminal: true }
|
||||
}),
|
||||
hwlabEvent: { eventId: "hwlab:evt-late" },
|
||||
hwlabTopic: "hwlab.event.v1",
|
||||
partitionKey: "trc_sealed",
|
||||
headers: {}
|
||||
});
|
||||
|
||||
assert.equal(result.suppressedAfterSeal, true);
|
||||
assert.equal(result.projectionWritten, false);
|
||||
assert.equal(result.projectedSeq, 9);
|
||||
assert.equal(queries.some(({ sql }) => sql.startsWith("INSERT INTO hwlab_kafka_outbox")), false);
|
||||
const inboxUpdate = queries.find(({ sql }) => sql.startsWith("UPDATE workbench_kafka_inbox"));
|
||||
assert.equal(inboxUpdate.params[0], 9);
|
||||
});
|
||||
|
||||
test("HWLAB outbox claim and completion use attempt and lease fencing", async () => {
|
||||
const calls = [];
|
||||
const row = { outbox_seq: 3, event_id: "evt-3", topic: "hwlab.event.v1", partition_key: "trc-3", payload_json: {}, headers_json: {}, attempt_count: 4, lease_owner: "relay-a", lease_expires_at: "2026-07-10T10:05:00.000Z", next_attempt_at: "2026-07-10T10:00:00.000Z", created_at: "2026-07-10T10:00:00.000Z" };
|
||||
const store = {
|
||||
now: () => "2026-07-10T10:04:00.000Z",
|
||||
async withDurableTransaction(_label, operation) { return operation({ query: async (sql, params) => { calls.push({ sql, params }); return { rows: [row] }; } }); },
|
||||
async query(sql, params) { calls.push({ sql, params }); return { rows: [{ outbox_seq: 3 }] }; }
|
||||
};
|
||||
const [item] = await claimHwlabKafkaOutbox(store, { owner: "relay-a", leaseMs: 30000, limit: 1 });
|
||||
await completeHwlabKafkaOutbox(store, item);
|
||||
await retryHwlabKafkaOutbox(store, item, new Error("retry"), "2026-07-10T10:04:30.000Z");
|
||||
|
||||
assert.match(calls[0].sql, /attempt_count = o\.attempt_count \+ 1/u);
|
||||
assert.match(calls[0].sql, /NOT EXISTS/u);
|
||||
assert.match(calls[1].sql, /lease_owner = \$3 AND attempt_count = \$4 AND lease_expires_at = \$5 AND lease_expires_at > \$1/u);
|
||||
assert.deepEqual(calls[1].params.slice(2), ["relay-a", 4, "2026-07-10T10:05:00.000Z"]);
|
||||
assert.match(calls[2].sql, /lease_owner = \$5 AND attempt_count = \$6 AND lease_expires_at = \$7 AND lease_expires_at > \$3/u);
|
||||
});
|
||||
|
||||
test("stale HWLAB outbox fencing token cannot complete a claim", async () => {
|
||||
const store = { now: () => "2026-07-10T10:06:00.000Z", async query() { return { rows: [] }; } };
|
||||
await assert.rejects(completeHwlabKafkaOutbox(store, { outboxSeq: 3, leaseOwner: "relay-old", attemptCount: 1, leaseExpiresAt: "2026-07-10T10:05:00.000Z" }), (error) => error?.code === "hwlab_kafka_outbox_lease_conflict");
|
||||
});
|
||||
|
||||
function transactionStore(client) {
|
||||
return {
|
||||
now: () => "2026-07-10T10:00:00.000Z",
|
||||
async withDurableTransaction(_label, operation) { return operation(client); }
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,418 @@
|
||||
/*
|
||||
* Durable AgentRun Kafka projector storage operations.
|
||||
* Kafka I/O is intentionally outside every transaction in this module.
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import {
|
||||
indexWorkbenchAggregateEvents,
|
||||
inputFactWithAggregateSeq,
|
||||
nonNegativeInteger,
|
||||
normalizeWorkbenchAggregateEventsForFacts,
|
||||
normalizeWorkbenchFacts,
|
||||
parseJsonColumn,
|
||||
stableJson,
|
||||
textOr
|
||||
} from "./runtime-store-core.ts";
|
||||
|
||||
export async function commitAgentRunKafkaProjection(store, params = {}) {
|
||||
const transport = normalizeTransport(params.transport);
|
||||
const canonicalEvent = objectValue(params.canonicalEvent);
|
||||
const projectedEvent = objectValue(params.projectedEvent);
|
||||
const sourceEventId = requiredText(canonicalEvent.eventId, "canonical AgentRun eventId");
|
||||
const traceId = requiredText(projectedEvent.traceId ?? canonicalEvent.traceId, "canonical AgentRun traceId");
|
||||
const sessionId = text(projectedEvent.sessionId ?? canonicalEvent.hwlabSessionId);
|
||||
const sourceSeq = nonNegativeInteger(canonicalEvent.sourceSeq ?? canonicalEvent.event?.seq);
|
||||
const inputSha256 = requiredText(transport.inputSha256, "Kafka input sha256");
|
||||
const now = store.now();
|
||||
const result = await store.withDurableTransaction("workbench.kafka-projection.commit", async (client) => {
|
||||
await lockKafkaInboxIdentity(client, transport, sourceEventId);
|
||||
await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`workbench-projection:${traceId}`]);
|
||||
const existing = await client.query(
|
||||
"SELECT source_topic, source_partition, source_offset, source_event_id, input_sha256, status, projected_seq FROM workbench_kafka_inbox WHERE (source_topic = $1 AND source_partition = $2 AND source_offset = $3) OR (source_topic = $1 AND source_event_id = $4) ORDER BY created_at ASC LIMIT 1",
|
||||
[transport.sourceTopic, transport.sourcePartition, transport.sourceOffset, sourceEventId]
|
||||
);
|
||||
const prior = existing.rows?.[0] ?? null;
|
||||
if (prior) {
|
||||
if (prior.input_sha256 !== inputSha256) {
|
||||
const diagnostic = { code: "workbench_kafka_inbox_hash_conflict", message: "Kafka transport or source event identity was reused with a different payload hash.", valuesRedacted: true };
|
||||
if (prior.status !== "projected") {
|
||||
await client.query(
|
||||
"UPDATE workbench_kafka_inbox SET status = 'failed', error_json = $1, updated_at = $2 WHERE source_topic = $3 AND source_partition = $4 AND source_offset = $5",
|
||||
[stableJson(diagnostic), now, prior.source_topic, prior.source_partition, prior.source_offset]
|
||||
);
|
||||
}
|
||||
await insertDlq(client, { ...transport, sourceEventId, inputSha256, error: diagnostic, envelope: redactedEnvelope(canonicalEvent), createdAt: now });
|
||||
return { conflict: true, diagnostic, duplicate: false, projectedSeq: nonNegativeInteger(prior.projected_seq) };
|
||||
}
|
||||
return { duplicate: true, conflict: false, status: prior.status, projectedSeq: nonNegativeInteger(prior.projected_seq) };
|
||||
}
|
||||
await client.query(
|
||||
"INSERT INTO workbench_kafka_inbox (source_topic, source_partition, source_offset, source_key, source_event_id, input_sha256, trace_id, session_id, run_id, command_id, source_seq, status, projected_seq, envelope_json, error_json, created_at, updated_at, projected_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,'processing',NULL,$12,'{}',$13,$13,NULL)",
|
||||
[transport.sourceTopic, transport.sourcePartition, transport.sourceOffset, transport.sourceKey, sourceEventId, inputSha256, traceId, sessionId, text(canonicalEvent.runId), text(canonicalEvent.commandId), sourceSeq, stableJson(redactedEnvelope(canonicalEvent)), now]
|
||||
);
|
||||
const checkpointResult = await client.query("SELECT checkpoint_json FROM workbench_projection_checkpoints WHERE trace_id = $1 FOR UPDATE", [traceId]);
|
||||
const previousCheckpoint = parseJsonColumn(checkpointResult.rows?.[0]?.checkpoint_json, null);
|
||||
const sessionResult = sessionId
|
||||
? await client.query("SELECT owner_user_id, project_id, conversation_id, thread_id, session_json FROM workbench_sessions WHERE session_id = $1 FOR UPDATE", [sessionId])
|
||||
: { rows: [] };
|
||||
const existingSession = sessionResult.rows?.[0] ?? null;
|
||||
const maxResult = await client.query(
|
||||
"SELECT GREATEST(COALESCE((SELECT MAX(projected_seq) FROM workbench_trace_events WHERE trace_id = $1), 0), COALESCE((SELECT projected_seq FROM workbench_projection_checkpoints WHERE trace_id = $1), 0))::int AS max_projected_seq",
|
||||
[traceId]
|
||||
);
|
||||
const projectedSeq = nonNegativeInteger(maxResult.rows?.[0]?.max_projected_seq) + 1;
|
||||
if (typeof params.factsFactory !== "function") throw codedError("workbench_kafka_facts_factory_required", "Kafka projection requires a pure facts factory.");
|
||||
const built = await params.factsFactory({ projectedSeq, previousCheckpoint, projectedAt: now });
|
||||
let facts = null;
|
||||
let persistedEvents = [];
|
||||
if (built?.written !== false) {
|
||||
facts = normalizeWorkbenchFacts(mergeKafkaSessionIdentity(built?.facts ?? {}, existingSession), params.requestMeta ?? {}, now);
|
||||
const persisted = await persistFacts(store, client, facts, params.requestMeta ?? {});
|
||||
persistedEvents = persisted.events;
|
||||
}
|
||||
const committedProjectedSeq = built?.written === false
|
||||
? nonNegativeInteger(built?.projectedSeq) || nonNegativeInteger(previousCheckpoint?.projectedSeq) || projectedSeq
|
||||
: projectedSeq;
|
||||
const projectionWritten = built?.written !== false;
|
||||
if (projectionWritten) {
|
||||
const hwlabEvent = objectValue(params.hwlabEvent);
|
||||
const hwlabEventId = requiredText(hwlabEvent.eventId, "HWLAB Kafka eventId");
|
||||
await client.query(
|
||||
"INSERT INTO hwlab_kafka_outbox (event_id, topic, partition_key, payload_json, headers_json, attempt_count, next_attempt_at, lease_owner, lease_expires_at, published_at, last_error, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,0,$6,NULL,NULL,NULL,NULL,$6,$6) ON CONFLICT (event_id) DO NOTHING",
|
||||
[hwlabEventId, requiredText(params.hwlabTopic, "HWLAB Kafka topic"), requiredText(params.partitionKey, "HWLAB Kafka partition key"), stableJson(hwlabEvent), stableJson(objectValue(params.headers)), now]
|
||||
);
|
||||
}
|
||||
await client.query(
|
||||
"UPDATE workbench_kafka_inbox SET status = 'projected', projected_seq = $1, updated_at = $2, projected_at = $2 WHERE source_topic = $3 AND source_partition = $4 AND source_offset = $5",
|
||||
[committedProjectedSeq, now, transport.sourceTopic, transport.sourcePartition, transport.sourceOffset]
|
||||
);
|
||||
return { duplicate: false, conflict: false, projectedSeq: committedProjectedSeq, facts, events: persistedEvents, projectionWritten, suppressedAfterSeal: built?.suppressedAfterSeal === true };
|
||||
});
|
||||
if (result.facts) store.memory.writeWorkbenchFacts({ facts: result.facts }, params.requestMeta ?? {});
|
||||
return { ...result, transport, sourceEventId, traceId, sessionId, sourceSeq, valuesPrinted: false };
|
||||
}
|
||||
|
||||
export async function recordFailedAgentRunKafkaMessage(store, params = {}) {
|
||||
const transport = normalizeTransport(params.transport);
|
||||
const now = store.now();
|
||||
const error = redactedError(params.error);
|
||||
const sourceEventId = text(params.sourceEventId);
|
||||
const inputSha256 = requiredText(transport.inputSha256, "Kafka input sha256");
|
||||
const result = await store.withDurableTransaction("workbench.kafka-projection.dlq", async (client) => {
|
||||
await lockKafkaInboxIdentity(client, transport, sourceEventId);
|
||||
const existing = await client.query("SELECT source_topic, source_partition, source_offset, input_sha256, status, projected_seq FROM workbench_kafka_inbox WHERE (source_topic = $1 AND source_partition = $2 AND source_offset = $3) OR ($4::text IS NOT NULL AND source_topic = $1 AND source_event_id = $4) ORDER BY created_at ASC LIMIT 1 FOR UPDATE", [transport.sourceTopic, transport.sourcePartition, transport.sourceOffset, sourceEventId]);
|
||||
const prior = existing.rows?.[0] ?? null;
|
||||
if (prior) {
|
||||
const sameTransport = prior.source_partition === transport.sourcePartition && String(prior.source_offset) === transport.sourceOffset;
|
||||
const diagnostic = prior.input_sha256 !== inputSha256
|
||||
? { ...error, code: "workbench_kafka_inbox_hash_conflict" }
|
||||
: sameTransport ? error : { ...error, code: "workbench_kafka_inbox_source_event_duplicate" };
|
||||
if (prior.status !== "projected") {
|
||||
await client.query("UPDATE workbench_kafka_inbox SET status = 'failed', error_json = $1, updated_at = $2 WHERE source_topic = $3 AND source_partition = $4 AND source_offset = $5", [stableJson(diagnostic), now, prior.source_topic, prior.source_partition, prior.source_offset]);
|
||||
}
|
||||
await insertDlq(client, { ...transport, sourceEventId, inputSha256, error: diagnostic, envelope: redactedEnvelope(params.envelope), createdAt: now });
|
||||
return { duplicate: sameTransport, conflict: !sameTransport || prior.input_sha256 !== inputSha256, priorStatus: prior.status, projectedSeq: nonNegativeInteger(prior.projected_seq), diagnostic };
|
||||
}
|
||||
await client.query(
|
||||
"INSERT INTO workbench_kafka_inbox (source_topic, source_partition, source_offset, source_key, source_event_id, input_sha256, trace_id, session_id, run_id, command_id, source_seq, status, projected_seq, envelope_json, error_json, created_at, updated_at, projected_at) VALUES ($1,$2,$3,$4,$5,$6,NULL,NULL,NULL,NULL,0,'failed',NULL,$7,$8,$9,$9,NULL) ON CONFLICT (source_topic, source_partition, source_offset) DO UPDATE SET status = 'failed', error_json = EXCLUDED.error_json, updated_at = EXCLUDED.updated_at",
|
||||
[transport.sourceTopic, transport.sourcePartition, transport.sourceOffset, transport.sourceKey, sourceEventId, inputSha256, stableJson(redactedEnvelope(params.envelope)), stableJson(error), now]
|
||||
);
|
||||
await insertDlq(client, { ...transport, sourceEventId, inputSha256, error, envelope: redactedEnvelope(params.envelope), createdAt: now });
|
||||
return { duplicate: false, conflict: false, priorStatus: null, diagnostic: error };
|
||||
});
|
||||
return { recorded: true, transport, sourceEventId, error: result.diagnostic, ...result, valuesPrinted: false };
|
||||
}
|
||||
|
||||
export async function recordIgnoredAgentRunKafkaMessage(store, params = {}) {
|
||||
const transport = normalizeTransport(params.transport);
|
||||
const envelope = objectValue(params.envelope);
|
||||
const sourceEventId = requiredText(envelope.eventId, "canonical AgentRun eventId");
|
||||
const inputSha256 = requiredText(transport.inputSha256, "Kafka input sha256");
|
||||
const now = store.now();
|
||||
return store.withDurableTransaction("workbench.kafka-projection.ignore", async (client) => {
|
||||
await lockKafkaInboxIdentity(client, transport, sourceEventId);
|
||||
const existing = await client.query("SELECT input_sha256, status FROM workbench_kafka_inbox WHERE (source_topic = $1 AND source_partition = $2 AND source_offset = $3) OR (source_topic = $1 AND source_event_id = $4) ORDER BY created_at ASC LIMIT 1", [transport.sourceTopic, transport.sourcePartition, transport.sourceOffset, sourceEventId]);
|
||||
const prior = existing.rows?.[0] ?? null;
|
||||
if (prior) {
|
||||
if (prior.input_sha256 !== inputSha256) {
|
||||
const diagnostic = { code: "workbench_kafka_inbox_hash_conflict", message: "Ignored Kafka identity was reused with a different payload hash.", valuesRedacted: true };
|
||||
await insertDlq(client, { ...transport, sourceEventId, inputSha256, error: diagnostic, envelope: redactedEnvelope(envelope), createdAt: now });
|
||||
return { ignored: true, duplicate: false, conflict: true, status: prior.status, diagnostic, valuesPrinted: false };
|
||||
}
|
||||
return { ignored: true, duplicate: true, status: prior.status, valuesPrinted: false };
|
||||
}
|
||||
await client.query(
|
||||
"INSERT INTO workbench_kafka_inbox (source_topic, source_partition, source_offset, source_key, source_event_id, input_sha256, trace_id, session_id, run_id, command_id, source_seq, status, projected_seq, envelope_json, error_json, created_at, updated_at, projected_at) VALUES ($1,$2,$3,$4,$5,$6,NULL,NULL,$7,$8,$9,'ignored',NULL,$10,'{}',$11,$11,NULL)",
|
||||
[transport.sourceTopic, transport.sourcePartition, transport.sourceOffset, transport.sourceKey, sourceEventId, inputSha256, text(envelope.runId), text(envelope.commandId), nonNegativeInteger(envelope.sourceSeq), stableJson(redactedEnvelope(envelope)), now]
|
||||
);
|
||||
return { ignored: true, duplicate: false, status: "ignored", valuesPrinted: false };
|
||||
});
|
||||
}
|
||||
|
||||
export async function claimHwlabKafkaOutbox(store, { owner, leaseMs, limit } = {}) {
|
||||
const leaseOwner = requiredText(owner, "HWLAB Kafka relay owner");
|
||||
const boundedLimit = boundedInteger(limit, 100, 1, 500);
|
||||
const now = store.now();
|
||||
const leaseExpiresAt = new Date(Date.parse(now) + boundedInteger(leaseMs, 60_000, 1_000, 3_600_000)).toISOString();
|
||||
const result = await store.withDurableTransaction("hwlab.kafka-outbox.claim", (client) => client.query(
|
||||
`WITH candidates AS (
|
||||
SELECT candidate.outbox_seq FROM hwlab_kafka_outbox candidate
|
||||
WHERE candidate.published_at IS NULL
|
||||
AND candidate.next_attempt_at <= $1
|
||||
AND (candidate.lease_expires_at IS NULL OR candidate.lease_expires_at <= $1)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM hwlab_kafka_outbox earlier
|
||||
WHERE earlier.published_at IS NULL
|
||||
AND earlier.partition_key = candidate.partition_key
|
||||
AND earlier.outbox_seq < candidate.outbox_seq
|
||||
)
|
||||
ORDER BY candidate.outbox_seq ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT $2
|
||||
)
|
||||
UPDATE hwlab_kafka_outbox o SET attempt_count = o.attempt_count + 1, lease_owner = $3, lease_expires_at = $4, updated_at = $1
|
||||
FROM candidates WHERE o.outbox_seq = candidates.outbox_seq
|
||||
RETURNING o.*`,
|
||||
[now, boundedLimit, leaseOwner, leaseExpiresAt]
|
||||
));
|
||||
return (result.rows ?? []).map(outboxRow);
|
||||
}
|
||||
|
||||
export async function completeHwlabKafkaOutbox(store, item = {}) {
|
||||
const result = await store.query(
|
||||
"UPDATE hwlab_kafka_outbox SET published_at = $1, lease_owner = NULL, lease_expires_at = NULL, last_error = NULL, updated_at = $1 WHERE outbox_seq = $2 AND lease_owner = $3 AND attempt_count = $4 AND lease_expires_at = $5 AND lease_expires_at > $1 AND published_at IS NULL RETURNING outbox_seq",
|
||||
[store.now(), item.outboxSeq, item.leaseOwner, item.attemptCount, item.leaseExpiresAt]
|
||||
);
|
||||
if (!result.rows?.[0]) throw codedError("hwlab_kafka_outbox_lease_conflict", "HWLAB Kafka outbox claim is stale.");
|
||||
return { completed: true, outboxSeq: Number(result.rows[0].outbox_seq), valuesPrinted: false };
|
||||
}
|
||||
|
||||
export async function retryHwlabKafkaOutbox(store, item = {}, error = null, retryAt = null) {
|
||||
const now = store.now();
|
||||
const nextAttemptAt = text(retryAt) ?? now;
|
||||
const result = await store.query(
|
||||
"UPDATE hwlab_kafka_outbox SET next_attempt_at = $1, lease_owner = NULL, lease_expires_at = NULL, last_error = $2, updated_at = $3 WHERE outbox_seq = $4 AND lease_owner = $5 AND attempt_count = $6 AND lease_expires_at = $7 AND lease_expires_at > $3 AND published_at IS NULL RETURNING outbox_seq",
|
||||
[nextAttemptAt, stableJson(redactedError(error)), now, item.outboxSeq, item.leaseOwner, item.attemptCount, item.leaseExpiresAt]
|
||||
);
|
||||
if (!result.rows?.[0]) throw codedError("hwlab_kafka_outbox_lease_conflict", "HWLAB Kafka outbox claim is stale.");
|
||||
return { retried: true, outboxSeq: Number(result.rows[0].outbox_seq), nextAttemptAt, valuesPrinted: false };
|
||||
}
|
||||
|
||||
export async function hwlabKafkaProjectorStatus(store) {
|
||||
const result = await store.query(
|
||||
`SELECT
|
||||
COUNT(*) FILTER (WHERE published_at IS NULL)::int AS backlog_count,
|
||||
COUNT(*) FILTER (WHERE published_at IS NULL AND attempt_count > 0)::int AS retry_count,
|
||||
MIN(created_at) FILTER (WHERE published_at IS NULL) AS oldest_pending_at,
|
||||
(SELECT COUNT(*)::int FROM workbench_kafka_inbox WHERE status = 'failed') AS failed_inbox_count,
|
||||
(SELECT COUNT(*)::int FROM workbench_kafka_dlq) AS dlq_count
|
||||
FROM hwlab_kafka_outbox`, []
|
||||
);
|
||||
const row = result.rows?.[0] ?? {};
|
||||
return { backlogCount: Number(row.backlog_count ?? 0), retryCount: Number(row.retry_count ?? 0), oldestPendingAt: row.oldest_pending_at ?? null, failedInboxCount: Number(row.failed_inbox_count ?? 0), dlqCount: Number(row.dlq_count ?? 0), valuesPrinted: false };
|
||||
}
|
||||
|
||||
export async function readAtomicWorkbenchProjectionSync(store, params = {}) {
|
||||
const since = nonNegativeInteger(params.afterOutboxSeq ?? params.afterSeq ?? params.since);
|
||||
const limit = boundedInteger(params.limit, 100, 1, 500);
|
||||
const traceId = text(params.traceId);
|
||||
const sessionId = text(params.sessionId);
|
||||
const snapshotOnly = params.snapshotOnly === true;
|
||||
const deltaOnly = params.deltaOnly === true;
|
||||
if (!traceId && !sessionId) throw codedError("workbench_sync_scope_required", "Atomic Workbench sync requires sessionId or traceId.");
|
||||
if (snapshotOnly && deltaOnly) throw codedError("workbench_sync_mode_invalid", "snapshotOnly and deltaOnly are mutually exclusive.");
|
||||
return store.withDurableTransaction("workbench.projection-sync.read", async (client) => {
|
||||
await client.query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY");
|
||||
const filter = traceId ? { column: "trace_id", value: traceId } : { column: "session_id", value: sessionId };
|
||||
const cutoffResult = await client.query(`SELECT COALESCE(MAX(outbox_seq), 0)::bigint AS cutoff FROM workbench_projection_outbox WHERE ${filter.column} = $1`, [filter.value]);
|
||||
const cutoff = Number(cutoffResult.rows?.[0]?.cutoff ?? 0);
|
||||
const outboxResult = snapshotOnly ? { rows: [] } : await client.query(
|
||||
`SELECT outbox_seq, outbox_event_id, entity_family, entity_id, event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at FROM workbench_projection_outbox WHERE outbox_seq > $1 AND outbox_seq <= $2 AND ${filter.column} = $3 ORDER BY outbox_seq ASC LIMIT $4`,
|
||||
[since, cutoff, filter.value, limit + 1]
|
||||
);
|
||||
const allRows = (outboxResult.rows ?? []).map(projectionOutboxRow);
|
||||
const events = allRows.slice(0, limit);
|
||||
const facts = deltaOnly ? {} : await readSyncFacts(client, { traceId, sessionId, limit: 500, families: snapshotOnly ? ["sessions", "messages", "turns"] : null });
|
||||
const hasMore = allRows.length > limit;
|
||||
return { facts, events, cutoffOutboxSeq: cutoff, cursorOutboxSeq: snapshotOnly ? cutoff : hasMore ? (events.at(-1)?.outboxSeq ?? since) : cutoff, hasMore, valuesPrinted: false };
|
||||
});
|
||||
}
|
||||
|
||||
async function persistFacts(store, client, facts, requestMeta) {
|
||||
const aggregateEvents = normalizeWorkbenchAggregateEventsForFacts(facts, requestMeta, store.now());
|
||||
const events = [];
|
||||
for (const event of aggregateEvents) events.push(await store.persistWorkbenchAggregateEvent(event, client));
|
||||
const eventIndex = indexWorkbenchAggregateEvents(events);
|
||||
const inputFacts = facts.inputs.map((fact) => inputFactWithAggregateSeq(fact, eventIndex));
|
||||
for (const fact of inputFacts) await store.persistWorkbenchSessionInputFact(fact, client);
|
||||
for (const fact of facts.sessions) await store.persistWorkbenchSessionFact(fact, client);
|
||||
for (const fact of facts.messages) await store.persistWorkbenchMessageFact(fact, client);
|
||||
for (const fact of facts.parts) await store.persistWorkbenchPartFact(fact, client);
|
||||
for (const fact of facts.turns) await store.persistWorkbenchTurnFact(fact, client);
|
||||
for (const fact of facts.traceEvents) await store.persistWorkbenchTraceEventFact(fact, client);
|
||||
for (const fact of facts.checkpoints) await store.persistWorkbenchProjectionCheckpoint(fact, client);
|
||||
for (const fact of facts.messages) {
|
||||
const event = eventIndex.get(`message:${fact.messageId}`) ?? eventIndex.get(`messages:${fact.messageId}`) ?? eventIndex.get(`sourceEvent:${fact.sourceEventId}`) ?? null;
|
||||
await store.persistWorkbenchProjectionOutbox(outboxRecord(fact, event, "messages", "message", { role: fact.role, status: fact.status }), client);
|
||||
}
|
||||
for (const fact of facts.traceEvents) {
|
||||
const event = eventIndex.get(`traceEvent:${fact.id}`) ?? eventIndex.get(`sourceEvent:${fact.sourceEventId}`) ?? null;
|
||||
await store.persistWorkbenchProjectionOutbox(outboxRecord(fact, event, "traceEvents", fact.terminal ? "terminal" : "event", { type: fact.eventType }), client);
|
||||
}
|
||||
for (const fact of facts.turns.filter((item) => item.terminal || item.sealed)) {
|
||||
const event = eventIndex.get(`turn:${fact.turnId}`) ?? eventIndex.get(`sourceEvent:${fact.sourceEventId}`) ?? null;
|
||||
await store.persistWorkbenchProjectionOutbox(outboxRecord(fact, event, "turns", "terminal", { status: fact.status, failureKind: fact.failureKind }), client);
|
||||
}
|
||||
return { events, inputFacts };
|
||||
}
|
||||
|
||||
function outboxRecord(fact, event, family, commitType, metadata) {
|
||||
const entityId = text(family === "messages" ? fact.messageId : family === "turns" ? fact.turnId : fact.id) ?? text(fact.traceId) ?? "projection";
|
||||
return {
|
||||
outboxEventId: projectionOutboxEventId({ fact, event, family, entityId, commitType }),
|
||||
entityFamily: family,
|
||||
entityId,
|
||||
eventSeq: event?.eventSeq ?? null,
|
||||
aggregateId: event?.aggregateId ?? null,
|
||||
aggregateSeq: event?.aggregateSeq ?? 0,
|
||||
projectionRevision: event?.projectionRevision ?? fact.projectedSeq,
|
||||
traceId: fact.traceId,
|
||||
sessionId: fact.sessionId,
|
||||
turnId: fact.turnId,
|
||||
messageId: fact.messageId,
|
||||
projectedSeq: fact.projectedSeq,
|
||||
sourceSeq: fact.sourceSeq,
|
||||
sourceEventId: fact.sourceEventId,
|
||||
commitType,
|
||||
terminal: Boolean(fact.terminal),
|
||||
sealed: Boolean(fact.sealed),
|
||||
payload: {
|
||||
family,
|
||||
fact,
|
||||
metadata: { ...metadata, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq, valuesRedacted: true },
|
||||
valuesRedacted: true
|
||||
},
|
||||
createdAt: fact.occurredAt ?? fact.updatedAt ?? fact.createdAt
|
||||
};
|
||||
}
|
||||
|
||||
export function projectionOutboxEventId({ fact = {}, event = null, family, entityId, commitType }) {
|
||||
const sourceEventId = text(fact.sourceEventId);
|
||||
if (sourceEventId) return `${sourceEventId}:${family}:${entityId}:${commitType}`;
|
||||
const identity = stableJson({
|
||||
family,
|
||||
entityId,
|
||||
commitType,
|
||||
eventSeq: event?.eventSeq ?? null,
|
||||
aggregateId: event?.aggregateId ?? null,
|
||||
aggregateSeq: event?.aggregateSeq ?? 0,
|
||||
projectionRevision: event?.projectionRevision ?? fact.projectedSeq ?? 0,
|
||||
projectedSeq: fact.projectedSeq ?? 0,
|
||||
sourceSeq: fact.sourceSeq ?? 0
|
||||
});
|
||||
return `projection:${createHash("sha256").update(identity).digest("hex")}`;
|
||||
}
|
||||
|
||||
function mergeKafkaSessionIdentity(facts, row) {
|
||||
if (!row) return facts;
|
||||
const existing = parseJsonColumn(row.session_json, {});
|
||||
const sessions = Array.isArray(facts?.sessions) ? facts.sessions.map((fact) => ({
|
||||
...existing,
|
||||
...fact,
|
||||
ownerUserId: fact.ownerUserId ?? row.owner_user_id ?? existing.ownerUserId ?? null,
|
||||
projectId: fact.projectId ?? row.project_id ?? existing.projectId ?? null,
|
||||
conversationId: fact.conversationId ?? row.conversation_id ?? existing.conversationId ?? null,
|
||||
threadId: fact.threadId ?? row.thread_id ?? existing.threadId ?? null,
|
||||
ownerRole: fact.ownerRole ?? existing.ownerRole ?? null
|
||||
})) : [];
|
||||
return { ...facts, sessions };
|
||||
}
|
||||
|
||||
async function readSyncFacts(client, { traceId, sessionId, limit, families = null }) {
|
||||
const familySet = Array.isArray(families) ? new Set(families) : null;
|
||||
const specs = [
|
||||
["sessions", "workbench_sessions", "session_json", traceId ? "last_trace_id" : "session_id"],
|
||||
["messages", "workbench_messages", "message_json", traceId ? "trace_id" : "session_id"],
|
||||
["parts", "workbench_parts", "part_json", traceId ? "trace_id" : "session_id"],
|
||||
["turns", "workbench_turns", "turn_json", traceId ? "trace_id" : "session_id"],
|
||||
["traceEvents", "workbench_trace_events", "event_json", traceId ? "trace_id" : "session_id"],
|
||||
["checkpoints", "workbench_projection_checkpoints", "checkpoint_json", traceId ? "trace_id" : "session_id"]
|
||||
].filter(([family]) => !familySet || familySet.has(family));
|
||||
const value = traceId ?? sessionId;
|
||||
const facts = {};
|
||||
for (const [family, table, jsonColumn, column] of specs) {
|
||||
const result = await client.query(`SELECT ${jsonColumn} FROM ${table} WHERE ${column} = $1 ORDER BY updated_at ASC LIMIT $2`, [value, limit]);
|
||||
facts[family] = (result.rows ?? []).map((row) => parseJsonColumn(row[jsonColumn], null)).filter(Boolean);
|
||||
}
|
||||
return facts;
|
||||
}
|
||||
|
||||
async function insertDlq(client, record) {
|
||||
await client.query(
|
||||
"INSERT INTO workbench_kafka_dlq (source_topic, source_partition, source_offset, source_event_id, input_sha256, error_code, error_message, envelope_json, created_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (source_topic, source_partition, source_offset) DO UPDATE SET error_code = EXCLUDED.error_code, error_message = EXCLUDED.error_message, envelope_json = EXCLUDED.envelope_json",
|
||||
[record.sourceTopic, record.sourcePartition, record.sourceOffset, record.sourceEventId, record.inputSha256, record.error.code, record.error.message, stableJson(record.envelope), record.createdAt]
|
||||
);
|
||||
}
|
||||
|
||||
async function lockKafkaInboxIdentity(client, transport, sourceEventId) {
|
||||
const transportIdentity = `${transport.sourceTopic}:transport:${transport.sourcePartition}:${transport.sourceOffset}`;
|
||||
await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`workbench-kafka-inbox:${transportIdentity}`]);
|
||||
if (sourceEventId) {
|
||||
const eventIdentity = `${transport.sourceTopic}:event:${sourceEventId}`;
|
||||
await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`workbench-kafka-inbox:${eventIdentity}`]);
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeTransport(value = {}) {
|
||||
const input = objectValue(value);
|
||||
const sourceTopic = requiredText(input.sourceTopic ?? input.topic, "Kafka source topic");
|
||||
const sourcePartition = Number(input.sourcePartition ?? input.partition);
|
||||
const sourceOffset = text(input.sourceOffset ?? input.offset);
|
||||
if (!Number.isInteger(sourcePartition) || sourcePartition < 0) throw codedError("workbench_kafka_partition_invalid", "Kafka source partition must be a non-negative integer.");
|
||||
if (!/^\d+$/u.test(sourceOffset ?? "")) throw codedError("workbench_kafka_offset_invalid", "Kafka source offset must be a non-negative integer string.");
|
||||
return { sourceTopic, sourcePartition, sourceOffset, inputSha256: text(input.inputSha256), sourceKey: text(input.sourceKey), valuesPrinted: false };
|
||||
}
|
||||
|
||||
function projectionOutboxRow(row) {
|
||||
return { outboxSeq: Number(row.outbox_seq), outboxEventId: row.outbox_event_id, entityFamily: row.entity_family, entityId: row.entity_id, eventSeq: row.event_seq == null ? null : Number(row.event_seq), aggregateId: row.aggregate_id, aggregateSeq: Number(row.aggregate_seq ?? 0), projectionRevision: Number(row.projection_revision ?? 0), traceId: row.trace_id, sessionId: row.session_id, turnId: row.turn_id, messageId: row.message_id, projectedSeq: Number(row.projected_seq ?? 0), sourceSeq: Number(row.source_seq ?? 0), sourceEventId: row.source_event_id, commitType: row.commit_type, terminal: Boolean(row.terminal), sealed: Boolean(row.sealed), payload: parseJsonColumn(row.payload_json, {}), createdAt: row.created_at, valuesPrinted: false };
|
||||
}
|
||||
|
||||
function outboxRow(row) {
|
||||
return { outboxSeq: Number(row.outbox_seq), eventId: row.event_id, topic: row.topic, partitionKey: row.partition_key, payload: parseJsonColumn(row.payload_json, {}), headers: parseJsonColumn(row.headers_json, {}), attemptCount: Number(row.attempt_count ?? 0), nextAttemptAt: row.next_attempt_at, leaseOwner: row.lease_owner, leaseExpiresAt: row.lease_expires_at, createdAt: row.created_at, valuesPrinted: false };
|
||||
}
|
||||
|
||||
function redactedEnvelope(value) {
|
||||
const input = objectValue(value);
|
||||
return { schema: text(input.schema), eventType: text(input.eventType), eventId: text(input.eventId), outboxSeq: nonNegativeInteger(input.outboxSeq), sourceSeq: nonNegativeInteger(input.sourceSeq), traceId: text(input.traceId), sessionId: text(input.sessionId), hwlabSessionId: text(input.hwlabSessionId), runId: text(input.runId), commandId: text(input.commandId), valuesRedacted: true };
|
||||
}
|
||||
|
||||
function redactedError(error) {
|
||||
return { code: text(error?.code) ?? "workbench_kafka_message_invalid", message: String(error?.message ?? error ?? "Kafka message is invalid.").slice(0, 500), valuesRedacted: true };
|
||||
}
|
||||
|
||||
function objectValue(value) {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
||||
}
|
||||
|
||||
function text(value) {
|
||||
const normalized = textOr(value, "");
|
||||
return normalized || null;
|
||||
}
|
||||
|
||||
function requiredText(value, label) {
|
||||
const normalized = text(value);
|
||||
if (!normalized) throw codedError("workbench_kafka_contract_invalid", `${label} is required.`);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function boundedInteger(value, fallback, min, max) {
|
||||
const parsed = Number(value);
|
||||
return Number.isInteger(parsed) && parsed >= min && parsed <= max ? parsed : fallback;
|
||||
}
|
||||
|
||||
function codedError(code, message) {
|
||||
const error = new Error(message);
|
||||
error.code = code;
|
||||
return error;
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { EventEmitter } from "node:events";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { subscribeWorkbenchProjectionCommits, WORKBENCH_PROJECTION_CHANNEL } from "./runtime-store-postgres-notify.ts";
|
||||
|
||||
test("Postgres LISTEN fans committed projection scope out and releases its dedicated connection", async () => {
|
||||
const queries = [];
|
||||
let released = 0;
|
||||
const client = new EventEmitter();
|
||||
client.query = async (sql) => { queries.push(sql); return { rows: [] }; };
|
||||
client.release = () => { released += 1; };
|
||||
const pool = { async connect() { return client; } };
|
||||
const store = { async getQueryClient() { return pool; }, logger: null };
|
||||
const signals = [];
|
||||
|
||||
const unsubscribe = await subscribeWorkbenchProjectionCommits(store, (signal) => signals.push(signal));
|
||||
assert.deepEqual(queries, [`LISTEN ${WORKBENCH_PROJECTION_CHANNEL}`]);
|
||||
|
||||
client.emit("notification", { channel: WORKBENCH_PROJECTION_CHANNEL, payload: JSON.stringify({ sessionId: "ses_notify", traceId: "trc_notify", outboxSeq: 12 }) });
|
||||
assert.deepEqual(signals, [{ sessionId: "ses_notify", traceId: "trc_notify", outboxSeq: 12, valuesRedacted: true }]);
|
||||
|
||||
await unsubscribe();
|
||||
assert.equal(queries.at(-1), `UNLISTEN ${WORKBENCH_PROJECTION_CHANNEL}`);
|
||||
assert.equal(released, 1);
|
||||
});
|
||||
@@ -0,0 +1,145 @@
|
||||
/* PostgreSQL projection notifications wake SSE readers; the outbox remains the payload authority. */
|
||||
|
||||
const WORKBENCH_PROJECTION_CHANNEL = "hwlab_workbench_projection";
|
||||
const RECONNECT_DELAY_MS = 500;
|
||||
const hubs = new WeakMap();
|
||||
|
||||
export async function subscribeWorkbenchProjectionCommits(store, listener) {
|
||||
if (typeof listener !== "function") throw codedError("workbench_projection_listener_invalid", "Projection notification listener must be a function.");
|
||||
let hub = hubs.get(store);
|
||||
if (!hub) {
|
||||
hub = { store, listeners: new Set(), client: null, connectPromise: null, reconnectTimer: null, connectedOnce: false, closing: false };
|
||||
hubs.set(store, hub);
|
||||
}
|
||||
hub.listeners.add(listener);
|
||||
hub.closing = false;
|
||||
await ensureConnected(hub);
|
||||
let subscribed = true;
|
||||
return async () => {
|
||||
if (!subscribed) return;
|
||||
subscribed = false;
|
||||
hub.listeners.delete(listener);
|
||||
if (hub.listeners.size === 0) await closeHub(hub);
|
||||
};
|
||||
}
|
||||
|
||||
async function ensureConnected(hub) {
|
||||
if (hub.client || hub.connectPromise || hub.closing || hub.listeners.size === 0) return hub.connectPromise;
|
||||
hub.connectPromise = (async () => {
|
||||
const queryClient = await hub.store.getQueryClient();
|
||||
const client = typeof queryClient?.connect === "function" ? await queryClient.connect() : queryClient;
|
||||
if (!client || typeof client.query !== "function" || typeof client.on !== "function") {
|
||||
client?.release?.();
|
||||
throw codedError("workbench_projection_notify_unsupported", "Postgres runtime adapter requires a dedicated LISTEN connection.");
|
||||
}
|
||||
const onNotification = (message) => {
|
||||
if (message?.channel !== WORKBENCH_PROJECTION_CHANNEL) return;
|
||||
const signal = parseSignal(message.payload);
|
||||
for (const notify of [...hub.listeners]) {
|
||||
try { notify(signal); } catch {}
|
||||
}
|
||||
};
|
||||
const onDisconnect = (error) => disconnectHubClient(hub, client, error);
|
||||
client.on("notification", onNotification);
|
||||
client.on("error", onDisconnect);
|
||||
client.on("end", onDisconnect);
|
||||
client.__hwlabProjectionNotifyHandlers = { onNotification, onDisconnect };
|
||||
try {
|
||||
await client.query(`LISTEN ${WORKBENCH_PROJECTION_CHANNEL}`);
|
||||
} catch (error) {
|
||||
detachClient(client, error);
|
||||
throw error;
|
||||
}
|
||||
if (hub.closing || hub.listeners.size === 0) {
|
||||
detachClient(client);
|
||||
return;
|
||||
}
|
||||
hub.client = client;
|
||||
if (hub.connectedOnce) emitRecovery(hub);
|
||||
hub.connectedOnce = true;
|
||||
})().finally(() => {
|
||||
hub.connectPromise = null;
|
||||
});
|
||||
return hub.connectPromise;
|
||||
}
|
||||
|
||||
function disconnectHubClient(hub, client, error) {
|
||||
if (client && hub.client !== client) return;
|
||||
if (client) {
|
||||
hub.client = null;
|
||||
detachClient(client, error);
|
||||
}
|
||||
scheduleReconnect(hub, error);
|
||||
}
|
||||
|
||||
function scheduleReconnect(hub, error) {
|
||||
if (hub.closing || hub.listeners.size === 0 || hub.reconnectTimer) return;
|
||||
hub.store.logger?.warn?.({ event: "workbench_projection_notify_disconnected", errorCode: error?.code ?? "UNKNOWN", valuesRedacted: true });
|
||||
hub.reconnectTimer = setTimeout(() => {
|
||||
hub.reconnectTimer = null;
|
||||
void ensureConnected(hub).catch((connectError) => scheduleReconnect(hub, connectError));
|
||||
}, RECONNECT_DELAY_MS);
|
||||
hub.reconnectTimer.unref?.();
|
||||
}
|
||||
|
||||
async function closeHub(hub) {
|
||||
hub.closing = true;
|
||||
if (hub.reconnectTimer) clearTimeout(hub.reconnectTimer);
|
||||
hub.reconnectTimer = null;
|
||||
const client = hub.client;
|
||||
hub.client = null;
|
||||
if (client) {
|
||||
try { await client.query(`UNLISTEN ${WORKBENCH_PROJECTION_CHANNEL}`); } catch {}
|
||||
detachClient(client);
|
||||
}
|
||||
hubs.delete(hub.store);
|
||||
}
|
||||
|
||||
function detachClient(client, error = null) {
|
||||
const handlers = client?.__hwlabProjectionNotifyHandlers;
|
||||
if (handlers) {
|
||||
client.off?.("notification", handlers.onNotification);
|
||||
client.off?.("error", handlers.onDisconnect);
|
||||
client.off?.("end", handlers.onDisconnect);
|
||||
delete client.__hwlabProjectionNotifyHandlers;
|
||||
}
|
||||
client?.release?.(error || undefined);
|
||||
}
|
||||
|
||||
function emitRecovery(hub) {
|
||||
for (const notify of [...hub.listeners]) {
|
||||
try { notify({ recovery: true, valuesRedacted: true }); } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
function parseSignal(payload) {
|
||||
try {
|
||||
const value = JSON.parse(String(payload ?? "{}"));
|
||||
return {
|
||||
sessionId: text(value.sessionId),
|
||||
traceId: text(value.traceId),
|
||||
outboxSeq: nonNegativeInteger(value.outboxSeq),
|
||||
valuesRedacted: true
|
||||
};
|
||||
} catch {
|
||||
return { recovery: true, valuesRedacted: true };
|
||||
}
|
||||
}
|
||||
|
||||
function text(value) {
|
||||
const normalized = String(value ?? "").trim();
|
||||
return normalized || null;
|
||||
}
|
||||
|
||||
function nonNegativeInteger(value) {
|
||||
const number = Number(value);
|
||||
return Number.isInteger(number) && number >= 0 ? number : 0;
|
||||
}
|
||||
|
||||
function codedError(code, message) {
|
||||
const error = new Error(message);
|
||||
error.code = code;
|
||||
return error;
|
||||
}
|
||||
|
||||
export { WORKBENCH_PROJECTION_CHANNEL };
|
||||
@@ -19,12 +19,26 @@ import {
|
||||
CLOUD_CORE_MIGRATION_ID,
|
||||
CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION,
|
||||
CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE,
|
||||
CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS
|
||||
CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS,
|
||||
CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID,
|
||||
CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION,
|
||||
CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS
|
||||
} from "./schema.ts";
|
||||
|
||||
|
||||
import { CloudRuntimeStore } from "./runtime-store-memory.ts";
|
||||
import * as runtimeCore from "./runtime-store-core.ts";
|
||||
import {
|
||||
claimHwlabKafkaOutbox as claimHwlabKafkaOutboxOperation,
|
||||
commitAgentRunKafkaProjection as commitAgentRunKafkaProjectionOperation,
|
||||
completeHwlabKafkaOutbox as completeHwlabKafkaOutboxOperation,
|
||||
hwlabKafkaProjectorStatus as hwlabKafkaProjectorStatusOperation,
|
||||
readAtomicWorkbenchProjectionSync as readAtomicWorkbenchProjectionSyncOperation,
|
||||
recordFailedAgentRunKafkaMessage as recordFailedAgentRunKafkaMessageOperation,
|
||||
recordIgnoredAgentRunKafkaMessage as recordIgnoredAgentRunKafkaMessageOperation,
|
||||
retryHwlabKafkaOutbox as retryHwlabKafkaOutboxOperation
|
||||
} from "./runtime-store-postgres-kafka.ts";
|
||||
import { subscribeWorkbenchProjectionCommits as subscribeWorkbenchProjectionCommitsOperation } from "./runtime-store-postgres-notify.ts";
|
||||
|
||||
const {
|
||||
RUNTIME_STORE_KIND,
|
||||
@@ -697,7 +711,7 @@ export class PostgresCloudRuntimeStore {
|
||||
commitType: "message",
|
||||
terminal: Boolean(fact.terminal),
|
||||
sealed: Boolean(fact.sealed),
|
||||
payload: { role: fact.role, status: fact.status, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq, valuesRedacted: true },
|
||||
payload: immutableProjectionOutboxPayload("messages", fact, { role: fact.role, status: fact.status, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq }),
|
||||
createdAt: fact.updatedAt ?? fact.createdAt ?? this.now()
|
||||
}, client);
|
||||
}
|
||||
@@ -718,7 +732,7 @@ export class PostgresCloudRuntimeStore {
|
||||
commitType: fact.terminal ? "terminal" : "event",
|
||||
terminal: Boolean(fact.terminal),
|
||||
sealed: Boolean(fact.sealed),
|
||||
payload: { type: fact.eventType, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq, valuesRedacted: true },
|
||||
payload: immutableProjectionOutboxPayload("traceEvents", fact, { type: fact.eventType, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq }),
|
||||
createdAt: fact.occurredAt ?? fact.updatedAt ?? this.now()
|
||||
}, client);
|
||||
}
|
||||
@@ -740,7 +754,7 @@ export class PostgresCloudRuntimeStore {
|
||||
commitType: "terminal",
|
||||
terminal: true,
|
||||
sealed: true,
|
||||
payload: { status: fact.status, failureKind: fact.failureKind, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, valuesRedacted: true },
|
||||
payload: immutableProjectionOutboxPayload("turns", fact, { status: fact.status, failureKind: fact.failureKind, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null }),
|
||||
createdAt: fact.updatedAt ?? this.now()
|
||||
}, client);
|
||||
}
|
||||
@@ -752,6 +766,135 @@ export class PostgresCloudRuntimeStore {
|
||||
return withPersistence({ written: true, facts: persistedFacts, events: txResult.events }, this.summary());
|
||||
}
|
||||
|
||||
async writeWorkbenchSessionAdmissionFact(params = {}, requestMeta = {}) {
|
||||
const readiness = this.summary();
|
||||
if (readiness.ready !== true || readiness.durable !== true || readiness.liveRuntimeEvidence !== true) {
|
||||
await this.assertReadyForWrites();
|
||||
}
|
||||
const facts = normalizeWorkbenchFacts({ sessions: [params.fact ?? params.session ?? params] }, requestMeta, this.now());
|
||||
const fact = facts.sessions[0];
|
||||
if (!fact?.sessionId) throw new Error("Workbench session admission fact requires sessionId.");
|
||||
await this.withDurableTransaction("workbench.session-admission.write", (client) => this.persistWorkbenchSessionAdmissionFact(fact, client));
|
||||
const memoryResult = this.memory.writeWorkbenchSessionAdmissionFact({ fact }, requestMeta);
|
||||
return withPersistence({ written: true, admissionOnly: true, fact: memoryResult.fact, valuesPrinted: false }, this.summary());
|
||||
}
|
||||
|
||||
async promoteWorkbenchTurnAdmission(params = {}, requestMeta = {}) {
|
||||
const readiness = this.summary();
|
||||
if (readiness.ready !== true || readiness.durable !== true || readiness.liveRuntimeEvidence !== true) {
|
||||
await this.assertReadyForWrites();
|
||||
}
|
||||
const facts = normalizeWorkbenchFacts(params.facts ?? params, requestMeta, this.now());
|
||||
const sessionFact = facts.sessions[0];
|
||||
const inputFact = facts.inputs[0];
|
||||
const turnFact = facts.turns[0];
|
||||
if (!sessionFact?.sessionId || !inputFact?.inputId || !turnFact?.turnId) throw new Error("Workbench turn admission promotion requires session, input, and turn facts.");
|
||||
const aggregateEvents = normalizeWorkbenchAggregateEventsForFacts(facts, requestMeta, this.now());
|
||||
const transaction = await this.withDurableTransaction("workbench.turn-admission.promote", async (client) => {
|
||||
await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`workbench-projection:${turnFact.traceId}`]);
|
||||
const existingTurnResult = await client.query(
|
||||
"SELECT projected_seq, sealed FROM workbench_turns WHERE turn_id = $1 FOR UPDATE",
|
||||
[turnFact.turnId]
|
||||
);
|
||||
const existingTurn = existingTurnResult.rows?.[0] ?? null;
|
||||
const supersededByProjection = nonNegativeInteger(existingTurn?.projected_seq) > 0 || existingTurn?.sealed === true;
|
||||
const persistedEvents = [];
|
||||
const transactionEvents = supersededByProjection
|
||||
? aggregateEvents.filter((event) => event.factFamily === "inputs")
|
||||
: aggregateEvents;
|
||||
for (const event of transactionEvents) persistedEvents.push(await this.persistWorkbenchAggregateEvent(event, client));
|
||||
const eventIndex = indexWorkbenchAggregateEvents(persistedEvents);
|
||||
const promotedInput = inputFactWithAggregateSeq(inputFact, eventIndex);
|
||||
await this.persistWorkbenchSessionInputFact(promotedInput, client);
|
||||
if (supersededByProjection) {
|
||||
return { persistedEvents, promotedInput, outbox: null, supersededByProjection: true };
|
||||
}
|
||||
await this.persistWorkbenchSessionAdmissionFact(sessionFact, client);
|
||||
for (const fact of facts.messages) await this.persistWorkbenchMessageAdmissionFact(fact, client);
|
||||
for (const fact of facts.parts) await this.persistWorkbenchPartAdmissionFact(fact, client);
|
||||
for (const fact of facts.turns) await this.persistWorkbenchTurnAdmissionFact(fact, client);
|
||||
const turnEvent = eventIndex.get(`turn:${turnFact.turnId}`) ?? eventIndex.get(`sourceEvent:${turnFact.sourceEventId}`) ?? null;
|
||||
const outbox = {
|
||||
entityFamily: "turns",
|
||||
entityId: turnFact.turnId,
|
||||
eventSeq: turnEvent?.eventSeq ?? null,
|
||||
aggregateId: turnEvent?.aggregateId ?? null,
|
||||
aggregateSeq: turnEvent?.aggregateSeq ?? 0,
|
||||
projectionRevision: turnEvent?.projectionRevision ?? turnFact.projectedSeq,
|
||||
traceId: turnFact.traceId,
|
||||
sessionId: turnFact.sessionId,
|
||||
turnId: turnFact.turnId,
|
||||
messageId: turnFact.messageId,
|
||||
projectedSeq: turnFact.projectedSeq,
|
||||
sourceSeq: turnFact.sourceSeq,
|
||||
sourceEventId: turnFact.sourceEventId,
|
||||
commitType: "admission",
|
||||
terminal: false,
|
||||
sealed: false,
|
||||
payload: immutableProjectionOutboxPayload("turns", turnFact, { status: turnFact.status, admissionState: "promoted", eventSeq: turnEvent?.eventSeq ?? null, aggregateSeq: turnEvent?.aggregateSeq ?? null }),
|
||||
createdAt: turnFact.updatedAt ?? this.now()
|
||||
};
|
||||
await this.persistWorkbenchProjectionOutbox(outbox, client);
|
||||
return { persistedEvents, promotedInput, outbox, supersededByProjection: false };
|
||||
});
|
||||
const memoryResult = transaction.supersededByProjection
|
||||
? this.memory.writeWorkbenchFacts({ facts: { inputs: [transaction.promotedInput] } }, requestMeta)
|
||||
: this.memory.promoteWorkbenchTurnAdmission({ facts: { ...facts, inputs: [transaction.promotedInput] } }, requestMeta);
|
||||
return withPersistence({
|
||||
written: true,
|
||||
promoted: true,
|
||||
outboxCommitted: !transaction.supersededByProjection,
|
||||
outbox: transaction.outbox,
|
||||
supersededByProjection: transaction.supersededByProjection,
|
||||
facts: memoryResult.facts,
|
||||
events: transaction.persistedEvents,
|
||||
valuesPrinted: false
|
||||
}, this.summary());
|
||||
}
|
||||
|
||||
async commitAgentRunKafkaProjection(params = {}) {
|
||||
await this.assertReadyForWrites();
|
||||
return commitAgentRunKafkaProjectionOperation(this, params);
|
||||
}
|
||||
|
||||
async recordFailedAgentRunKafkaMessage(params = {}) {
|
||||
await this.assertReadyForWrites();
|
||||
return recordFailedAgentRunKafkaMessageOperation(this, params);
|
||||
}
|
||||
|
||||
async recordIgnoredAgentRunKafkaMessage(params = {}) {
|
||||
await this.assertReadyForWrites();
|
||||
return recordIgnoredAgentRunKafkaMessageOperation(this, params);
|
||||
}
|
||||
|
||||
async claimHwlabKafkaOutbox(params = {}) {
|
||||
await this.assertReadyForWrites();
|
||||
return claimHwlabKafkaOutboxOperation(this, params);
|
||||
}
|
||||
|
||||
async completeHwlabKafkaOutbox(item = {}) {
|
||||
return completeHwlabKafkaOutboxOperation(this, item);
|
||||
}
|
||||
|
||||
async retryHwlabKafkaOutbox(item = {}, error = null, retryAt = null) {
|
||||
return retryHwlabKafkaOutboxOperation(this, item, error, retryAt);
|
||||
}
|
||||
|
||||
async hwlabKafkaProjectorStatus() {
|
||||
await this.assertReadyForDurableReads("hwlab.kafka-projector.status");
|
||||
return hwlabKafkaProjectorStatusOperation(this);
|
||||
}
|
||||
|
||||
async readAtomicWorkbenchProjectionSync(params = {}) {
|
||||
await this.assertReadyForDurableReads("workbench.projection-sync.read");
|
||||
return readAtomicWorkbenchProjectionSyncOperation(this, params);
|
||||
}
|
||||
|
||||
async subscribeWorkbenchProjectionCommits(listener) {
|
||||
await this.assertReadyForDurableReads("workbench.projection-notify.listen");
|
||||
return subscribeWorkbenchProjectionCommitsOperation(this, listener);
|
||||
}
|
||||
|
||||
async queryWorkbenchFacts(params = {}) {
|
||||
await this.assertReadyForDurableReads("workbench.facts.query");
|
||||
const families = workbenchFactFamilySet(params);
|
||||
@@ -1122,7 +1265,14 @@ export class PostgresCloudRuntimeStore {
|
||||
|
||||
async persistWorkbenchSessionFact(record, client = this) {
|
||||
await client.query(
|
||||
"INSERT INTO workbench_sessions (session_id, owner_user_id, project_id, conversation_id, thread_id, status, last_trace_id, projected_seq, source_seq, source_event_id, terminal, sealed, session_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (session_id) DO UPDATE SET owner_user_id = EXCLUDED.owner_user_id, project_id = EXCLUDED.project_id, conversation_id = EXCLUDED.conversation_id, thread_id = EXCLUDED.thread_id, status = EXCLUDED.status, last_trace_id = EXCLUDED.last_trace_id, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, session_json = EXCLUDED.session_json, updated_at = EXCLUDED.updated_at",
|
||||
"INSERT INTO workbench_sessions (session_id, owner_user_id, project_id, conversation_id, thread_id, status, last_trace_id, projected_seq, source_seq, source_event_id, terminal, sealed, session_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (session_id) DO UPDATE SET owner_user_id = COALESCE(EXCLUDED.owner_user_id, workbench_sessions.owner_user_id), project_id = COALESCE(EXCLUDED.project_id, workbench_sessions.project_id), conversation_id = COALESCE(EXCLUDED.conversation_id, workbench_sessions.conversation_id), thread_id = COALESCE(EXCLUDED.thread_id, workbench_sessions.thread_id), status = EXCLUDED.status, last_trace_id = EXCLUDED.last_trace_id, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, session_json = EXCLUDED.session_json, updated_at = EXCLUDED.updated_at",
|
||||
[record.sessionId, record.ownerUserId, record.projectId, record.conversationId, record.threadId, record.status, record.lastTraceId, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record), record.createdAt, record.updatedAt]
|
||||
);
|
||||
}
|
||||
|
||||
async persistWorkbenchSessionAdmissionFact(record, client = this) {
|
||||
await client.query(
|
||||
"INSERT INTO workbench_sessions (session_id, owner_user_id, project_id, conversation_id, thread_id, status, last_trace_id, projected_seq, source_seq, source_event_id, terminal, sealed, session_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (session_id) DO UPDATE SET owner_user_id = COALESCE(workbench_sessions.owner_user_id, EXCLUDED.owner_user_id), project_id = COALESCE(workbench_sessions.project_id, EXCLUDED.project_id), conversation_id = COALESCE(workbench_sessions.conversation_id, EXCLUDED.conversation_id), thread_id = COALESCE(workbench_sessions.thread_id, EXCLUDED.thread_id), status = CASE WHEN workbench_sessions.projected_seq <= 0 AND NOT workbench_sessions.sealed THEN EXCLUDED.status ELSE workbench_sessions.status END, last_trace_id = CASE WHEN workbench_sessions.projected_seq <= 0 AND NOT workbench_sessions.sealed THEN COALESCE(EXCLUDED.last_trace_id, workbench_sessions.last_trace_id) ELSE workbench_sessions.last_trace_id END, session_json = CASE WHEN workbench_sessions.projected_seq <= 0 AND NOT workbench_sessions.sealed THEN EXCLUDED.session_json ELSE workbench_sessions.session_json END, updated_at = CASE WHEN workbench_sessions.projected_seq <= 0 AND NOT workbench_sessions.sealed THEN EXCLUDED.updated_at ELSE workbench_sessions.updated_at END",
|
||||
[record.sessionId, record.ownerUserId, record.projectId, record.conversationId, record.threadId, record.status, record.lastTraceId, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record), record.createdAt, record.updatedAt]
|
||||
);
|
||||
}
|
||||
@@ -1141,6 +1291,13 @@ export class PostgresCloudRuntimeStore {
|
||||
);
|
||||
}
|
||||
|
||||
async persistWorkbenchMessageAdmissionFact(record, client = this) {
|
||||
await client.query(
|
||||
"INSERT INTO workbench_messages (message_id, session_id, turn_id, trace_id, role, status, projected_seq, source_seq, source_event_id, terminal, sealed, message_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT (message_id) DO UPDATE SET session_id = EXCLUDED.session_id, turn_id = EXCLUDED.turn_id, trace_id = EXCLUDED.trace_id, role = EXCLUDED.role, status = CASE WHEN workbench_messages.projected_seq <= 0 AND NOT workbench_messages.sealed THEN EXCLUDED.status ELSE workbench_messages.status END, source_event_id = CASE WHEN workbench_messages.projected_seq <= 0 AND NOT workbench_messages.sealed THEN EXCLUDED.source_event_id ELSE workbench_messages.source_event_id END, message_json = CASE WHEN workbench_messages.projected_seq <= 0 AND NOT workbench_messages.sealed THEN EXCLUDED.message_json ELSE workbench_messages.message_json END, updated_at = CASE WHEN workbench_messages.projected_seq <= 0 AND NOT workbench_messages.sealed THEN EXCLUDED.updated_at ELSE workbench_messages.updated_at END",
|
||||
[record.messageId, record.sessionId, record.turnId, record.traceId, record.role, record.status, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record), record.createdAt, record.updatedAt]
|
||||
);
|
||||
}
|
||||
|
||||
async persistWorkbenchPartFact(record, client = this) {
|
||||
await client.query(
|
||||
"INSERT INTO workbench_parts (part_id, message_id, session_id, turn_id, trace_id, part_index, part_type, status, projected_seq, source_seq, source_event_id, terminal, sealed, part_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) ON CONFLICT (part_id) DO UPDATE SET message_id = EXCLUDED.message_id, session_id = EXCLUDED.session_id, turn_id = EXCLUDED.turn_id, trace_id = EXCLUDED.trace_id, part_index = EXCLUDED.part_index, part_type = EXCLUDED.part_type, status = EXCLUDED.status, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, part_json = EXCLUDED.part_json, updated_at = EXCLUDED.updated_at",
|
||||
@@ -1148,6 +1305,13 @@ export class PostgresCloudRuntimeStore {
|
||||
);
|
||||
}
|
||||
|
||||
async persistWorkbenchPartAdmissionFact(record, client = this) {
|
||||
await client.query(
|
||||
"INSERT INTO workbench_parts (part_id, message_id, session_id, turn_id, trace_id, part_index, part_type, status, projected_seq, source_seq, source_event_id, terminal, sealed, part_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) ON CONFLICT (part_id) DO UPDATE SET message_id = EXCLUDED.message_id, session_id = EXCLUDED.session_id, turn_id = EXCLUDED.turn_id, trace_id = EXCLUDED.trace_id, part_index = EXCLUDED.part_index, part_type = EXCLUDED.part_type, status = CASE WHEN workbench_parts.projected_seq <= 0 AND NOT workbench_parts.sealed THEN EXCLUDED.status ELSE workbench_parts.status END, source_event_id = CASE WHEN workbench_parts.projected_seq <= 0 AND NOT workbench_parts.sealed THEN EXCLUDED.source_event_id ELSE workbench_parts.source_event_id END, part_json = CASE WHEN workbench_parts.projected_seq <= 0 AND NOT workbench_parts.sealed THEN EXCLUDED.part_json ELSE workbench_parts.part_json END, updated_at = CASE WHEN workbench_parts.projected_seq <= 0 AND NOT workbench_parts.sealed THEN EXCLUDED.updated_at ELSE workbench_parts.updated_at END",
|
||||
[record.partId, record.messageId, record.sessionId, record.turnId, record.traceId, record.partIndex, record.partType, record.status, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record), record.createdAt, record.updatedAt]
|
||||
);
|
||||
}
|
||||
|
||||
async persistWorkbenchTurnFact(record, client = this) {
|
||||
await client.query(
|
||||
"INSERT INTO workbench_turns (turn_id, session_id, trace_id, message_id, status, projected_seq, source_seq, source_event_id, terminal, sealed, final_response_json, diagnostic_json, turn_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (turn_id) DO UPDATE SET session_id = EXCLUDED.session_id, trace_id = EXCLUDED.trace_id, message_id = EXCLUDED.message_id, status = EXCLUDED.status, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, final_response_json = EXCLUDED.final_response_json, diagnostic_json = EXCLUDED.diagnostic_json, turn_json = EXCLUDED.turn_json, updated_at = EXCLUDED.updated_at",
|
||||
@@ -1155,6 +1319,13 @@ export class PostgresCloudRuntimeStore {
|
||||
);
|
||||
}
|
||||
|
||||
async persistWorkbenchTurnAdmissionFact(record, client = this) {
|
||||
await client.query(
|
||||
"INSERT INTO workbench_turns (turn_id, session_id, trace_id, message_id, status, projected_seq, source_seq, source_event_id, terminal, sealed, final_response_json, diagnostic_json, turn_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (turn_id) DO UPDATE SET session_id = EXCLUDED.session_id, trace_id = EXCLUDED.trace_id, message_id = CASE WHEN workbench_turns.projected_seq <= 0 AND NOT workbench_turns.sealed THEN EXCLUDED.message_id ELSE workbench_turns.message_id END, status = CASE WHEN workbench_turns.projected_seq <= 0 AND NOT workbench_turns.sealed THEN EXCLUDED.status ELSE workbench_turns.status END, source_event_id = CASE WHEN workbench_turns.projected_seq <= 0 AND NOT workbench_turns.sealed THEN EXCLUDED.source_event_id ELSE workbench_turns.source_event_id END, diagnostic_json = CASE WHEN workbench_turns.projected_seq <= 0 AND NOT workbench_turns.sealed THEN EXCLUDED.diagnostic_json ELSE workbench_turns.diagnostic_json END, turn_json = CASE WHEN workbench_turns.projected_seq <= 0 AND NOT workbench_turns.sealed THEN EXCLUDED.turn_json ELSE workbench_turns.turn_json END, updated_at = CASE WHEN workbench_turns.projected_seq <= 0 AND NOT workbench_turns.sealed THEN EXCLUDED.updated_at ELSE workbench_turns.updated_at END",
|
||||
[record.turnId, record.sessionId, record.traceId, record.messageId, record.status, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record.finalResponse), stableJson(record.diagnostic), stableJson(record), record.createdAt, record.updatedAt]
|
||||
);
|
||||
}
|
||||
|
||||
async persistWorkbenchTraceEventFact(record, client = this) {
|
||||
await client.query(
|
||||
"INSERT INTO workbench_trace_events (id, trace_id, session_id, turn_id, message_id, source_seq, source_event_id, projected_seq, event_type, terminal, sealed, event_json, occurred_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT (id) DO UPDATE SET trace_id = EXCLUDED.trace_id, session_id = EXCLUDED.session_id, turn_id = EXCLUDED.turn_id, message_id = EXCLUDED.message_id, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, projected_seq = EXCLUDED.projected_seq, event_type = EXCLUDED.event_type, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, event_json = EXCLUDED.event_json, occurred_at = EXCLUDED.occurred_at, updated_at = EXCLUDED.updated_at",
|
||||
@@ -1164,8 +1335,8 @@ export class PostgresCloudRuntimeStore {
|
||||
|
||||
async persistWorkbenchProjectionOutbox(record, client = this) {
|
||||
await client.query(
|
||||
"INSERT INTO workbench_projection_outbox (event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)",
|
||||
[record.eventSeq, record.aggregateId, nonNegativeInteger(record.aggregateSeq), nonNegativeInteger(record.projectionRevision), record.traceId, record.sessionId, record.turnId, record.messageId, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.commitType ?? "event", Boolean(record.terminal), Boolean(record.sealed), stableJson(record.payload ?? record), record.createdAt ?? this.now()]
|
||||
"INSERT INTO workbench_projection_outbox (outbox_event_id, entity_family, entity_id, event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19) ON CONFLICT (outbox_event_id) DO NOTHING",
|
||||
[record.outboxEventId ?? projectionOutboxEventId(record), record.entityFamily ?? projectionOutboxEntity(record).family, record.entityId ?? projectionOutboxEntity(record).id, record.eventSeq, record.aggregateId, nonNegativeInteger(record.aggregateSeq), nonNegativeInteger(record.projectionRevision), record.traceId, record.sessionId, record.turnId, record.messageId, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.commitType ?? "event", Boolean(record.terminal), Boolean(record.sealed), stableJson(record.payload ?? record), record.createdAt ?? this.now()]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1186,11 +1357,14 @@ export class PostgresCloudRuntimeStore {
|
||||
}
|
||||
params.push(Math.min(Math.max(Number(limit) || 100, 1), 500));
|
||||
const result = await this.query(
|
||||
`SELECT outbox_seq, event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at FROM workbench_projection_outbox WHERE ${where} ORDER BY outbox_seq ASC LIMIT $${paramIdx}`,
|
||||
`SELECT outbox_seq, outbox_event_id, entity_family, entity_id, event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at FROM workbench_projection_outbox WHERE ${where} ORDER BY outbox_seq ASC LIMIT $${paramIdx}`,
|
||||
params
|
||||
);
|
||||
return (result.rows ?? []).map((row) => ({
|
||||
outboxSeq: Number(row.outbox_seq),
|
||||
outboxEventId: row.outbox_event_id,
|
||||
entityFamily: row.entity_family,
|
||||
entityId: row.entity_id,
|
||||
eventSeq: row.event_seq === null || row.event_seq === undefined ? null : Number(row.event_seq),
|
||||
aggregateId: row.aggregate_id,
|
||||
aggregateSeq: Number(row.aggregate_seq ?? 0),
|
||||
@@ -1248,6 +1422,45 @@ export class PostgresCloudRuntimeStore {
|
||||
};
|
||||
}
|
||||
|
||||
async workbenchTransactionalRealtimeReadiness() {
|
||||
const schemaResult = await this.query(
|
||||
"SELECT table_name, column_name FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = ANY($1::text[])",
|
||||
[Object.keys(CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS)]
|
||||
);
|
||||
const schema = summarizeRuntimeSchema(schemaResult?.rows ?? [], CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS);
|
||||
const migrationResult = await this.query(
|
||||
`SELECT id, schema_version FROM ${CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE} WHERE id = $1 LIMIT 1`,
|
||||
[CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID]
|
||||
);
|
||||
const row = migrationResult.rows?.[0] ?? null;
|
||||
const migration = {
|
||||
checked: true,
|
||||
ready: row?.id === CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID && row?.schema_version === CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION,
|
||||
table: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE,
|
||||
requiredMigrationId: CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID,
|
||||
requiredSchemaVersion: CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION,
|
||||
appliedMigrationId: row?.id ?? null,
|
||||
appliedSchemaVersion: row?.schema_version ?? null,
|
||||
missing: !row
|
||||
};
|
||||
return {
|
||||
ready: schema.ready && migration.ready,
|
||||
schema,
|
||||
migration,
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
async assertWorkbenchTransactionalRealtimeReady() {
|
||||
const readiness = await this.workbenchTransactionalRealtimeReadiness();
|
||||
if (readiness.ready) return readiness;
|
||||
const error = new Error("Workbench transactional realtime schema or migration is not ready.");
|
||||
error.code = readiness.schema.ready ? "workbench_transactional_realtime_migration_blocked" : "workbench_transactional_realtime_schema_blocked";
|
||||
error.readiness = readiness;
|
||||
error.valuesRedacted = true;
|
||||
throw error;
|
||||
}
|
||||
|
||||
async ensureRuntimeReadIndexes() {
|
||||
if (this.runtimeReadIndexesReady) return;
|
||||
if (!this.runtimeReadIndexesReadyPromise) {
|
||||
@@ -1535,3 +1748,28 @@ export class PostgresCloudRuntimeStore {
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function projectionOutboxEventId(record = {}) {
|
||||
const sourceEventId = textOr(record.sourceEventId, "");
|
||||
const commitType = textOr(record.commitType, "event");
|
||||
const family = textOr(record.entityFamily ?? record.payload?.family, "") || (commitType === "message" ? "messages" : record.payload?.type ? "traceEvents" : "turns");
|
||||
const entityId = family === "traceEvents"
|
||||
? textOr(record.entityId ?? record.payload?.fact?.id ?? record.aggregateId ?? record.sourceEventId ?? record.traceId, "projection")
|
||||
: textOr(record.entityId ?? record.payload?.fact?.messageId ?? record.payload?.fact?.turnId ?? record.messageId ?? record.turnId ?? record.traceId ?? record.aggregateId, "projection");
|
||||
if (sourceEventId) return `${sourceEventId}:${family}:${entityId}:${commitType}`;
|
||||
const derived = stableJson({ family, entityId, commitType, eventSeq: record.eventSeq ?? null, aggregateId: record.aggregateId ?? null, aggregateSeq: record.aggregateSeq ?? 0, projectionRevision: record.projectionRevision ?? record.projectedSeq ?? 0, traceId: record.traceId ?? null, sessionId: record.sessionId ?? null });
|
||||
return `derived:${createHash("sha256").update(derived).digest("hex")}`;
|
||||
}
|
||||
|
||||
function projectionOutboxEntity(record = {}) {
|
||||
const commitType = textOr(record.commitType, "event");
|
||||
const family = textOr(record.entityFamily ?? record.payload?.family, "") || (commitType === "message" ? "messages" : record.payload?.type ? "traceEvents" : "turns");
|
||||
const id = family === "traceEvents"
|
||||
? textOr(record.entityId ?? record.payload?.fact?.id ?? record.aggregateId ?? record.sourceEventId ?? record.traceId, "projection")
|
||||
: textOr(record.entityId ?? record.payload?.fact?.messageId ?? record.payload?.fact?.turnId ?? record.messageId ?? record.turnId ?? record.traceId ?? record.aggregateId, "projection");
|
||||
return { family, id };
|
||||
}
|
||||
|
||||
function immutableProjectionOutboxPayload(family, fact, metadata = {}) {
|
||||
return { family, fact, metadata: { ...metadata, valuesRedacted: true }, valuesRedacted: true };
|
||||
}
|
||||
|
||||
@@ -1605,7 +1605,7 @@ function createFakePostgresClient({
|
||||
if (beforeWorkbenchFactRead) await beforeWorkbenchFactRead(sql, params);
|
||||
return workbenchFactRows(state.workbench_projection_checkpoints, "checkpoint_json", sql, params, readErrorCode);
|
||||
}
|
||||
if (sql.startsWith("SELECT outbox_seq, event_seq, aggregate_id")) {
|
||||
if (sql.startsWith("SELECT outbox_seq, outbox_event_id, entity_family, entity_id")) {
|
||||
const afterSeq = Number(params[0] ?? 0);
|
||||
const traceFilter = sql.includes("trace_id = $2") ? params[1] : null;
|
||||
const sessionFilter = sql.includes("session_id = $2") ? params[1] : sql.includes("session_id = $3") ? params[2] : null;
|
||||
@@ -1770,25 +1770,30 @@ function createFakePostgresClient({
|
||||
error.code = outboxErrorCode;
|
||||
throw error;
|
||||
}
|
||||
const existing = [...state.workbench_projection_outbox.values()].find((record) => record.outbox_event_id === params[0]);
|
||||
if (existing) return { rows: [] };
|
||||
const outboxSeq = state.workbench_projection_outbox.size + 1;
|
||||
state.workbench_projection_outbox.set(outboxSeq, {
|
||||
outbox_seq: outboxSeq,
|
||||
event_seq: params[0],
|
||||
aggregate_id: params[1],
|
||||
aggregate_seq: params[2],
|
||||
projection_revision: params[3],
|
||||
trace_id: params[4],
|
||||
session_id: params[5],
|
||||
turn_id: params[6],
|
||||
message_id: params[7],
|
||||
projected_seq: params[8],
|
||||
source_seq: params[9],
|
||||
source_event_id: params[10],
|
||||
commit_type: params[11],
|
||||
terminal: params[12],
|
||||
sealed: params[13],
|
||||
payload_json: params[14],
|
||||
created_at: params[15]
|
||||
outbox_event_id: params[0],
|
||||
entity_family: params[1],
|
||||
entity_id: params[2],
|
||||
event_seq: params[3],
|
||||
aggregate_id: params[4],
|
||||
aggregate_seq: params[5],
|
||||
projection_revision: params[6],
|
||||
trace_id: params[7],
|
||||
session_id: params[8],
|
||||
turn_id: params[9],
|
||||
message_id: params[10],
|
||||
projected_seq: params[11],
|
||||
source_seq: params[12],
|
||||
source_event_id: params[13],
|
||||
commit_type: params[14],
|
||||
terminal: params[15],
|
||||
sealed: params[16],
|
||||
payload_json: params[17],
|
||||
created_at: params[18]
|
||||
});
|
||||
return { rows: [] };
|
||||
}
|
||||
|
||||
@@ -12,6 +12,8 @@ import {
|
||||
CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE,
|
||||
CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE_COLUMNS,
|
||||
CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS,
|
||||
CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID,
|
||||
CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION,
|
||||
FROZEN_CLOUD_CORE_TABLES,
|
||||
assertFrozenCloudCoreTables,
|
||||
requiredRuntimeDurableColumns
|
||||
@@ -41,15 +43,17 @@ test("initial migration skeleton declares every frozen table", async () => {
|
||||
assert.match(sql, /\btimestamp\b/);
|
||||
});
|
||||
|
||||
test("initial migration exposes columns required by the durable runtime adapter", async () => {
|
||||
const sql = await readFile(path.join(repoRoot, "internal/db/migrations/0001_cloud_core_skeleton.sql"), "utf8");
|
||||
test("versioned migration chain exposes columns required by the durable runtime adapter", async () => {
|
||||
const sql = await readMigrationChain();
|
||||
|
||||
assert.ok(requiredRuntimeDurableColumns().length > 0);
|
||||
for (const [table, columns] of Object.entries(CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS)) {
|
||||
const tableMatch = sql.match(new RegExp(`CREATE TABLE IF NOT EXISTS ${table}\\s*\\(([\\s\\S]*?)\\);`, "u"));
|
||||
assert.ok(tableMatch, `missing ${table}`);
|
||||
for (const column of columns) {
|
||||
assert.match(tableMatch[1], new RegExp(`\\b${column}\\b`), `missing ${table}.${column}`);
|
||||
const declared = new RegExp(`\\b${column}\\b`).test(tableMatch[1]);
|
||||
const added = new RegExp(`ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS ${column}\\b`, "u").test(sql);
|
||||
assert.equal(declared || added, true, `missing ${table}.${column}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,6 +96,36 @@ test("initial migration declares Workbench fact backfill sources", async () => {
|
||||
assert.match(sql, /CREATE UNIQUE INDEX idx_workbench_trace_events_trace_projected_seq/u);
|
||||
});
|
||||
|
||||
test("v8 migration upgrades the existing Workbench outbox before enforcing realtime identity", async () => {
|
||||
const sql = await readFile(path.join(repoRoot, "internal/db/migrations/0008_workbench_kafka_realtime.sql"), "utf8");
|
||||
|
||||
for (const column of ["outbox_event_id", "entity_family", "entity_id"]) {
|
||||
assert.match(
|
||||
sql,
|
||||
new RegExp(`ALTER TABLE workbench_projection_outbox ADD COLUMN IF NOT EXISTS ${column} TEXT`, "u"),
|
||||
`missing additive upgrade for workbench_projection_outbox.${column}`
|
||||
);
|
||||
assert.match(
|
||||
sql,
|
||||
new RegExp(`ALTER TABLE workbench_projection_outbox ALTER COLUMN ${column} SET NOT NULL`, "u"),
|
||||
`missing post-backfill constraint for workbench_projection_outbox.${column}`
|
||||
);
|
||||
}
|
||||
|
||||
assert.match(sql, /SET outbox_event_id = COALESCE\(outbox_event_id, 'legacy:' \|\| outbox_seq::text\)/u);
|
||||
assert.match(sql, /entity_family = COALESCE\(entity_family, 'legacy'\)/u);
|
||||
assert.match(sql, /CREATE UNIQUE INDEX idx_workbench_projection_outbox_event_id ON workbench_projection_outbox\(outbox_event_id\)/u);
|
||||
assert.match(sql, /CREATE TRIGGER trg_notify_hwlab_workbench_projection/u);
|
||||
assert.match(sql, new RegExp(`'${CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID}'`, "u"));
|
||||
assert.match(sql, new RegExp(`'${CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION}'`, "u"));
|
||||
assert.match(sql, /ON CONFLICT \(id\) DO UPDATE SET\s+schema_version = EXCLUDED\.schema_version/u);
|
||||
});
|
||||
|
||||
async function readMigrationChain() {
|
||||
const sources = await Promise.all(CLOUD_CORE_MIGRATIONS.map((migration) => readFile(path.join(repoRoot, migration.path), "utf8")));
|
||||
return sources.join("\n");
|
||||
}
|
||||
|
||||
test("protocol record guards catch schema drift before runtime writes", () => {
|
||||
assertProtocolRecord("gatewaySession", {
|
||||
gatewaySessionId: "gws_01J00000000000000000000000",
|
||||
|
||||
+82
-1
@@ -6,6 +6,8 @@ import { TABLES } from "../protocol/index.mjs";
|
||||
|
||||
export const CLOUD_CORE_MIGRATION_ID = "0001_cloud_core_skeleton";
|
||||
export const CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION = "runtime-durable-postgres-v7";
|
||||
export const CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID = "0008_workbench_kafka_realtime";
|
||||
export const CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION = "runtime-durable-postgres-v8";
|
||||
export const CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE = "hwlab_schema_migrations";
|
||||
|
||||
export const FROZEN_CLOUD_CORE_TABLES = Object.freeze([...TABLES]);
|
||||
@@ -337,13 +339,86 @@ export const CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS = Object.freeze({
|
||||
])
|
||||
});
|
||||
|
||||
export const CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS = Object.freeze({
|
||||
workbench_projection_outbox: Object.freeze([
|
||||
"outbox_event_id",
|
||||
"entity_family",
|
||||
"entity_id"
|
||||
]),
|
||||
workbench_kafka_inbox: Object.freeze([
|
||||
"source_topic",
|
||||
"source_partition",
|
||||
"source_offset",
|
||||
"source_key",
|
||||
"source_event_id",
|
||||
"input_sha256",
|
||||
"trace_id",
|
||||
"session_id",
|
||||
"run_id",
|
||||
"command_id",
|
||||
"source_seq",
|
||||
"status",
|
||||
"projected_seq",
|
||||
"envelope_json",
|
||||
"error_json",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
"projected_at"
|
||||
]),
|
||||
workbench_kafka_dlq: Object.freeze([
|
||||
"dlq_seq",
|
||||
"source_topic",
|
||||
"source_partition",
|
||||
"source_offset",
|
||||
"source_event_id",
|
||||
"input_sha256",
|
||||
"error_code",
|
||||
"error_message",
|
||||
"envelope_json",
|
||||
"created_at"
|
||||
]),
|
||||
hwlab_kafka_outbox: Object.freeze([
|
||||
"outbox_seq",
|
||||
"event_id",
|
||||
"topic",
|
||||
"partition_key",
|
||||
"payload_json",
|
||||
"headers_json",
|
||||
"attempt_count",
|
||||
"next_attempt_at",
|
||||
"lease_owner",
|
||||
"lease_expires_at",
|
||||
"published_at",
|
||||
"last_error",
|
||||
"created_at",
|
||||
"updated_at"
|
||||
])
|
||||
});
|
||||
|
||||
export const CLOUD_TRANSACTIONAL_REALTIME_REQUIRED_TABLE_COLUMNS = Object.freeze({
|
||||
...CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS,
|
||||
...CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS,
|
||||
workbench_projection_outbox: Object.freeze([
|
||||
...CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS.workbench_projection_outbox,
|
||||
...CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS.workbench_projection_outbox
|
||||
])
|
||||
});
|
||||
|
||||
export const CLOUD_CORE_MIGRATIONS = Object.freeze([
|
||||
{
|
||||
id: CLOUD_CORE_MIGRATION_ID,
|
||||
path: "internal/db/migrations/0001_cloud_core_skeleton.sql",
|
||||
tables: FROZEN_CLOUD_CORE_TABLES,
|
||||
connectsToDatabase: false,
|
||||
runtimeDurableAdapterSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION,
|
||||
runtimeDurableAdapterSchemaVersion: "runtime-durable-postgres-v7",
|
||||
runtimeDurableMigrationTable: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE
|
||||
},
|
||||
{
|
||||
id: CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID,
|
||||
path: "internal/db/migrations/0008_workbench_kafka_realtime.sql",
|
||||
tables: Object.freeze(["workbench_projection_outbox", "workbench_kafka_inbox", "workbench_kafka_dlq", "hwlab_kafka_outbox"]),
|
||||
connectsToDatabase: false,
|
||||
runtimeDurableAdapterSchemaVersion: CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION,
|
||||
runtimeDurableMigrationTable: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE
|
||||
}
|
||||
]);
|
||||
@@ -360,3 +435,9 @@ export function requiredRuntimeDurableColumns() {
|
||||
columns.map((column) => `${table}.${column}`)
|
||||
);
|
||||
}
|
||||
|
||||
export function requiredTransactionalRealtimeColumns() {
|
||||
return Object.entries(CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS).flatMap(([table, columns]) =>
|
||||
columns.map((column) => `${table}.${column}`)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -390,7 +390,12 @@ function validateDisplayLocale(locale) {
|
||||
|
||||
function workbenchRuntimeConfigFromEnv() {
|
||||
const traceTimeline = {};
|
||||
const result = {};
|
||||
const result = {
|
||||
realtimeFeatures: {
|
||||
liveKafkaSse: requiredWorkbenchRealtimeFeature(process.env.HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED, "HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED"),
|
||||
projectionRealtime: requiredWorkbenchRealtimeFeature(process.env.HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED, "HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED")
|
||||
}
|
||||
};
|
||||
const autoExpandRunning = parseEnvBoolean(process.env.HWLAB_WORKBENCH_TRACE_AUTO_EXPAND_RUNNING);
|
||||
const autoCollapseTerminal = parseEnvBoolean(process.env.HWLAB_WORKBENCH_TRACE_AUTO_COLLAPSE_TERMINAL);
|
||||
const traceExplorerUrlTemplate = traceExplorerUrlTemplateFromEnv(process.env.HWLAB_WORKBENCH_TRACE_EXPLORER_URL_TEMPLATE);
|
||||
@@ -401,6 +406,13 @@ function workbenchRuntimeConfigFromEnv() {
|
||||
return Object.keys(result).length > 0 ? result : null;
|
||||
}
|
||||
|
||||
function requiredWorkbenchRealtimeFeature(value, name) {
|
||||
const normalized = String(value ?? "").trim().toLowerCase();
|
||||
if (normalized === "true" || normalized === "1") return true;
|
||||
if (normalized === "false" || normalized === "0") return false;
|
||||
throw new Error(`${name} is required and must be explicitly true or false`);
|
||||
}
|
||||
|
||||
function traceExplorerUrlTemplateFromEnv(value) {
|
||||
if (value === undefined || value === null || value === "") return null;
|
||||
const template = String(value).trim();
|
||||
|
||||
@@ -429,7 +429,9 @@ test("cloud web serves client deep links through the Vue shell", async () => {
|
||||
const restoreEnv = withEnv({
|
||||
HWLAB_CLOUD_WEB_DISPLAY_TIME_ZONE: "Asia/Shanghai",
|
||||
HWLAB_CLOUD_WEB_DISPLAY_TIME_LOCALE: "zh-CN",
|
||||
HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: "北京时间"
|
||||
HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: "北京时间",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false"
|
||||
});
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-cloud-web-runtime-"));
|
||||
await writeFile(path.join(root, "index.html"), "<div id=\"root\"></div>\n", "utf8");
|
||||
@@ -484,7 +486,9 @@ test("cloud web injects trace explorer runtime config from env", async () => {
|
||||
HWLAB_CLOUD_WEB_DISPLAY_TIME_ZONE: "Asia/Shanghai",
|
||||
HWLAB_CLOUD_WEB_DISPLAY_TIME_LOCALE: "zh-CN",
|
||||
HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: "北京时间",
|
||||
HWLAB_WORKBENCH_TRACE_EXPLORER_URL_TEMPLATE: "/v1/workbench/traces/{trace_id}/events"
|
||||
HWLAB_WORKBENCH_TRACE_EXPLORER_URL_TEMPLATE: "/v1/workbench/traces/{trace_id}/events",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false"
|
||||
});
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-cloud-web-runtime-"));
|
||||
await writeFile(path.join(root, "index.html"), "<html><head></head><body><div id=\"root\"></div></body></html>\n", "utf8");
|
||||
@@ -588,7 +592,9 @@ test("cloud web OpenCode frame-url endpoint mints traced tickets", async () => {
|
||||
HWLAB_CLOUD_WEB_DISPLAY_TIME_ZONE: "Asia/Shanghai",
|
||||
HWLAB_CLOUD_WEB_DISPLAY_TIME_LOCALE: "zh-CN",
|
||||
HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: "北京时间",
|
||||
HWLAB_CLOUD_WEB_OPENCODE_URL: "https://opencode.example.test"
|
||||
HWLAB_CLOUD_WEB_OPENCODE_URL: "https://opencode.example.test",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false"
|
||||
});
|
||||
const cloudApiRequests = [];
|
||||
const opencodeRequests = [];
|
||||
@@ -1028,7 +1034,9 @@ test("cloud web OpenCode proxy accepts short-lived tickets minted by the shell",
|
||||
HWLAB_CLOUD_WEB_DISPLAY_TIME_ZONE: "Asia/Shanghai",
|
||||
HWLAB_CLOUD_WEB_DISPLAY_TIME_LOCALE: "zh-CN",
|
||||
HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: "北京时间",
|
||||
HWLAB_CLOUD_WEB_OPENCODE_URL: "https://opencode.example.test"
|
||||
HWLAB_CLOUD_WEB_OPENCODE_URL: "https://opencode.example.test",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false"
|
||||
});
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-cloud-web-runtime-"));
|
||||
await writeFile(path.join(root, "index.html"), "<html><head></head><body><div id=\"root\"></div></body></html>\n", "utf8");
|
||||
|
||||
@@ -0,0 +1,493 @@
|
||||
package workbenchruntime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
realtimeSyncContractVersion = "workbench-runtime-sync-v1"
|
||||
realtimeSyncDefaultLimit = 100
|
||||
realtimeSyncMaxLimit = 500
|
||||
realtimeSyncMaxIdentityLen = 512
|
||||
)
|
||||
|
||||
type realtimeSyncQuery struct {
|
||||
SessionID string `json:"sessionId"`
|
||||
TraceID string `json:"traceId"`
|
||||
AfterOutboxSeq int64 `json:"afterOutboxSeq"`
|
||||
Limit int `json:"limit"`
|
||||
SnapshotOnly bool `json:"snapshotOnly,omitempty"`
|
||||
DeltaOnly bool `json:"deltaOnly,omitempty"`
|
||||
Actor sessionActor `json:"actor"`
|
||||
}
|
||||
|
||||
type realtimeSyncSnapshot struct {
|
||||
Facts map[string][]any `json:"facts"`
|
||||
Events []realtimeSyncEvent `json:"events"`
|
||||
CutoffOutboxSeq int64 `json:"cutoffOutboxSeq"`
|
||||
CursorOutboxSeq int64 `json:"cursorOutboxSeq"`
|
||||
HasMore bool `json:"hasMore"`
|
||||
}
|
||||
|
||||
type realtimeSyncEvent struct {
|
||||
OutboxSeq int64 `json:"outboxSeq"`
|
||||
OutboxEventID string `json:"outboxEventId"`
|
||||
EntityFamily string `json:"entityFamily"`
|
||||
EntityID string `json:"entityId"`
|
||||
EventSeq any `json:"eventSeq"`
|
||||
AggregateID any `json:"aggregateId"`
|
||||
AggregateSeq int64 `json:"aggregateSeq"`
|
||||
ProjectionRevision int64 `json:"projectionRevision"`
|
||||
TraceID string `json:"traceId"`
|
||||
SessionID any `json:"sessionId"`
|
||||
TurnID any `json:"turnId"`
|
||||
MessageID any `json:"messageId"`
|
||||
ProjectedSeq int64 `json:"projectedSeq"`
|
||||
SourceSeq int64 `json:"sourceSeq"`
|
||||
SourceEventID any `json:"sourceEventId"`
|
||||
CommitType string `json:"commitType"`
|
||||
Terminal bool `json:"terminal"`
|
||||
Sealed bool `json:"sealed"`
|
||||
Payload any `json:"payload"`
|
||||
CreatedAt string `json:"createdAt"`
|
||||
ValuesRedacted bool `json:"valuesRedacted"`
|
||||
}
|
||||
|
||||
type realtimeSyncFactSpec struct {
|
||||
Family string
|
||||
Table string
|
||||
JSONColumn string
|
||||
OrderBy string
|
||||
}
|
||||
|
||||
var realtimeSyncFactSpecs = []realtimeSyncFactSpec{
|
||||
{Family: "sessions", Table: "workbench_sessions", JSONColumn: "session_json", OrderBy: "updated_at ASC, session_id ASC"},
|
||||
{Family: "messages", Table: "workbench_messages", JSONColumn: "message_json", OrderBy: "updated_at ASC, message_id ASC"},
|
||||
{Family: "parts", Table: "workbench_parts", JSONColumn: "part_json", OrderBy: "updated_at ASC, part_index ASC, part_id ASC"},
|
||||
{Family: "turns", Table: "workbench_turns", JSONColumn: "turn_json", OrderBy: "updated_at ASC, turn_id ASC"},
|
||||
{Family: "traceEvents", Table: "workbench_trace_events", JSONColumn: "event_json", OrderBy: "projected_seq ASC, id ASC"},
|
||||
{Family: "checkpoints", Table: "workbench_projection_checkpoints", JSONColumn: "checkpoint_json", OrderBy: "updated_at ASC, trace_id ASC"},
|
||||
}
|
||||
|
||||
var realtimeSyncSnapshotFactSpecs = []realtimeSyncFactSpec{
|
||||
realtimeSyncFactSpecs[0],
|
||||
realtimeSyncFactSpecs[1],
|
||||
realtimeSyncFactSpecs[3],
|
||||
}
|
||||
|
||||
const realtimeSyncResolveScopeSQL = `
|
||||
SELECT sessions.session_id
|
||||
FROM workbench_sessions AS sessions
|
||||
WHERE ($1 = '' OR sessions.session_id = $1)
|
||||
AND (
|
||||
$2 = ''
|
||||
OR sessions.last_trace_id = $2
|
||||
OR EXISTS (
|
||||
SELECT 1
|
||||
FROM workbench_projection_outbox AS scoped_outbox
|
||||
WHERE scoped_outbox.session_id = sessions.session_id
|
||||
AND scoped_outbox.trace_id = $2
|
||||
)
|
||||
)
|
||||
AND ($3 OR sessions.owner_user_id = $4)
|
||||
ORDER BY
|
||||
CASE WHEN sessions.session_id = NULLIF($1, '') THEN 0 ELSE 1 END,
|
||||
sessions.updated_at DESC,
|
||||
sessions.session_id ASC
|
||||
LIMIT 1`
|
||||
|
||||
const realtimeSyncCutoffSQL = `
|
||||
SELECT COALESCE(MAX(outbox_seq), 0)::bigint
|
||||
FROM workbench_projection_outbox
|
||||
WHERE session_id = $1
|
||||
AND ($2 = '' OR trace_id = $2)`
|
||||
|
||||
const realtimeSyncEventsSQL = `
|
||||
SELECT
|
||||
outbox_seq,
|
||||
outbox_event_id,
|
||||
entity_family,
|
||||
entity_id,
|
||||
event_seq,
|
||||
aggregate_id,
|
||||
aggregate_seq,
|
||||
projection_revision,
|
||||
trace_id,
|
||||
session_id,
|
||||
turn_id,
|
||||
message_id,
|
||||
projected_seq,
|
||||
source_seq,
|
||||
source_event_id,
|
||||
commit_type,
|
||||
terminal,
|
||||
sealed,
|
||||
payload_json,
|
||||
created_at
|
||||
FROM workbench_projection_outbox
|
||||
WHERE session_id = $1
|
||||
AND ($2 = '' OR trace_id = $2)
|
||||
AND outbox_seq > $3
|
||||
AND outbox_seq <= $4
|
||||
ORDER BY outbox_seq ASC
|
||||
LIMIT $5`
|
||||
|
||||
func (s *Server) handleRealtimeSync(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
methodNotAllowed(w, "POST")
|
||||
return
|
||||
}
|
||||
query, ok := decodeRealtimeSyncQuery(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
query, validation := normalizeRealtimeSyncQuery(query)
|
||||
if validation != nil {
|
||||
writeAPIError(w, validation.Status, validation.Code, validation.Message)
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), s.config.QueryTimeout)
|
||||
defer cancel()
|
||||
snapshot, err := s.readRealtimeSync(ctx, query)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
writeAPIError(w, http.StatusNotFound, "workbench_sync_scope_not_found", "workbench sync scope was not found")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
writeQueryError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"ok": true,
|
||||
"status": "succeeded",
|
||||
"contractVersion": realtimeSyncContractVersion,
|
||||
"facts": snapshot.Facts,
|
||||
"events": snapshot.Events,
|
||||
"cutoffOutboxSeq": snapshot.CutoffOutboxSeq,
|
||||
"cursorOutboxSeq": snapshot.CursorOutboxSeq,
|
||||
"hasMore": snapshot.HasMore,
|
||||
"persistence": s.persistenceSummary(),
|
||||
"servedBy": serviceID,
|
||||
"valuesRedacted": true,
|
||||
"secretMaterialStored": false,
|
||||
})
|
||||
}
|
||||
|
||||
type realtimeSyncValidationError struct {
|
||||
Status int
|
||||
Code string
|
||||
Message string
|
||||
}
|
||||
|
||||
func decodeRealtimeSyncQuery(w http.ResponseWriter, r *http.Request) (realtimeSyncQuery, bool) {
|
||||
defer r.Body.Close()
|
||||
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20))
|
||||
decoder.DisallowUnknownFields()
|
||||
var query realtimeSyncQuery
|
||||
if err := decoder.Decode(&query); err != nil {
|
||||
writeAPIError(w, http.StatusBadRequest, "invalid_json", "request body must be a closed workbench sync JSON object")
|
||||
return realtimeSyncQuery{}, false
|
||||
}
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
writeAPIError(w, http.StatusBadRequest, "invalid_json", "request body must contain exactly one JSON object")
|
||||
return realtimeSyncQuery{}, false
|
||||
}
|
||||
return query, true
|
||||
}
|
||||
|
||||
func normalizeRealtimeSyncQuery(query realtimeSyncQuery) (realtimeSyncQuery, *realtimeSyncValidationError) {
|
||||
query.SessionID = strings.TrimSpace(query.SessionID)
|
||||
query.TraceID = strings.TrimSpace(query.TraceID)
|
||||
query.Actor.ID = strings.TrimSpace(query.Actor.ID)
|
||||
query.Actor.Role = strings.ToLower(strings.TrimSpace(query.Actor.Role))
|
||||
if query.SessionID == "" && query.TraceID == "" {
|
||||
return query, &realtimeSyncValidationError{Status: http.StatusBadRequest, Code: "workbench_sync_scope_required", Message: "sessionId or traceId is required"}
|
||||
}
|
||||
if len(query.SessionID) > realtimeSyncMaxIdentityLen || len(query.TraceID) > realtimeSyncMaxIdentityLen || len(query.Actor.ID) > realtimeSyncMaxIdentityLen {
|
||||
return query, &realtimeSyncValidationError{Status: http.StatusBadRequest, Code: "workbench_sync_identity_invalid", Message: "workbench sync identifiers are too long"}
|
||||
}
|
||||
if query.AfterOutboxSeq < 0 {
|
||||
return query, &realtimeSyncValidationError{Status: http.StatusBadRequest, Code: "workbench_sync_cursor_invalid", Message: "afterOutboxSeq must be non-negative"}
|
||||
}
|
||||
if query.Limit < 0 {
|
||||
return query, &realtimeSyncValidationError{Status: http.StatusBadRequest, Code: "workbench_sync_limit_invalid", Message: "limit must be non-negative"}
|
||||
}
|
||||
if query.SnapshotOnly && query.DeltaOnly {
|
||||
return query, &realtimeSyncValidationError{Status: http.StatusBadRequest, Code: "workbench_sync_mode_invalid", Message: "snapshotOnly and deltaOnly are mutually exclusive"}
|
||||
}
|
||||
query.Limit = boundedLimit(query.Limit, realtimeSyncDefaultLimit, realtimeSyncMaxLimit)
|
||||
if query.Actor.Role != "admin" && query.Actor.Role != "user" {
|
||||
return query, &realtimeSyncValidationError{Status: http.StatusBadRequest, Code: "workbench_sync_actor_invalid", Message: "actor.role must be user or admin"}
|
||||
}
|
||||
if query.Actor.Role != "admin" && query.Actor.ID == "" {
|
||||
return query, &realtimeSyncValidationError{Status: http.StatusForbidden, Code: "workbench_sync_actor_required", Message: "ordinary users require an actor id"}
|
||||
}
|
||||
return query, nil
|
||||
}
|
||||
|
||||
func realtimeSyncTxOptions() *sql.TxOptions {
|
||||
return &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true}
|
||||
}
|
||||
|
||||
func (s *Server) readRealtimeSync(ctx context.Context, query realtimeSyncQuery) (realtimeSyncSnapshot, error) {
|
||||
tx, err := s.db.BeginTx(ctx, realtimeSyncTxOptions())
|
||||
if err != nil {
|
||||
return realtimeSyncSnapshot{}, wrapQueryStage("workbench_runtime.db.sync.begin", err)
|
||||
}
|
||||
committed := false
|
||||
defer func() {
|
||||
if !committed {
|
||||
_ = tx.Rollback()
|
||||
}
|
||||
}()
|
||||
snapshot, err := readRealtimeSyncSnapshot(ctx, sqlRealtimeSyncQueryer{tx: tx}, query)
|
||||
if err != nil {
|
||||
return realtimeSyncSnapshot{}, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return realtimeSyncSnapshot{}, wrapQueryStage("workbench_runtime.db.sync.commit", err)
|
||||
}
|
||||
committed = true
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
type realtimeSyncRow interface {
|
||||
Scan(...any) error
|
||||
}
|
||||
|
||||
type realtimeSyncRows interface {
|
||||
Close() error
|
||||
Err() error
|
||||
Next() bool
|
||||
Scan(...any) error
|
||||
}
|
||||
|
||||
type realtimeSyncQueryer interface {
|
||||
QueryContext(context.Context, string, ...any) (realtimeSyncRows, error)
|
||||
QueryRowContext(context.Context, string, ...any) realtimeSyncRow
|
||||
}
|
||||
|
||||
type sqlRealtimeSyncQueryer struct {
|
||||
tx *sql.Tx
|
||||
}
|
||||
|
||||
func (queryer sqlRealtimeSyncQueryer) QueryContext(ctx context.Context, query string, args ...any) (realtimeSyncRows, error) {
|
||||
return queryer.tx.QueryContext(ctx, query, args...)
|
||||
}
|
||||
|
||||
func (queryer sqlRealtimeSyncQueryer) QueryRowContext(ctx context.Context, query string, args ...any) realtimeSyncRow {
|
||||
return queryer.tx.QueryRowContext(ctx, query, args...)
|
||||
}
|
||||
|
||||
func readRealtimeSyncSnapshot(ctx context.Context, tx realtimeSyncQueryer, query realtimeSyncQuery) (realtimeSyncSnapshot, error) {
|
||||
sessionID, err := resolveRealtimeSyncSession(ctx, tx, query)
|
||||
if err != nil {
|
||||
return realtimeSyncSnapshot{}, err
|
||||
}
|
||||
cutoff, err := readRealtimeSyncCutoff(ctx, tx, sessionID, query.TraceID)
|
||||
if err != nil {
|
||||
return realtimeSyncSnapshot{}, err
|
||||
}
|
||||
events := make([]realtimeSyncEvent, 0)
|
||||
hasMore := false
|
||||
if !query.SnapshotOnly {
|
||||
allEvents, readErr := readRealtimeSyncEvents(ctx, tx, sessionID, query.TraceID, query.AfterOutboxSeq, cutoff, query.Limit+1)
|
||||
if readErr != nil {
|
||||
return realtimeSyncSnapshot{}, readErr
|
||||
}
|
||||
hasMore = len(allEvents) > query.Limit
|
||||
events = allEvents
|
||||
if hasMore {
|
||||
events = allEvents[:query.Limit]
|
||||
}
|
||||
}
|
||||
facts := make(map[string][]any)
|
||||
if !query.DeltaOnly {
|
||||
specs := realtimeSyncFactSpecs
|
||||
if query.SnapshotOnly {
|
||||
specs = realtimeSyncSnapshotFactSpecs
|
||||
}
|
||||
facts, err = readRealtimeSyncFacts(ctx, tx, sessionID, query.TraceID, specs)
|
||||
if err != nil {
|
||||
return realtimeSyncSnapshot{}, err
|
||||
}
|
||||
}
|
||||
cursor := query.AfterOutboxSeq
|
||||
if query.SnapshotOnly || cursor > cutoff {
|
||||
cursor = cutoff
|
||||
}
|
||||
if len(events) > 0 {
|
||||
cursor = events[len(events)-1].OutboxSeq
|
||||
}
|
||||
return realtimeSyncSnapshot{Facts: facts, Events: events, CutoffOutboxSeq: cutoff, CursorOutboxSeq: cursor, HasMore: hasMore}, nil
|
||||
}
|
||||
|
||||
func resolveRealtimeSyncSession(ctx context.Context, tx realtimeSyncQueryer, query realtimeSyncQuery) (string, error) {
|
||||
var sessionID string
|
||||
err := tx.QueryRowContext(ctx, realtimeSyncResolveScopeSQL, query.SessionID, query.TraceID, query.Actor.Role == "admin", query.Actor.ID).Scan(&sessionID)
|
||||
if err != nil {
|
||||
return "", wrapQueryStage("workbench_runtime.db.sync.scope", err)
|
||||
}
|
||||
return sessionID, nil
|
||||
}
|
||||
|
||||
func readRealtimeSyncCutoff(ctx context.Context, tx realtimeSyncQueryer, sessionID string, traceID string) (int64, error) {
|
||||
var cutoff int64
|
||||
if err := tx.QueryRowContext(ctx, realtimeSyncCutoffSQL, sessionID, traceID).Scan(&cutoff); err != nil {
|
||||
return 0, wrapQueryStage("workbench_runtime.db.sync.cutoff", err)
|
||||
}
|
||||
return cutoff, nil
|
||||
}
|
||||
|
||||
func readRealtimeSyncEvents(ctx context.Context, tx realtimeSyncQueryer, sessionID string, traceID string, after int64, cutoff int64, limit int) ([]realtimeSyncEvent, error) {
|
||||
rows, err := tx.QueryContext(ctx, realtimeSyncEventsSQL, sessionID, traceID, after, cutoff, limit)
|
||||
if err != nil {
|
||||
return nil, wrapQueryStage("workbench_runtime.db.sync.events", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
events := make([]realtimeSyncEvent, 0)
|
||||
for rows.Next() {
|
||||
event, err := scanRealtimeSyncEvent(rows)
|
||||
if err != nil {
|
||||
return nil, wrapQueryStage("workbench_runtime.db.sync.events", err)
|
||||
}
|
||||
events = append(events, event)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, wrapQueryStage("workbench_runtime.db.sync.events", err)
|
||||
}
|
||||
return events, nil
|
||||
}
|
||||
|
||||
type realtimeSyncScanner interface {
|
||||
Scan(...any) error
|
||||
}
|
||||
|
||||
func scanRealtimeSyncEvent(scanner realtimeSyncScanner) (realtimeSyncEvent, error) {
|
||||
var event realtimeSyncEvent
|
||||
var outboxEventID, entityFamily, entityID, traceID, commitType, payloadJSON, createdAt sql.NullString
|
||||
var eventSeq, aggregateSeq, projectionRevision, projectedSeq, sourceSeq sql.NullInt64
|
||||
var aggregateID, sessionID, turnID, messageID, sourceEventID sql.NullString
|
||||
var terminal, sealed sql.NullBool
|
||||
if err := scanner.Scan(
|
||||
&event.OutboxSeq,
|
||||
&outboxEventID,
|
||||
&entityFamily,
|
||||
&entityID,
|
||||
&eventSeq,
|
||||
&aggregateID,
|
||||
&aggregateSeq,
|
||||
&projectionRevision,
|
||||
&traceID,
|
||||
&sessionID,
|
||||
&turnID,
|
||||
&messageID,
|
||||
&projectedSeq,
|
||||
&sourceSeq,
|
||||
&sourceEventID,
|
||||
&commitType,
|
||||
&terminal,
|
||||
&sealed,
|
||||
&payloadJSON,
|
||||
&createdAt,
|
||||
); err != nil {
|
||||
return realtimeSyncEvent{}, err
|
||||
}
|
||||
payload := any(map[string]any{})
|
||||
if payloadJSON.Valid && strings.TrimSpace(payloadJSON.String) != "" {
|
||||
if err := json.Unmarshal([]byte(payloadJSON.String), &payload); err != nil {
|
||||
return realtimeSyncEvent{}, fmt.Errorf("decode projection outbox payload: %w", err)
|
||||
}
|
||||
}
|
||||
event.OutboxEventID = nullableStringValue(outboxEventID)
|
||||
event.EntityFamily = nullableStringValue(entityFamily)
|
||||
event.EntityID = nullableStringValue(entityID)
|
||||
event.EventSeq = realtimeSyncNullableInt(eventSeq)
|
||||
event.AggregateID = nullableString(aggregateID)
|
||||
event.AggregateSeq = realtimeSyncInt(aggregateSeq)
|
||||
event.ProjectionRevision = realtimeSyncInt(projectionRevision)
|
||||
event.TraceID = nullableStringValue(traceID)
|
||||
event.SessionID = nullableString(sessionID)
|
||||
event.TurnID = nullableString(turnID)
|
||||
event.MessageID = nullableString(messageID)
|
||||
event.ProjectedSeq = realtimeSyncInt(projectedSeq)
|
||||
event.SourceSeq = realtimeSyncInt(sourceSeq)
|
||||
event.SourceEventID = nullableString(sourceEventID)
|
||||
event.CommitType = nullableStringValue(commitType)
|
||||
event.Terminal = nullableBool(terminal)
|
||||
event.Sealed = nullableBool(sealed)
|
||||
event.Payload = payload
|
||||
event.CreatedAt = nullableTimeText(createdAt)
|
||||
event.ValuesRedacted = true
|
||||
return event, nil
|
||||
}
|
||||
|
||||
func readRealtimeSyncFacts(ctx context.Context, tx realtimeSyncQueryer, sessionID string, traceID string, specs []realtimeSyncFactSpec) (map[string][]any, error) {
|
||||
facts := make(map[string][]any, len(specs))
|
||||
for _, spec := range specs {
|
||||
args := []any{sessionID}
|
||||
where := "session_id = $1"
|
||||
if spec.Family != "sessions" {
|
||||
args = append(args, traceID)
|
||||
where += " AND ($2 = '' OR trace_id = $2)"
|
||||
}
|
||||
sqlText := fmt.Sprintf("SELECT %s FROM %s WHERE %s ORDER BY %s", spec.JSONColumn, spec.Table, where, spec.OrderBy)
|
||||
rows, err := tx.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, wrapQueryStage("workbench_runtime.db.sync.facts."+spec.Family, err)
|
||||
}
|
||||
items, err := scanRealtimeSyncFacts(rows)
|
||||
if err != nil {
|
||||
return nil, wrapQueryStage("workbench_runtime.db.sync.facts."+spec.Family, err)
|
||||
}
|
||||
facts[spec.Family] = items
|
||||
}
|
||||
return facts, nil
|
||||
}
|
||||
|
||||
func scanRealtimeSyncFacts(rows realtimeSyncRows) ([]any, error) {
|
||||
defer rows.Close()
|
||||
items := make([]any, 0)
|
||||
for rows.Next() {
|
||||
var raw sql.NullString
|
||||
if err := rows.Scan(&raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !raw.Valid || strings.TrimSpace(raw.String) == "" {
|
||||
continue
|
||||
}
|
||||
var item any
|
||||
if err := json.Unmarshal([]byte(raw.String), &item); err != nil {
|
||||
return nil, fmt.Errorf("decode workbench fact: %w", err)
|
||||
}
|
||||
if item != nil {
|
||||
items = append(items, item)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func realtimeSyncNullableInt(value sql.NullInt64) any {
|
||||
if !value.Valid {
|
||||
return nil
|
||||
}
|
||||
return value.Int64
|
||||
}
|
||||
|
||||
func realtimeSyncInt(value sql.NullInt64) int64 {
|
||||
if !value.Valid {
|
||||
return 0
|
||||
}
|
||||
return value.Int64
|
||||
}
|
||||
@@ -0,0 +1,409 @@
|
||||
package workbenchruntime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRealtimeSyncTxOptionsAreReadOnlyRepeatableRead(t *testing.T) {
|
||||
options := realtimeSyncTxOptions()
|
||||
if options.Isolation != sql.LevelRepeatableRead {
|
||||
t.Fatalf("isolation=%v, want repeatable read", options.Isolation)
|
||||
}
|
||||
if !options.ReadOnly {
|
||||
t.Fatal("realtime sync transaction must be read only")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeRealtimeSyncQueryRequiresScopedActorAndBoundsLimit(t *testing.T) {
|
||||
query, validation := normalizeRealtimeSyncQuery(realtimeSyncQuery{
|
||||
SessionID: " ses_sync ",
|
||||
Limit: 900,
|
||||
Actor: sessionActor{ID: " usr_owner ", Role: " USER "},
|
||||
})
|
||||
if validation != nil {
|
||||
t.Fatalf("unexpected validation: %#v", validation)
|
||||
}
|
||||
if query.SessionID != "ses_sync" || query.Actor.ID != "usr_owner" || query.Actor.Role != "user" {
|
||||
t.Fatalf("query was not normalized: %#v", query)
|
||||
}
|
||||
if query.Limit != realtimeSyncMaxLimit {
|
||||
t.Fatalf("limit=%d, want %d", query.Limit, realtimeSyncMaxLimit)
|
||||
}
|
||||
|
||||
_, validation = normalizeRealtimeSyncQuery(realtimeSyncQuery{TraceID: "trc_sync", Actor: sessionActor{Role: "user"}})
|
||||
if validation == nil || validation.Status != http.StatusForbidden || validation.Code != "workbench_sync_actor_required" {
|
||||
t.Fatalf("ordinary user without owner id validation=%#v", validation)
|
||||
}
|
||||
|
||||
admin, validation := normalizeRealtimeSyncQuery(realtimeSyncQuery{TraceID: "trc_sync", Actor: sessionActor{Role: "admin"}})
|
||||
if validation != nil || admin.Actor.ID != "" {
|
||||
t.Fatalf("admin should not require owner id: query=%#v validation=%#v", admin, validation)
|
||||
}
|
||||
|
||||
_, validation = normalizeRealtimeSyncQuery(realtimeSyncQuery{Actor: sessionActor{ID: "usr_owner", Role: "user"}})
|
||||
if validation == nil || validation.Code != "workbench_sync_scope_required" {
|
||||
t.Fatalf("missing scope validation=%#v", validation)
|
||||
}
|
||||
|
||||
_, validation = normalizeRealtimeSyncQuery(realtimeSyncQuery{SessionID: "ses_sync", AfterOutboxSeq: -1, Actor: sessionActor{ID: "usr_owner", Role: "user"}})
|
||||
if validation == nil || validation.Code != "workbench_sync_cursor_invalid" {
|
||||
t.Fatalf("negative cursor validation=%#v", validation)
|
||||
}
|
||||
|
||||
_, validation = normalizeRealtimeSyncQuery(realtimeSyncQuery{SessionID: "ses_sync", SnapshotOnly: true, DeltaOnly: true, Actor: sessionActor{ID: "usr_owner", Role: "user"}})
|
||||
if validation == nil || validation.Code != "workbench_sync_mode_invalid" {
|
||||
t.Fatalf("conflicting sync mode validation=%#v", validation)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRealtimeSyncRouteIsPostOnlyAndRejectsUnknownFields(t *testing.T) {
|
||||
server := &Server{config: Config{}, mux: http.NewServeMux()}
|
||||
server.routes()
|
||||
|
||||
methodRecorder := httptest.NewRecorder()
|
||||
server.mux.ServeHTTP(methodRecorder, httptest.NewRequest(http.MethodGet, "/v1/workbench-runtime/sync", nil))
|
||||
if methodRecorder.Code != http.StatusMethodNotAllowed {
|
||||
t.Fatalf("GET status=%d, want %d", methodRecorder.Code, http.StatusMethodNotAllowed)
|
||||
}
|
||||
if methodRecorder.Header().Get("Allow") != "POST" {
|
||||
t.Fatalf("Allow=%q, want POST", methodRecorder.Header().Get("Allow"))
|
||||
}
|
||||
|
||||
unknownRecorder := httptest.NewRecorder()
|
||||
unknownRequest := httptest.NewRequest(http.MethodPost, "/v1/workbench-runtime/sync", strings.NewReader(`{"sessionId":"ses_sync","actor":{"id":"usr_owner","role":"user"},"unknown":true}`))
|
||||
server.mux.ServeHTTP(unknownRecorder, unknownRequest)
|
||||
if unknownRecorder.Code != http.StatusBadRequest || !strings.Contains(unknownRecorder.Body.String(), "invalid_json") {
|
||||
t.Fatalf("unknown field response=%d %s", unknownRecorder.Code, unknownRecorder.Body.String())
|
||||
}
|
||||
|
||||
trailingRecorder := httptest.NewRecorder()
|
||||
trailingRequest := httptest.NewRequest(http.MethodPost, "/v1/workbench-runtime/sync", strings.NewReader(`{"sessionId":"ses_sync","actor":{"id":"usr_owner","role":"user"}} {}`))
|
||||
server.mux.ServeHTTP(trailingRecorder, trailingRequest)
|
||||
if trailingRecorder.Code != http.StatusBadRequest || !strings.Contains(trailingRecorder.Body.String(), "invalid_json") {
|
||||
t.Fatalf("trailing JSON response=%d %s", trailingRecorder.Code, trailingRecorder.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadRealtimeSyncSnapshotUsesOwnerScopeAndSingleCutoff(t *testing.T) {
|
||||
queryer := newRealtimeSyncFakeQueryer("usr_owner")
|
||||
query := realtimeSyncQuery{
|
||||
SessionID: "ses_sync",
|
||||
TraceID: "trc_sync",
|
||||
AfterOutboxSeq: 0,
|
||||
Limit: 2,
|
||||
Actor: sessionActor{ID: "usr_owner", Role: "user"},
|
||||
}
|
||||
snapshot, err := readRealtimeSyncSnapshot(context.Background(), queryer, query)
|
||||
if err != nil {
|
||||
t.Fatalf("read sync snapshot: %v", err)
|
||||
}
|
||||
if snapshot.CutoffOutboxSeq != 3 || snapshot.CursorOutboxSeq != 2 || !snapshot.HasMore {
|
||||
t.Fatalf("unexpected cursor boundary: %#v", snapshot)
|
||||
}
|
||||
if len(snapshot.Events) != 2 {
|
||||
t.Fatalf("events=%d, want 2", len(snapshot.Events))
|
||||
}
|
||||
first := snapshot.Events[0]
|
||||
if first.OutboxEventID != "obe_1" || first.EntityFamily != "traceEvents" || first.EntityID != "evt_1" {
|
||||
t.Fatalf("event identity fields were not returned: %#v", first)
|
||||
}
|
||||
if first.EventSeq != int64(101) || first.AggregateID != "trc_sync" || first.AggregateSeq != 1 || first.ProjectionRevision != 11 {
|
||||
t.Fatalf("event aggregate fields were not returned: %#v", first)
|
||||
}
|
||||
if len(snapshot.Facts) != len(realtimeSyncFactSpecs) {
|
||||
t.Fatalf("fact families=%d, want %d: %#v", len(snapshot.Facts), len(realtimeSyncFactSpecs), snapshot.Facts)
|
||||
}
|
||||
for _, spec := range realtimeSyncFactSpecs {
|
||||
if len(snapshot.Facts[spec.Family]) != 1 {
|
||||
t.Fatalf("facts[%s]=%#v, want one fact", spec.Family, snapshot.Facts[spec.Family])
|
||||
}
|
||||
}
|
||||
if len(queryer.calls) != 3+len(realtimeSyncFactSpecs) {
|
||||
t.Fatalf("query calls=%v", queryer.calls)
|
||||
}
|
||||
if queryer.calls[0].kind != "scope" || queryer.calls[1].kind != "cutoff" || queryer.calls[2].kind != "events" {
|
||||
t.Fatalf("snapshot boundary query order=%v", queryer.calls)
|
||||
}
|
||||
if got := queryer.calls[0].args; len(got) != 4 || got[2] != false || got[3] != "usr_owner" {
|
||||
t.Fatalf("owner scope args=%#v", got)
|
||||
}
|
||||
if got := queryer.calls[2].args; len(got) != 5 || got[3] != int64(3) || got[4] != 3 {
|
||||
t.Fatalf("events must use cutoff and limit+1: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadRealtimeSyncSnapshotRejectsOtherOwnerBeforeReadingFacts(t *testing.T) {
|
||||
if !strings.Contains(realtimeSyncResolveScopeSQL, "sessions.owner_user_id = $4") {
|
||||
t.Fatalf("ordinary user scope must be constrained by workbench_sessions.owner_user_id: %s", realtimeSyncResolveScopeSQL)
|
||||
}
|
||||
queryer := newRealtimeSyncFakeQueryer("usr_owner")
|
||||
_, err := readRealtimeSyncSnapshot(context.Background(), queryer, realtimeSyncQuery{
|
||||
SessionID: "ses_sync",
|
||||
Limit: 100,
|
||||
Actor: sessionActor{ID: "usr_other", Role: "user"},
|
||||
})
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
t.Fatalf("error=%v, want sql.ErrNoRows", err)
|
||||
}
|
||||
if len(queryer.calls) != 1 || queryer.calls[0].kind != "scope" {
|
||||
t.Fatalf("unauthorized scope must stop before outbox/facts reads: %v", queryer.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadRealtimeSyncSnapshotOnlySkipsHistoryAndReadsBoundedCurrentFacts(t *testing.T) {
|
||||
queryer := newRealtimeSyncFakeQueryer("usr_owner")
|
||||
snapshot, err := readRealtimeSyncSnapshot(context.Background(), queryer, realtimeSyncQuery{
|
||||
SessionID: "ses_sync",
|
||||
Limit: 100,
|
||||
SnapshotOnly: true,
|
||||
Actor: sessionActor{ID: "usr_owner", Role: "user"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("read snapshot-only sync: %v", err)
|
||||
}
|
||||
if len(snapshot.Events) != 0 || snapshot.HasMore || snapshot.CursorOutboxSeq != snapshot.CutoffOutboxSeq {
|
||||
t.Fatalf("snapshot-only boundary=%#v", snapshot)
|
||||
}
|
||||
if len(snapshot.Facts) != len(realtimeSyncSnapshotFactSpecs) {
|
||||
t.Fatalf("snapshot-only facts=%#v", snapshot.Facts)
|
||||
}
|
||||
for _, call := range queryer.calls {
|
||||
if call.kind == "events" || call.kind == "facts.parts" || call.kind == "facts.traceEvents" || call.kind == "facts.checkpoints" {
|
||||
t.Fatalf("snapshot-only performed unbounded read: %v", queryer.calls)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadRealtimeSyncDeltaOnlyReadsOutboxWithoutFacts(t *testing.T) {
|
||||
queryer := newRealtimeSyncFakeQueryer("usr_owner")
|
||||
snapshot, err := readRealtimeSyncSnapshot(context.Background(), queryer, realtimeSyncQuery{
|
||||
SessionID: "ses_sync",
|
||||
AfterOutboxSeq: 1,
|
||||
Limit: 100,
|
||||
DeltaOnly: true,
|
||||
Actor: sessionActor{ID: "usr_owner", Role: "user"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("read delta-only sync: %v", err)
|
||||
}
|
||||
if len(snapshot.Events) == 0 || len(snapshot.Facts) != 0 {
|
||||
t.Fatalf("delta-only result=%#v", snapshot)
|
||||
}
|
||||
for _, call := range queryer.calls {
|
||||
if strings.HasPrefix(call.kind, "facts.") {
|
||||
t.Fatalf("delta-only read facts: %v", queryer.calls)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadRealtimeSyncSnapshotClampsForeignCursorToScopedCutoff(t *testing.T) {
|
||||
queryer := newRealtimeSyncFakeQueryer("usr_owner")
|
||||
queryer.events = [][]any{}
|
||||
snapshot, err := readRealtimeSyncSnapshot(context.Background(), queryer, realtimeSyncQuery{
|
||||
SessionID: "ses_sync",
|
||||
AfterOutboxSeq: 999,
|
||||
Limit: 100,
|
||||
Actor: sessionActor{ID: "usr_owner", Role: "user"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("read sync snapshot: %v", err)
|
||||
}
|
||||
if snapshot.CutoffOutboxSeq != 3 || snapshot.CursorOutboxSeq != 3 || snapshot.HasMore || len(snapshot.Events) != 0 {
|
||||
t.Fatalf("foreign cursor was not clamped to scoped cutoff: %#v", snapshot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadRealtimeSyncSnapshotAllowsAdminWithoutOwnerFilter(t *testing.T) {
|
||||
queryer := newRealtimeSyncFakeQueryer("usr_owner")
|
||||
snapshot, err := readRealtimeSyncSnapshot(context.Background(), queryer, realtimeSyncQuery{
|
||||
TraceID: "trc_sync",
|
||||
Limit: 3,
|
||||
Actor: sessionActor{Role: "admin"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("admin sync: %v", err)
|
||||
}
|
||||
if len(queryer.calls) == 0 || queryer.calls[0].args[2] != true || queryer.calls[0].args[3] != "" {
|
||||
t.Fatalf("admin scope args=%#v", queryer.calls)
|
||||
}
|
||||
if snapshot.CursorOutboxSeq != snapshot.CutoffOutboxSeq || snapshot.HasMore {
|
||||
t.Fatalf("admin complete page boundary=%#v", snapshot)
|
||||
}
|
||||
}
|
||||
|
||||
type realtimeSyncFakeCall struct {
|
||||
kind string
|
||||
query string
|
||||
args []any
|
||||
}
|
||||
|
||||
type realtimeSyncFakeQueryer struct {
|
||||
ownerID string
|
||||
sessionID string
|
||||
cutoff int64
|
||||
events [][]any
|
||||
facts map[string][][]any
|
||||
calls []realtimeSyncFakeCall
|
||||
}
|
||||
|
||||
func newRealtimeSyncFakeQueryer(ownerID string) *realtimeSyncFakeQueryer {
|
||||
facts := map[string][][]any{}
|
||||
for _, spec := range realtimeSyncFactSpecs {
|
||||
facts[spec.Table] = [][]any{{fmt.Sprintf(`{"family":%q,"sessionId":"ses_sync"}`, spec.Family)}}
|
||||
}
|
||||
return &realtimeSyncFakeQueryer{
|
||||
ownerID: ownerID,
|
||||
sessionID: "ses_sync",
|
||||
cutoff: 3,
|
||||
events: [][]any{realtimeSyncTestEventRow(1), realtimeSyncTestEventRow(2), realtimeSyncTestEventRow(3)},
|
||||
facts: facts,
|
||||
}
|
||||
}
|
||||
|
||||
func (queryer *realtimeSyncFakeQueryer) QueryRowContext(_ context.Context, query string, args ...any) realtimeSyncRow {
|
||||
if query == realtimeSyncResolveScopeSQL {
|
||||
queryer.record("scope", query, args)
|
||||
isAdmin, _ := args[2].(bool)
|
||||
actorID, _ := args[3].(string)
|
||||
if !isAdmin && actorID != queryer.ownerID {
|
||||
return realtimeSyncFakeRow{err: sql.ErrNoRows}
|
||||
}
|
||||
return realtimeSyncFakeRow{values: []any{queryer.sessionID}}
|
||||
}
|
||||
if query == realtimeSyncCutoffSQL {
|
||||
queryer.record("cutoff", query, args)
|
||||
return realtimeSyncFakeRow{values: []any{queryer.cutoff}}
|
||||
}
|
||||
return realtimeSyncFakeRow{err: fmt.Errorf("unexpected QueryRowContext: %s", query)}
|
||||
}
|
||||
|
||||
func (queryer *realtimeSyncFakeQueryer) QueryContext(_ context.Context, query string, args ...any) (realtimeSyncRows, error) {
|
||||
if query == realtimeSyncEventsSQL {
|
||||
queryer.record("events", query, args)
|
||||
return &realtimeSyncFakeRows{values: queryer.events}, nil
|
||||
}
|
||||
for _, spec := range realtimeSyncFactSpecs {
|
||||
if strings.Contains(query, "FROM "+spec.Table+" ") {
|
||||
queryer.record("facts."+spec.Family, query, args)
|
||||
return &realtimeSyncFakeRows{values: queryer.facts[spec.Table]}, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("unexpected QueryContext: %s", query)
|
||||
}
|
||||
|
||||
func (queryer *realtimeSyncFakeQueryer) record(kind string, query string, args []any) {
|
||||
cloned := append([]any(nil), args...)
|
||||
queryer.calls = append(queryer.calls, realtimeSyncFakeCall{kind: kind, query: query, args: cloned})
|
||||
}
|
||||
|
||||
type realtimeSyncFakeRow struct {
|
||||
values []any
|
||||
err error
|
||||
}
|
||||
|
||||
func (row realtimeSyncFakeRow) Scan(destinations ...any) error {
|
||||
if row.err != nil {
|
||||
return row.err
|
||||
}
|
||||
return assignRealtimeSyncFakeValues(destinations, row.values)
|
||||
}
|
||||
|
||||
type realtimeSyncFakeRows struct {
|
||||
values [][]any
|
||||
index int
|
||||
err error
|
||||
}
|
||||
|
||||
func (rows *realtimeSyncFakeRows) Close() error { return nil }
|
||||
|
||||
func (rows *realtimeSyncFakeRows) Err() error { return rows.err }
|
||||
|
||||
func (rows *realtimeSyncFakeRows) Next() bool { return rows.index < len(rows.values) }
|
||||
|
||||
func (rows *realtimeSyncFakeRows) Scan(destinations ...any) error {
|
||||
if rows.index >= len(rows.values) {
|
||||
return errors.New("realtime sync fake rows exhausted")
|
||||
}
|
||||
values := rows.values[rows.index]
|
||||
rows.index++
|
||||
return assignRealtimeSyncFakeValues(destinations, values)
|
||||
}
|
||||
|
||||
func assignRealtimeSyncFakeValues(destinations []any, values []any) error {
|
||||
if len(destinations) != len(values) {
|
||||
return fmt.Errorf("scan destinations=%d values=%d", len(destinations), len(values))
|
||||
}
|
||||
for index, destination := range destinations {
|
||||
value := values[index]
|
||||
switch target := destination.(type) {
|
||||
case *string:
|
||||
if value == nil {
|
||||
return fmt.Errorf("cannot scan nil into string")
|
||||
}
|
||||
*target = fmt.Sprint(value)
|
||||
case *int64:
|
||||
integer, ok := value.(int64)
|
||||
if !ok {
|
||||
return fmt.Errorf("cannot scan %T into int64", value)
|
||||
}
|
||||
*target = integer
|
||||
case *sql.NullString:
|
||||
if value == nil {
|
||||
*target = sql.NullString{}
|
||||
} else {
|
||||
*target = sql.NullString{String: fmt.Sprint(value), Valid: true}
|
||||
}
|
||||
case *sql.NullInt64:
|
||||
if value == nil {
|
||||
*target = sql.NullInt64{}
|
||||
} else if integer, ok := value.(int64); ok {
|
||||
*target = sql.NullInt64{Int64: integer, Valid: true}
|
||||
} else {
|
||||
return fmt.Errorf("cannot scan %T into NullInt64", value)
|
||||
}
|
||||
case *sql.NullBool:
|
||||
if value == nil {
|
||||
*target = sql.NullBool{}
|
||||
} else if boolean, ok := value.(bool); ok {
|
||||
*target = sql.NullBool{Bool: boolean, Valid: true}
|
||||
} else {
|
||||
return fmt.Errorf("cannot scan %T into NullBool", value)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unsupported scan destination %T", destination)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func realtimeSyncTestEventRow(sequence int64) []any {
|
||||
return []any{
|
||||
sequence,
|
||||
fmt.Sprintf("obe_%d", sequence),
|
||||
"traceEvents",
|
||||
fmt.Sprintf("evt_%d", sequence),
|
||||
int64(100) + sequence,
|
||||
"trc_sync",
|
||||
sequence,
|
||||
int64(10) + sequence,
|
||||
"trc_sync",
|
||||
"ses_sync",
|
||||
"turn_sync",
|
||||
nil,
|
||||
sequence,
|
||||
sequence,
|
||||
fmt.Sprintf("source_%d", sequence),
|
||||
"event",
|
||||
false,
|
||||
false,
|
||||
fmt.Sprintf(`{"sequence":%d}`, sequence),
|
||||
"2026-07-10T12:00:00Z",
|
||||
}
|
||||
}
|
||||
@@ -258,6 +258,7 @@ func (s *Server) routes() {
|
||||
s.mux.HandleFunc("/v1/workbench-runtime/trace-events", s.handleTraceEvents)
|
||||
s.mux.HandleFunc("/v1/workbench-runtime/sessions", s.handleSessions)
|
||||
s.mux.HandleFunc("/v1/workbench-runtime/projection-outbox", s.handleProjectionOutbox)
|
||||
s.mux.HandleFunc("/v1/workbench-runtime/sync", s.handleRealtimeSync)
|
||||
}
|
||||
|
||||
func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
+77
-5818
File diff suppressed because it is too large
Load Diff
@@ -1,10 +1,15 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import vm from "node:vm";
|
||||
|
||||
import { configString } from "./src/gitops-render/core.mjs";
|
||||
import { opencodeEgressProxyUrlForProfile } from "./src/gitops-render/runtime-manifests.mjs";
|
||||
|
||||
import { CLOUD_CORE_MIGRATIONS, CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID } from "../internal/db/schema.ts";
|
||||
|
||||
function taskByName(pipeline, name) {
|
||||
const task = pipeline.spec.tasks.find((item) => item.name === name);
|
||||
@@ -98,36 +103,11 @@ async function runtimeNoopDecision(gitopsPromoteScript, { oldRuntime, newRuntime
|
||||
}
|
||||
}
|
||||
|
||||
function extractNamedFunction(source, name) {
|
||||
const start = source.indexOf(`function ${name}`);
|
||||
assert.notEqual(start, -1, `missing function ${name}`);
|
||||
const bodyStart = source.indexOf("{", start);
|
||||
assert.notEqual(bodyStart, -1, `missing function body for ${name}`);
|
||||
let depth = 0;
|
||||
for (let index = bodyStart; index < source.length; index += 1) {
|
||||
const char = source[index];
|
||||
if (char === "{") depth += 1;
|
||||
else if (char === "}") {
|
||||
depth -= 1;
|
||||
if (depth === 0) return source.slice(start, index + 1);
|
||||
}
|
||||
}
|
||||
assert.fail(`unterminated function ${name}`);
|
||||
}
|
||||
|
||||
test("opencode APK proxy prefers explicit gitMirror proxyUrl", async () => {
|
||||
const source = await readFile("scripts/gitops-render.mjs", "utf8");
|
||||
const context = {};
|
||||
vm.runInNewContext(`${extractNamedFunction(source, "configString")}
|
||||
${extractNamedFunction(source, "opencodeEgressProxyUrlForProfile")}
|
||||
globalThis.result = {
|
||||
hostRoute: opencodeEgressProxyUrlForProfile({ lanes: { v03: { gitMirror: { egressProxy: { required: true, proxyUrl: "http://10.42.0.1:10808", namespace: "platform-infra", serviceName: "jd01-host-proxy", port: 10808 } } } } }, "v03"),
|
||||
serviceFallback: opencodeEgressProxyUrlForProfile({ lanes: { v03: { gitMirror: { egressProxy: { required: true, namespace: "platform-infra", serviceName: "sub2api-egress-proxy", port: 10808 } } } } }, "v03"),
|
||||
disabled: opencodeEgressProxyUrlForProfile({ lanes: { v03: { gitMirror: { egressProxy: { required: false, proxyUrl: "http://10.42.0.1:10808" } } } } }, "v03")
|
||||
};`, context, { filename: "gitops-render-opencode-proxy.test.mjs" });
|
||||
assert.equal(context.result.hostRoute, "http://10.42.0.1:10808");
|
||||
assert.equal(context.result.serviceFallback, "http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808");
|
||||
assert.equal(context.result.disabled, "");
|
||||
assert.equal(configString(" value "), "value");
|
||||
assert.equal(opencodeEgressProxyUrlForProfile({ lanes: { v03: { gitMirror: { egressProxy: { required: true, proxyUrl: "http://10.42.0.1:10808", namespace: "platform-infra", serviceName: "jd01-host-proxy", port: 10808 } } } } }, "v03"), "http://10.42.0.1:10808");
|
||||
assert.equal(opencodeEgressProxyUrlForProfile({ lanes: { v03: { gitMirror: { egressProxy: { required: true, namespace: "platform-infra", serviceName: "sub2api-egress-proxy", port: 10808 } } } } }, "v03"), "http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808");
|
||||
assert.equal(opencodeEgressProxyUrlForProfile({ lanes: { v03: { gitMirror: { egressProxy: { required: false, proxyUrl: "http://10.42.0.1:10808" } } } } }, "v03"), "");
|
||||
});
|
||||
|
||||
test("v02 render follows TypeScript runtime checks and does not self-patch bootstrap RBAC", async () => {
|
||||
@@ -499,12 +479,23 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots
|
||||
assert.equal(frpcDeployment.spec.template.metadata.labels["hwlab.pikastech.local/source-commit"], undefined);
|
||||
|
||||
const postgres = JSON.parse(await readFile(path.join(outDir, "runtime-v02", "postgres.yaml"), "utf8"));
|
||||
const postgresMigrationConfig = postgres.items.find((item) => item.kind === "ConfigMap" && item.metadata?.name === "hwlab-v02-postgres-init");
|
||||
const postgresStatefulSet = postgres.items.find((item) => item.kind === "StatefulSet");
|
||||
const postgresMigrationLabel = postgresStatefulSet.spec.template.metadata.labels["hwlab.pikastech.local/migration-sha256"];
|
||||
const postgresMigrationAnnotation = postgresStatefulSet.spec.template.metadata.annotations["hwlab.pikastech.local/migration-sha256"];
|
||||
const migrationSources = await Promise.all(CLOUD_CORE_MIGRATIONS.map(async (migration) => ({
|
||||
...migration,
|
||||
sql: await readFile(path.join(process.cwd(), migration.path), "utf8")
|
||||
})));
|
||||
const migrationData = Object.fromEntries(migrationSources.map((migration) => [path.basename(migration.path), migration.sql]));
|
||||
const migrationSha256 = createHash("sha256").update(migrationSources.map((migration) => migration.sql).join("\n")).digest("hex");
|
||||
assert.deepEqual(postgresMigrationConfig.data, migrationData);
|
||||
assert.deepEqual(Object.keys(postgresMigrationConfig.data), CLOUD_CORE_MIGRATIONS.map((migration) => path.basename(migration.path)));
|
||||
assert.match(postgresMigrationConfig.data["0008_workbench_kafka_realtime.sql"], new RegExp(`'${CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID}'`, "u"));
|
||||
assert.ok(postgresMigrationLabel);
|
||||
assert.equal(postgresMigrationLabel.length <= 63, true);
|
||||
assert.match(postgresMigrationAnnotation, /^[a-f0-9]{64}$/u);
|
||||
assert.equal(postgresMigrationAnnotation, migrationSha256);
|
||||
assert.equal(postgresMigrationAnnotation.startsWith(postgresMigrationLabel), true);
|
||||
assert.equal(postgresStatefulSet.spec.template.metadata.labels["hwlab.pikastech.local/source-commit"], undefined);
|
||||
|
||||
|
||||
@@ -4,11 +4,12 @@ import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
CLOUD_CORE_MIGRATION_ID,
|
||||
CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION,
|
||||
CLOUD_CORE_MIGRATIONS,
|
||||
CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID as CLOUD_RUNTIME_DURABLE_MIGRATION_ID,
|
||||
CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION as CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION,
|
||||
CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE,
|
||||
CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE_COLUMNS,
|
||||
CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS
|
||||
CLOUD_TRANSACTIONAL_REALTIME_REQUIRED_TABLE_COLUMNS as CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS
|
||||
} from "../../internal/db/schema.ts";
|
||||
import {
|
||||
PostgresCloudRuntimeStore,
|
||||
@@ -25,9 +26,10 @@ import { ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs";
|
||||
import { ensureNotRepoReportsPath, tempReportPath } from "./report-paths.mjs";
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const migrationPath = "internal/db/migrations/0001_cloud_core_skeleton.sql";
|
||||
const migrationPaths = CLOUD_CORE_MIGRATIONS.map((migration) => migration.path);
|
||||
const latestMigrationPath = migrationPaths.at(-1);
|
||||
const defaultReportPath = tempReportPath("dev-runtime-migration-report.json");
|
||||
const issue = "pikasTech/HWLAB#164";
|
||||
const issue = "pikasTech/HWLAB#2464";
|
||||
|
||||
export async function runDevRuntimeMigrationCli(argv = process.argv.slice(2), options = {}) {
|
||||
const args = parseArgs(argv);
|
||||
@@ -55,7 +57,11 @@ export async function buildDevRuntimeMigrationReport(args = {}, options = {}) {
|
||||
const env = options.env ?? process.env;
|
||||
const now = options.now ?? (() => new Date().toISOString());
|
||||
const root = options.repoRoot ?? repoRoot;
|
||||
const sql = await readFile(path.join(root, migrationPath), "utf8");
|
||||
const migrationSources = await Promise.all(migrationPaths.map(async (migrationPath) => ({
|
||||
path: migrationPath,
|
||||
sql: await readFile(path.join(root, migrationPath), "utf8")
|
||||
})));
|
||||
const sql = migrationSources.map((migration) => migration.sql).join("\n");
|
||||
const sourceCheck = validateMigrationSource(sql);
|
||||
const db = summarizeDbEnv(env);
|
||||
const report = {
|
||||
@@ -71,8 +77,9 @@ export async function buildDevRuntimeMigrationReport(args = {}, options = {}) {
|
||||
dbEndpointAuthority: DEV_DB_ENV_CONTRACT.endpointAuthority.source
|
||||
},
|
||||
migration: {
|
||||
id: CLOUD_CORE_MIGRATION_ID,
|
||||
path: migrationPath,
|
||||
id: CLOUD_RUNTIME_DURABLE_MIGRATION_ID,
|
||||
path: latestMigrationPath,
|
||||
paths: migrationPaths,
|
||||
sha256: sha256(sql),
|
||||
schemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION,
|
||||
ledgerTable: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE,
|
||||
@@ -195,7 +202,9 @@ export function validateMigrationSource(sql) {
|
||||
continue;
|
||||
}
|
||||
for (const column of columns) {
|
||||
if (!new RegExp(`\\b${column}\\b`, "u").test(tableMatch[1])) {
|
||||
const created = new RegExp(`\\b${column}\\b`, "u").test(tableMatch[1]);
|
||||
const added = new RegExp(`ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS ${column}\\b`, "u").test(sql);
|
||||
if (!created && !added) {
|
||||
missing.push(`column:${table}.${column}`);
|
||||
}
|
||||
}
|
||||
@@ -214,8 +223,8 @@ export function validateMigrationSource(sql) {
|
||||
}
|
||||
}
|
||||
|
||||
if (!new RegExp(`'${CLOUD_CORE_MIGRATION_ID}'`, "u").test(sql)) {
|
||||
missing.push(`ledger-id:${CLOUD_CORE_MIGRATION_ID}`);
|
||||
if (!new RegExp(`'${CLOUD_RUNTIME_DURABLE_MIGRATION_ID}'`, "u").test(sql)) {
|
||||
missing.push(`ledger-id:${CLOUD_RUNTIME_DURABLE_MIGRATION_ID}`);
|
||||
}
|
||||
if (!new RegExp(`'${CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION}'`, "u").test(sql)) {
|
||||
missing.push(`schema-version:${CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION}`);
|
||||
@@ -227,8 +236,9 @@ export function validateMigrationSource(sql) {
|
||||
return {
|
||||
ready: missing.length === 0,
|
||||
checked: true,
|
||||
migrationPath,
|
||||
requiredMigrationId: CLOUD_CORE_MIGRATION_ID,
|
||||
migrationPath: latestMigrationPath,
|
||||
migrationPaths,
|
||||
requiredMigrationId: CLOUD_RUNTIME_DURABLE_MIGRATION_ID,
|
||||
requiredSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION,
|
||||
requiredTables: Object.keys(CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS),
|
||||
requiredLedgerTable: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE,
|
||||
@@ -257,6 +267,10 @@ async function verifyRuntimeReadiness(report, env, options) {
|
||||
let runtime;
|
||||
try {
|
||||
runtime = await runtimeStore.readiness();
|
||||
if (runtime.gates?.schema?.ready === true && runtime.gates?.migration?.ready === true) {
|
||||
const transactionalReadiness = await runtimeStore.workbenchTransactionalRealtimeReadiness();
|
||||
runtime = mergeTransactionalRealtimeReadiness(runtime, transactionalReadiness);
|
||||
}
|
||||
} finally {
|
||||
if (!options.queryClient && typeof runtimeStore.pool?.end === "function") {
|
||||
await runtimeStore.pool.end();
|
||||
@@ -280,6 +294,56 @@ async function verifyRuntimeReadiness(report, env, options) {
|
||||
}
|
||||
}
|
||||
|
||||
function mergeTransactionalRealtimeReadiness(runtime, readiness = {}) {
|
||||
const schemaReady = readiness.schema?.ready === true;
|
||||
const migrationReady = readiness.migration?.ready === true;
|
||||
const transactionalReady = readiness.ready === true && schemaReady && migrationReady;
|
||||
const ready = runtime.ready === true && transactionalReady;
|
||||
const blocker = ready
|
||||
? null
|
||||
: !schemaReady
|
||||
? RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED
|
||||
: !migrationReady
|
||||
? RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED
|
||||
: runtime.blocker;
|
||||
return {
|
||||
...runtime,
|
||||
ready,
|
||||
status: ready ? "ready" : "blocked",
|
||||
blocker,
|
||||
reason: ready
|
||||
? "Postgres transactional Workbench realtime schema and migration are ready"
|
||||
: !schemaReady
|
||||
? "Postgres transactional Workbench realtime schema is incomplete"
|
||||
: !migrationReady
|
||||
? "Postgres transactional Workbench realtime migration is not recorded"
|
||||
: runtime.reason,
|
||||
liveRuntimeEvidence: runtime.liveRuntimeEvidence === true && ready,
|
||||
connection: {
|
||||
...runtime.connection,
|
||||
queryResult: ready
|
||||
? "transactional_realtime_readiness_ready"
|
||||
: !schemaReady
|
||||
? "schema_blocked"
|
||||
: !migrationReady
|
||||
? "migration_blocked"
|
||||
: runtime.connection?.queryResult
|
||||
},
|
||||
schema: readiness.schema,
|
||||
migration: readiness.migration,
|
||||
gates: {
|
||||
...runtime.gates,
|
||||
schema: schemaReady ? readyGate() : blockedGate(RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED),
|
||||
migration: !schemaReady
|
||||
? notCheckedGate()
|
||||
: migrationReady
|
||||
? readyGate()
|
||||
: blockedGate(RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED),
|
||||
durability: transactionalReady ? runtime.gates?.durability ?? notCheckedGate() : notCheckedGate()
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function applyMigration(report, sql, env, options) {
|
||||
report.actions.liveDbWriteAttempted = true;
|
||||
let client;
|
||||
@@ -456,7 +520,7 @@ function summarizeRuntimeMigration(migration = {}) {
|
||||
checked: Boolean(migration.checked),
|
||||
ready: Boolean(migration.ready),
|
||||
table: migration.table ?? CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE,
|
||||
requiredMigrationId: migration.requiredMigrationId ?? CLOUD_CORE_MIGRATION_ID,
|
||||
requiredMigrationId: migration.requiredMigrationId ?? CLOUD_RUNTIME_DURABLE_MIGRATION_ID,
|
||||
requiredSchemaVersion: migration.requiredSchemaVersion ?? CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION,
|
||||
appliedMigrationId: migration.appliedMigrationId ?? null,
|
||||
appliedSchemaVersion: migration.appliedSchemaVersion ?? null,
|
||||
@@ -485,6 +549,12 @@ function summarizeDbEnv(env = {}) {
|
||||
}
|
||||
|
||||
function buildApplyBoundary(args) {
|
||||
const schemaWriteScope = migrationPaths.map((migrationPath) =>
|
||||
`DEV Postgres schema objects declared by ${migrationPath}`
|
||||
);
|
||||
const ledgerWriteScope = CLOUD_CORE_MIGRATIONS.map((migration) =>
|
||||
`DEV ${CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE} ledger row ${migration.id}`
|
||||
);
|
||||
return {
|
||||
defaultMode: "check",
|
||||
mode: args.mode,
|
||||
@@ -493,10 +563,7 @@ function buildApplyBoundary(args) {
|
||||
requiresForLiveDbRead: ["--confirm-dev", "HWLAB_CLOUD_DB_URL from hwlab-cloud-api-dev-db/database-url"],
|
||||
requiresForApply: ["--apply", "--confirm-dev", "--confirmed-non-production", "HWLAB_CLOUD_DB_URL from hwlab-cloud-api-dev-db/database-url"],
|
||||
writeScope: args.mode === "apply"
|
||||
? [
|
||||
"DEV Postgres schema objects declared by internal/db/migrations/0001_cloud_core_skeleton.sql",
|
||||
"DEV hwlab_schema_migrations ledger row 0001_cloud_core_skeleton"
|
||||
]
|
||||
? [...schemaWriteScope, ...ledgerWriteScope]
|
||||
: [],
|
||||
noWriteScope: [
|
||||
"Kubernetes Secret resources",
|
||||
|
||||
@@ -10,8 +10,12 @@ import {
|
||||
} from "./dev-runtime-migration.mjs";
|
||||
import {
|
||||
CLOUD_CORE_MIGRATION_ID,
|
||||
CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION,
|
||||
CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS
|
||||
CLOUD_CORE_MIGRATIONS,
|
||||
CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION as CLOUD_CORE_SCHEMA_VERSION,
|
||||
CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID as CLOUD_RUNTIME_DURABLE_MIGRATION_ID,
|
||||
CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION as CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION,
|
||||
CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE,
|
||||
CLOUD_TRANSACTIONAL_REALTIME_REQUIRED_TABLE_COLUMNS as CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS
|
||||
} from "../../internal/db/schema.ts";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
@@ -34,6 +38,8 @@ test("source check is non-secret and does not attempt live DB access", async ()
|
||||
assert.equal(report.gates.schema.status, "ready");
|
||||
assert.equal(report.gates.migration.status, "ready");
|
||||
assert.equal(report.gates.readiness.status, "not_checked");
|
||||
assert.equal(report.issue, "pikasTech/HWLAB#2464");
|
||||
assert.deepEqual(report.applyBoundary.writeScope, []);
|
||||
assert.equal(report.safety.sourceStoresSecretValues, false);
|
||||
assert.equal(report.safety.printsSecretValues, false);
|
||||
assert.equal(JSON.stringify(report).includes("secret-password"), false);
|
||||
@@ -60,7 +66,7 @@ test("cloud-api image migration entrypoint exposes the same non-secret source ch
|
||||
assert.equal(report.actions.liveDbWriteAttempted, false);
|
||||
assert.equal(report.safety.printsSecretValues, false);
|
||||
assert.equal(report.safety.dbUrlValueRedacted, true);
|
||||
assert.equal(report.migration.id, CLOUD_CORE_MIGRATION_ID);
|
||||
assert.equal(report.migration.id, CLOUD_RUNTIME_DURABLE_MIGRATION_ID);
|
||||
assert.equal(JSON.stringify(report).includes("secret-password"), false);
|
||||
assert.equal(JSON.stringify(report).includes("db.example.test"), false);
|
||||
});
|
||||
@@ -120,6 +126,7 @@ test("live dry-run separates migration ledger blocker from auth and schema", asy
|
||||
assert.equal(report.gates.readiness.status, "not_checked");
|
||||
assert.equal(report.blockers[0].scope, "runtime-migration-ledger");
|
||||
assert.equal(report.runtime.migration.missing, true);
|
||||
assert.equal(report.runtime.migration.requiredMigrationId, CLOUD_RUNTIME_DURABLE_MIGRATION_ID);
|
||||
});
|
||||
|
||||
test("live dry-run separates SSL negotiation blocker from auth and schema", async () => {
|
||||
@@ -306,8 +313,17 @@ test("apply mode records ledger and verifies durable runtime readiness", async (
|
||||
assert.equal(report.gates.schema.status, "ready");
|
||||
assert.equal(report.gates.migration.status, "ready");
|
||||
assert.equal(report.gates.readiness.status, "ready");
|
||||
assert.equal(report.runtime.migration.appliedMigrationId, CLOUD_CORE_MIGRATION_ID);
|
||||
assert.equal(report.runtime.migration.appliedMigrationId, CLOUD_RUNTIME_DURABLE_MIGRATION_ID);
|
||||
assert.equal(report.runtime.migration.appliedSchemaVersion, CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION);
|
||||
assert.ok(queryClient.calls.some((call) => call.params[0] === CLOUD_CORE_MIGRATION_ID));
|
||||
assert.ok(queryClient.calls.some((call) => call.params[0] === CLOUD_RUNTIME_DURABLE_MIGRATION_ID));
|
||||
assert.deepEqual(report.applyBoundary.writeScope, [
|
||||
...CLOUD_CORE_MIGRATIONS.map((migration) => `DEV Postgres schema objects declared by ${migration.path}`),
|
||||
...CLOUD_CORE_MIGRATIONS.map((migration) => `DEV ${CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE} ledger row ${migration.id}`)
|
||||
]);
|
||||
assert.equal(report.applyBoundary.writeScope.length, CLOUD_CORE_MIGRATIONS.length * 2);
|
||||
assert.ok(report.applyBoundary.writeScope.some((entry) => entry.includes("0008_workbench_kafka_realtime.sql")));
|
||||
assert.ok(report.applyBoundary.writeScope.some((entry) => entry.endsWith(`ledger row ${CLOUD_RUNTIME_DURABLE_MIGRATION_ID}`)));
|
||||
assert.ok(queryClient.calls.some((call) => call.sql === "BEGIN"));
|
||||
assert.ok(queryClient.calls.some((call) => call.sql.includes("INSERT INTO hwlab_schema_migrations")));
|
||||
assert.ok(queryClient.calls.some((call) => call.sql === "COMMIT"));
|
||||
@@ -444,11 +460,17 @@ function createFakeQueryClient({ migrationReady, countErrorCode = null, migratio
|
||||
return { rows: schemaRows() };
|
||||
}
|
||||
if (sql.startsWith("SELECT id, schema_version FROM hwlab_schema_migrations")) {
|
||||
const requestedMigrationId = params[0];
|
||||
if (requestedMigrationId === CLOUD_CORE_MIGRATION_ID) {
|
||||
return {
|
||||
rows: [{ id: CLOUD_CORE_MIGRATION_ID, schema_version: CLOUD_CORE_SCHEMA_VERSION }]
|
||||
};
|
||||
}
|
||||
return {
|
||||
rows: state.migrationReady
|
||||
rows: requestedMigrationId === CLOUD_RUNTIME_DURABLE_MIGRATION_ID && state.migrationReady
|
||||
? [
|
||||
{
|
||||
id: CLOUD_CORE_MIGRATION_ID,
|
||||
id: CLOUD_RUNTIME_DURABLE_MIGRATION_ID,
|
||||
schema_version: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION
|
||||
}
|
||||
]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,254 @@
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
ciToolsRunnerImage,
|
||||
configObject,
|
||||
defaultOutDir,
|
||||
label,
|
||||
optionalConfigString,
|
||||
requiredConfigPositiveInteger,
|
||||
requiredConfigString,
|
||||
runtimeLaneMirrorSpecs
|
||||
} from "./core.mjs";
|
||||
import { renderTemplate, shellSingleQuote } from "./tekton-scripts.mjs";
|
||||
|
||||
function registryManifest() {
|
||||
return {
|
||||
apiVersion: "v1",
|
||||
kind: "List",
|
||||
items: [
|
||||
{ apiVersion: "v1", kind: "Namespace", metadata: { name: "hwlab-ci" } },
|
||||
{
|
||||
apiVersion: "apps/v1",
|
||||
kind: "Deployment",
|
||||
metadata: { name: "hwlab-registry", namespace: "hwlab-ci", labels: { "app.kubernetes.io/name": "hwlab-registry" } },
|
||||
spec: {
|
||||
replicas: 1,
|
||||
selector: { matchLabels: { "app.kubernetes.io/name": "hwlab-registry" } },
|
||||
template: {
|
||||
metadata: { labels: { "app.kubernetes.io/name": "hwlab-registry" } },
|
||||
spec: {
|
||||
hostNetwork: true,
|
||||
dnsPolicy: "ClusterFirstWithHostNet",
|
||||
containers: [{
|
||||
name: "registry",
|
||||
image: "registry:2.8.3",
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
ports: [{ name: "registry", containerPort: 5000, hostPort: 5000 }],
|
||||
env: [{ name: "REGISTRY_STORAGE_DELETE_ENABLED", value: "true" }],
|
||||
volumeMounts: [{ name: "storage", mountPath: "/var/lib/registry" }],
|
||||
readinessProbe: { httpGet: { path: "/v2/", port: "registry" } },
|
||||
livenessProbe: { httpGet: { path: "/v2/", port: "registry" } }
|
||||
}],
|
||||
volumes: [{ name: "storage", hostPath: { path: "/var/lib/hwlab/registry", type: "DirectoryOrCreate" } }]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
apiVersion: "v1",
|
||||
kind: "Service",
|
||||
metadata: { name: "hwlab-registry", namespace: "hwlab-ci", labels: { "app.kubernetes.io/name": "hwlab-registry" } },
|
||||
spec: { selector: { "app.kubernetes.io/name": "hwlab-registry" }, ports: [{ name: "registry", port: 5000, targetPort: "registry" }] }
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function uniqueStrings(values) {
|
||||
return Array.from(new Set(values.filter((value) => typeof value === "string" && value.length > 0)));
|
||||
}
|
||||
|
||||
function renderGitMirrorSshPrelude({ direct, proxySummary, proxyCommand, proxyUrl, proxyNoProxy }) {
|
||||
if (direct) {
|
||||
return renderTemplate("git-mirror-ssh-direct.sh", {
|
||||
"__HWLAB_PROXY_SUMMARY_SHELL__": shellSingleQuote(proxySummary)
|
||||
});
|
||||
}
|
||||
return renderTemplate("git-mirror-ssh-proxied.sh", {
|
||||
"// __HWLAB_GIT_MIRROR_PROXY_CONNECT_MJS__": renderTemplate("git-mirror-proxy-connect.mjs"),
|
||||
"__HWLAB_PROXY_SUMMARY_SHELL__": shellSingleQuote(proxySummary),
|
||||
"__HWLAB_PROXY_COMMAND_SHELL__": shellSingleQuote(proxyCommand),
|
||||
"__HWLAB_PROXY_URL_SHELL__": shellSingleQuote(proxyUrl),
|
||||
"__HWLAB_PROXY_NO_PROXY_SHELL__": shellSingleQuote(proxyNoProxy)
|
||||
});
|
||||
}
|
||||
|
||||
function gitMirrorConfigScripts({
|
||||
fetchConfigCommands,
|
||||
gitMirrorSshPrelude,
|
||||
gitopsBranchArgs,
|
||||
gitopsBranchesJson,
|
||||
mirrorBranchesJson,
|
||||
requiredRefArgs,
|
||||
sourceBranchArgs,
|
||||
sourceBranchesJson,
|
||||
sourceSnapshotRefPrefix
|
||||
}) {
|
||||
const gitopsBranchesJsonShell = shellSingleQuote(gitopsBranchesJson);
|
||||
return {
|
||||
"sync.sh": renderTemplate("git-mirror-sync.sh", {
|
||||
"# __HWLAB_GIT_MIRROR_SSH_PRELUDE__": gitMirrorSshPrelude,
|
||||
"__HWLAB_SOURCE_SNAPSHOT_REF_PREFIX_SHELL__": shellSingleQuote(sourceSnapshotRefPrefix),
|
||||
"# __HWLAB_FETCH_CONFIG_COMMANDS__": fetchConfigCommands,
|
||||
"__HWLAB_REQUIRED_REF_ARGS__": requiredRefArgs,
|
||||
"__HWLAB_SOURCE_BRANCH_ARGS__": sourceBranchArgs,
|
||||
"__HWLAB_GITOPS_BRANCH_ARGS__": gitopsBranchArgs,
|
||||
"__HWLAB_MIRROR_BRANCHES_JSON_SHELL__": shellSingleQuote(mirrorBranchesJson),
|
||||
"__HWLAB_SOURCE_BRANCHES_JSON_SHELL__": shellSingleQuote(sourceBranchesJson),
|
||||
"__HWLAB_GITOPS_BRANCHES_JSON_SHELL__": gitopsBranchesJsonShell
|
||||
}),
|
||||
"install-hooks.sh": renderTemplate("git-mirror-install-hooks.sh", {
|
||||
"__HWLAB_GITOPS_BRANCHES_JSON_SHELL__": gitopsBranchesJsonShell
|
||||
}),
|
||||
"flush.sh": renderTemplate("git-mirror-flush.sh", {
|
||||
"# __HWLAB_GIT_MIRROR_SSH_PRELUDE__": gitMirrorSshPrelude,
|
||||
"__HWLAB_GITOPS_BRANCH_ARGS__": gitopsBranchArgs,
|
||||
"__HWLAB_GITOPS_BRANCHES_JSON_SHELL__": gitopsBranchesJsonShell
|
||||
}),
|
||||
"http.mjs": renderTemplate("git-mirror-http.mjs")
|
||||
};
|
||||
}
|
||||
|
||||
function devopsInfraGitMirrorManifest(args, deploy) {
|
||||
const mirrorSpecs = runtimeLaneMirrorSpecs(deploy, { defaultNodeId: args.nodeId, defaultGitopsRoot: defaultOutDir });
|
||||
assert.ok(mirrorSpecs.length > 0, "runtime lane git mirror requires at least one deploy.lanes entry");
|
||||
const sourceBranches = uniqueStrings(mirrorSpecs.map((spec) => spec.sourceBranch));
|
||||
const gitopsBranches = uniqueStrings(mirrorSpecs.map((spec) => spec.gitopsBranch));
|
||||
const mirrorBranches = uniqueStrings([...sourceBranches, ...gitopsBranches]);
|
||||
const fetchConfigCommands = mirrorBranches
|
||||
.map((branch) => `git -C "$repo_path" config --add remote.origin.fetch '+refs/heads/${branch}:refs/mirror-stage/heads/${branch}'`)
|
||||
.join("\n");
|
||||
const requiredRefArgs = mirrorBranches.map((branch) => shellSingleQuote(`refs/mirror-stage/heads/${branch}`)).join(" ");
|
||||
const sourceBranchArgs = sourceBranches.map((branch) => shellSingleQuote(branch)).join(" ");
|
||||
const gitopsBranchArgs = gitopsBranches.map((branch) => shellSingleQuote(branch)).join(" ");
|
||||
const mirrorBranchesJson = JSON.stringify(mirrorBranches);
|
||||
const sourceBranchesJson = JSON.stringify(sourceBranches);
|
||||
const gitopsBranchesJson = JSON.stringify(gitopsBranches);
|
||||
const sourceSnapshotRefPrefix = "refs/unidesk/snapshots/hwlab-node-runtime";
|
||||
const laneConfig = configObject(configObject(deploy.lanes, "deploy.lanes")[args.lane], `deploy.lanes.${args.lane}`);
|
||||
const gitMirrorConfig = configObject(laneConfig.gitMirror, `deploy.lanes.${args.lane}.gitMirror`);
|
||||
const gitMirrorProxy = configObject(gitMirrorConfig.egressProxy, `deploy.lanes.${args.lane}.gitMirror.egressProxy`);
|
||||
const gitMirrorProxyMode = requiredConfigString(gitMirrorProxy, "mode", `deploy.lanes.${args.lane}.gitMirror.egressProxy`);
|
||||
assert.ok(gitMirrorProxyMode === "node-global" || gitMirrorProxyMode === "host-proxy-client" || gitMirrorProxyMode === "direct", `deploy.lanes.${args.lane}.gitMirror.egressProxy.mode must be node-global, host-proxy-client, or direct`);
|
||||
const useDirectGitMirrorProxy = gitMirrorProxyMode === "direct";
|
||||
const useHostGitMirrorProxy = gitMirrorProxyMode === "host-proxy-client";
|
||||
const gitMirrorNamespace = requiredConfigString(gitMirrorConfig, "namespace", `deploy.lanes.${args.lane}.gitMirror`);
|
||||
const gitMirrorReadName = requiredConfigString(gitMirrorConfig, "serviceReadName", `deploy.lanes.${args.lane}.gitMirror`);
|
||||
const gitMirrorWriteName = requiredConfigString(gitMirrorConfig, "serviceWriteName", `deploy.lanes.${args.lane}.gitMirror`);
|
||||
const gitMirrorCachePvcName = requiredConfigString(gitMirrorConfig, "cachePvcName", `deploy.lanes.${args.lane}.gitMirror`);
|
||||
const gitMirrorCachePvcStorage = requiredConfigString(gitMirrorConfig, "cachePvcStorage", `deploy.lanes.${args.lane}.gitMirror`);
|
||||
const gitMirrorCacheHostPath = optionalConfigString(gitMirrorConfig, "cacheHostPath", `deploy.lanes.${args.lane}.gitMirror`);
|
||||
const gitMirrorServicePort = requiredConfigPositiveInteger(gitMirrorConfig, "servicePort", `deploy.lanes.${args.lane}.gitMirror`);
|
||||
const gitMirrorConfigMapName = requiredConfigString(gitMirrorConfig, "syncConfigMapName", `deploy.lanes.${args.lane}.gitMirror`);
|
||||
const proxyNamespace = useDirectGitMirrorProxy || useHostGitMirrorProxy ? "" : requiredConfigString(gitMirrorProxy, "namespace", `deploy.lanes.${args.lane}.gitMirror.egressProxy`);
|
||||
const proxyServiceName = useDirectGitMirrorProxy || useHostGitMirrorProxy ? "" : requiredConfigString(gitMirrorProxy, "serviceName", `deploy.lanes.${args.lane}.gitMirror.egressProxy`);
|
||||
const proxyPort = useDirectGitMirrorProxy ? 0 : requiredConfigPositiveInteger(gitMirrorProxy, "port", `deploy.lanes.${args.lane}.gitMirror.egressProxy`);
|
||||
const proxyClientName = useDirectGitMirrorProxy ? "" : requiredConfigString(gitMirrorProxy, "clientName", `deploy.lanes.${args.lane}.gitMirror.egressProxy`);
|
||||
const proxyNoProxy = !useDirectGitMirrorProxy && Array.isArray(gitMirrorProxy.noProxy) ? gitMirrorProxy.noProxy.filter((value) => typeof value === "string" && value.length > 0).join(",") : "";
|
||||
const proxyHost = useHostGitMirrorProxy
|
||||
? requiredConfigString(gitMirrorProxy, "host", `deploy.lanes.${args.lane}.gitMirror.egressProxy`)
|
||||
: `${proxyServiceName}.${proxyNamespace}.svc.cluster.local`;
|
||||
const proxyUrl = `http://${proxyHost}:${proxyPort}`;
|
||||
const proxySummary = useDirectGitMirrorProxy
|
||||
? "git-mirror-egress-proxy mode=direct required=false transport=ssh source=yaml"
|
||||
: `git-mirror-egress-proxy client=${proxyClientName} mode=${gitMirrorProxyMode} required=true host=${proxyHost} port=${proxyPort} ssh=GIT_SSH-wrapper source=yaml`;
|
||||
const proxyCommand = useDirectGitMirrorProxy ? "" : `ProxyCommand=node /tmp/hwlab-github-proxy-connect.mjs ${proxyHost} ${proxyPort} %h %p`;
|
||||
const gitMirrorSshPrelude = renderGitMirrorSshPrelude({
|
||||
direct: useDirectGitMirrorProxy,
|
||||
proxySummary,
|
||||
proxyCommand,
|
||||
proxyUrl,
|
||||
proxyNoProxy
|
||||
});
|
||||
const cacheVolume = gitMirrorCacheHostPath === null
|
||||
? { name: "cache", persistentVolumeClaim: { claimName: gitMirrorCachePvcName } }
|
||||
: { name: "cache", hostPath: { path: gitMirrorCacheHostPath, type: "DirectoryOrCreate" } };
|
||||
const labels = {
|
||||
"app.kubernetes.io/part-of": "hwlab-node-control-plane",
|
||||
"hwlab.pikastech.local/node": args.nodeId,
|
||||
"hwlab.pikastech.local/lane": args.lane
|
||||
};
|
||||
const configLabels = { ...labels, "app.kubernetes.io/name": "git-mirror" };
|
||||
const readLabels = { ...labels, "app.kubernetes.io/name": gitMirrorReadName, "hwlab.pikastech.local/git-mirror-mode": "read" };
|
||||
const writeLabels = { ...labels, "app.kubernetes.io/name": gitMirrorWriteName, "hwlab.pikastech.local/git-mirror-mode": "write" };
|
||||
return {
|
||||
apiVersion: "v1",
|
||||
kind: "List",
|
||||
items: [
|
||||
{ apiVersion: "v1", kind: "Namespace", metadata: { name: gitMirrorNamespace } },
|
||||
...(gitMirrorCacheHostPath === null ? [{
|
||||
apiVersion: "v1",
|
||||
kind: "PersistentVolumeClaim",
|
||||
metadata: { name: gitMirrorCachePvcName, namespace: gitMirrorNamespace, labels: configLabels },
|
||||
spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: gitMirrorCachePvcStorage } } }
|
||||
}] : []),
|
||||
{
|
||||
apiVersion: "v1",
|
||||
kind: "ConfigMap",
|
||||
metadata: { name: gitMirrorConfigMapName, namespace: gitMirrorNamespace, labels: configLabels },
|
||||
data: gitMirrorConfigScripts({
|
||||
fetchConfigCommands,
|
||||
gitMirrorSshPrelude,
|
||||
gitopsBranchArgs,
|
||||
gitopsBranchesJson,
|
||||
mirrorBranchesJson,
|
||||
requiredRefArgs,
|
||||
sourceBranchArgs,
|
||||
sourceBranchesJson,
|
||||
sourceSnapshotRefPrefix
|
||||
})
|
||||
},
|
||||
{
|
||||
apiVersion: "apps/v1",
|
||||
kind: "Deployment",
|
||||
metadata: { name: gitMirrorReadName, namespace: gitMirrorNamespace, labels: readLabels },
|
||||
spec: {
|
||||
replicas: 1,
|
||||
selector: { matchLabels: { "app.kubernetes.io/name": gitMirrorReadName } },
|
||||
template: {
|
||||
metadata: { labels: readLabels },
|
||||
spec: {
|
||||
containers: [{
|
||||
name: "git-mirror",
|
||||
image: ciToolsRunnerImage,
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
command: ["node"],
|
||||
args: ["/etc/git-mirror/http.mjs"],
|
||||
ports: [{ name: "http", containerPort: 8080 }],
|
||||
readinessProbe: { httpGet: { path: "/pikasTech/HWLAB.git/HEAD", port: "http" }, initialDelaySeconds: 5, periodSeconds: 10 },
|
||||
livenessProbe: { httpGet: { path: "/", port: "http" }, initialDelaySeconds: 10, periodSeconds: 30 },
|
||||
volumeMounts: [
|
||||
{ name: "cache", mountPath: "/cache" },
|
||||
{ name: "config", mountPath: "/etc/git-mirror", readOnly: true }
|
||||
]
|
||||
}],
|
||||
volumes: [
|
||||
cacheVolume,
|
||||
{ name: "config", configMap: { name: gitMirrorConfigMapName, defaultMode: 0o555 } }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
apiVersion: "v1",
|
||||
kind: "Service",
|
||||
metadata: { name: gitMirrorReadName, namespace: gitMirrorNamespace, labels: readLabels },
|
||||
spec: { selector: { "app.kubernetes.io/name": gitMirrorReadName }, ports: [{ name: "http", port: gitMirrorServicePort, targetPort: "http" }] }
|
||||
},
|
||||
{
|
||||
apiVersion: "v1",
|
||||
kind: "Service",
|
||||
metadata: { name: gitMirrorWriteName, namespace: gitMirrorNamespace, labels: writeLabels },
|
||||
spec: { selector: { "app.kubernetes.io/name": gitMirrorReadName }, ports: [{ name: "http", port: gitMirrorServicePort, targetPort: "http" }] }
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
export {
|
||||
devopsInfraGitMirrorManifest,
|
||||
registryManifest,
|
||||
uniqueStrings
|
||||
};
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,830 @@
|
||||
import {
|
||||
argoApplicationName,
|
||||
buildkitRunnerImage,
|
||||
ciToolsRunnerImage,
|
||||
defaultDevBaseImage,
|
||||
defaultRegistryPrefix,
|
||||
defaultServiceIds,
|
||||
defaultSourceRepo,
|
||||
gitopsPathFor,
|
||||
gitopsPathForProfile,
|
||||
gitopsTargetForProfile,
|
||||
isRuntimeLane,
|
||||
namespaceNameForProfile,
|
||||
primitiveValidationTasks,
|
||||
proxyEnv,
|
||||
serviceIdsForLane,
|
||||
servicesParamForLane
|
||||
} from "./core.mjs";
|
||||
import {
|
||||
ciTimingShellFunction,
|
||||
collectArtifactsScript,
|
||||
controlPlaneReconcileScript,
|
||||
gitopsPromoteScript,
|
||||
perServicePublishScript,
|
||||
planArtifactsScript,
|
||||
pollerScript,
|
||||
prepareSourceScript,
|
||||
primitiveValidationTask,
|
||||
primitiveValidationTaskNames
|
||||
} from "./tekton-scripts.mjs";
|
||||
|
||||
function ciLaneSettings(argsOrLane = "node") {
|
||||
const lane = typeof argsOrLane === "string" ? argsOrLane : argsOrLane.lane;
|
||||
if (isRuntimeLane(lane)) {
|
||||
const namespace = namespaceNameForProfile(lane);
|
||||
return {
|
||||
lane,
|
||||
profile: lane,
|
||||
gitopsTarget: lane,
|
||||
pipelineName: `${namespace}-ci-image-publish`,
|
||||
pipelineRunPrefix: `${namespace}-ci-poll`,
|
||||
serviceAccountName: `${namespace}-tekton-runner`,
|
||||
pollerName: `${namespace}-branch-poller`,
|
||||
controlPlaneReconcilerName: `${namespace}-control-plane-reconciler`,
|
||||
tektonDir: `tekton-${lane}`,
|
||||
catalogPath: typeof argsOrLane === "object" ? argsOrLane.catalogPath : `deploy/artifact-catalog.${lane}.json`,
|
||||
imageTagMode: typeof argsOrLane === "object" ? argsOrLane.imageTagMode : "full",
|
||||
runtimeNamespace: namespace
|
||||
};
|
||||
}
|
||||
return {
|
||||
lane: "node",
|
||||
profile: "dev",
|
||||
gitopsTarget: "node",
|
||||
pipelineName: "hwlab-node-ci-image-publish",
|
||||
pipelineRunPrefix: "hwlab-node-ci-poll",
|
||||
serviceAccountName: "hwlab-tekton-runner",
|
||||
pollerName: "hwlab-node-branch-poller",
|
||||
controlPlaneReconcilerName: "hwlab-node-control-plane-reconciler",
|
||||
tektonDir: "tekton",
|
||||
catalogPath: "deploy/artifact-catalog.dev.json",
|
||||
imageTagMode: "short",
|
||||
runtimeNamespace: "hwlab-dev"
|
||||
};
|
||||
}
|
||||
|
||||
function tektonRbac(args = { lane: "node" }) {
|
||||
const settings = ciLaneSettings(args);
|
||||
if (isRuntimeLane(settings.lane)) {
|
||||
const runtimeObserverName = `${settings.runtimeNamespace}-runtime-observer`;
|
||||
const pipelineRunWriterName = `${settings.runtimeNamespace}-pipelinerun-writer`;
|
||||
const argoReconcilerName = `${settings.runtimeNamespace}-argocd-reconciler`;
|
||||
return {
|
||||
apiVersion: "v1",
|
||||
kind: "List",
|
||||
items: [
|
||||
{ apiVersion: "v1", kind: "Namespace", metadata: { name: "hwlab-ci" } },
|
||||
{ apiVersion: "v1", kind: "ServiceAccount", metadata: { name: settings.serviceAccountName, namespace: "hwlab-ci" } },
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "Role",
|
||||
metadata: { name: runtimeObserverName, namespace: settings.runtimeNamespace },
|
||||
rules: [
|
||||
{ apiGroups: [""], resources: ["pods", "services", "configmaps"], verbs: ["get", "list", "watch"] },
|
||||
{ apiGroups: ["apps"], resources: ["deployments", "statefulsets"], verbs: ["get", "list", "watch"] },
|
||||
{ apiGroups: ["batch"], resources: ["jobs"], verbs: ["get", "list", "watch"] }
|
||||
]
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "RoleBinding",
|
||||
metadata: { name: runtimeObserverName, namespace: settings.runtimeNamespace },
|
||||
subjects: [{ kind: "ServiceAccount", name: settings.serviceAccountName, namespace: "hwlab-ci" }],
|
||||
roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: runtimeObserverName }
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "Role",
|
||||
metadata: { name: pipelineRunWriterName, namespace: "hwlab-ci" },
|
||||
rules: [
|
||||
{ apiGroups: ["tekton.dev"], resources: ["pipelineruns"], verbs: ["get", "list", "create"] },
|
||||
{ apiGroups: ["tekton.dev"], resources: ["pipelines"], verbs: ["get", "patch", "create"] },
|
||||
{ apiGroups: ["batch"], resources: ["cronjobs"], verbs: ["get", "patch", "create"] },
|
||||
{ apiGroups: ["rbac.authorization.k8s.io"], resources: ["roles", "rolebindings"], verbs: ["get", "patch", "create"] },
|
||||
{ apiGroups: [""], resources: ["serviceaccounts"], verbs: ["get", "patch", "create"] }
|
||||
]
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "RoleBinding",
|
||||
metadata: { name: pipelineRunWriterName, namespace: "hwlab-ci" },
|
||||
subjects: [{ kind: "ServiceAccount", name: settings.serviceAccountName, namespace: "hwlab-ci" }],
|
||||
roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: pipelineRunWriterName }
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "Role",
|
||||
metadata: { name: argoReconcilerName, namespace: "argocd" },
|
||||
rules: [
|
||||
{ apiGroups: ["argoproj.io"], resources: ["applications", "appprojects"], verbs: ["get", "patch", "create"] },
|
||||
{ apiGroups: ["rbac.authorization.k8s.io"], resources: ["roles", "rolebindings"], verbs: ["get", "patch", "create"] }
|
||||
]
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "RoleBinding",
|
||||
metadata: { name: argoReconcilerName, namespace: "argocd" },
|
||||
subjects: [{ kind: "ServiceAccount", name: settings.serviceAccountName, namespace: "hwlab-ci" }],
|
||||
roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: argoReconcilerName }
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
return {
|
||||
apiVersion: "v1",
|
||||
kind: "List",
|
||||
items: [
|
||||
{ apiVersion: "v1", kind: "Namespace", metadata: { name: "hwlab-ci" } },
|
||||
{ apiVersion: "v1", kind: "ServiceAccount", metadata: { name: "hwlab-tekton-runner", namespace: "hwlab-ci" } },
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "Role",
|
||||
metadata: { name: "hwlab-node-control-plane-reconciler", namespace: "hwlab-dev" },
|
||||
rules: [
|
||||
{ apiGroups: ["rbac.authorization.k8s.io"], resources: ["roles", "rolebindings"], verbs: ["get", "patch", "create"] }
|
||||
]
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "RoleBinding",
|
||||
metadata: { name: "hwlab-node-control-plane-reconciler", namespace: "hwlab-dev" },
|
||||
subjects: [{ kind: "ServiceAccount", name: "hwlab-tekton-runner", namespace: "hwlab-ci" }],
|
||||
roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: "hwlab-node-control-plane-reconciler" }
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "Role",
|
||||
metadata: { name: "hwlab-tekton-runtime-observer", namespace: "hwlab-dev" },
|
||||
rules: [
|
||||
{ apiGroups: [""], resources: ["pods", "services", "configmaps"], verbs: ["get", "list", "watch"] },
|
||||
{ apiGroups: ["apps"], resources: ["deployments", "statefulsets"], verbs: ["get", "list", "watch"] },
|
||||
{ apiGroups: ["batch"], resources: ["jobs"], verbs: ["get", "list", "watch"] }
|
||||
]
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "RoleBinding",
|
||||
metadata: { name: "hwlab-tekton-runtime-observer", namespace: "hwlab-dev" },
|
||||
subjects: [{ kind: "ServiceAccount", name: "hwlab-tekton-runner", namespace: "hwlab-ci" }],
|
||||
roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: "hwlab-tekton-runtime-observer" }
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "Role",
|
||||
metadata: { name: "hwlab-tekton-pipelinerun-writer", namespace: "hwlab-ci" },
|
||||
rules: [
|
||||
{ apiGroups: ["tekton.dev"], resources: ["pipelineruns"], verbs: ["get", "list", "create"] },
|
||||
{ apiGroups: ["tekton.dev"], resources: ["pipelines"], verbs: ["get", "patch", "create"] },
|
||||
{ apiGroups: ["batch"], resources: ["cronjobs"], verbs: ["get", "patch", "create"] },
|
||||
{ apiGroups: ["rbac.authorization.k8s.io"], resources: ["roles", "rolebindings"], verbs: ["get", "patch", "create"] },
|
||||
{ apiGroups: [""], resources: ["serviceaccounts"], verbs: ["get", "patch", "create"] }
|
||||
]
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "RoleBinding",
|
||||
metadata: { name: "hwlab-tekton-pipelinerun-writer", namespace: "hwlab-ci" },
|
||||
subjects: [{ kind: "ServiceAccount", name: "hwlab-tekton-runner", namespace: "hwlab-ci" }],
|
||||
roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: "hwlab-tekton-pipelinerun-writer" }
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function buildTaskName(serviceId) {
|
||||
return `build-${serviceId}`;
|
||||
}
|
||||
|
||||
function affectedResultName(serviceId) {
|
||||
return `affected-${serviceId}`;
|
||||
}
|
||||
|
||||
function buildResultName(serviceId) {
|
||||
return `build-${serviceId}`;
|
||||
}
|
||||
|
||||
const serviceResultFields = Object.freeze([
|
||||
"status",
|
||||
"service-id",
|
||||
"image",
|
||||
"image-tag",
|
||||
"digest",
|
||||
"repository-digest",
|
||||
"source-commit-id",
|
||||
"component-input-hash",
|
||||
"environment-input-hash",
|
||||
"code-input-hash",
|
||||
"runtime-mode",
|
||||
"boot-repo",
|
||||
"boot-commit",
|
||||
"boot-sh",
|
||||
"build-created-at",
|
||||
"build-backend",
|
||||
"reused-from",
|
||||
"go-base-image",
|
||||
"go-base-image-status",
|
||||
"go-base-digest",
|
||||
"go-base-build-duration-ms",
|
||||
"go-base-buildkit-cache-ref"
|
||||
]);
|
||||
|
||||
function serviceResultParamName(serviceId, field) {
|
||||
return `${serviceId}-${field}`;
|
||||
}
|
||||
|
||||
function serviceResultEnvName(serviceId, field) {
|
||||
return `HWLAB_SERVICE_RESULT_${serviceId.toUpperCase().replace(/[^A-Z0-9]+/gu, "_")}_${field.toUpperCase().replace(/[^A-Z0-9]+/gu, "_")}`;
|
||||
}
|
||||
|
||||
function serviceResultParams(serviceIds = defaultServiceIds) {
|
||||
return serviceIds.flatMap((serviceId) => serviceResultFields.map((field) => ({ name: serviceResultParamName(serviceId, field), default: "" })));
|
||||
}
|
||||
|
||||
function serviceResultParamValues(serviceIds = defaultServiceIds) {
|
||||
return serviceIds.flatMap((serviceId) => serviceResultFields.map((field) => ({
|
||||
name: serviceResultParamName(serviceId, field),
|
||||
value: `$(tasks.${buildTaskName(serviceId)}.results.${field})`
|
||||
})));
|
||||
}
|
||||
|
||||
function serviceResultEnv(serviceIds = defaultServiceIds) {
|
||||
return serviceIds.flatMap((serviceId) => serviceResultFields.map((field) => ({
|
||||
name: serviceResultEnvName(serviceId, field),
|
||||
value: `$(params.${serviceResultParamName(serviceId, field)})`
|
||||
})));
|
||||
}
|
||||
|
||||
function serviceWorkVolume() {
|
||||
return { name: "service-work", emptyDir: {} };
|
||||
}
|
||||
|
||||
function buildkitBinVolume() {
|
||||
return { name: "buildkit-bin", emptyDir: {} };
|
||||
}
|
||||
|
||||
function buildkitRunVolume() {
|
||||
return { name: "buildkit-run", emptyDir: {} };
|
||||
}
|
||||
|
||||
function prepareBuildkitClientStep() {
|
||||
return {
|
||||
name: "prepare-buildkit-client",
|
||||
image: buildkitRunnerImage,
|
||||
securityContext: { runAsUser: 0, runAsGroup: 0 },
|
||||
volumeMounts: [{ name: "buildkit-bin", mountPath: "/workspace/buildkit-bin" }],
|
||||
script: `#!/bin/sh
|
||||
set -eu
|
||||
mkdir -p /workspace/buildkit-bin
|
||||
cp /usr/bin/buildctl /workspace/buildkit-bin/
|
||||
chmod +x /workspace/buildkit-bin/*
|
||||
`
|
||||
};
|
||||
}
|
||||
|
||||
function buildkitSidecar() {
|
||||
return {
|
||||
name: "buildkitd",
|
||||
image: buildkitRunnerImage,
|
||||
args: ["--addr", "unix:///workspace/buildkit-run/buildkitd.sock", "--oci-worker-no-process-sandbox"],
|
||||
env: proxyEnv(),
|
||||
volumeMounts: [{ name: "buildkit-run", mountPath: "/workspace/buildkit-run" }],
|
||||
securityContext: {
|
||||
runAsUser: 1000,
|
||||
runAsGroup: 1000,
|
||||
allowPrivilegeEscalation: true,
|
||||
appArmorProfile: { type: "Unconfined" },
|
||||
seccompProfile: { type: "Unconfined" }
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function planArtifactsTask({ serviceIds = defaultServiceIds } = {}) {
|
||||
return {
|
||||
name: "plan-artifacts",
|
||||
runAfter: primitiveValidationTaskNames(),
|
||||
workspaces: [{ name: "source", workspace: "source" }],
|
||||
taskSpec: {
|
||||
params: [
|
||||
{ name: "git-url" },
|
||||
{ name: "git-read-url" },
|
||||
{ name: "gitops-branch" },
|
||||
{ name: "lane" },
|
||||
{ name: "revision" },
|
||||
{ name: "catalog-path" },
|
||||
{ name: "registry-prefix" },
|
||||
{ name: "services" }
|
||||
],
|
||||
results: serviceIds.flatMap((serviceId) => [
|
||||
{ name: affectedResultName(serviceId), description: `${serviceId} rollout affected according to ci-plan` },
|
||||
{ name: buildResultName(serviceId), description: `${serviceId} image build required according to ci-plan` }
|
||||
]),
|
||||
workspaces: [{ name: "source" }],
|
||||
steps: [{
|
||||
name: "plan",
|
||||
image: ciToolsRunnerImage,
|
||||
env: [
|
||||
{ name: "HWLAB_SELECTED_SERVICES", value: "$(params.services)" },
|
||||
{ name: "HWLAB_DEV_REGISTRY_K8S_NATIVE", value: "1" },
|
||||
{ name: "HWLAB_TEKTON_PIPELINERUN", value: "$(context.pipelineRun.name)" },
|
||||
...proxyEnv()
|
||||
],
|
||||
script: planArtifactsScript()
|
||||
}]
|
||||
},
|
||||
params: [
|
||||
{ name: "git-url", value: "$(params.git-url)" },
|
||||
{ name: "git-read-url", value: "$(params.git-read-url)" },
|
||||
{ name: "gitops-branch", value: "$(params.gitops-branch)" },
|
||||
{ name: "lane", value: "$(params.lane)" },
|
||||
{ name: "revision", value: "$(params.revision)" },
|
||||
{ name: "catalog-path", value: "$(params.catalog-path)" },
|
||||
{ name: "registry-prefix", value: "$(params.registry-prefix)" },
|
||||
{ name: "services", value: "$(params.services)" }
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function perServiceBuildTask(serviceId, { runAfter = ["plan-artifacts"] } = {}) {
|
||||
return {
|
||||
name: buildTaskName(serviceId),
|
||||
runAfter,
|
||||
workspaces: [{ name: "source", workspace: "source" }],
|
||||
when: [{ input: `$(tasks.plan-artifacts.results.${buildResultName(serviceId)})`, operator: "in", values: ["true"] }],
|
||||
taskSpec: {
|
||||
params: [
|
||||
{ name: "revision" },
|
||||
{ name: "lane" },
|
||||
{ name: "catalog-path" },
|
||||
{ name: "image-tag-mode" },
|
||||
{ name: "registry-prefix" },
|
||||
{ name: "service-id" },
|
||||
{ name: "base-image" },
|
||||
{ name: "build-cache-mode" }
|
||||
],
|
||||
results: serviceResultFields.map((field) => ({ name: field, description: `${serviceId} ${field}` })),
|
||||
workspaces: [{ name: "source" }],
|
||||
volumes: [serviceWorkVolume(), buildkitBinVolume(), buildkitRunVolume()],
|
||||
sidecars: [buildkitSidecar()],
|
||||
steps: [
|
||||
prepareBuildkitClientStep(),
|
||||
{
|
||||
name: "publish",
|
||||
image: ciToolsRunnerImage,
|
||||
securityContext: { runAsUser: 1000, runAsGroup: 1000 },
|
||||
env: proxyEnv(),
|
||||
volumeMounts: [{ name: "service-work", mountPath: "/workspace/service-work" }, { name: "buildkit-bin", mountPath: "/workspace/buildkit-bin" }, { name: "buildkit-run", mountPath: "/workspace/buildkit-run" }],
|
||||
script: perServicePublishScript()
|
||||
}
|
||||
]
|
||||
},
|
||||
params: [
|
||||
{ name: "revision", value: "$(params.revision)" },
|
||||
{ name: "lane", value: "$(params.lane)" },
|
||||
{ name: "catalog-path", value: "$(params.catalog-path)" },
|
||||
{ name: "image-tag-mode", value: "$(params.image-tag-mode)" },
|
||||
{ name: "registry-prefix", value: "$(params.registry-prefix)" },
|
||||
{ name: "service-id", value: serviceId },
|
||||
{ name: "base-image", value: "$(params.base-image)" },
|
||||
{ name: "build-cache-mode", value: "$(params.build-cache-mode)" }
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function collectArtifactsTask({ serviceIds = defaultServiceIds } = {}) {
|
||||
return {
|
||||
name: "collect-artifacts",
|
||||
runAfter: serviceIds.map(buildTaskName),
|
||||
workspaces: [{ name: "source", workspace: "source" }],
|
||||
taskSpec: {
|
||||
params: [
|
||||
{ name: "revision" },
|
||||
{ name: "lane" },
|
||||
{ name: "catalog-path" },
|
||||
{ name: "image-tag-mode" },
|
||||
{ name: "registry-prefix" },
|
||||
{ name: "services" },
|
||||
{ name: "base-image" }
|
||||
],
|
||||
workspaces: [{ name: "source" }],
|
||||
steps: [{ name: "collect", image: ciToolsRunnerImage, env: proxyEnv(), script: collectArtifactsScript() }]
|
||||
},
|
||||
params: [
|
||||
{ name: "revision", value: "$(params.revision)" },
|
||||
{ name: "lane", value: "$(params.lane)" },
|
||||
{ name: "catalog-path", value: "$(params.catalog-path)" },
|
||||
{ name: "image-tag-mode", value: "$(params.image-tag-mode)" },
|
||||
{ name: "registry-prefix", value: "$(params.registry-prefix)" },
|
||||
{ name: "services", value: "$(params.services)" },
|
||||
{ name: "base-image", value: "$(params.base-image)" }
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function runtimeReadyScript() {
|
||||
return `#!/bin/sh
|
||||
set -eu
|
||||
${ciTimingShellFunction()}
|
||||
export HWLAB_TEKTON_PIPELINERUN="$(context.pipelineRun.name)"
|
||||
export HWLAB_TEKTON_TASKRUN="$(context.taskRun.name)"
|
||||
export HWLAB_TEKTON_TASK="runtime-ready"
|
||||
export HWLAB_SOURCE_REVISION="$(params.revision)"
|
||||
export HWLAB_RUNTIME_READY_TIMEOUT_MS="$(params.timeout-ms)"
|
||||
cd /workspace/source/repo
|
||||
node scripts/ci/runtime-ready.mjs
|
||||
`;
|
||||
}
|
||||
|
||||
function runtimeReadyTask({ profile = "dev" } = {}) {
|
||||
return {
|
||||
name: "runtime-ready",
|
||||
runAfter: ["gitops-promote"],
|
||||
workspaces: [{ name: "source", workspace: "source" }],
|
||||
taskSpec: {
|
||||
params: [{ name: "revision" }, { name: "runtime-namespace" }, { name: "gitops-target" }, { name: "argo-application" }, { name: "timeout-ms" }],
|
||||
workspaces: [{ name: "source" }],
|
||||
steps: [{
|
||||
name: "runtime-ready",
|
||||
image: ciToolsRunnerImage,
|
||||
env: [
|
||||
...proxyEnv(),
|
||||
{ name: "HWLAB_RUNTIME_NAMESPACE", value: "$(params.runtime-namespace)" },
|
||||
{ name: "HWLAB_GITOPS_TARGET", value: "$(params.gitops-target)" },
|
||||
{ name: "HWLAB_ARGO_APPLICATION", value: "$(params.argo-application)" }
|
||||
],
|
||||
script: runtimeReadyScript()
|
||||
}]
|
||||
},
|
||||
params: [
|
||||
{ name: "revision", value: "$(params.revision)" },
|
||||
{ name: "runtime-namespace", value: namespaceNameForProfile(profile) },
|
||||
{ name: "gitops-target", value: gitopsTargetForProfile(profile) },
|
||||
{ name: "argo-application", value: argoApplicationName(profile) },
|
||||
{ name: "timeout-ms", value: "$(params.runtime-ready-timeout-ms)" }
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function imagePublishTaskSet(args = { lane: "node" }, deploy = null) {
|
||||
const serviceIds = serviceIdsForLane(args.lane, deploy);
|
||||
const buildTasks = serviceIds.map((serviceId) => perServiceBuildTask(serviceId, {
|
||||
runAfter: ["plan-artifacts"]
|
||||
}));
|
||||
return [
|
||||
planArtifactsTask({ serviceIds }),
|
||||
...buildTasks,
|
||||
collectArtifactsTask({ serviceIds })
|
||||
];
|
||||
}
|
||||
|
||||
function tektonPipeline(args = { lane: "node" }, deploy = null) {
|
||||
const settings = ciLaneSettings(args);
|
||||
const runtimePath = gitopsPathForProfile(args, settings.profile);
|
||||
const preFlushRuntimeReadyTasks = isRuntimeLane(settings.lane)
|
||||
? []
|
||||
: [{
|
||||
...runtimeReadyTask({ profile: settings.profile }),
|
||||
when: [{ input: "$(tasks.gitops-promote.results.runtime-ready-required)", operator: "in", values: ["true"] }]
|
||||
}];
|
||||
return {
|
||||
apiVersion: "tekton.dev/v1",
|
||||
kind: "Pipeline",
|
||||
metadata: {
|
||||
name: settings.pipelineName,
|
||||
namespace: "hwlab-ci",
|
||||
labels: {
|
||||
"app.kubernetes.io/part-of": "hwlab",
|
||||
"hwlab.pikastech.local/gitops-target": settings.gitopsTarget
|
||||
},
|
||||
annotations: {
|
||||
"hwlab.pikastech.local/source-config": "scripts/gitops-render.mjs#tekton-native-primitive-ci",
|
||||
"hwlab.pikastech.local/ci-contract": "tekton-native-primitive-tasks",
|
||||
"hwlab.pikastech.local/policy": "native-per-service-taskrun-image-publish"
|
||||
}
|
||||
},
|
||||
spec: {
|
||||
params: [
|
||||
{ name: "git-url", type: "string", default: defaultSourceRepo },
|
||||
{ name: "git-read-url", type: "string", default: args.gitReadUrl },
|
||||
{ name: "git-write-url", type: "string", default: args.gitWriteUrl },
|
||||
{ name: "source-branch", type: "string", default: args.sourceBranch },
|
||||
{ name: "gitops-branch", type: "string", default: args.gitopsBranch },
|
||||
{ name: "lane", type: "string", default: settings.lane },
|
||||
{ name: "catalog-path", type: "string", default: args.catalogPath || settings.catalogPath },
|
||||
{ name: "image-tag-mode", type: "string", default: args.imageTagMode || settings.imageTagMode },
|
||||
{ name: "runtime-path", type: "string", default: runtimePath },
|
||||
{ name: "revision", type: "string", description: "Full git commit SHA; the only source truth for image tags." },
|
||||
{ name: "registry-prefix", type: "string", default: defaultRegistryPrefix },
|
||||
{ name: "services", type: "string", default: servicesParamForLane(args.lane, deploy) },
|
||||
{ name: "base-image", type: "string", default: defaultDevBaseImage },
|
||||
{ name: "build-cache-mode", type: "string", default: "registry" },
|
||||
{ name: "runtime-ready-timeout-ms", type: "string", default: "60000" }
|
||||
],
|
||||
workspaces: [
|
||||
{ name: "source" },
|
||||
{ name: "git-ssh" }
|
||||
],
|
||||
tasks: [
|
||||
{
|
||||
name: "prepare-source",
|
||||
workspaces: [
|
||||
{ name: "source", workspace: "source" },
|
||||
{ name: "git-ssh", workspace: "git-ssh" }
|
||||
],
|
||||
taskSpec: {
|
||||
params: [
|
||||
{ name: "git-url" },
|
||||
{ name: "git-read-url" },
|
||||
{ name: "source-branch" },
|
||||
{ name: "gitops-branch" },
|
||||
{ name: "lane" },
|
||||
{ name: "catalog-path" },
|
||||
{ name: "image-tag-mode" },
|
||||
{ name: "registry-prefix" },
|
||||
{ name: "services" },
|
||||
{ name: "revision" }
|
||||
],
|
||||
workspaces: [{ name: "source" }, { name: "git-ssh" }],
|
||||
steps: [{ name: "prepare-source", image: ciToolsRunnerImage, env: proxyEnv(), script: prepareSourceScript() }]
|
||||
},
|
||||
params: [
|
||||
{ name: "git-url", value: "$(params.git-url)" },
|
||||
{ name: "git-read-url", value: "$(params.git-read-url)" },
|
||||
{ name: "source-branch", value: "$(params.source-branch)" },
|
||||
{ name: "gitops-branch", value: "$(params.gitops-branch)" },
|
||||
{ name: "lane", value: "$(params.lane)" },
|
||||
{ name: "catalog-path", value: "$(params.catalog-path)" },
|
||||
{ name: "image-tag-mode", value: "$(params.image-tag-mode)" },
|
||||
{ name: "registry-prefix", value: "$(params.registry-prefix)" },
|
||||
{ name: "services", value: "$(params.services)" },
|
||||
{ name: "revision", value: "$(params.revision)" }
|
||||
]
|
||||
},
|
||||
...primitiveValidationTasks.map(primitiveValidationTask),
|
||||
...imagePublishTaskSet(args, deploy),
|
||||
{
|
||||
name: "gitops-promote",
|
||||
runAfter: ["collect-artifacts"],
|
||||
workspaces: [
|
||||
{ name: "source", workspace: "source" },
|
||||
{ name: "git-ssh", workspace: "git-ssh" }
|
||||
],
|
||||
taskSpec: {
|
||||
params: [
|
||||
{ name: "git-url" },
|
||||
{ name: "git-read-url" },
|
||||
{ name: "git-write-url" },
|
||||
{ name: "source-branch" },
|
||||
{ name: "gitops-branch" },
|
||||
{ name: "lane" },
|
||||
{ name: "catalog-path" },
|
||||
{ name: "image-tag-mode" },
|
||||
{ name: "runtime-path" },
|
||||
{ name: "revision" },
|
||||
{ name: "registry-prefix" }
|
||||
],
|
||||
results: [{ name: "runtime-ready-required", description: "true when GitOps promotion changed runtime desired state and runtime readiness must be observed" }],
|
||||
workspaces: [{ name: "source" }, { name: "git-ssh" }],
|
||||
steps: [{ name: "promote", image: ciToolsRunnerImage, env: proxyEnv(), script: gitopsPromoteScript() }]
|
||||
},
|
||||
params: [
|
||||
{ name: "git-url", value: "$(params.git-url)" },
|
||||
{ name: "git-read-url", value: "$(params.git-read-url)" },
|
||||
{ name: "git-write-url", value: "$(params.git-write-url)" },
|
||||
{ name: "source-branch", value: "$(params.source-branch)" },
|
||||
{ name: "gitops-branch", value: "$(params.gitops-branch)" },
|
||||
{ name: "lane", value: "$(params.lane)" },
|
||||
{ name: "catalog-path", value: "$(params.catalog-path)" },
|
||||
{ name: "image-tag-mode", value: "$(params.image-tag-mode)" },
|
||||
{ name: "runtime-path", value: "$(params.runtime-path)" },
|
||||
{ name: "revision", value: "$(params.revision)" },
|
||||
{ name: "registry-prefix", value: "$(params.registry-prefix)" }
|
||||
]
|
||||
},
|
||||
...preFlushRuntimeReadyTasks
|
||||
]
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function tektonPipelineRunTemplate({ source, args }) {
|
||||
const settings = ciLaneSettings(args);
|
||||
const runtimePath = gitopsPathForProfile(args, settings.profile);
|
||||
return {
|
||||
apiVersion: "tekton.dev/v1",
|
||||
kind: "PipelineRun",
|
||||
metadata: {
|
||||
generateName: `${settings.pipelineRunPrefix}-manual-`,
|
||||
namespace: "hwlab-ci",
|
||||
labels: {
|
||||
"app.kubernetes.io/part-of": "hwlab",
|
||||
"hwlab.pikastech.local/gitops-target": settings.gitopsTarget,
|
||||
"hwlab.pikastech.local/source-commit": source.full
|
||||
}
|
||||
},
|
||||
spec: {
|
||||
pipelineRef: { name: settings.pipelineName },
|
||||
taskRunTemplate: {
|
||||
serviceAccountName: settings.serviceAccountName,
|
||||
podTemplate: {
|
||||
hostNetwork: true,
|
||||
dnsPolicy: "ClusterFirstWithHostNet",
|
||||
securityContext: { fsGroup: 1000 }
|
||||
}
|
||||
},
|
||||
params: [
|
||||
{ name: "git-url", value: args.sourceRepo },
|
||||
{ name: "git-read-url", value: args.gitReadUrl },
|
||||
{ name: "git-write-url", value: args.gitWriteUrl },
|
||||
{ name: "source-branch", value: args.sourceBranch },
|
||||
{ name: "gitops-branch", value: args.gitopsBranch },
|
||||
{ name: "lane", value: settings.lane },
|
||||
{ name: "catalog-path", value: args.catalogPath || settings.catalogPath },
|
||||
{ name: "image-tag-mode", value: args.imageTagMode || settings.imageTagMode },
|
||||
{ name: "runtime-path", value: runtimePath },
|
||||
{ name: "revision", value: source.full },
|
||||
{ name: "registry-prefix", value: args.registryPrefix },
|
||||
{ name: "base-image", value: defaultDevBaseImage },
|
||||
{ name: "build-cache-mode", value: "registry" }
|
||||
],
|
||||
workspaces: [
|
||||
{ name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } },
|
||||
{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }
|
||||
]
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function tektonPollerCronJob(args, deploy) {
|
||||
const settings = ciLaneSettings(args);
|
||||
if (settings.lane === "v02") throw new Error("v02 CI/CD is manually triggered by UniDesk CLI and must not render a branch poller CronJob");
|
||||
return {
|
||||
apiVersion: "batch/v1",
|
||||
kind: "CronJob",
|
||||
metadata: {
|
||||
name: settings.pollerName,
|
||||
namespace: "hwlab-ci",
|
||||
labels: {
|
||||
"app.kubernetes.io/part-of": "hwlab",
|
||||
"hwlab.pikastech.local/gitops-target": settings.gitopsTarget
|
||||
},
|
||||
annotations: {
|
||||
"hwlab.pikastech.local/source-branch": args.sourceBranch,
|
||||
"hwlab.pikastech.local/gitops-branch": args.gitopsBranch
|
||||
}
|
||||
},
|
||||
spec: {
|
||||
schedule: settings.lane === "v02" ? "*/10 * * * *" : "* * * * *",
|
||||
concurrencyPolicy: "Forbid",
|
||||
successfulJobsHistoryLimit: 3,
|
||||
failedJobsHistoryLimit: 3,
|
||||
jobTemplate: {
|
||||
spec: {
|
||||
backoffLimit: 1,
|
||||
template: {
|
||||
metadata: {
|
||||
labels: {
|
||||
"app.kubernetes.io/part-of": "hwlab",
|
||||
"hwlab.pikastech.local/gitops-target": settings.gitopsTarget
|
||||
}
|
||||
},
|
||||
spec: {
|
||||
serviceAccountName: settings.serviceAccountName,
|
||||
restartPolicy: "Never",
|
||||
hostNetwork: true,
|
||||
dnsPolicy: "ClusterFirstWithHostNet",
|
||||
containers: [{
|
||||
name: "poll",
|
||||
image: ciToolsRunnerImage,
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
env: [
|
||||
...proxyEnv(),
|
||||
{ name: "POD_NAMESPACE", valueFrom: { fieldRef: { fieldPath: "metadata.namespace" } } },
|
||||
{ name: "PIPELINE_NAME", value: settings.pipelineName },
|
||||
{ name: "PIPELINERUN_PREFIX", value: settings.pipelineRunPrefix },
|
||||
{ name: "SERVICE_ACCOUNT_NAME", value: settings.serviceAccountName },
|
||||
{ name: "GITOPS_TARGET", value: settings.gitopsTarget },
|
||||
{ name: "LANE", value: settings.lane },
|
||||
{ name: "CATALOG_PATH", value: args.catalogPath || settings.catalogPath },
|
||||
{ name: "IMAGE_TAG_MODE", value: args.imageTagMode || settings.imageTagMode },
|
||||
{ name: "RUNTIME_PATH", value: gitopsPathForProfile(args, settings.profile) },
|
||||
{ name: "GIT_URL", value: args.sourceRepo },
|
||||
{ name: "GIT_READ_URL", value: args.gitReadUrl },
|
||||
{ name: "SOURCE_BRANCH", value: args.sourceBranch },
|
||||
{ name: "GITOPS_BRANCH", value: args.gitopsBranch },
|
||||
{ name: "REGISTRY_PREFIX", value: args.registryPrefix },
|
||||
{ name: "SERVICES", value: servicesParamForLane(args.lane, deploy) },
|
||||
{ name: "BASE_IMAGE", value: defaultDevBaseImage }
|
||||
],
|
||||
volumeMounts: [{ name: "git-ssh", mountPath: "/workspace/git-ssh", readOnly: true }],
|
||||
command: ["/bin/sh", "-c"],
|
||||
args: [pollerScript()]
|
||||
}],
|
||||
volumes: [{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function tektonControlPlaneReconcilerCronJob(args) {
|
||||
const settings = ciLaneSettings(args);
|
||||
if (settings.lane === "v02") throw new Error("v02 CI/CD is manually triggered by UniDesk CLI and must not render a control-plane reconciler CronJob");
|
||||
return {
|
||||
apiVersion: "batch/v1",
|
||||
kind: "CronJob",
|
||||
metadata: {
|
||||
name: settings.controlPlaneReconcilerName,
|
||||
namespace: "hwlab-ci",
|
||||
labels: {
|
||||
"app.kubernetes.io/part-of": "hwlab",
|
||||
"hwlab.pikastech.local/gitops-target": settings.gitopsTarget
|
||||
},
|
||||
annotations: {
|
||||
"hwlab.pikastech.local/source-branch": args.sourceBranch,
|
||||
"hwlab.pikastech.local/gitops-branch": args.gitopsBranch,
|
||||
"hwlab.pikastech.local/purpose": "render-and-apply-node-ci-control-plane"
|
||||
}
|
||||
},
|
||||
spec: {
|
||||
schedule: "*/10 * * * *",
|
||||
concurrencyPolicy: "Forbid",
|
||||
successfulJobsHistoryLimit: 3,
|
||||
failedJobsHistoryLimit: 3,
|
||||
jobTemplate: {
|
||||
spec: {
|
||||
backoffLimit: 1,
|
||||
template: {
|
||||
metadata: {
|
||||
labels: {
|
||||
"app.kubernetes.io/part-of": "hwlab",
|
||||
"hwlab.pikastech.local/gitops-target": settings.gitopsTarget
|
||||
}
|
||||
},
|
||||
spec: {
|
||||
serviceAccountName: settings.serviceAccountName,
|
||||
restartPolicy: "Never",
|
||||
hostNetwork: true,
|
||||
dnsPolicy: "ClusterFirstWithHostNet",
|
||||
containers: [{
|
||||
name: "reconcile",
|
||||
image: ciToolsRunnerImage,
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
env: [
|
||||
...proxyEnv(),
|
||||
{ name: "POD_NAMESPACE", valueFrom: { fieldRef: { fieldPath: "metadata.namespace" } } },
|
||||
{ name: "LANE", value: settings.lane },
|
||||
{ name: "CATALOG_PATH", value: args.catalogPath || settings.catalogPath },
|
||||
{ name: "IMAGE_TAG_MODE", value: args.imageTagMode || settings.imageTagMode },
|
||||
{ name: "TEKTON_DIR", value: settings.tektonDir },
|
||||
{ name: "ARGO_FILES", value: settings.lane === "v02" ? `${gitopsPathFor(args, "argocd/project.yaml")},${gitopsPathFor(args, "argocd/application-v02.yaml")}` : "" },
|
||||
{ name: "FIELD_MANAGER", value: settings.controlPlaneReconcilerName },
|
||||
{ name: "RECONCILE_EVENT", value: `${settings.gitopsTarget}-control-plane-reconciled` },
|
||||
{ name: "GIT_URL", value: args.sourceRepo },
|
||||
{ name: "GIT_READ_URL", value: args.gitReadUrl },
|
||||
{ name: "SOURCE_BRANCH", value: args.sourceBranch },
|
||||
{ name: "GITOPS_BRANCH", value: args.gitopsBranch },
|
||||
{ name: "REGISTRY_PREFIX", value: args.registryPrefix }
|
||||
],
|
||||
volumeMounts: [{ name: "git-ssh", mountPath: "/workspace/git-ssh", readOnly: true }],
|
||||
command: ["/bin/sh", "-c"],
|
||||
args: [controlPlaneReconcileScript()]
|
||||
}],
|
||||
volumes: [{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export {
|
||||
affectedResultName,
|
||||
buildResultName,
|
||||
buildTaskName,
|
||||
buildkitBinVolume,
|
||||
buildkitRunVolume,
|
||||
buildkitSidecar,
|
||||
ciLaneSettings,
|
||||
collectArtifactsTask,
|
||||
imagePublishTaskSet,
|
||||
perServiceBuildTask,
|
||||
planArtifactsTask,
|
||||
prepareBuildkitClientStep,
|
||||
runtimeReadyScript,
|
||||
runtimeReadyTask,
|
||||
serviceResultEnv,
|
||||
serviceResultEnvName,
|
||||
serviceResultParamName,
|
||||
serviceResultParams,
|
||||
serviceResultParamValues,
|
||||
serviceWorkVolume,
|
||||
tektonControlPlaneReconcilerCronJob,
|
||||
tektonPipeline,
|
||||
tektonPipelineRunTemplate,
|
||||
tektonPollerCronJob,
|
||||
tektonRbac
|
||||
};
|
||||
@@ -0,0 +1,187 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
ciToolsRunnerImage,
|
||||
defaultServiceIds,
|
||||
defaultV02RuntimeEndpoint,
|
||||
primitiveValidationTasks,
|
||||
proxyEnv
|
||||
} from "./core.mjs";
|
||||
|
||||
const templatesDir = path.join(path.dirname(fileURLToPath(import.meta.url)), "templates");
|
||||
const templateCache = new Map();
|
||||
|
||||
function readTemplate(name) {
|
||||
if (!templateCache.has(name)) {
|
||||
templateCache.set(name, readFileSync(path.join(templatesDir, name), "utf8"));
|
||||
}
|
||||
return templateCache.get(name);
|
||||
}
|
||||
|
||||
function renderTemplate(name, replacements = {}) {
|
||||
let rendered = readTemplate(name);
|
||||
for (const [token, value] of Object.entries(replacements)) {
|
||||
if (!rendered.includes(token)) throw new Error(`template ${name} is missing token ${token}`);
|
||||
rendered = rendered.replaceAll(token, String(value));
|
||||
}
|
||||
const unresolved = [...rendered.matchAll(/__HWLAB_[A-Z0-9_]+__/gu)].map((match) => match[0]);
|
||||
if (unresolved.length > 0) {
|
||||
throw new Error(`template ${name} has unresolved tokens: ${[...new Set(unresolved)].join(", ")}`);
|
||||
}
|
||||
return rendered;
|
||||
}
|
||||
|
||||
function shellSingleQuote(value) {
|
||||
return `'${String(value).replace(/'/g, `'"'"'`)}'`;
|
||||
}
|
||||
|
||||
function ciTimingShellFunction() {
|
||||
return readTemplate("ci-timing-functions.sh");
|
||||
}
|
||||
|
||||
function dependencyProxyProbeScript(phase, targetUrl) {
|
||||
if (!/^[a-z0-9-]+$/u.test(phase)) throw new Error(`invalid dependency proxy probe phase: ${phase}`);
|
||||
return renderTemplate("dependency-proxy-probe.sh", {
|
||||
"__HWLAB_PROXY_PHASE_SHELL__": shellSingleQuote(phase),
|
||||
"__HWLAB_PROXY_TARGET_SHELL__": shellSingleQuote(targetUrl),
|
||||
"__HWLAB_PROXY_PHASE__": phase
|
||||
});
|
||||
}
|
||||
|
||||
function curlDownloadProbeFunction() {
|
||||
return readTemplate("curl-download-probe.sh");
|
||||
}
|
||||
|
||||
function gitSshShellFunction() {
|
||||
return renderTemplate("git-ssh-functions.sh", {
|
||||
"// __HWLAB_GITHUB_PROXY_CONNECT_MJS__": readTemplate("github-proxy-connect.mjs")
|
||||
});
|
||||
}
|
||||
|
||||
function prepareSourceScript() {
|
||||
return renderTemplate("prepare-source.sh", {
|
||||
"# __HWLAB_CI_TIMING_SHELL__\n": ciTimingShellFunction(),
|
||||
"# __HWLAB_GIT_SSH_SHELL__\n": gitSshShellFunction(),
|
||||
"__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage,
|
||||
"__HWLAB_V02_RUNTIME_ENDPOINT__": JSON.stringify(defaultV02RuntimeEndpoint).slice(1, -1),
|
||||
"__HWLAB_VALIDATION_TASK_COUNT__": primitiveValidationTasks.length
|
||||
});
|
||||
}
|
||||
|
||||
function sourceValidationScript(commands) {
|
||||
return [
|
||||
"#!/bin/sh",
|
||||
"set -eu",
|
||||
"git config --global --add safe.directory /workspace/source/repo",
|
||||
"cd /workspace/source/repo",
|
||||
"test \"$(git rev-parse HEAD)\" = \"$(params.revision)\"",
|
||||
...commands
|
||||
].join("\n") + "\n";
|
||||
}
|
||||
|
||||
function primitiveValidationTaskNames() {
|
||||
return primitiveValidationTasks.map((task) => task.name);
|
||||
}
|
||||
|
||||
function primitiveValidationTask(task) {
|
||||
return {
|
||||
name: task.name,
|
||||
runAfter: ["prepare-source"],
|
||||
workspaces: [{ name: "source", workspace: "source" }],
|
||||
taskSpec: {
|
||||
params: [
|
||||
{ name: "revision" },
|
||||
{ name: "lane" },
|
||||
{ name: "catalog-path" },
|
||||
{ name: "image-tag-mode" },
|
||||
{ name: "source-branch" },
|
||||
{ name: "gitops-branch" }
|
||||
],
|
||||
workspaces: [{ name: "source" }],
|
||||
steps: [{ name: task.name, image: ciToolsRunnerImage, env: proxyEnv(), script: sourceValidationScript(task.commands) }]
|
||||
},
|
||||
params: [
|
||||
{ name: "revision", value: "$(params.revision)" },
|
||||
{ name: "lane", value: "$(params.lane)" },
|
||||
{ name: "catalog-path", value: "$(params.catalog-path)" },
|
||||
{ name: "image-tag-mode", value: "$(params.image-tag-mode)" },
|
||||
{ name: "source-branch", value: "$(params.source-branch)" },
|
||||
{ name: "gitops-branch", value: "$(params.gitops-branch)" }
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function planArtifactsScript() {
|
||||
return renderTemplate("plan-artifacts.sh", {
|
||||
"__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage,
|
||||
'["__HWLAB_DEFAULT_SERVICE_IDS__"]': JSON.stringify(defaultServiceIds)
|
||||
});
|
||||
}
|
||||
|
||||
function perServicePublishScript() {
|
||||
return renderTemplate("per-service-publish.sh", {
|
||||
"# __HWLAB_CI_TIMING_SHELL__\n": ciTimingShellFunction(),
|
||||
"# __HWLAB_DEPENDENCY_PROXY_PROBE__": dependencyProxyProbeScript(
|
||||
"service-image-publish-proxy-preflight",
|
||||
"http://deb.debian.org/debian/dists/bookworm/InRelease"
|
||||
),
|
||||
"__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage
|
||||
});
|
||||
}
|
||||
|
||||
function collectArtifactsScript() {
|
||||
return renderTemplate("collect-artifacts.sh", {
|
||||
"__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage
|
||||
});
|
||||
}
|
||||
|
||||
function gitopsPromoteScript() {
|
||||
return renderTemplate("gitops-promote.sh", {
|
||||
"# __HWLAB_CI_TIMING_SHELL__\n": ciTimingShellFunction(),
|
||||
"# __HWLAB_GIT_SSH_SHELL__\n": gitSshShellFunction(),
|
||||
"__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage
|
||||
});
|
||||
}
|
||||
|
||||
function controlPlaneReconcileScript() {
|
||||
return renderTemplate("control-plane-reconcile.sh", {
|
||||
"# __HWLAB_DEPENDENCY_PROXY_PROBE__": dependencyProxyProbeScript(
|
||||
"control-plane-proxy-preflight",
|
||||
"http://deb.debian.org/debian/dists/bookworm/InRelease"
|
||||
),
|
||||
"# __HWLAB_GIT_SSH_SHELL__\n": gitSshShellFunction(),
|
||||
"__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage
|
||||
});
|
||||
}
|
||||
|
||||
function pollerScript() {
|
||||
return renderTemplate("poller.sh", {
|
||||
"# __HWLAB_DEPENDENCY_PROXY_PROBE__": dependencyProxyProbeScript(
|
||||
"poller-proxy-preflight",
|
||||
"http://deb.debian.org/debian/dists/bookworm/InRelease"
|
||||
),
|
||||
"# __HWLAB_GIT_SSH_SHELL__\n": gitSshShellFunction(),
|
||||
"__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage
|
||||
});
|
||||
}
|
||||
|
||||
export {
|
||||
ciTimingShellFunction,
|
||||
collectArtifactsScript,
|
||||
controlPlaneReconcileScript,
|
||||
curlDownloadProbeFunction,
|
||||
dependencyProxyProbeScript,
|
||||
gitopsPromoteScript,
|
||||
gitSshShellFunction,
|
||||
perServicePublishScript,
|
||||
planArtifactsScript,
|
||||
pollerScript,
|
||||
prepareSourceScript,
|
||||
primitiveValidationTask,
|
||||
primitiveValidationTaskNames,
|
||||
renderTemplate,
|
||||
shellSingleQuote,
|
||||
sourceValidationScript
|
||||
};
|
||||
@@ -0,0 +1,13 @@
|
||||
ci_now_ms() {
|
||||
node -e 'console.log(Date.now())'
|
||||
}
|
||||
|
||||
ci_timing_emit() {
|
||||
stage="$1"
|
||||
status="$2"
|
||||
started_ms="$3"
|
||||
finished_ms="$(ci_now_ms)"
|
||||
duration_ms="$((finished_ms - started_ms))"
|
||||
service_id="${HWLAB_TIMING_SERVICE_ID:-}"
|
||||
node -e 'const [stage,status,durationMs,serviceId]=process.argv.slice(1); const payload={event:"node-cicd-timing",schemaVersion:"v1",stage,status,durationMs:Number(durationMs),pipelineRun:process.env.HWLAB_TEKTON_PIPELINERUN||null,taskRun:process.env.HWLAB_TEKTON_TASKRUN||null,task:process.env.HWLAB_TEKTON_TASK||null,revision:process.env.HWLAB_SOURCE_REVISION||null,serviceId:serviceId||null,source:"scripts/gitops-render.mjs",at:new Date().toISOString()}; console.log(JSON.stringify(payload));' "$stage" "$status" "$duration_ms" "$service_id"
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
echo '{"event":"ci-base-image","phase":"collect-artifacts","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apk"}'
|
||||
for tool in node git; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"collect-artifacts","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done
|
||||
cd /workspace/source/repo
|
||||
test "$(git rev-parse HEAD)" = "$(params.revision)"
|
||||
export HWLAB_CI_ARTIFACT_RUN_ID="$(context.pipelineRun.name)-collect"
|
||||
export HWLAB_CI_ARTIFACT_RUN_OWNER="tekton"
|
||||
export HWLAB_DEV_REGISTRY_PREFIX="$(params.registry-prefix)"
|
||||
export HWLAB_DEV_REGISTRY_K8S_NATIVE=1
|
||||
if [ -n "$(params.base-image)" ]; then export HWLAB_DEV_BASE_IMAGE="$(params.base-image)"; fi
|
||||
export HWLAB_ARTIFACT_LANE="$(params.lane)"
|
||||
export HWLAB_ARTIFACT_CATALOG_PATH="$(params.catalog-path)"
|
||||
export HWLAB_DEPLOY_MANIFEST_PATH="deploy/deploy.yaml"
|
||||
export HWLAB_ARTIFACT_IMAGE_TAG_MODE="$(params.image-tag-mode)"
|
||||
if [ -s /workspace/source/affected-services.json ] && node - /workspace/source/affected-services.json <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const plan = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
||||
const buildServices = Array.isArray(plan.buildServices) ? plan.buildServices : [];
|
||||
const affectedServices = Array.isArray(plan.affectedServices) ? plan.affectedServices : [];
|
||||
const willRunGitopsPromote = plan.ciCdPlan && plan.ciCdPlan.willRunGitopsPromote === true;
|
||||
process.exit(!willRunGitopsPromote && buildServices.length === 0 && affectedServices.length === 0 ? 0 : 1);
|
||||
NODE
|
||||
then
|
||||
rm -f /workspace/source/dev-artifacts.json
|
||||
echo '{"event":"collect-artifacts","status":"skipped","reason":"no-build-no-rollout-plan"}'
|
||||
exit 0
|
||||
fi
|
||||
HWLAB_CI_PLAN_VERIFY_REUSE_REGISTRY=1 node scripts/artifact-publish.mjs --publish --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --registry-prefix "$(params.registry-prefix)" --report /workspace/source/dev-artifacts.json --quiet-build --concurrency 1 --services "$(params.services)" --external-service-report-dir /workspace/source/service-results --ci-plan-path /workspace/source/affected-services.json
|
||||
node scripts/refresh-artifact-catalog.mjs --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --target-ref HEAD --publish-report /workspace/source/dev-artifacts.json --no-write
|
||||
@@ -0,0 +1,139 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# __HWLAB_DEPENDENCY_PROXY_PROBE__
|
||||
echo '{"event":"ci-base-image","phase":"control-plane-reconciler","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apk"}'
|
||||
for tool in node npm git python3 ssh curl timeout; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"control-plane-reconciler","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done
|
||||
# __HWLAB_GIT_SSH_SHELL__
|
||||
|
||||
git_ssh_setup
|
||||
workdir="$(mktemp -d)"
|
||||
git_timed control-plane-clone 180 git clone --depth 1 --branch "$SOURCE_BRANCH" "$GIT_READ_URL" "$workdir/repo"
|
||||
cd "$workdir/repo"
|
||||
git remote set-url origin "$GIT_READ_URL"
|
||||
revision="$(git rev-parse HEAD)"
|
||||
: "${LANE:?LANE is required}"
|
||||
: "${CATALOG_PATH:?CATALOG_PATH is required}"
|
||||
: "${IMAGE_TAG_MODE:?IMAGE_TAG_MODE is required}"
|
||||
: "${RUNTIME_PATH:?RUNTIME_PATH is required}"
|
||||
gitops_root="${RUNTIME_PATH%/*}"
|
||||
if [ "$gitops_root" = "$RUNTIME_PATH" ]; then gitops_root="."; fi
|
||||
node scripts/run-bun.mjs scripts/gitops-render.mjs --lane "$LANE" --catalog-path "$CATALOG_PATH" --image-tag-mode "$IMAGE_TAG_MODE" --source-revision "$revision" --source-repo "$GIT_URL" --source-branch "$SOURCE_BRANCH" --gitops-branch "$GITOPS_BRANCH" --gitops-root "$gitops_root" --out "$gitops_root" --registry-prefix "$REGISTRY_PREFIX"
|
||||
node scripts/run-bun.mjs scripts/gitops-render.mjs --lane "$LANE" --catalog-path "$CATALOG_PATH" --image-tag-mode "$IMAGE_TAG_MODE" --source-revision "$revision" --source-repo "$GIT_URL" --source-branch "$SOURCE_BRANCH" --gitops-branch "$GITOPS_BRANCH" --gitops-root "$gitops_root" --out "$gitops_root" --registry-prefix "$REGISTRY_PREFIX" --check
|
||||
node <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const https = require("node:https");
|
||||
|
||||
function requiredEnv(name) {
|
||||
const value = process.env[name];
|
||||
if (!value) throw new Error(name + " is required");
|
||||
return value;
|
||||
}
|
||||
|
||||
const namespace = requiredEnv("POD_NAMESPACE");
|
||||
const host = process.env.KUBERNETES_SERVICE_HOST;
|
||||
const port = process.env.KUBERNETES_SERVICE_PORT || "443";
|
||||
if (!host) throw new Error("KUBERNETES_SERVICE_HOST is not available");
|
||||
|
||||
const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8");
|
||||
const ca = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt");
|
||||
const tektonDir = requiredEnv("TEKTON_DIR");
|
||||
const argoFiles = String(process.env.ARGO_FILES || "").split(",").map((item) => item.trim()).filter(Boolean);
|
||||
const fieldManager = requiredEnv("FIELD_MANAGER");
|
||||
const eventName = requiredEnv("RECONCILE_EVENT");
|
||||
const manifestFiles = [
|
||||
"deploy/gitops/node/" + tektonDir + "/rbac.yaml",
|
||||
"deploy/gitops/node/" + tektonDir + "/pipeline.yaml",
|
||||
...(process.env.LANE === "v02" ? [] : [
|
||||
"deploy/gitops/node/" + tektonDir + "/poller.yaml",
|
||||
"deploy/gitops/node/" + tektonDir + "/control-plane-reconciler.yaml"
|
||||
]),
|
||||
...argoFiles
|
||||
];
|
||||
const plurals = new Map([
|
||||
["ServiceAccount", "serviceaccounts"],
|
||||
["Role", "roles"],
|
||||
["RoleBinding", "rolebindings"],
|
||||
["Pipeline", "pipelines"],
|
||||
["CronJob", "cronjobs"],
|
||||
["Application", "applications"],
|
||||
["AppProject", "appprojects"]
|
||||
]);
|
||||
|
||||
function encode(value) {
|
||||
return encodeURIComponent(String(value));
|
||||
}
|
||||
|
||||
function itemsFrom(file) {
|
||||
const parsed = JSON.parse(fs.readFileSync(file, "utf8"));
|
||||
return parsed.kind === "List" ? parsed.items || [] : [parsed];
|
||||
}
|
||||
|
||||
function apiPath(item) {
|
||||
if (item.kind === "Namespace") return null;
|
||||
const name = item.metadata?.name;
|
||||
if (!name) throw new Error("manifest item is missing metadata.name");
|
||||
const ns = item.metadata?.namespace || namespace;
|
||||
const plural = plurals.get(item.kind);
|
||||
if (!plural) throw new Error("unsupported reconciler manifest kind " + item.kind);
|
||||
if (item.apiVersion === "v1") return "/api/v1/namespaces/" + encode(ns) + "/" + plural + "/" + encode(name);
|
||||
const parts = String(item.apiVersion || "").split("/");
|
||||
if (parts.length !== 2) throw new Error("unsupported apiVersion " + item.apiVersion + " for " + item.kind);
|
||||
return "/apis/" + encode(parts[0]) + "/" + encode(parts[1]) + "/namespaces/" + encode(ns) + "/" + plural + "/" + encode(name);
|
||||
}
|
||||
|
||||
function reconcilePolicy(item) {
|
||||
const ns = item.metadata?.namespace || namespace;
|
||||
const crossNamespaceRbac = (item.kind === "Role" || item.kind === "RoleBinding") && ns !== namespace;
|
||||
if (crossNamespaceRbac && process.env.HWLAB_RECONCILE_CROSS_NAMESPACE_RBAC !== "1") {
|
||||
return { action: "skip", reason: "cross-namespace-rbac-bootstrap-managed", namespace: ns };
|
||||
}
|
||||
return { action: "apply", namespace: ns };
|
||||
}
|
||||
|
||||
function request(method, path, body) {
|
||||
const payload = body ? JSON.stringify(body) : "";
|
||||
const headers = { Authorization: "Bearer " + token };
|
||||
if (payload) {
|
||||
headers["Content-Type"] = "application/apply-patch+yaml";
|
||||
headers["Content-Length"] = Buffer.byteLength(payload);
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = https.request({ host, port, method, path, ca, headers }, (res) => {
|
||||
let data = "";
|
||||
res.setEncoding("utf8");
|
||||
res.on("data", (chunk) => { data += chunk; });
|
||||
res.on("end", () => resolve({ statusCode: res.statusCode || 0, body: data }));
|
||||
});
|
||||
req.on("error", reject);
|
||||
if (payload) req.write(payload);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const results = [];
|
||||
for (const file of manifestFiles) {
|
||||
for (const item of itemsFrom(file)) {
|
||||
const policy = reconcilePolicy(item);
|
||||
if (policy.action === "skip") {
|
||||
results.push({ file, kind: item.kind, name: item.metadata?.name || null, namespace: policy.namespace, status: "skipped", reason: policy.reason });
|
||||
continue;
|
||||
}
|
||||
const path = apiPath(item);
|
||||
if (!path) {
|
||||
results.push({ file, kind: item.kind, name: item.metadata?.name || null, status: "skipped-cluster-scoped" });
|
||||
continue;
|
||||
}
|
||||
const response = await request("PATCH", path + "?fieldManager=" + encode(fieldManager) + "&force=true", item);
|
||||
if (response.statusCode < 200 || response.statusCode > 299) {
|
||||
throw new Error("apply failed for " + item.kind + "/" + item.metadata.name + " status=" + response.statusCode + " body=" + response.body.slice(0, 1000));
|
||||
}
|
||||
results.push({ file, kind: item.kind, name: item.metadata.name, status: "applied", statusCode: response.statusCode });
|
||||
}
|
||||
}
|
||||
console.log(JSON.stringify({ event: eventName, sourceRevision: process.env.RECONCILE_REVISION || null, results }, null, 2));
|
||||
})().catch((error) => {
|
||||
console.error(error.stack || error.message);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
NODE
|
||||
@@ -0,0 +1,17 @@
|
||||
redact_proxy_value() {
|
||||
printf '%s' "${1:-}" | sed -E 's#(https?://)[^/@]+@#\1***@#; s#(socks5h?://)[^/@]+@#\1***@#'
|
||||
}
|
||||
|
||||
curl_dependency_probe() {
|
||||
phase="$1"
|
||||
url="$2"
|
||||
proxy="${HTTPS_PROXY:-${https_proxy:-${HTTP_PROXY:-${http_proxy:-}}}}"
|
||||
if [ -z "$proxy" ]; then
|
||||
echo '{"event":"dependency-curl-probe","phase":"'"$phase"'","ok":false,"reason":"proxy-env-missing"}'
|
||||
return 21
|
||||
fi
|
||||
safe_proxy="$(redact_proxy_value "$proxy")"
|
||||
echo '{"event":"dependency-curl-probe-start","phase":"'"$phase"'","proxy":"'"$safe_proxy"'","target":"'"$url"'"}'
|
||||
curl -sS -L --range 0-1048575 -o /dev/null --connect-timeout 15 --max-time 60 --proxy "$proxy" -w '{"event":"dependency-curl-probe","phase":"'"$phase"'","http_code":"%{http_code}","time_namelookup":%{time_namelookup},"time_connect":%{time_connect},"time_starttransfer":%{time_starttransfer},"time_total":%{time_total},"speed_download":%{speed_download},"size_download":%{size_download},"remote_ip":"%{remote_ip}"}
|
||||
' "$url"
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
HWLAB_PROXY_PROBE_PHASE=__HWLAB_PROXY_PHASE_SHELL__ HWLAB_PROXY_PROBE_URL=__HWLAB_PROXY_TARGET_SHELL__ node <<'NODE' || echo '{"event":"dependency-proxy-probe-nonblocking","phase":"__HWLAB_PROXY_PHASE__","ok":false,"reason":"probe-failed-but-continuing"}'
|
||||
const net = require("node:net");
|
||||
|
||||
const phase = process.env.HWLAB_PROXY_PROBE_PHASE || "unknown";
|
||||
const target = new URL(process.env.HWLAB_PROXY_PROBE_URL || "http://deb.debian.org/debian/dists/bookworm/InRelease");
|
||||
const proxyRaw = process.env.HTTP_PROXY || process.env.http_proxy || "";
|
||||
|
||||
function redactProxy(value) {
|
||||
if (!value) return "";
|
||||
try {
|
||||
const parsed = new URL(value);
|
||||
if (parsed.username || parsed.password) {
|
||||
parsed.username = "***";
|
||||
parsed.password = "";
|
||||
}
|
||||
return parsed.toString();
|
||||
} catch {
|
||||
return "<invalid-proxy-url>";
|
||||
}
|
||||
}
|
||||
|
||||
function emit(payload, exitCode = 0) {
|
||||
console.log(JSON.stringify({
|
||||
event: "dependency-proxy-probe",
|
||||
phase,
|
||||
target: target.href,
|
||||
proxy: redactProxy(proxyRaw),
|
||||
...payload
|
||||
}));
|
||||
if (exitCode) process.exit(exitCode);
|
||||
}
|
||||
|
||||
if (!proxyRaw) emit({ ok: false, reason: "proxy-env-missing" }, 21);
|
||||
|
||||
let proxy;
|
||||
try {
|
||||
proxy = new URL(proxyRaw);
|
||||
} catch (error) {
|
||||
emit({ ok: false, reason: "proxy-url-invalid", error: error.message }, 22);
|
||||
}
|
||||
|
||||
if (proxy.protocol !== "http:" && proxy.protocol !== "https:") {
|
||||
emit({ ok: false, reason: "http-proxy-required", protocol: proxy.protocol }, 23);
|
||||
}
|
||||
|
||||
const startedAt = Date.now();
|
||||
const socket = net.createConnection({ host: proxy.hostname, port: Number(proxy.port || (proxy.protocol === "https:" ? 443 : 80)) });
|
||||
let bytes = 0;
|
||||
let firstByteMs = null;
|
||||
let responseHead = "";
|
||||
let finished = false;
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
socket.destroy();
|
||||
emit({ ok: false, reason: "timeout", timeoutMs: 15000 }, 24);
|
||||
}, 15000);
|
||||
|
||||
socket.on("connect", () => {
|
||||
socket.write("GET " + target.href + " HTTP/1.1\r\nHost: " + target.host + "\r\nUser-Agent: hwlab-node-proxy-probe\r\nConnection: close\r\n\r\n");
|
||||
});
|
||||
|
||||
socket.on("data", (chunk) => {
|
||||
if (firstByteMs === null) firstByteMs = Date.now() - startedAt;
|
||||
bytes += chunk.length;
|
||||
if (responseHead.length < 240) responseHead += chunk.toString("utf8", 0, Math.min(chunk.length, 240 - responseHead.length));
|
||||
});
|
||||
|
||||
socket.on("end", () => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
clearTimeout(timeout);
|
||||
const totalMs = Date.now() - startedAt;
|
||||
const statusLine = responseHead.split("\r\n", 1)[0] || "";
|
||||
const statusCode = Number(statusLine.split(" ")[1] || 0);
|
||||
const ok = statusLine.startsWith("HTTP/") && statusCode >= 200 && statusCode < 400;
|
||||
emit({
|
||||
ok,
|
||||
reason: ok ? null : "bad-http-status",
|
||||
statusLine,
|
||||
statusCode,
|
||||
firstByteMs,
|
||||
totalMs,
|
||||
bytes,
|
||||
speedBytesPerSecond: totalMs > 0 ? Math.round(bytes * 1000 / totalMs) : 0
|
||||
}, ok ? 0 : 25);
|
||||
});
|
||||
|
||||
socket.on("error", (error) => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
clearTimeout(timeout);
|
||||
emit({ ok: false, reason: "proxy-connect-failed", error: error.message, totalMs: Date.now() - startedAt }, 26);
|
||||
});
|
||||
NODE
|
||||
@@ -0,0 +1,76 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
repo_url="ssh://git@ssh.github.com:443/pikasTech/HWLAB.git"
|
||||
repo_path="/cache/pikasTech/HWLAB.git"
|
||||
lock_dir="/cache/.hwlab-flush.lock"
|
||||
started_ms=$(node -e 'console.log(Date.now())')
|
||||
now_ms() { node -e 'console.log(Date.now())'; }
|
||||
emit_timing() {
|
||||
phase="$1"
|
||||
status="$2"
|
||||
started="$3"
|
||||
finished=$(now_ms)
|
||||
duration=$((finished - started))
|
||||
printf '{"event":"git-mirror-flush-timing","phase":"%s","status":"%s","durationMs":%s,"repository":"pikasTech/HWLAB"}
|
||||
' "$phase" "$status" "$duration"
|
||||
}
|
||||
if ! mkdir "$lock_dir" 2>/dev/null; then
|
||||
echo "git mirror flush already running"
|
||||
exit 0
|
||||
fi
|
||||
trap 'rmdir "$lock_dir" 2>/dev/null || true' EXIT INT TERM
|
||||
mkdir -p /root/.ssh
|
||||
cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa
|
||||
chmod 0400 /root/.ssh/id_rsa
|
||||
# __HWLAB_GIT_MIRROR_SSH_PRELUDE__
|
||||
/script/install-hooks.sh
|
||||
for gitops_branch in __HWLAB_GITOPS_BRANCH_ARGS__; do
|
||||
local_head=$(git -C "$repo_path" show-ref --verify --hash "refs/heads/$gitops_branch" 2>/dev/null || true)
|
||||
if [ -z "$local_head" ]; then
|
||||
echo "git mirror flush skips missing local refs/heads/$gitops_branch"
|
||||
continue
|
||||
fi
|
||||
fetch_started=$(now_ms)
|
||||
timeout 90 git -C "$repo_path" fetch --quiet "$repo_url" "refs/heads/$gitops_branch:refs/mirror-stage/heads/$gitops_branch"
|
||||
emit_timing "fetch-$gitops_branch" succeeded "$fetch_started"
|
||||
github_head=$(git -C "$repo_path" show-ref --verify --hash "refs/mirror-stage/heads/$gitops_branch" 2>/dev/null || true)
|
||||
if [ -n "$github_head" ] && [ "$github_head" != "$local_head" ] && ! git -C "$repo_path" merge-base --is-ancestor "$github_head" "$local_head"; then
|
||||
echo "git mirror flush rejected divergent $gitops_branch local=$local_head github=$github_head" >&2
|
||||
exit 52
|
||||
fi
|
||||
if [ "$local_head" != "$github_head" ]; then
|
||||
push_started=$(now_ms)
|
||||
timeout 120 git -C "$repo_path" push "$repo_url" "$local_head:refs/heads/$gitops_branch"
|
||||
emit_timing "push-$gitops_branch" succeeded "$push_started"
|
||||
else
|
||||
echo "git mirror flush $gitops_branch already matches GitHub"
|
||||
fi
|
||||
git -C "$repo_path" update-ref "refs/mirror-stage/heads/$gitops_branch" "$local_head"
|
||||
done
|
||||
git -C "$repo_path" update-server-info
|
||||
flushed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
export flushed_at
|
||||
export gitops_branches_json=__HWLAB_GITOPS_BRANCHES_JSON_SHELL__
|
||||
node - <<'NODE' | tee /cache/HWLAB.last-flush.json
|
||||
const { execFileSync } = require("node:child_process");
|
||||
const repo = "/cache/pikasTech/HWLAB.git";
|
||||
const gitopsBranches = JSON.parse(process.env.gitops_branches_json || "[]");
|
||||
function rev(ref) {
|
||||
try { return execFileSync("git", ["-C", repo, "rev-parse", ref], { encoding: "utf8" }).trim() || null; } catch { return null; }
|
||||
}
|
||||
const refs = {};
|
||||
for (const branch of gitopsBranches) {
|
||||
refs["refs/heads/" + branch] = rev("refs/heads/" + branch);
|
||||
refs["refs/mirror-stage/heads/" + branch] = rev("refs/mirror-stage/heads/" + branch);
|
||||
}
|
||||
const pendingFlush = gitopsBranches.some((branch) => refs["refs/heads/" + branch] && refs["refs/mirror-stage/heads/" + branch] && refs["refs/heads/" + branch] !== refs["refs/mirror-stage/heads/" + branch]);
|
||||
console.log(JSON.stringify({
|
||||
event: "git-mirror-flush",
|
||||
status: "flushed",
|
||||
repository: "pikasTech/HWLAB",
|
||||
flushedAt: process.env.flushed_at,
|
||||
pendingFlush,
|
||||
refs
|
||||
}));
|
||||
NODE
|
||||
emit_timing total succeeded "$started_ms"
|
||||
@@ -0,0 +1,196 @@
|
||||
import { createServer } from "node:http";
|
||||
import { spawn } from "node:child_process";
|
||||
import { createReadStream, statSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { createGunzip, createInflate } from "node:zlib";
|
||||
|
||||
const cacheRoot = process.env.GIT_MIRROR_CACHE_ROOT || "/cache";
|
||||
const port = Number(process.env.PORT || 8080);
|
||||
|
||||
createServer((req, res) => {
|
||||
handle(req, res).catch((error) => {
|
||||
if (!res.headersSent) res.writeHead(500, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify({ ok: false, error: error.message }));
|
||||
});
|
||||
}).listen(port, "0.0.0.0", () => {
|
||||
console.log(JSON.stringify({ event: "git-mirror-smart-http-listening", port, cacheRoot }));
|
||||
});
|
||||
|
||||
async function handle(req, res) {
|
||||
const url = new URL(req.url || "/", "http://git-mirror-http");
|
||||
const writeHost = isWriteHost(req.headers.host || "");
|
||||
if (req.method === "GET" && (url.pathname === "/" || url.pathname === "/health")) {
|
||||
res.writeHead(200, { "content-type": "application/json", "cache-control": "no-cache" });
|
||||
res.end(JSON.stringify({ ok: true, service: "git-mirror-smart-http", writeHost }));
|
||||
return;
|
||||
}
|
||||
if (req.method === "GET" && url.pathname.endsWith("/info/refs") && url.searchParams.get("service") === "git-upload-pack") {
|
||||
await runUploadPackAdvertisement(repoPathFromGitServicePath(url.pathname, "/info/refs"), res);
|
||||
return;
|
||||
}
|
||||
if (req.method === "POST" && url.pathname.endsWith("/git-upload-pack")) {
|
||||
await runUploadPackRpc(repoPathFromGitServicePath(url.pathname, "/git-upload-pack"), req, res);
|
||||
return;
|
||||
}
|
||||
if (req.method === "GET" && url.pathname.endsWith("/info/refs") && url.searchParams.get("service") === "git-receive-pack") {
|
||||
if (!writeHost) return rejectReadOnly(res);
|
||||
await runReceivePackAdvertisement(repoPathFromGitServicePath(url.pathname, "/info/refs"), res);
|
||||
return;
|
||||
}
|
||||
if (req.method === "POST" && url.pathname.endsWith("/git-receive-pack")) {
|
||||
if (!writeHost) return rejectReadOnly(res);
|
||||
await runReceivePackRpc(repoPathFromGitServicePath(url.pathname, "/git-receive-pack"), req, res);
|
||||
return;
|
||||
}
|
||||
if (req.method === "GET" || req.method === "HEAD") {
|
||||
serveStaticFile(url.pathname, req, res);
|
||||
return;
|
||||
}
|
||||
res.writeHead(405, { "content-type": "text/plain" });
|
||||
res.end("method not allowed\n");
|
||||
}
|
||||
|
||||
function isWriteHost(hostHeader) {
|
||||
const host = String(hostHeader || "").split(":", 1)[0].toLowerCase();
|
||||
return host === "git-mirror-write" || host.startsWith("git-mirror-write.");
|
||||
}
|
||||
|
||||
function rejectReadOnly(res) {
|
||||
res.writeHead(403, { "content-type": "application/json", "cache-control": "no-cache" });
|
||||
res.end(JSON.stringify({ ok: false, error: "git mirror receive-pack is only available through git-mirror-write" }));
|
||||
}
|
||||
|
||||
function repoPathFromGitServicePath(urlPath, suffix) {
|
||||
const repoUrlPath = urlPath.slice(0, -suffix.length);
|
||||
if (!repoUrlPath.endsWith(".git")) throw new Error(`unsupported git repo path: ${urlPath}`);
|
||||
return safePath(repoUrlPath);
|
||||
}
|
||||
|
||||
function safePath(urlPath) {
|
||||
const decoded = decodeURIComponent(urlPath.replace(/^[/]+/u, ""));
|
||||
const fullPath = path.resolve(cacheRoot, decoded);
|
||||
const normalizedRoot = path.resolve(cacheRoot);
|
||||
if (fullPath !== normalizedRoot && !fullPath.startsWith(`${normalizedRoot}${path.sep}`)) throw new Error("path escapes cache root");
|
||||
return fullPath;
|
||||
}
|
||||
|
||||
function smartHeaders(contentType) {
|
||||
return {
|
||||
"content-type": contentType,
|
||||
"cache-control": "no-cache, max-age=0, must-revalidate",
|
||||
expires: "Fri, 01 Jan 1980 00:00:00 GMT",
|
||||
pragma: "no-cache"
|
||||
};
|
||||
}
|
||||
|
||||
function pktLine(text) {
|
||||
const length = Buffer.byteLength(text) + 4;
|
||||
return `${length.toString(16).padStart(4, "0")}${text}`;
|
||||
}
|
||||
|
||||
function requestBodyStream(req) {
|
||||
const encoding = String(req.headers["content-encoding"] || "identity").toLowerCase();
|
||||
if (encoding === "identity" || encoding === "") return req;
|
||||
if (encoding === "gzip" || encoding === "x-gzip") return req.pipe(createGunzip());
|
||||
if (encoding === "deflate") return req.pipe(createInflate());
|
||||
throw new Error(`unsupported git upload-pack content-encoding: ${encoding}`);
|
||||
}
|
||||
|
||||
function runUploadPackAdvertisement(repoPath, res) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn("git-upload-pack", ["--stateless-rpc", "--advertise-refs", repoPath], { stdio: ["ignore", "pipe", "pipe"] });
|
||||
let stderr = "";
|
||||
child.stderr.on("data", (chunk) => { stderr += chunk; process.stderr.write(chunk); });
|
||||
child.on("error", reject);
|
||||
child.on("close", (code) => {
|
||||
if (code !== 0 && !res.headersSent) reject(new Error(`git-upload-pack advertise failed: ${stderr.trim()}`));
|
||||
else resolve();
|
||||
});
|
||||
res.writeHead(200, smartHeaders("application/x-git-upload-pack-advertisement"));
|
||||
res.write(pktLine("# service=git-upload-pack\n"));
|
||||
res.write("0000");
|
||||
child.stdout.pipe(res, { end: false });
|
||||
child.stdout.on("end", () => res.end());
|
||||
});
|
||||
}
|
||||
|
||||
function runUploadPackRpc(repoPath, req, res) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn("git-upload-pack", ["--stateless-rpc", repoPath], { stdio: ["pipe", "pipe", "pipe"] });
|
||||
let stderr = "";
|
||||
child.stderr.on("data", (chunk) => { stderr += chunk; process.stderr.write(chunk); });
|
||||
child.on("error", reject);
|
||||
child.on("close", (code) => {
|
||||
if (code !== 0 && !res.headersSent) reject(new Error(`git-upload-pack rpc failed: ${stderr.trim()}`));
|
||||
else resolve();
|
||||
});
|
||||
res.writeHead(200, smartHeaders("application/x-git-upload-pack-result"));
|
||||
const body = requestBodyStream(req);
|
||||
body.on("error", (error) => child.stdin.destroy(error));
|
||||
body.pipe(child.stdin);
|
||||
child.stdout.pipe(res);
|
||||
});
|
||||
}
|
||||
|
||||
function runReceivePackAdvertisement(repoPath, res) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn("git-receive-pack", ["--stateless-rpc", "--advertise-refs", repoPath], { stdio: ["ignore", "pipe", "pipe"] });
|
||||
let stderr = "";
|
||||
child.stderr.on("data", (chunk) => { stderr += chunk; process.stderr.write(chunk); });
|
||||
child.on("error", reject);
|
||||
child.on("close", (code) => {
|
||||
if (code !== 0 && !res.headersSent) reject(new Error(`git-receive-pack advertise failed: ${stderr.trim()}`));
|
||||
else resolve();
|
||||
});
|
||||
res.writeHead(200, smartHeaders("application/x-git-receive-pack-advertisement"));
|
||||
res.write(pktLine("# service=git-receive-pack\n"));
|
||||
res.write("0000");
|
||||
child.stdout.pipe(res, { end: false });
|
||||
child.stdout.on("end", () => res.end());
|
||||
});
|
||||
}
|
||||
|
||||
function runReceivePackRpc(repoPath, req, res) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn("git-receive-pack", ["--stateless-rpc", repoPath], { stdio: ["pipe", "pipe", "pipe"] });
|
||||
let stderr = "";
|
||||
child.stderr.on("data", (chunk) => { stderr += chunk; process.stderr.write(chunk); });
|
||||
child.on("error", reject);
|
||||
child.on("close", (code) => {
|
||||
if (code !== 0 && !res.headersSent) reject(new Error(`git-receive-pack rpc failed: ${stderr.trim()}`));
|
||||
else resolve();
|
||||
});
|
||||
res.writeHead(200, smartHeaders("application/x-git-receive-pack-result"));
|
||||
const body = requestBodyStream(req);
|
||||
body.on("error", (error) => child.stdin.destroy(error));
|
||||
body.pipe(child.stdin);
|
||||
child.stdout.pipe(res);
|
||||
});
|
||||
}
|
||||
|
||||
function serveStaticFile(urlPath, req, res) {
|
||||
const fullPath = safePath(urlPath);
|
||||
let stat;
|
||||
try { stat = statSync(fullPath); } catch {
|
||||
res.writeHead(404, { "content-type": "text/plain" });
|
||||
res.end("not found\n");
|
||||
return;
|
||||
}
|
||||
if (!stat.isFile()) {
|
||||
res.writeHead(404, { "content-type": "text/plain" });
|
||||
res.end("not found\n");
|
||||
return;
|
||||
}
|
||||
res.writeHead(200, { "content-length": stat.size, "content-type": contentType(fullPath), "cache-control": "no-cache" });
|
||||
if (req.method === "HEAD") {
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
createReadStream(fullPath).pipe(res);
|
||||
}
|
||||
|
||||
function contentType(filePath) {
|
||||
if (filePath.endsWith(".pack")) return "application/x-git-packed-objects";
|
||||
if (filePath.endsWith(".idx")) return "application/x-git-packed-objects-toc";
|
||||
return "text/plain";
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
repo_path="/cache/pikasTech/HWLAB.git"
|
||||
mkdir -p "$repo_path/hooks"
|
||||
cat > "$repo_path/hooks/pre-receive" <<'HOOK'
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
zero="0000000000000000000000000000000000000000"
|
||||
status=0
|
||||
while read -r old new ref; do
|
||||
if [ "$new" = "$zero" ]; then
|
||||
echo "git mirror write rejected: deleting $ref is forbidden" >&2
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
git cat-file -e "$new^{commit}" || status=1
|
||||
git cat-file -e "$new^{tree}" || status=1
|
||||
if git rev-list --objects --missing=print "$new" | grep -q '^?'; then
|
||||
echo "git mirror write rejected: missing objects for $new" >&2
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
if [ "$old" != "$zero" ] && ! git merge-base --is-ancestor "$old" "$new"; then
|
||||
echo "git mirror write rejected: non-fast-forward $ref" >&2
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
done
|
||||
exit "$status"
|
||||
HOOK
|
||||
cat > "$repo_path/hooks/post-receive" <<'HOOK'
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
repo_path=$(git rev-parse --git-dir)
|
||||
git -C "$repo_path" update-server-info
|
||||
written_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
export repo_path written_at
|
||||
export gitops_branches_json=__HWLAB_GITOPS_BRANCHES_JSON_SHELL__
|
||||
node - <<'NODE' > /cache/HWLAB.last-write.json
|
||||
const { execFileSync } = require("node:child_process");
|
||||
const repo = process.env.repo_path || "/cache/pikasTech/HWLAB.git";
|
||||
const gitopsBranches = JSON.parse(process.env.gitops_branches_json || "[]");
|
||||
function rev(ref) {
|
||||
try { return execFileSync("git", ["-C", repo, "rev-parse", ref], { encoding: "utf8" }).trim() || null; } catch { return null; }
|
||||
}
|
||||
const refs = {};
|
||||
for (const branch of gitopsBranches) {
|
||||
refs["refs/heads/" + branch] = rev("refs/heads/" + branch);
|
||||
refs["refs/mirror-stage/heads/" + branch] = rev("refs/mirror-stage/heads/" + branch);
|
||||
}
|
||||
const pendingFlush = gitopsBranches.some((branch) => refs["refs/heads/" + branch] && refs["refs/heads/" + branch] !== refs["refs/mirror-stage/heads/" + branch]);
|
||||
console.log(JSON.stringify({
|
||||
event: "git-mirror-write",
|
||||
status: "received",
|
||||
repository: "pikasTech/HWLAB",
|
||||
writtenAt: process.env.written_at,
|
||||
pendingFlush,
|
||||
refs
|
||||
}));
|
||||
NODE
|
||||
HOOK
|
||||
chmod 0755 "$repo_path/hooks/pre-receive" "$repo_path/hooks/post-receive"
|
||||
git -C "$repo_path" config http.receivepack true
|
||||
git -C "$repo_path" config receive.denyDeletes true
|
||||
git -C "$repo_path" config receive.denyNonFastForwards true
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env node
|
||||
import net from "node:net";
|
||||
const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);
|
||||
const proxyPort = Number.parseInt(proxyPortRaw || "", 10);
|
||||
const targetPort = Number.parseInt(targetPortRaw || "", 10);
|
||||
if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {
|
||||
console.error("hwlab git-mirror proxy-connect: invalid ProxyCommand arguments");
|
||||
process.exit(64);
|
||||
}
|
||||
let settled = false;
|
||||
let tunnelEstablished = false;
|
||||
function finish(code, message) {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (message) console.error("hwlab git-mirror proxy-connect: " + message);
|
||||
process.exit(code);
|
||||
}
|
||||
const socket = net.createConnection({ host: proxyHost, port: proxyPort });
|
||||
let buffer = Buffer.alloc(0);
|
||||
socket.setTimeout(15000, () => { socket.destroy(); finish(65, "timeout connecting via " + proxyHost + ":" + proxyPort + " to " + targetHost + ":" + targetPort); });
|
||||
socket.on("connect", () => socket.write("CONNECT " + targetHost + ":" + targetPort + " HTTP/1.1\r\nHost: " + targetHost + ":" + targetPort + "\r\nProxy-Connection: Keep-Alive\r\n\r\n"));
|
||||
socket.on("error", (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? "tunnel socket error: " : "tcp error connecting to proxy: ") + (error && error.message ? error.message : String(error))));
|
||||
socket.on("close", () => { if (!tunnelEstablished) finish(68, "proxy closed before CONNECT completed via " + proxyHost + ":" + proxyPort + " to " + targetHost + ":" + targetPort); else finish(0); });
|
||||
function onData(chunk) {
|
||||
buffer = Buffer.concat([buffer, chunk]);
|
||||
const headerEnd = buffer.indexOf("\r\n\r\n");
|
||||
if (headerEnd === -1 && buffer.length < 8192) return;
|
||||
if (headerEnd === -1) { socket.destroy(); finish(68, "proxy response header exceeded 8192 bytes before CONNECT status via " + proxyHost + ":" + proxyPort + " to " + targetHost + ":" + targetPort); return; }
|
||||
const head = buffer.slice(0, headerEnd + 4).toString("latin1");
|
||||
const statusLine = head.split("\r\n", 1)[0] || "";
|
||||
const statusCode = Number.parseInt(statusLine.split(" ")[1] || "", 10);
|
||||
if (!statusLine.startsWith("HTTP/1.") || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {
|
||||
const safeStatus = statusLine.replace(/[^\x20-\x7e]/g, "?").slice(0, 160);
|
||||
socket.destroy();
|
||||
finish(67, "proxy CONNECT failed via " + proxyHost + ":" + proxyPort + " to " + targetHost + ":" + targetPort + ": " + safeStatus);
|
||||
return;
|
||||
}
|
||||
socket.off("data", onData);
|
||||
socket.setTimeout(0);
|
||||
tunnelEstablished = true;
|
||||
const rest = buffer.slice(headerEnd + 4);
|
||||
if (rest.length) process.stdout.write(rest);
|
||||
process.stdin.on("error", () => {});
|
||||
process.stdout.on("error", () => {});
|
||||
process.stdin.pipe(socket);
|
||||
socket.pipe(process.stdout);
|
||||
}
|
||||
socket.on("data", onData);
|
||||
@@ -0,0 +1,11 @@
|
||||
printf '%s\n' __HWLAB_PROXY_SUMMARY_SHELL__ >&2
|
||||
cat > /tmp/hwlab-git-ssh-proxy.sh <<'SH_PROXY'
|
||||
#!/bin/sh
|
||||
exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 "$@"
|
||||
SH_PROXY
|
||||
chmod 0700 /tmp/hwlab-git-ssh-proxy.sh
|
||||
unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy
|
||||
export NO_PROXY='*'
|
||||
export no_proxy='*'
|
||||
export GIT_SSH=/tmp/hwlab-git-ssh-proxy.sh
|
||||
unset GIT_SSH_COMMAND
|
||||
@@ -0,0 +1,20 @@
|
||||
printf '%s\n' __HWLAB_PROXY_SUMMARY_SHELL__ >&2
|
||||
cat > /tmp/hwlab-github-proxy-connect.mjs <<'NODE_PROXY'
|
||||
// __HWLAB_GIT_MIRROR_PROXY_CONNECT_MJS__
|
||||
NODE_PROXY
|
||||
chmod 0700 /tmp/hwlab-github-proxy-connect.mjs
|
||||
cat > /tmp/hwlab-git-ssh-proxy.sh <<'SH_PROXY'
|
||||
#!/bin/sh
|
||||
exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o __HWLAB_PROXY_COMMAND_SHELL__ "$@"
|
||||
SH_PROXY
|
||||
chmod 0700 /tmp/hwlab-git-ssh-proxy.sh
|
||||
export HTTP_PROXY=__HWLAB_PROXY_URL_SHELL__
|
||||
export HTTPS_PROXY=__HWLAB_PROXY_URL_SHELL__
|
||||
export ALL_PROXY=__HWLAB_PROXY_URL_SHELL__
|
||||
export http_proxy=__HWLAB_PROXY_URL_SHELL__
|
||||
export https_proxy=__HWLAB_PROXY_URL_SHELL__
|
||||
export all_proxy=__HWLAB_PROXY_URL_SHELL__
|
||||
export NO_PROXY=__HWLAB_PROXY_NO_PROXY_SHELL__
|
||||
export no_proxy=__HWLAB_PROXY_NO_PROXY_SHELL__
|
||||
export GIT_SSH=/tmp/hwlab-git-ssh-proxy.sh
|
||||
unset GIT_SSH_COMMAND
|
||||
@@ -0,0 +1,181 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
repo_url="ssh://git@ssh.github.com:443/pikasTech/HWLAB.git"
|
||||
repo_path="/cache/pikasTech/HWLAB.git"
|
||||
source_snapshot_ref_prefix=__HWLAB_SOURCE_SNAPSHOT_REF_PREFIX_SHELL__
|
||||
lock_dir="/cache/.hwlab-sync.lock"
|
||||
sync_started_ms=$(node -e 'console.log(Date.now())')
|
||||
now_ms() { node -e 'console.log(Date.now())'; }
|
||||
emit_timing() {
|
||||
phase="$1"
|
||||
status="$2"
|
||||
started_ms="$3"
|
||||
finished_ms=$(now_ms)
|
||||
duration_ms=$((finished_ms - started_ms))
|
||||
printf '{"event":"git-mirror-sync-timing","phase":"%s","status":"%s","durationMs":%s,"repository":"pikasTech/HWLAB"}
|
||||
' "$phase" "$status" "$duration_ms"
|
||||
}
|
||||
mkdir -p /cache/pikasTech /root/.ssh
|
||||
if ! mkdir "$lock_dir" 2>/dev/null; then
|
||||
echo "git mirror sync already running"
|
||||
exit 0
|
||||
fi
|
||||
trap 'rmdir "$lock_dir" 2>/dev/null || true' EXIT INT TERM
|
||||
cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa
|
||||
chmod 0400 /root/.ssh/id_rsa
|
||||
# __HWLAB_GIT_MIRROR_SSH_PRELUDE__
|
||||
if [ -d "$repo_path/objects" ] && [ -f "$repo_path/HEAD" ]; then
|
||||
git -C "$repo_path" remote set-url origin "$repo_url" || git -C "$repo_path" remote add origin "$repo_url"
|
||||
else
|
||||
rm -rf "$repo_path"
|
||||
git init --bare "$repo_path"
|
||||
git -C "$repo_path" remote add origin "$repo_url"
|
||||
fi
|
||||
/script/install-hooks.sh
|
||||
git -C "$repo_path" config uploadpack.hideRefs refs/mirror-stage
|
||||
git -C "$repo_path" config uploadpack.allowReachableSHA1InWant true
|
||||
git -C "$repo_path" config uploadpack.allowAnySHA1InWant true
|
||||
git -C "$repo_path" config --unset-all remote.origin.fetch 2>/dev/null || true
|
||||
# __HWLAB_FETCH_CONFIG_COMMANDS__
|
||||
fetch_started_ms=$(now_ms)
|
||||
timeout 180 git -C "$repo_path" fetch --quiet --prune origin
|
||||
emit_timing fetch succeeded "$fetch_started_ms"
|
||||
git -C "$repo_path" for-each-ref --format='%(objectname) %(refname)' refs/mirror-stage/heads > /tmp/hwlab-stage-heads
|
||||
git -C "$repo_path" for-each-ref --format='%(refname)' refs/mirror-stage/heads > /tmp/hwlab-stage-head-refs
|
||||
git -C "$repo_path" for-each-ref --format='%(objectname) %(refname)' refs/mirror-stage/tags > /tmp/hwlab-stage-tags
|
||||
git -C "$repo_path" for-each-ref --format='%(refname)' refs/mirror-stage/tags > /tmp/hwlab-stage-tag-refs
|
||||
if [ ! -s /tmp/hwlab-stage-heads ]; then
|
||||
echo "git mirror sync fetched no branch refs" >&2
|
||||
exit 41
|
||||
fi
|
||||
for required_ref in __HWLAB_REQUIRED_REF_ARGS__; do
|
||||
git -C "$repo_path" show-ref --verify --quiet "$required_ref" || { echo "git mirror sync missing required ref $required_ref" >&2; exit 43; }
|
||||
done
|
||||
validate_started_ms=$(now_ms)
|
||||
while read -r sha ref; do
|
||||
[ -n "$sha" ] || continue
|
||||
git -C "$repo_path" cat-file -e "$sha^{commit}"
|
||||
git -C "$repo_path" cat-file -e "$sha^{tree}"
|
||||
if git -C "$repo_path" rev-list --objects --missing=print "$sha" | grep -q '^?'; then
|
||||
echo "git mirror sync found missing objects for $ref $sha" >&2
|
||||
exit 42
|
||||
fi
|
||||
done < /tmp/hwlab-stage-heads
|
||||
emit_timing validate succeeded "$validate_started_ms"
|
||||
publish_started_ms=$(now_ms)
|
||||
while read -r sha ref; do
|
||||
[ -n "$sha" ] || continue
|
||||
name=$(printf '%s
|
||||
' "$ref" | sed 's#^refs/mirror-stage/heads/##')
|
||||
is_gitops_branch=false
|
||||
for gitops_branch in __HWLAB_GITOPS_BRANCH_ARGS__; do
|
||||
if [ "$name" = "$gitops_branch" ]; then is_gitops_branch=true; break; fi
|
||||
done
|
||||
if [ "$is_gitops_branch" = true ]; then
|
||||
local_sha=$(git -C "$repo_path" show-ref --verify --hash "refs/heads/$name" 2>/dev/null || true)
|
||||
if [ -n "$local_sha" ] && [ "$local_sha" != "$sha" ]; then
|
||||
if git -C "$repo_path" merge-base --is-ancestor "$sha" "$local_sha"; then
|
||||
echo "git mirror sync keeps local $name ahead of GitHub"
|
||||
continue
|
||||
fi
|
||||
if ! git -C "$repo_path" merge-base --is-ancestor "$local_sha" "$sha"; then
|
||||
echo "git mirror sync found divergent $name local=$local_sha github=$sha" >&2
|
||||
exit 44
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
git -C "$repo_path" update-ref "refs/heads/$name" "$sha"
|
||||
done < /tmp/hwlab-stage-heads
|
||||
for source_branch in __HWLAB_SOURCE_BRANCH_ARGS__; do
|
||||
source_sha=$(git -C "$repo_path" show-ref --verify --hash "refs/heads/$source_branch" 2>/dev/null || true)
|
||||
if [ -n "$source_sha" ]; then
|
||||
source_stage_ref="${source_snapshot_ref_prefix%/}/$source_branch/$source_sha"
|
||||
git -C "$repo_path" update-ref "$source_stage_ref" "$source_sha"
|
||||
fi
|
||||
done
|
||||
while read -r sha ref; do
|
||||
[ -n "$sha" ] || continue
|
||||
name=$(printf '%s
|
||||
' "$ref" | sed 's#^refs/mirror-stage/tags/##')
|
||||
git -C "$repo_path" update-ref "refs/tags/$name" "$sha"
|
||||
done < /tmp/hwlab-stage-tags
|
||||
git -C "$repo_path" for-each-ref --format='%(refname)' refs/heads > /tmp/hwlab-public-heads
|
||||
while read -r ref; do
|
||||
[ -n "$ref" ] || continue
|
||||
name=$(printf '%s
|
||||
' "$ref" | sed 's#^refs/heads/##')
|
||||
if ! grep -Fxq "refs/mirror-stage/heads/$name" /tmp/hwlab-stage-head-refs; then
|
||||
git -C "$repo_path" update-ref -d "$ref"
|
||||
fi
|
||||
done < /tmp/hwlab-public-heads
|
||||
git -C "$repo_path" for-each-ref --format='%(refname)' refs/tags > /tmp/hwlab-public-tags
|
||||
while read -r ref; do
|
||||
[ -n "$ref" ] || continue
|
||||
name=$(printf '%s
|
||||
' "$ref" | sed 's#^refs/tags/##')
|
||||
if ! grep -Fxq "refs/mirror-stage/tags/$name" /tmp/hwlab-stage-tag-refs; then
|
||||
git -C "$repo_path" update-ref -d "$ref"
|
||||
fi
|
||||
done < /tmp/hwlab-public-tags
|
||||
emit_timing publish succeeded "$publish_started_ms"
|
||||
for head_branch in __HWLAB_SOURCE_BRANCH_ARGS__; do
|
||||
if git -C "$repo_path" show-ref --verify --quiet "refs/heads/$head_branch"; then
|
||||
git -C "$repo_path" symbolic-ref HEAD "refs/heads/$head_branch"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fsck_started_ms=$(now_ms)
|
||||
git -C "$repo_path" fsck --connectivity-only --no-dangling >/tmp/hwlab-git-fsck.out
|
||||
emit_timing fsck succeeded "$fsck_started_ms"
|
||||
git -C "$repo_path" update-server-info
|
||||
published_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
for source_branch in __HWLAB_SOURCE_BRANCH_ARGS__; do
|
||||
git -C "$repo_path" rev-parse "refs/heads/$source_branch" 2>/dev/null | tee "/cache/HWLAB-$source_branch.head" >/dev/null || true
|
||||
done
|
||||
printf '%s
|
||||
' "$published_at" > /cache/HWLAB.last-sync
|
||||
export published_at
|
||||
export source_snapshot_ref_prefix
|
||||
export mirror_branches_json=__HWLAB_MIRROR_BRANCHES_JSON_SHELL__
|
||||
export source_branches_json=__HWLAB_SOURCE_BRANCHES_JSON_SHELL__
|
||||
export gitops_branches_json=__HWLAB_GITOPS_BRANCHES_JSON_SHELL__
|
||||
node - <<'NODE' | tee /cache/HWLAB.last-sync.json
|
||||
const { execFileSync } = require("node:child_process");
|
||||
const repo = "/cache/pikasTech/HWLAB.git";
|
||||
const mirrorBranches = JSON.parse(process.env.mirror_branches_json || "[]");
|
||||
const sourceBranches = JSON.parse(process.env.source_branches_json || "[]");
|
||||
const gitopsBranches = JSON.parse(process.env.gitops_branches_json || "[]");
|
||||
const sourceSnapshotRefPrefix = (process.env.source_snapshot_ref_prefix || "refs/unidesk/snapshots/hwlab-node-runtime").replace(//+$/u, "");
|
||||
function rev(ref) {
|
||||
try { return execFileSync("git", ["-C", repo, "rev-parse", ref], { encoding: "utf8" }).trim() || null; } catch { return null; }
|
||||
}
|
||||
const refs = {};
|
||||
for (const branch of mirrorBranches) {
|
||||
refs["refs/heads/" + branch] = rev("refs/heads/" + branch);
|
||||
refs["refs/mirror-stage/heads/" + branch] = rev("refs/mirror-stage/heads/" + branch);
|
||||
}
|
||||
const sourceSnapshots = {};
|
||||
for (const branch of sourceBranches) {
|
||||
const source = refs["refs/heads/" + branch] || null;
|
||||
const stageRef = source ? sourceSnapshotRefPrefix + "/" + branch + "/" + source : null;
|
||||
const sourceSnapshot = stageRef ? rev(stageRef) : null;
|
||||
if (stageRef) refs[stageRef] = sourceSnapshot;
|
||||
sourceSnapshots[branch] = { stageRef, sourceSnapshot };
|
||||
}
|
||||
const sourceInSync = sourceBranches.every((branch) => refs["refs/heads/" + branch] && refs["refs/heads/" + branch] === refs["refs/mirror-stage/heads/" + branch] && sourceSnapshots[branch]?.sourceSnapshot === refs["refs/mirror-stage/heads/" + branch]);
|
||||
const gitopsInSync = gitopsBranches.every((branch) => refs["refs/heads/" + branch] && refs["refs/heads/" + branch] === refs["refs/mirror-stage/heads/" + branch]);
|
||||
const pendingFlush = gitopsBranches.some((branch) => refs["refs/heads/" + branch] && refs["refs/mirror-stage/heads/" + branch] && refs["refs/heads/" + branch] !== refs["refs/mirror-stage/heads/" + branch]);
|
||||
const payload = {
|
||||
event: "git-mirror-sync",
|
||||
status: "published",
|
||||
repository: "pikasTech/HWLAB",
|
||||
publishedAt: process.env.published_at,
|
||||
pendingFlush,
|
||||
sourceInSync,
|
||||
gitopsInSync,
|
||||
sourceSnapshots,
|
||||
refs
|
||||
};
|
||||
console.log(JSON.stringify(payload));
|
||||
NODE
|
||||
emit_timing total succeeded "$sync_started_ms"
|
||||
@@ -0,0 +1,46 @@
|
||||
git_ssh_setup() {
|
||||
mkdir -p /root/.ssh
|
||||
cp /workspace/git-ssh/ssh-privatekey /root/.ssh/id_rsa
|
||||
chmod 600 /root/.ssh/id_rsa
|
||||
timeout 10 ssh-keyscan github.com >> /root/.ssh/known_hosts 2>/dev/null || true
|
||||
timeout 10 ssh-keyscan -p 443 ssh.github.com >> /root/.ssh/known_hosts 2>/dev/null || true
|
||||
write_github_proxy_command
|
||||
export GIT_SSH_COMMAND="ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ServerAliveInterval=10 -o ServerAliveCountMax=2 -o ProxyCommand='node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808 %h %p'"
|
||||
git config --global url."ssh://git@ssh.github.com:443/".insteadOf "git@github.com:"
|
||||
git config --global url."ssh://git@ssh.github.com:443/".insteadOf "ssh://git@github.com/"
|
||||
}
|
||||
|
||||
write_github_proxy_command() {
|
||||
cat > /tmp/hwlab-github-proxy-connect.mjs <<'NODE_PROXY'
|
||||
// __HWLAB_GITHUB_PROXY_CONNECT_MJS__
|
||||
NODE_PROXY
|
||||
chmod 0700 /tmp/hwlab-github-proxy-connect.mjs
|
||||
}
|
||||
|
||||
git_now_ms() {
|
||||
node -e 'console.log(Date.now())' 2>/dev/null || date +%s000
|
||||
}
|
||||
|
||||
git_timed() {
|
||||
phase="$1"
|
||||
timeout_seconds="$2"
|
||||
shift 2
|
||||
started_ms="$(git_now_ms)"
|
||||
echo '{"event":"git-operation","phase":"'"$phase"'","status":"started","timeoutSeconds":'"$timeout_seconds"'}' >&2
|
||||
set +e
|
||||
timeout "$timeout_seconds" "$@"
|
||||
status="$?"
|
||||
set -e
|
||||
finished_ms="$(git_now_ms)"
|
||||
duration_ms="$((finished_ms - started_ms))"
|
||||
if [ "$status" -eq 0 ]; then
|
||||
echo '{"event":"git-operation","phase":"'"$phase"'","status":"succeeded","durationMs":'"$duration_ms"'}' >&2
|
||||
return 0
|
||||
fi
|
||||
if [ "$status" -eq 124 ] || [ "$status" -eq 137 ]; then
|
||||
echo '{"event":"git-operation","phase":"'"$phase"'","status":"failed","reason":"timeout","durationMs":'"$duration_ms"',"timeoutSeconds":'"$timeout_seconds"'}' >&2
|
||||
else
|
||||
echo '{"event":"git-operation","phase":"'"$phase"'","status":"failed","exitCode":'"$status"',"durationMs":'"$duration_ms"'}' >&2
|
||||
fi
|
||||
return "$status"
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env node
|
||||
import net from "node:net";
|
||||
|
||||
const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);
|
||||
const proxyPort = Number.parseInt(proxyPortRaw || "", 10);
|
||||
const targetPort = Number.parseInt(targetPortRaw || "", 10);
|
||||
if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {
|
||||
console.error("usage: hwlab-github-proxy-connect <proxyHost> <proxyPort> <targetHost> <targetPort>");
|
||||
process.exit(64);
|
||||
}
|
||||
|
||||
const socket = net.createConnection({ host: proxyHost, port: proxyPort });
|
||||
let buffer = Buffer.alloc(0);
|
||||
|
||||
socket.setTimeout(10000, () => {
|
||||
console.error("proxy connect timeout " + proxyHost + ":" + proxyPort + " -> " + targetHost + ":" + targetPort);
|
||||
socket.destroy();
|
||||
process.exit(65);
|
||||
});
|
||||
|
||||
socket.on("connect", () => {
|
||||
socket.write("CONNECT " + targetHost + ":" + targetPort + " HTTP/1.1\r\nHost: " + targetHost + ":" + targetPort + "\r\nProxy-Connection: Keep-Alive\r\n\r\n");
|
||||
});
|
||||
|
||||
socket.on("error", (error) => {
|
||||
console.error("proxy connect failed: " + error.message);
|
||||
process.exit(66);
|
||||
});
|
||||
|
||||
function onData(chunk) {
|
||||
buffer = Buffer.concat([buffer, chunk]);
|
||||
const headerEnd = buffer.indexOf("\r\n\r\n");
|
||||
if (headerEnd === -1 && buffer.length < 8192) return;
|
||||
const head = buffer.slice(0, headerEnd + 4).toString("latin1");
|
||||
const statusLine = head.split("\r\n", 1)[0] || "";
|
||||
const statusCode = Number.parseInt(statusLine.split(" ")[1] || "", 10);
|
||||
if (!statusLine.startsWith("HTTP/1.") || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {
|
||||
console.error("proxy CONNECT rejected: " + (statusLine || "missing-status"));
|
||||
socket.destroy();
|
||||
process.exit(67);
|
||||
}
|
||||
socket.off("data", onData);
|
||||
socket.setTimeout(0);
|
||||
const rest = buffer.slice(headerEnd + 4);
|
||||
if (rest.length) process.stdout.write(rest);
|
||||
process.stdin.pipe(socket);
|
||||
socket.pipe(process.stdout);
|
||||
}
|
||||
|
||||
socket.on("data", onData);
|
||||
socket.on("close", () => process.exit(0));
|
||||
@@ -0,0 +1,331 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# __HWLAB_CI_TIMING_SHELL__
|
||||
|
||||
export HWLAB_TEKTON_PIPELINERUN="$(context.pipelineRun.name)"
|
||||
export HWLAB_TEKTON_TASKRUN="$(context.taskRun.name)"
|
||||
export HWLAB_TEKTON_TASK="gitops-promote"
|
||||
export HWLAB_SOURCE_REVISION="$(params.revision)"
|
||||
echo '{"event":"ci-base-image","phase":"gitops-promote","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apt"}'
|
||||
for tool in node git timeout; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"gitops-promote","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done
|
||||
# __HWLAB_GIT_SSH_SHELL__
|
||||
|
||||
git_url_requires_ssh() { case "$1" in git@*|ssh://*) return 0 ;; *) return 1 ;; esac; }
|
||||
lane="$(params.lane)"
|
||||
case "$lane" in
|
||||
v[0-9][0-9]*) runtime_lane=true ;;
|
||||
*) runtime_lane=false ;;
|
||||
esac
|
||||
if [ "$runtime_lane" = "true" ]; then
|
||||
printf 'false' > /tekton/results/runtime-ready-required
|
||||
else
|
||||
printf 'true' > /tekton/results/runtime-ready-required
|
||||
fi
|
||||
source_head_url_for_setup="$(params.git-url)"
|
||||
gitops_read_url_for_setup="$(params.git-url)"
|
||||
if [ "$runtime_lane" = "true" ]; then
|
||||
source_head_url_for_setup="$(params.git-read-url)"
|
||||
gitops_read_url_for_setup="$(params.git-read-url)"
|
||||
fi
|
||||
gitops_write_url_for_setup="$(params.git-write-url)"
|
||||
if git_url_requires_ssh "$source_head_url_for_setup" || git_url_requires_ssh "$gitops_read_url_for_setup" || git_url_requires_ssh "$gitops_write_url_for_setup"; then
|
||||
for tool in ssh ssh-keyscan; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"gitops-promote","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done
|
||||
git_ssh_setup
|
||||
else
|
||||
echo '{"event":"git-ssh-setup","phase":"gitops-promote","status":"skipped","reason":"non-ssh-git-url"}'
|
||||
fi
|
||||
cd /workspace/source/repo
|
||||
test "$(git rev-parse HEAD)" = "$(params.revision)"
|
||||
|
||||
check_source_head() {
|
||||
phase="$1"
|
||||
expected="${2:-$(params.revision)}"
|
||||
source_head_url="$(params.git-url)"
|
||||
if [ "$runtime_lane" = "true" ]; then
|
||||
source_head_url="$(params.git-read-url)"
|
||||
fi
|
||||
latest_file="$(mktemp)"
|
||||
if ! git_timed "source-head-$phase" 45 git ls-remote "$source_head_url" "refs/heads/$(params.source-branch)" > "$latest_file"; then
|
||||
echo '{"status":"failed","reason":"source-head-check-failed","phase":"'"$phase"'","sourceBranch":"'"$(params.source-branch)"'","sourceRevision":"'"$(params.revision)"'"}'
|
||||
exit 1
|
||||
fi
|
||||
latest="$(cut -f1 "$latest_file" | head -n 1)"
|
||||
if [ -z "$latest" ]; then
|
||||
echo '{"status":"failed","reason":"source-head-unresolved","phase":"'"$phase"'","sourceBranch":"'"$(params.source-branch)"'","sourceRevision":"'"$(params.revision)"'"}'
|
||||
exit 1
|
||||
fi
|
||||
if [ "$latest" != "$expected" ]; then
|
||||
printf 'false' > /tekton/results/runtime-ready-required || true
|
||||
echo '{"status":"skipped-stale-source","verdict":"superseded","phase":"'"$phase"'","sourceBranch":"'"$(params.source-branch)"'","sourceRevision":"'"$(params.revision)"'","expectedRevision":"'"$expected"'","latestRevision":"'"$latest"'"}'
|
||||
exit 0
|
||||
fi
|
||||
}
|
||||
|
||||
check_source_head before-render
|
||||
if [ -s /workspace/source/affected-services.json ] && node - /workspace/source/affected-services.json <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const plan = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
||||
const buildServices = Array.isArray(plan.buildServices) ? plan.buildServices : [];
|
||||
const affectedServices = Array.isArray(plan.affectedServices) ? plan.affectedServices : [];
|
||||
const willRunGitopsPromote = plan.ciCdPlan && plan.ciCdPlan.willRunGitopsPromote === true;
|
||||
process.exit(!willRunGitopsPromote && buildServices.length === 0 && affectedServices.length === 0 ? 0 : 1);
|
||||
NODE
|
||||
then
|
||||
printf 'false' > /tekton/results/runtime-ready-required || true
|
||||
echo '{"event":"gitops-promote","status":"skipped","reason":"no-build-no-rollout-plan"}'
|
||||
exit 0
|
||||
fi
|
||||
git config --global user.name "HWLAB node GitOps Bot"
|
||||
git config --global user.email "hwlab-node-gitops-bot@users.noreply.github.com"
|
||||
catalog_path="$(params.catalog-path)"
|
||||
runtime_path="$(params.runtime-path)"
|
||||
gitops_root_from_runtime_path() {
|
||||
path="$1"
|
||||
case "$path" in
|
||||
*/*) printf '%s
|
||||
' "${path%/*}" ;;
|
||||
*) printf '.
|
||||
' ;;
|
||||
esac
|
||||
}
|
||||
gitops_root="$(gitops_root_from_runtime_path "$runtime_path")"
|
||||
|
||||
argo_hard_refresh_runtime_lane() {
|
||||
if [ "$runtime_lane" != "true" ]; then return 0; fi
|
||||
ARGO_APPLICATION="hwlab-node-$lane" ARGO_FIELD_MANAGER="hwlab-$lane-gitops-promote" node <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const https = require("node:https");
|
||||
|
||||
const host = process.env.KUBERNETES_SERVICE_HOST;
|
||||
const port = process.env.KUBERNETES_SERVICE_PORT || "443";
|
||||
const startedAt = Date.now();
|
||||
const application = process.env.ARGO_APPLICATION || "hwlab-node-v02";
|
||||
const fieldManager = process.env.ARGO_FIELD_MANAGER || "hwlab-v02-gitops-promote";
|
||||
const pipelineRun = process.env.HWLAB_TEKTON_PIPELINERUN || null;
|
||||
const taskRun = process.env.HWLAB_TEKTON_TASKRUN || null;
|
||||
const task = process.env.HWLAB_TEKTON_TASK || null;
|
||||
const revision = process.env.HWLAB_SOURCE_REVISION || null;
|
||||
|
||||
function emit(payload) {
|
||||
console.log(JSON.stringify({
|
||||
event: "node-cicd-timing",
|
||||
schemaVersion: "v1",
|
||||
stage: "argo-hard-refresh",
|
||||
pipelineRun,
|
||||
taskRun,
|
||||
task,
|
||||
revision,
|
||||
application,
|
||||
source: "scripts/gitops-render.mjs",
|
||||
at: new Date().toISOString(),
|
||||
...payload
|
||||
}));
|
||||
}
|
||||
|
||||
function safeJson(text) {
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function request(method, path, body, contentType = "application/merge-patch+json") {
|
||||
const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8");
|
||||
const ca = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt");
|
||||
const payload = body ? JSON.stringify(body) : "";
|
||||
const headers = { Authorization: "Bearer " + token };
|
||||
if (payload) {
|
||||
headers["Content-Type"] = contentType;
|
||||
headers["Content-Length"] = Buffer.byteLength(payload);
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = https.request({ host, port, method, path, ca, headers }, (res) => {
|
||||
let data = "";
|
||||
res.setEncoding("utf8");
|
||||
res.on("data", (chunk) => { data += chunk; });
|
||||
res.on("end", () => resolve({ statusCode: res.statusCode || 0, body: data, json: safeJson(data) }));
|
||||
});
|
||||
req.on("error", reject);
|
||||
if (payload) req.write(payload);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
(async () => {
|
||||
if (!host) {
|
||||
emit({ status: "skipped", reason: "kubernetes-service-host-missing", durationMs: Date.now() - startedAt });
|
||||
return;
|
||||
}
|
||||
const response = await request(
|
||||
"PATCH",
|
||||
"/apis/argoproj.io/v1alpha1/namespaces/argocd/applications/" + encodeURIComponent(application) + "?fieldManager=" + encodeURIComponent(fieldManager),
|
||||
{ metadata: { annotations: { "argocd.argoproj.io/refresh": "hard" } } }
|
||||
);
|
||||
if (response.statusCode >= 200 && response.statusCode < 300) {
|
||||
emit({ status: "succeeded", durationMs: Date.now() - startedAt, statusCode: response.statusCode });
|
||||
return;
|
||||
}
|
||||
emit({ status: "degraded", reason: "argo-refresh-patch-failed", durationMs: Date.now() - startedAt, statusCode: response.statusCode, body: response.body.slice(0, 1000) });
|
||||
})().catch((error) => {
|
||||
emit({ status: "degraded", reason: "argo-refresh-patch-error", durationMs: Date.now() - startedAt, error: error.message });
|
||||
});
|
||||
NODE
|
||||
}
|
||||
|
||||
if [ -s /workspace/source/dev-artifacts.json ]; then
|
||||
node scripts/refresh-artifact-catalog.mjs --lane "$lane" --catalog-path "$catalog_path" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --target-ref "$(params.revision)" --publish-report /workspace/source/dev-artifacts.json --write
|
||||
fi
|
||||
gitops_render_started_ms="$(ci_now_ms)"
|
||||
node scripts/run-bun.mjs scripts/gitops-render.mjs --lane "$lane" --catalog-path "$catalog_path" --image-tag-mode "$(params.image-tag-mode)" --source-revision "$(params.revision)" --source-repo "$(params.git-url)" --source-branch "$(params.source-branch)" --gitops-branch "$(params.gitops-branch)" --gitops-root "$gitops_root" --out "$gitops_root" --registry-prefix "$(params.registry-prefix)" --use-deploy-images
|
||||
node scripts/run-bun.mjs scripts/gitops-render.mjs --lane "$lane" --catalog-path "$catalog_path" --image-tag-mode "$(params.image-tag-mode)" --source-revision "$(params.revision)" --source-repo "$(params.git-url)" --source-branch "$(params.source-branch)" --gitops-branch "$(params.gitops-branch)" --gitops-root "$gitops_root" --out "$gitops_root" --registry-prefix "$(params.registry-prefix)" --use-deploy-images --check
|
||||
ci_timing_emit gitops-render succeeded "$gitops_render_started_ms"
|
||||
check_source_head before-push
|
||||
workdir="$(mktemp -d)"
|
||||
gitops_read_url="$(params.git-url)"
|
||||
gitops_write_url="$(params.git-write-url)"
|
||||
if [ "$runtime_lane" = "true" ]; then
|
||||
gitops_read_url="$(params.git-read-url)"
|
||||
fi
|
||||
gitops_clone_started_ms="$(ci_now_ms)"
|
||||
git_timed gitops-clone 180 git clone --no-checkout "$gitops_read_url" "$workdir/gitops"
|
||||
cd "$workdir/gitops"
|
||||
git remote set-url origin "$gitops_write_url"
|
||||
old_runtime_snapshot="$workdir/old-runtime-snapshot"
|
||||
if git_timed gitops-branch-ls 45 git ls-remote --exit-code --heads origin "$(params.gitops-branch)" >/dev/null; then
|
||||
git_timed gitops-fetch 90 git fetch origin "$(params.gitops-branch)"
|
||||
git checkout -B "$(params.gitops-branch)" "origin/$(params.gitops-branch)"
|
||||
if [ "$runtime_lane" = "true" ] && [ -d "$runtime_path" ]; then
|
||||
mkdir -p "$old_runtime_snapshot"
|
||||
cp -a "$runtime_path"/. "$old_runtime_snapshot"/
|
||||
fi
|
||||
if [ "$runtime_lane" = "true" ]; then rm -rf "$runtime_path" "$catalog_path"; else rm -rf deploy/gitops/node "$catalog_path"; fi
|
||||
else
|
||||
git checkout --orphan "$(params.gitops-branch)"
|
||||
git rm -rf . >/dev/null 2>&1 || true
|
||||
fi
|
||||
ci_timing_emit gitops-clone succeeded "$gitops_clone_started_ms"
|
||||
mkdir -p "$(dirname "$runtime_path")" "$(dirname "$catalog_path")"
|
||||
if [ "$runtime_lane" = "true" ]; then
|
||||
cp -a "/workspace/source/repo/$runtime_path" "$runtime_path"
|
||||
cp "/workspace/source/repo/$catalog_path" "$catalog_path"
|
||||
git add "$catalog_path" "$runtime_path"
|
||||
else
|
||||
mkdir -p deploy/gitops
|
||||
cp -a /workspace/source/repo/deploy/gitops/node deploy/gitops/node
|
||||
cp "/workspace/source/repo/$catalog_path" "$catalog_path"
|
||||
git add "$catalog_path" deploy/gitops/node
|
||||
fi
|
||||
if [ "$runtime_lane" = "true" ] && [ -s /workspace/source/affected-services.json ]; then
|
||||
runtime_noop_decision="$(node - "$runtime_path" "$old_runtime_snapshot" /workspace/source/affected-services.json <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
|
||||
const [runtimePath, oldRuntimePath, planPath] = process.argv.slice(2);
|
||||
|
||||
function readJson(filePath) {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf8"));
|
||||
}
|
||||
|
||||
function stable(value) {
|
||||
if (Array.isArray(value)) return "[" + value.map(stable).join(",") + "]";
|
||||
if (value && typeof value === "object") {
|
||||
return "{" + Object.keys(value).sort().map((key) => JSON.stringify(key) + ":" + stable(value[key])).join(",") + "}";
|
||||
}
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
function scrub(value) {
|
||||
if (Array.isArray(value)) return value.map(scrub);
|
||||
if (!value || typeof value !== "object") return value;
|
||||
const result = {};
|
||||
for (const [key, child] of Object.entries(value)) {
|
||||
if (key === "hwlab.pikastech.local/source-commit" ||
|
||||
key === "hwlab.pikastech.local/artifact-source-commit" ||
|
||||
key === "hwlab.pikastech.local/boot-commit" ||
|
||||
key === "sourceCommitId" ||
|
||||
key === "bootCommit") {
|
||||
result[key] = "<runtime-source-identity>";
|
||||
continue;
|
||||
}
|
||||
const envName = String(value.name || "");
|
||||
if (key === "value" && /^HWLAB_.*COMMIT/.test(envName)) {
|
||||
result[key] = "<runtime-source-identity>";
|
||||
continue;
|
||||
}
|
||||
result[key] = scrub(child);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function listFiles(rootDir) {
|
||||
if (!rootDir || !fs.existsSync(rootDir)) return null;
|
||||
const files = [];
|
||||
function walk(current) {
|
||||
for (const entry of fs.readdirSync(current, { withFileTypes: true })) {
|
||||
const fullPath = path.join(current, entry.name);
|
||||
if (entry.isDirectory()) walk(fullPath);
|
||||
else if (entry.isFile()) files.push(path.relative(rootDir, fullPath).replaceAll(path.sep, "/"));
|
||||
}
|
||||
}
|
||||
walk(rootDir);
|
||||
return files.sort();
|
||||
}
|
||||
|
||||
function normalizedTree(rootDir) {
|
||||
const files = listFiles(rootDir);
|
||||
if (!files) return null;
|
||||
const entries = {};
|
||||
for (const relativePath of files) {
|
||||
const filePath = path.join(rootDir, relativePath);
|
||||
const text = fs.readFileSync(filePath, "utf8");
|
||||
try {
|
||||
entries[relativePath] = stable(scrub(JSON.parse(text)));
|
||||
} catch {
|
||||
entries[relativePath] = text.replace(/[a-f0-9]{40}/gu, "<runtime-source-identity>");
|
||||
}
|
||||
}
|
||||
return stable(entries);
|
||||
}
|
||||
|
||||
const plan = readJson(planPath);
|
||||
const buildServices = Array.isArray(plan.buildServices) ? plan.buildServices : [];
|
||||
const rolloutServices = Array.isArray(plan.rolloutServices) ? plan.rolloutServices : [];
|
||||
if (buildServices.length > 0 || rolloutServices.length > 0) {
|
||||
process.stdout.write("runtime-required");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const oldTree = normalizedTree(oldRuntimePath);
|
||||
const newTree = normalizedTree(runtimePath);
|
||||
process.stdout.write(oldTree && newTree && oldTree === newTree ? "runtime-identity-only" : "runtime-required");
|
||||
NODE
|
||||
)"
|
||||
if [ "$runtime_noop_decision" = "runtime-identity-only" ]; then
|
||||
printf 'false' > /tekton/results/runtime-ready-required
|
||||
echo '{"status":"skipped-runtime-unchanged","reason":"runtime-identity-only","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}'
|
||||
ci_timing_emit gitops-commit skipped "$(ci_now_ms)"
|
||||
ci_timing_emit gitops-push skipped "$(ci_now_ms)"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
if git diff --cached --quiet; then
|
||||
printf 'false' > /tekton/results/runtime-ready-required
|
||||
echo '{"status":"unchanged","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}'
|
||||
ci_timing_emit gitops-commit unchanged "$(ci_now_ms)"
|
||||
ci_timing_emit gitops-push unchanged "$(ci_now_ms)"
|
||||
exit 0
|
||||
fi
|
||||
short="$(printf '%.7s' "$(params.revision)")"
|
||||
gitops_commit_started_ms="$(ci_now_ms)"
|
||||
git commit -m "chore: promote node GitOps source $short"
|
||||
ci_timing_emit gitops-commit succeeded "$gitops_commit_started_ms"
|
||||
gitops_push_started_ms="$(ci_now_ms)"
|
||||
git_timed gitops-push 120 git push origin "HEAD:$(params.gitops-branch)"
|
||||
ci_timing_emit gitops-push succeeded "$gitops_push_started_ms"
|
||||
if [ "$runtime_lane" = "true" ]; then
|
||||
printf 'false' > /tekton/results/runtime-ready-required
|
||||
echo '{"event":"runtime-ready","phase":"gitops-promote","status":"delegated-post-flush-closeout","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}'
|
||||
fi
|
||||
argo_hard_refresh_runtime_lane
|
||||
echo '{"status":"pushed","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'","gitopsWriteUrl":"'"$gitops_write_url"'"}'
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# __HWLAB_CI_TIMING_SHELL__
|
||||
|
||||
export HWLAB_TEKTON_PIPELINERUN="$(context.pipelineRun.name)"
|
||||
export HWLAB_TEKTON_TASKRUN="$(context.taskRun.name)"
|
||||
export HWLAB_TEKTON_TASK="build-$(params.service-id)"
|
||||
export HWLAB_SOURCE_REVISION="$(params.revision)"
|
||||
export HWLAB_TIMING_SERVICE_ID="$(params.service-id)"
|
||||
# __HWLAB_DEPENDENCY_PROXY_PROBE__
|
||||
echo '{"event":"ci-base-image","phase":"service-image-publish","serviceId":"'"$(params.service-id)"'","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apk"}'
|
||||
for tool in node npm git python3 ssh curl; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"service-image-publish","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done
|
||||
mkdir -p /workspace/service-work/home
|
||||
export HOME=/workspace/service-work/home
|
||||
git config --global --add safe.directory /workspace/source/repo
|
||||
cd /workspace/source/repo
|
||||
test "$(git rev-parse HEAD)" = "$(params.revision)"
|
||||
export HWLAB_ARTIFACT_LANE="$(params.lane)"
|
||||
export HWLAB_ARTIFACT_CATALOG_PATH="$(params.catalog-path)"
|
||||
export HWLAB_DEPLOY_MANIFEST_PATH="deploy/deploy.yaml"
|
||||
export HWLAB_ARTIFACT_IMAGE_TAG_MODE="$(params.image-tag-mode)"
|
||||
mkdir -p /workspace/source/service-results
|
||||
if [ -s /workspace/source/affected-services.json ] && ! node -e 'const fs=require("node:fs"); const p=JSON.parse(fs.readFileSync("/workspace/source/affected-services.json","utf8")); const service=process.argv[1]; const item=(p.entries||[]).find((entry)=>entry.serviceId===service); process.exit(item && item.buildRequired ? 0 : 1);' "$(params.service-id)"; then
|
||||
node - "$(params.service-id)" <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const serviceId = process.argv[2];
|
||||
const catalog = JSON.parse(fs.readFileSync(process.env.HWLAB_ARTIFACT_CATALOG_PATH, "utf8"));
|
||||
const plan = fs.existsSync("/workspace/source/affected-services.json") ? JSON.parse(fs.readFileSync("/workspace/source/affected-services.json", "utf8")) : {};
|
||||
const service = (catalog.services || []).find((item) => item.serviceId === serviceId) || {};
|
||||
const planned = (plan.services || []).find((item) => item.serviceId === serviceId) || {};
|
||||
const envReuse = planned.runtimeMode === "env-reuse-git-mirror-checkout" || service.runtimeMode === "env-reuse-git-mirror-checkout" || planned.envReuse === true || service.envReuse === true;
|
||||
const image = envReuse ? (service.environmentImage || service.image || "") : (service.image || "");
|
||||
const digest = envReuse ? (service.environmentDigest || service.digest || "not_published") : (service.digest || "not_published");
|
||||
const imageTag = service.imageTag || (image.includes(":") ? image.slice(image.lastIndexOf(":") + 1) : "");
|
||||
const repository = image.includes(":") ? image.slice(0, image.lastIndexOf(":")) : image;
|
||||
const revision = process.env.HWLAB_SOURCE_REVISION || planned.bootCommit || service.bootCommit || service.sourceCommitId || service.commitId || imageTag;
|
||||
const componentInputHash = envReuse ? (planned.componentInputHash || service.componentInputHash || "") : (service.componentInputHash || "");
|
||||
const environmentInputHash = envReuse ? (service.environmentInputHash || planned.environmentInputHash || "") : (service.environmentInputHash || "");
|
||||
const codeInputHash = envReuse ? (planned.codeInputHash || service.codeInputHash || "") : (service.codeInputHash || "");
|
||||
const values = {
|
||||
"service-id": serviceId,
|
||||
status: /^sha256:[a-f0-9]{64}$/.test(digest) ? "reused" : "blocked_reuse_unavailable",
|
||||
image,
|
||||
"image-tag": imageTag,
|
||||
digest,
|
||||
"repository-digest": /^sha256:[a-f0-9]{64}$/.test(digest) && repository ? repository + "@" + digest : "",
|
||||
"source-commit-id": envReuse ? revision : (service.sourceCommitId || service.commitId || imageTag),
|
||||
"component-input-hash": componentInputHash,
|
||||
"environment-input-hash": environmentInputHash,
|
||||
"code-input-hash": codeInputHash,
|
||||
"runtime-mode": envReuse ? "env-reuse-git-mirror-checkout" : "service-image",
|
||||
"boot-repo": planned.bootRepo || service.bootRepo || "",
|
||||
"boot-commit": envReuse ? revision : (service.bootCommit || ""),
|
||||
"boot-sh": planned.bootSh || service.bootSh || "",
|
||||
"build-created-at": service.buildCreatedAt || "",
|
||||
"build-backend": envReuse ? "reused-env-catalog" : "reused-catalog",
|
||||
"reused-from": (envReuse ? service.environmentInputHash : service.componentInputHash) || service.commitId || imageTag || "catalog"
|
||||
};
|
||||
for (const [name, value] of Object.entries(values)) fs.writeFileSync("/tekton/results/" + name, String(value || ""));
|
||||
NODE
|
||||
echo '{"event":"service-build-skip","serviceId":"'"$(params.service-id)"'","reason":"component-inputs-unchanged"}'
|
||||
exit 0
|
||||
fi
|
||||
rm -rf /workspace/service-work/repo
|
||||
mkdir -p /workspace/service-work/repo
|
||||
tar -C /workspace/source/repo -cf - . | tar -C /workspace/service-work/repo -xo -f -
|
||||
cd /workspace/service-work/repo
|
||||
export HWLAB_CI_ARTIFACT_RUN_ID="$(context.pipelineRun.name)-$(params.service-id)"
|
||||
export HWLAB_CI_ARTIFACT_RUN_OWNER="tekton"
|
||||
export HWLAB_DEV_REGISTRY_PREFIX="$(params.registry-prefix)"
|
||||
export HWLAB_DEV_REGISTRY_K8S_NATIVE=1
|
||||
export HWLAB_NODE_CICD_TIMING=1
|
||||
export HWLAB_TEKTON_PIPELINERUN
|
||||
export HWLAB_TEKTON_TASKRUN
|
||||
export HWLAB_TEKTON_TASK
|
||||
export HWLAB_SOURCE_REVISION
|
||||
export PATH="/workspace/buildkit-bin:$PATH"
|
||||
export HWLAB_BUILDKIT_ADDR="unix:///workspace/buildkit-run/buildkitd.sock"
|
||||
if [ -n "$(params.base-image)" ]; then export HWLAB_DEV_BASE_IMAGE="$(params.base-image)"; fi
|
||||
if [ -n "${HWLAB_DEV_BASE_IMAGE:-}" ]; then
|
||||
echo '{"event":"dependency-local-registry-probe-start","phase":"service-image-publish-pre-base-image","serviceId":"'"$(params.service-id)"'","target":"http://127.0.0.1:5000/v2/"}'
|
||||
registry_started_at="$(date +%s)"
|
||||
curl -fsS --max-time 10 http://127.0.0.1:5000/v2/ >/dev/null
|
||||
registry_finished_at="$(date +%s)"
|
||||
echo '{"event":"dependency-local-registry-probe","phase":"service-image-publish-pre-base-image","serviceId":"'"$(params.service-id)"'","ok":true,"durationSeconds":'"$((registry_finished_at - registry_started_at))"'}'
|
||||
fi
|
||||
buildkit_ready=0
|
||||
for attempt in $(seq 1 60); do
|
||||
if /workspace/buildkit-bin/buildctl --addr "$HWLAB_BUILDKIT_ADDR" debug workers >/dev/null 2>&1; then
|
||||
buildkit_ready=1
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "$buildkit_ready" != "1" ]; then
|
||||
echo '{"event":"buildkit-sidecar-not-ready","serviceId":"'"$(params.service-id)"'","addr":"'"$HWLAB_BUILDKIT_ADDR"'"}' >&2
|
||||
exit 1
|
||||
fi
|
||||
HWLAB_CI_PLAN_VERIFY_REUSE_REGISTRY=1 node scripts/artifact-publish.mjs --publish --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --registry-prefix "$(params.registry-prefix)" --report "/workspace/source/service-results/$(params.service-id).json" --tekton-results-dir /tekton/results --quiet-build --concurrency 1 --services "$(params.service-id)" --buildkit-command /workspace/buildkit-bin/buildctl --buildkit-addr "$HWLAB_BUILDKIT_ADDR" --build-cache-mode "$(params.build-cache-mode)"
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
echo '{"event":"ci-base-image","phase":"plan-artifacts","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apk"}'
|
||||
for tool in node git; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"plan-artifacts","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done
|
||||
cd /workspace/source/repo
|
||||
ci_node_deps="${HWLAB_CI_NODE_DEPS:-/opt/hwlab-ci-node-deps/node_modules}"
|
||||
if [ -d "$ci_node_deps/yaml" ] && [ ! -e node_modules/yaml ]; then
|
||||
mkdir -p node_modules
|
||||
ln -s "$ci_node_deps/yaml" node_modules/yaml
|
||||
fi
|
||||
test "$(git rev-parse HEAD)" = "$(params.revision)"
|
||||
HWLAB_CATALOG_PATH="$(params.catalog-path)" HWLAB_GIT_URL="$(params.git-url)" HWLAB_GIT_READ_URL="$(params.git-read-url)" HWLAB_GITOPS_BRANCH="$(params.gitops-branch)" node scripts/ci/restore-artifact-catalog.mjs
|
||||
node scripts/ci-plan.mjs --lane "$(params.lane)" --target-ref HEAD --deploy-config deploy/deploy.yaml --artifact-catalog "$(params.catalog-path)" --registry-prefix "$(params.registry-prefix)" --services "$(params.services)" --verify-reuse-registry > /workspace/source/ci-plan.json
|
||||
node - <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const plan = JSON.parse(fs.readFileSync("/workspace/source/ci-plan.json", "utf8"));
|
||||
const selected = String(process.env.HWLAB_SELECTED_SERVICES || "").split(",").map((item) => item.trim()).filter(Boolean);
|
||||
const allServices = selected.length > 0 ? selected : ["__HWLAB_DEFAULT_SERVICE_IDS__"];
|
||||
const affected = new Set(plan.affectedServices || []);
|
||||
const plannedBuildServices = Array.isArray(plan.buildServices) ? new Set(plan.buildServices) : null;
|
||||
const selectedSet = new Set(selected);
|
||||
const byService = new Map((plan.services || []).map((service) => [service.serviceId, service]));
|
||||
const entries = allServices.map((serviceId) => {
|
||||
const service = byService.get(serviceId) || {};
|
||||
const serviceSelected = selectedSet.has(serviceId);
|
||||
const rolloutAffected = serviceSelected && affected.has(serviceId);
|
||||
const envReuse = service.runtimeMode === "env-reuse-git-mirror-checkout" || service.envReuse === true;
|
||||
const buildRequired = serviceSelected && (plannedBuildServices ? plannedBuildServices.has(serviceId) : (envReuse ? service.envChanged === true : rolloutAffected));
|
||||
return {
|
||||
serviceId,
|
||||
selected: serviceSelected,
|
||||
affected: rolloutAffected,
|
||||
buildRequired,
|
||||
rolloutAffected,
|
||||
runtimeMode: service.runtimeMode || "service-image",
|
||||
envChanged: service.envChanged ?? null,
|
||||
codeChanged: service.codeChanged ?? null
|
||||
};
|
||||
});
|
||||
for (const entry of entries) {
|
||||
fs.writeFileSync("/tekton/results/affected-" + entry.serviceId, entry.affected ? "true" : "false");
|
||||
fs.writeFileSync("/tekton/results/build-" + entry.serviceId, entry.buildRequired ? "true" : "false");
|
||||
}
|
||||
fs.writeFileSync("/workspace/source/affected-services.json", JSON.stringify({
|
||||
sourceCommitId: plan.sourceCommitId,
|
||||
affectedServices: plan.affectedServices || [],
|
||||
rolloutServices: plan.rolloutServices || plan.affectedServices || [],
|
||||
buildServices: plan.buildServices || entries.filter((entry) => entry.buildRequired).map((entry) => entry.serviceId),
|
||||
reusedServices: plan.reusedServices || [],
|
||||
buildSkippedCount: plan.buildSkippedCount || 0,
|
||||
envArtifactGroups: plan.envArtifactGroups || [],
|
||||
changedPathSummary: plan.changedPathSummary || null,
|
||||
ciCdPlan: plan.ciCdPlan || null,
|
||||
services: plan.services || [],
|
||||
entries
|
||||
}, null, 2) + String.fromCharCode(10));
|
||||
console.log(JSON.stringify({ event: "g14-ci-plan", sourceCommitId: plan.sourceCommitId, affectedServices: plan.affectedServices || [], rolloutServices: plan.rolloutServices || plan.affectedServices || [], buildServices: plan.buildServices || [], reusedServices: plan.reusedServices || [], buildSkippedCount: plan.buildSkippedCount || 0, envArtifactGroups: plan.envArtifactGroups || [] }));
|
||||
NODE
|
||||
@@ -0,0 +1,142 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# __HWLAB_DEPENDENCY_PROXY_PROBE__
|
||||
echo '{"event":"ci-base-image","phase":"poller","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apk"}'
|
||||
for tool in node git ssh curl timeout; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"poller","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done
|
||||
# __HWLAB_GIT_SSH_SHELL__
|
||||
|
||||
git_ssh_setup
|
||||
workdir="$(mktemp -d)"
|
||||
git_timed poller-clone 180 git clone --depth 1 --branch "$SOURCE_BRANCH" "$GIT_READ_URL" "$workdir/repo"
|
||||
cd "$workdir/repo"
|
||||
git remote set-url origin "$GIT_READ_URL"
|
||||
revision="$(git rev-parse HEAD)"
|
||||
subject="$(git log -1 --pretty=%s)"
|
||||
case "$subject" in
|
||||
"chore: promote node GitOps source "*)
|
||||
echo "Skipping generated GitOps promotion commit $revision"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
short="$(printf '%.12s' "$revision")"
|
||||
: "${PIPELINERUN_PREFIX:?PIPELINERUN_PREFIX is required}"
|
||||
: "${GITOPS_TARGET:?GITOPS_TARGET is required}"
|
||||
prefix="$PIPELINERUN_PREFIX"
|
||||
name="$prefix-$short"
|
||||
export POLLER_REVISION="$revision"
|
||||
export POLLER_PIPELINERUN="$name"
|
||||
export POLLER_SOURCE_SUBJECT="$subject"
|
||||
echo "Checking $GITOPS_TARGET source $revision with PipelineRun $name"
|
||||
node <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const https = require("node:https");
|
||||
|
||||
function requiredEnv(name) {
|
||||
const value = process.env[name];
|
||||
if (!value) throw new Error(name + " is required");
|
||||
return value;
|
||||
}
|
||||
|
||||
function optionalEnv(name) {
|
||||
return process.env[name] || "";
|
||||
}
|
||||
|
||||
const namespace = requiredEnv("POD_NAMESPACE");
|
||||
const name = requiredEnv("POLLER_PIPELINERUN");
|
||||
const revision = requiredEnv("POLLER_REVISION");
|
||||
const host = process.env.KUBERNETES_SERVICE_HOST;
|
||||
const port = process.env.KUBERNETES_SERVICE_PORT || "443";
|
||||
if (!host) throw new Error("KUBERNETES_SERVICE_HOST is not available");
|
||||
|
||||
const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8");
|
||||
const ca = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt");
|
||||
|
||||
function request(method, path, body) {
|
||||
const payload = body ? JSON.stringify(body) : "";
|
||||
const headers = { Authorization: "Bearer " + token };
|
||||
if (payload) {
|
||||
headers["Content-Type"] = "application/json";
|
||||
headers["Content-Length"] = Buffer.byteLength(payload);
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = https.request({ host, port, method, path, ca, headers }, (res) => {
|
||||
let data = "";
|
||||
res.setEncoding("utf8");
|
||||
res.on("data", (chunk) => { data += chunk; });
|
||||
res.on("end", () => resolve({ statusCode: res.statusCode || 0, body: data }));
|
||||
});
|
||||
req.on("error", reject);
|
||||
if (payload) req.write(payload);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
const labels = {
|
||||
"app.kubernetes.io/part-of": "hwlab",
|
||||
"hwlab.pikastech.local/gitops-target": requiredEnv("GITOPS_TARGET"),
|
||||
"hwlab.pikastech.local/source-commit": revision,
|
||||
"hwlab.pikastech.local/trigger": "polling"
|
||||
};
|
||||
|
||||
const pipelineRun = {
|
||||
apiVersion: "tekton.dev/v1",
|
||||
kind: "PipelineRun",
|
||||
metadata: {
|
||||
name,
|
||||
namespace,
|
||||
labels,
|
||||
annotations: {
|
||||
"hwlab.pikastech.local/source-subject": optionalEnv("POLLER_SOURCE_SUBJECT"),
|
||||
"hwlab.pikastech.local/source-branch": requiredEnv("SOURCE_BRANCH"),
|
||||
"hwlab.pikastech.local/gitops-branch": requiredEnv("GITOPS_BRANCH")
|
||||
}
|
||||
},
|
||||
spec: {
|
||||
pipelineRef: { name: requiredEnv("PIPELINE_NAME") },
|
||||
taskRunTemplate: {
|
||||
serviceAccountName: requiredEnv("SERVICE_ACCOUNT_NAME"),
|
||||
podTemplate: { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet", securityContext: { fsGroup: 1000 } }
|
||||
},
|
||||
params: [
|
||||
{ name: "git-url", value: requiredEnv("GIT_URL") },
|
||||
{ name: "git-read-url", value: requiredEnv("GIT_READ_URL") },
|
||||
{ name: "source-branch", value: requiredEnv("SOURCE_BRANCH") },
|
||||
{ name: "gitops-branch", value: requiredEnv("GITOPS_BRANCH") },
|
||||
{ name: "lane", value: requiredEnv("LANE") },
|
||||
{ name: "catalog-path", value: requiredEnv("CATALOG_PATH") },
|
||||
{ name: "image-tag-mode", value: requiredEnv("IMAGE_TAG_MODE") },
|
||||
{ name: "runtime-path", value: requiredEnv("RUNTIME_PATH") },
|
||||
{ name: "revision", value: revision },
|
||||
{ name: "registry-prefix", value: requiredEnv("REGISTRY_PREFIX") },
|
||||
{ name: "services", value: requiredEnv("SERVICES") },
|
||||
{ name: "base-image", value: requiredEnv("BASE_IMAGE") }
|
||||
],
|
||||
workspaces: [
|
||||
{ name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } },
|
||||
{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
const base = "/apis/tekton.dev/v1/namespaces/" + encodeURIComponent(namespace) + "/pipelineruns";
|
||||
const item = base + "/" + encodeURIComponent(name);
|
||||
|
||||
(async () => {
|
||||
const existing = await request("GET", item);
|
||||
if (existing.statusCode === 200) {
|
||||
console.log(JSON.stringify({ status: "exists", pipelineRun: name, revision }));
|
||||
return;
|
||||
}
|
||||
if (existing.statusCode !== 404) {
|
||||
throw new Error("unexpected PipelineRun lookup status " + existing.statusCode + ": " + existing.body.slice(0, 500));
|
||||
}
|
||||
const created = await request("POST", base, pipelineRun);
|
||||
if (created.statusCode < 200 || created.statusCode > 299) {
|
||||
throw new Error("PipelineRun create failed with status " + created.statusCode + ": " + created.body.slice(0, 1000));
|
||||
}
|
||||
console.log(JSON.stringify({ status: "created", pipelineRun: name, revision }));
|
||||
})().catch((error) => {
|
||||
console.error(error.stack || error.message);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
NODE
|
||||
@@ -0,0 +1,117 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# __HWLAB_CI_TIMING_SHELL__
|
||||
|
||||
export HWLAB_TEKTON_PIPELINERUN="$(context.pipelineRun.name)"
|
||||
export HWLAB_TEKTON_TASKRUN="$(context.taskRun.name)"
|
||||
export HWLAB_TEKTON_TASK="prepare-source"
|
||||
export HWLAB_SOURCE_REVISION="$(params.revision)"
|
||||
echo '{"event":"ci-base-image","phase":"prepare-source","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apk"}'
|
||||
for tool in node git timeout; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done
|
||||
node -e 'console.log(JSON.stringify({event:"ci-base-image",phase:"prepare-source",ok:true,node:process.version}))'
|
||||
# __HWLAB_GIT_SSH_SHELL__
|
||||
|
||||
git_url_requires_ssh() { case "$1" in git@*|ssh://*) return 0 ;; *) return 1 ;; esac; }
|
||||
git_read_url="$(params.git-read-url)"
|
||||
if git_url_requires_ssh "$git_read_url"; then
|
||||
for tool in ssh ssh-keyscan; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done
|
||||
git_ssh_setup
|
||||
else
|
||||
echo '{"event":"git-ssh-setup","phase":"prepare-source","status":"skipped","reason":"non-ssh-git-read-url"}'
|
||||
fi
|
||||
rm -rf /workspace/source/repo
|
||||
source_clone_started_ms="$(ci_now_ms)"
|
||||
git_timed source-clone 180 git clone --branch "$(params.source-branch)" "$git_read_url" /workspace/source/repo
|
||||
cd /workspace/source/repo
|
||||
git config --global --add safe.directory /workspace/source/repo
|
||||
git remote set-url origin "$git_read_url"
|
||||
git checkout "$(params.revision)"
|
||||
test "$(git rev-parse HEAD)" = "$(params.revision)"
|
||||
git merge-base --is-ancestor "$(params.revision)" "origin/$(params.source-branch)" || { echo '{"event":"source-ancestry","status":"failed","revision":"'"$(params.revision)"'","sourceBranch":"'"$(params.source-branch)"'"}'; exit 32; }
|
||||
ci_timing_emit source-clone succeeded "$source_clone_started_ms"
|
||||
prepare_source_dependencies_started_ms="$(ci_now_ms)"
|
||||
echo '{"event":"prepare-source-dependencies","status":"skipped","reason":"renderer-dependency-install-disabled","dependency":"yaml"}'
|
||||
ci_timing_emit prepare-source-dependencies succeeded "$prepare_source_dependencies_started_ms"
|
||||
catalog_path="$(params.catalog-path)"
|
||||
mkdir -p "$(dirname "$catalog_path")"
|
||||
catalog_fetch_started_ms="$(ci_now_ms)"
|
||||
if git_timed catalog-ls-remote 45 git ls-remote --exit-code --heads "$git_read_url" "$(params.gitops-branch)" >/dev/null; then
|
||||
git remote set-url gitops-catalog "$git_read_url" 2>/dev/null || git remote add gitops-catalog "$git_read_url"
|
||||
git_timed catalog-fetch 90 git fetch --depth 1 gitops-catalog "$(params.gitops-branch)" >/dev/null || true
|
||||
if git cat-file -e FETCH_HEAD:"$catalog_path" 2>/dev/null; then
|
||||
git show FETCH_HEAD:"$catalog_path" > /tmp/hwlab-gitops-artifact-catalog.json
|
||||
if node --input-type=module - /tmp/hwlab-gitops-artifact-catalog.json deploy/deploy.yaml "$(params.services)" <<'NODE'
|
||||
import fs from "node:fs";
|
||||
const [catalogPath, deployPath, selectedServices] = process.argv.slice(2);
|
||||
const ids = (doc) => (doc.services || []).map((service) => service.serviceId).filter(Boolean);
|
||||
const topLevelServiceIds = (text) => {
|
||||
const values = [];
|
||||
let inServices = false;
|
||||
for (const line of text.split(/\r?\n/u)) {
|
||||
if (/^services:\s*$/u.test(line)) { inServices = true; continue; }
|
||||
if (!inServices) continue;
|
||||
if (/^\S/u.test(line)) break;
|
||||
const match = line.match(/^\s*-\s+serviceId:\s*['"]?([^'"#\s]+)['"]?/u);
|
||||
if (match) values.push(match[1]);
|
||||
}
|
||||
return values;
|
||||
};
|
||||
const selected = (selectedServices || "").split(",").map((item) => item.trim()).filter(Boolean);
|
||||
const uniqueSorted = (items) => [...new Set(items)].sort();
|
||||
const gitops = JSON.parse(fs.readFileSync(catalogPath, "utf8"));
|
||||
const expected = selected.length ? selected : topLevelServiceIds(fs.readFileSync(deployPath, "utf8"));
|
||||
const actual = ids(gitops);
|
||||
const expectedSet = uniqueSorted(expected);
|
||||
const actualSet = uniqueSorted(actual);
|
||||
const sameServices = expectedSet.length === actualSet.length && expectedSet.every((item, index) => item === actualSet[index]);
|
||||
if (!sameServices) {
|
||||
const missing = expectedSet.filter((item) => !actualSet.includes(item));
|
||||
const extra = actualSet.filter((item) => !expectedSet.includes(item));
|
||||
console.error(JSON.stringify({ event: "gitops-artifact-catalog", phase: "prepare-source", status: "ignored-stale", reason: "service-ids-mismatch", expected, actual, missing, extra }));
|
||||
process.exit(42);
|
||||
}
|
||||
NODE
|
||||
then
|
||||
cp /tmp/hwlab-gitops-artifact-catalog.json "$catalog_path"
|
||||
echo '{"event":"gitops-artifact-catalog","phase":"prepare-source","status":"loaded","branch":"'"$(params.gitops-branch)"'"}'
|
||||
else
|
||||
echo '{"event":"gitops-artifact-catalog","phase":"prepare-source","status":"ignored-stale","reason":"service-ids-mismatch","branch":"'"$(params.gitops-branch)"'"}'
|
||||
fi
|
||||
else
|
||||
echo '{"event":"gitops-artifact-catalog","phase":"prepare-source","status":"seed","reason":"missing-on-gitops-branch"}'
|
||||
fi
|
||||
else
|
||||
echo '{"event":"gitops-artifact-catalog","phase":"prepare-source","status":"seed","reason":"gitops-branch-missing"}'
|
||||
fi
|
||||
if [ ! -s "$catalog_path" ] && [ "$(params.lane)" = "v02" ]; then
|
||||
node --input-type=module - "$catalog_path" "$(params.revision)" "$(params.registry-prefix)" "$(params.image-tag-mode)" "$(params.services)" <<'NODE'
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
const [catalogPath, revision, registryPrefix, imageTagMode, selectedServices] = process.argv.slice(2);
|
||||
const topLevelServiceIds = (text) => {
|
||||
const values = [];
|
||||
let inServices = false;
|
||||
for (const line of text.split(/\r?\n/u)) {
|
||||
if (/^services:\s*$/u.test(line)) { inServices = true; continue; }
|
||||
if (!inServices) continue;
|
||||
if (/^\S/u.test(line)) break;
|
||||
const match = line.match(/^\s*-\s+serviceId:\s*['"]?([^'"#\s]+)['"]?/u);
|
||||
if (match) values.push(match[1]);
|
||||
}
|
||||
return values;
|
||||
};
|
||||
const tag = imageTagMode === 'full' ? revision : revision.slice(0, 7);
|
||||
const namespace = 'hwlab-v02';
|
||||
const selected = (selectedServices || '').split(',').filter(Boolean);
|
||||
const serviceIds = selected.length ? selected : topLevelServiceIds(fs.readFileSync('deploy/deploy.yaml', 'utf8'));
|
||||
const services = serviceIds.map((serviceId) => {
|
||||
const service = { serviceId, profile: 'v02', namespace, commitId: tag, sourceCommitId: revision, image: `${registryPrefix}/${serviceId}:${tag}`, imageTag: tag, digest: null, buildBackend: 'contract-skeleton' };
|
||||
return service;
|
||||
});
|
||||
fs.mkdirSync(path.dirname(catalogPath), { recursive: true });
|
||||
fs.writeFileSync(catalogPath, JSON.stringify({ catalogVersion: 'v1', kind: 'hwlab-artifact-catalog', environment: 'v02', profile: 'v02', namespace, endpoint: "__HWLAB_V02_RUNTIME_ENDPOINT__", commitId: tag, artifactState: 'contract-skeleton', publish: { registryPrefix, sourceCommitId: revision, imageTag: tag, publishedAt: null }, allowedProfiles: ['v02'], forbiddenProfiles: ['dev', 'prod'], services }, null, 2) + '\n');
|
||||
NODE
|
||||
echo '{"event":"gitops-artifact-catalog","phase":"prepare-source","status":"seeded-v02-skeleton"}'
|
||||
fi
|
||||
ci_timing_emit catalog-fetch succeeded "$catalog_fetch_started_ms"
|
||||
echo 'node prepare-source complete; validation task count=__HWLAB_VALIDATION_TASK_COUNT__'
|
||||
@@ -98,7 +98,7 @@ const workbenchHealthRuntimeSource = readWeb("src/utils/workbench-health.ts");
|
||||
const traceTimelineSource = readWeb("src/components/agent/TraceTimeline.vue");
|
||||
const messageTraceDebugSource = readWeb("src/components/workbench/MessageTraceDebugPanel.vue");
|
||||
const conversationPanelSource = readWeb("src/components/workbench/ConversationPanel.vue");
|
||||
const serverWorkbenchHttpSource = fs.readFileSync(path.resolve(rootDir, "..", "..", "internal/cloud/server-workbench-http.ts"), "utf8");
|
||||
const serverWorkbenchRealtimeHttpSource = fs.readFileSync(path.resolve(rootDir, "..", "..", "internal/cloud/server-workbench-realtime-http.ts"), "utf8");
|
||||
|
||||
assert.match(html, /<div id="app"><\/div>/u, "index.html must expose Vue mount root");
|
||||
assert.match(html, /src="\/src\/main\.ts"/u, "index.html must load Vue TS entry");
|
||||
@@ -208,9 +208,9 @@ assert.doesNotMatch(workbenchStoreSource, /refreshActiveTraceFromRest[\s\S]{0,12
|
||||
assert.doesNotMatch(workbenchStoreSource, /refreshTerminalTraceFromRest[\s\S]{0,1200}refreshSessions\([^;]+force:\s*true/u, "Terminal trace REST refresh must not force-refresh the full session list");
|
||||
assert.doesNotMatch(workbenchStoreSource, /message\.runnerTrace(?:\?\.|\.)status/u, "Workbench message lifecycle must not be inferred from runnerTrace.status");
|
||||
assert.doesNotMatch(conversationPanelSource, /message\.runnerTrace(?:\?\.|\.)status/u, "ConversationPanel must not override message completion from runnerTrace.status");
|
||||
assert.doesNotMatch(serverWorkbenchHttpSource, /return\s+turnSnapshot\(context\)|\.\.\.turnSnapshot\(context\)|blockedTraceEventProjection\(context\.projection/u, "Realtime turn snapshots must not fall back to legacy trace/result projection");
|
||||
assertIncludes(serverWorkbenchHttpSource, "blockedRealtimeTurnSnapshot", "Realtime read-model gaps must produce blocked diagnostic turn snapshots");
|
||||
assertIncludes(serverWorkbenchHttpSource, "workbench_facts_session_missing", "Realtime read-model gaps must expose a facts-missing blocker");
|
||||
assert.doesNotMatch(serverWorkbenchRealtimeHttpSource, /return\s+turnSnapshot\(context\)|\.\.\.turnSnapshot\(context\)|blockedTraceEventProjection\(context\.projection/u, "Realtime turn snapshots must not fall back to legacy trace/result projection");
|
||||
assertIncludes(serverWorkbenchRealtimeHttpSource, "blockedRealtimeTurnSnapshot", "Realtime read-model gaps must produce blocked diagnostic turn snapshots");
|
||||
assertIncludes(serverWorkbenchRealtimeHttpSource, "workbench_facts_session_missing", "Realtime read-model gaps must expose a facts-missing blocker");
|
||||
assertIncludes(appSource, "/v1/workbench/events", "Workbench realtime client must use the RESTful same-origin events endpoint");
|
||||
assertIncludes(appSource, "/v1/workbench/traces/", "trace detail read must use Workbench read-model trace API");
|
||||
assert.doesNotMatch(appSource, /\/v1\/agent\/(?:turns|traces|chat\/result)\//u, "Cloud Web must not call legacy Code Agent read-through turn/trace/result APIs");
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user