mirrors/pikasTech-HWLAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: make Kafka projector the Workbench realtime authority

Browse Source
This commit is contained in:
root
2026-07-10 05:23:03 +02:00
parent a685be0b78
commit ce83ffdf49
74 changed files with 7746 additions and 9863 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/db/runtime-store-postgres.ts
+93 -11
View File
@@ -16,7 +16,7 @@ import {
deriveProtocolActorFromMeta
} from "../audit/index.mjs";
import {
CLOUD_CORE_MIGRATION_ID,
CLOUD_RUNTIME_DURABLE_MIGRATION_ID,
CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION,
CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE,
CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS
@@ -25,6 +25,17 @@ import {
import { CloudRuntimeStore } from "./runtime-store-memory.ts";
import * as runtimeCore from "./runtime-store-core.ts";
import {
claimHwlabKafkaOutbox as claimHwlabKafkaOutboxOperation,
commitAgentRunKafkaProjection as commitAgentRunKafkaProjectionOperation,
completeHwlabKafkaOutbox as completeHwlabKafkaOutboxOperation,
hwlabKafkaProjectorStatus as hwlabKafkaProjectorStatusOperation,
readAtomicWorkbenchProjectionSync as readAtomicWorkbenchProjectionSyncOperation,
recordFailedAgentRunKafkaMessage as recordFailedAgentRunKafkaMessageOperation,
recordIgnoredAgentRunKafkaMessage as recordIgnoredAgentRunKafkaMessageOperation,
retryHwlabKafkaOutbox as retryHwlabKafkaOutboxOperation
} from "./runtime-store-postgres-kafka.ts";
import { subscribeWorkbenchProjectionCommits as subscribeWorkbenchProjectionCommitsOperation } from "./runtime-store-postgres-notify.ts";
const {
RUNTIME_STORE_KIND,
@@ -697,7 +708,7 @@ export class PostgresCloudRuntimeStore {
commitType: "message",
terminal: Boolean(fact.terminal),
sealed: Boolean(fact.sealed),
payload: { role: fact.role, status: fact.status, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq, valuesRedacted: true },
payload: immutableProjectionOutboxPayload("messages", fact, { role: fact.role, status: fact.status, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq }),
createdAt: fact.updatedAt ?? fact.createdAt ?? this.now()
}, client);
}
@@ -718,7 +729,7 @@ export class PostgresCloudRuntimeStore {
commitType: fact.terminal ? "terminal" : "event",
terminal: Boolean(fact.terminal),
sealed: Boolean(fact.sealed),
payload: { type: fact.eventType, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq, valuesRedacted: true },
payload: immutableProjectionOutboxPayload("traceEvents", fact, { type: fact.eventType, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, projectedSeq: fact.projectedSeq }),
createdAt: fact.occurredAt ?? fact.updatedAt ?? this.now()
}, client);
}
@@ -740,7 +751,7 @@ export class PostgresCloudRuntimeStore {
commitType: "terminal",
terminal: true,
sealed: true,
payload: { status: fact.status, failureKind: fact.failureKind, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null, valuesRedacted: true },
payload: immutableProjectionOutboxPayload("turns", fact, { status: fact.status, failureKind: fact.failureKind, eventSeq: event?.eventSeq ?? null, aggregateSeq: event?.aggregateSeq ?? null }),
createdAt: fact.updatedAt ?? this.now()
}, client);
}
@@ -752,6 +763,49 @@ export class PostgresCloudRuntimeStore {
return withPersistence({ written: true, facts: persistedFacts, events: txResult.events }, this.summary());
}
async commitAgentRunKafkaProjection(params = {}) {
await this.assertReadyForWrites();
return commitAgentRunKafkaProjectionOperation(this, params);
}
async recordFailedAgentRunKafkaMessage(params = {}) {
await this.assertReadyForWrites();
return recordFailedAgentRunKafkaMessageOperation(this, params);
}
async recordIgnoredAgentRunKafkaMessage(params = {}) {
await this.assertReadyForWrites();
return recordIgnoredAgentRunKafkaMessageOperation(this, params);
}
async claimHwlabKafkaOutbox(params = {}) {
await this.assertReadyForWrites();
return claimHwlabKafkaOutboxOperation(this, params);
}
async completeHwlabKafkaOutbox(item = {}) {
return completeHwlabKafkaOutboxOperation(this, item);
}
async retryHwlabKafkaOutbox(item = {}, error = null, retryAt = null) {
return retryHwlabKafkaOutboxOperation(this, item, error, retryAt);
}
async hwlabKafkaProjectorStatus() {
await this.assertReadyForDurableReads("hwlab.kafka-projector.status");
return hwlabKafkaProjectorStatusOperation(this);
}
async readAtomicWorkbenchProjectionSync(params = {}) {
await this.assertReadyForDurableReads("workbench.projection-sync.read");
return readAtomicWorkbenchProjectionSyncOperation(this, params);
}
async subscribeWorkbenchProjectionCommits(listener) {
await this.assertReadyForDurableReads("workbench.projection-notify.listen");
return subscribeWorkbenchProjectionCommitsOperation(this, listener);
}
async queryWorkbenchFacts(params = {}) {
await this.assertReadyForDurableReads("workbench.facts.query");
const families = workbenchFactFamilySet(params);
@@ -1122,7 +1176,7 @@ export class PostgresCloudRuntimeStore {
async persistWorkbenchSessionFact(record, client = this) {
await client.query(
"INSERT INTO workbench_sessions (session_id, owner_user_id, project_id, conversation_id, thread_id, status, last_trace_id, projected_seq, source_seq, source_event_id, terminal, sealed, session_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (session_id) DO UPDATE SET owner_user_id = EXCLUDED.owner_user_id, project_id = EXCLUDED.project_id, conversation_id = EXCLUDED.conversation_id, thread_id = EXCLUDED.thread_id, status = EXCLUDED.status, last_trace_id = EXCLUDED.last_trace_id, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, session_json = EXCLUDED.session_json, updated_at = EXCLUDED.updated_at",
"INSERT INTO workbench_sessions (session_id, owner_user_id, project_id, conversation_id, thread_id, status, last_trace_id, projected_seq, source_seq, source_event_id, terminal, sealed, session_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (session_id) DO UPDATE SET owner_user_id = COALESCE(EXCLUDED.owner_user_id, workbench_sessions.owner_user_id), project_id = COALESCE(EXCLUDED.project_id, workbench_sessions.project_id), conversation_id = COALESCE(EXCLUDED.conversation_id, workbench_sessions.conversation_id), thread_id = COALESCE(EXCLUDED.thread_id, workbench_sessions.thread_id), status = EXCLUDED.status, last_trace_id = EXCLUDED.last_trace_id, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, session_json = EXCLUDED.session_json, updated_at = EXCLUDED.updated_at",
[record.sessionId, record.ownerUserId, record.projectId, record.conversationId, record.threadId, record.status, record.lastTraceId, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record), record.createdAt, record.updatedAt]
);
}
@@ -1164,8 +1218,8 @@ export class PostgresCloudRuntimeStore {
async persistWorkbenchProjectionOutbox(record, client = this) {
await client.query(
"INSERT INTO workbench_projection_outbox (event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)",
[record.eventSeq, record.aggregateId, nonNegativeInteger(record.aggregateSeq), nonNegativeInteger(record.projectionRevision), record.traceId, record.sessionId, record.turnId, record.messageId, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.commitType ?? "event", Boolean(record.terminal), Boolean(record.sealed), stableJson(record.payload ?? record), record.createdAt ?? this.now()]
"INSERT INTO workbench_projection_outbox (outbox_event_id, entity_family, entity_id, event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19) ON CONFLICT (outbox_event_id) DO NOTHING",
[record.outboxEventId ?? projectionOutboxEventId(record), record.entityFamily ?? projectionOutboxEntity(record).family, record.entityId ?? projectionOutboxEntity(record).id, record.eventSeq, record.aggregateId, nonNegativeInteger(record.aggregateSeq), nonNegativeInteger(record.projectionRevision), record.traceId, record.sessionId, record.turnId, record.messageId, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.commitType ?? "event", Boolean(record.terminal), Boolean(record.sealed), stableJson(record.payload ?? record), record.createdAt ?? this.now()]
);
}
@@ -1186,11 +1240,14 @@ export class PostgresCloudRuntimeStore {
}
params.push(Math.min(Math.max(Number(limit) || 100, 1), 500));
const result = await this.query(
`SELECT outbox_seq, event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at FROM workbench_projection_outbox WHERE ${where} ORDER BY outbox_seq ASC LIMIT $${paramIdx}`,
`SELECT outbox_seq, outbox_event_id, entity_family, entity_id, event_seq, aggregate_id, aggregate_seq, projection_revision, trace_id, session_id, turn_id, message_id, projected_seq, source_seq, source_event_id, commit_type, terminal, sealed, payload_json, created_at FROM workbench_projection_outbox WHERE ${where} ORDER BY outbox_seq ASC LIMIT $${paramIdx}`,
params
);
return (result.rows ?? []).map((row) => ({
outboxSeq: Number(row.outbox_seq),
outboxEventId: row.outbox_event_id,
entityFamily: row.entity_family,
entityId: row.entity_id,
eventSeq: row.event_seq === null || row.event_seq === undefined ? null : Number(row.event_seq),
aggregateId: row.aggregate_id,
aggregateSeq: Number(row.aggregate_seq ?? 0),
@@ -1230,17 +1287,17 @@ export class PostgresCloudRuntimeStore {
async readMigrationReadiness() {
const result = await this.query(
`SELECT id, schema_version FROM ${CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE} WHERE id = $1 LIMIT 1`,
[CLOUD_CORE_MIGRATION_ID]
[CLOUD_RUNTIME_DURABLE_MIGRATION_ID]
);
const row = result.rows?.[0] ?? null;
const ready =
row?.id === CLOUD_CORE_MIGRATION_ID &&
row?.id === CLOUD_RUNTIME_DURABLE_MIGRATION_ID &&
row?.schema_version === CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION;
return {
checked: true,
ready,
table: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE,
requiredMigrationId: CLOUD_CORE_MIGRATION_ID,
requiredMigrationId: CLOUD_RUNTIME_DURABLE_MIGRATION_ID,
requiredSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION,
appliedMigrationId: row?.id ?? null,
appliedSchemaVersion: row?.schema_version ?? null,
@@ -1535,3 +1592,28 @@ export class PostgresCloudRuntimeStore {
});
}
}
function projectionOutboxEventId(record = {}) {
const sourceEventId = textOr(record.sourceEventId, "");
const commitType = textOr(record.commitType, "event");
const family = textOr(record.entityFamily ?? record.payload?.family, "") || (commitType === "message" ? "messages" : record.payload?.type ? "traceEvents" : "turns");
const entityId = family === "traceEvents"
? textOr(record.entityId ?? record.payload?.fact?.id ?? record.aggregateId ?? record.sourceEventId ?? record.traceId, "projection")
: textOr(record.entityId ?? record.payload?.fact?.messageId ?? record.payload?.fact?.turnId ?? record.messageId ?? record.turnId ?? record.traceId ?? record.aggregateId, "projection");
if (sourceEventId) return `${sourceEventId}:${family}:${entityId}:${commitType}`;
const derived = stableJson({ family, entityId, commitType, eventSeq: record.eventSeq ?? null, aggregateId: record.aggregateId ?? null, aggregateSeq: record.aggregateSeq ?? 0, projectionRevision: record.projectionRevision ?? record.projectedSeq ?? 0, traceId: record.traceId ?? null, sessionId: record.sessionId ?? null });
return `derived:${createHash("sha256").update(derived).digest("hex")}`;
}
function projectionOutboxEntity(record = {}) {
const commitType = textOr(record.commitType, "event");
const family = textOr(record.entityFamily ?? record.payload?.family, "") || (commitType === "message" ? "messages" : record.payload?.type ? "traceEvents" : "turns");
const id = family === "traceEvents"
? textOr(record.entityId ?? record.payload?.fact?.id ?? record.aggregateId ?? record.sourceEventId ?? record.traceId, "projection")
: textOr(record.entityId ?? record.payload?.fact?.messageId ?? record.payload?.fact?.turnId ?? record.messageId ?? record.turnId ?? record.traceId ?? record.aggregateId, "projection");
return { family, id };
}
function immutableProjectionOutboxPayload(family, fact, metadata = {}) {
return { family, fact, metadata: { ...metadata, valuesRedacted: true }, valuesRedacted: true };
}