diff --git a/scripts/gitops-render.mjs b/scripts/gitops-render.mjs index da33cc8a..2d09a93b 100644 --- a/scripts/gitops-render.mjs +++ b/scripts/gitops-render.mjs @@ -261,7 +261,16 @@ async function plannedFiles(args) { if (runtimeSecretPlane) putJson(`${runtimePath}/external-secrets.yaml`, runtimeSecretPlaneManifest({ config: runtimeSecretPlane, namespace, labels: profileLabels, annotations })); putJson(`${runtimePath}/workloads.yaml`, transformWorkloads({ workloads, deploy, catalog: artifactCatalog, source, sourceBranch: args.sourceBranch, gitReadUrl: args.gitReadUrl, registryPrefix: args.registryPrefix, runtimeEndpoint: endpoints.runtimeEndpoint, webEndpoint: endpoints.webEndpoint, profile, nodeId: args.nodeId, useDeployImages: args.useDeployImages, metricsSidecarSha256 })); putJson(`${runtimePath}/deepseek-proxy.yaml`, deepSeekProxyManifest({ profile, source, sourceBranch: args.sourceBranch, sourceRepo: args.sourceRepo, gitReadUrl: args.gitReadUrl, deploy, registryPrefix: args.registryPrefix, catalog: artifactCatalog, useDeployImages: args.useDeployImages, metricsSidecarSha256 })); - if (isRuntimeLane(profile) && externalPostgres) putJson(`${runtimePath}/external-postgres.yaml`, externalPostgresManifest({ profile, config: externalPostgres, source, deploy })); + if (isRuntimeLane(profile) && externalPostgres) putJson(`${runtimePath}/external-postgres.yaml`, externalPostgresManifest({ + profile, + config: externalPostgres, + source, + deploy, + migrationSources, + catalog: artifactCatalog, + registryPrefix: args.registryPrefix, + useDeployImages: args.useDeployImages + })); if (isRuntimeLane(profile) && !externalPostgres) putJson(`${runtimePath}/postgres.yaml`, v02PostgresManifest({ profile, migrationSources, source, image: runtimePostgresImageForProfile(deploy, profile), deploy })); if (workbenchRedis) putJson(`${runtimePath}/workbench-redis.yaml`, workbenchRuntimeRedisManifest({ profile, config: workbenchRedis, source, deploy })); if (isRuntimeLane(profile)) putJson(`${runtimePath}/openfga.yaml`, v02OpenFgaManifest({ profile, source, deploy })); diff --git a/scripts/gitops-render.test.ts b/scripts/gitops-render.test.ts index af3f13f6..5ce4f333 100644 --- a/scripts/gitops-render.test.ts +++ b/scripts/gitops-render.test.ts @@ -728,7 +728,22 @@ test("v03 render keeps node identity as data instead of generated structure", as assert.ok((projectManagementContainer?.volumeMounts ?? []).some((mount) => mount.name === "project-management-bootstrap" && mount.mountPath === "/etc/hwlab/project-management")); assert.ok((projectManagementContainer?.env ?? []).some((entry) => entry.name === "HWLAB_PROJECT_MANAGEMENT_BOOTSTRAP_SOURCES_PATH" && entry.value === "/etc/hwlab/project-management/sources.json")); const externalPostgres = JSON.parse(await readFile(path.join(outDir, "runtime-v03", "external-postgres.yaml"), "utf8")); - assert.deepEqual((externalPostgres.items ?? []).map((item) => item.kind), ["Service", "EndpointSlice"]); + assert.deepEqual((externalPostgres.items ?? []).map((item) => item.kind), ["ConfigMap", "Job", "Service", "EndpointSlice"]); + const externalPostgresMigrationConfig = (externalPostgres.items ?? []).find((item) => item.kind === "ConfigMap"); + const externalPostgresMigrationJob = (externalPostgres.items ?? []).find((item) => item.kind === "Job"); + assert.deepEqual( + Object.keys(externalPostgresMigrationConfig?.data ?? {}).filter((name) => name.endsWith(".sql")), + CLOUD_CORE_MIGRATIONS.map((migration) => path.basename(migration.path)) + ); + assert.equal(externalPostgresMigrationJob?.metadata?.annotations?.["argocd.argoproj.io/sync-wave"], "-1"); + assert.equal( + externalPostgresMigrationJob?.spec?.template?.spec?.containers?.[0]?.image, + `127.0.0.1:5000/hwlab/hwlab-cloud-api-env@sha256:${"1".repeat(64)}` + ); + assert.deepEqual(externalPostgresMigrationJob?.spec?.template?.spec?.containers?.[0]?.env?.[0]?.valueFrom?.secretKeyRef, { + name: "hwlab-cloud-api-v03-db", + key: "database-url" + }); const externalPostgresSlice = (externalPostgres.items ?? []).find((item) => item.kind === "EndpointSlice"); assert.equal(externalPostgresSlice?.metadata?.name, "jd01-pk01-platform-postgres-host"); assert.equal(externalPostgresSlice?.metadata?.labels?.["kubernetes.io/service-name"], "jd01-pk01-platform-postgres"); diff --git a/scripts/src/gitops-render/runtime-manifests.mjs b/scripts/src/gitops-render/runtime-manifests.mjs index b873c0ca..09889826 100644 --- a/scripts/src/gitops-render/runtime-manifests.mjs +++ b/scripts/src/gitops-render/runtime-manifests.mjs @@ -740,8 +740,24 @@ function externalPostgresConfigForLane(deploy, profile, args = {}) { return { serviceName: effective.serviceName, endpointAddress: effective.endpointAddress, port }; } -function externalPostgresManifest({ profile = "v03", config, source, deploy = null }) { +function externalPostgresManifest({ profile = "v03", config, source, deploy = null, migrationSources = [], catalog = null, registryPrefix = defaultRegistryPrefix, useDeployImages = false }) { const namespace = namespaceNameForProfile(profile, deploy); + assert.ok(Array.isArray(migrationSources) && migrationSources.length > 0, "external Postgres migration chain is required"); + const migrationData = Object.fromEntries(migrationSources.map((migration) => [path.basename(migration.path), migration.sql])); + const migrationSha256 = createHash("sha256").update(migrationSources.map((migration) => migration.sql).join("\n")).digest("hex"); + const migrationRunner = externalPostgresMigrationRunner(); + const migrationImage = runtimeImageForService({ + catalog, + deploy, + serviceId: "hwlab-cloud-api", + source, + registryPrefix, + useDeployImages, + digestPin: true, + envReuseServiceIds: envReuseServiceIdsForLane(deploy, profile) + }); + const executionSha256 = createHash("sha256").update(`${migrationSha256}\n${migrationImage}\n${migrationRunner}`).digest("hex"); + const migrationName = `${namespace}-external-postgres-migrate-${executionSha256.slice(0, 12)}`; const labels = { "app.kubernetes.io/name": config.serviceName, "app.kubernetes.io/part-of": "hwlab", @@ -756,6 +772,50 @@ function externalPostgresManifest({ profile = "v03", config, source, deploy = nu apiVersion: "v1", kind: "List", items: [ + { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { name: migrationName, namespace, labels, annotations: { ...annotations, "hwlab.pikastech.local/migration-sha256": migrationSha256 } }, + data: { + "run.mjs": migrationRunner, + ...migrationData + } + }, + { + apiVersion: "batch/v1", + kind: "Job", + metadata: { + name: migrationName, + namespace, + labels, + annotations: { + ...annotations, + "argocd.argoproj.io/sync-wave": "-1", + "hwlab.pikastech.local/migration-sha256": migrationSha256 + } + }, + spec: { + backoffLimit: 2, + template: { + metadata: { labels: { ...labels, "app.kubernetes.io/name": migrationName } }, + spec: { + restartPolicy: "Never", + containers: [{ + name: "migrate", + image: migrationImage, + imagePullPolicy: "IfNotPresent", + command: ["node", "/opt/hwlab-migrations/run.mjs"], + env: [{ + name: "DATABASE_URL", + valueFrom: { secretKeyRef: { name: `hwlab-cloud-api-${profile}-db`, key: "database-url" } } + }], + volumeMounts: [{ name: "migrations", mountPath: "/opt/hwlab-migrations", readOnly: true }] + }], + volumes: [{ name: "migrations", configMap: { name: migrationName } }] + } + } + } + }, { apiVersion: "v1", kind: "Service", @@ -774,6 +834,27 @@ function externalPostgresManifest({ profile = "v03", config, source, deploy = nu }; } +function externalPostgresMigrationRunner() { + return [ + 'import { readdir, readFile } from "node:fs/promises";', + 'import { createRequire } from "node:module";', + 'const require = createRequire("/opt/hwlab-env/package.json");', + 'const { Pool } = require("/opt/hwlab-env/node_modules/pg");', + 'const root = "/opt/hwlab-migrations";', + 'const files = (await readdir(root)).filter((name) => /^\\d+_.*\\.sql$/u.test(name)).sort();', + 'const pool = new Pool({ connectionString: process.env.DATABASE_URL, max: 1 });', + 'try {', + ' for (const file of files) {', + ' await pool.query(await readFile(`${root}/${file}`, "utf8"));', + ' console.error(JSON.stringify({ event: "hwlab-runtime-migration-applied", file, valuesPrinted: false }));', + ' }', + '} finally {', + ' await pool.end();', + '}', + '' + ].join("\n"); +} + function v02OpenFgaManifest({ profile = "v02", source, deploy = null }) { const namespace = namespaceNameForProfile(profile, deploy); const name = "hwlab-openfga";