diff --git a/docs/reference/spec-v02-observability-monitoring.md b/docs/reference/spec-v02-observability-monitoring.md index 3df4bc33..03eb2cb4 100644 --- a/docs/reference/spec-v02-observability-monitoring.md +++ b/docs/reference/spec-v02-observability-monitoring.md @@ -24,7 +24,7 @@ hwlab-* service 第一阶段优先使用 `ServiceMonitor`，因为 v0.2 保留服务均有 ClusterIP Service。动态 Job、AgentRun runner 或临时 debug Pod 的指标如需接入，后续使用 `PodMonitor`，但不得把 AgentRun runner 变成 HWLAB 自有运行面。 -所有 HWLAB `/metrics` endpoint 必须是内部指标入口，不参与公网 FRP 暴露。`hwlab-cloud-web` 和 `hwlab-edge-proxy` 不得把 `/metrics` 代理给普通浏览器或公网 API 调用方。Prometheus 抓取路径只允许从集群内 Service 访问。 +所有 HWLAB `/metrics` endpoint 必须是内部指标入口，不参与公网 FRP 暴露。第一阶段使用独立 `metrics` named Service port 暴露 sidecar 指标；公网 FRP 仍只转发业务端口，所以 `19666/19667` 上的 `/metrics` 必须继续是负向结果。`hwlab-cloud-web` 和 `hwlab-edge-proxy` 不得把 `/metrics` 代理给普通浏览器或公网 API 调用方。Prometheus 抓取路径只允许从集群内 Service 访问。 ## API 接口说明 diff --git a/internal/dev-entrypoint/metrics-sidecar.mjs b/internal/dev-entrypoint/metrics-sidecar.mjs new file mode 100644 index 00000000..4c1940b4 --- /dev/null +++ b/internal/dev-entrypoint/metrics-sidecar.mjs @@ -0,0 +1,124 @@ +import { createServer } from "node:http"; +import { pathToFileURL } from "node:url"; + +const DEFAULT_PORT = 9100; +const DEFAULT_TARGET_TIMEOUT_MS = 2000; +const startedAtMs = Date.now(); + +export function sanitizeLabelValue(value, fallback = "unknown") { + const text = String(value ?? "").trim(); + const sanitized = text.replace(/[^A-Za-z0-9_.:-]/gu, "_").slice(0, 120); + return sanitized || fallback; +} + +export function escapeLabelValue(value) { + return String(value ?? "").replace(/\\/gu, "\\\\").replace(/\n/gu, "\\n").replace(/"/gu, "\\\""); +} + +function metricLabels(env) { + const service = sanitizeLabelValue(env.HWLAB_METRICS_SERVICE_ID || env.HWLAB_SERVICE_ID); + const namespace = sanitizeLabelValue(env.HWLAB_METRICS_NAMESPACE || env.POD_NAMESPACE || "hwlab-v02"); + const gitopsTarget = sanitizeLabelValue(env.HWLAB_METRICS_GITOPS_TARGET || env.HWLAB_GITOPS_TARGET || "v02"); + return `service="${escapeLabelValue(service)}",namespace="${escapeLabelValue(namespace)}",gitops_target="${escapeLabelValue(gitopsTarget)}"`; +} + +export async function probeTarget({ targetUrl, timeoutMs = DEFAULT_TARGET_TIMEOUT_MS, fetchImpl = globalThis.fetch } = {}) { + const configured = typeof targetUrl === "string" && targetUrl.trim().length > 0; + if (!configured || typeof fetchImpl !== "function") { + return { configured: false, success: false, statusCode: 0, durationSeconds: 0 }; + } + const controller = new AbortController(); + const started = Date.now(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetchImpl(targetUrl, { signal: controller.signal }); + return { + configured: true, + success: response.ok, + statusCode: response.status || 0, + durationSeconds: (Date.now() - started) / 1000 + }; + } catch { + return { configured: true, success: false, statusCode: 0, durationSeconds: (Date.now() - started) / 1000 }; + } finally { + clearTimeout(timeout); + } +} + +export function buildMetricsText({ env = process.env, probe = null, now = Date.now() } = {}) { + const labels = metricLabels(env); + const uptimeSeconds = Math.max(0, (now - startedAtMs) / 1000); + const healthProbe = probe ?? { configured: false, success: false, statusCode: 0, durationSeconds: 0 }; + return [ + "# HELP hwlab_service_up HWLAB metrics sidecar availability.", + "# TYPE hwlab_service_up gauge", + `hwlab_service_up{${labels}} 1`, + "# HELP hwlab_service_info Static HWLAB service metrics identity without high-cardinality request identifiers.", + "# TYPE hwlab_service_info gauge", + `hwlab_service_info{${labels}} 1`, + "# HELP hwlab_service_process_uptime_seconds Metrics sidecar process uptime in seconds.", + "# TYPE hwlab_service_process_uptime_seconds gauge", + `hwlab_service_process_uptime_seconds{${labels}} ${uptimeSeconds.toFixed(3)}`, + "# HELP hwlab_service_health_probe_configured Whether the sidecar has a cluster-local health probe target.", + "# TYPE hwlab_service_health_probe_configured gauge", + `hwlab_service_health_probe_configured{${labels}} ${healthProbe.configured ? 1 : 0}`, + "# HELP hwlab_service_health_probe_success Whether the most recent cluster-local health probe succeeded.", + "# TYPE hwlab_service_health_probe_success gauge", + `hwlab_service_health_probe_success{${labels}} ${healthProbe.success ? 1 : 0}`, + "# HELP hwlab_service_health_probe_status_code Most recent cluster-local health probe HTTP status code, or 0 when unavailable.", + "# TYPE hwlab_service_health_probe_status_code gauge", + `hwlab_service_health_probe_status_code{${labels}} ${Number.isInteger(healthProbe.statusCode) ? healthProbe.statusCode : 0}`, + "# HELP hwlab_service_health_probe_duration_seconds Most recent cluster-local health probe duration in seconds.", + "# TYPE hwlab_service_health_probe_duration_seconds gauge", + `hwlab_service_health_probe_duration_seconds{${labels}} ${Number(healthProbe.durationSeconds || 0).toFixed(3)}`, + "" + ].join("\n"); +} + +function sendJson(response, statusCode, body) { + const payload = JSON.stringify(body) + "\n"; + response.writeHead(statusCode, { + "content-type": "application/json; charset=utf-8", + "content-length": Buffer.byteLength(payload) + }); + response.end(payload); +} + +export function createMetricsSidecarServer({ env = process.env, fetchImpl = globalThis.fetch } = {}) { + return createServer(async (request, response) => { + const url = new URL(request.url || "/", "http://hwlab-metrics.local"); + if (request.method === "GET" && (url.pathname === "/health" || url.pathname === "/health/live")) { + sendJson(response, 200, { + serviceId: sanitizeLabelValue(env.HWLAB_METRICS_SERVICE_ID || env.HWLAB_SERVICE_ID), + status: "ok", + metricsPath: "/metrics" + }); + return; + } + if (request.method === "GET" && url.pathname === "/metrics") { + const timeoutMs = Number.parseInt(env.HWLAB_METRICS_TARGET_TIMEOUT_MS || "", 10) || DEFAULT_TARGET_TIMEOUT_MS; + const probe = await probeTarget({ targetUrl: env.HWLAB_METRICS_TARGET_URL, timeoutMs, fetchImpl }); + const payload = buildMetricsText({ env, probe }); + response.writeHead(200, { + "content-type": "text/plain; version=0.0.4; charset=utf-8", + "content-length": Buffer.byteLength(payload) + }); + response.end(payload); + return; + } + sendJson(response, 404, { error: "not_found", path: url.pathname }); + }); +} + +export function startMetricsSidecar({ env = process.env } = {}) { + const port = Number.parseInt(env.HWLAB_METRICS_PORT || env.PORT || "", 10) || DEFAULT_PORT; + const server = createMetricsSidecarServer({ env }); + server.listen(port, "0.0.0.0", () => { + process.stdout.write(JSON.stringify({ serviceId: env.HWLAB_METRICS_SERVICE_ID || "hwlab-unknown", status: "metrics-listening", port }) + "\n"); + }); + return server; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + startMetricsSidecar(); +} diff --git a/internal/dev-entrypoint/metrics-sidecar.test.mjs b/internal/dev-entrypoint/metrics-sidecar.test.mjs new file mode 100644 index 00000000..348b9b9d --- /dev/null +++ b/internal/dev-entrypoint/metrics-sidecar.test.mjs @@ -0,0 +1,25 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { buildMetricsText, sanitizeLabelValue } from "./metrics-sidecar.mjs"; + +test("metrics text exposes stable HWLAB labels without sensitive identifiers", () => { + const text = buildMetricsText({ + env: { + HWLAB_METRICS_SERVICE_ID: "hwlab-cloud-api", + HWLAB_METRICS_NAMESPACE: "hwlab-v02", + HWLAB_METRICS_GITOPS_TARGET: "v02" + }, + probe: { configured: true, success: true, statusCode: 200, durationSeconds: 0.012 } + }); + + assert.match(text, /hwlab_service_up\{service="hwlab-cloud-api",namespace="hwlab-v02",gitops_target="v02"\} 1/u); + assert.match(text, /hwlab_service_health_probe_success\{service="hwlab-cloud-api",namespace="hwlab-v02",gitops_target="v02"\} 1/u); + assert.doesNotMatch(text, /traceId|sessionId|conversationId|threadId|runId|commandId|jobId/iu); + assert.doesNotMatch(text, /prompt|assistant text|device output|secret|api key/iu); +}); + +test("label sanitization keeps metrics labels low-risk", () => { + assert.equal(sanitizeLabelValue("hwlab/cloud api\ntraceId=trc_1"), "hwlab_cloud_api_traceId_trc_1"); + assert.equal(sanitizeLabelValue(""), "unknown"); +}); diff --git a/scripts/g14-gitops-render.mjs b/scripts/g14-gitops-render.mjs index bae522d4..4108068c 100644 --- a/scripts/g14-gitops-render.mjs +++ b/scripts/g14-gitops-render.mjs @@ -46,6 +46,15 @@ const v02RuntimeServiceIds = Object.freeze([ "hwlab-edge-proxy", "hwlab-agent-skills" ]); +const v02ObservableServices = Object.freeze([ + { serviceId: "hwlab-cloud-api", port: 6667, healthPath: "/health/live" }, + { serviceId: "hwlab-cloud-web", port: 8080, healthPath: "/health/live" }, + { serviceId: "hwlab-device-pod", port: 7601, healthPath: "/health/live" }, + { serviceId: "hwlab-edge-proxy", port: 6667, healthPath: "/health" }, + { serviceId: "hwlab-agent-skills", port: 7430, healthPath: "/health/live" }, + { serviceId: "hwlab-deepseek-proxy", port: 4000, healthPath: "/health/live" } +]); +const v02ObservableServiceIds = new Set(v02ObservableServices.map((item) => item.serviceId)); const v02RemovedServiceIds = new Set([ "hwlab-agent-mgr", "hwlab-agent-worker", @@ -526,6 +535,10 @@ function serviceIdsForProfile(profile) { return profile === "v02" ? v02RuntimeServiceIds : defaultServiceIds; } +function v02ObservableService(serviceId) { + return v02ObservableServices.find((item) => item.serviceId === serviceId) ?? null; +} + function servicesParamForLane(lane) { return serviceIdsForLane(lane).join(","); } @@ -774,6 +787,56 @@ function deployServicesForProfile(deploy, profile) { return services; } +function v02MetricsLabels(serviceId) { + return { + "hwlab.pikastech.local/monitoring": "enabled", + "hwlab.pikastech.local/service-id": serviceId + }; +} + +function v02MetricsSidecarVolume() { + return { name: "hwlab-metrics-sidecar", configMap: { name: "hwlab-v02-metrics-sidecar" } }; +} + +function v02MetricsSidecarContainer({ serviceId, namespace, gitopsTarget }) { + const service = v02ObservableService(serviceId); + assert.ok(service, `unknown v0.2 observable service ${serviceId}`); + return { + name: "hwlab-metrics", + image: ciToolsRunnerImage, + imagePullPolicy: "IfNotPresent", + command: ["node", "/metrics/metrics-sidecar.mjs"], + env: [ + { name: "HWLAB_METRICS_SERVICE_ID", value: serviceId }, + { name: "HWLAB_METRICS_NAMESPACE", value: namespace }, + { name: "HWLAB_METRICS_GITOPS_TARGET", value: gitopsTarget }, + { name: "HWLAB_METRICS_PORT", value: "9100" }, + { name: "HWLAB_METRICS_TARGET_URL", value: `http://127.0.0.1:${service.port}${service.healthPath}` }, + { name: "HWLAB_METRICS_TARGET_TIMEOUT_MS", value: "2000" } + ], + ports: [{ name: "metrics", containerPort: 9100 }], + readinessProbe: { httpGet: { path: "/health/live", port: "metrics" }, initialDelaySeconds: 3, periodSeconds: 10 }, + livenessProbe: { httpGet: { path: "/health/live", port: "metrics" }, initialDelaySeconds: 10, periodSeconds: 20 }, + volumeMounts: [{ name: "hwlab-metrics-sidecar", mountPath: "/metrics", readOnly: true }] + }; +} + +function upsertV02MetricsSidecar(podSpec, options) { + if (!podSpec || !Array.isArray(podSpec.containers)) return; + podSpec.containers = podSpec.containers.filter((container) => container?.name !== "hwlab-metrics"); + podSpec.containers.push(v02MetricsSidecarContainer(options)); + podSpec.volumes ??= []; + podSpec.volumes = podSpec.volumes.filter((volume) => volume?.name !== "hwlab-metrics-sidecar"); + podSpec.volumes.push(v02MetricsSidecarVolume()); +} + +function upsertV02MetricsPort(service) { + service.spec ??= {}; + service.spec.ports ??= []; + service.spec.ports = service.spec.ports.filter((port) => port?.name !== "metrics"); + service.spec.ports.push({ name: "metrics", port: 9100, targetPort: "metrics" }); +} + function transformWorkloads({ workloads, deploy, catalog, source, sourceBranch = defaultBranch, registryPrefix, runtimeEndpoint, webEndpoint, profile = "dev", useDeployImages = false }) { const result = cloneJson(workloads); const deployServices = deployServicesForProfile(deploy, profile); @@ -822,6 +885,11 @@ function transformWorkloads({ workloads, deploy, catalog, source, sourceBranch = "hwlab.pikastech.local/source-commit": templateSourceCommit, "hwlab.pikastech.local/artifact-source-commit": templateArtifactCommit }); + if (profile === "v02" && v02ObservableServiceIds.has(templateServiceId)) { + label(item.metadata, v02MetricsLabels(templateServiceId)); + label(podTemplate.metadata, v02MetricsLabels(templateServiceId)); + upsertV02MetricsSidecar(podTemplate.spec, { serviceId: templateServiceId, namespace, gitopsTarget }); + } if ((item.kind === "Deployment" || item.kind === "StatefulSet") && item.spec?.selector?.matchLabels) { item.spec.selector.matchLabels = { ...item.spec.selector.matchLabels, @@ -983,10 +1051,88 @@ function transformServices({ services, namespace, labels, annotations, profile = const result = transformListNamespace(services, namespace, labels, annotations); if (result.kind === "List") { result.items = (result.items ?? []).filter((item) => !(profile === "v02" && isV02RemovedServiceId(serviceIdForWorkload(item, null)))); + for (const item of result.items) { + const serviceId = serviceIdForWorkload(item, null); + if (profile !== "v02" || !v02ObservableServiceIds.has(serviceId)) continue; + label(item.metadata ??= {}, v02MetricsLabels(serviceId)); + upsertV02MetricsPort(item); + } } return result; } +function observabilityManifest({ namespace, labels, annotations, metricsSidecarScript }) { + const baseLabels = { + ...labels, + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/monitoring": "enabled" + }; + const serviceMonitors = v02ObservableServices.map((service) => ({ + apiVersion: "monitoring.coreos.com/v1", + kind: "ServiceMonitor", + metadata: { + name: `hwlab-v02-${service.serviceId}`, + namespace, + labels: { ...baseLabels, "hwlab.pikastech.local/service-id": service.serviceId }, + annotations + }, + spec: { + namespaceSelector: { matchNames: [namespace] }, + selector: { + matchLabels: { + "hwlab.pikastech.local/gitops-target": "v02", + "hwlab.pikastech.local/monitoring": "enabled", + "hwlab.pikastech.local/service-id": service.serviceId + } + }, + targetLabels: [ + "hwlab.pikastech.local/gitops-target", + "hwlab.pikastech.local/service-id" + ], + endpoints: [{ port: "metrics", path: "/metrics", interval: "30s", scrapeTimeout: "10s" }] + } + })); + return { + apiVersion: "v1", + kind: "List", + items: [ + { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { name: "hwlab-v02-metrics-sidecar", namespace, labels: baseLabels, annotations }, + data: { "metrics-sidecar.mjs": metricsSidecarScript } + }, + ...serviceMonitors, + { + apiVersion: "monitoring.coreos.com/v1", + kind: "PrometheusRule", + metadata: { name: "hwlab-v02-observability", namespace, labels: baseLabels, annotations }, + spec: { + groups: [{ + name: "hwlab-v02-observability", + rules: [ + { + alert: "HWLABV02MetricsTargetDown", + expr: 'up{namespace="hwlab-v02"} == 0', + for: "5m", + labels: { severity: "warning", lane: "v02" }, + annotations: { summary: "HWLAB v0.2 metrics target is down", description: "Prometheus cannot scrape a HWLAB v0.2 metrics target." } + }, + { + alert: "HWLABV02ServiceHealthProbeFailed", + expr: 'hwlab_service_health_probe_success{namespace="hwlab-v02"} == 0', + for: "5m", + labels: { severity: "warning", lane: "v02" }, + annotations: { summary: "HWLAB v0.2 service health probe failed", description: "The metrics sidecar cannot reach the service health endpoint inside the pod network." } + } + ] + }] + } + } + ] + }; +} + function shellSingleQuote(value) { return `'${String(value).replace(/'/g, `'"'"'`)}'`; } @@ -3914,6 +4060,7 @@ function deepSeekProxyManifest({ profile = "dev", source, registryPrefix = defau "hwlab.pikastech.local/service-id": "hwlab-deepseek-proxy", "hwlab.pikastech.local/source-commit": source.full }; + if (profile === "v02") Object.assign(labels, v02MetricsLabels("hwlab-deepseek-proxy")); const templateLabels = { ...labels, "hwlab.pikastech.local/source-commit": bridgeCommit, @@ -3987,11 +4134,12 @@ function deepSeekProxyManifest({ profile = "dev", source, registryPrefix = defau { name: "moonbridge-config", mountPath: "/config", readOnly: true }, { name: "moonbridge-data", mountPath: "/data" } ] - }], + }, ...(profile === "v02" ? [v02MetricsSidecarContainer({ serviceId: "hwlab-deepseek-proxy", namespace, gitopsTarget: gitopsTargetForProfile(profile) })] : [])], volumes: [ { name: "moonbridge-scripts", configMap: { name: "hwlab-deepseek-proxy-config" } }, { name: "moonbridge-config", emptyDir: {} }, - { name: "moonbridge-data", emptyDir: {} } + { name: "moonbridge-data", emptyDir: {} }, + ...(profile === "v02" ? [v02MetricsSidecarVolume()] : []) ] } } @@ -4001,7 +4149,7 @@ function deepSeekProxyManifest({ profile = "dev", source, registryPrefix = defau apiVersion: "v1", kind: "Service", metadata: { name: "hwlab-deepseek-proxy", namespace, labels }, - spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": "hwlab-deepseek-proxy" }, ports: [{ name: "http", port: 4000, targetPort: "http" }] } + spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": "hwlab-deepseek-proxy" }, ports: [{ name: "http", port: 4000, targetPort: "http" }, ...(profile === "v02" ? [{ name: "metrics", port: 9100, targetPort: "metrics" }] : [])] } } ] }; @@ -4370,7 +4518,7 @@ function v02PostgresManifest({ migrationSql, source }) { function runtimeKustomization({ profile = "dev", includeDeviceAgent = profile === "dev" } = {}) { const resources = ["namespace.yaml", "services.yaml", "health-contract.yaml", "workloads.yaml", "deepseek-proxy.yaml"]; if (profile !== "v02") resources.splice(1, 0, "code-agent-codex-config.yaml"); - if (profile === "v02") resources.push("postgres.yaml"); + if (profile === "v02") resources.push("postgres.yaml", "observability.yaml"); if (includeDeviceAgent) resources.push("device-agent-71-freq.yaml"); resources.push("g14-frpc.yaml"); return { @@ -4436,6 +4584,9 @@ async function plannedFiles(args) { const migrationSql = args.lane === "v02" ? await readFile(path.join(repoRoot, "internal/db/migrations/0001_cloud_core_skeleton.sql"), "utf8") : null; + const metricsSidecarScript = args.lane === "v02" + ? await readFile(path.join(repoRoot, "internal/dev-entrypoint/metrics-sidecar.mjs"), "utf8") + : null; const baseLabels = { "hwlab.pikastech.local/gitops-target": settings.gitopsTarget, "hwlab.pikastech.local/source-commit": source.full }; const annotations = { "hwlab.pikastech.local/rendered-by": "scripts/g14-gitops-render.mjs" }; const files = new Map(); @@ -4503,6 +4654,7 @@ async function plannedFiles(args) { putJson(`${runtimePath}/workloads.yaml`, transformWorkloads({ workloads, deploy, catalog: artifactCatalog, source, sourceBranch: args.sourceBranch, registryPrefix: args.registryPrefix, runtimeEndpoint: endpoints.runtimeEndpoint, webEndpoint: endpoints.webEndpoint, profile, useDeployImages: args.useDeployImages })); putJson(`${runtimePath}/deepseek-proxy.yaml`, deepSeekProxyManifest({ profile, source, registryPrefix: args.registryPrefix, catalog: artifactCatalog, useDeployImages: args.useDeployImages })); if (profile === "v02") putJson(`${runtimePath}/postgres.yaml`, v02PostgresManifest({ migrationSql, source })); + if (profile === "v02") putJson(`${runtimePath}/observability.yaml`, observabilityManifest({ namespace, labels: profileLabels, annotations, metricsSidecarScript })); if (includeDeviceAgent) putJson(`${runtimePath}/device-agent-71-freq.yaml`, deviceAgent71FreqManifest({ profile, source, registryPrefix: args.registryPrefix, catalog: artifactCatalog, useDeployImages: args.useDeployImages })); putJson(`${runtimePath}/g14-frpc.yaml`, g14FrpcManifest({ profile, source })); } diff --git a/scripts/g14-gitops-render.test.ts b/scripts/g14-gitops-render.test.ts index 060fe95c..3eb41b0f 100644 --- a/scripts/g14-gitops-render.test.ts +++ b/scripts/g14-gitops-render.test.ts @@ -212,10 +212,61 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots const workloads = await readFile(path.join(outDir, "runtime-v02", "workloads.yaml"), "utf8"); const workloadsJson = JSON.parse(workloads); const services = await readFile(path.join(outDir, "runtime-v02", "services.yaml"), "utf8"); + const servicesJson = JSON.parse(services); + const deepseekProxy = JSON.parse(await readFile(path.join(outDir, "runtime-v02", "deepseek-proxy.yaml"), "utf8")); + const observability = await readFile(path.join(outDir, "runtime-v02", "observability.yaml"), "utf8"); + const observabilityJson = JSON.parse(observability); for (const serviceId of removedServiceIds) { assert.doesNotMatch(workloads, new RegExp(serviceId, "u")); assert.doesNotMatch(services, new RegExp(serviceId, "u")); } + const observableRuntimeServiceIds = [ + "hwlab-agent-skills", + "hwlab-cloud-api", + "hwlab-cloud-web", + "hwlab-device-pod", + "hwlab-edge-proxy" + ]; + for (const serviceId of observableRuntimeServiceIds) { + const service = servicesJson.items.find((item) => item.kind === "Service" && item.metadata?.name === serviceId); + assert.ok(service, `missing observable service ${serviceId}`); + assert.equal(service.metadata.labels["hwlab.pikastech.local/monitoring"], "enabled"); + assert.ok(service.spec.ports.some((port) => port.name === "metrics" && port.port === 9100 && port.targetPort === "metrics"), `missing metrics port for ${serviceId}`); + const workload = workloadsJson.items.find((item) => item.kind === "Deployment" && item.metadata?.name === serviceId); + assert.ok(workload, `missing observable workload ${serviceId}`); + assert.equal(workload.metadata.labels["hwlab.pikastech.local/monitoring"], "enabled"); + const sidecar = workload.spec.template.spec.containers.find((container) => container.name === "hwlab-metrics"); + assert.ok(sidecar, `missing metrics sidecar for ${serviceId}`); + assert.ok(sidecar.env.some((entry) => entry.name === "HWLAB_METRICS_SERVICE_ID" && entry.value === serviceId)); + assert.ok(sidecar.ports.some((port) => port.name === "metrics" && port.containerPort === 9100)); + } + const deepseekService = deepseekProxy.items.find((item) => item.kind === "Service" && item.metadata?.name === "hwlab-deepseek-proxy"); + assert.equal(deepseekService.metadata.labels["hwlab.pikastech.local/monitoring"], "enabled"); + assert.ok(deepseekService.spec.ports.some((port) => port.name === "metrics" && port.port === 9100)); + const deepseekDeployment = deepseekProxy.items.find((item) => item.kind === "Deployment" && item.metadata?.name === "hwlab-deepseek-proxy"); + assert.ok(deepseekDeployment.spec.template.spec.containers.some((container) => container.name === "hwlab-metrics")); + const observabilityItems = observabilityJson.items ?? []; + const metricsConfig = observabilityItems.find((item) => item.kind === "ConfigMap" && item.metadata?.name === "hwlab-v02-metrics-sidecar"); + assert.ok(metricsConfig, "missing metrics sidecar ConfigMap"); + assert.match(metricsConfig.data["metrics-sidecar.mjs"], /hwlab_service_up/u); + const serviceMonitors = observabilityItems.filter((item) => item.kind === "ServiceMonitor"); + assert.deepEqual(serviceMonitors.map((item) => item.metadata.name).sort(), [ + "hwlab-v02-hwlab-agent-skills", + "hwlab-v02-hwlab-cloud-api", + "hwlab-v02-hwlab-cloud-web", + "hwlab-v02-hwlab-deepseek-proxy", + "hwlab-v02-hwlab-device-pod", + "hwlab-v02-hwlab-edge-proxy" + ]); + for (const monitor of serviceMonitors) { + assert.equal(monitor.metadata.namespace, "hwlab-v02"); + assert.equal(monitor.metadata.labels["hwlab.pikastech.local/monitoring"], "enabled"); + assert.deepEqual(monitor.spec.namespaceSelector.matchNames, ["hwlab-v02"]); + assert.equal(monitor.spec.selector.matchLabels["hwlab.pikastech.local/gitops-target"], "v02"); + assert.equal(monitor.spec.selector.matchLabels["hwlab.pikastech.local/monitoring"], "enabled"); + assert.deepEqual(monitor.spec.endpoints, [{ port: "metrics", path: "/metrics", interval: "30s", scrapeTimeout: "10s" }]); + } + assert.ok(observabilityItems.some((item) => item.kind === "PrometheusRule" && item.metadata?.name === "hwlab-v02-observability")); assert.match(workloads, /"name": "HWLAB_ACCESS_CONTROL_REQUIRED"[\s\S]{0,80}"value": "1"/u); assert.match(workloads, /"name": "HWLAB_BOOTSTRAP_ADMIN_PASSWORD_HASH"/u); assert.match(workloads, /"name": "hwlab-v02-bootstrap-admin"/u);