feat: add v02 observability metrics sidecars
This commit is contained in:
Codex Agent
@@ -46,6 +46,15 @@ const v02RuntimeServiceIds = Object.freeze([
|
||||
"hwlab-edge-proxy",
|
||||
"hwlab-agent-skills"
|
||||
]);
|
||||
const v02ObservableServices = Object.freeze([
|
||||
{ serviceId: "hwlab-cloud-api", port: 6667, healthPath: "/health/live" },
|
||||
{ serviceId: "hwlab-cloud-web", port: 8080, healthPath: "/health/live" },
|
||||
{ serviceId: "hwlab-device-pod", port: 7601, healthPath: "/health/live" },
|
||||
{ serviceId: "hwlab-edge-proxy", port: 6667, healthPath: "/health" },
|
||||
{ serviceId: "hwlab-agent-skills", port: 7430, healthPath: "/health/live" },
|
||||
{ serviceId: "hwlab-deepseek-proxy", port: 4000, healthPath: "/health/live" }
|
||||
]);
|
||||
const v02ObservableServiceIds = new Set(v02ObservableServices.map((item) => item.serviceId));
|
||||
const v02RemovedServiceIds = new Set([
|
||||
"hwlab-agent-mgr",
|
||||
"hwlab-agent-worker",
|
||||
@@ -526,6 +535,10 @@ function serviceIdsForProfile(profile) {
|
||||
return profile === "v02" ? v02RuntimeServiceIds : defaultServiceIds;
|
||||
}
|
||||
|
||||
function v02ObservableService(serviceId) {
|
||||
return v02ObservableServices.find((item) => item.serviceId === serviceId) ?? null;
|
||||
}
|
||||
|
||||
function servicesParamForLane(lane) {
|
||||
return serviceIdsForLane(lane).join(",");
|
||||
}
|
||||
@@ -774,6 +787,56 @@ function deployServicesForProfile(deploy, profile) {
|
||||
return services;
|
||||
}
|
||||
|
||||
function v02MetricsLabels(serviceId) {
|
||||
return {
|
||||
"hwlab.pikastech.local/monitoring": "enabled",
|
||||
"hwlab.pikastech.local/service-id": serviceId
|
||||
};
|
||||
}
|
||||
|
||||
function v02MetricsSidecarVolume() {
|
||||
return { name: "hwlab-metrics-sidecar", configMap: { name: "hwlab-v02-metrics-sidecar" } };
|
||||
}
|
||||
|
||||
function v02MetricsSidecarContainer({ serviceId, namespace, gitopsTarget }) {
|
||||
const service = v02ObservableService(serviceId);
|
||||
assert.ok(service, `unknown v0.2 observable service ${serviceId}`);
|
||||
return {
|
||||
name: "hwlab-metrics",
|
||||
image: ciToolsRunnerImage,
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
command: ["node", "/metrics/metrics-sidecar.mjs"],
|
||||
env: [
|
||||
{ name: "HWLAB_METRICS_SERVICE_ID", value: serviceId },
|
||||
{ name: "HWLAB_METRICS_NAMESPACE", value: namespace },
|
||||
{ name: "HWLAB_METRICS_GITOPS_TARGET", value: gitopsTarget },
|
||||
{ name: "HWLAB_METRICS_PORT", value: "9100" },
|
||||
{ name: "HWLAB_METRICS_TARGET_URL", value: `http://127.0.0.1:${service.port}${service.healthPath}` },
|
||||
{ name: "HWLAB_METRICS_TARGET_TIMEOUT_MS", value: "2000" }
|
||||
],
|
||||
ports: [{ name: "metrics", containerPort: 9100 }],
|
||||
readinessProbe: { httpGet: { path: "/health/live", port: "metrics" }, initialDelaySeconds: 3, periodSeconds: 10 },
|
||||
livenessProbe: { httpGet: { path: "/health/live", port: "metrics" }, initialDelaySeconds: 10, periodSeconds: 20 },
|
||||
volumeMounts: [{ name: "hwlab-metrics-sidecar", mountPath: "/metrics", readOnly: true }]
|
||||
};
|
||||
}
|
||||
|
||||
function upsertV02MetricsSidecar(podSpec, options) {
|
||||
if (!podSpec || !Array.isArray(podSpec.containers)) return;
|
||||
podSpec.containers = podSpec.containers.filter((container) => container?.name !== "hwlab-metrics");
|
||||
podSpec.containers.push(v02MetricsSidecarContainer(options));
|
||||
podSpec.volumes ??= [];
|
||||
podSpec.volumes = podSpec.volumes.filter((volume) => volume?.name !== "hwlab-metrics-sidecar");
|
||||
podSpec.volumes.push(v02MetricsSidecarVolume());
|
||||
}
|
||||
|
||||
function upsertV02MetricsPort(service) {
|
||||
service.spec ??= {};
|
||||
service.spec.ports ??= [];
|
||||
service.spec.ports = service.spec.ports.filter((port) => port?.name !== "metrics");
|
||||
service.spec.ports.push({ name: "metrics", port: 9100, targetPort: "metrics" });
|
||||
}
|
||||
|
||||
function transformWorkloads({ workloads, deploy, catalog, source, sourceBranch = defaultBranch, registryPrefix, runtimeEndpoint, webEndpoint, profile = "dev", useDeployImages = false }) {
|
||||
const result = cloneJson(workloads);
|
||||
const deployServices = deployServicesForProfile(deploy, profile);
|
||||
@@ -822,6 +885,11 @@ function transformWorkloads({ workloads, deploy, catalog, source, sourceBranch =
|
||||
"hwlab.pikastech.local/source-commit": templateSourceCommit,
|
||||
"hwlab.pikastech.local/artifact-source-commit": templateArtifactCommit
|
||||
});
|
||||
if (profile === "v02" && v02ObservableServiceIds.has(templateServiceId)) {
|
||||
label(item.metadata, v02MetricsLabels(templateServiceId));
|
||||
label(podTemplate.metadata, v02MetricsLabels(templateServiceId));
|
||||
upsertV02MetricsSidecar(podTemplate.spec, { serviceId: templateServiceId, namespace, gitopsTarget });
|
||||
}
|
||||
if ((item.kind === "Deployment" || item.kind === "StatefulSet") && item.spec?.selector?.matchLabels) {
|
||||
item.spec.selector.matchLabels = {
|
||||
...item.spec.selector.matchLabels,
|
||||
@@ -983,10 +1051,88 @@ function transformServices({ services, namespace, labels, annotations, profile =
|
||||
const result = transformListNamespace(services, namespace, labels, annotations);
|
||||
if (result.kind === "List") {
|
||||
result.items = (result.items ?? []).filter((item) => !(profile === "v02" && isV02RemovedServiceId(serviceIdForWorkload(item, null))));
|
||||
for (const item of result.items) {
|
||||
const serviceId = serviceIdForWorkload(item, null);
|
||||
if (profile !== "v02" || !v02ObservableServiceIds.has(serviceId)) continue;
|
||||
label(item.metadata ??= {}, v02MetricsLabels(serviceId));
|
||||
upsertV02MetricsPort(item);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function observabilityManifest({ namespace, labels, annotations, metricsSidecarScript }) {
|
||||
const baseLabels = {
|
||||
...labels,
|
||||
"app.kubernetes.io/part-of": "hwlab",
|
||||
"hwlab.pikastech.local/monitoring": "enabled"
|
||||
};
|
||||
const serviceMonitors = v02ObservableServices.map((service) => ({
|
||||
apiVersion: "monitoring.coreos.com/v1",
|
||||
kind: "ServiceMonitor",
|
||||
metadata: {
|
||||
name: `hwlab-v02-${service.serviceId}`,
|
||||
namespace,
|
||||
labels: { ...baseLabels, "hwlab.pikastech.local/service-id": service.serviceId },
|
||||
annotations
|
||||
},
|
||||
spec: {
|
||||
namespaceSelector: { matchNames: [namespace] },
|
||||
selector: {
|
||||
matchLabels: {
|
||||
"hwlab.pikastech.local/gitops-target": "v02",
|
||||
"hwlab.pikastech.local/monitoring": "enabled",
|
||||
"hwlab.pikastech.local/service-id": service.serviceId
|
||||
}
|
||||
},
|
||||
targetLabels: [
|
||||
"hwlab.pikastech.local/gitops-target",
|
||||
"hwlab.pikastech.local/service-id"
|
||||
],
|
||||
endpoints: [{ port: "metrics", path: "/metrics", interval: "30s", scrapeTimeout: "10s" }]
|
||||
}
|
||||
}));
|
||||
return {
|
||||
apiVersion: "v1",
|
||||
kind: "List",
|
||||
items: [
|
||||
{
|
||||
apiVersion: "v1",
|
||||
kind: "ConfigMap",
|
||||
metadata: { name: "hwlab-v02-metrics-sidecar", namespace, labels: baseLabels, annotations },
|
||||
data: { "metrics-sidecar.mjs": metricsSidecarScript }
|
||||
},
|
||||
...serviceMonitors,
|
||||
{
|
||||
apiVersion: "monitoring.coreos.com/v1",
|
||||
kind: "PrometheusRule",
|
||||
metadata: { name: "hwlab-v02-observability", namespace, labels: baseLabels, annotations },
|
||||
spec: {
|
||||
groups: [{
|
||||
name: "hwlab-v02-observability",
|
||||
rules: [
|
||||
{
|
||||
alert: "HWLABV02MetricsTargetDown",
|
||||
expr: 'up{namespace="hwlab-v02"} == 0',
|
||||
for: "5m",
|
||||
labels: { severity: "warning", lane: "v02" },
|
||||
annotations: { summary: "HWLAB v0.2 metrics target is down", description: "Prometheus cannot scrape a HWLAB v0.2 metrics target." }
|
||||
},
|
||||
{
|
||||
alert: "HWLABV02ServiceHealthProbeFailed",
|
||||
expr: 'hwlab_service_health_probe_success{namespace="hwlab-v02"} == 0',
|
||||
for: "5m",
|
||||
labels: { severity: "warning", lane: "v02" },
|
||||
annotations: { summary: "HWLAB v0.2 service health probe failed", description: "The metrics sidecar cannot reach the service health endpoint inside the pod network." }
|
||||
}
|
||||
]
|
||||
}]
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function shellSingleQuote(value) {
|
||||
return `'${String(value).replace(/'/g, `'"'"'`)}'`;
|
||||
}
|
||||
@@ -3914,6 +4060,7 @@ function deepSeekProxyManifest({ profile = "dev", source, registryPrefix = defau
|
||||
"hwlab.pikastech.local/service-id": "hwlab-deepseek-proxy",
|
||||
"hwlab.pikastech.local/source-commit": source.full
|
||||
};
|
||||
if (profile === "v02") Object.assign(labels, v02MetricsLabels("hwlab-deepseek-proxy"));
|
||||
const templateLabels = {
|
||||
...labels,
|
||||
"hwlab.pikastech.local/source-commit": bridgeCommit,
|
||||
@@ -3987,11 +4134,12 @@ function deepSeekProxyManifest({ profile = "dev", source, registryPrefix = defau
|
||||
{ name: "moonbridge-config", mountPath: "/config", readOnly: true },
|
||||
{ name: "moonbridge-data", mountPath: "/data" }
|
||||
]
|
||||
}],
|
||||
}, ...(profile === "v02" ? [v02MetricsSidecarContainer({ serviceId: "hwlab-deepseek-proxy", namespace, gitopsTarget: gitopsTargetForProfile(profile) })] : [])],
|
||||
volumes: [
|
||||
{ name: "moonbridge-scripts", configMap: { name: "hwlab-deepseek-proxy-config" } },
|
||||
{ name: "moonbridge-config", emptyDir: {} },
|
||||
{ name: "moonbridge-data", emptyDir: {} }
|
||||
{ name: "moonbridge-data", emptyDir: {} },
|
||||
...(profile === "v02" ? [v02MetricsSidecarVolume()] : [])
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -4001,7 +4149,7 @@ function deepSeekProxyManifest({ profile = "dev", source, registryPrefix = defau
|
||||
apiVersion: "v1",
|
||||
kind: "Service",
|
||||
metadata: { name: "hwlab-deepseek-proxy", namespace, labels },
|
||||
spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": "hwlab-deepseek-proxy" }, ports: [{ name: "http", port: 4000, targetPort: "http" }] }
|
||||
spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": "hwlab-deepseek-proxy" }, ports: [{ name: "http", port: 4000, targetPort: "http" }, ...(profile === "v02" ? [{ name: "metrics", port: 9100, targetPort: "metrics" }] : [])] }
|
||||
}
|
||||
]
|
||||
};
|
||||
@@ -4370,7 +4518,7 @@ function v02PostgresManifest({ migrationSql, source }) {
|
||||
function runtimeKustomization({ profile = "dev", includeDeviceAgent = profile === "dev" } = {}) {
|
||||
const resources = ["namespace.yaml", "services.yaml", "health-contract.yaml", "workloads.yaml", "deepseek-proxy.yaml"];
|
||||
if (profile !== "v02") resources.splice(1, 0, "code-agent-codex-config.yaml");
|
||||
if (profile === "v02") resources.push("postgres.yaml");
|
||||
if (profile === "v02") resources.push("postgres.yaml", "observability.yaml");
|
||||
if (includeDeviceAgent) resources.push("device-agent-71-freq.yaml");
|
||||
resources.push("g14-frpc.yaml");
|
||||
return {
|
||||
@@ -4436,6 +4584,9 @@ async function plannedFiles(args) {
|
||||
const migrationSql = args.lane === "v02"
|
||||
? await readFile(path.join(repoRoot, "internal/db/migrations/0001_cloud_core_skeleton.sql"), "utf8")
|
||||
: null;
|
||||
const metricsSidecarScript = args.lane === "v02"
|
||||
? await readFile(path.join(repoRoot, "internal/dev-entrypoint/metrics-sidecar.mjs"), "utf8")
|
||||
: null;
|
||||
const baseLabels = { "hwlab.pikastech.local/gitops-target": settings.gitopsTarget, "hwlab.pikastech.local/source-commit": source.full };
|
||||
const annotations = { "hwlab.pikastech.local/rendered-by": "scripts/g14-gitops-render.mjs" };
|
||||
const files = new Map();
|
||||
@@ -4503,6 +4654,7 @@ async function plannedFiles(args) {
|
||||
putJson(`${runtimePath}/workloads.yaml`, transformWorkloads({ workloads, deploy, catalog: artifactCatalog, source, sourceBranch: args.sourceBranch, registryPrefix: args.registryPrefix, runtimeEndpoint: endpoints.runtimeEndpoint, webEndpoint: endpoints.webEndpoint, profile, useDeployImages: args.useDeployImages }));
|
||||
putJson(`${runtimePath}/deepseek-proxy.yaml`, deepSeekProxyManifest({ profile, source, registryPrefix: args.registryPrefix, catalog: artifactCatalog, useDeployImages: args.useDeployImages }));
|
||||
if (profile === "v02") putJson(`${runtimePath}/postgres.yaml`, v02PostgresManifest({ migrationSql, source }));
|
||||
if (profile === "v02") putJson(`${runtimePath}/observability.yaml`, observabilityManifest({ namespace, labels: profileLabels, annotations, metricsSidecarScript }));
|
||||
if (includeDeviceAgent) putJson(`${runtimePath}/device-agent-71-freq.yaml`, deviceAgent71FreqManifest({ profile, source, registryPrefix: args.registryPrefix, catalog: artifactCatalog, useDeployImages: args.useDeployImages }));
|
||||
putJson(`${runtimePath}/g14-frpc.yaml`, g14FrpcManifest({ profile, source }));
|
||||
}
|
||||
|
||||
@@ -212,10 +212,61 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots
|
||||
const workloads = await readFile(path.join(outDir, "runtime-v02", "workloads.yaml"), "utf8");
|
||||
const workloadsJson = JSON.parse(workloads);
|
||||
const services = await readFile(path.join(outDir, "runtime-v02", "services.yaml"), "utf8");
|
||||
const servicesJson = JSON.parse(services);
|
||||
const deepseekProxy = JSON.parse(await readFile(path.join(outDir, "runtime-v02", "deepseek-proxy.yaml"), "utf8"));
|
||||
const observability = await readFile(path.join(outDir, "runtime-v02", "observability.yaml"), "utf8");
|
||||
const observabilityJson = JSON.parse(observability);
|
||||
for (const serviceId of removedServiceIds) {
|
||||
assert.doesNotMatch(workloads, new RegExp(serviceId, "u"));
|
||||
assert.doesNotMatch(services, new RegExp(serviceId, "u"));
|
||||
}
|
||||
const observableRuntimeServiceIds = [
|
||||
"hwlab-agent-skills",
|
||||
"hwlab-cloud-api",
|
||||
"hwlab-cloud-web",
|
||||
"hwlab-device-pod",
|
||||
"hwlab-edge-proxy"
|
||||
];
|
||||
for (const serviceId of observableRuntimeServiceIds) {
|
||||
const service = servicesJson.items.find((item) => item.kind === "Service" && item.metadata?.name === serviceId);
|
||||
assert.ok(service, `missing observable service ${serviceId}`);
|
||||
assert.equal(service.metadata.labels["hwlab.pikastech.local/monitoring"], "enabled");
|
||||
assert.ok(service.spec.ports.some((port) => port.name === "metrics" && port.port === 9100 && port.targetPort === "metrics"), `missing metrics port for ${serviceId}`);
|
||||
const workload = workloadsJson.items.find((item) => item.kind === "Deployment" && item.metadata?.name === serviceId);
|
||||
assert.ok(workload, `missing observable workload ${serviceId}`);
|
||||
assert.equal(workload.metadata.labels["hwlab.pikastech.local/monitoring"], "enabled");
|
||||
const sidecar = workload.spec.template.spec.containers.find((container) => container.name === "hwlab-metrics");
|
||||
assert.ok(sidecar, `missing metrics sidecar for ${serviceId}`);
|
||||
assert.ok(sidecar.env.some((entry) => entry.name === "HWLAB_METRICS_SERVICE_ID" && entry.value === serviceId));
|
||||
assert.ok(sidecar.ports.some((port) => port.name === "metrics" && port.containerPort === 9100));
|
||||
}
|
||||
const deepseekService = deepseekProxy.items.find((item) => item.kind === "Service" && item.metadata?.name === "hwlab-deepseek-proxy");
|
||||
assert.equal(deepseekService.metadata.labels["hwlab.pikastech.local/monitoring"], "enabled");
|
||||
assert.ok(deepseekService.spec.ports.some((port) => port.name === "metrics" && port.port === 9100));
|
||||
const deepseekDeployment = deepseekProxy.items.find((item) => item.kind === "Deployment" && item.metadata?.name === "hwlab-deepseek-proxy");
|
||||
assert.ok(deepseekDeployment.spec.template.spec.containers.some((container) => container.name === "hwlab-metrics"));
|
||||
const observabilityItems = observabilityJson.items ?? [];
|
||||
const metricsConfig = observabilityItems.find((item) => item.kind === "ConfigMap" && item.metadata?.name === "hwlab-v02-metrics-sidecar");
|
||||
assert.ok(metricsConfig, "missing metrics sidecar ConfigMap");
|
||||
assert.match(metricsConfig.data["metrics-sidecar.mjs"], /hwlab_service_up/u);
|
||||
const serviceMonitors = observabilityItems.filter((item) => item.kind === "ServiceMonitor");
|
||||
assert.deepEqual(serviceMonitors.map((item) => item.metadata.name).sort(), [
|
||||
"hwlab-v02-hwlab-agent-skills",
|
||||
"hwlab-v02-hwlab-cloud-api",
|
||||
"hwlab-v02-hwlab-cloud-web",
|
||||
"hwlab-v02-hwlab-deepseek-proxy",
|
||||
"hwlab-v02-hwlab-device-pod",
|
||||
"hwlab-v02-hwlab-edge-proxy"
|
||||
]);
|
||||
for (const monitor of serviceMonitors) {
|
||||
assert.equal(monitor.metadata.namespace, "hwlab-v02");
|
||||
assert.equal(monitor.metadata.labels["hwlab.pikastech.local/monitoring"], "enabled");
|
||||
assert.deepEqual(monitor.spec.namespaceSelector.matchNames, ["hwlab-v02"]);
|
||||
assert.equal(monitor.spec.selector.matchLabels["hwlab.pikastech.local/gitops-target"], "v02");
|
||||
assert.equal(monitor.spec.selector.matchLabels["hwlab.pikastech.local/monitoring"], "enabled");
|
||||
assert.deepEqual(monitor.spec.endpoints, [{ port: "metrics", path: "/metrics", interval: "30s", scrapeTimeout: "10s" }]);
|
||||
}
|
||||
assert.ok(observabilityItems.some((item) => item.kind === "PrometheusRule" && item.metadata?.name === "hwlab-v02-observability"));
|
||||
assert.match(workloads, /"name": "HWLAB_ACCESS_CONTROL_REQUIRED"[\s\S]{0,80}"value": "1"/u);
|
||||
assert.match(workloads, /"name": "HWLAB_BOOTSTRAP_ADMIN_PASSWORD_HASH"/u);
|
||||
assert.match(workloads, /"name": "hwlab-v02-bootstrap-admin"/u);
|
||||
|
||||
Reference in New Issue
Block a user