mirrors/pikasTech-HWLAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
387846c81095726d1bf31552401ee7b75ec81553
pikasTech-HWLAB/deploy/gitops/node/runtime-v03/observability.yaml
T
HWLAB node GitOps Bot 387846c810 chore: promote node GitOps source f3a8615
2026-06-09 01:35:12 +00:00

313 lines
21 KiB
YAML
Raw Blame History

{
"apiVersion": "v1",
"kind": "List",
"items": [
{
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": {
"name": "hwlab-v03-metrics-sidecar",
"namespace": "hwlab-v03",
"labels": {
"hwlab.pikastech.local/gitops-target": "v03",
"hwlab.pikastech.local/source-commit": "f3a8615bd5af9b20ae6b0aa7c83b5f6a2942c126",
"hwlab.pikastech.local/environment": "v03",
"hwlab.pikastech.local/profile": "v03",
"app.kubernetes.io/part-of": "hwlab",
"hwlab.pikastech.local/monitoring": "enabled"
},
"annotations": {
"hwlab.pikastech.local/rendered-by": "scripts/gitops-render.mjs"
}
},
"data": {
"metrics-sidecar.mjs": "import { createServer, request as httpRequest } from \"node:http\";\nimport { request as httpsRequest } from \"node:https\";\nimport { realpathSync } from \"node:fs\";\nimport { pathToFileURL } from \"node:url\";\n\nconst DEFAULT_PORT = 9100;\nconst DEFAULT_TARGET_TIMEOUT_MS = 2000;\nconst DEFAULT_EXTRA_METRICS_MAX_BYTES = 64 * 1024;\nconst startedAtMs = Date.now();\n\nexport function sanitizeLabelValue(value, fallback = \"unknown\") {\n const text = String(value ?? \"\").trim();\n const sanitized = text.replace(/[^A-Za-z0-9_.:-]/gu, \"_\").slice(0, 120);\n return sanitized || fallback;\n}\n\nexport function escapeLabelValue(value) {\n return String(value ?? \"\").replace(/\\\\/gu, \"\\\\\\\\\").replace(/\\n/gu, \"\\\\n\").replace(/\"/gu, \"\\\\\\\"\");\n}\n\nfunction metricLabels(env) {\n const service = sanitizeLabelValue(env.HWLAB_METRICS_SERVICE_ID || env.HWLAB_SERVICE_ID);\n const namespace = sanitizeLabelValue(env.HWLAB_METRICS_NAMESPACE || env.POD_NAMESPACE || \"hwlab-v02\");\n const gitopsTarget = sanitizeLabelValue(env.HWLAB_METRICS_GITOPS_TARGET || env.HWLAB_GITOPS_TARGET || \"v02\");\n return `service=\"${escapeLabelValue(service)}\",namespace=\"${escapeLabelValue(namespace)}\",gitops_target=\"${escapeLabelValue(gitopsTarget)}\"`;\n}\n\nexport function requestTarget({ targetUrl, timeoutMs = DEFAULT_TARGET_TIMEOUT_MS, collectBody = false, maxBodyBytes = DEFAULT_EXTRA_METRICS_MAX_BYTES } = {}) {\n const configured = typeof targetUrl === \"string\" && targetUrl.trim().length > 0;\n if (!configured) return Promise.resolve({ configured: false, success: false, statusCode: 0, durationSeconds: 0 });\n\n let url;\n try {\n url = new URL(targetUrl);\n } catch {\n return Promise.resolve({ configured: true, success: false, statusCode: 0, durationSeconds: 0 });\n }\n\n const requestImpl = url.protocol === \"https:\" ? httpsRequest : url.protocol === \"http:\" ? httpRequest : null;\n if (!requestImpl) return Promise.resolve({ configured: true, success: false, statusCode: 0, durationSeconds: 0 });\n\n const started = Date.now();\n return new Promise((resolve) => {\n const finish = (result) => resolve({\n configured: true,\n success: false,\n statusCode: 0,\n durationSeconds: (Date.now() - started) / 1000,\n ...result\n });\n const request = requestImpl(url, { method: \"GET\", timeout: timeoutMs }, (response) => {\n const chunks = [];\n let bytes = 0;\n if (collectBody) {\n response.on(\"data\", (chunk) => {\n const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);\n bytes += buffer.length;\n if (bytes <= maxBodyBytes) chunks.push(buffer);\n });\n } else {\n response.resume();\n }\n response.on(\"end\", () => finish({\n success: Boolean(response.statusCode && response.statusCode >= 200 && response.statusCode < 300),\n statusCode: response.statusCode || 0,\n body: collectBody ? Buffer.concat(chunks).toString(\"utf8\") : undefined,\n bodyBytes: collectBody ? Math.min(bytes, maxBodyBytes) : 0,\n truncated: collectBody ? bytes > maxBodyBytes : false\n }));\n });\n request.on(\"timeout\", () => request.destroy());\n request.on(\"error\", () => finish({}));\n request.end();\n });\n}\n\nexport async function probeExtraMetrics({ targetUrl, timeoutMs = DEFAULT_TARGET_TIMEOUT_MS, maxBodyBytes = DEFAULT_EXTRA_METRICS_MAX_BYTES, probeImpl = requestTarget } = {}) {\n const configured = typeof targetUrl === \"string\" && targetUrl.trim().length > 0;\n if (!configured || typeof probeImpl !== \"function\") {\n return { configured: false, success: false, statusCode: 0, durationSeconds: 0, text: \"\" };\n }\n try {\n const result = await probeImpl({ targetUrl, timeoutMs, collectBody: true, maxBodyBytes });\n return {\n configured: true,\n success: Boolean(result.success),\n statusCode: result.statusCode || 0,\n durationSeconds: result.durationSeconds || 0,\n text: result.success ? sanitizeExtraMetricsText(result.body || \"\") : \"\"\n };\n } catch {\n return { configured: true, success: false, statusCode: 0, durationSeconds: 0, text: \"\" };\n }\n}\n\nexport function sanitizeExtraMetricsText(text) {\n const lines = String(text ?? \"\").split(/\\r?\\n/gu);\n return lines.filter((line) => {\n if (!line) return true;\n if (/^#\\s+(?:HELP|TYPE)\\s+hwlab_[A-Za-z_:][A-Za-z0-9_:]*\\b/u.test(line)) return true;\n return /^hwlab_[A-Za-z_:][A-Za-z0-9_:]*(?:\\{[^\\n{}]*\\})?\\s+[-+0-9.eE]+$/u.test(line);\n }).join(\"\\n\");\n}\n\nexport async function probeTarget({ targetUrl, timeoutMs = DEFAULT_TARGET_TIMEOUT_MS, probeImpl = requestTarget } = {}) {\n const configured = typeof targetUrl === \"string\" && targetUrl.trim().length > 0;\n if (!configured || typeof probeImpl !== \"function\") {\n return { configured: false, success: false, statusCode: 0, durationSeconds: 0 };\n }\n try {\n return await probeImpl({ targetUrl, timeoutMs });\n } catch {\n const started = Date.now();\n return { configured: true, success: false, statusCode: 0, durationSeconds: (Date.now() - started) / 1000 };\n }\n}\n\nexport function buildMetricsText({ env = process.env, probe = null, extraMetrics = null, now = Date.now() } = {}) {\n const labels = metricLabels(env);\n const uptimeSeconds = Math.max(0, (now - startedAtMs) / 1000);\n const healthProbe = probe ?? { configured: false, success: false, statusCode: 0, durationSeconds: 0 };\n const extraProbe = extraMetrics ?? { configured: false, success: false, statusCode: 0, durationSeconds: 0, text: \"\" };\n const lines = [\n \"# HELP hwlab_service_up HWLAB metrics sidecar availability.\",\n \"# TYPE hwlab_service_up gauge\",\n `hwlab_service_up{${labels}} 1`,\n \"# HELP hwlab_service_info Static HWLAB service metrics identity without high-cardinality request identifiers.\",\n \"# TYPE hwlab_service_info gauge\",\n `hwlab_service_info{${labels}} 1`,\n \"# HELP hwlab_service_process_uptime_seconds Metrics sidecar process uptime in seconds.\",\n \"# TYPE hwlab_service_process_uptime_seconds gauge\",\n `hwlab_service_process_uptime_seconds{${labels}} ${uptimeSeconds.toFixed(3)}`,\n \"# HELP hwlab_service_health_probe_configured Whether the sidecar has a cluster-local health probe target.\",\n \"# TYPE hwlab_service_health_probe_configured gauge\",\n `hwlab_service_health_probe_configured{${labels}} ${healthProbe.configured ? 1 : 0}`,\n \"# HELP hwlab_service_health_probe_success Whether the most recent cluster-local health probe succeeded.\",\n \"# TYPE hwlab_service_health_probe_success gauge\",\n `hwlab_service_health_probe_success{${labels}} ${healthProbe.success ? 1 : 0}`,\n \"# HELP hwlab_service_health_probe_status_code Most recent cluster-local health probe HTTP status code, or 0 when unavailable.\",\n \"# TYPE hwlab_service_health_probe_status_code gauge\",\n `hwlab_service_health_probe_status_code{${labels}} ${Number.isInteger(healthProbe.statusCode) ? healthProbe.statusCode : 0}`,\n \"# HELP hwlab_service_health_probe_duration_seconds Most recent cluster-local health probe duration in seconds.\",\n \"# TYPE hwlab_service_health_probe_duration_seconds gauge\",\n `hwlab_service_health_probe_duration_seconds{${labels}} ${Number(healthProbe.durationSeconds || 0).toFixed(3)}`,\n \"# HELP hwlab_service_extra_metrics_configured Whether the sidecar has an additional cluster-local metrics target.\",\n \"# TYPE hwlab_service_extra_metrics_configured gauge\",\n `hwlab_service_extra_metrics_configured{${labels}} ${extraProbe.configured ? 1 : 0}`,\n \"# HELP hwlab_service_extra_metrics_success Whether the most recent additional metrics fetch succeeded.\",\n \"# TYPE hwlab_service_extra_metrics_success gauge\",\n `hwlab_service_extra_metrics_success{${labels}} ${extraProbe.success ? 1 : 0}`,\n \"# HELP hwlab_service_extra_metrics_status_code Most recent additional metrics HTTP status code, or 0 when unavailable.\",\n \"# TYPE hwlab_service_extra_metrics_status_code gauge\",\n `hwlab_service_extra_metrics_status_code{${labels}} ${Number.isInteger(extraProbe.statusCode) ? extraProbe.statusCode : 0}`,\n \"# HELP hwlab_service_extra_metrics_duration_seconds Most recent additional metrics fetch duration in seconds.\",\n \"# TYPE hwlab_service_extra_metrics_duration_seconds gauge\",\n `hwlab_service_extra_metrics_duration_seconds{${labels}} ${Number(extraProbe.durationSeconds || 0).toFixed(3)}`,\n \"\"\n ];\n if (extraProbe.text) {\n lines.push(extraProbe.text.trimEnd(), \"\");\n }\n return lines.join(\"\\n\");\n}\n\nfunction sendJson(response, statusCode, body) {\n const payload = JSON.stringify(body) + \"\\n\";\n response.writeHead(statusCode, {\n \"content-type\": \"application/json; charset=utf-8\",\n \"content-length\": Buffer.byteLength(payload)\n });\n response.end(payload);\n}\n\nexport function createMetricsSidecarServer({ env = process.env, probeImpl = requestTarget } = {}) {\n return createServer(async (request, response) => {\n const url = new URL(request.url || \"/\", \"http://hwlab-metrics.local\");\n if (request.method === \"GET\" && (url.pathname === \"/health\" || url.pathname === \"/health/live\")) {\n sendJson(response, 200, {\n serviceId: sanitizeLabelValue(env.HWLAB_METRICS_SERVICE_ID || env.HWLAB_SERVICE_ID),\n status: \"ok\",\n metricsPath: \"/metrics\"\n });\n return;\n }\n if (request.method === \"GET\" && url.pathname === \"/metrics\") {\n const timeoutMs = Number.parseInt(env.HWLAB_METRICS_TARGET_TIMEOUT_MS || \"\", 10) || DEFAULT_TARGET_TIMEOUT_MS;\n const probe = await probeTarget({ targetUrl: env.HWLAB_METRICS_TARGET_URL, timeoutMs, probeImpl });\n const extraTimeoutMs = Number.parseInt(env.HWLAB_METRICS_EXTRA_TIMEOUT_MS || \"\", 10) || DEFAULT_TARGET_TIMEOUT_MS;\n const extraMetrics = await probeExtraMetrics({ targetUrl: env.HWLAB_METRICS_EXTRA_URL, timeoutMs: extraTimeoutMs, probeImpl });\n const payload = buildMetricsText({ env, probe, extraMetrics });\n response.writeHead(200, {\n \"content-type\": \"text/plain; version=0.0.4; charset=utf-8\",\n \"content-length\": Buffer.byteLength(payload)\n });\n response.end(payload);\n return;\n }\n sendJson(response, 404, { error: \"not_found\", path: url.pathname });\n });\n}\n\nexport function startMetricsSidecar({ env = process.env } = {}) {\n const port = Number.parseInt(env.HWLAB_METRICS_PORT || env.PORT || \"\", 10) || DEFAULT_PORT;\n const server = createMetricsSidecarServer({ env });\n server.listen(port, \"0.0.0.0\", () => {\n process.stdout.write(JSON.stringify({ serviceId: env.HWLAB_METRICS_SERVICE_ID || \"hwlab-unknown\", status: \"metrics-listening\", port }) + \"\\n\");\n });\n return server;\n}\n\nexport function isMainModule({ moduleUrl = import.meta.url, argvPath = process.argv[1], realpath = realpathSync } = {}) {\n if (!argvPath) return false;\n if (moduleUrl === pathToFileURL(argvPath).href) return true;\n try {\n return moduleUrl === pathToFileURL(realpath(argvPath)).href;\n } catch {\n return false;\n }\n}\n\nif (isMainModule()) {\n startMetricsSidecar();\n}\n"
}
},
{
"apiVersion": "monitoring.coreos.com/v1",
"kind": "ServiceMonitor",
"metadata": {
"name": "hwlab-v03-hwlab-cloud-api",
"namespace": "hwlab-v03",
"labels": {
"hwlab.pikastech.local/gitops-target": "v03",
"hwlab.pikastech.local/source-commit": "f3a8615bd5af9b20ae6b0aa7c83b5f6a2942c126",
"hwlab.pikastech.local/environment": "v03",
"hwlab.pikastech.local/profile": "v03",
"app.kubernetes.io/part-of": "hwlab",
"hwlab.pikastech.local/monitoring": "enabled",
"hwlab.pikastech.local/service-id": "hwlab-cloud-api"
},
"annotations": {
"hwlab.pikastech.local/rendered-by": "scripts/gitops-render.mjs"
}
},
"spec": {
"namespaceSelector": {
"matchNames": [
"hwlab-v03"
]
},
"selector": {
"matchLabels": {
"hwlab.pikastech.local/gitops-target": "v03",
"hwlab.pikastech.local/monitoring": "enabled",
"hwlab.pikastech.local/service-id": "hwlab-cloud-api"
}
},
"targetLabels": [
"hwlab.pikastech.local/gitops-target",
"hwlab.pikastech.local/service-id"
],
"endpoints": [
{
"port": "metrics",
"path": "/metrics",
"interval": "30s",
"scrapeTimeout": "10s"
}
]
}
},
{
"apiVersion": "monitoring.coreos.com/v1",
"kind": "ServiceMonitor",
"metadata": {
"name": "hwlab-v03-hwlab-cloud-web",
"namespace": "hwlab-v03",
"labels": {
"hwlab.pikastech.local/gitops-target": "v03",
"hwlab.pikastech.local/source-commit": "f3a8615bd5af9b20ae6b0aa7c83b5f6a2942c126",
"hwlab.pikastech.local/environment": "v03",
"hwlab.pikastech.local/profile": "v03",
"app.kubernetes.io/part-of": "hwlab",
"hwlab.pikastech.local/monitoring": "enabled",
"hwlab.pikastech.local/service-id": "hwlab-cloud-web"
},
"annotations": {
"hwlab.pikastech.local/rendered-by": "scripts/gitops-render.mjs"
}
},
"spec": {
"namespaceSelector": {
"matchNames": [
"hwlab-v03"
]
},
"selector": {
"matchLabels": {
"hwlab.pikastech.local/gitops-target": "v03",
"hwlab.pikastech.local/monitoring": "enabled",
"hwlab.pikastech.local/service-id": "hwlab-cloud-web"
}
},
"targetLabels": [
"hwlab.pikastech.local/gitops-target",
"hwlab.pikastech.local/service-id"
],
"endpoints": [
{
"port": "metrics",
"path": "/metrics",
"interval": "30s",
"scrapeTimeout": "10s"
}
]
}
},
{
"apiVersion": "monitoring.coreos.com/v1",
"kind": "ServiceMonitor",
"metadata": {
"name": "hwlab-v03-hwlab-edge-proxy",
"namespace": "hwlab-v03",
"labels": {
"hwlab.pikastech.local/gitops-target": "v03",
"hwlab.pikastech.local/source-commit": "f3a8615bd5af9b20ae6b0aa7c83b5f6a2942c126",
"hwlab.pikastech.local/environment": "v03",
"hwlab.pikastech.local/profile": "v03",
"app.kubernetes.io/part-of": "hwlab",
"hwlab.pikastech.local/monitoring": "enabled",
"hwlab.pikastech.local/service-id": "hwlab-edge-proxy"
},
"annotations": {
"hwlab.pikastech.local/rendered-by": "scripts/gitops-render.mjs"
}
},
"spec": {
"namespaceSelector": {
"matchNames": [
"hwlab-v03"
]
},
"selector": {
"matchLabels": {
"hwlab.pikastech.local/gitops-target": "v03",
"hwlab.pikastech.local/monitoring": "enabled",
"hwlab.pikastech.local/service-id": "hwlab-edge-proxy"
}
},
"targetLabels": [
"hwlab.pikastech.local/gitops-target",
"hwlab.pikastech.local/service-id"
],
"endpoints": [
{
"port": "metrics",
"path": "/metrics",
"interval": "30s",
"scrapeTimeout": "10s"
}
]
}
},
{
"apiVersion": "monitoring.coreos.com/v1",
"kind": "ServiceMonitor",
"metadata": {
"name": "hwlab-v03-hwlab-agent-skills",
"namespace": "hwlab-v03",
"labels": {
"hwlab.pikastech.local/gitops-target": "v03",
"hwlab.pikastech.local/source-commit": "f3a8615bd5af9b20ae6b0aa7c83b5f6a2942c126",
"hwlab.pikastech.local/environment": "v03",
"hwlab.pikastech.local/profile": "v03",
"app.kubernetes.io/part-of": "hwlab",
"hwlab.pikastech.local/monitoring": "enabled",
"hwlab.pikastech.local/service-id": "hwlab-agent-skills"
},
"annotations": {
"hwlab.pikastech.local/rendered-by": "scripts/gitops-render.mjs"
}
},
"spec": {
"namespaceSelector": {
"matchNames": [
"hwlab-v03"
]
},
"selector": {
"matchLabels": {
"hwlab.pikastech.local/gitops-target": "v03",
"hwlab.pikastech.local/monitoring": "enabled",
"hwlab.pikastech.local/service-id": "hwlab-agent-skills"
}
},
"targetLabels": [
"hwlab.pikastech.local/gitops-target",
"hwlab.pikastech.local/service-id"
],
"endpoints": [
{
"port": "metrics",
"path": "/metrics",
"interval": "30s",
"scrapeTimeout": "10s"
}
]
}
},
{
"apiVersion": "monitoring.coreos.com/v1",
"kind": "ServiceMonitor",
"metadata": {
"name": "hwlab-v03-hwlab-deepseek-proxy",
"namespace": "hwlab-v03",
"labels": {
"hwlab.pikastech.local/gitops-target": "v03",
"hwlab.pikastech.local/source-commit": "f3a8615bd5af9b20ae6b0aa7c83b5f6a2942c126",
"hwlab.pikastech.local/environment": "v03",
"hwlab.pikastech.local/profile": "v03",
"app.kubernetes.io/part-of": "hwlab",
"hwlab.pikastech.local/monitoring": "enabled",
"hwlab.pikastech.local/service-id": "hwlab-deepseek-proxy"
},
"annotations": {
"hwlab.pikastech.local/rendered-by": "scripts/gitops-render.mjs"
}
},
"spec": {
"namespaceSelector": {
"matchNames": [
"hwlab-v03"
]
},
"selector": {
"matchLabels": {
"hwlab.pikastech.local/gitops-target": "v03",
"hwlab.pikastech.local/monitoring": "enabled",
"hwlab.pikastech.local/service-id": "hwlab-deepseek-proxy"
}
},
"targetLabels": [
"hwlab.pikastech.local/gitops-target",
"hwlab.pikastech.local/service-id"
],
"endpoints": [
{
"port": "metrics",
"path": "/metrics",
"interval": "30s",
"scrapeTimeout": "10s"
}
]
}
},
{
"apiVersion": "monitoring.coreos.com/v1",
"kind": "PrometheusRule",
"metadata": {
"name": "hwlab-v03-observability",
"namespace": "hwlab-v03",
"labels": {
"hwlab.pikastech.local/gitops-target": "v03",
"hwlab.pikastech.local/source-commit": "f3a8615bd5af9b20ae6b0aa7c83b5f6a2942c126",
"hwlab.pikastech.local/environment": "v03",
"hwlab.pikastech.local/profile": "v03",
"app.kubernetes.io/part-of": "hwlab",
"hwlab.pikastech.local/monitoring": "enabled"
},
"annotations": {
"hwlab.pikastech.local/rendered-by": "scripts/gitops-render.mjs"
}
},
"spec": {
"groups": [
{
"name": "hwlab-v03-observability",
"rules": [
{
"alert": "HWLABV03MetricsTargetDown",
"expr": "up{namespace=\"hwlab-v03\"} == 0",
"for": "5m",
"labels": {
"severity": "warning",
"lane": "v03"
},
"annotations": {
"summary": "HWLAB v0.3 metrics target is down",
"description": "Prometheus cannot scrape a HWLAB v0.3 metrics target."
}
},
{
"alert": "HWLABV03ServiceHealthProbeFailed",
"expr": "hwlab_service_health_probe_success{namespace=\"hwlab-v03\"} == 0",
"for": "5m",
"labels": {
"severity": "warning",
"lane": "v03"
},
"annotations": {
"summary": "HWLAB v0.2 service health probe failed",
"description": "The metrics sidecar cannot reach the service health endpoint inside the pod network."
}
}
]
}
]
}
}
]
}
Reference in New Issue View Git Blame Copy Permalink