refactor: split GitOps renderer by responsibility
This commit is contained in:
root
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,254 @@
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
ciToolsRunnerImage,
|
||||
configObject,
|
||||
defaultOutDir,
|
||||
label,
|
||||
optionalConfigString,
|
||||
requiredConfigPositiveInteger,
|
||||
requiredConfigString,
|
||||
runtimeLaneMirrorSpecs
|
||||
} from "./core.mjs";
|
||||
import { renderTemplate, shellSingleQuote } from "./tekton-scripts.mjs";
|
||||
|
||||
function registryManifest() {
|
||||
return {
|
||||
apiVersion: "v1",
|
||||
kind: "List",
|
||||
items: [
|
||||
{ apiVersion: "v1", kind: "Namespace", metadata: { name: "hwlab-ci" } },
|
||||
{
|
||||
apiVersion: "apps/v1",
|
||||
kind: "Deployment",
|
||||
metadata: { name: "hwlab-registry", namespace: "hwlab-ci", labels: { "app.kubernetes.io/name": "hwlab-registry" } },
|
||||
spec: {
|
||||
replicas: 1,
|
||||
selector: { matchLabels: { "app.kubernetes.io/name": "hwlab-registry" } },
|
||||
template: {
|
||||
metadata: { labels: { "app.kubernetes.io/name": "hwlab-registry" } },
|
||||
spec: {
|
||||
hostNetwork: true,
|
||||
dnsPolicy: "ClusterFirstWithHostNet",
|
||||
containers: [{
|
||||
name: "registry",
|
||||
image: "registry:2.8.3",
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
ports: [{ name: "registry", containerPort: 5000, hostPort: 5000 }],
|
||||
env: [{ name: "REGISTRY_STORAGE_DELETE_ENABLED", value: "true" }],
|
||||
volumeMounts: [{ name: "storage", mountPath: "/var/lib/registry" }],
|
||||
readinessProbe: { httpGet: { path: "/v2/", port: "registry" } },
|
||||
livenessProbe: { httpGet: { path: "/v2/", port: "registry" } }
|
||||
}],
|
||||
volumes: [{ name: "storage", hostPath: { path: "/var/lib/hwlab/registry", type: "DirectoryOrCreate" } }]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
apiVersion: "v1",
|
||||
kind: "Service",
|
||||
metadata: { name: "hwlab-registry", namespace: "hwlab-ci", labels: { "app.kubernetes.io/name": "hwlab-registry" } },
|
||||
spec: { selector: { "app.kubernetes.io/name": "hwlab-registry" }, ports: [{ name: "registry", port: 5000, targetPort: "registry" }] }
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function uniqueStrings(values) {
|
||||
return Array.from(new Set(values.filter((value) => typeof value === "string" && value.length > 0)));
|
||||
}
|
||||
|
||||
function renderGitMirrorSshPrelude({ direct, proxySummary, proxyCommand, proxyUrl, proxyNoProxy }) {
|
||||
if (direct) {
|
||||
return renderTemplate("git-mirror-ssh-direct.sh", {
|
||||
"__HWLAB_PROXY_SUMMARY_SHELL__": shellSingleQuote(proxySummary)
|
||||
});
|
||||
}
|
||||
return renderTemplate("git-mirror-ssh-proxied.sh", {
|
||||
"// __HWLAB_GIT_MIRROR_PROXY_CONNECT_MJS__": renderTemplate("git-mirror-proxy-connect.mjs"),
|
||||
"__HWLAB_PROXY_SUMMARY_SHELL__": shellSingleQuote(proxySummary),
|
||||
"__HWLAB_PROXY_COMMAND_SHELL__": shellSingleQuote(proxyCommand),
|
||||
"__HWLAB_PROXY_URL_SHELL__": shellSingleQuote(proxyUrl),
|
||||
"__HWLAB_PROXY_NO_PROXY_SHELL__": shellSingleQuote(proxyNoProxy)
|
||||
});
|
||||
}
|
||||
|
||||
function gitMirrorConfigScripts({
|
||||
fetchConfigCommands,
|
||||
gitMirrorSshPrelude,
|
||||
gitopsBranchArgs,
|
||||
gitopsBranchesJson,
|
||||
mirrorBranchesJson,
|
||||
requiredRefArgs,
|
||||
sourceBranchArgs,
|
||||
sourceBranchesJson,
|
||||
sourceSnapshotRefPrefix
|
||||
}) {
|
||||
const gitopsBranchesJsonShell = shellSingleQuote(gitopsBranchesJson);
|
||||
return {
|
||||
"sync.sh": renderTemplate("git-mirror-sync.sh", {
|
||||
"# __HWLAB_GIT_MIRROR_SSH_PRELUDE__": gitMirrorSshPrelude,
|
||||
"__HWLAB_SOURCE_SNAPSHOT_REF_PREFIX_SHELL__": shellSingleQuote(sourceSnapshotRefPrefix),
|
||||
"# __HWLAB_FETCH_CONFIG_COMMANDS__": fetchConfigCommands,
|
||||
"__HWLAB_REQUIRED_REF_ARGS__": requiredRefArgs,
|
||||
"__HWLAB_SOURCE_BRANCH_ARGS__": sourceBranchArgs,
|
||||
"__HWLAB_GITOPS_BRANCH_ARGS__": gitopsBranchArgs,
|
||||
"__HWLAB_MIRROR_BRANCHES_JSON_SHELL__": shellSingleQuote(mirrorBranchesJson),
|
||||
"__HWLAB_SOURCE_BRANCHES_JSON_SHELL__": shellSingleQuote(sourceBranchesJson),
|
||||
"__HWLAB_GITOPS_BRANCHES_JSON_SHELL__": gitopsBranchesJsonShell
|
||||
}),
|
||||
"install-hooks.sh": renderTemplate("git-mirror-install-hooks.sh", {
|
||||
"__HWLAB_GITOPS_BRANCHES_JSON_SHELL__": gitopsBranchesJsonShell
|
||||
}),
|
||||
"flush.sh": renderTemplate("git-mirror-flush.sh", {
|
||||
"# __HWLAB_GIT_MIRROR_SSH_PRELUDE__": gitMirrorSshPrelude,
|
||||
"__HWLAB_GITOPS_BRANCH_ARGS__": gitopsBranchArgs,
|
||||
"__HWLAB_GITOPS_BRANCHES_JSON_SHELL__": gitopsBranchesJsonShell
|
||||
}),
|
||||
"http.mjs": renderTemplate("git-mirror-http.mjs")
|
||||
};
|
||||
}
|
||||
|
||||
function devopsInfraGitMirrorManifest(args, deploy) {
|
||||
const mirrorSpecs = runtimeLaneMirrorSpecs(deploy, { defaultNodeId: args.nodeId, defaultGitopsRoot: defaultOutDir });
|
||||
assert.ok(mirrorSpecs.length > 0, "runtime lane git mirror requires at least one deploy.lanes entry");
|
||||
const sourceBranches = uniqueStrings(mirrorSpecs.map((spec) => spec.sourceBranch));
|
||||
const gitopsBranches = uniqueStrings(mirrorSpecs.map((spec) => spec.gitopsBranch));
|
||||
const mirrorBranches = uniqueStrings([...sourceBranches, ...gitopsBranches]);
|
||||
const fetchConfigCommands = mirrorBranches
|
||||
.map((branch) => `git -C "$repo_path" config --add remote.origin.fetch '+refs/heads/${branch}:refs/mirror-stage/heads/${branch}'`)
|
||||
.join("\n");
|
||||
const requiredRefArgs = mirrorBranches.map((branch) => shellSingleQuote(`refs/mirror-stage/heads/${branch}`)).join(" ");
|
||||
const sourceBranchArgs = sourceBranches.map((branch) => shellSingleQuote(branch)).join(" ");
|
||||
const gitopsBranchArgs = gitopsBranches.map((branch) => shellSingleQuote(branch)).join(" ");
|
||||
const mirrorBranchesJson = JSON.stringify(mirrorBranches);
|
||||
const sourceBranchesJson = JSON.stringify(sourceBranches);
|
||||
const gitopsBranchesJson = JSON.stringify(gitopsBranches);
|
||||
const sourceSnapshotRefPrefix = "refs/unidesk/snapshots/hwlab-node-runtime";
|
||||
const laneConfig = configObject(configObject(deploy.lanes, "deploy.lanes")[args.lane], `deploy.lanes.${args.lane}`);
|
||||
const gitMirrorConfig = configObject(laneConfig.gitMirror, `deploy.lanes.${args.lane}.gitMirror`);
|
||||
const gitMirrorProxy = configObject(gitMirrorConfig.egressProxy, `deploy.lanes.${args.lane}.gitMirror.egressProxy`);
|
||||
const gitMirrorProxyMode = requiredConfigString(gitMirrorProxy, "mode", `deploy.lanes.${args.lane}.gitMirror.egressProxy`);
|
||||
assert.ok(gitMirrorProxyMode === "node-global" || gitMirrorProxyMode === "host-proxy-client" || gitMirrorProxyMode === "direct", `deploy.lanes.${args.lane}.gitMirror.egressProxy.mode must be node-global, host-proxy-client, or direct`);
|
||||
const useDirectGitMirrorProxy = gitMirrorProxyMode === "direct";
|
||||
const useHostGitMirrorProxy = gitMirrorProxyMode === "host-proxy-client";
|
||||
const gitMirrorNamespace = requiredConfigString(gitMirrorConfig, "namespace", `deploy.lanes.${args.lane}.gitMirror`);
|
||||
const gitMirrorReadName = requiredConfigString(gitMirrorConfig, "serviceReadName", `deploy.lanes.${args.lane}.gitMirror`);
|
||||
const gitMirrorWriteName = requiredConfigString(gitMirrorConfig, "serviceWriteName", `deploy.lanes.${args.lane}.gitMirror`);
|
||||
const gitMirrorCachePvcName = requiredConfigString(gitMirrorConfig, "cachePvcName", `deploy.lanes.${args.lane}.gitMirror`);
|
||||
const gitMirrorCachePvcStorage = requiredConfigString(gitMirrorConfig, "cachePvcStorage", `deploy.lanes.${args.lane}.gitMirror`);
|
||||
const gitMirrorCacheHostPath = optionalConfigString(gitMirrorConfig, "cacheHostPath", `deploy.lanes.${args.lane}.gitMirror`);
|
||||
const gitMirrorServicePort = requiredConfigPositiveInteger(gitMirrorConfig, "servicePort", `deploy.lanes.${args.lane}.gitMirror`);
|
||||
const gitMirrorConfigMapName = requiredConfigString(gitMirrorConfig, "syncConfigMapName", `deploy.lanes.${args.lane}.gitMirror`);
|
||||
const proxyNamespace = useDirectGitMirrorProxy || useHostGitMirrorProxy ? "" : requiredConfigString(gitMirrorProxy, "namespace", `deploy.lanes.${args.lane}.gitMirror.egressProxy`);
|
||||
const proxyServiceName = useDirectGitMirrorProxy || useHostGitMirrorProxy ? "" : requiredConfigString(gitMirrorProxy, "serviceName", `deploy.lanes.${args.lane}.gitMirror.egressProxy`);
|
||||
const proxyPort = useDirectGitMirrorProxy ? 0 : requiredConfigPositiveInteger(gitMirrorProxy, "port", `deploy.lanes.${args.lane}.gitMirror.egressProxy`);
|
||||
const proxyClientName = useDirectGitMirrorProxy ? "" : requiredConfigString(gitMirrorProxy, "clientName", `deploy.lanes.${args.lane}.gitMirror.egressProxy`);
|
||||
const proxyNoProxy = !useDirectGitMirrorProxy && Array.isArray(gitMirrorProxy.noProxy) ? gitMirrorProxy.noProxy.filter((value) => typeof value === "string" && value.length > 0).join(",") : "";
|
||||
const proxyHost = useHostGitMirrorProxy
|
||||
? requiredConfigString(gitMirrorProxy, "host", `deploy.lanes.${args.lane}.gitMirror.egressProxy`)
|
||||
: `${proxyServiceName}.${proxyNamespace}.svc.cluster.local`;
|
||||
const proxyUrl = `http://${proxyHost}:${proxyPort}`;
|
||||
const proxySummary = useDirectGitMirrorProxy
|
||||
? "git-mirror-egress-proxy mode=direct required=false transport=ssh source=yaml"
|
||||
: `git-mirror-egress-proxy client=${proxyClientName} mode=${gitMirrorProxyMode} required=true host=${proxyHost} port=${proxyPort} ssh=GIT_SSH-wrapper source=yaml`;
|
||||
const proxyCommand = useDirectGitMirrorProxy ? "" : `ProxyCommand=node /tmp/hwlab-github-proxy-connect.mjs ${proxyHost} ${proxyPort} %h %p`;
|
||||
const gitMirrorSshPrelude = renderGitMirrorSshPrelude({
|
||||
direct: useDirectGitMirrorProxy,
|
||||
proxySummary,
|
||||
proxyCommand,
|
||||
proxyUrl,
|
||||
proxyNoProxy
|
||||
});
|
||||
const cacheVolume = gitMirrorCacheHostPath === null
|
||||
? { name: "cache", persistentVolumeClaim: { claimName: gitMirrorCachePvcName } }
|
||||
: { name: "cache", hostPath: { path: gitMirrorCacheHostPath, type: "DirectoryOrCreate" } };
|
||||
const labels = {
|
||||
"app.kubernetes.io/part-of": "hwlab-node-control-plane",
|
||||
"hwlab.pikastech.local/node": args.nodeId,
|
||||
"hwlab.pikastech.local/lane": args.lane
|
||||
};
|
||||
const configLabels = { ...labels, "app.kubernetes.io/name": "git-mirror" };
|
||||
const readLabels = { ...labels, "app.kubernetes.io/name": gitMirrorReadName, "hwlab.pikastech.local/git-mirror-mode": "read" };
|
||||
const writeLabels = { ...labels, "app.kubernetes.io/name": gitMirrorWriteName, "hwlab.pikastech.local/git-mirror-mode": "write" };
|
||||
return {
|
||||
apiVersion: "v1",
|
||||
kind: "List",
|
||||
items: [
|
||||
{ apiVersion: "v1", kind: "Namespace", metadata: { name: gitMirrorNamespace } },
|
||||
...(gitMirrorCacheHostPath === null ? [{
|
||||
apiVersion: "v1",
|
||||
kind: "PersistentVolumeClaim",
|
||||
metadata: { name: gitMirrorCachePvcName, namespace: gitMirrorNamespace, labels: configLabels },
|
||||
spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: gitMirrorCachePvcStorage } } }
|
||||
}] : []),
|
||||
{
|
||||
apiVersion: "v1",
|
||||
kind: "ConfigMap",
|
||||
metadata: { name: gitMirrorConfigMapName, namespace: gitMirrorNamespace, labels: configLabels },
|
||||
data: gitMirrorConfigScripts({
|
||||
fetchConfigCommands,
|
||||
gitMirrorSshPrelude,
|
||||
gitopsBranchArgs,
|
||||
gitopsBranchesJson,
|
||||
mirrorBranchesJson,
|
||||
requiredRefArgs,
|
||||
sourceBranchArgs,
|
||||
sourceBranchesJson,
|
||||
sourceSnapshotRefPrefix
|
||||
})
|
||||
},
|
||||
{
|
||||
apiVersion: "apps/v1",
|
||||
kind: "Deployment",
|
||||
metadata: { name: gitMirrorReadName, namespace: gitMirrorNamespace, labels: readLabels },
|
||||
spec: {
|
||||
replicas: 1,
|
||||
selector: { matchLabels: { "app.kubernetes.io/name": gitMirrorReadName } },
|
||||
template: {
|
||||
metadata: { labels: readLabels },
|
||||
spec: {
|
||||
containers: [{
|
||||
name: "git-mirror",
|
||||
image: ciToolsRunnerImage,
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
command: ["node"],
|
||||
args: ["/etc/git-mirror/http.mjs"],
|
||||
ports: [{ name: "http", containerPort: 8080 }],
|
||||
readinessProbe: { httpGet: { path: "/pikasTech/HWLAB.git/HEAD", port: "http" }, initialDelaySeconds: 5, periodSeconds: 10 },
|
||||
livenessProbe: { httpGet: { path: "/", port: "http" }, initialDelaySeconds: 10, periodSeconds: 30 },
|
||||
volumeMounts: [
|
||||
{ name: "cache", mountPath: "/cache" },
|
||||
{ name: "config", mountPath: "/etc/git-mirror", readOnly: true }
|
||||
]
|
||||
}],
|
||||
volumes: [
|
||||
cacheVolume,
|
||||
{ name: "config", configMap: { name: gitMirrorConfigMapName, defaultMode: 0o555 } }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
apiVersion: "v1",
|
||||
kind: "Service",
|
||||
metadata: { name: gitMirrorReadName, namespace: gitMirrorNamespace, labels: readLabels },
|
||||
spec: { selector: { "app.kubernetes.io/name": gitMirrorReadName }, ports: [{ name: "http", port: gitMirrorServicePort, targetPort: "http" }] }
|
||||
},
|
||||
{
|
||||
apiVersion: "v1",
|
||||
kind: "Service",
|
||||
metadata: { name: gitMirrorWriteName, namespace: gitMirrorNamespace, labels: writeLabels },
|
||||
spec: { selector: { "app.kubernetes.io/name": gitMirrorReadName }, ports: [{ name: "http", port: gitMirrorServicePort, targetPort: "http" }] }
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
export {
|
||||
devopsInfraGitMirrorManifest,
|
||||
registryManifest,
|
||||
uniqueStrings
|
||||
};
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,830 @@
|
||||
import {
|
||||
argoApplicationName,
|
||||
buildkitRunnerImage,
|
||||
ciToolsRunnerImage,
|
||||
defaultDevBaseImage,
|
||||
defaultRegistryPrefix,
|
||||
defaultServiceIds,
|
||||
defaultSourceRepo,
|
||||
gitopsPathFor,
|
||||
gitopsPathForProfile,
|
||||
gitopsTargetForProfile,
|
||||
isRuntimeLane,
|
||||
namespaceNameForProfile,
|
||||
primitiveValidationTasks,
|
||||
proxyEnv,
|
||||
serviceIdsForLane,
|
||||
servicesParamForLane
|
||||
} from "./core.mjs";
|
||||
import {
|
||||
ciTimingShellFunction,
|
||||
collectArtifactsScript,
|
||||
controlPlaneReconcileScript,
|
||||
gitopsPromoteScript,
|
||||
perServicePublishScript,
|
||||
planArtifactsScript,
|
||||
pollerScript,
|
||||
prepareSourceScript,
|
||||
primitiveValidationTask,
|
||||
primitiveValidationTaskNames
|
||||
} from "./tekton-scripts.mjs";
|
||||
|
||||
function ciLaneSettings(argsOrLane = "node") {
|
||||
const lane = typeof argsOrLane === "string" ? argsOrLane : argsOrLane.lane;
|
||||
if (isRuntimeLane(lane)) {
|
||||
const namespace = namespaceNameForProfile(lane);
|
||||
return {
|
||||
lane,
|
||||
profile: lane,
|
||||
gitopsTarget: lane,
|
||||
pipelineName: `${namespace}-ci-image-publish`,
|
||||
pipelineRunPrefix: `${namespace}-ci-poll`,
|
||||
serviceAccountName: `${namespace}-tekton-runner`,
|
||||
pollerName: `${namespace}-branch-poller`,
|
||||
controlPlaneReconcilerName: `${namespace}-control-plane-reconciler`,
|
||||
tektonDir: `tekton-${lane}`,
|
||||
catalogPath: typeof argsOrLane === "object" ? argsOrLane.catalogPath : `deploy/artifact-catalog.${lane}.json`,
|
||||
imageTagMode: typeof argsOrLane === "object" ? argsOrLane.imageTagMode : "full",
|
||||
runtimeNamespace: namespace
|
||||
};
|
||||
}
|
||||
return {
|
||||
lane: "node",
|
||||
profile: "dev",
|
||||
gitopsTarget: "node",
|
||||
pipelineName: "hwlab-node-ci-image-publish",
|
||||
pipelineRunPrefix: "hwlab-node-ci-poll",
|
||||
serviceAccountName: "hwlab-tekton-runner",
|
||||
pollerName: "hwlab-node-branch-poller",
|
||||
controlPlaneReconcilerName: "hwlab-node-control-plane-reconciler",
|
||||
tektonDir: "tekton",
|
||||
catalogPath: "deploy/artifact-catalog.dev.json",
|
||||
imageTagMode: "short",
|
||||
runtimeNamespace: "hwlab-dev"
|
||||
};
|
||||
}
|
||||
|
||||
function tektonRbac(args = { lane: "node" }) {
|
||||
const settings = ciLaneSettings(args);
|
||||
if (isRuntimeLane(settings.lane)) {
|
||||
const runtimeObserverName = `${settings.runtimeNamespace}-runtime-observer`;
|
||||
const pipelineRunWriterName = `${settings.runtimeNamespace}-pipelinerun-writer`;
|
||||
const argoReconcilerName = `${settings.runtimeNamespace}-argocd-reconciler`;
|
||||
return {
|
||||
apiVersion: "v1",
|
||||
kind: "List",
|
||||
items: [
|
||||
{ apiVersion: "v1", kind: "Namespace", metadata: { name: "hwlab-ci" } },
|
||||
{ apiVersion: "v1", kind: "ServiceAccount", metadata: { name: settings.serviceAccountName, namespace: "hwlab-ci" } },
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "Role",
|
||||
metadata: { name: runtimeObserverName, namespace: settings.runtimeNamespace },
|
||||
rules: [
|
||||
{ apiGroups: [""], resources: ["pods", "services", "configmaps"], verbs: ["get", "list", "watch"] },
|
||||
{ apiGroups: ["apps"], resources: ["deployments", "statefulsets"], verbs: ["get", "list", "watch"] },
|
||||
{ apiGroups: ["batch"], resources: ["jobs"], verbs: ["get", "list", "watch"] }
|
||||
]
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "RoleBinding",
|
||||
metadata: { name: runtimeObserverName, namespace: settings.runtimeNamespace },
|
||||
subjects: [{ kind: "ServiceAccount", name: settings.serviceAccountName, namespace: "hwlab-ci" }],
|
||||
roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: runtimeObserverName }
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "Role",
|
||||
metadata: { name: pipelineRunWriterName, namespace: "hwlab-ci" },
|
||||
rules: [
|
||||
{ apiGroups: ["tekton.dev"], resources: ["pipelineruns"], verbs: ["get", "list", "create"] },
|
||||
{ apiGroups: ["tekton.dev"], resources: ["pipelines"], verbs: ["get", "patch", "create"] },
|
||||
{ apiGroups: ["batch"], resources: ["cronjobs"], verbs: ["get", "patch", "create"] },
|
||||
{ apiGroups: ["rbac.authorization.k8s.io"], resources: ["roles", "rolebindings"], verbs: ["get", "patch", "create"] },
|
||||
{ apiGroups: [""], resources: ["serviceaccounts"], verbs: ["get", "patch", "create"] }
|
||||
]
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "RoleBinding",
|
||||
metadata: { name: pipelineRunWriterName, namespace: "hwlab-ci" },
|
||||
subjects: [{ kind: "ServiceAccount", name: settings.serviceAccountName, namespace: "hwlab-ci" }],
|
||||
roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: pipelineRunWriterName }
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "Role",
|
||||
metadata: { name: argoReconcilerName, namespace: "argocd" },
|
||||
rules: [
|
||||
{ apiGroups: ["argoproj.io"], resources: ["applications", "appprojects"], verbs: ["get", "patch", "create"] },
|
||||
{ apiGroups: ["rbac.authorization.k8s.io"], resources: ["roles", "rolebindings"], verbs: ["get", "patch", "create"] }
|
||||
]
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "RoleBinding",
|
||||
metadata: { name: argoReconcilerName, namespace: "argocd" },
|
||||
subjects: [{ kind: "ServiceAccount", name: settings.serviceAccountName, namespace: "hwlab-ci" }],
|
||||
roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: argoReconcilerName }
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
return {
|
||||
apiVersion: "v1",
|
||||
kind: "List",
|
||||
items: [
|
||||
{ apiVersion: "v1", kind: "Namespace", metadata: { name: "hwlab-ci" } },
|
||||
{ apiVersion: "v1", kind: "ServiceAccount", metadata: { name: "hwlab-tekton-runner", namespace: "hwlab-ci" } },
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "Role",
|
||||
metadata: { name: "hwlab-node-control-plane-reconciler", namespace: "hwlab-dev" },
|
||||
rules: [
|
||||
{ apiGroups: ["rbac.authorization.k8s.io"], resources: ["roles", "rolebindings"], verbs: ["get", "patch", "create"] }
|
||||
]
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "RoleBinding",
|
||||
metadata: { name: "hwlab-node-control-plane-reconciler", namespace: "hwlab-dev" },
|
||||
subjects: [{ kind: "ServiceAccount", name: "hwlab-tekton-runner", namespace: "hwlab-ci" }],
|
||||
roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: "hwlab-node-control-plane-reconciler" }
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "Role",
|
||||
metadata: { name: "hwlab-tekton-runtime-observer", namespace: "hwlab-dev" },
|
||||
rules: [
|
||||
{ apiGroups: [""], resources: ["pods", "services", "configmaps"], verbs: ["get", "list", "watch"] },
|
||||
{ apiGroups: ["apps"], resources: ["deployments", "statefulsets"], verbs: ["get", "list", "watch"] },
|
||||
{ apiGroups: ["batch"], resources: ["jobs"], verbs: ["get", "list", "watch"] }
|
||||
]
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "RoleBinding",
|
||||
metadata: { name: "hwlab-tekton-runtime-observer", namespace: "hwlab-dev" },
|
||||
subjects: [{ kind: "ServiceAccount", name: "hwlab-tekton-runner", namespace: "hwlab-ci" }],
|
||||
roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: "hwlab-tekton-runtime-observer" }
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "Role",
|
||||
metadata: { name: "hwlab-tekton-pipelinerun-writer", namespace: "hwlab-ci" },
|
||||
rules: [
|
||||
{ apiGroups: ["tekton.dev"], resources: ["pipelineruns"], verbs: ["get", "list", "create"] },
|
||||
{ apiGroups: ["tekton.dev"], resources: ["pipelines"], verbs: ["get", "patch", "create"] },
|
||||
{ apiGroups: ["batch"], resources: ["cronjobs"], verbs: ["get", "patch", "create"] },
|
||||
{ apiGroups: ["rbac.authorization.k8s.io"], resources: ["roles", "rolebindings"], verbs: ["get", "patch", "create"] },
|
||||
{ apiGroups: [""], resources: ["serviceaccounts"], verbs: ["get", "patch", "create"] }
|
||||
]
|
||||
},
|
||||
{
|
||||
apiVersion: "rbac.authorization.k8s.io/v1",
|
||||
kind: "RoleBinding",
|
||||
metadata: { name: "hwlab-tekton-pipelinerun-writer", namespace: "hwlab-ci" },
|
||||
subjects: [{ kind: "ServiceAccount", name: "hwlab-tekton-runner", namespace: "hwlab-ci" }],
|
||||
roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: "hwlab-tekton-pipelinerun-writer" }
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function buildTaskName(serviceId) {
|
||||
return `build-${serviceId}`;
|
||||
}
|
||||
|
||||
function affectedResultName(serviceId) {
|
||||
return `affected-${serviceId}`;
|
||||
}
|
||||
|
||||
function buildResultName(serviceId) {
|
||||
return `build-${serviceId}`;
|
||||
}
|
||||
|
||||
const serviceResultFields = Object.freeze([
|
||||
"status",
|
||||
"service-id",
|
||||
"image",
|
||||
"image-tag",
|
||||
"digest",
|
||||
"repository-digest",
|
||||
"source-commit-id",
|
||||
"component-input-hash",
|
||||
"environment-input-hash",
|
||||
"code-input-hash",
|
||||
"runtime-mode",
|
||||
"boot-repo",
|
||||
"boot-commit",
|
||||
"boot-sh",
|
||||
"build-created-at",
|
||||
"build-backend",
|
||||
"reused-from",
|
||||
"go-base-image",
|
||||
"go-base-image-status",
|
||||
"go-base-digest",
|
||||
"go-base-build-duration-ms",
|
||||
"go-base-buildkit-cache-ref"
|
||||
]);
|
||||
|
||||
function serviceResultParamName(serviceId, field) {
|
||||
return `${serviceId}-${field}`;
|
||||
}
|
||||
|
||||
function serviceResultEnvName(serviceId, field) {
|
||||
return `HWLAB_SERVICE_RESULT_${serviceId.toUpperCase().replace(/[^A-Z0-9]+/gu, "_")}_${field.toUpperCase().replace(/[^A-Z0-9]+/gu, "_")}`;
|
||||
}
|
||||
|
||||
function serviceResultParams(serviceIds = defaultServiceIds) {
|
||||
return serviceIds.flatMap((serviceId) => serviceResultFields.map((field) => ({ name: serviceResultParamName(serviceId, field), default: "" })));
|
||||
}
|
||||
|
||||
function serviceResultParamValues(serviceIds = defaultServiceIds) {
|
||||
return serviceIds.flatMap((serviceId) => serviceResultFields.map((field) => ({
|
||||
name: serviceResultParamName(serviceId, field),
|
||||
value: `$(tasks.${buildTaskName(serviceId)}.results.${field})`
|
||||
})));
|
||||
}
|
||||
|
||||
function serviceResultEnv(serviceIds = defaultServiceIds) {
|
||||
return serviceIds.flatMap((serviceId) => serviceResultFields.map((field) => ({
|
||||
name: serviceResultEnvName(serviceId, field),
|
||||
value: `$(params.${serviceResultParamName(serviceId, field)})`
|
||||
})));
|
||||
}
|
||||
|
||||
function serviceWorkVolume() {
|
||||
return { name: "service-work", emptyDir: {} };
|
||||
}
|
||||
|
||||
function buildkitBinVolume() {
|
||||
return { name: "buildkit-bin", emptyDir: {} };
|
||||
}
|
||||
|
||||
function buildkitRunVolume() {
|
||||
return { name: "buildkit-run", emptyDir: {} };
|
||||
}
|
||||
|
||||
function prepareBuildkitClientStep() {
|
||||
return {
|
||||
name: "prepare-buildkit-client",
|
||||
image: buildkitRunnerImage,
|
||||
securityContext: { runAsUser: 0, runAsGroup: 0 },
|
||||
volumeMounts: [{ name: "buildkit-bin", mountPath: "/workspace/buildkit-bin" }],
|
||||
script: `#!/bin/sh
|
||||
set -eu
|
||||
mkdir -p /workspace/buildkit-bin
|
||||
cp /usr/bin/buildctl /workspace/buildkit-bin/
|
||||
chmod +x /workspace/buildkit-bin/*
|
||||
`
|
||||
};
|
||||
}
|
||||
|
||||
function buildkitSidecar() {
|
||||
return {
|
||||
name: "buildkitd",
|
||||
image: buildkitRunnerImage,
|
||||
args: ["--addr", "unix:///workspace/buildkit-run/buildkitd.sock", "--oci-worker-no-process-sandbox"],
|
||||
env: proxyEnv(),
|
||||
volumeMounts: [{ name: "buildkit-run", mountPath: "/workspace/buildkit-run" }],
|
||||
securityContext: {
|
||||
runAsUser: 1000,
|
||||
runAsGroup: 1000,
|
||||
allowPrivilegeEscalation: true,
|
||||
appArmorProfile: { type: "Unconfined" },
|
||||
seccompProfile: { type: "Unconfined" }
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function planArtifactsTask({ serviceIds = defaultServiceIds } = {}) {
|
||||
return {
|
||||
name: "plan-artifacts",
|
||||
runAfter: primitiveValidationTaskNames(),
|
||||
workspaces: [{ name: "source", workspace: "source" }],
|
||||
taskSpec: {
|
||||
params: [
|
||||
{ name: "git-url" },
|
||||
{ name: "git-read-url" },
|
||||
{ name: "gitops-branch" },
|
||||
{ name: "lane" },
|
||||
{ name: "revision" },
|
||||
{ name: "catalog-path" },
|
||||
{ name: "registry-prefix" },
|
||||
{ name: "services" }
|
||||
],
|
||||
results: serviceIds.flatMap((serviceId) => [
|
||||
{ name: affectedResultName(serviceId), description: `${serviceId} rollout affected according to ci-plan` },
|
||||
{ name: buildResultName(serviceId), description: `${serviceId} image build required according to ci-plan` }
|
||||
]),
|
||||
workspaces: [{ name: "source" }],
|
||||
steps: [{
|
||||
name: "plan",
|
||||
image: ciToolsRunnerImage,
|
||||
env: [
|
||||
{ name: "HWLAB_SELECTED_SERVICES", value: "$(params.services)" },
|
||||
{ name: "HWLAB_DEV_REGISTRY_K8S_NATIVE", value: "1" },
|
||||
{ name: "HWLAB_TEKTON_PIPELINERUN", value: "$(context.pipelineRun.name)" },
|
||||
...proxyEnv()
|
||||
],
|
||||
script: planArtifactsScript()
|
||||
}]
|
||||
},
|
||||
params: [
|
||||
{ name: "git-url", value: "$(params.git-url)" },
|
||||
{ name: "git-read-url", value: "$(params.git-read-url)" },
|
||||
{ name: "gitops-branch", value: "$(params.gitops-branch)" },
|
||||
{ name: "lane", value: "$(params.lane)" },
|
||||
{ name: "revision", value: "$(params.revision)" },
|
||||
{ name: "catalog-path", value: "$(params.catalog-path)" },
|
||||
{ name: "registry-prefix", value: "$(params.registry-prefix)" },
|
||||
{ name: "services", value: "$(params.services)" }
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function perServiceBuildTask(serviceId, { runAfter = ["plan-artifacts"] } = {}) {
|
||||
return {
|
||||
name: buildTaskName(serviceId),
|
||||
runAfter,
|
||||
workspaces: [{ name: "source", workspace: "source" }],
|
||||
when: [{ input: `$(tasks.plan-artifacts.results.${buildResultName(serviceId)})`, operator: "in", values: ["true"] }],
|
||||
taskSpec: {
|
||||
params: [
|
||||
{ name: "revision" },
|
||||
{ name: "lane" },
|
||||
{ name: "catalog-path" },
|
||||
{ name: "image-tag-mode" },
|
||||
{ name: "registry-prefix" },
|
||||
{ name: "service-id" },
|
||||
{ name: "base-image" },
|
||||
{ name: "build-cache-mode" }
|
||||
],
|
||||
results: serviceResultFields.map((field) => ({ name: field, description: `${serviceId} ${field}` })),
|
||||
workspaces: [{ name: "source" }],
|
||||
volumes: [serviceWorkVolume(), buildkitBinVolume(), buildkitRunVolume()],
|
||||
sidecars: [buildkitSidecar()],
|
||||
steps: [
|
||||
prepareBuildkitClientStep(),
|
||||
{
|
||||
name: "publish",
|
||||
image: ciToolsRunnerImage,
|
||||
securityContext: { runAsUser: 1000, runAsGroup: 1000 },
|
||||
env: proxyEnv(),
|
||||
volumeMounts: [{ name: "service-work", mountPath: "/workspace/service-work" }, { name: "buildkit-bin", mountPath: "/workspace/buildkit-bin" }, { name: "buildkit-run", mountPath: "/workspace/buildkit-run" }],
|
||||
script: perServicePublishScript()
|
||||
}
|
||||
]
|
||||
},
|
||||
params: [
|
||||
{ name: "revision", value: "$(params.revision)" },
|
||||
{ name: "lane", value: "$(params.lane)" },
|
||||
{ name: "catalog-path", value: "$(params.catalog-path)" },
|
||||
{ name: "image-tag-mode", value: "$(params.image-tag-mode)" },
|
||||
{ name: "registry-prefix", value: "$(params.registry-prefix)" },
|
||||
{ name: "service-id", value: serviceId },
|
||||
{ name: "base-image", value: "$(params.base-image)" },
|
||||
{ name: "build-cache-mode", value: "$(params.build-cache-mode)" }
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function collectArtifactsTask({ serviceIds = defaultServiceIds } = {}) {
|
||||
return {
|
||||
name: "collect-artifacts",
|
||||
runAfter: serviceIds.map(buildTaskName),
|
||||
workspaces: [{ name: "source", workspace: "source" }],
|
||||
taskSpec: {
|
||||
params: [
|
||||
{ name: "revision" },
|
||||
{ name: "lane" },
|
||||
{ name: "catalog-path" },
|
||||
{ name: "image-tag-mode" },
|
||||
{ name: "registry-prefix" },
|
||||
{ name: "services" },
|
||||
{ name: "base-image" }
|
||||
],
|
||||
workspaces: [{ name: "source" }],
|
||||
steps: [{ name: "collect", image: ciToolsRunnerImage, env: proxyEnv(), script: collectArtifactsScript() }]
|
||||
},
|
||||
params: [
|
||||
{ name: "revision", value: "$(params.revision)" },
|
||||
{ name: "lane", value: "$(params.lane)" },
|
||||
{ name: "catalog-path", value: "$(params.catalog-path)" },
|
||||
{ name: "image-tag-mode", value: "$(params.image-tag-mode)" },
|
||||
{ name: "registry-prefix", value: "$(params.registry-prefix)" },
|
||||
{ name: "services", value: "$(params.services)" },
|
||||
{ name: "base-image", value: "$(params.base-image)" }
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function runtimeReadyScript() {
|
||||
return `#!/bin/sh
|
||||
set -eu
|
||||
${ciTimingShellFunction()}
|
||||
export HWLAB_TEKTON_PIPELINERUN="$(context.pipelineRun.name)"
|
||||
export HWLAB_TEKTON_TASKRUN="$(context.taskRun.name)"
|
||||
export HWLAB_TEKTON_TASK="runtime-ready"
|
||||
export HWLAB_SOURCE_REVISION="$(params.revision)"
|
||||
export HWLAB_RUNTIME_READY_TIMEOUT_MS="$(params.timeout-ms)"
|
||||
cd /workspace/source/repo
|
||||
node scripts/ci/runtime-ready.mjs
|
||||
`;
|
||||
}
|
||||
|
||||
function runtimeReadyTask({ profile = "dev" } = {}) {
|
||||
return {
|
||||
name: "runtime-ready",
|
||||
runAfter: ["gitops-promote"],
|
||||
workspaces: [{ name: "source", workspace: "source" }],
|
||||
taskSpec: {
|
||||
params: [{ name: "revision" }, { name: "runtime-namespace" }, { name: "gitops-target" }, { name: "argo-application" }, { name: "timeout-ms" }],
|
||||
workspaces: [{ name: "source" }],
|
||||
steps: [{
|
||||
name: "runtime-ready",
|
||||
image: ciToolsRunnerImage,
|
||||
env: [
|
||||
...proxyEnv(),
|
||||
{ name: "HWLAB_RUNTIME_NAMESPACE", value: "$(params.runtime-namespace)" },
|
||||
{ name: "HWLAB_GITOPS_TARGET", value: "$(params.gitops-target)" },
|
||||
{ name: "HWLAB_ARGO_APPLICATION", value: "$(params.argo-application)" }
|
||||
],
|
||||
script: runtimeReadyScript()
|
||||
}]
|
||||
},
|
||||
params: [
|
||||
{ name: "revision", value: "$(params.revision)" },
|
||||
{ name: "runtime-namespace", value: namespaceNameForProfile(profile) },
|
||||
{ name: "gitops-target", value: gitopsTargetForProfile(profile) },
|
||||
{ name: "argo-application", value: argoApplicationName(profile) },
|
||||
{ name: "timeout-ms", value: "$(params.runtime-ready-timeout-ms)" }
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function imagePublishTaskSet(args = { lane: "node" }, deploy = null) {
|
||||
const serviceIds = serviceIdsForLane(args.lane, deploy);
|
||||
const buildTasks = serviceIds.map((serviceId) => perServiceBuildTask(serviceId, {
|
||||
runAfter: ["plan-artifacts"]
|
||||
}));
|
||||
return [
|
||||
planArtifactsTask({ serviceIds }),
|
||||
...buildTasks,
|
||||
collectArtifactsTask({ serviceIds })
|
||||
];
|
||||
}
|
||||
|
||||
function tektonPipeline(args = { lane: "node" }, deploy = null) {
|
||||
const settings = ciLaneSettings(args);
|
||||
const runtimePath = gitopsPathForProfile(args, settings.profile);
|
||||
const preFlushRuntimeReadyTasks = isRuntimeLane(settings.lane)
|
||||
? []
|
||||
: [{
|
||||
...runtimeReadyTask({ profile: settings.profile }),
|
||||
when: [{ input: "$(tasks.gitops-promote.results.runtime-ready-required)", operator: "in", values: ["true"] }]
|
||||
}];
|
||||
return {
|
||||
apiVersion: "tekton.dev/v1",
|
||||
kind: "Pipeline",
|
||||
metadata: {
|
||||
name: settings.pipelineName,
|
||||
namespace: "hwlab-ci",
|
||||
labels: {
|
||||
"app.kubernetes.io/part-of": "hwlab",
|
||||
"hwlab.pikastech.local/gitops-target": settings.gitopsTarget
|
||||
},
|
||||
annotations: {
|
||||
"hwlab.pikastech.local/source-config": "scripts/gitops-render.mjs#tekton-native-primitive-ci",
|
||||
"hwlab.pikastech.local/ci-contract": "tekton-native-primitive-tasks",
|
||||
"hwlab.pikastech.local/policy": "native-per-service-taskrun-image-publish"
|
||||
}
|
||||
},
|
||||
spec: {
|
||||
params: [
|
||||
{ name: "git-url", type: "string", default: defaultSourceRepo },
|
||||
{ name: "git-read-url", type: "string", default: args.gitReadUrl },
|
||||
{ name: "git-write-url", type: "string", default: args.gitWriteUrl },
|
||||
{ name: "source-branch", type: "string", default: args.sourceBranch },
|
||||
{ name: "gitops-branch", type: "string", default: args.gitopsBranch },
|
||||
{ name: "lane", type: "string", default: settings.lane },
|
||||
{ name: "catalog-path", type: "string", default: args.catalogPath || settings.catalogPath },
|
||||
{ name: "image-tag-mode", type: "string", default: args.imageTagMode || settings.imageTagMode },
|
||||
{ name: "runtime-path", type: "string", default: runtimePath },
|
||||
{ name: "revision", type: "string", description: "Full git commit SHA; the only source truth for image tags." },
|
||||
{ name: "registry-prefix", type: "string", default: defaultRegistryPrefix },
|
||||
{ name: "services", type: "string", default: servicesParamForLane(args.lane, deploy) },
|
||||
{ name: "base-image", type: "string", default: defaultDevBaseImage },
|
||||
{ name: "build-cache-mode", type: "string", default: "registry" },
|
||||
{ name: "runtime-ready-timeout-ms", type: "string", default: "60000" }
|
||||
],
|
||||
workspaces: [
|
||||
{ name: "source" },
|
||||
{ name: "git-ssh" }
|
||||
],
|
||||
tasks: [
|
||||
{
|
||||
name: "prepare-source",
|
||||
workspaces: [
|
||||
{ name: "source", workspace: "source" },
|
||||
{ name: "git-ssh", workspace: "git-ssh" }
|
||||
],
|
||||
taskSpec: {
|
||||
params: [
|
||||
{ name: "git-url" },
|
||||
{ name: "git-read-url" },
|
||||
{ name: "source-branch" },
|
||||
{ name: "gitops-branch" },
|
||||
{ name: "lane" },
|
||||
{ name: "catalog-path" },
|
||||
{ name: "image-tag-mode" },
|
||||
{ name: "registry-prefix" },
|
||||
{ name: "services" },
|
||||
{ name: "revision" }
|
||||
],
|
||||
workspaces: [{ name: "source" }, { name: "git-ssh" }],
|
||||
steps: [{ name: "prepare-source", image: ciToolsRunnerImage, env: proxyEnv(), script: prepareSourceScript() }]
|
||||
},
|
||||
params: [
|
||||
{ name: "git-url", value: "$(params.git-url)" },
|
||||
{ name: "git-read-url", value: "$(params.git-read-url)" },
|
||||
{ name: "source-branch", value: "$(params.source-branch)" },
|
||||
{ name: "gitops-branch", value: "$(params.gitops-branch)" },
|
||||
{ name: "lane", value: "$(params.lane)" },
|
||||
{ name: "catalog-path", value: "$(params.catalog-path)" },
|
||||
{ name: "image-tag-mode", value: "$(params.image-tag-mode)" },
|
||||
{ name: "registry-prefix", value: "$(params.registry-prefix)" },
|
||||
{ name: "services", value: "$(params.services)" },
|
||||
{ name: "revision", value: "$(params.revision)" }
|
||||
]
|
||||
},
|
||||
...primitiveValidationTasks.map(primitiveValidationTask),
|
||||
...imagePublishTaskSet(args, deploy),
|
||||
{
|
||||
name: "gitops-promote",
|
||||
runAfter: ["collect-artifacts"],
|
||||
workspaces: [
|
||||
{ name: "source", workspace: "source" },
|
||||
{ name: "git-ssh", workspace: "git-ssh" }
|
||||
],
|
||||
taskSpec: {
|
||||
params: [
|
||||
{ name: "git-url" },
|
||||
{ name: "git-read-url" },
|
||||
{ name: "git-write-url" },
|
||||
{ name: "source-branch" },
|
||||
{ name: "gitops-branch" },
|
||||
{ name: "lane" },
|
||||
{ name: "catalog-path" },
|
||||
{ name: "image-tag-mode" },
|
||||
{ name: "runtime-path" },
|
||||
{ name: "revision" },
|
||||
{ name: "registry-prefix" }
|
||||
],
|
||||
results: [{ name: "runtime-ready-required", description: "true when GitOps promotion changed runtime desired state and runtime readiness must be observed" }],
|
||||
workspaces: [{ name: "source" }, { name: "git-ssh" }],
|
||||
steps: [{ name: "promote", image: ciToolsRunnerImage, env: proxyEnv(), script: gitopsPromoteScript() }]
|
||||
},
|
||||
params: [
|
||||
{ name: "git-url", value: "$(params.git-url)" },
|
||||
{ name: "git-read-url", value: "$(params.git-read-url)" },
|
||||
{ name: "git-write-url", value: "$(params.git-write-url)" },
|
||||
{ name: "source-branch", value: "$(params.source-branch)" },
|
||||
{ name: "gitops-branch", value: "$(params.gitops-branch)" },
|
||||
{ name: "lane", value: "$(params.lane)" },
|
||||
{ name: "catalog-path", value: "$(params.catalog-path)" },
|
||||
{ name: "image-tag-mode", value: "$(params.image-tag-mode)" },
|
||||
{ name: "runtime-path", value: "$(params.runtime-path)" },
|
||||
{ name: "revision", value: "$(params.revision)" },
|
||||
{ name: "registry-prefix", value: "$(params.registry-prefix)" }
|
||||
]
|
||||
},
|
||||
...preFlushRuntimeReadyTasks
|
||||
]
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function tektonPipelineRunTemplate({ source, args }) {
|
||||
const settings = ciLaneSettings(args);
|
||||
const runtimePath = gitopsPathForProfile(args, settings.profile);
|
||||
return {
|
||||
apiVersion: "tekton.dev/v1",
|
||||
kind: "PipelineRun",
|
||||
metadata: {
|
||||
generateName: `${settings.pipelineRunPrefix}-manual-`,
|
||||
namespace: "hwlab-ci",
|
||||
labels: {
|
||||
"app.kubernetes.io/part-of": "hwlab",
|
||||
"hwlab.pikastech.local/gitops-target": settings.gitopsTarget,
|
||||
"hwlab.pikastech.local/source-commit": source.full
|
||||
}
|
||||
},
|
||||
spec: {
|
||||
pipelineRef: { name: settings.pipelineName },
|
||||
taskRunTemplate: {
|
||||
serviceAccountName: settings.serviceAccountName,
|
||||
podTemplate: {
|
||||
hostNetwork: true,
|
||||
dnsPolicy: "ClusterFirstWithHostNet",
|
||||
securityContext: { fsGroup: 1000 }
|
||||
}
|
||||
},
|
||||
params: [
|
||||
{ name: "git-url", value: args.sourceRepo },
|
||||
{ name: "git-read-url", value: args.gitReadUrl },
|
||||
{ name: "git-write-url", value: args.gitWriteUrl },
|
||||
{ name: "source-branch", value: args.sourceBranch },
|
||||
{ name: "gitops-branch", value: args.gitopsBranch },
|
||||
{ name: "lane", value: settings.lane },
|
||||
{ name: "catalog-path", value: args.catalogPath || settings.catalogPath },
|
||||
{ name: "image-tag-mode", value: args.imageTagMode || settings.imageTagMode },
|
||||
{ name: "runtime-path", value: runtimePath },
|
||||
{ name: "revision", value: source.full },
|
||||
{ name: "registry-prefix", value: args.registryPrefix },
|
||||
{ name: "base-image", value: defaultDevBaseImage },
|
||||
{ name: "build-cache-mode", value: "registry" }
|
||||
],
|
||||
workspaces: [
|
||||
{ name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } },
|
||||
{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }
|
||||
]
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function tektonPollerCronJob(args, deploy) {
|
||||
const settings = ciLaneSettings(args);
|
||||
if (settings.lane === "v02") throw new Error("v02 CI/CD is manually triggered by UniDesk CLI and must not render a branch poller CronJob");
|
||||
return {
|
||||
apiVersion: "batch/v1",
|
||||
kind: "CronJob",
|
||||
metadata: {
|
||||
name: settings.pollerName,
|
||||
namespace: "hwlab-ci",
|
||||
labels: {
|
||||
"app.kubernetes.io/part-of": "hwlab",
|
||||
"hwlab.pikastech.local/gitops-target": settings.gitopsTarget
|
||||
},
|
||||
annotations: {
|
||||
"hwlab.pikastech.local/source-branch": args.sourceBranch,
|
||||
"hwlab.pikastech.local/gitops-branch": args.gitopsBranch
|
||||
}
|
||||
},
|
||||
spec: {
|
||||
schedule: settings.lane === "v02" ? "*/10 * * * *" : "* * * * *",
|
||||
concurrencyPolicy: "Forbid",
|
||||
successfulJobsHistoryLimit: 3,
|
||||
failedJobsHistoryLimit: 3,
|
||||
jobTemplate: {
|
||||
spec: {
|
||||
backoffLimit: 1,
|
||||
template: {
|
||||
metadata: {
|
||||
labels: {
|
||||
"app.kubernetes.io/part-of": "hwlab",
|
||||
"hwlab.pikastech.local/gitops-target": settings.gitopsTarget
|
||||
}
|
||||
},
|
||||
spec: {
|
||||
serviceAccountName: settings.serviceAccountName,
|
||||
restartPolicy: "Never",
|
||||
hostNetwork: true,
|
||||
dnsPolicy: "ClusterFirstWithHostNet",
|
||||
containers: [{
|
||||
name: "poll",
|
||||
image: ciToolsRunnerImage,
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
env: [
|
||||
...proxyEnv(),
|
||||
{ name: "POD_NAMESPACE", valueFrom: { fieldRef: { fieldPath: "metadata.namespace" } } },
|
||||
{ name: "PIPELINE_NAME", value: settings.pipelineName },
|
||||
{ name: "PIPELINERUN_PREFIX", value: settings.pipelineRunPrefix },
|
||||
{ name: "SERVICE_ACCOUNT_NAME", value: settings.serviceAccountName },
|
||||
{ name: "GITOPS_TARGET", value: settings.gitopsTarget },
|
||||
{ name: "LANE", value: settings.lane },
|
||||
{ name: "CATALOG_PATH", value: args.catalogPath || settings.catalogPath },
|
||||
{ name: "IMAGE_TAG_MODE", value: args.imageTagMode || settings.imageTagMode },
|
||||
{ name: "RUNTIME_PATH", value: gitopsPathForProfile(args, settings.profile) },
|
||||
{ name: "GIT_URL", value: args.sourceRepo },
|
||||
{ name: "GIT_READ_URL", value: args.gitReadUrl },
|
||||
{ name: "SOURCE_BRANCH", value: args.sourceBranch },
|
||||
{ name: "GITOPS_BRANCH", value: args.gitopsBranch },
|
||||
{ name: "REGISTRY_PREFIX", value: args.registryPrefix },
|
||||
{ name: "SERVICES", value: servicesParamForLane(args.lane, deploy) },
|
||||
{ name: "BASE_IMAGE", value: defaultDevBaseImage }
|
||||
],
|
||||
volumeMounts: [{ name: "git-ssh", mountPath: "/workspace/git-ssh", readOnly: true }],
|
||||
command: ["/bin/sh", "-c"],
|
||||
args: [pollerScript()]
|
||||
}],
|
||||
volumes: [{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function tektonControlPlaneReconcilerCronJob(args) {
|
||||
const settings = ciLaneSettings(args);
|
||||
if (settings.lane === "v02") throw new Error("v02 CI/CD is manually triggered by UniDesk CLI and must not render a control-plane reconciler CronJob");
|
||||
return {
|
||||
apiVersion: "batch/v1",
|
||||
kind: "CronJob",
|
||||
metadata: {
|
||||
name: settings.controlPlaneReconcilerName,
|
||||
namespace: "hwlab-ci",
|
||||
labels: {
|
||||
"app.kubernetes.io/part-of": "hwlab",
|
||||
"hwlab.pikastech.local/gitops-target": settings.gitopsTarget
|
||||
},
|
||||
annotations: {
|
||||
"hwlab.pikastech.local/source-branch": args.sourceBranch,
|
||||
"hwlab.pikastech.local/gitops-branch": args.gitopsBranch,
|
||||
"hwlab.pikastech.local/purpose": "render-and-apply-node-ci-control-plane"
|
||||
}
|
||||
},
|
||||
spec: {
|
||||
schedule: "*/10 * * * *",
|
||||
concurrencyPolicy: "Forbid",
|
||||
successfulJobsHistoryLimit: 3,
|
||||
failedJobsHistoryLimit: 3,
|
||||
jobTemplate: {
|
||||
spec: {
|
||||
backoffLimit: 1,
|
||||
template: {
|
||||
metadata: {
|
||||
labels: {
|
||||
"app.kubernetes.io/part-of": "hwlab",
|
||||
"hwlab.pikastech.local/gitops-target": settings.gitopsTarget
|
||||
}
|
||||
},
|
||||
spec: {
|
||||
serviceAccountName: settings.serviceAccountName,
|
||||
restartPolicy: "Never",
|
||||
hostNetwork: true,
|
||||
dnsPolicy: "ClusterFirstWithHostNet",
|
||||
containers: [{
|
||||
name: "reconcile",
|
||||
image: ciToolsRunnerImage,
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
env: [
|
||||
...proxyEnv(),
|
||||
{ name: "POD_NAMESPACE", valueFrom: { fieldRef: { fieldPath: "metadata.namespace" } } },
|
||||
{ name: "LANE", value: settings.lane },
|
||||
{ name: "CATALOG_PATH", value: args.catalogPath || settings.catalogPath },
|
||||
{ name: "IMAGE_TAG_MODE", value: args.imageTagMode || settings.imageTagMode },
|
||||
{ name: "TEKTON_DIR", value: settings.tektonDir },
|
||||
{ name: "ARGO_FILES", value: settings.lane === "v02" ? `${gitopsPathFor(args, "argocd/project.yaml")},${gitopsPathFor(args, "argocd/application-v02.yaml")}` : "" },
|
||||
{ name: "FIELD_MANAGER", value: settings.controlPlaneReconcilerName },
|
||||
{ name: "RECONCILE_EVENT", value: `${settings.gitopsTarget}-control-plane-reconciled` },
|
||||
{ name: "GIT_URL", value: args.sourceRepo },
|
||||
{ name: "GIT_READ_URL", value: args.gitReadUrl },
|
||||
{ name: "SOURCE_BRANCH", value: args.sourceBranch },
|
||||
{ name: "GITOPS_BRANCH", value: args.gitopsBranch },
|
||||
{ name: "REGISTRY_PREFIX", value: args.registryPrefix }
|
||||
],
|
||||
volumeMounts: [{ name: "git-ssh", mountPath: "/workspace/git-ssh", readOnly: true }],
|
||||
command: ["/bin/sh", "-c"],
|
||||
args: [controlPlaneReconcileScript()]
|
||||
}],
|
||||
volumes: [{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export {
|
||||
affectedResultName,
|
||||
buildResultName,
|
||||
buildTaskName,
|
||||
buildkitBinVolume,
|
||||
buildkitRunVolume,
|
||||
buildkitSidecar,
|
||||
ciLaneSettings,
|
||||
collectArtifactsTask,
|
||||
imagePublishTaskSet,
|
||||
perServiceBuildTask,
|
||||
planArtifactsTask,
|
||||
prepareBuildkitClientStep,
|
||||
runtimeReadyScript,
|
||||
runtimeReadyTask,
|
||||
serviceResultEnv,
|
||||
serviceResultEnvName,
|
||||
serviceResultParamName,
|
||||
serviceResultParams,
|
||||
serviceResultParamValues,
|
||||
serviceWorkVolume,
|
||||
tektonControlPlaneReconcilerCronJob,
|
||||
tektonPipeline,
|
||||
tektonPipelineRunTemplate,
|
||||
tektonPollerCronJob,
|
||||
tektonRbac
|
||||
};
|
||||
@@ -0,0 +1,187 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
ciToolsRunnerImage,
|
||||
defaultServiceIds,
|
||||
defaultV02RuntimeEndpoint,
|
||||
primitiveValidationTasks,
|
||||
proxyEnv
|
||||
} from "./core.mjs";
|
||||
|
||||
const templatesDir = path.join(path.dirname(fileURLToPath(import.meta.url)), "templates");
|
||||
const templateCache = new Map();
|
||||
|
||||
function readTemplate(name) {
|
||||
if (!templateCache.has(name)) {
|
||||
templateCache.set(name, readFileSync(path.join(templatesDir, name), "utf8"));
|
||||
}
|
||||
return templateCache.get(name);
|
||||
}
|
||||
|
||||
function renderTemplate(name, replacements = {}) {
|
||||
let rendered = readTemplate(name);
|
||||
for (const [token, value] of Object.entries(replacements)) {
|
||||
if (!rendered.includes(token)) throw new Error(`template ${name} is missing token ${token}`);
|
||||
rendered = rendered.replaceAll(token, String(value));
|
||||
}
|
||||
const unresolved = [...rendered.matchAll(/__HWLAB_[A-Z0-9_]+__/gu)].map((match) => match[0]);
|
||||
if (unresolved.length > 0) {
|
||||
throw new Error(`template ${name} has unresolved tokens: ${[...new Set(unresolved)].join(", ")}`);
|
||||
}
|
||||
return rendered;
|
||||
}
|
||||
|
||||
function shellSingleQuote(value) {
|
||||
return `'${String(value).replace(/'/g, `'"'"'`)}'`;
|
||||
}
|
||||
|
||||
function ciTimingShellFunction() {
|
||||
return readTemplate("ci-timing-functions.sh");
|
||||
}
|
||||
|
||||
function dependencyProxyProbeScript(phase, targetUrl) {
|
||||
if (!/^[a-z0-9-]+$/u.test(phase)) throw new Error(`invalid dependency proxy probe phase: ${phase}`);
|
||||
return renderTemplate("dependency-proxy-probe.sh", {
|
||||
"__HWLAB_PROXY_PHASE_SHELL__": shellSingleQuote(phase),
|
||||
"__HWLAB_PROXY_TARGET_SHELL__": shellSingleQuote(targetUrl),
|
||||
"__HWLAB_PROXY_PHASE__": phase
|
||||
});
|
||||
}
|
||||
|
||||
function curlDownloadProbeFunction() {
|
||||
return readTemplate("curl-download-probe.sh");
|
||||
}
|
||||
|
||||
function gitSshShellFunction() {
|
||||
return renderTemplate("git-ssh-functions.sh", {
|
||||
"// __HWLAB_GITHUB_PROXY_CONNECT_MJS__": readTemplate("github-proxy-connect.mjs")
|
||||
});
|
||||
}
|
||||
|
||||
function prepareSourceScript() {
|
||||
return renderTemplate("prepare-source.sh", {
|
||||
"# __HWLAB_CI_TIMING_SHELL__\n": ciTimingShellFunction(),
|
||||
"# __HWLAB_GIT_SSH_SHELL__\n": gitSshShellFunction(),
|
||||
"__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage,
|
||||
"__HWLAB_V02_RUNTIME_ENDPOINT__": JSON.stringify(defaultV02RuntimeEndpoint).slice(1, -1),
|
||||
"__HWLAB_VALIDATION_TASK_COUNT__": primitiveValidationTasks.length
|
||||
});
|
||||
}
|
||||
|
||||
function sourceValidationScript(commands) {
|
||||
return [
|
||||
"#!/bin/sh",
|
||||
"set -eu",
|
||||
"git config --global --add safe.directory /workspace/source/repo",
|
||||
"cd /workspace/source/repo",
|
||||
"test \"$(git rev-parse HEAD)\" = \"$(params.revision)\"",
|
||||
...commands
|
||||
].join("\n") + "\n";
|
||||
}
|
||||
|
||||
function primitiveValidationTaskNames() {
|
||||
return primitiveValidationTasks.map((task) => task.name);
|
||||
}
|
||||
|
||||
function primitiveValidationTask(task) {
|
||||
return {
|
||||
name: task.name,
|
||||
runAfter: ["prepare-source"],
|
||||
workspaces: [{ name: "source", workspace: "source" }],
|
||||
taskSpec: {
|
||||
params: [
|
||||
{ name: "revision" },
|
||||
{ name: "lane" },
|
||||
{ name: "catalog-path" },
|
||||
{ name: "image-tag-mode" },
|
||||
{ name: "source-branch" },
|
||||
{ name: "gitops-branch" }
|
||||
],
|
||||
workspaces: [{ name: "source" }],
|
||||
steps: [{ name: task.name, image: ciToolsRunnerImage, env: proxyEnv(), script: sourceValidationScript(task.commands) }]
|
||||
},
|
||||
params: [
|
||||
{ name: "revision", value: "$(params.revision)" },
|
||||
{ name: "lane", value: "$(params.lane)" },
|
||||
{ name: "catalog-path", value: "$(params.catalog-path)" },
|
||||
{ name: "image-tag-mode", value: "$(params.image-tag-mode)" },
|
||||
{ name: "source-branch", value: "$(params.source-branch)" },
|
||||
{ name: "gitops-branch", value: "$(params.gitops-branch)" }
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function planArtifactsScript() {
|
||||
return renderTemplate("plan-artifacts.sh", {
|
||||
"__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage,
|
||||
'["__HWLAB_DEFAULT_SERVICE_IDS__"]': JSON.stringify(defaultServiceIds)
|
||||
});
|
||||
}
|
||||
|
||||
function perServicePublishScript() {
|
||||
return renderTemplate("per-service-publish.sh", {
|
||||
"# __HWLAB_CI_TIMING_SHELL__\n": ciTimingShellFunction(),
|
||||
"# __HWLAB_DEPENDENCY_PROXY_PROBE__": dependencyProxyProbeScript(
|
||||
"service-image-publish-proxy-preflight",
|
||||
"http://deb.debian.org/debian/dists/bookworm/InRelease"
|
||||
),
|
||||
"__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage
|
||||
});
|
||||
}
|
||||
|
||||
function collectArtifactsScript() {
|
||||
return renderTemplate("collect-artifacts.sh", {
|
||||
"__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage
|
||||
});
|
||||
}
|
||||
|
||||
function gitopsPromoteScript() {
|
||||
return renderTemplate("gitops-promote.sh", {
|
||||
"# __HWLAB_CI_TIMING_SHELL__\n": ciTimingShellFunction(),
|
||||
"# __HWLAB_GIT_SSH_SHELL__\n": gitSshShellFunction(),
|
||||
"__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage
|
||||
});
|
||||
}
|
||||
|
||||
function controlPlaneReconcileScript() {
|
||||
return renderTemplate("control-plane-reconcile.sh", {
|
||||
"# __HWLAB_DEPENDENCY_PROXY_PROBE__": dependencyProxyProbeScript(
|
||||
"control-plane-proxy-preflight",
|
||||
"http://deb.debian.org/debian/dists/bookworm/InRelease"
|
||||
),
|
||||
"# __HWLAB_GIT_SSH_SHELL__\n": gitSshShellFunction(),
|
||||
"__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage
|
||||
});
|
||||
}
|
||||
|
||||
function pollerScript() {
|
||||
return renderTemplate("poller.sh", {
|
||||
"# __HWLAB_DEPENDENCY_PROXY_PROBE__": dependencyProxyProbeScript(
|
||||
"poller-proxy-preflight",
|
||||
"http://deb.debian.org/debian/dists/bookworm/InRelease"
|
||||
),
|
||||
"# __HWLAB_GIT_SSH_SHELL__\n": gitSshShellFunction(),
|
||||
"__HWLAB_CI_TOOLS_IMAGE__": ciToolsRunnerImage
|
||||
});
|
||||
}
|
||||
|
||||
export {
|
||||
ciTimingShellFunction,
|
||||
collectArtifactsScript,
|
||||
controlPlaneReconcileScript,
|
||||
curlDownloadProbeFunction,
|
||||
dependencyProxyProbeScript,
|
||||
gitopsPromoteScript,
|
||||
gitSshShellFunction,
|
||||
perServicePublishScript,
|
||||
planArtifactsScript,
|
||||
pollerScript,
|
||||
prepareSourceScript,
|
||||
primitiveValidationTask,
|
||||
primitiveValidationTaskNames,
|
||||
renderTemplate,
|
||||
shellSingleQuote,
|
||||
sourceValidationScript
|
||||
};
|
||||
@@ -0,0 +1,13 @@
|
||||
ci_now_ms() {
|
||||
node -e 'console.log(Date.now())'
|
||||
}
|
||||
|
||||
ci_timing_emit() {
|
||||
stage="$1"
|
||||
status="$2"
|
||||
started_ms="$3"
|
||||
finished_ms="$(ci_now_ms)"
|
||||
duration_ms="$((finished_ms - started_ms))"
|
||||
service_id="${HWLAB_TIMING_SERVICE_ID:-}"
|
||||
node -e 'const [stage,status,durationMs,serviceId]=process.argv.slice(1); const payload={event:"node-cicd-timing",schemaVersion:"v1",stage,status,durationMs:Number(durationMs),pipelineRun:process.env.HWLAB_TEKTON_PIPELINERUN||null,taskRun:process.env.HWLAB_TEKTON_TASKRUN||null,task:process.env.HWLAB_TEKTON_TASK||null,revision:process.env.HWLAB_SOURCE_REVISION||null,serviceId:serviceId||null,source:"scripts/gitops-render.mjs",at:new Date().toISOString()}; console.log(JSON.stringify(payload));' "$stage" "$status" "$duration_ms" "$service_id"
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
echo '{"event":"ci-base-image","phase":"collect-artifacts","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apk"}'
|
||||
for tool in node git; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"collect-artifacts","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done
|
||||
cd /workspace/source/repo
|
||||
test "$(git rev-parse HEAD)" = "$(params.revision)"
|
||||
export HWLAB_CI_ARTIFACT_RUN_ID="$(context.pipelineRun.name)-collect"
|
||||
export HWLAB_CI_ARTIFACT_RUN_OWNER="tekton"
|
||||
export HWLAB_DEV_REGISTRY_PREFIX="$(params.registry-prefix)"
|
||||
export HWLAB_DEV_REGISTRY_K8S_NATIVE=1
|
||||
if [ -n "$(params.base-image)" ]; then export HWLAB_DEV_BASE_IMAGE="$(params.base-image)"; fi
|
||||
export HWLAB_ARTIFACT_LANE="$(params.lane)"
|
||||
export HWLAB_ARTIFACT_CATALOG_PATH="$(params.catalog-path)"
|
||||
export HWLAB_DEPLOY_MANIFEST_PATH="deploy/deploy.yaml"
|
||||
export HWLAB_ARTIFACT_IMAGE_TAG_MODE="$(params.image-tag-mode)"
|
||||
if [ -s /workspace/source/affected-services.json ] && node - /workspace/source/affected-services.json <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const plan = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
||||
const buildServices = Array.isArray(plan.buildServices) ? plan.buildServices : [];
|
||||
const affectedServices = Array.isArray(plan.affectedServices) ? plan.affectedServices : [];
|
||||
const willRunGitopsPromote = plan.ciCdPlan && plan.ciCdPlan.willRunGitopsPromote === true;
|
||||
process.exit(!willRunGitopsPromote && buildServices.length === 0 && affectedServices.length === 0 ? 0 : 1);
|
||||
NODE
|
||||
then
|
||||
rm -f /workspace/source/dev-artifacts.json
|
||||
echo '{"event":"collect-artifacts","status":"skipped","reason":"no-build-no-rollout-plan"}'
|
||||
exit 0
|
||||
fi
|
||||
HWLAB_CI_PLAN_VERIFY_REUSE_REGISTRY=1 node scripts/artifact-publish.mjs --publish --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --registry-prefix "$(params.registry-prefix)" --report /workspace/source/dev-artifacts.json --quiet-build --concurrency 1 --services "$(params.services)" --external-service-report-dir /workspace/source/service-results --ci-plan-path /workspace/source/affected-services.json
|
||||
node scripts/refresh-artifact-catalog.mjs --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --target-ref HEAD --publish-report /workspace/source/dev-artifacts.json --no-write
|
||||
@@ -0,0 +1,139 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# __HWLAB_DEPENDENCY_PROXY_PROBE__
|
||||
echo '{"event":"ci-base-image","phase":"control-plane-reconciler","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apk"}'
|
||||
for tool in node npm git python3 ssh curl timeout; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"control-plane-reconciler","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done
|
||||
# __HWLAB_GIT_SSH_SHELL__
|
||||
|
||||
git_ssh_setup
|
||||
workdir="$(mktemp -d)"
|
||||
git_timed control-plane-clone 180 git clone --depth 1 --branch "$SOURCE_BRANCH" "$GIT_READ_URL" "$workdir/repo"
|
||||
cd "$workdir/repo"
|
||||
git remote set-url origin "$GIT_READ_URL"
|
||||
revision="$(git rev-parse HEAD)"
|
||||
: "${LANE:?LANE is required}"
|
||||
: "${CATALOG_PATH:?CATALOG_PATH is required}"
|
||||
: "${IMAGE_TAG_MODE:?IMAGE_TAG_MODE is required}"
|
||||
: "${RUNTIME_PATH:?RUNTIME_PATH is required}"
|
||||
gitops_root="${RUNTIME_PATH%/*}"
|
||||
if [ "$gitops_root" = "$RUNTIME_PATH" ]; then gitops_root="."; fi
|
||||
node scripts/run-bun.mjs scripts/gitops-render.mjs --lane "$LANE" --catalog-path "$CATALOG_PATH" --image-tag-mode "$IMAGE_TAG_MODE" --source-revision "$revision" --source-repo "$GIT_URL" --source-branch "$SOURCE_BRANCH" --gitops-branch "$GITOPS_BRANCH" --gitops-root "$gitops_root" --out "$gitops_root" --registry-prefix "$REGISTRY_PREFIX"
|
||||
node scripts/run-bun.mjs scripts/gitops-render.mjs --lane "$LANE" --catalog-path "$CATALOG_PATH" --image-tag-mode "$IMAGE_TAG_MODE" --source-revision "$revision" --source-repo "$GIT_URL" --source-branch "$SOURCE_BRANCH" --gitops-branch "$GITOPS_BRANCH" --gitops-root "$gitops_root" --out "$gitops_root" --registry-prefix "$REGISTRY_PREFIX" --check
|
||||
node <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const https = require("node:https");
|
||||
|
||||
function requiredEnv(name) {
|
||||
const value = process.env[name];
|
||||
if (!value) throw new Error(name + " is required");
|
||||
return value;
|
||||
}
|
||||
|
||||
const namespace = requiredEnv("POD_NAMESPACE");
|
||||
const host = process.env.KUBERNETES_SERVICE_HOST;
|
||||
const port = process.env.KUBERNETES_SERVICE_PORT || "443";
|
||||
if (!host) throw new Error("KUBERNETES_SERVICE_HOST is not available");
|
||||
|
||||
const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8");
|
||||
const ca = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt");
|
||||
const tektonDir = requiredEnv("TEKTON_DIR");
|
||||
const argoFiles = String(process.env.ARGO_FILES || "").split(",").map((item) => item.trim()).filter(Boolean);
|
||||
const fieldManager = requiredEnv("FIELD_MANAGER");
|
||||
const eventName = requiredEnv("RECONCILE_EVENT");
|
||||
const manifestFiles = [
|
||||
"deploy/gitops/node/" + tektonDir + "/rbac.yaml",
|
||||
"deploy/gitops/node/" + tektonDir + "/pipeline.yaml",
|
||||
...(process.env.LANE === "v02" ? [] : [
|
||||
"deploy/gitops/node/" + tektonDir + "/poller.yaml",
|
||||
"deploy/gitops/node/" + tektonDir + "/control-plane-reconciler.yaml"
|
||||
]),
|
||||
...argoFiles
|
||||
];
|
||||
const plurals = new Map([
|
||||
["ServiceAccount", "serviceaccounts"],
|
||||
["Role", "roles"],
|
||||
["RoleBinding", "rolebindings"],
|
||||
["Pipeline", "pipelines"],
|
||||
["CronJob", "cronjobs"],
|
||||
["Application", "applications"],
|
||||
["AppProject", "appprojects"]
|
||||
]);
|
||||
|
||||
function encode(value) {
|
||||
return encodeURIComponent(String(value));
|
||||
}
|
||||
|
||||
function itemsFrom(file) {
|
||||
const parsed = JSON.parse(fs.readFileSync(file, "utf8"));
|
||||
return parsed.kind === "List" ? parsed.items || [] : [parsed];
|
||||
}
|
||||
|
||||
function apiPath(item) {
|
||||
if (item.kind === "Namespace") return null;
|
||||
const name = item.metadata?.name;
|
||||
if (!name) throw new Error("manifest item is missing metadata.name");
|
||||
const ns = item.metadata?.namespace || namespace;
|
||||
const plural = plurals.get(item.kind);
|
||||
if (!plural) throw new Error("unsupported reconciler manifest kind " + item.kind);
|
||||
if (item.apiVersion === "v1") return "/api/v1/namespaces/" + encode(ns) + "/" + plural + "/" + encode(name);
|
||||
const parts = String(item.apiVersion || "").split("/");
|
||||
if (parts.length !== 2) throw new Error("unsupported apiVersion " + item.apiVersion + " for " + item.kind);
|
||||
return "/apis/" + encode(parts[0]) + "/" + encode(parts[1]) + "/namespaces/" + encode(ns) + "/" + plural + "/" + encode(name);
|
||||
}
|
||||
|
||||
function reconcilePolicy(item) {
|
||||
const ns = item.metadata?.namespace || namespace;
|
||||
const crossNamespaceRbac = (item.kind === "Role" || item.kind === "RoleBinding") && ns !== namespace;
|
||||
if (crossNamespaceRbac && process.env.HWLAB_RECONCILE_CROSS_NAMESPACE_RBAC !== "1") {
|
||||
return { action: "skip", reason: "cross-namespace-rbac-bootstrap-managed", namespace: ns };
|
||||
}
|
||||
return { action: "apply", namespace: ns };
|
||||
}
|
||||
|
||||
function request(method, path, body) {
|
||||
const payload = body ? JSON.stringify(body) : "";
|
||||
const headers = { Authorization: "Bearer " + token };
|
||||
if (payload) {
|
||||
headers["Content-Type"] = "application/apply-patch+yaml";
|
||||
headers["Content-Length"] = Buffer.byteLength(payload);
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = https.request({ host, port, method, path, ca, headers }, (res) => {
|
||||
let data = "";
|
||||
res.setEncoding("utf8");
|
||||
res.on("data", (chunk) => { data += chunk; });
|
||||
res.on("end", () => resolve({ statusCode: res.statusCode || 0, body: data }));
|
||||
});
|
||||
req.on("error", reject);
|
||||
if (payload) req.write(payload);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const results = [];
|
||||
for (const file of manifestFiles) {
|
||||
for (const item of itemsFrom(file)) {
|
||||
const policy = reconcilePolicy(item);
|
||||
if (policy.action === "skip") {
|
||||
results.push({ file, kind: item.kind, name: item.metadata?.name || null, namespace: policy.namespace, status: "skipped", reason: policy.reason });
|
||||
continue;
|
||||
}
|
||||
const path = apiPath(item);
|
||||
if (!path) {
|
||||
results.push({ file, kind: item.kind, name: item.metadata?.name || null, status: "skipped-cluster-scoped" });
|
||||
continue;
|
||||
}
|
||||
const response = await request("PATCH", path + "?fieldManager=" + encode(fieldManager) + "&force=true", item);
|
||||
if (response.statusCode < 200 || response.statusCode > 299) {
|
||||
throw new Error("apply failed for " + item.kind + "/" + item.metadata.name + " status=" + response.statusCode + " body=" + response.body.slice(0, 1000));
|
||||
}
|
||||
results.push({ file, kind: item.kind, name: item.metadata.name, status: "applied", statusCode: response.statusCode });
|
||||
}
|
||||
}
|
||||
console.log(JSON.stringify({ event: eventName, sourceRevision: process.env.RECONCILE_REVISION || null, results }, null, 2));
|
||||
})().catch((error) => {
|
||||
console.error(error.stack || error.message);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
NODE
|
||||
@@ -0,0 +1,17 @@
|
||||
redact_proxy_value() {
|
||||
printf '%s' "${1:-}" | sed -E 's#(https?://)[^/@]+@#\1***@#; s#(socks5h?://)[^/@]+@#\1***@#'
|
||||
}
|
||||
|
||||
curl_dependency_probe() {
|
||||
phase="$1"
|
||||
url="$2"
|
||||
proxy="${HTTPS_PROXY:-${https_proxy:-${HTTP_PROXY:-${http_proxy:-}}}}"
|
||||
if [ -z "$proxy" ]; then
|
||||
echo '{"event":"dependency-curl-probe","phase":"'"$phase"'","ok":false,"reason":"proxy-env-missing"}'
|
||||
return 21
|
||||
fi
|
||||
safe_proxy="$(redact_proxy_value "$proxy")"
|
||||
echo '{"event":"dependency-curl-probe-start","phase":"'"$phase"'","proxy":"'"$safe_proxy"'","target":"'"$url"'"}'
|
||||
curl -sS -L --range 0-1048575 -o /dev/null --connect-timeout 15 --max-time 60 --proxy "$proxy" -w '{"event":"dependency-curl-probe","phase":"'"$phase"'","http_code":"%{http_code}","time_namelookup":%{time_namelookup},"time_connect":%{time_connect},"time_starttransfer":%{time_starttransfer},"time_total":%{time_total},"speed_download":%{speed_download},"size_download":%{size_download},"remote_ip":"%{remote_ip}"}
|
||||
' "$url"
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
HWLAB_PROXY_PROBE_PHASE=__HWLAB_PROXY_PHASE_SHELL__ HWLAB_PROXY_PROBE_URL=__HWLAB_PROXY_TARGET_SHELL__ node <<'NODE' || echo '{"event":"dependency-proxy-probe-nonblocking","phase":"__HWLAB_PROXY_PHASE__","ok":false,"reason":"probe-failed-but-continuing"}'
|
||||
const net = require("node:net");
|
||||
|
||||
const phase = process.env.HWLAB_PROXY_PROBE_PHASE || "unknown";
|
||||
const target = new URL(process.env.HWLAB_PROXY_PROBE_URL || "http://deb.debian.org/debian/dists/bookworm/InRelease");
|
||||
const proxyRaw = process.env.HTTP_PROXY || process.env.http_proxy || "";
|
||||
|
||||
function redactProxy(value) {
|
||||
if (!value) return "";
|
||||
try {
|
||||
const parsed = new URL(value);
|
||||
if (parsed.username || parsed.password) {
|
||||
parsed.username = "***";
|
||||
parsed.password = "";
|
||||
}
|
||||
return parsed.toString();
|
||||
} catch {
|
||||
return "<invalid-proxy-url>";
|
||||
}
|
||||
}
|
||||
|
||||
function emit(payload, exitCode = 0) {
|
||||
console.log(JSON.stringify({
|
||||
event: "dependency-proxy-probe",
|
||||
phase,
|
||||
target: target.href,
|
||||
proxy: redactProxy(proxyRaw),
|
||||
...payload
|
||||
}));
|
||||
if (exitCode) process.exit(exitCode);
|
||||
}
|
||||
|
||||
if (!proxyRaw) emit({ ok: false, reason: "proxy-env-missing" }, 21);
|
||||
|
||||
let proxy;
|
||||
try {
|
||||
proxy = new URL(proxyRaw);
|
||||
} catch (error) {
|
||||
emit({ ok: false, reason: "proxy-url-invalid", error: error.message }, 22);
|
||||
}
|
||||
|
||||
if (proxy.protocol !== "http:" && proxy.protocol !== "https:") {
|
||||
emit({ ok: false, reason: "http-proxy-required", protocol: proxy.protocol }, 23);
|
||||
}
|
||||
|
||||
const startedAt = Date.now();
|
||||
const socket = net.createConnection({ host: proxy.hostname, port: Number(proxy.port || (proxy.protocol === "https:" ? 443 : 80)) });
|
||||
let bytes = 0;
|
||||
let firstByteMs = null;
|
||||
let responseHead = "";
|
||||
let finished = false;
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
socket.destroy();
|
||||
emit({ ok: false, reason: "timeout", timeoutMs: 15000 }, 24);
|
||||
}, 15000);
|
||||
|
||||
socket.on("connect", () => {
|
||||
socket.write("GET " + target.href + " HTTP/1.1\r\nHost: " + target.host + "\r\nUser-Agent: hwlab-node-proxy-probe\r\nConnection: close\r\n\r\n");
|
||||
});
|
||||
|
||||
socket.on("data", (chunk) => {
|
||||
if (firstByteMs === null) firstByteMs = Date.now() - startedAt;
|
||||
bytes += chunk.length;
|
||||
if (responseHead.length < 240) responseHead += chunk.toString("utf8", 0, Math.min(chunk.length, 240 - responseHead.length));
|
||||
});
|
||||
|
||||
socket.on("end", () => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
clearTimeout(timeout);
|
||||
const totalMs = Date.now() - startedAt;
|
||||
const statusLine = responseHead.split("\r\n", 1)[0] || "";
|
||||
const statusCode = Number(statusLine.split(" ")[1] || 0);
|
||||
const ok = statusLine.startsWith("HTTP/") && statusCode >= 200 && statusCode < 400;
|
||||
emit({
|
||||
ok,
|
||||
reason: ok ? null : "bad-http-status",
|
||||
statusLine,
|
||||
statusCode,
|
||||
firstByteMs,
|
||||
totalMs,
|
||||
bytes,
|
||||
speedBytesPerSecond: totalMs > 0 ? Math.round(bytes * 1000 / totalMs) : 0
|
||||
}, ok ? 0 : 25);
|
||||
});
|
||||
|
||||
socket.on("error", (error) => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
clearTimeout(timeout);
|
||||
emit({ ok: false, reason: "proxy-connect-failed", error: error.message, totalMs: Date.now() - startedAt }, 26);
|
||||
});
|
||||
NODE
|
||||
@@ -0,0 +1,76 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
repo_url="ssh://git@ssh.github.com:443/pikasTech/HWLAB.git"
|
||||
repo_path="/cache/pikasTech/HWLAB.git"
|
||||
lock_dir="/cache/.hwlab-flush.lock"
|
||||
started_ms=$(node -e 'console.log(Date.now())')
|
||||
now_ms() { node -e 'console.log(Date.now())'; }
|
||||
emit_timing() {
|
||||
phase="$1"
|
||||
status="$2"
|
||||
started="$3"
|
||||
finished=$(now_ms)
|
||||
duration=$((finished - started))
|
||||
printf '{"event":"git-mirror-flush-timing","phase":"%s","status":"%s","durationMs":%s,"repository":"pikasTech/HWLAB"}
|
||||
' "$phase" "$status" "$duration"
|
||||
}
|
||||
if ! mkdir "$lock_dir" 2>/dev/null; then
|
||||
echo "git mirror flush already running"
|
||||
exit 0
|
||||
fi
|
||||
trap 'rmdir "$lock_dir" 2>/dev/null || true' EXIT INT TERM
|
||||
mkdir -p /root/.ssh
|
||||
cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa
|
||||
chmod 0400 /root/.ssh/id_rsa
|
||||
# __HWLAB_GIT_MIRROR_SSH_PRELUDE__
|
||||
/script/install-hooks.sh
|
||||
for gitops_branch in __HWLAB_GITOPS_BRANCH_ARGS__; do
|
||||
local_head=$(git -C "$repo_path" show-ref --verify --hash "refs/heads/$gitops_branch" 2>/dev/null || true)
|
||||
if [ -z "$local_head" ]; then
|
||||
echo "git mirror flush skips missing local refs/heads/$gitops_branch"
|
||||
continue
|
||||
fi
|
||||
fetch_started=$(now_ms)
|
||||
timeout 90 git -C "$repo_path" fetch --quiet "$repo_url" "refs/heads/$gitops_branch:refs/mirror-stage/heads/$gitops_branch"
|
||||
emit_timing "fetch-$gitops_branch" succeeded "$fetch_started"
|
||||
github_head=$(git -C "$repo_path" show-ref --verify --hash "refs/mirror-stage/heads/$gitops_branch" 2>/dev/null || true)
|
||||
if [ -n "$github_head" ] && [ "$github_head" != "$local_head" ] && ! git -C "$repo_path" merge-base --is-ancestor "$github_head" "$local_head"; then
|
||||
echo "git mirror flush rejected divergent $gitops_branch local=$local_head github=$github_head" >&2
|
||||
exit 52
|
||||
fi
|
||||
if [ "$local_head" != "$github_head" ]; then
|
||||
push_started=$(now_ms)
|
||||
timeout 120 git -C "$repo_path" push "$repo_url" "$local_head:refs/heads/$gitops_branch"
|
||||
emit_timing "push-$gitops_branch" succeeded "$push_started"
|
||||
else
|
||||
echo "git mirror flush $gitops_branch already matches GitHub"
|
||||
fi
|
||||
git -C "$repo_path" update-ref "refs/mirror-stage/heads/$gitops_branch" "$local_head"
|
||||
done
|
||||
git -C "$repo_path" update-server-info
|
||||
flushed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
export flushed_at
|
||||
export gitops_branches_json=__HWLAB_GITOPS_BRANCHES_JSON_SHELL__
|
||||
node - <<'NODE' | tee /cache/HWLAB.last-flush.json
|
||||
const { execFileSync } = require("node:child_process");
|
||||
const repo = "/cache/pikasTech/HWLAB.git";
|
||||
const gitopsBranches = JSON.parse(process.env.gitops_branches_json || "[]");
|
||||
function rev(ref) {
|
||||
try { return execFileSync("git", ["-C", repo, "rev-parse", ref], { encoding: "utf8" }).trim() || null; } catch { return null; }
|
||||
}
|
||||
const refs = {};
|
||||
for (const branch of gitopsBranches) {
|
||||
refs["refs/heads/" + branch] = rev("refs/heads/" + branch);
|
||||
refs["refs/mirror-stage/heads/" + branch] = rev("refs/mirror-stage/heads/" + branch);
|
||||
}
|
||||
const pendingFlush = gitopsBranches.some((branch) => refs["refs/heads/" + branch] && refs["refs/mirror-stage/heads/" + branch] && refs["refs/heads/" + branch] !== refs["refs/mirror-stage/heads/" + branch]);
|
||||
console.log(JSON.stringify({
|
||||
event: "git-mirror-flush",
|
||||
status: "flushed",
|
||||
repository: "pikasTech/HWLAB",
|
||||
flushedAt: process.env.flushed_at,
|
||||
pendingFlush,
|
||||
refs
|
||||
}));
|
||||
NODE
|
||||
emit_timing total succeeded "$started_ms"
|
||||
@@ -0,0 +1,196 @@
|
||||
import { createServer } from "node:http";
|
||||
import { spawn } from "node:child_process";
|
||||
import { createReadStream, statSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { createGunzip, createInflate } from "node:zlib";
|
||||
|
||||
const cacheRoot = process.env.GIT_MIRROR_CACHE_ROOT || "/cache";
|
||||
const port = Number(process.env.PORT || 8080);
|
||||
|
||||
createServer((req, res) => {
|
||||
handle(req, res).catch((error) => {
|
||||
if (!res.headersSent) res.writeHead(500, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify({ ok: false, error: error.message }));
|
||||
});
|
||||
}).listen(port, "0.0.0.0", () => {
|
||||
console.log(JSON.stringify({ event: "git-mirror-smart-http-listening", port, cacheRoot }));
|
||||
});
|
||||
|
||||
async function handle(req, res) {
|
||||
const url = new URL(req.url || "/", "http://git-mirror-http");
|
||||
const writeHost = isWriteHost(req.headers.host || "");
|
||||
if (req.method === "GET" && (url.pathname === "/" || url.pathname === "/health")) {
|
||||
res.writeHead(200, { "content-type": "application/json", "cache-control": "no-cache" });
|
||||
res.end(JSON.stringify({ ok: true, service: "git-mirror-smart-http", writeHost }));
|
||||
return;
|
||||
}
|
||||
if (req.method === "GET" && url.pathname.endsWith("/info/refs") && url.searchParams.get("service") === "git-upload-pack") {
|
||||
await runUploadPackAdvertisement(repoPathFromGitServicePath(url.pathname, "/info/refs"), res);
|
||||
return;
|
||||
}
|
||||
if (req.method === "POST" && url.pathname.endsWith("/git-upload-pack")) {
|
||||
await runUploadPackRpc(repoPathFromGitServicePath(url.pathname, "/git-upload-pack"), req, res);
|
||||
return;
|
||||
}
|
||||
if (req.method === "GET" && url.pathname.endsWith("/info/refs") && url.searchParams.get("service") === "git-receive-pack") {
|
||||
if (!writeHost) return rejectReadOnly(res);
|
||||
await runReceivePackAdvertisement(repoPathFromGitServicePath(url.pathname, "/info/refs"), res);
|
||||
return;
|
||||
}
|
||||
if (req.method === "POST" && url.pathname.endsWith("/git-receive-pack")) {
|
||||
if (!writeHost) return rejectReadOnly(res);
|
||||
await runReceivePackRpc(repoPathFromGitServicePath(url.pathname, "/git-receive-pack"), req, res);
|
||||
return;
|
||||
}
|
||||
if (req.method === "GET" || req.method === "HEAD") {
|
||||
serveStaticFile(url.pathname, req, res);
|
||||
return;
|
||||
}
|
||||
res.writeHead(405, { "content-type": "text/plain" });
|
||||
res.end("method not allowed\n");
|
||||
}
|
||||
|
||||
function isWriteHost(hostHeader) {
|
||||
const host = String(hostHeader || "").split(":", 1)[0].toLowerCase();
|
||||
return host === "git-mirror-write" || host.startsWith("git-mirror-write.");
|
||||
}
|
||||
|
||||
function rejectReadOnly(res) {
|
||||
res.writeHead(403, { "content-type": "application/json", "cache-control": "no-cache" });
|
||||
res.end(JSON.stringify({ ok: false, error: "git mirror receive-pack is only available through git-mirror-write" }));
|
||||
}
|
||||
|
||||
function repoPathFromGitServicePath(urlPath, suffix) {
|
||||
const repoUrlPath = urlPath.slice(0, -suffix.length);
|
||||
if (!repoUrlPath.endsWith(".git")) throw new Error(`unsupported git repo path: ${urlPath}`);
|
||||
return safePath(repoUrlPath);
|
||||
}
|
||||
|
||||
function safePath(urlPath) {
|
||||
const decoded = decodeURIComponent(urlPath.replace(/^[/]+/u, ""));
|
||||
const fullPath = path.resolve(cacheRoot, decoded);
|
||||
const normalizedRoot = path.resolve(cacheRoot);
|
||||
if (fullPath !== normalizedRoot && !fullPath.startsWith(`${normalizedRoot}${path.sep}`)) throw new Error("path escapes cache root");
|
||||
return fullPath;
|
||||
}
|
||||
|
||||
function smartHeaders(contentType) {
|
||||
return {
|
||||
"content-type": contentType,
|
||||
"cache-control": "no-cache, max-age=0, must-revalidate",
|
||||
expires: "Fri, 01 Jan 1980 00:00:00 GMT",
|
||||
pragma: "no-cache"
|
||||
};
|
||||
}
|
||||
|
||||
function pktLine(text) {
|
||||
const length = Buffer.byteLength(text) + 4;
|
||||
return `${length.toString(16).padStart(4, "0")}${text}`;
|
||||
}
|
||||
|
||||
function requestBodyStream(req) {
|
||||
const encoding = String(req.headers["content-encoding"] || "identity").toLowerCase();
|
||||
if (encoding === "identity" || encoding === "") return req;
|
||||
if (encoding === "gzip" || encoding === "x-gzip") return req.pipe(createGunzip());
|
||||
if (encoding === "deflate") return req.pipe(createInflate());
|
||||
throw new Error(`unsupported git upload-pack content-encoding: ${encoding}`);
|
||||
}
|
||||
|
||||
function runUploadPackAdvertisement(repoPath, res) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn("git-upload-pack", ["--stateless-rpc", "--advertise-refs", repoPath], { stdio: ["ignore", "pipe", "pipe"] });
|
||||
let stderr = "";
|
||||
child.stderr.on("data", (chunk) => { stderr += chunk; process.stderr.write(chunk); });
|
||||
child.on("error", reject);
|
||||
child.on("close", (code) => {
|
||||
if (code !== 0 && !res.headersSent) reject(new Error(`git-upload-pack advertise failed: ${stderr.trim()}`));
|
||||
else resolve();
|
||||
});
|
||||
res.writeHead(200, smartHeaders("application/x-git-upload-pack-advertisement"));
|
||||
res.write(pktLine("# service=git-upload-pack\n"));
|
||||
res.write("0000");
|
||||
child.stdout.pipe(res, { end: false });
|
||||
child.stdout.on("end", () => res.end());
|
||||
});
|
||||
}
|
||||
|
||||
function runUploadPackRpc(repoPath, req, res) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn("git-upload-pack", ["--stateless-rpc", repoPath], { stdio: ["pipe", "pipe", "pipe"] });
|
||||
let stderr = "";
|
||||
child.stderr.on("data", (chunk) => { stderr += chunk; process.stderr.write(chunk); });
|
||||
child.on("error", reject);
|
||||
child.on("close", (code) => {
|
||||
if (code !== 0 && !res.headersSent) reject(new Error(`git-upload-pack rpc failed: ${stderr.trim()}`));
|
||||
else resolve();
|
||||
});
|
||||
res.writeHead(200, smartHeaders("application/x-git-upload-pack-result"));
|
||||
const body = requestBodyStream(req);
|
||||
body.on("error", (error) => child.stdin.destroy(error));
|
||||
body.pipe(child.stdin);
|
||||
child.stdout.pipe(res);
|
||||
});
|
||||
}
|
||||
|
||||
function runReceivePackAdvertisement(repoPath, res) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn("git-receive-pack", ["--stateless-rpc", "--advertise-refs", repoPath], { stdio: ["ignore", "pipe", "pipe"] });
|
||||
let stderr = "";
|
||||
child.stderr.on("data", (chunk) => { stderr += chunk; process.stderr.write(chunk); });
|
||||
child.on("error", reject);
|
||||
child.on("close", (code) => {
|
||||
if (code !== 0 && !res.headersSent) reject(new Error(`git-receive-pack advertise failed: ${stderr.trim()}`));
|
||||
else resolve();
|
||||
});
|
||||
res.writeHead(200, smartHeaders("application/x-git-receive-pack-advertisement"));
|
||||
res.write(pktLine("# service=git-receive-pack\n"));
|
||||
res.write("0000");
|
||||
child.stdout.pipe(res, { end: false });
|
||||
child.stdout.on("end", () => res.end());
|
||||
});
|
||||
}
|
||||
|
||||
function runReceivePackRpc(repoPath, req, res) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn("git-receive-pack", ["--stateless-rpc", repoPath], { stdio: ["pipe", "pipe", "pipe"] });
|
||||
let stderr = "";
|
||||
child.stderr.on("data", (chunk) => { stderr += chunk; process.stderr.write(chunk); });
|
||||
child.on("error", reject);
|
||||
child.on("close", (code) => {
|
||||
if (code !== 0 && !res.headersSent) reject(new Error(`git-receive-pack rpc failed: ${stderr.trim()}`));
|
||||
else resolve();
|
||||
});
|
||||
res.writeHead(200, smartHeaders("application/x-git-receive-pack-result"));
|
||||
const body = requestBodyStream(req);
|
||||
body.on("error", (error) => child.stdin.destroy(error));
|
||||
body.pipe(child.stdin);
|
||||
child.stdout.pipe(res);
|
||||
});
|
||||
}
|
||||
|
||||
function serveStaticFile(urlPath, req, res) {
|
||||
const fullPath = safePath(urlPath);
|
||||
let stat;
|
||||
try { stat = statSync(fullPath); } catch {
|
||||
res.writeHead(404, { "content-type": "text/plain" });
|
||||
res.end("not found\n");
|
||||
return;
|
||||
}
|
||||
if (!stat.isFile()) {
|
||||
res.writeHead(404, { "content-type": "text/plain" });
|
||||
res.end("not found\n");
|
||||
return;
|
||||
}
|
||||
res.writeHead(200, { "content-length": stat.size, "content-type": contentType(fullPath), "cache-control": "no-cache" });
|
||||
if (req.method === "HEAD") {
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
createReadStream(fullPath).pipe(res);
|
||||
}
|
||||
|
||||
function contentType(filePath) {
|
||||
if (filePath.endsWith(".pack")) return "application/x-git-packed-objects";
|
||||
if (filePath.endsWith(".idx")) return "application/x-git-packed-objects-toc";
|
||||
return "text/plain";
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
repo_path="/cache/pikasTech/HWLAB.git"
|
||||
mkdir -p "$repo_path/hooks"
|
||||
cat > "$repo_path/hooks/pre-receive" <<'HOOK'
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
zero="0000000000000000000000000000000000000000"
|
||||
status=0
|
||||
while read -r old new ref; do
|
||||
if [ "$new" = "$zero" ]; then
|
||||
echo "git mirror write rejected: deleting $ref is forbidden" >&2
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
git cat-file -e "$new^{commit}" || status=1
|
||||
git cat-file -e "$new^{tree}" || status=1
|
||||
if git rev-list --objects --missing=print "$new" | grep -q '^?'; then
|
||||
echo "git mirror write rejected: missing objects for $new" >&2
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
if [ "$old" != "$zero" ] && ! git merge-base --is-ancestor "$old" "$new"; then
|
||||
echo "git mirror write rejected: non-fast-forward $ref" >&2
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
done
|
||||
exit "$status"
|
||||
HOOK
|
||||
cat > "$repo_path/hooks/post-receive" <<'HOOK'
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
repo_path=$(git rev-parse --git-dir)
|
||||
git -C "$repo_path" update-server-info
|
||||
written_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
export repo_path written_at
|
||||
export gitops_branches_json=__HWLAB_GITOPS_BRANCHES_JSON_SHELL__
|
||||
node - <<'NODE' > /cache/HWLAB.last-write.json
|
||||
const { execFileSync } = require("node:child_process");
|
||||
const repo = process.env.repo_path || "/cache/pikasTech/HWLAB.git";
|
||||
const gitopsBranches = JSON.parse(process.env.gitops_branches_json || "[]");
|
||||
function rev(ref) {
|
||||
try { return execFileSync("git", ["-C", repo, "rev-parse", ref], { encoding: "utf8" }).trim() || null; } catch { return null; }
|
||||
}
|
||||
const refs = {};
|
||||
for (const branch of gitopsBranches) {
|
||||
refs["refs/heads/" + branch] = rev("refs/heads/" + branch);
|
||||
refs["refs/mirror-stage/heads/" + branch] = rev("refs/mirror-stage/heads/" + branch);
|
||||
}
|
||||
const pendingFlush = gitopsBranches.some((branch) => refs["refs/heads/" + branch] && refs["refs/heads/" + branch] !== refs["refs/mirror-stage/heads/" + branch]);
|
||||
console.log(JSON.stringify({
|
||||
event: "git-mirror-write",
|
||||
status: "received",
|
||||
repository: "pikasTech/HWLAB",
|
||||
writtenAt: process.env.written_at,
|
||||
pendingFlush,
|
||||
refs
|
||||
}));
|
||||
NODE
|
||||
HOOK
|
||||
chmod 0755 "$repo_path/hooks/pre-receive" "$repo_path/hooks/post-receive"
|
||||
git -C "$repo_path" config http.receivepack true
|
||||
git -C "$repo_path" config receive.denyDeletes true
|
||||
git -C "$repo_path" config receive.denyNonFastForwards true
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env node
|
||||
import net from "node:net";
|
||||
const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);
|
||||
const proxyPort = Number.parseInt(proxyPortRaw || "", 10);
|
||||
const targetPort = Number.parseInt(targetPortRaw || "", 10);
|
||||
if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {
|
||||
console.error("hwlab git-mirror proxy-connect: invalid ProxyCommand arguments");
|
||||
process.exit(64);
|
||||
}
|
||||
let settled = false;
|
||||
let tunnelEstablished = false;
|
||||
function finish(code, message) {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (message) console.error("hwlab git-mirror proxy-connect: " + message);
|
||||
process.exit(code);
|
||||
}
|
||||
const socket = net.createConnection({ host: proxyHost, port: proxyPort });
|
||||
let buffer = Buffer.alloc(0);
|
||||
socket.setTimeout(15000, () => { socket.destroy(); finish(65, "timeout connecting via " + proxyHost + ":" + proxyPort + " to " + targetHost + ":" + targetPort); });
|
||||
socket.on("connect", () => socket.write("CONNECT " + targetHost + ":" + targetPort + " HTTP/1.1\r\nHost: " + targetHost + ":" + targetPort + "\r\nProxy-Connection: Keep-Alive\r\n\r\n"));
|
||||
socket.on("error", (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? "tunnel socket error: " : "tcp error connecting to proxy: ") + (error && error.message ? error.message : String(error))));
|
||||
socket.on("close", () => { if (!tunnelEstablished) finish(68, "proxy closed before CONNECT completed via " + proxyHost + ":" + proxyPort + " to " + targetHost + ":" + targetPort); else finish(0); });
|
||||
function onData(chunk) {
|
||||
buffer = Buffer.concat([buffer, chunk]);
|
||||
const headerEnd = buffer.indexOf("\r\n\r\n");
|
||||
if (headerEnd === -1 && buffer.length < 8192) return;
|
||||
if (headerEnd === -1) { socket.destroy(); finish(68, "proxy response header exceeded 8192 bytes before CONNECT status via " + proxyHost + ":" + proxyPort + " to " + targetHost + ":" + targetPort); return; }
|
||||
const head = buffer.slice(0, headerEnd + 4).toString("latin1");
|
||||
const statusLine = head.split("\r\n", 1)[0] || "";
|
||||
const statusCode = Number.parseInt(statusLine.split(" ")[1] || "", 10);
|
||||
if (!statusLine.startsWith("HTTP/1.") || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {
|
||||
const safeStatus = statusLine.replace(/[^\x20-\x7e]/g, "?").slice(0, 160);
|
||||
socket.destroy();
|
||||
finish(67, "proxy CONNECT failed via " + proxyHost + ":" + proxyPort + " to " + targetHost + ":" + targetPort + ": " + safeStatus);
|
||||
return;
|
||||
}
|
||||
socket.off("data", onData);
|
||||
socket.setTimeout(0);
|
||||
tunnelEstablished = true;
|
||||
const rest = buffer.slice(headerEnd + 4);
|
||||
if (rest.length) process.stdout.write(rest);
|
||||
process.stdin.on("error", () => {});
|
||||
process.stdout.on("error", () => {});
|
||||
process.stdin.pipe(socket);
|
||||
socket.pipe(process.stdout);
|
||||
}
|
||||
socket.on("data", onData);
|
||||
@@ -0,0 +1,11 @@
|
||||
printf '%s\n' __HWLAB_PROXY_SUMMARY_SHELL__ >&2
|
||||
cat > /tmp/hwlab-git-ssh-proxy.sh <<'SH_PROXY'
|
||||
#!/bin/sh
|
||||
exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 "$@"
|
||||
SH_PROXY
|
||||
chmod 0700 /tmp/hwlab-git-ssh-proxy.sh
|
||||
unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy
|
||||
export NO_PROXY='*'
|
||||
export no_proxy='*'
|
||||
export GIT_SSH=/tmp/hwlab-git-ssh-proxy.sh
|
||||
unset GIT_SSH_COMMAND
|
||||
@@ -0,0 +1,20 @@
|
||||
printf '%s\n' __HWLAB_PROXY_SUMMARY_SHELL__ >&2
|
||||
cat > /tmp/hwlab-github-proxy-connect.mjs <<'NODE_PROXY'
|
||||
// __HWLAB_GIT_MIRROR_PROXY_CONNECT_MJS__
|
||||
NODE_PROXY
|
||||
chmod 0700 /tmp/hwlab-github-proxy-connect.mjs
|
||||
cat > /tmp/hwlab-git-ssh-proxy.sh <<'SH_PROXY'
|
||||
#!/bin/sh
|
||||
exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o __HWLAB_PROXY_COMMAND_SHELL__ "$@"
|
||||
SH_PROXY
|
||||
chmod 0700 /tmp/hwlab-git-ssh-proxy.sh
|
||||
export HTTP_PROXY=__HWLAB_PROXY_URL_SHELL__
|
||||
export HTTPS_PROXY=__HWLAB_PROXY_URL_SHELL__
|
||||
export ALL_PROXY=__HWLAB_PROXY_URL_SHELL__
|
||||
export http_proxy=__HWLAB_PROXY_URL_SHELL__
|
||||
export https_proxy=__HWLAB_PROXY_URL_SHELL__
|
||||
export all_proxy=__HWLAB_PROXY_URL_SHELL__
|
||||
export NO_PROXY=__HWLAB_PROXY_NO_PROXY_SHELL__
|
||||
export no_proxy=__HWLAB_PROXY_NO_PROXY_SHELL__
|
||||
export GIT_SSH=/tmp/hwlab-git-ssh-proxy.sh
|
||||
unset GIT_SSH_COMMAND
|
||||
@@ -0,0 +1,181 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
repo_url="ssh://git@ssh.github.com:443/pikasTech/HWLAB.git"
|
||||
repo_path="/cache/pikasTech/HWLAB.git"
|
||||
source_snapshot_ref_prefix=__HWLAB_SOURCE_SNAPSHOT_REF_PREFIX_SHELL__
|
||||
lock_dir="/cache/.hwlab-sync.lock"
|
||||
sync_started_ms=$(node -e 'console.log(Date.now())')
|
||||
now_ms() { node -e 'console.log(Date.now())'; }
|
||||
emit_timing() {
|
||||
phase="$1"
|
||||
status="$2"
|
||||
started_ms="$3"
|
||||
finished_ms=$(now_ms)
|
||||
duration_ms=$((finished_ms - started_ms))
|
||||
printf '{"event":"git-mirror-sync-timing","phase":"%s","status":"%s","durationMs":%s,"repository":"pikasTech/HWLAB"}
|
||||
' "$phase" "$status" "$duration_ms"
|
||||
}
|
||||
mkdir -p /cache/pikasTech /root/.ssh
|
||||
if ! mkdir "$lock_dir" 2>/dev/null; then
|
||||
echo "git mirror sync already running"
|
||||
exit 0
|
||||
fi
|
||||
trap 'rmdir "$lock_dir" 2>/dev/null || true' EXIT INT TERM
|
||||
cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa
|
||||
chmod 0400 /root/.ssh/id_rsa
|
||||
# __HWLAB_GIT_MIRROR_SSH_PRELUDE__
|
||||
if [ -d "$repo_path/objects" ] && [ -f "$repo_path/HEAD" ]; then
|
||||
git -C "$repo_path" remote set-url origin "$repo_url" || git -C "$repo_path" remote add origin "$repo_url"
|
||||
else
|
||||
rm -rf "$repo_path"
|
||||
git init --bare "$repo_path"
|
||||
git -C "$repo_path" remote add origin "$repo_url"
|
||||
fi
|
||||
/script/install-hooks.sh
|
||||
git -C "$repo_path" config uploadpack.hideRefs refs/mirror-stage
|
||||
git -C "$repo_path" config uploadpack.allowReachableSHA1InWant true
|
||||
git -C "$repo_path" config uploadpack.allowAnySHA1InWant true
|
||||
git -C "$repo_path" config --unset-all remote.origin.fetch 2>/dev/null || true
|
||||
# __HWLAB_FETCH_CONFIG_COMMANDS__
|
||||
fetch_started_ms=$(now_ms)
|
||||
timeout 180 git -C "$repo_path" fetch --quiet --prune origin
|
||||
emit_timing fetch succeeded "$fetch_started_ms"
|
||||
git -C "$repo_path" for-each-ref --format='%(objectname) %(refname)' refs/mirror-stage/heads > /tmp/hwlab-stage-heads
|
||||
git -C "$repo_path" for-each-ref --format='%(refname)' refs/mirror-stage/heads > /tmp/hwlab-stage-head-refs
|
||||
git -C "$repo_path" for-each-ref --format='%(objectname) %(refname)' refs/mirror-stage/tags > /tmp/hwlab-stage-tags
|
||||
git -C "$repo_path" for-each-ref --format='%(refname)' refs/mirror-stage/tags > /tmp/hwlab-stage-tag-refs
|
||||
if [ ! -s /tmp/hwlab-stage-heads ]; then
|
||||
echo "git mirror sync fetched no branch refs" >&2
|
||||
exit 41
|
||||
fi
|
||||
for required_ref in __HWLAB_REQUIRED_REF_ARGS__; do
|
||||
git -C "$repo_path" show-ref --verify --quiet "$required_ref" || { echo "git mirror sync missing required ref $required_ref" >&2; exit 43; }
|
||||
done
|
||||
validate_started_ms=$(now_ms)
|
||||
while read -r sha ref; do
|
||||
[ -n "$sha" ] || continue
|
||||
git -C "$repo_path" cat-file -e "$sha^{commit}"
|
||||
git -C "$repo_path" cat-file -e "$sha^{tree}"
|
||||
if git -C "$repo_path" rev-list --objects --missing=print "$sha" | grep -q '^?'; then
|
||||
echo "git mirror sync found missing objects for $ref $sha" >&2
|
||||
exit 42
|
||||
fi
|
||||
done < /tmp/hwlab-stage-heads
|
||||
emit_timing validate succeeded "$validate_started_ms"
|
||||
publish_started_ms=$(now_ms)
|
||||
while read -r sha ref; do
|
||||
[ -n "$sha" ] || continue
|
||||
name=$(printf '%s
|
||||
' "$ref" | sed 's#^refs/mirror-stage/heads/##')
|
||||
is_gitops_branch=false
|
||||
for gitops_branch in __HWLAB_GITOPS_BRANCH_ARGS__; do
|
||||
if [ "$name" = "$gitops_branch" ]; then is_gitops_branch=true; break; fi
|
||||
done
|
||||
if [ "$is_gitops_branch" = true ]; then
|
||||
local_sha=$(git -C "$repo_path" show-ref --verify --hash "refs/heads/$name" 2>/dev/null || true)
|
||||
if [ -n "$local_sha" ] && [ "$local_sha" != "$sha" ]; then
|
||||
if git -C "$repo_path" merge-base --is-ancestor "$sha" "$local_sha"; then
|
||||
echo "git mirror sync keeps local $name ahead of GitHub"
|
||||
continue
|
||||
fi
|
||||
if ! git -C "$repo_path" merge-base --is-ancestor "$local_sha" "$sha"; then
|
||||
echo "git mirror sync found divergent $name local=$local_sha github=$sha" >&2
|
||||
exit 44
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
git -C "$repo_path" update-ref "refs/heads/$name" "$sha"
|
||||
done < /tmp/hwlab-stage-heads
|
||||
for source_branch in __HWLAB_SOURCE_BRANCH_ARGS__; do
|
||||
source_sha=$(git -C "$repo_path" show-ref --verify --hash "refs/heads/$source_branch" 2>/dev/null || true)
|
||||
if [ -n "$source_sha" ]; then
|
||||
source_stage_ref="${source_snapshot_ref_prefix%/}/$source_branch/$source_sha"
|
||||
git -C "$repo_path" update-ref "$source_stage_ref" "$source_sha"
|
||||
fi
|
||||
done
|
||||
while read -r sha ref; do
|
||||
[ -n "$sha" ] || continue
|
||||
name=$(printf '%s
|
||||
' "$ref" | sed 's#^refs/mirror-stage/tags/##')
|
||||
git -C "$repo_path" update-ref "refs/tags/$name" "$sha"
|
||||
done < /tmp/hwlab-stage-tags
|
||||
git -C "$repo_path" for-each-ref --format='%(refname)' refs/heads > /tmp/hwlab-public-heads
|
||||
while read -r ref; do
|
||||
[ -n "$ref" ] || continue
|
||||
name=$(printf '%s
|
||||
' "$ref" | sed 's#^refs/heads/##')
|
||||
if ! grep -Fxq "refs/mirror-stage/heads/$name" /tmp/hwlab-stage-head-refs; then
|
||||
git -C "$repo_path" update-ref -d "$ref"
|
||||
fi
|
||||
done < /tmp/hwlab-public-heads
|
||||
git -C "$repo_path" for-each-ref --format='%(refname)' refs/tags > /tmp/hwlab-public-tags
|
||||
while read -r ref; do
|
||||
[ -n "$ref" ] || continue
|
||||
name=$(printf '%s
|
||||
' "$ref" | sed 's#^refs/tags/##')
|
||||
if ! grep -Fxq "refs/mirror-stage/tags/$name" /tmp/hwlab-stage-tag-refs; then
|
||||
git -C "$repo_path" update-ref -d "$ref"
|
||||
fi
|
||||
done < /tmp/hwlab-public-tags
|
||||
emit_timing publish succeeded "$publish_started_ms"
|
||||
for head_branch in __HWLAB_SOURCE_BRANCH_ARGS__; do
|
||||
if git -C "$repo_path" show-ref --verify --quiet "refs/heads/$head_branch"; then
|
||||
git -C "$repo_path" symbolic-ref HEAD "refs/heads/$head_branch"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fsck_started_ms=$(now_ms)
|
||||
git -C "$repo_path" fsck --connectivity-only --no-dangling >/tmp/hwlab-git-fsck.out
|
||||
emit_timing fsck succeeded "$fsck_started_ms"
|
||||
git -C "$repo_path" update-server-info
|
||||
published_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
for source_branch in __HWLAB_SOURCE_BRANCH_ARGS__; do
|
||||
git -C "$repo_path" rev-parse "refs/heads/$source_branch" 2>/dev/null | tee "/cache/HWLAB-$source_branch.head" >/dev/null || true
|
||||
done
|
||||
printf '%s
|
||||
' "$published_at" > /cache/HWLAB.last-sync
|
||||
export published_at
|
||||
export source_snapshot_ref_prefix
|
||||
export mirror_branches_json=__HWLAB_MIRROR_BRANCHES_JSON_SHELL__
|
||||
export source_branches_json=__HWLAB_SOURCE_BRANCHES_JSON_SHELL__
|
||||
export gitops_branches_json=__HWLAB_GITOPS_BRANCHES_JSON_SHELL__
|
||||
node - <<'NODE' | tee /cache/HWLAB.last-sync.json
|
||||
const { execFileSync } = require("node:child_process");
|
||||
const repo = "/cache/pikasTech/HWLAB.git";
|
||||
const mirrorBranches = JSON.parse(process.env.mirror_branches_json || "[]");
|
||||
const sourceBranches = JSON.parse(process.env.source_branches_json || "[]");
|
||||
const gitopsBranches = JSON.parse(process.env.gitops_branches_json || "[]");
|
||||
const sourceSnapshotRefPrefix = (process.env.source_snapshot_ref_prefix || "refs/unidesk/snapshots/hwlab-node-runtime").replace(//+$/u, "");
|
||||
function rev(ref) {
|
||||
try { return execFileSync("git", ["-C", repo, "rev-parse", ref], { encoding: "utf8" }).trim() || null; } catch { return null; }
|
||||
}
|
||||
const refs = {};
|
||||
for (const branch of mirrorBranches) {
|
||||
refs["refs/heads/" + branch] = rev("refs/heads/" + branch);
|
||||
refs["refs/mirror-stage/heads/" + branch] = rev("refs/mirror-stage/heads/" + branch);
|
||||
}
|
||||
const sourceSnapshots = {};
|
||||
for (const branch of sourceBranches) {
|
||||
const source = refs["refs/heads/" + branch] || null;
|
||||
const stageRef = source ? sourceSnapshotRefPrefix + "/" + branch + "/" + source : null;
|
||||
const sourceSnapshot = stageRef ? rev(stageRef) : null;
|
||||
if (stageRef) refs[stageRef] = sourceSnapshot;
|
||||
sourceSnapshots[branch] = { stageRef, sourceSnapshot };
|
||||
}
|
||||
const sourceInSync = sourceBranches.every((branch) => refs["refs/heads/" + branch] && refs["refs/heads/" + branch] === refs["refs/mirror-stage/heads/" + branch] && sourceSnapshots[branch]?.sourceSnapshot === refs["refs/mirror-stage/heads/" + branch]);
|
||||
const gitopsInSync = gitopsBranches.every((branch) => refs["refs/heads/" + branch] && refs["refs/heads/" + branch] === refs["refs/mirror-stage/heads/" + branch]);
|
||||
const pendingFlush = gitopsBranches.some((branch) => refs["refs/heads/" + branch] && refs["refs/mirror-stage/heads/" + branch] && refs["refs/heads/" + branch] !== refs["refs/mirror-stage/heads/" + branch]);
|
||||
const payload = {
|
||||
event: "git-mirror-sync",
|
||||
status: "published",
|
||||
repository: "pikasTech/HWLAB",
|
||||
publishedAt: process.env.published_at,
|
||||
pendingFlush,
|
||||
sourceInSync,
|
||||
gitopsInSync,
|
||||
sourceSnapshots,
|
||||
refs
|
||||
};
|
||||
console.log(JSON.stringify(payload));
|
||||
NODE
|
||||
emit_timing total succeeded "$sync_started_ms"
|
||||
@@ -0,0 +1,46 @@
|
||||
git_ssh_setup() {
|
||||
mkdir -p /root/.ssh
|
||||
cp /workspace/git-ssh/ssh-privatekey /root/.ssh/id_rsa
|
||||
chmod 600 /root/.ssh/id_rsa
|
||||
timeout 10 ssh-keyscan github.com >> /root/.ssh/known_hosts 2>/dev/null || true
|
||||
timeout 10 ssh-keyscan -p 443 ssh.github.com >> /root/.ssh/known_hosts 2>/dev/null || true
|
||||
write_github_proxy_command
|
||||
export GIT_SSH_COMMAND="ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ServerAliveInterval=10 -o ServerAliveCountMax=2 -o ProxyCommand='node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808 %h %p'"
|
||||
git config --global url."ssh://git@ssh.github.com:443/".insteadOf "git@github.com:"
|
||||
git config --global url."ssh://git@ssh.github.com:443/".insteadOf "ssh://git@github.com/"
|
||||
}
|
||||
|
||||
write_github_proxy_command() {
|
||||
cat > /tmp/hwlab-github-proxy-connect.mjs <<'NODE_PROXY'
|
||||
// __HWLAB_GITHUB_PROXY_CONNECT_MJS__
|
||||
NODE_PROXY
|
||||
chmod 0700 /tmp/hwlab-github-proxy-connect.mjs
|
||||
}
|
||||
|
||||
git_now_ms() {
|
||||
node -e 'console.log(Date.now())' 2>/dev/null || date +%s000
|
||||
}
|
||||
|
||||
git_timed() {
|
||||
phase="$1"
|
||||
timeout_seconds="$2"
|
||||
shift 2
|
||||
started_ms="$(git_now_ms)"
|
||||
echo '{"event":"git-operation","phase":"'"$phase"'","status":"started","timeoutSeconds":'"$timeout_seconds"'}' >&2
|
||||
set +e
|
||||
timeout "$timeout_seconds" "$@"
|
||||
status="$?"
|
||||
set -e
|
||||
finished_ms="$(git_now_ms)"
|
||||
duration_ms="$((finished_ms - started_ms))"
|
||||
if [ "$status" -eq 0 ]; then
|
||||
echo '{"event":"git-operation","phase":"'"$phase"'","status":"succeeded","durationMs":'"$duration_ms"'}' >&2
|
||||
return 0
|
||||
fi
|
||||
if [ "$status" -eq 124 ] || [ "$status" -eq 137 ]; then
|
||||
echo '{"event":"git-operation","phase":"'"$phase"'","status":"failed","reason":"timeout","durationMs":'"$duration_ms"',"timeoutSeconds":'"$timeout_seconds"'}' >&2
|
||||
else
|
||||
echo '{"event":"git-operation","phase":"'"$phase"'","status":"failed","exitCode":'"$status"',"durationMs":'"$duration_ms"'}' >&2
|
||||
fi
|
||||
return "$status"
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env node
|
||||
import net from "node:net";
|
||||
|
||||
const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);
|
||||
const proxyPort = Number.parseInt(proxyPortRaw || "", 10);
|
||||
const targetPort = Number.parseInt(targetPortRaw || "", 10);
|
||||
if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {
|
||||
console.error("usage: hwlab-github-proxy-connect <proxyHost> <proxyPort> <targetHost> <targetPort>");
|
||||
process.exit(64);
|
||||
}
|
||||
|
||||
const socket = net.createConnection({ host: proxyHost, port: proxyPort });
|
||||
let buffer = Buffer.alloc(0);
|
||||
|
||||
socket.setTimeout(10000, () => {
|
||||
console.error("proxy connect timeout " + proxyHost + ":" + proxyPort + " -> " + targetHost + ":" + targetPort);
|
||||
socket.destroy();
|
||||
process.exit(65);
|
||||
});
|
||||
|
||||
socket.on("connect", () => {
|
||||
socket.write("CONNECT " + targetHost + ":" + targetPort + " HTTP/1.1\r\nHost: " + targetHost + ":" + targetPort + "\r\nProxy-Connection: Keep-Alive\r\n\r\n");
|
||||
});
|
||||
|
||||
socket.on("error", (error) => {
|
||||
console.error("proxy connect failed: " + error.message);
|
||||
process.exit(66);
|
||||
});
|
||||
|
||||
function onData(chunk) {
|
||||
buffer = Buffer.concat([buffer, chunk]);
|
||||
const headerEnd = buffer.indexOf("\r\n\r\n");
|
||||
if (headerEnd === -1 && buffer.length < 8192) return;
|
||||
const head = buffer.slice(0, headerEnd + 4).toString("latin1");
|
||||
const statusLine = head.split("\r\n", 1)[0] || "";
|
||||
const statusCode = Number.parseInt(statusLine.split(" ")[1] || "", 10);
|
||||
if (!statusLine.startsWith("HTTP/1.") || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {
|
||||
console.error("proxy CONNECT rejected: " + (statusLine || "missing-status"));
|
||||
socket.destroy();
|
||||
process.exit(67);
|
||||
}
|
||||
socket.off("data", onData);
|
||||
socket.setTimeout(0);
|
||||
const rest = buffer.slice(headerEnd + 4);
|
||||
if (rest.length) process.stdout.write(rest);
|
||||
process.stdin.pipe(socket);
|
||||
socket.pipe(process.stdout);
|
||||
}
|
||||
|
||||
socket.on("data", onData);
|
||||
socket.on("close", () => process.exit(0));
|
||||
@@ -0,0 +1,331 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# __HWLAB_CI_TIMING_SHELL__
|
||||
|
||||
export HWLAB_TEKTON_PIPELINERUN="$(context.pipelineRun.name)"
|
||||
export HWLAB_TEKTON_TASKRUN="$(context.taskRun.name)"
|
||||
export HWLAB_TEKTON_TASK="gitops-promote"
|
||||
export HWLAB_SOURCE_REVISION="$(params.revision)"
|
||||
echo '{"event":"ci-base-image","phase":"gitops-promote","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apt"}'
|
||||
for tool in node git timeout; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"gitops-promote","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done
|
||||
# __HWLAB_GIT_SSH_SHELL__
|
||||
|
||||
git_url_requires_ssh() { case "$1" in git@*|ssh://*) return 0 ;; *) return 1 ;; esac; }
|
||||
lane="$(params.lane)"
|
||||
case "$lane" in
|
||||
v[0-9][0-9]*) runtime_lane=true ;;
|
||||
*) runtime_lane=false ;;
|
||||
esac
|
||||
if [ "$runtime_lane" = "true" ]; then
|
||||
printf 'false' > /tekton/results/runtime-ready-required
|
||||
else
|
||||
printf 'true' > /tekton/results/runtime-ready-required
|
||||
fi
|
||||
source_head_url_for_setup="$(params.git-url)"
|
||||
gitops_read_url_for_setup="$(params.git-url)"
|
||||
if [ "$runtime_lane" = "true" ]; then
|
||||
source_head_url_for_setup="$(params.git-read-url)"
|
||||
gitops_read_url_for_setup="$(params.git-read-url)"
|
||||
fi
|
||||
gitops_write_url_for_setup="$(params.git-write-url)"
|
||||
if git_url_requires_ssh "$source_head_url_for_setup" || git_url_requires_ssh "$gitops_read_url_for_setup" || git_url_requires_ssh "$gitops_write_url_for_setup"; then
|
||||
for tool in ssh ssh-keyscan; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"gitops-promote","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done
|
||||
git_ssh_setup
|
||||
else
|
||||
echo '{"event":"git-ssh-setup","phase":"gitops-promote","status":"skipped","reason":"non-ssh-git-url"}'
|
||||
fi
|
||||
cd /workspace/source/repo
|
||||
test "$(git rev-parse HEAD)" = "$(params.revision)"
|
||||
|
||||
check_source_head() {
|
||||
phase="$1"
|
||||
expected="${2:-$(params.revision)}"
|
||||
source_head_url="$(params.git-url)"
|
||||
if [ "$runtime_lane" = "true" ]; then
|
||||
source_head_url="$(params.git-read-url)"
|
||||
fi
|
||||
latest_file="$(mktemp)"
|
||||
if ! git_timed "source-head-$phase" 45 git ls-remote "$source_head_url" "refs/heads/$(params.source-branch)" > "$latest_file"; then
|
||||
echo '{"status":"failed","reason":"source-head-check-failed","phase":"'"$phase"'","sourceBranch":"'"$(params.source-branch)"'","sourceRevision":"'"$(params.revision)"'"}'
|
||||
exit 1
|
||||
fi
|
||||
latest="$(cut -f1 "$latest_file" | head -n 1)"
|
||||
if [ -z "$latest" ]; then
|
||||
echo '{"status":"failed","reason":"source-head-unresolved","phase":"'"$phase"'","sourceBranch":"'"$(params.source-branch)"'","sourceRevision":"'"$(params.revision)"'"}'
|
||||
exit 1
|
||||
fi
|
||||
if [ "$latest" != "$expected" ]; then
|
||||
printf 'false' > /tekton/results/runtime-ready-required || true
|
||||
echo '{"status":"skipped-stale-source","verdict":"superseded","phase":"'"$phase"'","sourceBranch":"'"$(params.source-branch)"'","sourceRevision":"'"$(params.revision)"'","expectedRevision":"'"$expected"'","latestRevision":"'"$latest"'"}'
|
||||
exit 0
|
||||
fi
|
||||
}
|
||||
|
||||
check_source_head before-render
|
||||
if [ -s /workspace/source/affected-services.json ] && node - /workspace/source/affected-services.json <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const plan = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
||||
const buildServices = Array.isArray(plan.buildServices) ? plan.buildServices : [];
|
||||
const affectedServices = Array.isArray(plan.affectedServices) ? plan.affectedServices : [];
|
||||
const willRunGitopsPromote = plan.ciCdPlan && plan.ciCdPlan.willRunGitopsPromote === true;
|
||||
process.exit(!willRunGitopsPromote && buildServices.length === 0 && affectedServices.length === 0 ? 0 : 1);
|
||||
NODE
|
||||
then
|
||||
printf 'false' > /tekton/results/runtime-ready-required || true
|
||||
echo '{"event":"gitops-promote","status":"skipped","reason":"no-build-no-rollout-plan"}'
|
||||
exit 0
|
||||
fi
|
||||
git config --global user.name "HWLAB node GitOps Bot"
|
||||
git config --global user.email "hwlab-node-gitops-bot@users.noreply.github.com"
|
||||
catalog_path="$(params.catalog-path)"
|
||||
runtime_path="$(params.runtime-path)"
|
||||
gitops_root_from_runtime_path() {
|
||||
path="$1"
|
||||
case "$path" in
|
||||
*/*) printf '%s
|
||||
' "${path%/*}" ;;
|
||||
*) printf '.
|
||||
' ;;
|
||||
esac
|
||||
}
|
||||
gitops_root="$(gitops_root_from_runtime_path "$runtime_path")"
|
||||
|
||||
argo_hard_refresh_runtime_lane() {
|
||||
if [ "$runtime_lane" != "true" ]; then return 0; fi
|
||||
ARGO_APPLICATION="hwlab-node-$lane" ARGO_FIELD_MANAGER="hwlab-$lane-gitops-promote" node <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const https = require("node:https");
|
||||
|
||||
const host = process.env.KUBERNETES_SERVICE_HOST;
|
||||
const port = process.env.KUBERNETES_SERVICE_PORT || "443";
|
||||
const startedAt = Date.now();
|
||||
const application = process.env.ARGO_APPLICATION || "hwlab-node-v02";
|
||||
const fieldManager = process.env.ARGO_FIELD_MANAGER || "hwlab-v02-gitops-promote";
|
||||
const pipelineRun = process.env.HWLAB_TEKTON_PIPELINERUN || null;
|
||||
const taskRun = process.env.HWLAB_TEKTON_TASKRUN || null;
|
||||
const task = process.env.HWLAB_TEKTON_TASK || null;
|
||||
const revision = process.env.HWLAB_SOURCE_REVISION || null;
|
||||
|
||||
function emit(payload) {
|
||||
console.log(JSON.stringify({
|
||||
event: "node-cicd-timing",
|
||||
schemaVersion: "v1",
|
||||
stage: "argo-hard-refresh",
|
||||
pipelineRun,
|
||||
taskRun,
|
||||
task,
|
||||
revision,
|
||||
application,
|
||||
source: "scripts/gitops-render.mjs",
|
||||
at: new Date().toISOString(),
|
||||
...payload
|
||||
}));
|
||||
}
|
||||
|
||||
function safeJson(text) {
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function request(method, path, body, contentType = "application/merge-patch+json") {
|
||||
const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8");
|
||||
const ca = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt");
|
||||
const payload = body ? JSON.stringify(body) : "";
|
||||
const headers = { Authorization: "Bearer " + token };
|
||||
if (payload) {
|
||||
headers["Content-Type"] = contentType;
|
||||
headers["Content-Length"] = Buffer.byteLength(payload);
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = https.request({ host, port, method, path, ca, headers }, (res) => {
|
||||
let data = "";
|
||||
res.setEncoding("utf8");
|
||||
res.on("data", (chunk) => { data += chunk; });
|
||||
res.on("end", () => resolve({ statusCode: res.statusCode || 0, body: data, json: safeJson(data) }));
|
||||
});
|
||||
req.on("error", reject);
|
||||
if (payload) req.write(payload);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
(async () => {
|
||||
if (!host) {
|
||||
emit({ status: "skipped", reason: "kubernetes-service-host-missing", durationMs: Date.now() - startedAt });
|
||||
return;
|
||||
}
|
||||
const response = await request(
|
||||
"PATCH",
|
||||
"/apis/argoproj.io/v1alpha1/namespaces/argocd/applications/" + encodeURIComponent(application) + "?fieldManager=" + encodeURIComponent(fieldManager),
|
||||
{ metadata: { annotations: { "argocd.argoproj.io/refresh": "hard" } } }
|
||||
);
|
||||
if (response.statusCode >= 200 && response.statusCode < 300) {
|
||||
emit({ status: "succeeded", durationMs: Date.now() - startedAt, statusCode: response.statusCode });
|
||||
return;
|
||||
}
|
||||
emit({ status: "degraded", reason: "argo-refresh-patch-failed", durationMs: Date.now() - startedAt, statusCode: response.statusCode, body: response.body.slice(0, 1000) });
|
||||
})().catch((error) => {
|
||||
emit({ status: "degraded", reason: "argo-refresh-patch-error", durationMs: Date.now() - startedAt, error: error.message });
|
||||
});
|
||||
NODE
|
||||
}
|
||||
|
||||
if [ -s /workspace/source/dev-artifacts.json ]; then
|
||||
node scripts/refresh-artifact-catalog.mjs --lane "$lane" --catalog-path "$catalog_path" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --target-ref "$(params.revision)" --publish-report /workspace/source/dev-artifacts.json --write
|
||||
fi
|
||||
gitops_render_started_ms="$(ci_now_ms)"
|
||||
node scripts/run-bun.mjs scripts/gitops-render.mjs --lane "$lane" --catalog-path "$catalog_path" --image-tag-mode "$(params.image-tag-mode)" --source-revision "$(params.revision)" --source-repo "$(params.git-url)" --source-branch "$(params.source-branch)" --gitops-branch "$(params.gitops-branch)" --gitops-root "$gitops_root" --out "$gitops_root" --registry-prefix "$(params.registry-prefix)" --use-deploy-images
|
||||
node scripts/run-bun.mjs scripts/gitops-render.mjs --lane "$lane" --catalog-path "$catalog_path" --image-tag-mode "$(params.image-tag-mode)" --source-revision "$(params.revision)" --source-repo "$(params.git-url)" --source-branch "$(params.source-branch)" --gitops-branch "$(params.gitops-branch)" --gitops-root "$gitops_root" --out "$gitops_root" --registry-prefix "$(params.registry-prefix)" --use-deploy-images --check
|
||||
ci_timing_emit gitops-render succeeded "$gitops_render_started_ms"
|
||||
check_source_head before-push
|
||||
workdir="$(mktemp -d)"
|
||||
gitops_read_url="$(params.git-url)"
|
||||
gitops_write_url="$(params.git-write-url)"
|
||||
if [ "$runtime_lane" = "true" ]; then
|
||||
gitops_read_url="$(params.git-read-url)"
|
||||
fi
|
||||
gitops_clone_started_ms="$(ci_now_ms)"
|
||||
git_timed gitops-clone 180 git clone --no-checkout "$gitops_read_url" "$workdir/gitops"
|
||||
cd "$workdir/gitops"
|
||||
git remote set-url origin "$gitops_write_url"
|
||||
old_runtime_snapshot="$workdir/old-runtime-snapshot"
|
||||
if git_timed gitops-branch-ls 45 git ls-remote --exit-code --heads origin "$(params.gitops-branch)" >/dev/null; then
|
||||
git_timed gitops-fetch 90 git fetch origin "$(params.gitops-branch)"
|
||||
git checkout -B "$(params.gitops-branch)" "origin/$(params.gitops-branch)"
|
||||
if [ "$runtime_lane" = "true" ] && [ -d "$runtime_path" ]; then
|
||||
mkdir -p "$old_runtime_snapshot"
|
||||
cp -a "$runtime_path"/. "$old_runtime_snapshot"/
|
||||
fi
|
||||
if [ "$runtime_lane" = "true" ]; then rm -rf "$runtime_path" "$catalog_path"; else rm -rf deploy/gitops/node "$catalog_path"; fi
|
||||
else
|
||||
git checkout --orphan "$(params.gitops-branch)"
|
||||
git rm -rf . >/dev/null 2>&1 || true
|
||||
fi
|
||||
ci_timing_emit gitops-clone succeeded "$gitops_clone_started_ms"
|
||||
mkdir -p "$(dirname "$runtime_path")" "$(dirname "$catalog_path")"
|
||||
if [ "$runtime_lane" = "true" ]; then
|
||||
cp -a "/workspace/source/repo/$runtime_path" "$runtime_path"
|
||||
cp "/workspace/source/repo/$catalog_path" "$catalog_path"
|
||||
git add "$catalog_path" "$runtime_path"
|
||||
else
|
||||
mkdir -p deploy/gitops
|
||||
cp -a /workspace/source/repo/deploy/gitops/node deploy/gitops/node
|
||||
cp "/workspace/source/repo/$catalog_path" "$catalog_path"
|
||||
git add "$catalog_path" deploy/gitops/node
|
||||
fi
|
||||
if [ "$runtime_lane" = "true" ] && [ -s /workspace/source/affected-services.json ]; then
|
||||
runtime_noop_decision="$(node - "$runtime_path" "$old_runtime_snapshot" /workspace/source/affected-services.json <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
|
||||
const [runtimePath, oldRuntimePath, planPath] = process.argv.slice(2);
|
||||
|
||||
function readJson(filePath) {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf8"));
|
||||
}
|
||||
|
||||
function stable(value) {
|
||||
if (Array.isArray(value)) return "[" + value.map(stable).join(",") + "]";
|
||||
if (value && typeof value === "object") {
|
||||
return "{" + Object.keys(value).sort().map((key) => JSON.stringify(key) + ":" + stable(value[key])).join(",") + "}";
|
||||
}
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
function scrub(value) {
|
||||
if (Array.isArray(value)) return value.map(scrub);
|
||||
if (!value || typeof value !== "object") return value;
|
||||
const result = {};
|
||||
for (const [key, child] of Object.entries(value)) {
|
||||
if (key === "hwlab.pikastech.local/source-commit" ||
|
||||
key === "hwlab.pikastech.local/artifact-source-commit" ||
|
||||
key === "hwlab.pikastech.local/boot-commit" ||
|
||||
key === "sourceCommitId" ||
|
||||
key === "bootCommit") {
|
||||
result[key] = "<runtime-source-identity>";
|
||||
continue;
|
||||
}
|
||||
const envName = String(value.name || "");
|
||||
if (key === "value" && /^HWLAB_.*COMMIT/.test(envName)) {
|
||||
result[key] = "<runtime-source-identity>";
|
||||
continue;
|
||||
}
|
||||
result[key] = scrub(child);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function listFiles(rootDir) {
|
||||
if (!rootDir || !fs.existsSync(rootDir)) return null;
|
||||
const files = [];
|
||||
function walk(current) {
|
||||
for (const entry of fs.readdirSync(current, { withFileTypes: true })) {
|
||||
const fullPath = path.join(current, entry.name);
|
||||
if (entry.isDirectory()) walk(fullPath);
|
||||
else if (entry.isFile()) files.push(path.relative(rootDir, fullPath).replaceAll(path.sep, "/"));
|
||||
}
|
||||
}
|
||||
walk(rootDir);
|
||||
return files.sort();
|
||||
}
|
||||
|
||||
function normalizedTree(rootDir) {
|
||||
const files = listFiles(rootDir);
|
||||
if (!files) return null;
|
||||
const entries = {};
|
||||
for (const relativePath of files) {
|
||||
const filePath = path.join(rootDir, relativePath);
|
||||
const text = fs.readFileSync(filePath, "utf8");
|
||||
try {
|
||||
entries[relativePath] = stable(scrub(JSON.parse(text)));
|
||||
} catch {
|
||||
entries[relativePath] = text.replace(/[a-f0-9]{40}/gu, "<runtime-source-identity>");
|
||||
}
|
||||
}
|
||||
return stable(entries);
|
||||
}
|
||||
|
||||
const plan = readJson(planPath);
|
||||
const buildServices = Array.isArray(plan.buildServices) ? plan.buildServices : [];
|
||||
const rolloutServices = Array.isArray(plan.rolloutServices) ? plan.rolloutServices : [];
|
||||
if (buildServices.length > 0 || rolloutServices.length > 0) {
|
||||
process.stdout.write("runtime-required");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const oldTree = normalizedTree(oldRuntimePath);
|
||||
const newTree = normalizedTree(runtimePath);
|
||||
process.stdout.write(oldTree && newTree && oldTree === newTree ? "runtime-identity-only" : "runtime-required");
|
||||
NODE
|
||||
)"
|
||||
if [ "$runtime_noop_decision" = "runtime-identity-only" ]; then
|
||||
printf 'false' > /tekton/results/runtime-ready-required
|
||||
echo '{"status":"skipped-runtime-unchanged","reason":"runtime-identity-only","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}'
|
||||
ci_timing_emit gitops-commit skipped "$(ci_now_ms)"
|
||||
ci_timing_emit gitops-push skipped "$(ci_now_ms)"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
if git diff --cached --quiet; then
|
||||
printf 'false' > /tekton/results/runtime-ready-required
|
||||
echo '{"status":"unchanged","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}'
|
||||
ci_timing_emit gitops-commit unchanged "$(ci_now_ms)"
|
||||
ci_timing_emit gitops-push unchanged "$(ci_now_ms)"
|
||||
exit 0
|
||||
fi
|
||||
short="$(printf '%.7s' "$(params.revision)")"
|
||||
gitops_commit_started_ms="$(ci_now_ms)"
|
||||
git commit -m "chore: promote node GitOps source $short"
|
||||
ci_timing_emit gitops-commit succeeded "$gitops_commit_started_ms"
|
||||
gitops_push_started_ms="$(ci_now_ms)"
|
||||
git_timed gitops-push 120 git push origin "HEAD:$(params.gitops-branch)"
|
||||
ci_timing_emit gitops-push succeeded "$gitops_push_started_ms"
|
||||
if [ "$runtime_lane" = "true" ]; then
|
||||
printf 'false' > /tekton/results/runtime-ready-required
|
||||
echo '{"event":"runtime-ready","phase":"gitops-promote","status":"delegated-post-flush-closeout","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'"}'
|
||||
fi
|
||||
argo_hard_refresh_runtime_lane
|
||||
echo '{"status":"pushed","gitopsBranch":"'"$(params.gitops-branch)"'","sourceRevision":"'"$(params.revision)"'","gitopsWriteUrl":"'"$gitops_write_url"'"}'
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# __HWLAB_CI_TIMING_SHELL__
|
||||
|
||||
export HWLAB_TEKTON_PIPELINERUN="$(context.pipelineRun.name)"
|
||||
export HWLAB_TEKTON_TASKRUN="$(context.taskRun.name)"
|
||||
export HWLAB_TEKTON_TASK="build-$(params.service-id)"
|
||||
export HWLAB_SOURCE_REVISION="$(params.revision)"
|
||||
export HWLAB_TIMING_SERVICE_ID="$(params.service-id)"
|
||||
# __HWLAB_DEPENDENCY_PROXY_PROBE__
|
||||
echo '{"event":"ci-base-image","phase":"service-image-publish","serviceId":"'"$(params.service-id)"'","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apk"}'
|
||||
for tool in node npm git python3 ssh curl; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"service-image-publish","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done
|
||||
mkdir -p /workspace/service-work/home
|
||||
export HOME=/workspace/service-work/home
|
||||
git config --global --add safe.directory /workspace/source/repo
|
||||
cd /workspace/source/repo
|
||||
test "$(git rev-parse HEAD)" = "$(params.revision)"
|
||||
export HWLAB_ARTIFACT_LANE="$(params.lane)"
|
||||
export HWLAB_ARTIFACT_CATALOG_PATH="$(params.catalog-path)"
|
||||
export HWLAB_DEPLOY_MANIFEST_PATH="deploy/deploy.yaml"
|
||||
export HWLAB_ARTIFACT_IMAGE_TAG_MODE="$(params.image-tag-mode)"
|
||||
mkdir -p /workspace/source/service-results
|
||||
if [ -s /workspace/source/affected-services.json ] && ! node -e 'const fs=require("node:fs"); const p=JSON.parse(fs.readFileSync("/workspace/source/affected-services.json","utf8")); const service=process.argv[1]; const item=(p.entries||[]).find((entry)=>entry.serviceId===service); process.exit(item && item.buildRequired ? 0 : 1);' "$(params.service-id)"; then
|
||||
node - "$(params.service-id)" <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const serviceId = process.argv[2];
|
||||
const catalog = JSON.parse(fs.readFileSync(process.env.HWLAB_ARTIFACT_CATALOG_PATH, "utf8"));
|
||||
const plan = fs.existsSync("/workspace/source/affected-services.json") ? JSON.parse(fs.readFileSync("/workspace/source/affected-services.json", "utf8")) : {};
|
||||
const service = (catalog.services || []).find((item) => item.serviceId === serviceId) || {};
|
||||
const planned = (plan.services || []).find((item) => item.serviceId === serviceId) || {};
|
||||
const envReuse = planned.runtimeMode === "env-reuse-git-mirror-checkout" || service.runtimeMode === "env-reuse-git-mirror-checkout" || planned.envReuse === true || service.envReuse === true;
|
||||
const image = envReuse ? (service.environmentImage || service.image || "") : (service.image || "");
|
||||
const digest = envReuse ? (service.environmentDigest || service.digest || "not_published") : (service.digest || "not_published");
|
||||
const imageTag = service.imageTag || (image.includes(":") ? image.slice(image.lastIndexOf(":") + 1) : "");
|
||||
const repository = image.includes(":") ? image.slice(0, image.lastIndexOf(":")) : image;
|
||||
const revision = process.env.HWLAB_SOURCE_REVISION || planned.bootCommit || service.bootCommit || service.sourceCommitId || service.commitId || imageTag;
|
||||
const componentInputHash = envReuse ? (planned.componentInputHash || service.componentInputHash || "") : (service.componentInputHash || "");
|
||||
const environmentInputHash = envReuse ? (service.environmentInputHash || planned.environmentInputHash || "") : (service.environmentInputHash || "");
|
||||
const codeInputHash = envReuse ? (planned.codeInputHash || service.codeInputHash || "") : (service.codeInputHash || "");
|
||||
const values = {
|
||||
"service-id": serviceId,
|
||||
status: /^sha256:[a-f0-9]{64}$/.test(digest) ? "reused" : "blocked_reuse_unavailable",
|
||||
image,
|
||||
"image-tag": imageTag,
|
||||
digest,
|
||||
"repository-digest": /^sha256:[a-f0-9]{64}$/.test(digest) && repository ? repository + "@" + digest : "",
|
||||
"source-commit-id": envReuse ? revision : (service.sourceCommitId || service.commitId || imageTag),
|
||||
"component-input-hash": componentInputHash,
|
||||
"environment-input-hash": environmentInputHash,
|
||||
"code-input-hash": codeInputHash,
|
||||
"runtime-mode": envReuse ? "env-reuse-git-mirror-checkout" : "service-image",
|
||||
"boot-repo": planned.bootRepo || service.bootRepo || "",
|
||||
"boot-commit": envReuse ? revision : (service.bootCommit || ""),
|
||||
"boot-sh": planned.bootSh || service.bootSh || "",
|
||||
"build-created-at": service.buildCreatedAt || "",
|
||||
"build-backend": envReuse ? "reused-env-catalog" : "reused-catalog",
|
||||
"reused-from": (envReuse ? service.environmentInputHash : service.componentInputHash) || service.commitId || imageTag || "catalog"
|
||||
};
|
||||
for (const [name, value] of Object.entries(values)) fs.writeFileSync("/tekton/results/" + name, String(value || ""));
|
||||
NODE
|
||||
echo '{"event":"service-build-skip","serviceId":"'"$(params.service-id)"'","reason":"component-inputs-unchanged"}'
|
||||
exit 0
|
||||
fi
|
||||
rm -rf /workspace/service-work/repo
|
||||
mkdir -p /workspace/service-work/repo
|
||||
tar -C /workspace/source/repo -cf - . | tar -C /workspace/service-work/repo -xo -f -
|
||||
cd /workspace/service-work/repo
|
||||
export HWLAB_CI_ARTIFACT_RUN_ID="$(context.pipelineRun.name)-$(params.service-id)"
|
||||
export HWLAB_CI_ARTIFACT_RUN_OWNER="tekton"
|
||||
export HWLAB_DEV_REGISTRY_PREFIX="$(params.registry-prefix)"
|
||||
export HWLAB_DEV_REGISTRY_K8S_NATIVE=1
|
||||
export HWLAB_NODE_CICD_TIMING=1
|
||||
export HWLAB_TEKTON_PIPELINERUN
|
||||
export HWLAB_TEKTON_TASKRUN
|
||||
export HWLAB_TEKTON_TASK
|
||||
export HWLAB_SOURCE_REVISION
|
||||
export PATH="/workspace/buildkit-bin:$PATH"
|
||||
export HWLAB_BUILDKIT_ADDR="unix:///workspace/buildkit-run/buildkitd.sock"
|
||||
if [ -n "$(params.base-image)" ]; then export HWLAB_DEV_BASE_IMAGE="$(params.base-image)"; fi
|
||||
if [ -n "${HWLAB_DEV_BASE_IMAGE:-}" ]; then
|
||||
echo '{"event":"dependency-local-registry-probe-start","phase":"service-image-publish-pre-base-image","serviceId":"'"$(params.service-id)"'","target":"http://127.0.0.1:5000/v2/"}'
|
||||
registry_started_at="$(date +%s)"
|
||||
curl -fsS --max-time 10 http://127.0.0.1:5000/v2/ >/dev/null
|
||||
registry_finished_at="$(date +%s)"
|
||||
echo '{"event":"dependency-local-registry-probe","phase":"service-image-publish-pre-base-image","serviceId":"'"$(params.service-id)"'","ok":true,"durationSeconds":'"$((registry_finished_at - registry_started_at))"'}'
|
||||
fi
|
||||
buildkit_ready=0
|
||||
for attempt in $(seq 1 60); do
|
||||
if /workspace/buildkit-bin/buildctl --addr "$HWLAB_BUILDKIT_ADDR" debug workers >/dev/null 2>&1; then
|
||||
buildkit_ready=1
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "$buildkit_ready" != "1" ]; then
|
||||
echo '{"event":"buildkit-sidecar-not-ready","serviceId":"'"$(params.service-id)"'","addr":"'"$HWLAB_BUILDKIT_ADDR"'"}' >&2
|
||||
exit 1
|
||||
fi
|
||||
HWLAB_CI_PLAN_VERIFY_REUSE_REGISTRY=1 node scripts/artifact-publish.mjs --publish --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --registry-prefix "$(params.registry-prefix)" --report "/workspace/source/service-results/$(params.service-id).json" --tekton-results-dir /tekton/results --quiet-build --concurrency 1 --services "$(params.service-id)" --buildkit-command /workspace/buildkit-bin/buildctl --buildkit-addr "$HWLAB_BUILDKIT_ADDR" --build-cache-mode "$(params.build-cache-mode)"
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
echo '{"event":"ci-base-image","phase":"plan-artifacts","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apk"}'
|
||||
for tool in node git; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"plan-artifacts","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done
|
||||
cd /workspace/source/repo
|
||||
ci_node_deps="${HWLAB_CI_NODE_DEPS:-/opt/hwlab-ci-node-deps/node_modules}"
|
||||
if [ -d "$ci_node_deps/yaml" ] && [ ! -e node_modules/yaml ]; then
|
||||
mkdir -p node_modules
|
||||
ln -s "$ci_node_deps/yaml" node_modules/yaml
|
||||
fi
|
||||
test "$(git rev-parse HEAD)" = "$(params.revision)"
|
||||
HWLAB_CATALOG_PATH="$(params.catalog-path)" HWLAB_GIT_URL="$(params.git-url)" HWLAB_GIT_READ_URL="$(params.git-read-url)" HWLAB_GITOPS_BRANCH="$(params.gitops-branch)" node scripts/ci/restore-artifact-catalog.mjs
|
||||
node scripts/ci-plan.mjs --lane "$(params.lane)" --target-ref HEAD --deploy-config deploy/deploy.yaml --artifact-catalog "$(params.catalog-path)" --registry-prefix "$(params.registry-prefix)" --services "$(params.services)" --verify-reuse-registry > /workspace/source/ci-plan.json
|
||||
node - <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const plan = JSON.parse(fs.readFileSync("/workspace/source/ci-plan.json", "utf8"));
|
||||
const selected = String(process.env.HWLAB_SELECTED_SERVICES || "").split(",").map((item) => item.trim()).filter(Boolean);
|
||||
const allServices = selected.length > 0 ? selected : ["__HWLAB_DEFAULT_SERVICE_IDS__"];
|
||||
const affected = new Set(plan.affectedServices || []);
|
||||
const plannedBuildServices = Array.isArray(plan.buildServices) ? new Set(plan.buildServices) : null;
|
||||
const selectedSet = new Set(selected);
|
||||
const byService = new Map((plan.services || []).map((service) => [service.serviceId, service]));
|
||||
const entries = allServices.map((serviceId) => {
|
||||
const service = byService.get(serviceId) || {};
|
||||
const serviceSelected = selectedSet.has(serviceId);
|
||||
const rolloutAffected = serviceSelected && affected.has(serviceId);
|
||||
const envReuse = service.runtimeMode === "env-reuse-git-mirror-checkout" || service.envReuse === true;
|
||||
const buildRequired = serviceSelected && (plannedBuildServices ? plannedBuildServices.has(serviceId) : (envReuse ? service.envChanged === true : rolloutAffected));
|
||||
return {
|
||||
serviceId,
|
||||
selected: serviceSelected,
|
||||
affected: rolloutAffected,
|
||||
buildRequired,
|
||||
rolloutAffected,
|
||||
runtimeMode: service.runtimeMode || "service-image",
|
||||
envChanged: service.envChanged ?? null,
|
||||
codeChanged: service.codeChanged ?? null
|
||||
};
|
||||
});
|
||||
for (const entry of entries) {
|
||||
fs.writeFileSync("/tekton/results/affected-" + entry.serviceId, entry.affected ? "true" : "false");
|
||||
fs.writeFileSync("/tekton/results/build-" + entry.serviceId, entry.buildRequired ? "true" : "false");
|
||||
}
|
||||
fs.writeFileSync("/workspace/source/affected-services.json", JSON.stringify({
|
||||
sourceCommitId: plan.sourceCommitId,
|
||||
affectedServices: plan.affectedServices || [],
|
||||
rolloutServices: plan.rolloutServices || plan.affectedServices || [],
|
||||
buildServices: plan.buildServices || entries.filter((entry) => entry.buildRequired).map((entry) => entry.serviceId),
|
||||
reusedServices: plan.reusedServices || [],
|
||||
buildSkippedCount: plan.buildSkippedCount || 0,
|
||||
envArtifactGroups: plan.envArtifactGroups || [],
|
||||
changedPathSummary: plan.changedPathSummary || null,
|
||||
ciCdPlan: plan.ciCdPlan || null,
|
||||
services: plan.services || [],
|
||||
entries
|
||||
}, null, 2) + String.fromCharCode(10));
|
||||
console.log(JSON.stringify({ event: "g14-ci-plan", sourceCommitId: plan.sourceCommitId, affectedServices: plan.affectedServices || [], rolloutServices: plan.rolloutServices || plan.affectedServices || [], buildServices: plan.buildServices || [], reusedServices: plan.reusedServices || [], buildSkippedCount: plan.buildSkippedCount || 0, envArtifactGroups: plan.envArtifactGroups || [] }));
|
||||
NODE
|
||||
@@ -0,0 +1,142 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# __HWLAB_DEPENDENCY_PROXY_PROBE__
|
||||
echo '{"event":"ci-base-image","phase":"poller","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apk"}'
|
||||
for tool in node git ssh curl timeout; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"poller","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done
|
||||
# __HWLAB_GIT_SSH_SHELL__
|
||||
|
||||
git_ssh_setup
|
||||
workdir="$(mktemp -d)"
|
||||
git_timed poller-clone 180 git clone --depth 1 --branch "$SOURCE_BRANCH" "$GIT_READ_URL" "$workdir/repo"
|
||||
cd "$workdir/repo"
|
||||
git remote set-url origin "$GIT_READ_URL"
|
||||
revision="$(git rev-parse HEAD)"
|
||||
subject="$(git log -1 --pretty=%s)"
|
||||
case "$subject" in
|
||||
"chore: promote node GitOps source "*)
|
||||
echo "Skipping generated GitOps promotion commit $revision"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
short="$(printf '%.12s' "$revision")"
|
||||
: "${PIPELINERUN_PREFIX:?PIPELINERUN_PREFIX is required}"
|
||||
: "${GITOPS_TARGET:?GITOPS_TARGET is required}"
|
||||
prefix="$PIPELINERUN_PREFIX"
|
||||
name="$prefix-$short"
|
||||
export POLLER_REVISION="$revision"
|
||||
export POLLER_PIPELINERUN="$name"
|
||||
export POLLER_SOURCE_SUBJECT="$subject"
|
||||
echo "Checking $GITOPS_TARGET source $revision with PipelineRun $name"
|
||||
node <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const https = require("node:https");
|
||||
|
||||
function requiredEnv(name) {
|
||||
const value = process.env[name];
|
||||
if (!value) throw new Error(name + " is required");
|
||||
return value;
|
||||
}
|
||||
|
||||
function optionalEnv(name) {
|
||||
return process.env[name] || "";
|
||||
}
|
||||
|
||||
const namespace = requiredEnv("POD_NAMESPACE");
|
||||
const name = requiredEnv("POLLER_PIPELINERUN");
|
||||
const revision = requiredEnv("POLLER_REVISION");
|
||||
const host = process.env.KUBERNETES_SERVICE_HOST;
|
||||
const port = process.env.KUBERNETES_SERVICE_PORT || "443";
|
||||
if (!host) throw new Error("KUBERNETES_SERVICE_HOST is not available");
|
||||
|
||||
const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8");
|
||||
const ca = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt");
|
||||
|
||||
function request(method, path, body) {
|
||||
const payload = body ? JSON.stringify(body) : "";
|
||||
const headers = { Authorization: "Bearer " + token };
|
||||
if (payload) {
|
||||
headers["Content-Type"] = "application/json";
|
||||
headers["Content-Length"] = Buffer.byteLength(payload);
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = https.request({ host, port, method, path, ca, headers }, (res) => {
|
||||
let data = "";
|
||||
res.setEncoding("utf8");
|
||||
res.on("data", (chunk) => { data += chunk; });
|
||||
res.on("end", () => resolve({ statusCode: res.statusCode || 0, body: data }));
|
||||
});
|
||||
req.on("error", reject);
|
||||
if (payload) req.write(payload);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
const labels = {
|
||||
"app.kubernetes.io/part-of": "hwlab",
|
||||
"hwlab.pikastech.local/gitops-target": requiredEnv("GITOPS_TARGET"),
|
||||
"hwlab.pikastech.local/source-commit": revision,
|
||||
"hwlab.pikastech.local/trigger": "polling"
|
||||
};
|
||||
|
||||
const pipelineRun = {
|
||||
apiVersion: "tekton.dev/v1",
|
||||
kind: "PipelineRun",
|
||||
metadata: {
|
||||
name,
|
||||
namespace,
|
||||
labels,
|
||||
annotations: {
|
||||
"hwlab.pikastech.local/source-subject": optionalEnv("POLLER_SOURCE_SUBJECT"),
|
||||
"hwlab.pikastech.local/source-branch": requiredEnv("SOURCE_BRANCH"),
|
||||
"hwlab.pikastech.local/gitops-branch": requiredEnv("GITOPS_BRANCH")
|
||||
}
|
||||
},
|
||||
spec: {
|
||||
pipelineRef: { name: requiredEnv("PIPELINE_NAME") },
|
||||
taskRunTemplate: {
|
||||
serviceAccountName: requiredEnv("SERVICE_ACCOUNT_NAME"),
|
||||
podTemplate: { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet", securityContext: { fsGroup: 1000 } }
|
||||
},
|
||||
params: [
|
||||
{ name: "git-url", value: requiredEnv("GIT_URL") },
|
||||
{ name: "git-read-url", value: requiredEnv("GIT_READ_URL") },
|
||||
{ name: "source-branch", value: requiredEnv("SOURCE_BRANCH") },
|
||||
{ name: "gitops-branch", value: requiredEnv("GITOPS_BRANCH") },
|
||||
{ name: "lane", value: requiredEnv("LANE") },
|
||||
{ name: "catalog-path", value: requiredEnv("CATALOG_PATH") },
|
||||
{ name: "image-tag-mode", value: requiredEnv("IMAGE_TAG_MODE") },
|
||||
{ name: "runtime-path", value: requiredEnv("RUNTIME_PATH") },
|
||||
{ name: "revision", value: revision },
|
||||
{ name: "registry-prefix", value: requiredEnv("REGISTRY_PREFIX") },
|
||||
{ name: "services", value: requiredEnv("SERVICES") },
|
||||
{ name: "base-image", value: requiredEnv("BASE_IMAGE") }
|
||||
],
|
||||
workspaces: [
|
||||
{ name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } },
|
||||
{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
const base = "/apis/tekton.dev/v1/namespaces/" + encodeURIComponent(namespace) + "/pipelineruns";
|
||||
const item = base + "/" + encodeURIComponent(name);
|
||||
|
||||
(async () => {
|
||||
const existing = await request("GET", item);
|
||||
if (existing.statusCode === 200) {
|
||||
console.log(JSON.stringify({ status: "exists", pipelineRun: name, revision }));
|
||||
return;
|
||||
}
|
||||
if (existing.statusCode !== 404) {
|
||||
throw new Error("unexpected PipelineRun lookup status " + existing.statusCode + ": " + existing.body.slice(0, 500));
|
||||
}
|
||||
const created = await request("POST", base, pipelineRun);
|
||||
if (created.statusCode < 200 || created.statusCode > 299) {
|
||||
throw new Error("PipelineRun create failed with status " + created.statusCode + ": " + created.body.slice(0, 1000));
|
||||
}
|
||||
console.log(JSON.stringify({ status: "created", pipelineRun: name, revision }));
|
||||
})().catch((error) => {
|
||||
console.error(error.stack || error.message);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
NODE
|
||||
@@ -0,0 +1,117 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# __HWLAB_CI_TIMING_SHELL__
|
||||
|
||||
export HWLAB_TEKTON_PIPELINERUN="$(context.pipelineRun.name)"
|
||||
export HWLAB_TEKTON_TASKRUN="$(context.taskRun.name)"
|
||||
export HWLAB_TEKTON_TASK="prepare-source"
|
||||
export HWLAB_SOURCE_REVISION="$(params.revision)"
|
||||
echo '{"event":"ci-base-image","phase":"prepare-source","image":"__HWLAB_CI_TOOLS_IMAGE__","policy":"no-runtime-apk"}'
|
||||
for tool in node git timeout; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done
|
||||
node -e 'console.log(JSON.stringify({event:"ci-base-image",phase:"prepare-source",ok:true,node:process.version}))'
|
||||
# __HWLAB_GIT_SSH_SHELL__
|
||||
|
||||
git_url_requires_ssh() { case "$1" in git@*|ssh://*) return 0 ;; *) return 1 ;; esac; }
|
||||
git_read_url="$(params.git-read-url)"
|
||||
if git_url_requires_ssh "$git_read_url"; then
|
||||
for tool in ssh ssh-keyscan; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done
|
||||
git_ssh_setup
|
||||
else
|
||||
echo '{"event":"git-ssh-setup","phase":"prepare-source","status":"skipped","reason":"non-ssh-git-read-url"}'
|
||||
fi
|
||||
rm -rf /workspace/source/repo
|
||||
source_clone_started_ms="$(ci_now_ms)"
|
||||
git_timed source-clone 180 git clone --branch "$(params.source-branch)" "$git_read_url" /workspace/source/repo
|
||||
cd /workspace/source/repo
|
||||
git config --global --add safe.directory /workspace/source/repo
|
||||
git remote set-url origin "$git_read_url"
|
||||
git checkout "$(params.revision)"
|
||||
test "$(git rev-parse HEAD)" = "$(params.revision)"
|
||||
git merge-base --is-ancestor "$(params.revision)" "origin/$(params.source-branch)" || { echo '{"event":"source-ancestry","status":"failed","revision":"'"$(params.revision)"'","sourceBranch":"'"$(params.source-branch)"'"}'; exit 32; }
|
||||
ci_timing_emit source-clone succeeded "$source_clone_started_ms"
|
||||
prepare_source_dependencies_started_ms="$(ci_now_ms)"
|
||||
echo '{"event":"prepare-source-dependencies","status":"skipped","reason":"renderer-dependency-install-disabled","dependency":"yaml"}'
|
||||
ci_timing_emit prepare-source-dependencies succeeded "$prepare_source_dependencies_started_ms"
|
||||
catalog_path="$(params.catalog-path)"
|
||||
mkdir -p "$(dirname "$catalog_path")"
|
||||
catalog_fetch_started_ms="$(ci_now_ms)"
|
||||
if git_timed catalog-ls-remote 45 git ls-remote --exit-code --heads "$git_read_url" "$(params.gitops-branch)" >/dev/null; then
|
||||
git remote set-url gitops-catalog "$git_read_url" 2>/dev/null || git remote add gitops-catalog "$git_read_url"
|
||||
git_timed catalog-fetch 90 git fetch --depth 1 gitops-catalog "$(params.gitops-branch)" >/dev/null || true
|
||||
if git cat-file -e FETCH_HEAD:"$catalog_path" 2>/dev/null; then
|
||||
git show FETCH_HEAD:"$catalog_path" > /tmp/hwlab-gitops-artifact-catalog.json
|
||||
if node --input-type=module - /tmp/hwlab-gitops-artifact-catalog.json deploy/deploy.yaml "$(params.services)" <<'NODE'
|
||||
import fs from "node:fs";
|
||||
const [catalogPath, deployPath, selectedServices] = process.argv.slice(2);
|
||||
const ids = (doc) => (doc.services || []).map((service) => service.serviceId).filter(Boolean);
|
||||
const topLevelServiceIds = (text) => {
|
||||
const values = [];
|
||||
let inServices = false;
|
||||
for (const line of text.split(/\r?\n/u)) {
|
||||
if (/^services:\s*$/u.test(line)) { inServices = true; continue; }
|
||||
if (!inServices) continue;
|
||||
if (/^\S/u.test(line)) break;
|
||||
const match = line.match(/^\s*-\s+serviceId:\s*['"]?([^'"#\s]+)['"]?/u);
|
||||
if (match) values.push(match[1]);
|
||||
}
|
||||
return values;
|
||||
};
|
||||
const selected = (selectedServices || "").split(",").map((item) => item.trim()).filter(Boolean);
|
||||
const uniqueSorted = (items) => [...new Set(items)].sort();
|
||||
const gitops = JSON.parse(fs.readFileSync(catalogPath, "utf8"));
|
||||
const expected = selected.length ? selected : topLevelServiceIds(fs.readFileSync(deployPath, "utf8"));
|
||||
const actual = ids(gitops);
|
||||
const expectedSet = uniqueSorted(expected);
|
||||
const actualSet = uniqueSorted(actual);
|
||||
const sameServices = expectedSet.length === actualSet.length && expectedSet.every((item, index) => item === actualSet[index]);
|
||||
if (!sameServices) {
|
||||
const missing = expectedSet.filter((item) => !actualSet.includes(item));
|
||||
const extra = actualSet.filter((item) => !expectedSet.includes(item));
|
||||
console.error(JSON.stringify({ event: "gitops-artifact-catalog", phase: "prepare-source", status: "ignored-stale", reason: "service-ids-mismatch", expected, actual, missing, extra }));
|
||||
process.exit(42);
|
||||
}
|
||||
NODE
|
||||
then
|
||||
cp /tmp/hwlab-gitops-artifact-catalog.json "$catalog_path"
|
||||
echo '{"event":"gitops-artifact-catalog","phase":"prepare-source","status":"loaded","branch":"'"$(params.gitops-branch)"'"}'
|
||||
else
|
||||
echo '{"event":"gitops-artifact-catalog","phase":"prepare-source","status":"ignored-stale","reason":"service-ids-mismatch","branch":"'"$(params.gitops-branch)"'"}'
|
||||
fi
|
||||
else
|
||||
echo '{"event":"gitops-artifact-catalog","phase":"prepare-source","status":"seed","reason":"missing-on-gitops-branch"}'
|
||||
fi
|
||||
else
|
||||
echo '{"event":"gitops-artifact-catalog","phase":"prepare-source","status":"seed","reason":"gitops-branch-missing"}'
|
||||
fi
|
||||
if [ ! -s "$catalog_path" ] && [ "$(params.lane)" = "v02" ]; then
|
||||
node --input-type=module - "$catalog_path" "$(params.revision)" "$(params.registry-prefix)" "$(params.image-tag-mode)" "$(params.services)" <<'NODE'
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
const [catalogPath, revision, registryPrefix, imageTagMode, selectedServices] = process.argv.slice(2);
|
||||
const topLevelServiceIds = (text) => {
|
||||
const values = [];
|
||||
let inServices = false;
|
||||
for (const line of text.split(/\r?\n/u)) {
|
||||
if (/^services:\s*$/u.test(line)) { inServices = true; continue; }
|
||||
if (!inServices) continue;
|
||||
if (/^\S/u.test(line)) break;
|
||||
const match = line.match(/^\s*-\s+serviceId:\s*['"]?([^'"#\s]+)['"]?/u);
|
||||
if (match) values.push(match[1]);
|
||||
}
|
||||
return values;
|
||||
};
|
||||
const tag = imageTagMode === 'full' ? revision : revision.slice(0, 7);
|
||||
const namespace = 'hwlab-v02';
|
||||
const selected = (selectedServices || '').split(',').filter(Boolean);
|
||||
const serviceIds = selected.length ? selected : topLevelServiceIds(fs.readFileSync('deploy/deploy.yaml', 'utf8'));
|
||||
const services = serviceIds.map((serviceId) => {
|
||||
const service = { serviceId, profile: 'v02', namespace, commitId: tag, sourceCommitId: revision, image: `${registryPrefix}/${serviceId}:${tag}`, imageTag: tag, digest: null, buildBackend: 'contract-skeleton' };
|
||||
return service;
|
||||
});
|
||||
fs.mkdirSync(path.dirname(catalogPath), { recursive: true });
|
||||
fs.writeFileSync(catalogPath, JSON.stringify({ catalogVersion: 'v1', kind: 'hwlab-artifact-catalog', environment: 'v02', profile: 'v02', namespace, endpoint: "__HWLAB_V02_RUNTIME_ENDPOINT__", commitId: tag, artifactState: 'contract-skeleton', publish: { registryPrefix, sourceCommitId: revision, imageTag: tag, publishedAt: null }, allowedProfiles: ['v02'], forbiddenProfiles: ['dev', 'prod'], services }, null, 2) + '\n');
|
||||
NODE
|
||||
echo '{"event":"gitops-artifact-catalog","phase":"prepare-source","status":"seeded-v02-skeleton"}'
|
||||
fi
|
||||
ci_timing_emit catalog-fetch succeeded "$catalog_fetch_started_ms"
|
||||
echo 'node prepare-source complete; validation task count=__HWLAB_VALIDATION_TASK_COUNT__'
|
||||
Reference in New Issue
Block a user