diff --git a/cmd/hwlab-user-billing/main.go b/cmd/hwlab-user-billing/main.go new file mode 100644 index 00000000..f6c867bf --- /dev/null +++ b/cmd/hwlab-user-billing/main.go @@ -0,0 +1,16 @@ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/pikasTech/HWLAB/internal/userbilling" +) + +func main() { + if err := userbilling.Run(context.Background()); err != nil { + fmt.Fprintf(os.Stderr, "hwlab-user-billing: %v\n", err) + os.Exit(1) + } +} diff --git a/deploy/deploy.schema.json b/deploy/deploy.schema.json index 22b3273c..8cd5292a 100644 --- a/deploy/deploy.schema.json +++ b/deploy/deploy.schema.json @@ -69,6 +69,7 @@ "type": "string", "enum": [ "hwlab-cloud-api", + "hwlab-user-billing", "hwlab-cloud-web", "hwlab-gateway", "hwlab-edge-proxy", @@ -377,9 +378,9 @@ "type": "object", "required": ["runtimeKind", "entrypoint", "componentPaths"], "properties": { - "runtimeKind": { "type": "string", "enum": ["bun-command", "cloud-web", "skills-bundle", "node-command"] }, + "runtimeKind": { "type": "string", "enum": ["bun-command", "cloud-web", "skills-bundle", "node-command", "go-service"] }, "entrypoint": { "$ref": "#/$defs/repoPath" }, - "artifactKind": { "type": "string", "enum": ["bun-command", "cloud-web", "skills-bundle", "node-command"] }, + "artifactKind": { "type": "string", "enum": ["bun-command", "cloud-web", "skills-bundle", "node-command", "go-service"] }, "healthPath": { "type": "string", "pattern": "^/" }, "healthPort": { "type": "integer", "minimum": 1, "maximum": 65535 }, "componentPaths": { @@ -408,6 +409,9 @@ "bunVersion": { "type": "string", "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+" }, "launcherPath": { "$ref": "#/$defs/repoPath" }, "runtimeNodeModulesPath": { "type": "string", "pattern": "^/" }, + "runtimeGoModCachePath": { "type": "string", "pattern": "^/" }, + "runtimeGoBuildCachePath": { "type": "string", "pattern": "^/" }, + "goToolchain": { "type": "string", "minLength": 1 }, "additionalEnvPaths": { "type": "array", "items": { "$ref": "#/$defs/repoPath" }, @@ -416,6 +420,13 @@ "hwpodAliases": { "type": "array", "items": { "$ref": "#/$defs/hwpodAlias" } + }, + "downloadStack": { + "type": "object", + "additionalProperties": true, + "properties": { + "goProxy": { "type": "string", "minLength": 1 } + } } }, "additionalProperties": false diff --git a/deploy/deploy.yaml b/deploy/deploy.yaml index d7646642..cfb41e6e 100644 --- a/deploy/deploy.yaml +++ b/deploy/deploy.yaml @@ -256,12 +256,14 @@ lanes: port: 5432 envReuseServices: - hwlab-cloud-api + - hwlab-user-billing - hwlab-cloud-web - hwlab-gateway - hwlab-edge-proxy - hwlab-agent-skills bootScripts: hwlab-cloud-api: deploy/runtime/boot/hwlab-cloud-api.sh + hwlab-user-billing: deploy/runtime/boot/hwlab-user-billing.sh hwlab-cloud-web: deploy/runtime/boot/hwlab-cloud-web.sh hwlab-gateway: deploy/runtime/boot/hwlab-gateway.sh hwlab-edge-proxy: deploy/runtime/boot/hwlab-edge-proxy.sh @@ -287,6 +289,20 @@ lanes: - skills/hwlab-agent-runtime/ env: {} observable: true + hwlab-user-billing: + runtimeKind: go-service + entrypoint: cmd/hwlab-user-billing/main.go + artifactKind: go-service + healthPath: /health/live + healthPort: 6670 + componentPaths: + - go.mod + - go.sum + - cmd/hwlab-user-billing/ + - internal/userbilling/ + - deploy/runtime/boot/hwlab-user-billing.sh + env: {} + observable: true hwlab-cloud-web: runtimeKind: cloud-web entrypoint: web/hwlab-cloud-web/index.html @@ -337,6 +353,7 @@ lanes: osPackages: - ca-certificates - git + - golang-go bunVersion: 1.3.13 downloadStack: httpProxy: http://127.0.0.1:10808 @@ -349,11 +366,15 @@ lanes: - .hyueapi.com npmRegistry: https://registry.npmjs.org/ npmFetchTimeoutMs: 300000 + goProxy: https://goproxy.cn,direct bunPackages: x64: '@oven/bun-linux-x64' arm64: '@oven/bun-linux-aarch64' launcherPath: deploy/runtime/launcher/hwlab-env-reuse-launcher.ts runtimeNodeModulesPath: /opt/hwlab-env/node_modules + runtimeGoModCachePath: /opt/hwlab-env/go/pkg/mod + runtimeGoBuildCachePath: /opt/hwlab-env/go/cache + goToolchain: local additionalEnvPaths: - internal/dev-entrypoint/artifact-runtime.mjs - scripts/artifact-publish.mjs @@ -388,6 +409,17 @@ lanes: HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE: agentrun-v01 HWLAB_CODE_AGENT_AGENTRUN_SECRET_NAMESPACE: agentrun-v01 HWLAB_CODE_AGENT_AGENTRUN_REPO_URL: http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git + - serviceId: hwlab-user-billing + env: + HWLAB_USER_BILLING_DB_URL: secretRef:hwlab-cloud-api-v03-db/database-url + HWLAB_USER_BILLING_STATE_AUTHORITY: pk01-postgres + HWLAB_USER_BILLING_EXTERNAL_DB_LABEL: pk01-external-postgres + HWLAB_USER_BILLING_PORT: "6670" + HWLAB_USER_BILLING_REGISTRATION_ENABLED: "1" + HWLAB_USER_BILLING_INITIAL_CREDITS: "0" + HWLAB_USER_BILLING_CREDIT_PER_1K_TOKENS: "1" + HWLAB_USER_BILLING_MIGRATE_ON_START: "1" + HWLAB_USER_BILLING_REDIS_URL: redis://hwlab-user-billing-redis.hwlab-v03.svc.cluster.local:6379/0 services: - serviceId: hwlab-cloud-api namespace: hwlab-dev diff --git a/deploy/k8s/base/services.yaml b/deploy/k8s/base/services.yaml index 0a903928..2afdc458 100644 --- a/deploy/k8s/base/services.yaml +++ b/deploy/k8s/base/services.yaml @@ -27,6 +27,31 @@ ] } }, + { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": "hwlab-user-billing", + "namespace": "hwlab-dev", + "labels": { + "app.kubernetes.io/name": "hwlab-user-billing", + "hwlab.pikastech.local/service-id": "hwlab-user-billing" + } + }, + "spec": { + "type": "ClusterIP", + "selector": { + "app.kubernetes.io/name": "hwlab-user-billing" + }, + "ports": [ + { + "name": "http", + "port": 6670, + "targetPort": "http" + } + ] + } + }, { "apiVersion": "v1", "kind": "Service", diff --git a/deploy/k8s/base/workloads.yaml b/deploy/k8s/base/workloads.yaml index f7e5875c..e58bf499 100644 --- a/deploy/k8s/base/workloads.yaml +++ b/deploy/k8s/base/workloads.yaml @@ -418,6 +418,74 @@ } } }, + { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "hwlab-user-billing", + "namespace": "hwlab-dev", + "labels": { + "app.kubernetes.io/name": "hwlab-user-billing", + "hwlab.pikastech.local/service-id": "hwlab-user-billing" + } + }, + "spec": { + "replicas": 1, + "selector": { + "matchLabels": { + "app.kubernetes.io/name": "hwlab-user-billing" + } + }, + "template": { + "metadata": { + "labels": { + "app.kubernetes.io/name": "hwlab-user-billing", + "hwlab.pikastech.local/service-id": "hwlab-user-billing" + } + }, + "spec": { + "containers": [ + { + "name": "hwlab-user-billing", + "image": "127.0.0.1:5000/hwlab/hwlab-user-billing:af46386", + "ports": [ + { + "name": "http", + "containerPort": 6670 + } + ], + "env": [ + { + "name": "HWLAB_USER_BILLING_PORT", + "value": "6670" + }, + { + "name": "HWLAB_USER_BILLING_STATE_AUTHORITY", + "value": "pk01-postgres" + }, + { + "name": "HWLAB_USER_BILLING_STATELESS", + "value": "true" + } + ], + "readinessProbe": { + "httpGet": { + "path": "/health/live", + "port": "http" + } + }, + "livenessProbe": { + "httpGet": { + "path": "/health/live", + "port": "http" + } + } + } + ] + } + } + } + }, { "apiVersion": "apps/v1", "kind": "Deployment", diff --git a/deploy/runtime/boot/hwlab-user-billing.sh b/deploy/runtime/boot/hwlab-user-billing.sh new file mode 100644 index 00000000..3cebd2c8 --- /dev/null +++ b/deploy/runtime/boot/hwlab-user-billing.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env sh +set -eu + +cd /workspace/hwlab-boot/repo + +export HWLAB_SERVICE_ID="${HWLAB_SERVICE_ID:-hwlab-user-billing}" +export HWLAB_ARTIFACT_KIND="${HWLAB_ARTIFACT_KIND:-go-service}" +export HWLAB_USER_BILLING_PORT="${HWLAB_USER_BILLING_PORT:-6670}" +export PORT="${PORT:-$HWLAB_USER_BILLING_PORT}" +export GOMODCACHE="${GOMODCACHE:-/opt/hwlab-env/go/pkg/mod}" +export GOCACHE="${GOCACHE:-/opt/hwlab-env/go/cache}" +export GOPROXY="${GOPROXY:-https://goproxy.cn,direct}" +export GOTOOLCHAIN="${GOTOOLCHAIN:-local}" + +mkdir -p "$GOMODCACHE" "$GOCACHE" /tmp/hwlab-user-billing +go build -trimpath -o /tmp/hwlab-user-billing/hwlab-user-billing ./cmd/hwlab-user-billing +exec /tmp/hwlab-user-billing/hwlab-user-billing diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..ae410d40 --- /dev/null +++ b/go.mod @@ -0,0 +1,17 @@ +module github.com/pikasTech/HWLAB + +go 1.18 + +require github.com/jackc/pgx/v4 v4.18.3 + +require ( + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.14.3 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.3.3 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgtype v1.14.0 // indirect + golang.org/x/crypto v0.27.0 // indirect + golang.org/x/text v0.18.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..359d9b85 --- /dev/null +++ b/go.sum @@ -0,0 +1,178 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= +github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= +github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= +github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA= +github.com/jackc/pgx/v4 v4.18.3/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/internal/userbilling/migrations/0001_user_billing.sql b/internal/userbilling/migrations/0001_user_billing.sql new file mode 100644 index 00000000..1b93d157 --- /dev/null +++ b/internal/userbilling/migrations/0001_user_billing.sql @@ -0,0 +1,105 @@ +CREATE TABLE IF NOT EXISTS hwlab_users ( + id TEXT PRIMARY KEY, + email TEXT NOT NULL UNIQUE, + username TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL DEFAULT '', + password_hash TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active', + role TEXT NOT NULL DEFAULT 'user', + email_verified BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK (status IN ('active', 'disabled', 'pending')), + CHECK (role IN ('user', 'admin')) +); + +CREATE TABLE IF NOT EXISTS hwlab_user_sessions ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL REFERENCES hwlab_users(id) ON DELETE CASCADE, + token_hash TEXT NOT NULL UNIQUE, + expires_at TIMESTAMPTZ NOT NULL, + revoked_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS hwlab_user_sessions_user_id_idx ON hwlab_user_sessions(user_id); +CREATE INDEX IF NOT EXISTS hwlab_user_sessions_token_hash_idx ON hwlab_user_sessions(token_hash); + +CREATE TABLE IF NOT EXISTS hwlab_api_keys ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL REFERENCES hwlab_users(id) ON DELETE CASCADE, + name TEXT NOT NULL, + key_prefix TEXT NOT NULL, + key_hash TEXT NOT NULL UNIQUE, + scopes_json JSONB NOT NULL DEFAULT '["api"]'::jsonb, + status TEXT NOT NULL DEFAULT 'active', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_used_at TIMESTAMPTZ, + revoked_at TIMESTAMPTZ, + CHECK (status IN ('active', 'revoked')) +); + +CREATE INDEX IF NOT EXISTS hwlab_api_keys_user_id_idx ON hwlab_api_keys(user_id); +CREATE INDEX IF NOT EXISTS hwlab_api_keys_key_hash_idx ON hwlab_api_keys(key_hash); + +CREATE TABLE IF NOT EXISTS hwlab_credit_accounts ( + user_id TEXT PRIMARY KEY REFERENCES hwlab_users(id) ON DELETE CASCADE, + balance_credits BIGINT NOT NULL DEFAULT 0, + reserved_credits BIGINT NOT NULL DEFAULT 0, + plan_id TEXT NOT NULL DEFAULT 'default', + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK (balance_credits >= 0), + CHECK (reserved_credits >= 0) +); + +CREATE TABLE IF NOT EXISTS hwlab_credit_ledger ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL REFERENCES hwlab_users(id) ON DELETE CASCADE, + delta_credits BIGINT NOT NULL, + balance_after BIGINT NOT NULL, + kind TEXT NOT NULL, + reason TEXT NOT NULL DEFAULT '', + idempotency_key TEXT UNIQUE, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS hwlab_credit_ledger_user_id_idx ON hwlab_credit_ledger(user_id); + +CREATE TABLE IF NOT EXISTS hwlab_billing_reservations ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL REFERENCES hwlab_users(id) ON DELETE CASCADE, + service_id TEXT NOT NULL, + estimated_credits BIGINT NOT NULL, + estimated_tokens BIGINT NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'reserved', + idempotency_key TEXT UNIQUE, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK (estimated_credits >= 0), + CHECK (estimated_tokens >= 0), + CHECK (status IN ('reserved', 'completed', 'cancelled', 'expired')) +); + +CREATE INDEX IF NOT EXISTS hwlab_billing_reservations_user_id_idx ON hwlab_billing_reservations(user_id); + +CREATE TABLE IF NOT EXISTS hwlab_usage_records ( + id TEXT PRIMARY KEY, + reservation_id TEXT REFERENCES hwlab_billing_reservations(id) ON DELETE SET NULL, + user_id TEXT NOT NULL REFERENCES hwlab_users(id) ON DELETE CASCADE, + service_id TEXT NOT NULL, + unit TEXT NOT NULL DEFAULT 'token', + quantity BIGINT NOT NULL DEFAULT 0, + credits BIGINT NOT NULL DEFAULT 0, + idempotency_key TEXT UNIQUE, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK (quantity >= 0), + CHECK (credits >= 0) +); + +CREATE INDEX IF NOT EXISTS hwlab_usage_records_user_id_idx ON hwlab_usage_records(user_id); +CREATE INDEX IF NOT EXISTS hwlab_usage_records_service_id_idx ON hwlab_usage_records(service_id); + diff --git a/internal/userbilling/service.go b/internal/userbilling/service.go new file mode 100644 index 00000000..1cc69678 --- /dev/null +++ b/internal/userbilling/service.go @@ -0,0 +1,1081 @@ +package userbilling + +import ( + "context" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "database/sql" + "embed" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/fs" + "log" + "math" + "net/http" + "os" + "os/signal" + "sort" + "strconv" + "strings" + "syscall" + "time" + + _ "github.com/jackc/pgx/v4/stdlib" +) + +const serviceID = "hwlab-user-billing" + +//go:embed migrations/*.sql +var migrationFS embed.FS + +type Config struct { + Addr string + DatabaseURL string + RedisURL string + InitialCredits int64 + CreditPerThousandTok int64 + SessionTTL time.Duration + ReservationTTL time.Duration + RegistrationEnabled bool + InternalToken string + RequireInternalToken bool + MigrateOnStart bool + StateAuthority string + RuntimeNamespace string + ExternalDatabaseLabel string +} + +type Server struct { + config Config + db *sql.DB + mux *http.ServeMux +} + +type User struct { + ID string `json:"id"` + Email string `json:"email"` + Username string `json:"username"` + DisplayName string `json:"displayName"` + Status string `json:"status"` + Role string `json:"role"` + EmailVerified bool `json:"emailVerified"` + CreatedAt time.Time `json:"createdAt"` +} + +type Principal struct { + UserID string `json:"userId"` + Email string `json:"email"` + Username string `json:"username"` + Role string `json:"role"` + Scopes []string `json:"scopes"` + AuthType string `json:"authType"` + KeyID string `json:"keyId,omitempty"` +} + +type apiError struct { + Code string `json:"code"` + Message string `json:"message"` +} + +func Run(parent context.Context) error { + cfg := LoadConfig(os.Environ()) + server, err := NewServer(parent, cfg) + if err != nil { + return err + } + defer server.Close() + + ctx, stop := signal.NotifyContext(parent, syscall.SIGINT, syscall.SIGTERM) + defer stop() + + httpServer := &http.Server{ + Addr: cfg.Addr, + Handler: server, + ReadHeaderTimeout: 10 * time.Second, + } + + errCh := make(chan error, 1) + go func() { + log.Printf("%s listening on %s", serviceID, cfg.Addr) + errCh <- httpServer.ListenAndServe() + }() + + select { + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = httpServer.Shutdown(shutdownCtx) + return nil + case err := <-errCh: + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err + } +} + +func LoadConfig(env []string) Config { + values := map[string]string{} + for _, item := range env { + name, value, ok := strings.Cut(item, "=") + if ok { + values[name] = value + } + } + port := first(values["HWLAB_USER_BILLING_PORT"], values["PORT"], "6670") + cfg := Config{ + Addr: first(values["HWLAB_USER_BILLING_ADDR"], ":"+port), + DatabaseURL: first(values["HWLAB_USER_BILLING_DB_URL"], values["DATABASE_URL"]), + RedisURL: values["HWLAB_USER_BILLING_REDIS_URL"], + InitialCredits: parseInt64(values["HWLAB_USER_BILLING_INITIAL_CREDITS"], 0), + CreditPerThousandTok: parseInt64(values["HWLAB_USER_BILLING_CREDIT_PER_1K_TOKENS"], 1), + SessionTTL: parseDurationHours(values["HWLAB_USER_BILLING_SESSION_TTL_HOURS"], 720), + ReservationTTL: parseDurationMinutes(values["HWLAB_USER_BILLING_RESERVATION_TTL_MINUTES"], 30), + RegistrationEnabled: values["HWLAB_USER_BILLING_REGISTRATION_ENABLED"] != "0", + InternalToken: values["HWLAB_USER_BILLING_INTERNAL_TOKEN"], + RequireInternalToken: values["HWLAB_USER_BILLING_REQUIRE_INTERNAL_TOKEN"] == "1", + MigrateOnStart: values["HWLAB_USER_BILLING_MIGRATE_ON_START"] != "0", + StateAuthority: first(values["HWLAB_USER_BILLING_STATE_AUTHORITY"], "pk01-postgres"), + RuntimeNamespace: first(values["POD_NAMESPACE"], values["HWLAB_NAMESPACE"], "hwlab-v03"), + ExternalDatabaseLabel: first(values["HWLAB_USER_BILLING_EXTERNAL_DB_LABEL"], "pk01-external-postgres"), + } + return cfg +} + +func NewServer(ctx context.Context, cfg Config) (*Server, error) { + server := &Server{config: cfg, mux: http.NewServeMux()} + if cfg.DatabaseURL != "" { + db, err := sql.Open("pgx", cfg.DatabaseURL) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(8) + db.SetMaxIdleConns(4) + db.SetConnMaxLifetime(30 * time.Minute) + server.db = db + if cfg.MigrateOnStart { + migrateCtx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + if err := server.migrate(migrateCtx); err != nil { + return nil, fmt.Errorf("migrate user billing schema: %w", err) + } + } + } + server.routes() + return server, nil +} + +func (s *Server) Close() { + if s.db != nil { + _ = s.db.Close() + } +} + +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mux.ServeHTTP(w, r) +} + +func (s *Server) routes() { + s.route(http.MethodGet, "/health/live", s.handleLive) + s.route(http.MethodGet, "/health/ready", s.handleReady) + s.route(http.MethodPost, "/v1/auth/register", s.handleRegister) + s.route(http.MethodPost, "/v1/auth/login", s.handleLogin) + s.route(http.MethodPost, "/v1/auth/logout", s.handleLogout) + s.route(http.MethodPost, "/v1/auth/refresh", s.handleRefresh) + s.route(http.MethodGet, "/v1/me", s.handleMe) + s.route(http.MethodPost, "/v1/api-keys", s.handleCreateAPIKey) + s.route(http.MethodPost, "/internal/auth/introspect", s.handleIntrospect) + s.route(http.MethodPost, "/internal/billing/preflight", s.handleBillingPreflight) + s.route(http.MethodPost, "/internal/billing/record", s.handleBillingRecord) + s.route(http.MethodPost, "/internal/admin/credits/adjust", s.handleAdminCreditAdjust) + s.route(http.MethodPost, "/internal/admin/users/status", s.handleAdminUserStatus) +} + +func (s *Server) route(method, path string, handler http.HandlerFunc) { + s.mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { + if r.Method != method { + w.Header().Set("Allow", method) + writeAPIError(w, http.StatusMethodNotAllowed, "method_not_allowed", "method not allowed") + return + } + handler(w, r) + }) +} + +func (s *Server) migrate(ctx context.Context) error { + if s.db == nil { + return nil + } + files, err := fs.Glob(migrationFS, "migrations/*.sql") + if err != nil { + return err + } + sort.Strings(files) + for _, name := range files { + body, err := migrationFS.ReadFile(name) + if err != nil { + return err + } + for _, statement := range splitSQLStatements(string(body)) { + if _, err := s.db.ExecContext(ctx, statement); err != nil { + return fmt.Errorf("%s: %w", name, err) + } + } + } + return nil +} + +func splitSQLStatements(text string) []string { + parts := strings.Split(text, ";\n") + statements := make([]string, 0, len(parts)) + for _, part := range parts { + stmt := strings.TrimSpace(part) + if stmt == "" { + continue + } + if !strings.HasSuffix(stmt, ";") { + stmt += ";" + } + statements = append(statements, stmt) + } + return statements +} + +func (s *Server) handleLive(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{ + "serviceId": serviceID, + "status": "ok", + "stateless": true, + "stateAuthority": s.config.StateAuthority, + "database": map[string]any{"configured": s.db != nil, "authority": s.config.ExternalDatabaseLabel}, + "redis": map[string]any{"configured": s.config.RedisURL != "", "role": "cache-only"}, + "namespace": s.config.RuntimeNamespace, + "registrationEnabled": s.config.RegistrationEnabled, + "internalTokenRequired": s.config.RequireInternalToken, + }) +} + +func (s *Server) handleReady(w http.ResponseWriter, r *http.Request) { + if s.db == nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]any{"serviceId": serviceID, "status": "degraded", "reason": "database-url-not-configured"}) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + if err := s.db.PingContext(ctx); err != nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]any{"serviceId": serviceID, "status": "degraded", "reason": "database-ping-failed"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"serviceId": serviceID, "status": "ready"}) +} + +func (s *Server) handleRegister(w http.ResponseWriter, r *http.Request) { + if !s.registrationAvailable(w) { + return + } + var req struct { + Email string `json:"email"` + Username string `json:"username"` + DisplayName string `json:"displayName"` + Password string `json:"password"` + } + if !decodeJSON(w, r, &req) { + return + } + email := strings.ToLower(strings.TrimSpace(req.Email)) + username := normalizeUsername(req.Username) + if email == "" || !strings.Contains(email, "@") || username == "" || len(req.Password) < 10 { + writeAPIError(w, http.StatusBadRequest, "invalid_registration", "email, username and a password with at least 10 characters are required") + return + } + passwordHash, err := hashPassword(req.Password) + if err != nil { + writeAPIError(w, http.StatusInternalServerError, "password_hash_failed", "could not hash password") + return + } + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{}) + if err != nil { + writeAPIError(w, http.StatusInternalServerError, "transaction_failed", "could not start transaction") + return + } + defer tx.Rollback() + user := User{ID: newID("usr"), Email: email, Username: username, DisplayName: strings.TrimSpace(req.DisplayName), Status: "active", Role: "user"} + err = tx.QueryRowContext(ctx, `INSERT INTO hwlab_users (id, email, username, display_name, password_hash, status, role) VALUES ($1, $2, $3, $4, $5, 'active', 'user') RETURNING created_at`, user.ID, user.Email, user.Username, user.DisplayName, passwordHash).Scan(&user.CreatedAt) + if err != nil { + if isDuplicate(err) { + writeAPIError(w, http.StatusConflict, "user_exists", "email or username already exists") + return + } + writeAPIError(w, http.StatusInternalServerError, "user_create_failed", "could not create user") + return + } + _, err = tx.ExecContext(ctx, `INSERT INTO hwlab_credit_accounts (user_id, balance_credits, reserved_credits, plan_id) VALUES ($1, $2, 0, 'default')`, user.ID, s.config.InitialCredits) + if err != nil { + writeAPIError(w, http.StatusInternalServerError, "credit_account_failed", "could not create credit account") + return + } + if s.config.InitialCredits > 0 { + _, err = tx.ExecContext(ctx, `INSERT INTO hwlab_credit_ledger (id, user_id, delta_credits, balance_after, kind, reason, metadata) VALUES ($1, $2, $3, $3, 'grant', 'registration_initial_credit', '{}'::jsonb)`, newID("led"), user.ID, s.config.InitialCredits) + if err != nil { + writeAPIError(w, http.StatusInternalServerError, "credit_ledger_failed", "could not write initial credit ledger") + return + } + } + if err := tx.Commit(); err != nil { + writeAPIError(w, http.StatusInternalServerError, "transaction_commit_failed", "could not commit registration") + return + } + writeJSON(w, http.StatusCreated, map[string]any{"user": user, "initialCredits": s.config.InitialCredits}) +} + +func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) { + if !s.databaseAvailable(w) { + return + } + var req struct { + Email string `json:"email"` + Username string `json:"username"` + Password string `json:"password"` + } + if !decodeJSON(w, r, &req) { + return + } + login := strings.ToLower(strings.TrimSpace(first(req.Email, req.Username))) + if login == "" || req.Password == "" { + writeAPIError(w, http.StatusBadRequest, "invalid_login", "email or username and password are required") + return + } + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + var user User + var passwordHash string + err := s.db.QueryRowContext(ctx, `SELECT id, email, username, display_name, password_hash, status, role, email_verified, created_at FROM hwlab_users WHERE lower(email) = $1 OR lower(username) = $1`, login).Scan(&user.ID, &user.Email, &user.Username, &user.DisplayName, &passwordHash, &user.Status, &user.Role, &user.EmailVerified, &user.CreatedAt) + if errors.Is(err, sql.ErrNoRows) { + writeAPIError(w, http.StatusUnauthorized, "invalid_credentials", "invalid email, username or password") + return + } + if err != nil { + writeAPIError(w, http.StatusInternalServerError, "login_failed", "could not load user") + return + } + if !verifyPassword(passwordHash, req.Password) { + writeAPIError(w, http.StatusUnauthorized, "invalid_credentials", "invalid email, username or password") + return + } + if user.Status != "active" { + writeAPIError(w, http.StatusForbidden, "user_disabled", "user is not active") + return + } + sessionToken := "hws_" + randomToken(32) + expiresAt := time.Now().UTC().Add(s.config.SessionTTL) + _, err = s.db.ExecContext(ctx, `INSERT INTO hwlab_user_sessions (id, user_id, token_hash, expires_at) VALUES ($1, $2, $3, $4)`, newID("ses"), user.ID, tokenHash(sessionToken), expiresAt) + if err != nil { + writeAPIError(w, http.StatusInternalServerError, "session_create_failed", "could not create session") + return + } + writeJSON(w, http.StatusOK, map[string]any{"token": sessionToken, "tokenType": "Bearer", "expiresAt": expiresAt, "user": user}) +} + +func (s *Server) handleLogout(w http.ResponseWriter, r *http.Request) { + if !s.databaseAvailable(w) { + return + } + token := bearerToken(r) + if token == "" { + writeAPIError(w, http.StatusUnauthorized, "missing_bearer", "authorization bearer token is required") + return + } + ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second) + defer cancel() + _, _ = s.db.ExecContext(ctx, `UPDATE hwlab_user_sessions SET revoked_at = now() WHERE token_hash = $1 AND revoked_at IS NULL`, tokenHash(token)) + writeJSON(w, http.StatusOK, map[string]any{"ok": true}) +} + +func (s *Server) handleRefresh(w http.ResponseWriter, r *http.Request) { + principal, ok := s.requirePrincipal(w, r) + if !ok { + return + } + if principal.AuthType != "session" { + writeAPIError(w, http.StatusForbidden, "session_required", "only session tokens can be refreshed") + return + } + expiresAt := time.Now().UTC().Add(s.config.SessionTTL) + ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second) + defer cancel() + _, err := s.db.ExecContext(ctx, `UPDATE hwlab_user_sessions SET expires_at = $1 WHERE token_hash = $2 AND revoked_at IS NULL`, expiresAt, tokenHash(bearerToken(r))) + if err != nil { + writeAPIError(w, http.StatusInternalServerError, "session_refresh_failed", "could not refresh session") + return + } + writeJSON(w, http.StatusOK, map[string]any{"expiresAt": expiresAt}) +} + +func (s *Server) handleMe(w http.ResponseWriter, r *http.Request) { + principal, ok := s.requirePrincipal(w, r) + if !ok { + return + } + balance, reserved, _ := s.accountBalance(r.Context(), principal.UserID) + writeJSON(w, http.StatusOK, map[string]any{"principal": principal, "credits": map[string]any{"balance": balance, "reserved": reserved, "available": balance - reserved}}) +} + +func (s *Server) handleCreateAPIKey(w http.ResponseWriter, r *http.Request) { + principal, ok := s.requirePrincipal(w, r) + if !ok { + return + } + var req struct { + Name string `json:"name"` + Scopes []string `json:"scopes"` + } + if !decodeJSON(w, r, &req) { + return + } + name := strings.TrimSpace(first(req.Name, "default")) + scopes := req.Scopes + if len(scopes) == 0 { + scopes = []string{"api"} + } + scopesJSON, _ := json.Marshal(scopes) + key := "hwl_" + randomToken(32) + keyID := newID("key") + prefix := key[:min(len(key), 16)] + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + _, err := s.db.ExecContext(ctx, `INSERT INTO hwlab_api_keys (id, user_id, name, key_prefix, key_hash, scopes_json) VALUES ($1, $2, $3, $4, $5, $6::jsonb)`, keyID, principal.UserID, name, prefix, tokenHash(key), string(scopesJSON)) + if err != nil { + writeAPIError(w, http.StatusInternalServerError, "api_key_create_failed", "could not create API key") + return + } + writeJSON(w, http.StatusCreated, map[string]any{"id": keyID, "key": key, "prefix": prefix, "scopes": scopes}) +} + +func (s *Server) handleIntrospect(w http.ResponseWriter, r *http.Request) { + if !s.internalAllowed(w, r) || !s.databaseAvailable(w) { + return + } + var req struct { + Token string `json:"token"` + APIKey string `json:"apiKey"` + } + if !decodeJSON(w, r, &req) { + return + } + token := first(req.Token, req.APIKey) + principal, err := s.principalFromToken(r.Context(), token) + if err != nil { + writeJSON(w, http.StatusOK, map[string]any{"active": false}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"active": true, "principal": principal}) +} + +func (s *Server) handleBillingPreflight(w http.ResponseWriter, r *http.Request) { + if !s.internalAllowed(w, r) || !s.databaseAvailable(w) { + return + } + var req struct { + Token string `json:"token"` + APIKey string `json:"apiKey"` + UserID string `json:"userId"` + ServiceID string `json:"serviceId"` + EstimatedCredits int64 `json:"estimatedCredits"` + EstimatedTokens int64 `json:"estimatedTokens"` + IdempotencyKey string `json:"idempotencyKey"` + Metadata map[string]any `json:"metadata"` + } + if !decodeJSON(w, r, &req) { + return + } + principal, err := s.principalForBilling(r.Context(), r, req.Token, req.APIKey, req.UserID) + if err != nil { + writeAPIError(w, http.StatusUnauthorized, "invalid_subject", "billing subject is not active") + return + } + service := strings.TrimSpace(req.ServiceID) + if service == "" { + writeAPIError(w, http.StatusBadRequest, "missing_service", "serviceId is required") + return + } + estimated := s.estimateCredits(req.EstimatedCredits, req.EstimatedTokens) + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + reservation, err := s.reserveCredits(ctx, principal.UserID, service, estimated, req.EstimatedTokens, strings.TrimSpace(req.IdempotencyKey), req.Metadata) + if err != nil { + if errors.Is(err, errInsufficientCredits) { + writeJSON(w, http.StatusPaymentRequired, map[string]any{"allowed": false, "reason": "insufficient_credits"}) + return + } + writeAPIError(w, http.StatusInternalServerError, "billing_preflight_failed", "could not reserve credits") + return + } + writeJSON(w, http.StatusOK, reservation) +} + +func (s *Server) handleBillingRecord(w http.ResponseWriter, r *http.Request) { + if !s.internalAllowed(w, r) || !s.databaseAvailable(w) { + return + } + var req struct { + ReservationID string `json:"reservationId"` + Token string `json:"token"` + APIKey string `json:"apiKey"` + UserID string `json:"userId"` + ServiceID string `json:"serviceId"` + UsedCredits int64 `json:"usedCredits"` + UsedTokens int64 `json:"usedTokens"` + IdempotencyKey string `json:"idempotencyKey"` + Metadata map[string]any `json:"metadata"` + } + if !decodeJSON(w, r, &req) { + return + } + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + result, err := s.recordUsage(ctx, r, req.ReservationID, req.Token, req.APIKey, req.UserID, req.ServiceID, req.UsedCredits, req.UsedTokens, req.IdempotencyKey, req.Metadata) + if err != nil { + writeAPIError(w, http.StatusBadRequest, "billing_record_failed", err.Error()) + return + } + writeJSON(w, http.StatusOK, result) +} + +func (s *Server) handleAdminCreditAdjust(w http.ResponseWriter, r *http.Request) { + if !s.internalAllowed(w, r) || !s.databaseAvailable(w) { + return + } + var req struct { + UserID string `json:"userId"` + DeltaCredits int64 `json:"deltaCredits"` + Reason string `json:"reason"` + IdempotencyKey string `json:"idempotencyKey"` + Metadata map[string]any `json:"metadata"` + } + if !decodeJSON(w, r, &req) { + return + } + if strings.TrimSpace(req.UserID) == "" || req.DeltaCredits == 0 { + writeAPIError(w, http.StatusBadRequest, "invalid_adjustment", "userId and non-zero deltaCredits are required") + return + } + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + balance, err := s.adjustCredits(ctx, req.UserID, req.DeltaCredits, "admin_adjust", req.Reason, req.IdempotencyKey, req.Metadata) + if err != nil { + writeAPIError(w, http.StatusBadRequest, "credit_adjust_failed", err.Error()) + return + } + writeJSON(w, http.StatusOK, map[string]any{"userId": req.UserID, "balance": balance}) +} + +func (s *Server) handleAdminUserStatus(w http.ResponseWriter, r *http.Request) { + if !s.internalAllowed(w, r) || !s.databaseAvailable(w) { + return + } + var req struct { + UserID string `json:"userId"` + Status string `json:"status"` + } + if !decodeJSON(w, r, &req) { + return + } + if req.Status != "active" && req.Status != "disabled" && req.Status != "pending" { + writeAPIError(w, http.StatusBadRequest, "invalid_status", "status must be active, disabled or pending") + return + } + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + res, err := s.db.ExecContext(ctx, `UPDATE hwlab_users SET status = $2, updated_at = now() WHERE id = $1`, req.UserID, req.Status) + if err != nil { + writeAPIError(w, http.StatusInternalServerError, "status_update_failed", "could not update user status") + return + } + affected, _ := res.RowsAffected() + if affected == 0 { + writeAPIError(w, http.StatusNotFound, "user_not_found", "user not found") + return + } + writeJSON(w, http.StatusOK, map[string]any{"userId": req.UserID, "status": req.Status}) +} + +var errInsufficientCredits = errors.New("insufficient credits") + +func (s *Server) reserveCredits(ctx context.Context, userID, service string, credits, tokens int64, idempotencyKey string, metadata map[string]any) (map[string]any, error) { + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{}) + if err != nil { + return nil, err + } + defer tx.Rollback() + if idempotencyKey != "" { + var existingID, status string + var estimated int64 + err := tx.QueryRowContext(ctx, `SELECT id, status, estimated_credits FROM hwlab_billing_reservations WHERE idempotency_key = $1`, idempotencyKey).Scan(&existingID, &status, &estimated) + if err == nil { + return map[string]any{"allowed": status == "reserved", "reservationId": existingID, "estimatedCredits": estimated, "status": status, "idempotent": true}, tx.Commit() + } + if !errors.Is(err, sql.ErrNoRows) { + return nil, err + } + } + var balance, reserved int64 + if err := tx.QueryRowContext(ctx, `SELECT balance_credits, reserved_credits FROM hwlab_credit_accounts WHERE user_id = $1 FOR UPDATE`, userID).Scan(&balance, &reserved); err != nil { + return nil, err + } + available := balance - reserved + if available < credits { + return nil, errInsufficientCredits + } + reservationID := newID("res") + metadataJSON := jsonObject(metadata) + expiresAt := time.Now().UTC().Add(s.config.ReservationTTL) + _, err = tx.ExecContext(ctx, `INSERT INTO hwlab_billing_reservations (id, user_id, service_id, estimated_credits, estimated_tokens, idempotency_key, metadata, expires_at) VALUES ($1, $2, $3, $4, $5, nullif($6, ''), $7::jsonb, $8)`, reservationID, userID, service, credits, nonNegative(tokens), idempotencyKey, metadataJSON, expiresAt) + if err != nil { + return nil, err + } + _, err = tx.ExecContext(ctx, `UPDATE hwlab_credit_accounts SET reserved_credits = reserved_credits + $2, updated_at = now() WHERE user_id = $1`, userID, credits) + if err != nil { + return nil, err + } + if err := tx.Commit(); err != nil { + return nil, err + } + return map[string]any{"allowed": true, "reservationId": reservationID, "estimatedCredits": credits, "estimatedTokens": tokens, "availableBefore": available, "expiresAt": expiresAt}, nil +} + +func (s *Server) recordUsage(ctx context.Context, r *http.Request, reservationID, token, apiKey, userID, serviceID string, usedCredits, usedTokens int64, idempotencyKey string, metadata map[string]any) (map[string]any, error) { + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{}) + if err != nil { + return nil, err + } + defer tx.Rollback() + if idempotencyKey != "" { + var existingID string + err := tx.QueryRowContext(ctx, `SELECT id FROM hwlab_usage_records WHERE idempotency_key = $1`, idempotencyKey).Scan(&existingID) + if err == nil { + return map[string]any{"recordId": existingID, "idempotent": true}, tx.Commit() + } + if !errors.Is(err, sql.ErrNoRows) { + return nil, err + } + } + var reserved int64 + if reservationID != "" { + var status string + err := tx.QueryRowContext(ctx, `SELECT user_id, service_id, estimated_credits, status FROM hwlab_billing_reservations WHERE id = $1 FOR UPDATE`, reservationID).Scan(&userID, &serviceID, &reserved, &status) + if err != nil { + return nil, err + } + if status != "reserved" { + return nil, fmt.Errorf("reservation status is %s", status) + } + } else { + principal, err := s.principalForBilling(ctx, r, token, apiKey, userID) + if err != nil { + return nil, err + } + userID = principal.UserID + serviceID = strings.TrimSpace(serviceID) + } + if serviceID == "" { + return nil, errors.New("serviceId is required") + } + credits := s.estimateCredits(usedCredits, usedTokens) + var balance, accountReserved int64 + if err := tx.QueryRowContext(ctx, `SELECT balance_credits, reserved_credits FROM hwlab_credit_accounts WHERE user_id = $1 FOR UPDATE`, userID).Scan(&balance, &accountReserved); err != nil { + return nil, err + } + if reservationID == "" && balance-accountReserved < credits { + return nil, errInsufficientCredits + } + newReserved := accountReserved - reserved + if newReserved < 0 { + newReserved = 0 + } + newBalance := balance - credits + if newBalance < 0 { + return nil, errInsufficientCredits + } + _, err = tx.ExecContext(ctx, `UPDATE hwlab_credit_accounts SET balance_credits = $2, reserved_credits = $3, updated_at = now() WHERE user_id = $1`, userID, newBalance, newReserved) + if err != nil { + return nil, err + } + recordID := newID("use") + metadataJSON := jsonObject(metadata) + _, err = tx.ExecContext(ctx, `INSERT INTO hwlab_usage_records (id, reservation_id, user_id, service_id, quantity, credits, idempotency_key, metadata) VALUES ($1, nullif($2, ''), $3, $4, $5, $6, nullif($7, ''), $8::jsonb)`, recordID, reservationID, userID, serviceID, nonNegative(usedTokens), credits, idempotencyKey, metadataJSON) + if err != nil { + return nil, err + } + usageLedgerKey := "" + if idempotencyKey != "" { + usageLedgerKey = "usage:" + idempotencyKey + } + _, err = tx.ExecContext(ctx, `INSERT INTO hwlab_credit_ledger (id, user_id, delta_credits, balance_after, kind, reason, idempotency_key, metadata) VALUES ($1, $2, $3, $4, 'usage', $5, nullif($6, ''), $7::jsonb)`, newID("led"), userID, -credits, newBalance, serviceID, usageLedgerKey, metadataJSON) + if err != nil { + return nil, err + } + if reservationID != "" { + _, err = tx.ExecContext(ctx, `UPDATE hwlab_billing_reservations SET status = 'completed', updated_at = now() WHERE id = $1`, reservationID) + if err != nil { + return nil, err + } + } + if err := tx.Commit(); err != nil { + return nil, err + } + return map[string]any{"recordId": recordID, "userId": userID, "serviceId": serviceID, "credits": credits, "balance": newBalance}, nil +} + +func (s *Server) adjustCredits(ctx context.Context, userID string, delta int64, kind, reason, idempotencyKey string, metadata map[string]any) (int64, error) { + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{}) + if err != nil { + return 0, err + } + defer tx.Rollback() + if idempotencyKey != "" { + var balance int64 + err := tx.QueryRowContext(ctx, `SELECT balance_after FROM hwlab_credit_ledger WHERE idempotency_key = $1`, idempotencyKey).Scan(&balance) + if err == nil { + return balance, tx.Commit() + } + if !errors.Is(err, sql.ErrNoRows) { + return 0, err + } + } + var balance int64 + if err := tx.QueryRowContext(ctx, `SELECT balance_credits FROM hwlab_credit_accounts WHERE user_id = $1 FOR UPDATE`, userID).Scan(&balance); err != nil { + return 0, err + } + newBalance := balance + delta + if newBalance < 0 { + return 0, errInsufficientCredits + } + _, err = tx.ExecContext(ctx, `UPDATE hwlab_credit_accounts SET balance_credits = $2, updated_at = now() WHERE user_id = $1`, userID, newBalance) + if err != nil { + return 0, err + } + _, err = tx.ExecContext(ctx, `INSERT INTO hwlab_credit_ledger (id, user_id, delta_credits, balance_after, kind, reason, idempotency_key, metadata) VALUES ($1, $2, $3, $4, $5, $6, nullif($7, ''), $8::jsonb)`, newID("led"), userID, delta, newBalance, kind, reason, idempotencyKey, jsonObject(metadata)) + if err != nil { + return 0, err + } + return newBalance, tx.Commit() +} + +func (s *Server) principalForBilling(ctx context.Context, r *http.Request, token, apiKey, userID string) (*Principal, error) { + if token == "" && apiKey == "" { + token = bearerToken(r) + } + if token != "" || apiKey != "" { + return s.principalFromToken(ctx, first(token, apiKey)) + } + userID = strings.TrimSpace(userID) + if userID == "" { + return nil, errors.New("billing subject is required") + } + var principal Principal + err := s.db.QueryRowContext(ctx, `SELECT id, email, username, role FROM hwlab_users WHERE id = $1 AND status = 'active'`, userID).Scan(&principal.UserID, &principal.Email, &principal.Username, &principal.Role) + if err != nil { + return nil, err + } + principal.AuthType = "internal-user-id" + principal.Scopes = []string{"api"} + return &principal, nil +} + +func (s *Server) principalFromToken(ctx context.Context, token string) (*Principal, error) { + token = strings.TrimSpace(token) + if token == "" { + return nil, errors.New("empty token") + } + hash := tokenHash(token) + var principal Principal + if strings.HasPrefix(token, "hwl_") { + var scopesRaw string + err := s.db.QueryRowContext(ctx, `SELECT u.id, u.email, u.username, u.role, k.id, k.scopes_json::text FROM hwlab_api_keys k JOIN hwlab_users u ON u.id = k.user_id WHERE k.key_hash = $1 AND k.status = 'active' AND k.revoked_at IS NULL AND u.status = 'active'`, hash).Scan(&principal.UserID, &principal.Email, &principal.Username, &principal.Role, &principal.KeyID, &scopesRaw) + if err != nil { + return nil, err + } + _ = json.Unmarshal([]byte(scopesRaw), &principal.Scopes) + if len(principal.Scopes) == 0 { + principal.Scopes = []string{"api"} + } + principal.AuthType = "api-key" + _, _ = s.db.ExecContext(ctx, `UPDATE hwlab_api_keys SET last_used_at = now() WHERE id = $1`, principal.KeyID) + return &principal, nil + } + err := s.db.QueryRowContext(ctx, `SELECT u.id, u.email, u.username, u.role FROM hwlab_user_sessions s JOIN hwlab_users u ON u.id = s.user_id WHERE s.token_hash = $1 AND s.revoked_at IS NULL AND s.expires_at > now() AND u.status = 'active'`, hash).Scan(&principal.UserID, &principal.Email, &principal.Username, &principal.Role) + if err != nil { + return nil, err + } + principal.AuthType = "session" + principal.Scopes = []string{"session"} + return &principal, nil +} + +func (s *Server) requirePrincipal(w http.ResponseWriter, r *http.Request) (*Principal, bool) { + if !s.databaseAvailable(w) { + return nil, false + } + principal, err := s.principalFromToken(r.Context(), bearerToken(r)) + if err != nil { + writeAPIError(w, http.StatusUnauthorized, "unauthorized", "valid bearer session or API key is required") + return nil, false + } + return principal, true +} + +func (s *Server) internalAllowed(w http.ResponseWriter, r *http.Request) bool { + if s.config.InternalToken == "" { + if s.config.RequireInternalToken { + writeAPIError(w, http.StatusServiceUnavailable, "internal_token_missing", "internal token is required but not configured") + return false + } + return true + } + candidate := strings.TrimSpace(r.Header.Get("X-HWLAB-Internal-Token")) + if candidate == "" { + candidate = bearerToken(r) + } + if subtle.ConstantTimeCompare([]byte(candidate), []byte(s.config.InternalToken)) != 1 { + writeAPIError(w, http.StatusUnauthorized, "internal_auth_failed", "valid internal token is required") + return false + } + return true +} + +func (s *Server) databaseAvailable(w http.ResponseWriter) bool { + if s.db != nil { + return true + } + writeAPIError(w, http.StatusServiceUnavailable, "state_backend_not_configured", "HWLAB_USER_BILLING_DB_URL must point to the PK01 external PostgreSQL authority") + return false +} + +func (s *Server) registrationAvailable(w http.ResponseWriter) bool { + if !s.RegistrationEnabled() { + writeAPIError(w, http.StatusForbidden, "registration_disabled", "registration is disabled") + return false + } + return s.databaseAvailable(w) +} + +func (s *Server) RegistrationEnabled() bool { + return s.config.RegistrationEnabled +} + +func (s *Server) accountBalance(ctx context.Context, userID string) (int64, int64, error) { + var balance, reserved int64 + err := s.db.QueryRowContext(ctx, `SELECT balance_credits, reserved_credits FROM hwlab_credit_accounts WHERE user_id = $1`, userID).Scan(&balance, &reserved) + return balance, reserved, err +} + +func (s *Server) estimateCredits(explicit, tokens int64) int64 { + if explicit > 0 { + return explicit + } + if tokens <= 0 { + return 1 + } + per := s.config.CreditPerThousandTok + if per <= 0 { + per = 1 + } + return int64(math.Ceil(float64(tokens*per) / 1000.0)) +} + +func decodeJSON(w http.ResponseWriter, r *http.Request, target any) bool { + defer r.Body.Close() + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(target); err != nil { + writeAPIError(w, http.StatusBadRequest, "invalid_json", "request body must be valid JSON") + return false + } + return true +} + +func writeAPIError(w http.ResponseWriter, status int, code, message string) { + writeJSON(w, status, map[string]any{"error": apiError{Code: code, Message: message}}) +} + +func writeJSON(w http.ResponseWriter, status int, payload any) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(payload) +} + +func bearerToken(r *http.Request) string { + value := strings.TrimSpace(r.Header.Get("Authorization")) + if strings.HasPrefix(strings.ToLower(value), "bearer ") { + return strings.TrimSpace(value[7:]) + } + return "" +} + +func normalizeUsername(value string) string { + value = strings.ToLower(strings.TrimSpace(value)) + value = strings.Map(func(r rune) rune { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' || r == '_' { + return r + } + return -1 + }, value) + if len(value) < 3 || len(value) > 64 { + return "" + } + return value +} + +func hashPassword(password string) (string, error) { + salt, err := randomBytes(16) + if err != nil { + return "", err + } + iterations := 210000 + digest := pbkdf2SHA256([]byte(password), salt, iterations, 32) + return fmt.Sprintf("pbkdf2_sha256$%d$%s$%s", iterations, b64(salt), b64(digest)), nil +} + +func verifyPassword(encoded, password string) bool { + parts := strings.Split(encoded, "$") + if len(parts) != 4 || parts[0] != "pbkdf2_sha256" { + return false + } + iterations, err := strconv.Atoi(parts[1]) + if err != nil || iterations <= 0 { + return false + } + salt, err := base64.RawURLEncoding.DecodeString(parts[2]) + if err != nil { + return false + } + expected, err := base64.RawURLEncoding.DecodeString(parts[3]) + if err != nil { + return false + } + actual := pbkdf2SHA256([]byte(password), salt, iterations, len(expected)) + return subtle.ConstantTimeCompare(actual, expected) == 1 +} + +func pbkdf2SHA256(password, salt []byte, iterations, keyLen int) []byte { + hLen := sha256.Size + numBlocks := (keyLen + hLen - 1) / hLen + var output []byte + for block := 1; block <= numBlocks; block++ { + u := pbkdf2Block(password, salt, block) + t := append([]byte(nil), u...) + for i := 1; i < iterations; i++ { + u = hmacSHA256(password, u) + for j := range t { + t[j] ^= u[j] + } + } + output = append(output, t...) + } + return output[:keyLen] +} + +func pbkdf2Block(password, salt []byte, block int) []byte { + buf := make([]byte, len(salt)+4) + copy(buf, salt) + binary.BigEndian.PutUint32(buf[len(salt):], uint32(block)) + return hmacSHA256(password, buf) +} + +func hmacSHA256(key, data []byte) []byte { + h := hmac.New(sha256.New, key) + _, _ = h.Write(data) + return h.Sum(nil) +} + +func tokenHash(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} + +func newID(prefix string) string { + return prefix + "_" + randomToken(16) +} + +func randomToken(size int) string { + buf, err := randomBytes(size) + if err != nil { + panic(err) + } + return b64(buf) +} + +func randomBytes(size int) ([]byte, error) { + buf := make([]byte, size) + _, err := rand.Read(buf) + return buf, err +} + +func b64(value []byte) string { + return base64.RawURLEncoding.EncodeToString(value) +} + +func jsonObject(value map[string]any) string { + if value == nil { + return "{}" + } + encoded, err := json.Marshal(value) + if err != nil { + return "{}" + } + return string(encoded) +} + +func parseInt64(value string, fallback int64) int64 { + if strings.TrimSpace(value) == "" { + return fallback + } + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return fallback + } + return parsed +} + +func parseDurationHours(value string, fallbackHours int64) time.Duration { + return time.Duration(parseInt64(value, fallbackHours)) * time.Hour +} + +func parseDurationMinutes(value string, fallbackMinutes int64) time.Duration { + return time.Duration(parseInt64(value, fallbackMinutes)) * time.Minute +} + +func first(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} + +func nonNegative(value int64) int64 { + if value < 0 { + return 0 + } + return value +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func isDuplicate(err error) bool { + return err != nil && strings.Contains(strings.ToLower(err.Error()), "duplicate") +} diff --git a/scripts/artifact-publish.mjs b/scripts/artifact-publish.mjs index 7ed4a323..3416812a 100644 --- a/scripts/artifact-publish.mjs +++ b/scripts/artifact-publish.mjs @@ -62,6 +62,7 @@ const ciArtifactIdentityEnvNames = [ const servicePorts = new Map([ ["hwlab-cloud-api", 6667], + ["hwlab-user-billing", 6670], ["hwlab-cloud-web", 8080], ["hwlab-gateway", 7001], ["hwlab-edge-proxy", 6667], @@ -803,8 +804,8 @@ function repoLabelFromRemote(remoteUrl) { } } -async function resolveServices(serviceIds, catalog) { - return resolveDevArtifactServices(repoRoot, serviceIds, SERVICE_IDS, catalog); +async function resolveServices(serviceIds, catalog, allowedServiceIds = SERVICE_IDS) { + return resolveDevArtifactServices(repoRoot, serviceIds, allowedServiceIds, catalog); } function dockerfile(baseImage, port, labels = []) { @@ -863,27 +864,51 @@ function dockerfile(baseImage, port, labels = []) { ].join("\n"); } -function envReuseDockerfile(baseImage, labels = [], recipe) { +function envReuseDockerfile(baseImage, labels = [], recipe, service = null) { const normalizedRecipe = normalizeEnvReuseRecipe(recipe); const runtimeRoot = normalizedRecipe.runtimeNodeModulesPath.replace(/\/node_modules$/u, ""); const downloadEnvPrefix = shellEnvPrefix(downloadEnvPairs(normalizedRecipe.downloadStack)); const npmConfigScript = npmDownloadConfigScript(normalizedRecipe.downloadStack); + const goService = service?.runtimeKind === "go-service" || service?.artifactKind === "go-service"; + const goCacheEnabled = Boolean(goService && normalizedRecipe.runtimeGoModCachePath && normalizedRecipe.runtimeGoBuildCachePath); + const goProxy = normalizedRecipe.downloadStack.goProxy || "https://proxy.golang.org,direct"; + const goEnv = goCacheEnabled ? [ + `ENV GOMODCACHE=${normalizedRecipe.runtimeGoModCachePath}`, + `ENV GOCACHE=${normalizedRecipe.runtimeGoBuildCachePath}`, + `ENV GOPROXY=${goProxy}`, + `ENV GOTOOLCHAIN=${normalizedRecipe.goToolchain || "local"}` + ] : []; + const goBuildEnvPrefix = goCacheEnabled ? shellEnvPrefix([ + ["GOMODCACHE", normalizedRecipe.runtimeGoModCachePath], + ["GOCACHE", normalizedRecipe.runtimeGoBuildCachePath], + ["GOPROXY", goProxy], + ["GOTOOLCHAIN", normalizedRecipe.goToolchain || "local"] + ]) : ""; + const goDownloadScript = goCacheEnabled + ? `RUN ${downloadEnvPrefix}${goBuildEnvPrefix}set -eu; mkdir -p ${shellQuote(normalizedRecipe.runtimeGoModCachePath)} ${shellQuote(normalizedRecipe.runtimeGoBuildCachePath)}; go mod download` + : null; + const goReadiness = goCacheEnabled + ? `; mkdir -p ${shellQuote(normalizedRecipe.runtimeGoModCachePath)} ${shellQuote(normalizedRecipe.runtimeGoBuildCachePath)}; command -v go >/dev/null; go version >/tmp/hwlab-env-go-version.txt` + : ""; const downloadDependencyScript = `set -eu; ${npmConfigScript}; if [ -f package-lock.json ]; then npm ci --omit=dev --include=optional --ignore-scripts; else npm install --omit=dev --include=optional --ignore-scripts; fi; if command -v bun >/dev/null 2>&1; then bun_path="$(command -v bun)"; if [ "$bun_path" != "/usr/local/bin/bun" ]; then ln -sf "$bun_path" /usr/local/bin/bun; fi; fi; if [ ! -x /usr/local/bin/bun ]; then bun_pkg=""; case "$(uname -m)" in x86_64|amd64) bun_pkg="${normalizedRecipe.downloadStack.bunPackages.x64}@${normalizedRecipe.bunVersion}" ;; aarch64|arm64) bun_pkg="${normalizedRecipe.downloadStack.bunPackages.arm64}@${normalizedRecipe.bunVersion}" ;; *) echo "unsupported bun architecture: $(uname -m)" >&2; false ;; esac; npm install --no-save --omit=dev --include=optional --ignore-scripts "$bun_pkg"; bun_bin="node_modules/\${bun_pkg%@*}/bin/bun"; test -x "$bun_bin"; ln -sf "${runtimeRoot}/$bun_bin" /usr/local/bin/bun; fi`; const osPackageScript = `set -eu; apt-get update; apt-get install -y --no-install-recommends ${normalizedRecipe.osPackages.map(shellQuote).join(" ")}; rm -rf /var/lib/apt/lists/*`; const aliasScript = normalizedRecipe.hwpodAliases.map((alias) => `printf '%s\\n' '#!/usr/bin/env sh' 'exec node /workspace/hwlab-boot/repo/scripts/run-bun.mjs /workspace/hwlab-boot/repo/${alias.script} \"$@\"' > /usr/local/bin/${alias.command}; chmod 755 /usr/local/bin/${alias.command}`).join("; "); - const readinessScript = `set -eu; command -v node >/dev/null; command -v npm >/dev/null; command -v git >/dev/null; command -v sh >/dev/null; test -d ${shellQuote(normalizedRecipe.runtimeNodeModulesPath)}; test -x /usr/local/bin/bun; /usr/local/bin/bun --version >/tmp/hwlab-env-bun-version.txt; ${aliasScript}`; + const readinessScript = `set -eu; command -v node >/dev/null; command -v npm >/dev/null; command -v git >/dev/null; command -v sh >/dev/null; test -d ${shellQuote(normalizedRecipe.runtimeNodeModulesPath)}; test -x /usr/local/bin/bun; /usr/local/bin/bun --version >/tmp/hwlab-env-bun-version.txt${goReadiness}; ${aliasScript}`; return [ `FROM ${baseImage}`, `WORKDIR ${runtimeRoot}`, ...labels.map(([name, value]) => `LABEL ${name}=${dockerfileQuote(value)}`), "COPY package.json ./package.json", "COPY package-lock.json* ./", + ...(goCacheEnabled ? ["COPY go.mod ./go.mod", "COPY go.sum* ./"] : []), `RUN ${downloadEnvPrefix}${osPackageScript}`, `RUN ${downloadEnvPrefix}${downloadDependencyScript}`, + ...(goDownloadScript ? [goDownloadScript] : []), `COPY ${normalizedRecipe.launcherPath} /usr/local/bin/hwlab-env-reuse-launcher.ts`, `RUN ${readinessScript}`, "ENV HWLAB_RUNTIME_MODE=env-reuse-git-mirror-checkout", `ENV HWLAB_RUNTIME_NODE_MODULES=${normalizedRecipe.runtimeNodeModulesPath}`, + ...goEnv, "ENV PATH=/usr/local/bin:$PATH", "CMD [\"/usr/local/bin/bun\", \"/usr/local/bin/hwlab-env-reuse-launcher.ts\"]", "" @@ -897,6 +922,9 @@ function normalizeEnvReuseRecipe(recipe) { bunVersion: String(recipe.bunVersion ?? "").trim(), launcherPath: normalizeRepoPath(recipe.launcherPath), runtimeNodeModulesPath: normalizeAbsolutePath(recipe.runtimeNodeModulesPath), + runtimeGoModCachePath: normalizeAbsolutePath(recipe.runtimeGoModCachePath), + runtimeGoBuildCachePath: normalizeAbsolutePath(recipe.runtimeGoBuildCachePath), + goToolchain: String(recipe.goToolchain ?? "").trim(), downloadStack: normalizeDownloadStack(recipe.downloadStack), hwpodAliases: normalizeHwpodAliases(recipe.hwpodAliases) }; @@ -917,6 +945,7 @@ function normalizeDownloadStack(value) { noProxy: normalizeStringList(value.noProxy), npmRegistry: normalizeOptionalHttpUrl(value.npmRegistry), npmFetchTimeoutMs: Number(value.npmFetchTimeoutMs), + goProxy: String(value.goProxy ?? "").trim(), bunPackages: { x64: normalizePackageName(value.bunPackages?.x64), arm64: normalizePackageName(value.bunPackages?.arm64) @@ -1242,7 +1271,7 @@ async function buildEnvReuseServiceWithBuildkit({ args, service, ref, buildCreat const buildkitAddrArgs = args.buildkitAddr ? ["--addr", args.buildkitAddr] : []; const startedAt = Date.now(); try { - await writeFile(dockerfilePath, envReuseDockerfile(args.baseImage, labels, service.envReuseRecipe)); + await writeFile(dockerfilePath, envReuseDockerfile(args.baseImage, labels, service.envReuseRecipe, service)); const argsList = [ ...buildkitAddrArgs, "build", @@ -2277,7 +2306,8 @@ async function main() { baseImage: args.baseImage ?? undefined, ...(args.servicesExplicit ? { services: args.services } : {}) }); - const services = enrichServicesWithCiPlan(await resolveServices(args.services, catalog), ciPlan); + const allowedServiceIds = isRuntimeArtifactLane(args.lane) ? runtimeLaneServiceIdsFromDeploy(deployManifest, args.lane) : SERVICE_IDS; + const services = enrichServicesWithCiPlan(await resolveServices(args.services, catalog, allowedServiceIds), ciPlan); const catalogByServiceId = new Map((catalog?.services ?? []).map((service) => [service.serviceId, service])); const affectedServiceIds = new Set(ciPlan?.affectedServices ?? []); const plannedBuildServices = Array.isArray(ciPlan?.buildServices) ? ciPlan.buildServices : null; diff --git a/scripts/gitops-render.mjs b/scripts/gitops-render.mjs index ed488a1d..d27f81ca 100644 --- a/scripts/gitops-render.mjs +++ b/scripts/gitops-render.mjs @@ -642,6 +642,10 @@ function v02ObservableService(deploy, serviceId) { return v02ObservableServicesForDeploy(deploy).find((item) => item.serviceId === serviceId) ?? null; } +function runtimeLaneObservableService(deploy, lane, serviceId) { + return runtimeLaneObservableServicesForDeploy(deploy, lane).find((item) => item.serviceId === serviceId) ?? null; +} + function runtimeLaneObservableServiceIdsForDeploy(deploy, lane) { return new Set(runtimeLaneObservableServicesForDeploy(deploy, lane).map((item) => item.serviceId)); } @@ -943,9 +947,9 @@ function v02MetricsSidecarAnnotations(metricsSidecarSha256) { return metricsSidecarSha256 ? { "hwlab.pikastech.local/metrics-sidecar-sha256": metricsSidecarSha256 } : {}; } -function v02MetricsSidecarContainer({ deploy, serviceId, namespace, gitopsTarget }) { - const service = v02ObservableService(deploy, serviceId); - assert.ok(service, `unknown v0.2 observable service ${serviceId}`); +function v02MetricsSidecarContainer({ deploy, serviceId, namespace, gitopsTarget, profile = "v02" }) { + const service = runtimeLaneObservableService(deploy, profile, serviceId); + assert.ok(service, `unknown ${profile} observable service ${serviceId}`); const extraMetricsEnv = serviceId === "hwlab-cloud-api" ? [ { name: "HWLAB_METRICS_EXTRA_URL", value: `http://127.0.0.1:${service.port}/v1/web-performance/metrics` }, diff --git a/scripts/refresh-artifact-catalog.mjs b/scripts/refresh-artifact-catalog.mjs index 7495165f..957abcc0 100644 --- a/scripts/refresh-artifact-catalog.mjs +++ b/scripts/refresh-artifact-catalog.mjs @@ -431,7 +431,7 @@ function refreshDocuments({ deploy, catalog, target, publishRecords, serviceInve const records = publishRecords?.records ?? null; const requiredIds = new Set(publishRecords?.requiredServiceIds ?? serviceInventory.requiredServiceIds); const catalogByService = new Map(catalog.services.map((service) => [service.serviceId, service])); - const deployByService = new Map(deploy.services.map((service) => [service.serviceId, service])); + const deployByService = deployServiceMapForLane(deploy, lane, serviceIds); const inventoryByService = new Map(serviceInventory.services.map((service) => [service.serviceId, service])); const refreshedServices = []; const nextCatalogServices = []; @@ -539,6 +539,17 @@ function refreshDocuments({ deploy, catalog, target, publishRecords, serviceInve return refreshedServices; } +function deployServiceMapForLane(deploy, lane, serviceIds) { + const base = new Map((deploy.services ?? []).map((service) => [service.serviceId, service])); + if (!isRuntimeArtifactLane(lane)) return base; + const overlay = new Map((laneDeployConfig(deploy, lane)?.services ?? []).map((service) => [service.serviceId, service])); + return new Map(serviceIds.map((serviceId) => [serviceId, { + serviceId, + ...(base.get(serviceId) ?? {}), + ...(overlay.get(serviceId) ?? {}) + }])); +} + function publishReportProvenance(publishReport, fallbackPath) { const producer = publishReport?.artifactPublish?.producer; if (producer?.kind && producer?.runId) return `${producer.kind}:${producer.runId}`; @@ -566,7 +577,7 @@ async function main() { const serviceIds = serviceIdsForLane(args.lane, deploy); const publishRecords = publishReport ? publishRecordsFromReport(publishReport, target, serviceIds) : null; const serviceInventory = publishReport?.artifactPublish?.serviceInventory ?? serviceInventoryFromServices( - await resolveDevArtifactServices(repoRoot, serviceIds, SERVICE_IDS) + await resolveDevArtifactServices(repoRoot, serviceIds, serviceIds) ); const services = refreshDocuments({ deploy, diff --git a/scripts/src/ci-plan-lib.mjs b/scripts/src/ci-plan-lib.mjs index 9d28d93b..3e4e8e24 100644 --- a/scripts/src/ci-plan-lib.mjs +++ b/scripts/src/ci-plan-lib.mjs @@ -18,6 +18,11 @@ export const DEFAULT_RUNTIME_DEP_PATHS = Object.freeze([ "package-lock.json" ]); +export const GO_RUNTIME_DEP_PATHS = Object.freeze([ + "go.mod", + "go.sum" +]); + export const DEFAULT_RUNTIME_PACKAGE_FIELDS = Object.freeze([ "dependencies", "optionalDependencies", @@ -75,9 +80,11 @@ export async function createCiPlan(options = {}) { const componentModels = componentModelsForServices(serviceIdResolution.serviceIds, laneConfig, lane); const envReuseServices = enabledEnvReuseServices(deployJson, lane, serviceIdResolution.serviceIds); const globalChange = classifyGlobalChange(normalizedChangedPaths); + const hasGoService = componentModels.some((model) => model.runtimeKind === "go-service"); const dockerfileHash = await hashGitPaths(repoRoot, targetRef, [ ...envReuseRecipe.additionalEnvPaths, - ...DEFAULT_RUNTIME_DEP_PATHS + ...DEFAULT_RUNTIME_DEP_PATHS, + ...(hasGoService ? GO_RUNTIME_DEP_PATHS : []) ], { semanticPackageJson: true }); const catalogByServiceId = new Map((artifactCatalog?.services ?? []).map((service) => [service.serviceId, service])); @@ -297,12 +304,20 @@ export function componentModelsForServices(serviceIds, laneConfig = null, lane = healthPort: declaration.healthPort, componentPaths: uniqueSorted(declaration.componentPaths.map(normalizeRepoPath)), sharedPaths: uniqueSorted(DEFAULT_SHARED_RUNTIME_PATHS.map(normalizeRepoPath)), - runtimeDeps: uniqueSorted(DEFAULT_RUNTIME_DEP_PATHS.map(normalizeRepoPath)), + runtimeDeps: runtimeDependencyPathsForDeclaration(declaration), buildSystemPaths: uniqueSorted(envReuseRecipeForLaneConfig(laneConfig, lane).additionalEnvPaths) }; }); } +function runtimeDependencyPathsForDeclaration(declaration) { + const paths = [...DEFAULT_RUNTIME_DEP_PATHS]; + if (declaration.runtimeKind === "go-service" || declaration.artifactKind === "go-service") { + paths.push(...GO_RUNTIME_DEP_PATHS); + } + return uniqueSorted(paths.map(normalizeRepoPath)); +} + function serviceDeclarationFor(laneConfig, serviceId, lane) { const configured = laneConfig?.serviceDeclarations?.[serviceId]; const runtimeKind = requireDeclarationString(configured?.runtimeKind, lane, `serviceDeclarations.${serviceId}.runtimeKind`); @@ -351,6 +366,9 @@ function envReuseRecipeForLaneConfig(laneConfig, lane = "v02") { const bunVersion = requireDeclarationString(configured.bunVersion, lane, "envRecipe.bunVersion"); const launcherPath = normalizeRepoPath(requireDeclarationString(configured.launcherPath, lane, "envRecipe.launcherPath")); const runtimeNodeModulesPath = normalizeAbsolutePath(configured.runtimeNodeModulesPath); + const runtimeGoModCachePath = normalizeAbsolutePath(configured.runtimeGoModCachePath); + const runtimeGoBuildCachePath = normalizeAbsolutePath(configured.runtimeGoBuildCachePath); + const goToolchain = normalizeDeclarationString(configured.goToolchain); if (!runtimeNodeModulesPath) throw new Error(`deploy.lanes.${lane}.envRecipe.runtimeNodeModulesPath must be absolute`); const additionalEnvPaths = uniqueSorted(normalizeStringList(configured.additionalEnvPaths, []).map(normalizeRepoPath)); const launcherInputPaths = uniqueSorted([ @@ -367,6 +385,9 @@ function envReuseRecipeForLaneConfig(laneConfig, lane = "v02") { bunVersion, launcherPath, runtimeNodeModulesPath, + runtimeGoModCachePath, + runtimeGoBuildCachePath, + goToolchain, additionalEnvPaths, launcherInputPaths, downloadStack, @@ -376,6 +397,9 @@ function envReuseRecipeForLaneConfig(laneConfig, lane = "v02") { bunVersion, launcherPath, runtimeNodeModulesPath, + runtimeGoModCachePath, + runtimeGoBuildCachePath, + goToolchain, additionalEnvPaths, downloadStack, hwpodAliases @@ -389,6 +413,7 @@ function normalizeDownloadStack(value, lane) { const httpProxy = normalizeOptionalHttpUrl(configured.httpProxy, `deploy.lanes.${lane}.envRecipe.downloadStack.httpProxy`); const httpsProxy = normalizeOptionalHttpUrl(configured.httpsProxy, `deploy.lanes.${lane}.envRecipe.downloadStack.httpsProxy`); const npmRegistry = normalizeOptionalHttpUrl(configured.npmRegistry, `deploy.lanes.${lane}.envRecipe.downloadStack.npmRegistry`); + const goProxy = normalizeDeclarationString(configured.goProxy); const npmFetchTimeoutMs = Number(configured.npmFetchTimeoutMs); const bunPackages = configured.bunPackages && typeof configured.bunPackages === "object" ? configured.bunPackages : null; const x64 = normalizePackageName(bunPackages?.x64); @@ -404,6 +429,7 @@ function normalizeDownloadStack(value, lane) { noProxy: normalizeStringList(configured.noProxy, []), npmRegistry, npmFetchTimeoutMs, + goProxy, bunPackages: { x64, arm64 } }; } diff --git a/scripts/src/dev-artifact-services.mjs b/scripts/src/dev-artifact-services.mjs index 76799856..c73a05ce 100644 --- a/scripts/src/dev-artifact-services.mjs +++ b/scripts/src/dev-artifact-services.mjs @@ -85,6 +85,17 @@ export async function resolveDevArtifactService(repoRoot, serviceId, catalog = n }); } + const goCmdPath = `cmd/${serviceId}/main.go`; + if (await pathExists(repoRoot, goCmdPath)) { + return withServicePublishPolicy({ + serviceId, + runtimeKind: "go-service", + entrypoint: goCmdPath, + implementationState: "repo-entrypoint", + sourceState + }); + } + if (serviceId === "hwlab-cloud-web") { return withServicePublishPolicy({ serviceId,