diff --git a/deploy/deploy.yaml b/deploy/deploy.yaml index c716a451..1ad90164 100644 --- a/deploy/deploy.yaml +++ b/deploy/deploy.yaml @@ -294,7 +294,7 @@ lanes: runtimeKind: go-service entrypoint: cmd/hwlab-user-billing/main.go artifactKind: go-service - healthPath: /health/live + healthPath: /health/ready healthPort: 6670 componentPaths: - go.mod @@ -404,6 +404,8 @@ lanes: HWLAB_BOOTSTRAP_ADMIN_API_KEY_ID: key_master_server_admin HWLAB_BOOTSTRAP_ADMIN_API_KEY: secretRef:hwlab-v03-master-server-admin-api-key/api-key HWLAB_USER_BILLING_URL: http://hwlab-user-billing.hwlab-v03.svc.cluster.local:6670 + HWLAB_USER_BILLING_LOGIN_RETRY_ATTEMPTS: "2" + HWLAB_USER_BILLING_LOGIN_RETRY_DELAY_MS: "250" HWLAB_USER_BILLING_CODE_AGENT_ENABLED: "1" HWLAB_USER_BILLING_CODE_AGENT_ESTIMATED_CREDITS: "1" HWLAB_USER_BILLING_CODE_AGENT_USED_CREDITS: "1" diff --git a/deploy/k8s/base/workloads.yaml b/deploy/k8s/base/workloads.yaml index e58bf499..746b1fed 100644 --- a/deploy/k8s/base/workloads.yaml +++ b/deploy/k8s/base/workloads.yaml @@ -470,7 +470,7 @@ ], "readinessProbe": { "httpGet": { - "path": "/health/live", + "path": "/health/ready", "port": "http" } }, diff --git a/internal/cloud/access-control.ts b/internal/cloud/access-control.ts index 03d93ff1..9a968ffd 100644 --- a/internal/cloud/access-control.ts +++ b/internal/cloud/access-control.ts @@ -807,19 +807,21 @@ class AccessController { const body = await jsonBody(request); const userBilling = await this.userBillingLogin(body, request, response); if (userBilling?.handled) return; - const user = await this.store.findUserByUsername(requiredText(body.username, "username")); - if (!user || user.status !== "active" || !verifyPassword(user.passwordHash, requiredText(body.password, "password"))) { + const password = requiredText(body.password, "password"); + const username = textOr(body.username, ""); + if (!username) { + if (userBilling?.retryable) return sendJson(response, userBilling.status ?? 503, userBillingDependencyErrorPayload(userBilling.result, "user_billing_login_failed", "user-billing login failed", userBilling.status ?? 503)); + requiredText(body.username, "username"); + } + const user = await this.store.findUserByUsername(username); + if (user && (user.status !== "active" || !verifyPassword(user.passwordHash, password))) { return sendJson(response, 401, errorPayload("invalid_credentials", "Username or password is invalid", 401)); } - const token = randomBytes(32).toString("base64url"); - const now = this.now(); - const session = await this.store.createSession({ - userId: user.id, - tokenHash: sha256(token), - now, - expiresAt: new Date(Date.parse(now) + SESSION_MAX_AGE_SECONDS * 1000).toISOString() - }); - await this.store.ensureDefaultApiKeyForUser?.({ userId: user.id, now }) ?? null; + if (!user) { + if (userBilling?.retryable) return sendJson(response, userBilling.status ?? 503, userBillingDependencyErrorPayload(userBilling.result, "user_billing_login_failed", "user-billing login failed", userBilling.status ?? 503)); + return sendJson(response, 401, errorPayload("invalid_credentials", "Username or password is invalid", 401)); + } + const { token, session } = await this.issueWebSessionForActor(user); setSessionCookie(response, token, SESSION_MAX_AGE_SECONDS, request, this.env); return sendJson(response, 200, { authenticated: true, @@ -866,9 +868,10 @@ class AccessController { const result = await this.issueUserBillingWebSession({ username, password: requiredText(body.password, "password") }, request, response); if (!result.ok) { if (result.status === 401) return { handled: false }; + if (isRetryableUserBillingResult(result)) return { handled: false, retryable: true, status: result.status ?? 503, result }; return { handled: true, - response: sendJson(response, result.status ?? 502, errorPayload(result.error?.code ?? "user_billing_login_failed", result.error?.message ?? "user-billing login failed", result.status ?? 502)) + response: sendJson(response, result.status ?? 502, userBillingDependencyErrorPayload(result, "user_billing_login_failed", "user-billing login failed", result.status ?? 502)) }; } sendJson(response, 200, { @@ -886,7 +889,15 @@ class AccessController { async issueUserBillingWebSession({ username, password }, request, response) { const login = await this.userBilling.login({ username, email: username, password }); if (!login.ok || !login.body?.token) { - return { ok: false, status: login.status, error: login.error }; + return { + ok: false, + status: login.status, + error: login.error, + retryable: login.retryable, + retryCount: login.retryCount, + transientObserved: login.transientObserved, + valuesRedacted: login.valuesRedacted + }; } const token = textOr(login.body.token, ""); const auth = await this.authenticateUserBillingToken(token); @@ -2922,6 +2933,36 @@ function shouldUseSecureSessionCookie(request, env = process.env) { return false; } function errorPayload(code, message, status) { return { ok: false, status, error: { code, message } }; } + +function userBillingDependencyErrorPayload(result, fallbackCode, fallbackMessage, fallbackStatus) { + const status = result?.status ?? fallbackStatus; + const payload = errorPayload(result?.error?.code ?? fallbackCode, result?.error?.message ?? fallbackMessage, status); + const retryable = isRetryableUserBillingResult(result); + payload.error.retryable = retryable; + payload.error.dependency = { + serviceId: "hwlab-user-billing", + retryCount: nonNegativeResultInteger(result?.retryCount), + transientObserved: Boolean(result?.transientObserved ?? retryable), + valuesRedacted: true + }; + return payload; +} + +function isRetryableUserBillingResult(result) { + if (!result || result.ok) return false; + if (result.retryable === true) return true; + const code = textOr(result.error?.code, ""); + if (code === "user_billing_not_configured") return false; + if (code === "user_billing_unreachable") return true; + const status = Number(result.status ?? 0); + if ([500, 502, 503, 504].includes(status)) return true; + return false; +} + +function nonNegativeResultInteger(value) { + const parsed = Number.parseInt(String(value ?? ""), 10); + return Number.isInteger(parsed) && parsed >= 0 ? parsed : 0; +} function positiveInteger(value, fallback = 0) { const number = Number(value); return Number.isFinite(number) && number > 0 ? Math.floor(number) : fallback; } async function resolveBeforeDeadline(promise, timeoutMs) { /** @type {ReturnType | null} */ diff --git a/internal/cloud/user-billing-client.test.ts b/internal/cloud/user-billing-client.test.ts new file mode 100644 index 00000000..1beafbe2 --- /dev/null +++ b/internal/cloud/user-billing-client.test.ts @@ -0,0 +1,64 @@ +import assert from "node:assert/strict"; +import { test } from "bun:test"; + +import { createUserBillingClient } from "./user-billing-client.ts"; + +test("user-billing login retries transient dependency failures only", async () => { + const calls = []; + const client = createUserBillingClient({ + env: { + HWLAB_USER_BILLING_URL: "http://user-billing.test", + HWLAB_USER_BILLING_LOGIN_RETRY_ATTEMPTS: "2", + HWLAB_USER_BILLING_LOGIN_RETRY_DELAY_MS: "1" + }, + fetchImpl: async (url, init) => { + calls.push({ url: String(url), method: init?.method }); + if (calls.length === 1) throw new Error("The operation timed out."); + return new Response(JSON.stringify({ token: "hws_retry_ok", tokenType: "Bearer" }), { status: 200, headers: { "content-type": "application/json" } }); + } + }); + + const result = await client.login({ username: "admin", password: "redacted" }); + assert.equal(result.ok, true); + assert.equal(result.retryCount, 1); + assert.equal(result.transientObserved, true); + assert.equal(calls.length, 2); + + const authFailures = []; + const invalidClient = createUserBillingClient({ + env: { + HWLAB_USER_BILLING_URL: "http://user-billing.test", + HWLAB_USER_BILLING_LOGIN_RETRY_ATTEMPTS: "2", + HWLAB_USER_BILLING_LOGIN_RETRY_DELAY_MS: "1" + }, + fetchImpl: async (url, init) => { + authFailures.push({ url: String(url), method: init?.method }); + return new Response(JSON.stringify({ error: { code: "invalid_credentials", message: "invalid email, username or password" } }), { status: 401, headers: { "content-type": "application/json" } }); + } + }); + + const invalid = await invalidClient.login({ username: "admin", password: "wrong" }); + assert.equal(invalid.ok, false); + assert.equal(invalid.status, 401); + assert.equal(invalid.error.code, "invalid_credentials"); + assert.equal(authFailures.length, 1); + + const unexpectedCalls = []; + const unconfiguredClient = createUserBillingClient({ + env: { + HWLAB_USER_BILLING_LOGIN_RETRY_ATTEMPTS: "2", + HWLAB_USER_BILLING_LOGIN_RETRY_DELAY_MS: "1" + }, + fetchImpl: async (url, init) => { + unexpectedCalls.push({ url: String(url), method: init?.method }); + throw new Error("fetch should not run without a user-billing URL"); + } + }); + + const unconfigured = await unconfiguredClient.login({ username: "admin", password: "redacted" }); + assert.equal(unconfigured.ok, false); + assert.equal(unconfigured.status, 503); + assert.equal(unconfigured.error.code, "user_billing_not_configured"); + assert.equal(unconfigured.retryable, undefined); + assert.equal(unexpectedCalls.length, 0); +}); diff --git a/internal/cloud/user-billing-client.ts b/internal/cloud/user-billing-client.ts index d383a69b..c184fba9 100644 --- a/internal/cloud/user-billing-client.ts +++ b/internal/cloud/user-billing-client.ts @@ -1,4 +1,7 @@ const DEFAULT_TIMEOUT_MS = 5000; +const DEFAULT_LOGIN_RETRY_ATTEMPTS = 2; +const DEFAULT_LOGIN_RETRY_DELAY_MS = 250; +const NON_RETRYABLE_USER_BILLING_ERROR_CODES = new Set(["user_billing_not_configured"]); export function createUserBillingClient({ env = process.env, fetchImpl = fetch } = {}) { const baseUrl = normalizeBaseUrl(firstText( @@ -8,8 +11,34 @@ export function createUserBillingClient({ env = process.env, fetchImpl = fetch } )); const internalToken = text(env.HWLAB_USER_BILLING_INTERNAL_TOKEN); const timeoutMs = positiveInteger(env.HWLAB_USER_BILLING_TIMEOUT_MS, DEFAULT_TIMEOUT_MS); + const loginRetryAttempts = nonNegativeInteger(env.HWLAB_USER_BILLING_LOGIN_RETRY_ATTEMPTS, DEFAULT_LOGIN_RETRY_ATTEMPTS); + const loginRetryDelayMs = positiveInteger(env.HWLAB_USER_BILLING_LOGIN_RETRY_DELAY_MS, DEFAULT_LOGIN_RETRY_DELAY_MS); async function requestJson(path, { method = "GET", body = null, bearerToken = "" } = {}) { + return requestJsonWithRetry(path, { method, body, bearerToken }); + } + + async function requestJsonWithRetry(path, { method = "GET", body = null, bearerToken = "", retry = null } = {}) { + const retryAttempts = nonNegativeInteger(retry?.attempts, 0); + const retryDelayMs = positiveInteger(retry?.delayMs, 0); + let retryCount = 0; + let transientObserved = false; + for (;;) { + const result = await requestJsonOnce(path, { method, body, bearerToken }); + const retryable = typeof retry?.shouldRetry === "function" ? retry.shouldRetry(result) : false; + if (!retryable || retryCount >= retryAttempts) { + if (retryCount > 0 || transientObserved || retryable) { + return { ...result, retryable, retryCount, transientObserved: transientObserved || retryable, valuesRedacted: true }; + } + return result; + } + retryCount += 1; + transientObserved = true; + if (retryDelayMs > 0) await sleep(retryDelayMs * retryCount); + } + } + + async function requestJsonOnce(path, { method = "GET", body = null, bearerToken = "" } = {}) { if (!baseUrl) { return { ok: false, status: 503, error: { code: "user_billing_not_configured", message: "HWLAB user-billing URL is not configured" } }; } @@ -94,7 +123,11 @@ export function createUserBillingClient({ env = process.env, fetchImpl = fetch } return publicPost("/v1/auth/register", body); }, async login(body = {}) { - return publicPost("/v1/auth/login", body); + return requestJsonWithRetry("/v1/auth/login", { + method: "POST", + body, + retry: { attempts: loginRetryAttempts, delayMs: loginRetryDelayMs, shouldRetry: isRetryableUserBillingResult } + }); }, async logout(token) { return requestJson("/v1/auth/logout", { method: "POST", body: {}, bearerToken: token }); @@ -245,6 +278,25 @@ function positiveInteger(value, fallback) { return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; } +function nonNegativeInteger(value, fallback) { + const parsed = Number.parseInt(String(value ?? ""), 10); + return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback; +} + +function isRetryableUserBillingResult(result) { + if (!result || result.ok) return false; + const code = text(result.error?.code); + if (NON_RETRYABLE_USER_BILLING_ERROR_CODES.has(code)) return false; + if (code === "user_billing_unreachable") return true; + const status = Number(result.status ?? 0); + if ([500, 502, 503, 504].includes(status)) return true; + return false; +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, Math.max(0, Number(ms) || 0))); +} + function firstText(...values) { for (const value of values) { const current = text(value); diff --git a/internal/cloud/user-billing-integration.test.ts b/internal/cloud/user-billing-integration.test.ts index 1950628d..b6d5ce37 100644 --- a/internal/cloud/user-billing-integration.test.ts +++ b/internal/cloud/user-billing-integration.test.ts @@ -939,6 +939,68 @@ test("cloud api bridges user-billing register/login into HttpOnly web session co } }); +test("cloud api falls back to local bootstrap login when user-billing login is transient", async () => { + const calls = []; + const userBillingClient = { + configured: true, + async login(body = {}) { + calls.push({ op: "login", username: body.username, passwordSeen: Boolean(body.password) }); + return { + ok: false, + status: 503, + retryable: true, + retryCount: 2, + transientObserved: true, + error: { code: "user_billing_unreachable", message: "The operation timed out." } + }; + } + }; + const server = createCloudApiServer({ + env: { + HWLAB_ACCESS_CONTROL_REQUIRED: "1", + HWLAB_BOOTSTRAP_ADMIN_USERNAME: "admin", + HWLAB_BOOTSTRAP_ADMIN_PASSWORD: "local-admin-password-1353" + }, + userBillingClient + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const { port } = server.address(); + const login = await fetch(`http://127.0.0.1:${port}/auth/login`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ username: "admin", password: "local-admin-password-1353" }) + }); + assert.equal(login.status, 200); + assert.match(login.headers.get("set-cookie") ?? "", /hwlab_session=/u); + const body = await login.json(); + assert.equal(body.actor.username, "admin"); + + const wrongLocalPassword = await fetch(`http://127.0.0.1:${port}/auth/login`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ username: "admin", password: "wrong-local-password" }) + }); + assert.equal(wrongLocalPassword.status, 401); + + const nonLocal = await fetch(`http://127.0.0.1:${port}/auth/login`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ username: "billing-only", password: "billing-only-password" }) + }); + assert.equal(nonLocal.status, 503); + const error = await nonLocal.json(); + assert.equal(error.error.code, "user_billing_unreachable"); + assert.equal(error.error.retryable, true); + assert.equal(error.error.dependency.serviceId, "hwlab-user-billing"); + assert.equal(error.error.dependency.retryCount, 2); + assert.equal(error.error.dependency.transientObserved, true); + assert.equal(calls.length, 3); + } finally { + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + } +}); + test("cloud api reports auth_required before user-billing profile/password proxy config errors", async () => { const userBillingClient = { configured: true,