Merge pull request #1356 from pikasTech/fix/1353-user-billing-login-resilience
fix: harden user-billing login transient handling
This commit is contained in:
Lyon
+3
-1
@@ -294,7 +294,7 @@ lanes:
|
||||
runtimeKind: go-service
|
||||
entrypoint: cmd/hwlab-user-billing/main.go
|
||||
artifactKind: go-service
|
||||
healthPath: /health/live
|
||||
healthPath: /health/ready
|
||||
healthPort: 6670
|
||||
componentPaths:
|
||||
- go.mod
|
||||
@@ -404,6 +404,8 @@ lanes:
|
||||
HWLAB_BOOTSTRAP_ADMIN_API_KEY_ID: key_master_server_admin
|
||||
HWLAB_BOOTSTRAP_ADMIN_API_KEY: secretRef:hwlab-v03-master-server-admin-api-key/api-key
|
||||
HWLAB_USER_BILLING_URL: http://hwlab-user-billing.hwlab-v03.svc.cluster.local:6670
|
||||
HWLAB_USER_BILLING_LOGIN_RETRY_ATTEMPTS: "2"
|
||||
HWLAB_USER_BILLING_LOGIN_RETRY_DELAY_MS: "250"
|
||||
HWLAB_USER_BILLING_CODE_AGENT_ENABLED: "1"
|
||||
HWLAB_USER_BILLING_CODE_AGENT_ESTIMATED_CREDITS: "1"
|
||||
HWLAB_USER_BILLING_CODE_AGENT_USED_CREDITS: "1"
|
||||
|
||||
@@ -470,7 +470,7 @@
|
||||
],
|
||||
"readinessProbe": {
|
||||
"httpGet": {
|
||||
"path": "/health/live",
|
||||
"path": "/health/ready",
|
||||
"port": "http"
|
||||
}
|
||||
},
|
||||
|
||||
@@ -807,19 +807,21 @@ class AccessController {
|
||||
const body = await jsonBody(request);
|
||||
const userBilling = await this.userBillingLogin(body, request, response);
|
||||
if (userBilling?.handled) return;
|
||||
const user = await this.store.findUserByUsername(requiredText(body.username, "username"));
|
||||
if (!user || user.status !== "active" || !verifyPassword(user.passwordHash, requiredText(body.password, "password"))) {
|
||||
const password = requiredText(body.password, "password");
|
||||
const username = textOr(body.username, "");
|
||||
if (!username) {
|
||||
if (userBilling?.retryable) return sendJson(response, userBilling.status ?? 503, userBillingDependencyErrorPayload(userBilling.result, "user_billing_login_failed", "user-billing login failed", userBilling.status ?? 503));
|
||||
requiredText(body.username, "username");
|
||||
}
|
||||
const user = await this.store.findUserByUsername(username);
|
||||
if (user && (user.status !== "active" || !verifyPassword(user.passwordHash, password))) {
|
||||
return sendJson(response, 401, errorPayload("invalid_credentials", "Username or password is invalid", 401));
|
||||
}
|
||||
const token = randomBytes(32).toString("base64url");
|
||||
const now = this.now();
|
||||
const session = await this.store.createSession({
|
||||
userId: user.id,
|
||||
tokenHash: sha256(token),
|
||||
now,
|
||||
expiresAt: new Date(Date.parse(now) + SESSION_MAX_AGE_SECONDS * 1000).toISOString()
|
||||
});
|
||||
await this.store.ensureDefaultApiKeyForUser?.({ userId: user.id, now }) ?? null;
|
||||
if (!user) {
|
||||
if (userBilling?.retryable) return sendJson(response, userBilling.status ?? 503, userBillingDependencyErrorPayload(userBilling.result, "user_billing_login_failed", "user-billing login failed", userBilling.status ?? 503));
|
||||
return sendJson(response, 401, errorPayload("invalid_credentials", "Username or password is invalid", 401));
|
||||
}
|
||||
const { token, session } = await this.issueWebSessionForActor(user);
|
||||
setSessionCookie(response, token, SESSION_MAX_AGE_SECONDS, request, this.env);
|
||||
return sendJson(response, 200, {
|
||||
authenticated: true,
|
||||
@@ -866,9 +868,10 @@ class AccessController {
|
||||
const result = await this.issueUserBillingWebSession({ username, password: requiredText(body.password, "password") }, request, response);
|
||||
if (!result.ok) {
|
||||
if (result.status === 401) return { handled: false };
|
||||
if (isRetryableUserBillingResult(result)) return { handled: false, retryable: true, status: result.status ?? 503, result };
|
||||
return {
|
||||
handled: true,
|
||||
response: sendJson(response, result.status ?? 502, errorPayload(result.error?.code ?? "user_billing_login_failed", result.error?.message ?? "user-billing login failed", result.status ?? 502))
|
||||
response: sendJson(response, result.status ?? 502, userBillingDependencyErrorPayload(result, "user_billing_login_failed", "user-billing login failed", result.status ?? 502))
|
||||
};
|
||||
}
|
||||
sendJson(response, 200, {
|
||||
@@ -886,7 +889,15 @@ class AccessController {
|
||||
async issueUserBillingWebSession({ username, password }, request, response) {
|
||||
const login = await this.userBilling.login({ username, email: username, password });
|
||||
if (!login.ok || !login.body?.token) {
|
||||
return { ok: false, status: login.status, error: login.error };
|
||||
return {
|
||||
ok: false,
|
||||
status: login.status,
|
||||
error: login.error,
|
||||
retryable: login.retryable,
|
||||
retryCount: login.retryCount,
|
||||
transientObserved: login.transientObserved,
|
||||
valuesRedacted: login.valuesRedacted
|
||||
};
|
||||
}
|
||||
const token = textOr(login.body.token, "");
|
||||
const auth = await this.authenticateUserBillingToken(token);
|
||||
@@ -2922,6 +2933,36 @@ function shouldUseSecureSessionCookie(request, env = process.env) {
|
||||
return false;
|
||||
}
|
||||
function errorPayload(code, message, status) { return { ok: false, status, error: { code, message } }; }
|
||||
|
||||
function userBillingDependencyErrorPayload(result, fallbackCode, fallbackMessage, fallbackStatus) {
|
||||
const status = result?.status ?? fallbackStatus;
|
||||
const payload = errorPayload(result?.error?.code ?? fallbackCode, result?.error?.message ?? fallbackMessage, status);
|
||||
const retryable = isRetryableUserBillingResult(result);
|
||||
payload.error.retryable = retryable;
|
||||
payload.error.dependency = {
|
||||
serviceId: "hwlab-user-billing",
|
||||
retryCount: nonNegativeResultInteger(result?.retryCount),
|
||||
transientObserved: Boolean(result?.transientObserved ?? retryable),
|
||||
valuesRedacted: true
|
||||
};
|
||||
return payload;
|
||||
}
|
||||
|
||||
function isRetryableUserBillingResult(result) {
|
||||
if (!result || result.ok) return false;
|
||||
if (result.retryable === true) return true;
|
||||
const code = textOr(result.error?.code, "");
|
||||
if (code === "user_billing_not_configured") return false;
|
||||
if (code === "user_billing_unreachable") return true;
|
||||
const status = Number(result.status ?? 0);
|
||||
if ([500, 502, 503, 504].includes(status)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function nonNegativeResultInteger(value) {
|
||||
const parsed = Number.parseInt(String(value ?? ""), 10);
|
||||
return Number.isInteger(parsed) && parsed >= 0 ? parsed : 0;
|
||||
}
|
||||
function positiveInteger(value, fallback = 0) { const number = Number(value); return Number.isFinite(number) && number > 0 ? Math.floor(number) : fallback; }
|
||||
async function resolveBeforeDeadline(promise, timeoutMs) {
|
||||
/** @type {ReturnType<typeof setTimeout> | null} */
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { createUserBillingClient } from "./user-billing-client.ts";
|
||||
|
||||
test("user-billing login retries transient dependency failures only", async () => {
|
||||
const calls = [];
|
||||
const client = createUserBillingClient({
|
||||
env: {
|
||||
HWLAB_USER_BILLING_URL: "http://user-billing.test",
|
||||
HWLAB_USER_BILLING_LOGIN_RETRY_ATTEMPTS: "2",
|
||||
HWLAB_USER_BILLING_LOGIN_RETRY_DELAY_MS: "1"
|
||||
},
|
||||
fetchImpl: async (url, init) => {
|
||||
calls.push({ url: String(url), method: init?.method });
|
||||
if (calls.length === 1) throw new Error("The operation timed out.");
|
||||
return new Response(JSON.stringify({ token: "hws_retry_ok", tokenType: "Bearer" }), { status: 200, headers: { "content-type": "application/json" } });
|
||||
}
|
||||
});
|
||||
|
||||
const result = await client.login({ username: "admin", password: "redacted" });
|
||||
assert.equal(result.ok, true);
|
||||
assert.equal(result.retryCount, 1);
|
||||
assert.equal(result.transientObserved, true);
|
||||
assert.equal(calls.length, 2);
|
||||
|
||||
const authFailures = [];
|
||||
const invalidClient = createUserBillingClient({
|
||||
env: {
|
||||
HWLAB_USER_BILLING_URL: "http://user-billing.test",
|
||||
HWLAB_USER_BILLING_LOGIN_RETRY_ATTEMPTS: "2",
|
||||
HWLAB_USER_BILLING_LOGIN_RETRY_DELAY_MS: "1"
|
||||
},
|
||||
fetchImpl: async (url, init) => {
|
||||
authFailures.push({ url: String(url), method: init?.method });
|
||||
return new Response(JSON.stringify({ error: { code: "invalid_credentials", message: "invalid email, username or password" } }), { status: 401, headers: { "content-type": "application/json" } });
|
||||
}
|
||||
});
|
||||
|
||||
const invalid = await invalidClient.login({ username: "admin", password: "wrong" });
|
||||
assert.equal(invalid.ok, false);
|
||||
assert.equal(invalid.status, 401);
|
||||
assert.equal(invalid.error.code, "invalid_credentials");
|
||||
assert.equal(authFailures.length, 1);
|
||||
|
||||
const unexpectedCalls = [];
|
||||
const unconfiguredClient = createUserBillingClient({
|
||||
env: {
|
||||
HWLAB_USER_BILLING_LOGIN_RETRY_ATTEMPTS: "2",
|
||||
HWLAB_USER_BILLING_LOGIN_RETRY_DELAY_MS: "1"
|
||||
},
|
||||
fetchImpl: async (url, init) => {
|
||||
unexpectedCalls.push({ url: String(url), method: init?.method });
|
||||
throw new Error("fetch should not run without a user-billing URL");
|
||||
}
|
||||
});
|
||||
|
||||
const unconfigured = await unconfiguredClient.login({ username: "admin", password: "redacted" });
|
||||
assert.equal(unconfigured.ok, false);
|
||||
assert.equal(unconfigured.status, 503);
|
||||
assert.equal(unconfigured.error.code, "user_billing_not_configured");
|
||||
assert.equal(unconfigured.retryable, undefined);
|
||||
assert.equal(unexpectedCalls.length, 0);
|
||||
});
|
||||
@@ -1,4 +1,7 @@
|
||||
const DEFAULT_TIMEOUT_MS = 5000;
|
||||
const DEFAULT_LOGIN_RETRY_ATTEMPTS = 2;
|
||||
const DEFAULT_LOGIN_RETRY_DELAY_MS = 250;
|
||||
const NON_RETRYABLE_USER_BILLING_ERROR_CODES = new Set(["user_billing_not_configured"]);
|
||||
|
||||
export function createUserBillingClient({ env = process.env, fetchImpl = fetch } = {}) {
|
||||
const baseUrl = normalizeBaseUrl(firstText(
|
||||
@@ -8,8 +11,34 @@ export function createUserBillingClient({ env = process.env, fetchImpl = fetch }
|
||||
));
|
||||
const internalToken = text(env.HWLAB_USER_BILLING_INTERNAL_TOKEN);
|
||||
const timeoutMs = positiveInteger(env.HWLAB_USER_BILLING_TIMEOUT_MS, DEFAULT_TIMEOUT_MS);
|
||||
const loginRetryAttempts = nonNegativeInteger(env.HWLAB_USER_BILLING_LOGIN_RETRY_ATTEMPTS, DEFAULT_LOGIN_RETRY_ATTEMPTS);
|
||||
const loginRetryDelayMs = positiveInteger(env.HWLAB_USER_BILLING_LOGIN_RETRY_DELAY_MS, DEFAULT_LOGIN_RETRY_DELAY_MS);
|
||||
|
||||
async function requestJson(path, { method = "GET", body = null, bearerToken = "" } = {}) {
|
||||
return requestJsonWithRetry(path, { method, body, bearerToken });
|
||||
}
|
||||
|
||||
async function requestJsonWithRetry(path, { method = "GET", body = null, bearerToken = "", retry = null } = {}) {
|
||||
const retryAttempts = nonNegativeInteger(retry?.attempts, 0);
|
||||
const retryDelayMs = positiveInteger(retry?.delayMs, 0);
|
||||
let retryCount = 0;
|
||||
let transientObserved = false;
|
||||
for (;;) {
|
||||
const result = await requestJsonOnce(path, { method, body, bearerToken });
|
||||
const retryable = typeof retry?.shouldRetry === "function" ? retry.shouldRetry(result) : false;
|
||||
if (!retryable || retryCount >= retryAttempts) {
|
||||
if (retryCount > 0 || transientObserved || retryable) {
|
||||
return { ...result, retryable, retryCount, transientObserved: transientObserved || retryable, valuesRedacted: true };
|
||||
}
|
||||
return result;
|
||||
}
|
||||
retryCount += 1;
|
||||
transientObserved = true;
|
||||
if (retryDelayMs > 0) await sleep(retryDelayMs * retryCount);
|
||||
}
|
||||
}
|
||||
|
||||
async function requestJsonOnce(path, { method = "GET", body = null, bearerToken = "" } = {}) {
|
||||
if (!baseUrl) {
|
||||
return { ok: false, status: 503, error: { code: "user_billing_not_configured", message: "HWLAB user-billing URL is not configured" } };
|
||||
}
|
||||
@@ -94,7 +123,11 @@ export function createUserBillingClient({ env = process.env, fetchImpl = fetch }
|
||||
return publicPost("/v1/auth/register", body);
|
||||
},
|
||||
async login(body = {}) {
|
||||
return publicPost("/v1/auth/login", body);
|
||||
return requestJsonWithRetry("/v1/auth/login", {
|
||||
method: "POST",
|
||||
body,
|
||||
retry: { attempts: loginRetryAttempts, delayMs: loginRetryDelayMs, shouldRetry: isRetryableUserBillingResult }
|
||||
});
|
||||
},
|
||||
async logout(token) {
|
||||
return requestJson("/v1/auth/logout", { method: "POST", body: {}, bearerToken: token });
|
||||
@@ -245,6 +278,25 @@ function positiveInteger(value, fallback) {
|
||||
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function nonNegativeInteger(value, fallback) {
|
||||
const parsed = Number.parseInt(String(value ?? ""), 10);
|
||||
return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function isRetryableUserBillingResult(result) {
|
||||
if (!result || result.ok) return false;
|
||||
const code = text(result.error?.code);
|
||||
if (NON_RETRYABLE_USER_BILLING_ERROR_CODES.has(code)) return false;
|
||||
if (code === "user_billing_unreachable") return true;
|
||||
const status = Number(result.status ?? 0);
|
||||
if ([500, 502, 503, 504].includes(status)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, Math.max(0, Number(ms) || 0)));
|
||||
}
|
||||
|
||||
function firstText(...values) {
|
||||
for (const value of values) {
|
||||
const current = text(value);
|
||||
|
||||
@@ -939,6 +939,68 @@ test("cloud api bridges user-billing register/login into HttpOnly web session co
|
||||
}
|
||||
});
|
||||
|
||||
test("cloud api falls back to local bootstrap login when user-billing login is transient", async () => {
|
||||
const calls = [];
|
||||
const userBillingClient = {
|
||||
configured: true,
|
||||
async login(body = {}) {
|
||||
calls.push({ op: "login", username: body.username, passwordSeen: Boolean(body.password) });
|
||||
return {
|
||||
ok: false,
|
||||
status: 503,
|
||||
retryable: true,
|
||||
retryCount: 2,
|
||||
transientObserved: true,
|
||||
error: { code: "user_billing_unreachable", message: "The operation timed out." }
|
||||
};
|
||||
}
|
||||
};
|
||||
const server = createCloudApiServer({
|
||||
env: {
|
||||
HWLAB_ACCESS_CONTROL_REQUIRED: "1",
|
||||
HWLAB_BOOTSTRAP_ADMIN_USERNAME: "admin",
|
||||
HWLAB_BOOTSTRAP_ADMIN_PASSWORD: "local-admin-password-1353"
|
||||
},
|
||||
userBillingClient
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const login = await fetch(`http://127.0.0.1:${port}/auth/login`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ username: "admin", password: "local-admin-password-1353" })
|
||||
});
|
||||
assert.equal(login.status, 200);
|
||||
assert.match(login.headers.get("set-cookie") ?? "", /hwlab_session=/u);
|
||||
const body = await login.json();
|
||||
assert.equal(body.actor.username, "admin");
|
||||
|
||||
const wrongLocalPassword = await fetch(`http://127.0.0.1:${port}/auth/login`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ username: "admin", password: "wrong-local-password" })
|
||||
});
|
||||
assert.equal(wrongLocalPassword.status, 401);
|
||||
|
||||
const nonLocal = await fetch(`http://127.0.0.1:${port}/auth/login`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ username: "billing-only", password: "billing-only-password" })
|
||||
});
|
||||
assert.equal(nonLocal.status, 503);
|
||||
const error = await nonLocal.json();
|
||||
assert.equal(error.error.code, "user_billing_unreachable");
|
||||
assert.equal(error.error.retryable, true);
|
||||
assert.equal(error.error.dependency.serviceId, "hwlab-user-billing");
|
||||
assert.equal(error.error.dependency.retryCount, 2);
|
||||
assert.equal(error.error.dependency.transientObserved, true);
|
||||
assert.equal(calls.length, 3);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())));
|
||||
}
|
||||
});
|
||||
|
||||
test("cloud api reports auth_required before user-billing profile/password proxy config errors", async () => {
|
||||
const userBillingClient = {
|
||||
configured: true,
|
||||
|
||||
Reference in New Issue
Block a user