Merge branch 'main' into release/3.6.59

Author: forgetso

.changeset/admin-endpoints-in-maintenance-mode.md

@@ -0,0 +1,8 @@
+---
+"@prosopo/env": patch
+"@prosopo/provider": patch
+---
+
+Keep the provider admin endpoints working while `MAINTENANCE_MODE` is on. Previously the admin/access-rule router was skipped entirely at boot in maintenance mode — `Environment.isReady()` never connected the DB, so `env.getDb()` threw and the DB-backed `Tasks` couldn't be constructed — which meant adding/removing site keys (access rules), detector keys and decision machines all 404'd on a node in maintenance mode.
+
+Now, in maintenance mode `Environment.isReady()` creates the `ProviderDatabase` handle and connects in the **background** (without awaiting), so a slow or unavailable Mongo/Redis socket still can't gate boot, but `env.getDb()` returns a usable handle and the admin endpoints register and function. The captcha request path is unchanged — it still short-circuits to a maintenance "pass" before touching the DB. `blockMiddleware` now has an explicit maintenance-mode skip (it previously relied on `env.getDb()` throwing to no-op) so the blocklist/Redis lookup stays off the captcha hot path.

packages/env/src/env.ts

@@ -136,13 +136,21 @@ export class Environment implements ProsopoEnvironment {
 
 			const maintenanceMode = isMaintenanceMode();
 
-			// In maintenance mode we skip the DB connect entirely so a slow
-			// Mongo socket can't gate boot. Handlers short-circuit before
-			// touching the DB while the flag is on.
+			// In maintenance mode the captcha request path is kept DB-free (the
+			// handlers short-circuit before touching the DB), but the admin
+			// endpoints — access rules, detector keys, site keys, decision
+			// machines — still need a DB. So we create the DB handle and connect
+			// in the BACKGROUND: env.getDb() returns a usable handle (admin
+			// routes register and keep working), while a slow or unavailable
+			// Mongo/Redis socket still can't gate boot because we never await
+			// the connection here.
 			if (maintenanceMode) {
 				this.logger.warn(() => ({
-					msg: "MAINTENANCE_MODE=true — skipping DB import on startup",
+					msg: "MAINTENANCE_MODE=true — connecting to DB in the background so admin endpoints stay available; captcha path short-circuits",
 				}));
+				if (!this.db) {
+					this.connectDatabaseInBackground();
+				}
 			} else if (!this.db) {
 				await this.importDatabase();
 			} else if (this.db && !this.db.connected) {
@@ -154,7 +162,9 @@ export class Environment implements ProsopoEnvironment {
 			}
 			// Resolve the default datasetId from the DB. Clients no longer
 			// send one — providers pick from this fallback for image challenges.
-			if (this.db && !this.datasetId) {
+			// Skip in maintenance mode: the connection is still settling in the
+			// background and this is a DB read we don't want to gate boot on.
+			if (!maintenanceMode && this.db && !this.datasetId) {
 				try {
 					this.datasetId = await this.db.getMostRecentDatasetId();
 					if (this.datasetId) {
@@ -185,25 +195,36 @@ export class Environment implements ProsopoEnvironment {
 		}
 	}
 
+	// Build the ProviderDatabase handle from config WITHOUT connecting. Returns
+	// undefined when no database is configured for the current environment.
+	buildDatabase(): ProviderDatabase | undefined {
+		if (!this.config.database) {
+			return undefined;
+		}
+		const dbConfig = this.config.database[this.defaultEnvironment];
+		if (!dbConfig) {
+			return undefined;
+		}
+		return new ProviderDatabase({
+			mongo: {
+				url: dbConfig.endpoint,
+				dbname: dbConfig.dbname,
+				authSource: dbConfig.authSource,
+			},
+			redis: {
+				url: this.config.redisConnection.url,
+				password: this.config.redisConnection.password,
+			},
+			logger: this.logger,
+		});
+	}
+
 	async importDatabase(): Promise<void> {
 		try {
-			if (this.config.database) {
-				const dbConfig = this.config.database[this.defaultEnvironment];
-				if (dbConfig) {
-					this.db = new ProviderDatabase({
-						mongo: {
-							url: dbConfig.endpoint,
-							dbname: dbConfig.dbname,
-							authSource: dbConfig.authSource,
-						},
-						redis: {
-							url: this.config.redisConnection.url,
-							password: this.config.redisConnection.password,
-						},
-						logger: this.logger,
-					});
-					await this.db.connect();
-				}
+			const db = this.buildDatabase();
+			if (db) {
+				this.db = db;
+				await this.db.connect();
 			}
 		} catch (error) {
 			throw new ProsopoEnvError("DATABASE.DATABASE_IMPORT_FAILED", {
@@ -218,6 +239,33 @@ export class Environment implements ProsopoEnvironment {
 			});
 		}
 	}
+
+	// Create the DB handle and kick off the connection WITHOUT awaiting it, so
+	// startup is never gated on the DB socket. Used in maintenance mode: the
+	// captcha request path never touches the DB (it short-circuits), but the
+	// admin endpoints do, so the handle must exist. A failed connection is
+	// logged rather than thrown — admin queries will surface the error (or
+	// succeed once the socket recovers) when they actually run.
+	connectDatabaseInBackground(): void {
+		try {
+			const db = this.buildDatabase();
+			if (!db) {
+				return;
+			}
+			this.db = db;
+			db.connect().catch((error: unknown) => {
+				this.logger.warn(() => ({
+					msg: "Background DB connection failed in maintenance mode; admin endpoints will retry on demand",
+					data: { error },
+				}));
+			});
+		} catch (error) {
+			this.logger.warn(() => ({
+				msg: "Failed to create DB handle in maintenance mode; admin endpoints will be unavailable until restart",
+				data: { error },
+			}));
+		}
+	}
 }
 
 // Read directly from process.env to avoid a cyclic dep on the provider

packages/env/src/tests/env.maintenance.unit.test.ts

@@ -102,18 +102,50 @@ describe("Environment.isReady — maintenance mode startup tolerance", () => {
 		});
 	});
 
-	it("skips DB import entirely when maintenance mode is on", async () => {
+	it("creates the DB handle and connects in the background when maintenance mode is on", async () => {
 		process.env.MAINTENANCE_MODE = "true";
+		const connect = vi.fn().mockResolvedValue(undefined);
+		mockProviderDatabase.mockImplementation(() => ({
+			connect,
+			connected: false,
+			connection: { readyState: 0 },
+		}));
 
 		const env = buildEnv();
 		await env.isReady();
+
 		expect(env.ready).toBe(true);
-		// Importantly: we never even construct ProviderDatabase, so a slow
-		// Mongo socket can't gate boot.
-		expect(mockProviderDatabase).not.toHaveBeenCalled();
+		// The handle IS created (and connecting) so admin endpoints — which need
+		// a DB even during maintenance — keep working. The captcha path stays
+		// DB-free via per-handler short-circuits, not by leaving env.db unset.
+		expect(mockProviderDatabase).toHaveBeenCalledTimes(1);
+		expect(connect).toHaveBeenCalledTimes(1);
+		expect(env.db).toBeDefined();
 		expect(mockIpInfoInit).toHaveBeenCalled();
 	});
 
+	it("does not gate boot when the background DB connect fails in maintenance mode", async () => {
+		process.env.MAINTENANCE_MODE = "true";
+		const connect = vi.fn().mockRejectedValue(new Error("ECONNREFUSED"));
+		mockProviderDatabase.mockImplementation(() => ({
+			connect,
+			connected: false,
+			connection: { readyState: 0 },
+		}));
+
+		const env = buildEnv();
+		// Unlike the maintenance-off path, a failing connect must NOT reject
+		// isReady() — the connection is fire-and-forget in the background.
+		await expect(env.isReady()).resolves.toBeUndefined();
+		expect(env.ready).toBe(true);
+		expect(env.db).toBeDefined();
+		expect(connect).toHaveBeenCalledTimes(1);
+		// Flush the background rejection so its .catch handler runs within the
+		// test rather than surfacing as an unhandled rejection afterwards.
+		await Promise.resolve();
+		expect(env.logger.warn).toHaveBeenCalled();
+	});
+
 	it("still completes the normal connect path when Mongo is up", async () => {
 		const connect = vi.fn().mockResolvedValue(undefined);
 		mockProviderDatabase.mockImplementation(() => ({

packages/provider/src/api/block.ts

@@ -14,6 +14,7 @@
 
 import type { ProviderEnvironment } from "@prosopo/types-env";
 import type { NextFunction, Request, Response } from "express";
+import { getMaintenanceMode } from "./admin/apiToggleMaintenanceModeEndpoint.js";
 import { BlacklistRequestInspector } from "./blacklistRequestInspector.js";
 
 export const blockMiddleware = (providerEnvironment: ProviderEnvironment) => {
@@ -27,6 +28,15 @@ export const blockMiddleware = (providerEnvironment: ProviderEnvironment) => {
 		providerEnvironment.isReady.bind(providerEnvironment);
 
 	return (req: Request, res: Response, next: NextFunction) => {
+		// In maintenance mode the captcha path short-circuits to a pass and the
+		// access-rules store (Redis) may be unavailable — skip the blocklist
+		// check so a slow or down store can't gate requests. env.getDb() now
+		// returns a handle during maintenance (so the admin endpoints work), so
+		// this explicit guard — not a thrown getDb() — is what keeps the
+		// blocklist check off the hot path.
+		if (getMaintenanceMode()) {
+			return next();
+		}
 		if (!blacklistRequestInspector) {
 			try {
 				const db = providerEnvironment.getDb();

packages/provider/src/api/startProviderApi.ts

@@ -147,9 +147,13 @@ export async function startProviderApi(
 	const apiEndpointAdapter = createApiExpressDefaultEndpointAdapter(
 		parseLogLevel(env.config.logLevel),
 	);
-	// Maintenance-mode / DB-down startup: skip the admin/access-rule wiring
-	// since both rely on DB-backed Tasks construction. Captcha endpoints
-	// short-circuit on maintenance-mode at the handler.
+	// Admin + access-rule routes both rely on a DB-backed Tasks/storage. In
+	// maintenance mode the DB handle is created and connected in the background
+	// (see Environment.isReady), so env.getDb() returns a handle and these
+	// routes register and keep working — that's what keeps admin operations
+	// (rules, detector keys, site keys, decision machines) available while the
+	// captcha path short-circuits. The try/catch is a safety net for the case
+	// where no DB is configured at all.
 	let apiRuleRoutesProvider: AccessRuleApiRoutes | undefined;
 	let apiAdminRoutesProvider:
 		| ReturnType<typeof createApiAdminRoutesProvider>