feat: CCP-6026-Redis Monitoring

api-gateway/templates/routes.yaml

@@ -628,6 +628,40 @@ routes:
     request_buffering: true
     response_buffering: true
 
+  # ============================================================================
+  # MONITORING ADMIN ROUTES (admin only)
+  # ============================================================================
+  # /api/v1/frontend/admin/monitoring/queues (queue, worker and Redis health)
+  - name: ${ROUTE_PREFIX}admin-monitoring-queues-frontend
+    tags: [${CONFIG_TAG}]
+    hosts:
+      - ${GATEWAY_HOSTNAME}
+    paths:
+      - ${PATH_PREFIX}/api/v1/frontend/admin/monitoring/queues
+    methods:
+      - GET
+    strip_path: true
+    https_redirect_status_code: 426
+    path_handling: v0
+    request_buffering: true
+    response_buffering: true
+
+  # /api/v1/frontend/admin/monitoring/queues/events (SSE change signals)
+  # Buffering off, as for notification-request-events: a buffered stream never reaches the page.
+  - name: ${ROUTE_PREFIX}admin-monitoring-queue-events-frontend
+    tags: [${CONFIG_TAG}]
+    hosts:
+      - ${GATEWAY_HOSTNAME}
+    paths:
+      - ${PATH_PREFIX}/api/v1/frontend/admin/monitoring/queues/events
+    methods:
+      - GET
+    strip_path: true
+    https_redirect_status_code: 426
+    path_handling: v0
+    request_buffering: false
+    response_buffering: false
+
   # ============================================================================
   # FEATURE FLAG ADMIN ROUTES (admin only)
   # ============================================================================
@@ -1679,6 +1713,23 @@ plugins:
     route: ${ROUTE_PREFIX}admin-users-list-frontend
     config: *jwtConfig
 
+  # ============================================================================
+  # MONITORING ADMIN ROUTES PLUGINS
+  # ============================================================================
+  # Admin Monitoring Queues - Frontend Admin Token
+  - name: jwt-keycloak
+    tags: [${CONFIG_TAG}]
+    enabled: true
+    route: ${ROUTE_PREFIX}admin-monitoring-queues-frontend
+    config: *jwtConfig
+
+  # Admin Monitoring Queue Events (SSE) - Frontend Admin Token
+  - name: jwt-keycloak
+    tags: [${CONFIG_TAG}]
+    enabled: true
+    route: ${ROUTE_PREFIX}admin-monitoring-queue-events-frontend
+    config: *jwtConfig
+
   # ============================================================================
   # FEATURE FLAG ADMIN ROUTES PLUGINS
   # ============================================================================

backend/Dockerfile

@@ -27,6 +27,7 @@ COPY --from=build /app/dist ./dist
 EXPOSE 3000
 HEALTHCHECK --interval=30s --timeout=3s CMD curl -f http://localhost:3000/api
 
-# Nonroot user, limit heap size to 150 MB
+# Nonroot user. The heap limit is set per environment by the Helm chart (NODE_OPTIONS from
+# backend.heapMb), not baked in here, so it can be raised without rebuilding the image.
 USER nonroot
-CMD ["--max-old-space-size=150", "/app/dist/main"]
+CMD ["/app/dist/main"]

backend/src/adapters/adapters.module.spec.ts

@@ -11,14 +11,16 @@ import { NodemailerEmailTransport } from '../../src/adapters/implementations/del
 import { LogEmailTransport } from '../../src/adapters/implementations/delivery/email/log/log-email.adapter'
 import { TwilioSmsTransport } from '../../src/adapters/implementations/delivery/sms/twilio/twilio-sms.adapter'
 import { AcsSmsTransport } from '../../src/adapters/implementations/delivery/sms/acs/acs-sms.adapter'
+import { SmsCircuitBreaker } from '../../src/adapters/implementations/delivery/sms/sms-circuit-breaker'
 
 describe('AdaptersModule', () => {
   it('forRoot returns dynamic module with all adapters and maps', () => {
     const dynamic = AdaptersModule.forRoot()
 
     expect(dynamic.module).toBe(AdaptersModule)
     expect(dynamic.global).toBe(true)
-    expect(dynamic.providers).toHaveLength(11)
+    expect(dynamic.providers).toHaveLength(12)
+    expect(dynamic.providers).toContain(SmsCircuitBreaker)
     expect(dynamic.exports).toContain(EMAIL_ADAPTER)
     expect(dynamic.exports).toContain(SMS_ADAPTER)
     expect(dynamic.exports).toContain(EMAIL_ADAPTER_MAP)
@@ -39,4 +41,18 @@ describe('AdaptersModule', () => {
     expect(adapterProviders).toContain(TwilioSmsTransport)
     expect(adapterProviders).toContain(AcsSmsTransport)
   })
+
+  it('sends SMS through the circuit breaker, whichever provider is configured', () => {
+    const provider = (AdaptersModule.forRoot().providers ?? []).find(
+      (p) => (p as { provide?: unknown }).provide === SMS_ADAPTER,
+    ) as { useFactory: (...args: unknown[]) => unknown }
+    const acs = { name: 'acs', send: vi.fn() }
+    const wrapped = { name: 'acs', send: vi.fn() }
+    const breaker = { wrap: vi.fn(() => wrapped) }
+
+    const adapter = provider.useFactory({ acs }, { get: () => 'acs' }, breaker)
+
+    expect(breaker.wrap).toHaveBeenCalledWith(acs)
+    expect(adapter).toBe(wrapped)
+  })
 })

backend/src/adapters/adapters.module.ts

@@ -5,6 +5,7 @@ import { NodemailerEmailTransport } from './implementations/delivery/email/nodem
 import { LogEmailTransport } from './implementations/delivery/email/log/log-email.adapter'
 import { TwilioSmsTransport } from './implementations/delivery/sms/twilio/twilio-sms.adapter'
 import { AcsSmsTransport } from './implementations/delivery/sms/acs/acs-sms.adapter'
+import { SmsCircuitBreaker } from './implementations/delivery/sms/sms-circuit-breaker'
 import {
   EMAIL_ADAPTER,
   EMAIL_ADAPTER_MAP,
@@ -37,6 +38,7 @@ export class AdaptersModule {
         LogEmailTransport,
         TwilioSmsTransport,
         AcsSmsTransport,
+        SmsCircuitBreaker,
         {
           provide: EMAIL_ADAPTER_MAP,
           useFactory: (
@@ -83,16 +85,17 @@ export class AdaptersModule {
           useFactory: (
             map: Record<string, ISmsTransport>,
             configService: ConfigService,
+            breaker: SmsCircuitBreaker,
           ): ISmsTransport => {
             const key = configService.get<string>('delivery.sms') ?? 'acs'
             // As above: a `:passthrough` suffix names no ISmsTransport, so fall back for DI.
             if (key?.includes(':passthrough')) {
-              return map['acs']
+              return breaker.wrap(map['acs'])
             }
             const provider = key?.includes(':') ? key.split(':')[0] : key
-            return map[provider] ?? map['acs']
+            return breaker.wrap(map[provider] ?? map['acs'])
           },
-          inject: [SMS_ADAPTER_MAP, ConfigService],
+          inject: [SMS_ADAPTER_MAP, ConfigService, SmsCircuitBreaker],
         },
         InMemoryTemplateStore,
         { provide: SENDER_STORE, useClass: InMemorySenderStore },

backend/src/adapters/delivery-errors.ts

@@ -0,0 +1,75 @@
+import { HttpException, HttpStatus } from '@nestjs/common'
+
+/**
+ * The provider could not take the message right now - it is down, restarting, overloaded or
+ * rate limiting - and it accepted nothing. The message itself is fine: send it again later.
+ *
+ * Workers must not mark a recipient failed for this. A provider outage of a few seconds would
+ * otherwise fail every recipient that reached it in those seconds, and since such errors come
+ * back instantly, a large send would burn through its whole backlog. The work is left owed and
+ * retried; the circuit breaker in front of the provider stops calls while it recovers.
+ *
+ * Not used when the outcome is unknown - a timeout after the request was sent may have been
+ * delivered, and sending it again could deliver it twice.
+ */
+export class TransientDeliveryError extends HttpException {
+  readonly transient = true
+
+  constructor(message: string, status: number = HttpStatus.SERVICE_UNAVAILABLE) {
+    super(message, status)
+  }
+}
+
+export function isTransientDeliveryError(error: unknown): error is TransientDeliveryError {
+  return error instanceof TransientDeliveryError
+}
+
+/** HTTP statuses that mean "not now" rather than "never": the request was not acted on. */
+export function isTransientHttpStatus(status: number): boolean {
+  return status === 408 || status === 429 || status >= 500
+}
+
+/**
+ * Network errors raised before a request reached the provider: nothing was sent. A connection
+ * reset or a body cut off mid-response is not here - the provider may already have acted.
+ */
+const NOT_SENT_NETWORK_CODES = new Set(['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN', 'EHOSTUNREACH'])
+
+/**
+ * A network error raised before the request reached the provider. The code is on the error itself
+ * (Azure SDK, axios) or on its `cause` (fetch's `TypeError('fetch failed')`).
+ */
+export function isNotSentNetworkError(error: unknown): boolean {
+  const { code, cause } = (error ?? {}) as { code?: unknown; cause?: { code?: unknown } }
+  return [code, cause?.code].some(
+    (value) => typeof value === 'string' && NOT_SENT_NETWORK_CODES.has(value),
+  )
+}
+
+/**
+ * An SDK error from a provider, as a TransientDeliveryError when it means the provider took
+ * nothing: an HTTP status of 408, 429 or 5xx (`statusCode` on Azure's RestError, `status` on
+ * Twilio's RestException), or a network error before the request was sent. Null otherwise.
+ */
+export function transientFromProviderError(
+  error: unknown,
+  label: string,
+): TransientDeliveryError | null {
+  if (isTransientDeliveryError(error)) return error
+  // Our own exceptions have a runtime `status` too; only a raw SDK error is classified here.
+  if (error instanceof HttpException) return null
+  const { statusCode, status, message } = (error ?? {}) as {
+    statusCode?: unknown
+    status?: unknown
+    message?: unknown
+  }
+  const httpStatus = typeof statusCode === 'number' ? statusCode : status
+  const text = typeof message === 'string' ? message : String(error)
+  if (typeof httpStatus === 'number' && isTransientHttpStatus(httpStatus)) {
+    return new TransientDeliveryError(`${label}: upstream ${httpStatus} - ${text}`, 502)
+  }
+  if (isNotSentNetworkError(error)) {
+    return new TransientDeliveryError(`${label} unreachable: ${text}`)
+  }
+  return null
+}

backend/src/adapters/delivery-keys.ts

@@ -0,0 +1,8 @@
+import { redisKey } from '../common/redis/redis-namespace'
+
+/** CHES email requests in flight across all pods (RedisConcurrencyLimiter). */
+export const CHES_IN_FLIGHT_KEY = redisKey('ches:in-flight')
+/** The circuit breaker in front of CHES (RedisCircuitBreaker); read by admin monitoring. */
+export const CHES_CIRCUIT_KEY = redisKey('ches:circuit')
+/** The circuit breaker in front of the SMS provider; read by admin monitoring. */
+export const SMS_CIRCUIT_KEY = redisKey('sms:circuit')