From 7f83808abdb6d49ddf03e9dbaa5f9846c8a89d6e Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Thu, 1 Oct 2026 15:18:39 -0700 Subject: [PATCH 01/23] Add queue monitoring metrics and heartbeats Adds backend foundations for queue monitoring: new monitoring thresholds and DTO schemas, Bull throughput helpers, and worker heartbeat publishing to Redis so admin views can report live worker state across pods. Queue creation now enables Bull per-minute metrics, tracks added jobs with TTL-based Redis counters, and exposes a configurable history window (`QUEUE_METRICS_MAX_DATA_POINTS`, default 1440). QueueModule now starts/stops heartbeat publishing and registers tracked queues with their concurrency. Includes unit coverage to ensure queue metrics are enabled and default retention is one day. --- api-gateway/templates/routes.yaml | 28 ++ .../admin/monitoring/monitoring-thresholds.ts | 19 + .../monitoring/monitoring.controller.spec.ts | 17 + .../admin/monitoring/monitoring.controller.ts | 31 ++ .../api/admin/monitoring/monitoring.module.ts | 14 + .../monitoring/monitoring.service.spec.ts | 313 +++++++++++++++ .../admin/monitoring/monitoring.service.ts | 373 ++++++++++++++++++ .../schemas/queue-monitoring.dto.ts | 97 +++++ backend/src/queue/queue-metrics.spec.ts | 86 ++++ backend/src/queue/queue-metrics.ts | 90 +++++ backend/src/queue/queue.module.ts | 31 +- backend/src/queue/redis-connection.spec.ts | 34 ++ backend/src/queue/redis-connection.ts | 12 + backend/src/queue/worker-heartbeat.spec.ts | 75 ++++ backend/src/queue/worker-heartbeat.ts | 114 ++++++ frontend/src/api/monitoring.api.ts | 24 ++ frontend/src/components/MeterBar.tsx | 35 ++ frontend/src/components/Sidebar.spec.tsx | 23 ++ frontend/src/components/Sidebar.tsx | 13 + frontend/src/components/Sparkline.tsx | 54 +++ frontend/src/components/StatTile.tsx | 20 + frontend/src/components/StatusBadge.tsx | 4 + frontend/src/hooks/usePolling.ts | 65 +++ .../interfaces/queueMonitoring.interface.ts | 78 ++++ .../admin/monitoring/QueueMonitoring.spec.tsx | 240 +++++++++++ .../admin/monitoring/QueueMonitoring.tsx | 122 ++++++ .../monitoring/sections/QueuesSection.tsx | 118 ++++++ .../sections/RecentFailuresSection.tsx | 56 +++ .../monitoring/sections/RedisSection.tsx | 79 ++++ .../monitoring/sections/WorkersSection.tsx | 80 ++++ frontend/src/routeTree.gen.ts | 21 + frontend/src/routes/admin/monitoring.tsx | 17 + frontend/src/scss/components/meter-bar.scss | 26 ++ .../src/scss/components/queue-monitoring.scss | 64 +++ frontend/src/scss/components/sparkline.scss | 38 ++ frontend/src/scss/components/stat-tile.scss | 39 ++ frontend/src/utils/monitoring.spec.ts | 35 ++ frontend/src/utils/monitoring.ts | 52 +++ 38 files changed, 2635 insertions(+), 2 deletions(-) create mode 100644 backend/src/api/admin/monitoring/monitoring-thresholds.ts create mode 100644 backend/src/api/admin/monitoring/monitoring.controller.spec.ts create mode 100644 backend/src/api/admin/monitoring/monitoring.controller.ts create mode 100644 backend/src/api/admin/monitoring/monitoring.module.ts create mode 100644 backend/src/api/admin/monitoring/monitoring.service.spec.ts create mode 100644 backend/src/api/admin/monitoring/monitoring.service.ts create mode 100644 backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts create mode 100644 backend/src/queue/queue-metrics.spec.ts create mode 100644 backend/src/queue/queue-metrics.ts create mode 100644 backend/src/queue/redis-connection.spec.ts create mode 100644 backend/src/queue/worker-heartbeat.spec.ts create mode 100644 backend/src/queue/worker-heartbeat.ts create mode 100644 frontend/src/api/monitoring.api.ts create mode 100644 frontend/src/components/MeterBar.tsx create mode 100644 frontend/src/components/Sparkline.tsx create mode 100644 frontend/src/components/StatTile.tsx create mode 100644 frontend/src/hooks/usePolling.ts create mode 100644 frontend/src/interfaces/queueMonitoring.interface.ts create mode 100644 frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx create mode 100644 frontend/src/pages/admin/monitoring/QueueMonitoring.tsx create mode 100644 frontend/src/pages/admin/monitoring/sections/QueuesSection.tsx create mode 100644 frontend/src/pages/admin/monitoring/sections/RecentFailuresSection.tsx create mode 100644 frontend/src/pages/admin/monitoring/sections/RedisSection.tsx create mode 100644 frontend/src/pages/admin/monitoring/sections/WorkersSection.tsx create mode 100644 frontend/src/routes/admin/monitoring.tsx create mode 100644 frontend/src/scss/components/meter-bar.scss create mode 100644 frontend/src/scss/components/queue-monitoring.scss create mode 100644 frontend/src/scss/components/sparkline.scss create mode 100644 frontend/src/scss/components/stat-tile.scss create mode 100644 frontend/src/utils/monitoring.spec.ts create mode 100644 frontend/src/utils/monitoring.ts diff --git a/api-gateway/templates/routes.yaml b/api-gateway/templates/routes.yaml index e7d1f825d..0589be2ae 100644 --- a/api-gateway/templates/routes.yaml +++ b/api-gateway/templates/routes.yaml @@ -628,6 +628,24 @@ routes: request_buffering: true response_buffering: true + # ============================================================================ + # MONITORING ADMIN ROUTES (admin only) + # ============================================================================ + # /api/v1/frontend/admin/monitoring/queues (queue, worker and Redis health) + - name: ${ROUTE_PREFIX}admin-monitoring-queues-frontend + tags: [${CONFIG_TAG}] + hosts: + - ${GATEWAY_HOSTNAME} + paths: + - ${PATH_PREFIX}/api/v1/frontend/admin/monitoring/queues + methods: + - GET + strip_path: true + https_redirect_status_code: 426 + path_handling: v0 + request_buffering: true + response_buffering: true + # ============================================================================ # FEATURE FLAG ADMIN ROUTES (admin only) # ============================================================================ @@ -1679,6 +1697,16 @@ plugins: route: ${ROUTE_PREFIX}admin-users-list-frontend config: *jwtConfig + # ============================================================================ + # MONITORING ADMIN ROUTES PLUGINS + # ============================================================================ + # Admin Monitoring Queues - Frontend Admin Token + - name: jwt-keycloak + tags: [${CONFIG_TAG}] + enabled: true + route: ${ROUTE_PREFIX}admin-monitoring-queues-frontend + config: *jwtConfig + # ============================================================================ # FEATURE FLAG ADMIN ROUTES PLUGINS # ============================================================================ diff --git a/backend/src/api/admin/monitoring/monitoring-thresholds.ts b/backend/src/api/admin/monitoring/monitoring-thresholds.ts new file mode 100644 index 000000000..64d14b739 --- /dev/null +++ b/backend/src/api/admin/monitoring/monitoring-thresholds.ts @@ -0,0 +1,19 @@ +const intFromEnv = (name: string, fallback: number): number => { + const value = parseInt(process.env[name] || '', 10) + return Number.isFinite(value) ? value : fallback +} + +/** + * Where the queue monitoring page turns amber and red. Redis is capped with `noeviction`, so + * once memory is full every enqueue fails - the memory thresholds leave room to react first. + */ +export const MONITORING_THRESHOLDS = { + redisMemoryWarningPercent: intFromEnv('MONITORING_REDIS_MEMORY_WARNING_PERCENT', 70), + redisMemoryCriticalPercent: intFromEnv('MONITORING_REDIS_MEMORY_CRITICAL_PERCENT', 85), + oldestWaitingWarningMs: intFromEnv('MONITORING_OLDEST_WAITING_WARNING_SECONDS', 120) * 1000, + oldestWaitingCriticalMs: intFromEnv('MONITORING_OLDEST_WAITING_CRITICAL_SECONDS', 600) * 1000, + failureRateWarningPercent: intFromEnv('MONITORING_FAILURE_RATE_WARNING_PERCENT', 5), + failureRateCriticalPercent: intFromEnv('MONITORING_FAILURE_RATE_CRITICAL_PERCENT', 25), + /** Whole minutes averaged for in/out rates; the current partial minute is excluded. */ + rateWindowMinutes: 5, +} diff --git a/backend/src/api/admin/monitoring/monitoring.controller.spec.ts b/backend/src/api/admin/monitoring/monitoring.controller.spec.ts new file mode 100644 index 000000000..596d25653 --- /dev/null +++ b/backend/src/api/admin/monitoring/monitoring.controller.spec.ts @@ -0,0 +1,17 @@ +import { describe, it, expect } from 'vitest' +import { GUARDS_METADATA } from '@nestjs/common/constants' +import { MonitoringController } from './monitoring.controller' +import { NotifyAdminGuard } from '../../../common/guards/notify-admin.guard' +import { ROLES_KEY } from '../../../common/decorators/roles.decorator' +import { SsoRole } from '../../../enum/sso-role.enum' + +describe('MonitoringController', () => { + it('is guarded by NotifyAdminGuard', () => { + expect(Reflect.getMetadata(GUARDS_METADATA, MonitoringController)).toContain(NotifyAdminGuard) + }) + + it('requires the NOTIFY_ADMIN SSO role for the queue endpoint', () => { + const roles = Reflect.getMetadata(ROLES_KEY, MonitoringController.prototype.getQueues) + expect(roles).toEqual([SsoRole.NOTIFY_ADMIN]) + }) +}) diff --git a/backend/src/api/admin/monitoring/monitoring.controller.ts b/backend/src/api/admin/monitoring/monitoring.controller.ts new file mode 100644 index 000000000..99e2fb397 --- /dev/null +++ b/backend/src/api/admin/monitoring/monitoring.controller.ts @@ -0,0 +1,31 @@ +import { Controller, Get, UseGuards, Version } from '@nestjs/common' +import { ApiBearerAuth, ApiExcludeController, ApiOperation, ApiTags } from '@nestjs/swagger' +import { NotifyAdminGuard } from '../../../common/guards/notify-admin.guard' +import { Roles } from '../../../common/decorators/roles.decorator' +import { SsoRole } from '../../../enum/sso-role.enum' +import { MonitoringService } from './monitoring.service' +import type { QueueMonitoringResponseDto } from './schemas/queue-monitoring.dto' + +/** + * Admin monitoring of the Bull queues, their workers and Redis. + * + * Route: /api/v1/frontend/admin/monitoring + * Guarded by NotifyAdminGuard + NOTIFY_ADMIN (SSO role); not tenant-scoped. + */ +@ApiTags('monitoring') +// Not part of the service API; kept out of the published spec. +@ApiExcludeController() +@Controller('frontend/admin/monitoring') +@UseGuards(NotifyAdminGuard) +@ApiBearerAuth() +export class MonitoringController { + constructor(private readonly monitoringService: MonitoringService) {} + + @Version('1') + @Get('queues') + @Roles(SsoRole.NOTIFY_ADMIN) + @ApiOperation({ summary: 'Queue, worker and Redis health (admin only)' }) + getQueues(): Promise { + return this.monitoringService.getQueueMonitoring() + } +} diff --git a/backend/src/api/admin/monitoring/monitoring.module.ts b/backend/src/api/admin/monitoring/monitoring.module.ts new file mode 100644 index 000000000..d85c21ed0 --- /dev/null +++ b/backend/src/api/admin/monitoring/monitoring.module.ts @@ -0,0 +1,14 @@ +import { Module } from '@nestjs/common' +import { TypeOrmModule } from '@nestjs/typeorm' +import { Tenant } from '../tenants/entities/tenant.entity' +import { QueueModule } from '../../../queue/queue.module' +import { NotifyAdminGuard } from '../../../common/guards/notify-admin.guard' +import { MonitoringController } from './monitoring.controller' +import { MonitoringService } from './monitoring.service' + +@Module({ + imports: [TypeOrmModule.forFeature([Tenant]), QueueModule], + controllers: [MonitoringController], + providers: [MonitoringService, NotifyAdminGuard], +}) +export class MonitoringModule {} diff --git a/backend/src/api/admin/monitoring/monitoring.service.spec.ts b/backend/src/api/admin/monitoring/monitoring.service.spec.ts new file mode 100644 index 000000000..a280c9caa --- /dev/null +++ b/backend/src/api/admin/monitoring/monitoring.service.spec.ts @@ -0,0 +1,313 @@ +import { describe, it, expect, vi } from 'vitest' +import { ServiceUnavailableException } from '@nestjs/common' +import type Bull from 'bull' +import type Redis from 'ioredis' +import type { Repository } from 'typeorm' +import { + MonitoringService, + averageOfWholeMinutes, + buildRedisStats, + evaluateQueue, + maskPersonalData, + parseRedisInfo, + worstStatus, +} from './monitoring.service' +import { MONITORING_WINDOW_MINUTES } from '../../../queue/queue-metrics' +import type { Tenant } from '../tenants/entities/tenant.entity' + +const MB = 1024 * 1024 + +const healthyInfo = (overrides: Record = {}) => + parseRedisInfo( + Object.entries({ + redis_version: '7.2.4', + used_memory: 300 * MB, + used_memory_peak: 400 * MB, + maxmemory: 768 * MB, + maxmemory_policy: 'noeviction', + mem_fragmentation_ratio: '1.12', + evicted_keys: 0, + rejected_connections: 0, + connected_clients: 12, + blocked_clients: 4, + instantaneous_ops_per_sec: 250, + uptime_in_seconds: 3600, + ...overrides, + }) + .map(([key, value]) => `${key}:${value}`) + .join('\r\n'), + ) + +const series = (wholeMinuteValue: number, current = 0) => [ + ...Array(MONITORING_WINDOW_MINUTES - 1).fill(wholeMinuteValue), + current, +] + +const baseQueue = { + name: 'email-delivery', + isPaused: false, + counts: { waiting: 0, active: 0, delayed: 0, failed: 0, paused: 0 }, + oldestWaitingAgeMs: null, + throughput: { + added: series(10), + completed: series(10), + failed: series(0), + inPerMinute: 10, + outPerMinute: 10, + failureRatePercent: 0, + }, + liveWorkerPods: 2, +} + +describe('parseRedisInfo', () => { + it('reads field:value lines and skips section headers', () => { + expect( + parseRedisInfo('# Memory\r\nused_memory:1024\r\nmaxmemory_policy:noeviction\r\n'), + ).toEqual({ used_memory: '1024', maxmemory_policy: 'noeviction' }) + }) +}) + +describe('buildRedisStats', () => { + it('is healthy below the warning threshold', () => { + const stats = buildRedisStats(healthyInfo()) + expect(stats.status).toBe('healthy') + expect(stats.usedMemoryPercent).toBe(39.1) + expect(stats.reasons).toEqual([]) + }) + + it('warns at 70% of the memory cap and is critical at 85%', () => { + expect(buildRedisStats(healthyInfo({ used_memory: 560 * MB })).status).toBe('warning') + expect(buildRedisStats(healthyInfo({ used_memory: 660 * MB })).status).toBe('critical') + }) + + it('is critical when Redis could evict Bull keys', () => { + const stats = buildRedisStats(healthyInfo({ maxmemory_policy: 'allkeys-lru', evicted_keys: 3 })) + expect(stats.status).toBe('critical') + expect(stats.reasons).toHaveLength(2) + }) + + it('reports no percentage when Redis has no memory cap', () => { + const stats = buildRedisStats(healthyInfo({ maxmemory: 0 })) + expect(stats.maxMemoryBytes).toBeNull() + expect(stats.usedMemoryPercent).toBeNull() + expect(stats.status).toBe('healthy') + }) +}) + +describe('evaluateQueue', () => { + it('is healthy with an empty backlog and estimates no drain time', () => { + const queue = evaluateQueue(baseQueue) + expect(queue.status).toBe('healthy') + expect(queue.estimatedDrainMinutes).toBe(0) + }) + + it('estimates drain time from the gap between in and out rates', () => { + const queue = evaluateQueue({ + ...baseQueue, + counts: { ...baseQueue.counts, waiting: 100 }, + throughput: { ...baseQueue.throughput, inPerMinute: 10, outPerMinute: 30 }, + }) + expect(queue.estimatedDrainMinutes).toBe(5) + }) + + it('warns when jobs arrive faster than they finish', () => { + const queue = evaluateQueue({ + ...baseQueue, + counts: { ...baseQueue.counts, waiting: 100 }, + throughput: { ...baseQueue.throughput, inPerMinute: 30, outPerMinute: 10 }, + }) + expect(queue.status).toBe('warning') + expect(queue.estimatedDrainMinutes).toBeNull() + }) + + it('is critical when jobs are waiting and no pod is processing the queue', () => { + const queue = evaluateQueue({ + ...baseQueue, + counts: { ...baseQueue.counts, waiting: 5 }, + liveWorkerPods: 0, + }) + expect(queue.status).toBe('critical') + }) + + it('grades the oldest waiting job against both thresholds', () => { + const waiting = { ...baseQueue.counts, waiting: 1 } + expect( + evaluateQueue({ ...baseQueue, counts: waiting, oldestWaitingAgeMs: 3 * 60_000 }).status, + ).toBe('warning') + expect( + evaluateQueue({ ...baseQueue, counts: waiting, oldestWaitingAgeMs: 11 * 60_000 }).status, + ).toBe('critical') + }) + + it('ignores the failure rate until enough jobs have finished', () => { + const throughput = { ...baseQueue.throughput, outPerMinute: 0.2, failureRatePercent: 100 } + expect(evaluateQueue({ ...baseQueue, throughput }).status).toBe('healthy') + }) + + it('flags a high failure rate once there is enough volume', () => { + const throughput = { ...baseQueue.throughput, outPerMinute: 10, failureRatePercent: 30 } + expect(evaluateQueue({ ...baseQueue, throughput }).status).toBe('critical') + }) + + it('warns when the queue is paused', () => { + expect(evaluateQueue({ ...baseQueue, isPaused: true }).reasons).toContain('Queue is paused') + }) +}) + +describe('helpers', () => { + it('averages whole minutes and leaves out the current partial minute', () => { + expect(averageOfWholeMinutes([1, 2, 4, 6, 8, 10, 999], 5)).toBe(6) + }) + + it('picks the worst status', () => { + expect(worstStatus(['healthy', 'critical', 'warning'])).toBe('critical') + expect(worstStatus([])).toBe('healthy') + }) + + it('masks email addresses and phone numbers in failure reasons', () => { + expect( + maskPersonalData('Invalid recipient jane.doe@gov.bc.ca, sms +1 (250) 555-0123 rejected'), + ).toBe('Invalid recipient [email], sms [phone] rejected') + }) +}) + +describe('MonitoringService', () => { + const now = Date.now() + + function fakeQueue(name: string, overrides: Partial> = {}) { + return { + name, + getJobCounts: vi.fn().mockResolvedValue({ + waiting: 2, + active: 1, + completed: 50, + failed: 1, + delayed: 0, + paused: 0, + }), + isPaused: vi.fn().mockResolvedValue(false), + getMetrics: vi.fn().mockResolvedValue({ + meta: { count: 0, prevTS: 0, prevCount: 0 }, + data: [], + count: 0, + }), + getJob: vi.fn().mockResolvedValue({ timestamp: now - 30_000, opts: {} }), + getFailed: vi.fn().mockResolvedValue([]), + client: { + lindex: vi.fn().mockResolvedValue('job-1'), + mget: vi.fn((...keys: string[]) => Promise.resolve(keys.map(() => null))), + }, + ...overrides, + } as unknown as Bull.Queue + } + + function fakeRedis(workers: object[] = []) { + return { + info: vi + .fn() + .mockResolvedValue( + 'redis_version:7.2.4\r\nused_memory:1000\r\nmaxmemory:10000\r\nmaxmemory_policy:noeviction', + ), + zremrangebyscore: vi.fn().mockResolvedValue(0), + zrange: vi.fn().mockResolvedValue(workers.map((_, i) => `pod-${i}`)), + mget: vi.fn().mockResolvedValue(workers.map((worker) => JSON.stringify(worker))), + } as unknown as Redis + } + + const tenantRepository = { + find: vi.fn().mockResolvedValue([{ id: 't-1', name: 'Health Ministry' }]), + } as unknown as Repository + + const livePod = { + podId: 'pod-0', + startedAt: now - 60_000, + heartbeatAt: now - 2_000, + queues: [ + { + queue: 'email-delivery', + concurrency: 2, + active: 1, + completed: 40, + failed: 1, + lastFinishedAt: now - 1_000, + }, + ], + } + + it('reports Redis, every queue, live workers and the oldest waiting job', async () => { + const service = new MonitoringService( + fakeRedis([livePod]), + null, + fakeQueue('email-delivery'), + null, + null, + tenantRepository, + ) + + const result = await service.getQueueMonitoring() + + expect(result.redis.usedMemoryPercent).toBe(10) + expect(result.queues).toHaveLength(1) + expect(result.queues[0].liveWorkerPods).toBe(1) + expect(result.queues[0].oldestWaitingAgeMs).toBeGreaterThanOrEqual(30_000) + expect(result.workers[0].podId).toBe('pod-0') + expect(result.status).toBe('healthy') + }) + + it('returns recent failures newest first with tenant names and masked reasons', async () => { + const failedJob = (id: string, finishedOn: number) => ({ + id, + finishedOn, + attemptsMade: 3, + failedReason: 'CHES rejected bob@example.com', + data: { notifyId: `n-${id}`, tenantId: 't-1', request: { secret: 'body' } }, + }) + const queue = fakeQueue('email-delivery', { + getFailed: vi.fn().mockResolvedValue([failedJob('1', now - 5_000), failedJob('2', now)]), + }) + const service = new MonitoringService( + fakeRedis([livePod]), + null, + queue, + null, + null, + tenantRepository, + ) + + const { recentFailures } = await service.getQueueMonitoring() + + expect(recentFailures.map((failure) => failure.jobId)).toEqual(['2', '1']) + expect(recentFailures[0]).toEqual({ + queue: 'email-delivery', + jobId: '2', + notificationId: 'n-2', + tenantId: 't-1', + tenantName: 'Health Ministry', + reason: 'CHES rejected [email]', + attemptsMade: 3, + failedAt: new Date(now).toISOString(), + }) + }) + + it('drops pods whose heartbeat key has expired', async () => { + const redis = fakeRedis([livePod]) + ;(redis.zrange as ReturnType).mockResolvedValue(['pod-0', 'pod-gone']) + ;(redis.mget as ReturnType).mockResolvedValue([JSON.stringify(livePod), null]) + const service = new MonitoringService( + redis, + null, + fakeQueue('email-delivery'), + null, + null, + tenantRepository, + ) + + const { workers } = await service.getQueueMonitoring() + expect(workers.map((worker) => worker.podId)).toEqual(['pod-0']) + }) + + it('is unavailable when Redis is not configured', async () => { + const service = new MonitoringService(null, null, null, null, null, tenantRepository) + await expect(service.getQueueMonitoring()).rejects.toBeInstanceOf(ServiceUnavailableException) + }) +}) diff --git a/backend/src/api/admin/monitoring/monitoring.service.ts b/backend/src/api/admin/monitoring/monitoring.service.ts new file mode 100644 index 000000000..dcbd26abb --- /dev/null +++ b/backend/src/api/admin/monitoring/monitoring.service.ts @@ -0,0 +1,373 @@ +import { Inject, Injectable, Optional, ServiceUnavailableException } from '@nestjs/common' +import { InjectRepository } from '@nestjs/typeorm' +import { In, Repository } from 'typeorm' +import type Bull from 'bull' +import type Redis from 'ioredis' +import { QueueName } from '../../../enum/queue-name.enum' +import { ProviderToken } from '../../../enum/provider-token.enum' +import { Tenant } from '../tenants/entities/tenant.entity' +import { + MONITORING_WINDOW_MINUTES, + addedCounterKey, + alignBullMetrics, + bullKey, + minuteOf, +} from '../../../queue/queue-metrics' +import type { BullMetrics } from '../../../queue/queue-metrics' +import { + HEARTBEAT_TTL_SECONDS, + WORKER_INDEX_KEY, + workerHeartbeatKey, +} from '../../../queue/worker-heartbeat' +import type { WorkerHeartbeatPayload } from '../../../queue/worker-heartbeat' +import { MONITORING_THRESHOLDS } from './monitoring-thresholds' +import type { + HealthStatus, + QueueMonitoringResponseDto, + QueueStatsDto, + RecentFailureDto, + RedisStatsDto, + WorkerPodDto, +} from './schemas/queue-monitoring.dto' + +const FAILURES_PER_QUEUE = 10 +const RECENT_FAILURES_LIMIT = 20 +/** A failure rate over a handful of jobs is noise; below this many finished jobs it is ignored. */ +const MIN_JOBS_FOR_FAILURE_RATE = 10 + +const SEVERITY: Record = { healthy: 0, warning: 1, critical: 2 } + +export function worstStatus(statuses: HealthStatus[]): HealthStatus { + return statuses.reduce( + (worst, status) => (SEVERITY[status] > SEVERITY[worst] ? status : worst), + 'healthy', + ) +} + +/** Parse the `field:value` lines of Redis INFO into a flat map. */ +export function parseRedisInfo(info: string): Record { + const fields: Record = {} + for (const line of info.split(/\r?\n/)) { + const index = line.indexOf(':') + if (index > 0 && !line.startsWith('#')) { + fields[line.slice(0, index)] = line.slice(index + 1).trim() + } + } + return fields +} + +/** + * Bull's failure reasons are provider error messages, which can quote the recipient. The page + * is for spotting patterns, so addresses and numbers are masked before they leave the backend. + */ +export function maskPersonalData(text: string): string { + return text + .replace(/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g, '[email]') + .replace(/\+?\d[\d\s().-]{8,}\d/g, '[phone]') +} + +export function buildRedisStats(info: Record): RedisStatsDto { + const num = (key: string) => Number(info[key] ?? 0) || 0 + const usedMemoryBytes = num('used_memory') + const maxMemoryBytes = num('maxmemory') || null + const usedMemoryPercent = + maxMemoryBytes === null ? null : Math.round((usedMemoryBytes / maxMemoryBytes) * 1000) / 10 + const maxMemoryPolicy = info.maxmemory_policy ?? 'unknown' + const evictedKeys = num('evicted_keys') + const rejectedConnections = num('rejected_connections') + + const reasons: string[] = [] + const statuses: HealthStatus[] = [] + const { redisMemoryCriticalPercent, redisMemoryWarningPercent } = MONITORING_THRESHOLDS + + if (usedMemoryPercent !== null && usedMemoryPercent >= redisMemoryCriticalPercent) { + statuses.push('critical') + reasons.push(`Memory is at ${usedMemoryPercent}% of the cap`) + } else if (usedMemoryPercent !== null && usedMemoryPercent >= redisMemoryWarningPercent) { + statuses.push('warning') + reasons.push(`Memory is at ${usedMemoryPercent}% of the cap`) + } + if (maxMemoryPolicy !== 'noeviction') { + statuses.push('critical') + reasons.push(`Eviction policy is ${maxMemoryPolicy}; Bull jobs can be silently dropped`) + } + if (evictedKeys > 0) { + statuses.push('critical') + reasons.push(`${evictedKeys.toLocaleString()} keys have been evicted`) + } + if (rejectedConnections > 0) { + statuses.push('warning') + reasons.push(`${rejectedConnections.toLocaleString()} connections have been rejected`) + } + + const fragmentation = Number(info.mem_fragmentation_ratio) + return { + status: worstStatus(statuses), + reasons, + usedMemoryBytes, + peakMemoryBytes: num('used_memory_peak'), + maxMemoryBytes, + usedMemoryPercent, + maxMemoryPolicy, + fragmentationRatio: Number.isFinite(fragmentation) ? fragmentation : null, + evictedKeys, + rejectedConnections, + connectedClients: num('connected_clients'), + blockedClients: num('blocked_clients'), + opsPerSecond: num('instantaneous_ops_per_sec'), + uptimeSeconds: num('uptime_in_seconds'), + version: info.redis_version ?? 'unknown', + } +} + +/** Mean of the last `window` whole minutes; the final entry is the current partial minute. */ +export function averageOfWholeMinutes(series: number[], window: number): number { + const whole = series.slice(-(window + 1), -1) + if (whole.length === 0) return 0 + const total = whole.reduce((sum, value) => sum + value, 0) + return Math.round((total / whole.length) * 10) / 10 +} + +type QueueEvaluationInput = Omit + +export function evaluateQueue(input: QueueEvaluationInput): QueueStatsDto { + const { counts, throughput, oldestWaitingAgeMs, isPaused, liveWorkerPods } = input + const t = MONITORING_THRESHOLDS + const statuses: HealthStatus[] = [] + const reasons: string[] = [] + const backlog = counts.waiting + counts.paused + + if (isPaused) { + statuses.push('warning') + reasons.push('Queue is paused') + } + if (backlog > 0 && liveWorkerPods === 0) { + statuses.push('critical') + reasons.push('Jobs are waiting but no pod is processing this queue') + } + if (oldestWaitingAgeMs !== null && oldestWaitingAgeMs >= t.oldestWaitingCriticalMs) { + statuses.push('critical') + reasons.push('Oldest job has waited longer than the critical threshold') + } else if (oldestWaitingAgeMs !== null && oldestWaitingAgeMs >= t.oldestWaitingWarningMs) { + statuses.push('warning') + reasons.push('Oldest job has waited longer than the warning threshold') + } + + const recentFinished = throughput.outPerMinute * t.rateWindowMinutes + if (recentFinished >= MIN_JOBS_FOR_FAILURE_RATE) { + if (throughput.failureRatePercent >= t.failureRateCriticalPercent) { + statuses.push('critical') + reasons.push(`${throughput.failureRatePercent}% of recent jobs failed`) + } else if (throughput.failureRatePercent >= t.failureRateWarningPercent) { + statuses.push('warning') + reasons.push(`${throughput.failureRatePercent}% of recent jobs failed`) + } + } + + const growing = throughput.inPerMinute > throughput.outPerMinute + if (backlog > 0 && growing) { + statuses.push('warning') + reasons.push('Jobs are arriving faster than they are finishing') + } + + let estimatedDrainMinutes: number | null = null + if (backlog === 0) { + estimatedDrainMinutes = 0 + } else if (throughput.outPerMinute > throughput.inPerMinute) { + estimatedDrainMinutes = Math.ceil(backlog / (throughput.outPerMinute - throughput.inPerMinute)) + } + + return { ...input, status: worstStatus(statuses), reasons, estimatedDrainMinutes } +} + +type JobIdentifiers = { notifyId?: unknown; notificationId?: unknown; tenantId?: unknown } + +const asString = (value: unknown): string | null => (typeof value === 'string' ? value : null) + +/** + * Read-only view of the Bull queues, their workers and the Redis instance behind them, for the + * admin monitoring page. Everything is read from Redis on each request, so the answer is the + * same whichever pod serves it. + */ +@Injectable() +export class MonitoringService { + private readonly queues: Bull.Queue[] + + constructor( + @Optional() @Inject(ProviderToken.REDIS_CLIENT) private readonly redis: Redis | null, + @Optional() @Inject(QueueName.INGESTION) ingestionQueue: Bull.Queue | null, + @Optional() @Inject(QueueName.EMAIL_DELIVERY) emailQueue: Bull.Queue | null, + @Optional() @Inject(QueueName.SMS_DELIVERY) smsQueue: Bull.Queue | null, + @Optional() @Inject(QueueName.WEBHOOK_DELIVERY) webhookQueue: Bull.Queue | null, + @InjectRepository(Tenant) private readonly tenantRepository: Repository, + ) { + this.queues = [ingestionQueue, emailQueue, smsQueue, webhookQueue].filter( + (queue): queue is Bull.Queue => !!queue, + ) + } + + async getQueueMonitoring(): Promise { + if (!this.redis || this.queues.length === 0) { + throw new ServiceUnavailableException( + 'Queue monitoring is unavailable: Redis is not configured', + ) + } + + const now = Date.now() + const [info, workers] = await Promise.all([this.redis.info(), this.getWorkers(now)]) + const redis = buildRedisStats(parseRedisInfo(info)) + const queues = await Promise.all( + this.queues.map((queue) => this.getQueueStats(queue, workers, now)), + ) + const recentFailures = await this.getRecentFailures() + + return { + generatedAt: new Date(now).toISOString(), + status: worstStatus([redis.status, ...queues.map((queue) => queue.status)]), + redis, + queues, + workers, + recentFailures, + } + } + + private async getQueueStats( + queue: Bull.Queue, + workers: WorkerPodDto[], + now: number, + ): Promise { + const [counts, isPaused, oldestWaitingAgeMs, added, completed, failed] = await Promise.all([ + queue.getJobCounts(), + queue.isPaused(), + this.getOldestWaitingAge(queue, now), + this.getAddedSeries(queue, now), + queue.getMetrics('completed', 0, MONITORING_WINDOW_MINUTES - 1), + queue.getMetrics('failed', 0, MONITORING_WINDOW_MINUTES - 1), + ]) + + const completedSeries = alignBullMetrics(completed as BullMetrics, now) + const failedSeries = alignBullMetrics(failed as BullMetrics, now) + const window = MONITORING_THRESHOLDS.rateWindowMinutes + const completedPerMinute = averageOfWholeMinutes(completedSeries, window) + const failedPerMinute = averageOfWholeMinutes(failedSeries, window) + const outPerMinute = Math.round((completedPerMinute + failedPerMinute) * 10) / 10 + const failureRatePercent = + outPerMinute > 0 ? Math.round((failedPerMinute / outPerMinute) * 1000) / 10 : 0 + + return evaluateQueue({ + name: queue.name, + isPaused, + counts: { + waiting: counts.waiting, + active: counts.active, + delayed: counts.delayed, + failed: counts.failed, + paused: (counts as Bull.JobCounts & { paused?: number }).paused ?? 0, + }, + oldestWaitingAgeMs, + throughput: { + added, + completed: completedSeries, + failed: failedSeries, + inPerMinute: averageOfWholeMinutes(added, window), + outPerMinute, + failureRatePercent, + }, + liveWorkerPods: workers.filter((pod) => pod.queues.some((q) => q.queue === queue.name)) + .length, + }) + } + + /** + * Workers take from the right of the wait list, so its last entry is the next job out. A + * delayed job (scheduled send or retry backoff) only became ready at timestamp + delay, which + * is what its wait is measured from. Pausing moves the wait list to `paused`. + */ + private async getOldestWaitingAge(queue: Bull.Queue, now: number): Promise { + const jobId = + (await queue.client.lindex(bullKey(queue.name, 'wait'), -1)) ?? + (await queue.client.lindex(bullKey(queue.name, 'paused'), -1)) + if (!jobId) return null + const job = await queue.getJob(jobId) + if (!job) return null + const readyAt = job.timestamp + (job.opts?.delay ?? 0) + return Math.max(0, now - readyAt) + } + + private async getAddedSeries(queue: Bull.Queue, now: number): Promise { + const currentMinute = minuteOf(now) + const keys: string[] = [] + for (let offset = MONITORING_WINDOW_MINUTES - 1; offset >= 0; offset--) { + keys.push(addedCounterKey(queue.name, currentMinute - offset)) + } + const values = await queue.client.mget(...keys) + return values.map((value) => Number(value ?? 0) || 0) + } + + private async getWorkers(now: number): Promise { + const redis = this.redis as Redis + // Pods that stopped without cleaning up stay in the index until pruned here. + await redis.zremrangebyscore(WORKER_INDEX_KEY, '-inf', now - HEARTBEAT_TTL_SECONDS * 2000) + const podIds = await redis.zrange(WORKER_INDEX_KEY, 0, -1) + if (podIds.length === 0) return [] + + const payloads = await redis.mget(...podIds.map(workerHeartbeatKey)) + return payloads + .filter((payload): payload is string => !!payload) + .map((payload) => JSON.parse(payload) as WorkerHeartbeatPayload) + .map((pod) => ({ + podId: pod.podId, + startedAt: new Date(pod.startedAt).toISOString(), + lastHeartbeatAt: new Date(pod.heartbeatAt).toISOString(), + queues: pod.queues.map((q) => ({ + ...q, + lastFinishedAt: q.lastFinishedAt ? new Date(q.lastFinishedAt).toISOString() : null, + })), + })) + .sort((a, b) => a.podId.localeCompare(b.podId)) + } + + private async getRecentFailures(): Promise { + const perQueue = await Promise.all( + this.queues.map(async (queue) => { + const jobs = await queue.getFailed(0, FAILURES_PER_QUEUE - 1) + return jobs.filter(Boolean).map((job) => ({ queue: queue.name, job })) + }), + ) + + const failures = perQueue + .flat() + .sort((a, b) => (b.job.finishedOn ?? 0) - (a.job.finishedOn ?? 0)) + .slice(0, RECENT_FAILURES_LIMIT) + + const tenantIds = [ + ...new Set( + failures + .map(({ job }) => asString((job.data as JobIdentifiers)?.tenantId)) + .filter((id): id is string => !!id), + ), + ] + const tenants = tenantIds.length + ? await this.tenantRepository.find({ + where: { id: In(tenantIds) }, + select: { id: true, name: true }, + }) + : [] + const tenantNames = new Map(tenants.map((tenant) => [tenant.id, tenant.name])) + + return failures.map(({ queue, job }) => { + const data = (job.data ?? {}) as JobIdentifiers + const tenantId = asString(data.tenantId) + return { + queue, + jobId: String(job.id), + notificationId: asString(data.notifyId) ?? asString(data.notificationId), + tenantId, + tenantName: tenantId ? (tenantNames.get(tenantId) ?? null) : null, + reason: maskPersonalData(job.failedReason ?? 'No reason recorded'), + attemptsMade: job.attemptsMade, + failedAt: job.finishedOn ? new Date(job.finishedOn).toISOString() : null, + } + }) + } +} diff --git a/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts b/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts new file mode 100644 index 000000000..628aa4260 --- /dev/null +++ b/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts @@ -0,0 +1,97 @@ +/** + * Response shape for GET /api/v1/frontend/admin/monitoring/queues. + * Mirrored by frontend/src/interfaces/queueMonitoring.interface.ts. + */ + +export type HealthStatus = 'healthy' | 'warning' | 'critical' + +export interface StatusReason { + status: HealthStatus + /** Plain-language reason shown next to the badge; empty when healthy. */ + reasons: string[] +} + +export interface RedisStatsDto extends StatusReason { + usedMemoryBytes: number + peakMemoryBytes: number + /** Null when Redis has no memory cap configured. */ + maxMemoryBytes: number | null + usedMemoryPercent: number | null + maxMemoryPolicy: string + fragmentationRatio: number | null + evictedKeys: number + rejectedConnections: number + connectedClients: number + blockedClients: number + opsPerSecond: number + uptimeSeconds: number + version: string +} + +export interface QueueCountsDto { + waiting: number + active: number + delayed: number + failed: number + paused: number +} + +export interface QueueThroughputDto { + /** Per-minute counts, oldest first; the last entry is the current, partial minute. */ + added: number[] + completed: number[] + failed: number[] + /** Averages over the last few whole minutes. */ + inPerMinute: number + outPerMinute: number + failureRatePercent: number +} + +export interface QueueStatsDto extends StatusReason { + name: string + isPaused: boolean + counts: QueueCountsDto + /** Age of the job that has waited longest, or null when nothing is waiting. */ + oldestWaitingAgeMs: number | null + throughput: QueueThroughputDto + /** Minutes until the backlog clears at the current rates; null when it is not shrinking. */ + estimatedDrainMinutes: number | null + liveWorkerPods: number +} + +export interface WorkerQueueDto { + queue: string + concurrency: number + active: number + completed: number + failed: number + lastFinishedAt: string | null +} + +export interface WorkerPodDto { + podId: string + startedAt: string + lastHeartbeatAt: string + queues: WorkerQueueDto[] +} + +export interface RecentFailureDto { + queue: string + jobId: string + notificationId: string | null + tenantId: string | null + tenantName: string | null + /** Bull's failure reason with email addresses and phone numbers masked. */ + reason: string + attemptsMade: number + failedAt: string | null +} + +export interface QueueMonitoringResponseDto { + generatedAt: string + status: HealthStatus + redis: RedisStatsDto + queues: QueueStatsDto[] + workers: WorkerPodDto[] + recentFailures: RecentFailureDto[] +} diff --git a/backend/src/queue/queue-metrics.spec.ts b/backend/src/queue/queue-metrics.spec.ts new file mode 100644 index 000000000..80e04343d --- /dev/null +++ b/backend/src/queue/queue-metrics.spec.ts @@ -0,0 +1,86 @@ +import { describe, it, expect, vi } from 'vitest' +import type Bull from 'bull' +import { + MONITORING_WINDOW_MINUTES, + addedCounterKey, + alignBullMetrics, + countAddedJobs, + minuteOf, +} from './queue-metrics' + +const MINUTE = 60_000 + +describe('alignBullMetrics', () => { + const now = 1_000 * MINUTE + 30_000 + + it('returns a zeroed window when the queue has never finished a job', () => { + const series = alignBullMetrics({ meta: { count: 0, prevTS: 0, prevCount: 0 }, data: [] }, now) + expect(series).toHaveLength(MONITORING_WINDOW_MINUTES) + expect(series.every((value) => value === 0)).toBe(true) + }) + + it('puts unflushed jobs in the current minute and flushed points behind it', () => { + const series = alignBullMetrics( + { meta: { count: 12, prevTS: now - 5_000, prevCount: 9 }, data: ['4', '7'] }, + now, + ) + expect(series.at(-1)).toBe(3) + expect(series.at(-2)).toBe(4) + expect(series.at(-3)).toBe(7) + }) + + it('anchors points to the last flush, not to now, when the queue has gone quiet', () => { + // Last job finished three minutes ago; Bull has not written a point since. + const series = alignBullMetrics( + { meta: { count: 10, prevTS: now - 3 * MINUTE, prevCount: 8 }, data: ['5'] }, + now, + ) + expect(series.slice(-3)).toEqual([0, 0, 0]) + expect(series.at(-4)).toBe(2) + expect(series.at(-5)).toBe(5) + }) +}) + +describe('countAddedJobs', () => { + function fakeQueue() { + const exec = vi.fn().mockResolvedValue([]) + const multi = { incr: vi.fn(), expire: vi.fn(), exec } + multi.incr.mockReturnValue(multi) + multi.expire.mockReturnValue(multi) + const job = { id: '1' } + const queue = { + name: 'email-delivery', + add: vi.fn().mockResolvedValue(job), + client: { multi: vi.fn(() => multi) }, + } + return { queue, multi, job, exec } + } + + it('increments the current minute counter after a job is added', async () => { + const { queue, multi, job } = fakeQueue() + countAddedJobs(queue as unknown as Bull.Queue) + + await expect(queue.add({ notifyId: 'n1' })).resolves.toBe(job) + + const key = addedCounterKey('email-delivery', minuteOf(Date.now())) + expect(multi.incr).toHaveBeenCalledWith(key) + expect(multi.expire).toHaveBeenCalledWith(key, expect.any(Number)) + }) + + it('does not fail the add when counting fails', async () => { + const { queue, job, exec } = fakeQueue() + exec.mockRejectedValue(new Error('redis down')) + countAddedJobs(queue as unknown as Bull.Queue) + + await expect(queue.add({})).resolves.toBe(job) + }) + + it('does not count a job whose add failed', async () => { + const { queue, multi } = fakeQueue() + queue.add.mockRejectedValue(new Error('OOM command not allowed')) + countAddedJobs(queue as unknown as Bull.Queue) + + await expect(queue.add({})).rejects.toThrow('OOM') + expect(multi.incr).not.toHaveBeenCalled() + }) +}) diff --git a/backend/src/queue/queue-metrics.ts b/backend/src/queue/queue-metrics.ts new file mode 100644 index 000000000..ca389bc0a --- /dev/null +++ b/backend/src/queue/queue-metrics.ts @@ -0,0 +1,90 @@ +import type Bull from 'bull' +import { Logger } from '@nestjs/common' +import { REDIS_KEY_PREFIX } from '../common/redis/redis-namespace' + +const logger = new Logger('QueueMetrics') + +const MINUTE_MS = 60_000 + +/** + * Per-minute "added" counters live this long. Bull records completed/failed itself (see + * QUEUE_METRICS_MAX_DATA_POINTS) but has no equivalent for adds, so the fill rate is ours. + */ +const ADDED_COUNTER_TTL_SECONDS = 2 * 60 * 60 + +/** Minutes of history the monitoring endpoint returns per series. */ +export const MONITORING_WINDOW_MINUTES = 60 + +export function minuteOf(timestampMs: number): number { + return Math.floor(timestampMs / MINUTE_MS) +} + +/** The key Bull itself uses for `suffix` on this queue, given our `prefix` option. */ +export function bullKey(queueName: string, suffix: string): string { + return `${REDIS_KEY_PREFIX}:${queueName}:${suffix}` +} + +export function addedCounterKey(queueName: string, minute: number): string { + return bullKey(queueName, `metrics:added:${minute}`) +} + +/** + * Count every job added to `queue` into a per-minute Redis counter. + * + * Jobs are added from a dozen call sites, so this wraps `add` once where the queue is built + * rather than at each of them. Counting is best-effort: a Redis error is logged and never + * fails the add, which has already succeeded by then. A re-add Bull deduplicates by jobId is + * still counted, which slightly overstates the fill rate during a pending-sweep retry. + */ +export function countAddedJobs(queue: Bull.Queue): void { + const add = queue.add.bind(queue) as (...args: unknown[]) => Promise + const counted = async (...args: unknown[]): Promise => { + const job = await add(...args) + const key = addedCounterKey(queue.name, minuteOf(Date.now())) + queue.client + .multi() + .incr(key) + .expire(key, ADDED_COUNTER_TTL_SECONDS) + .exec() + .catch((error: Error) => { + logger.warn(`Failed to count added job on ${queue.name}: ${error.message}`) + }) + return job + } + queue.add = counted as Bull.Queue['add'] +} + +/** Bull's `getMetrics()` result, as returned at runtime (list entries are strings). */ +export interface BullMetrics { + meta: { count: number; prevTS: number; prevCount: number } + data: Array +} + +/** + * Re-align Bull's completed/failed data points onto wall-clock minutes, oldest first. + * + * Bull only writes a data point when a job finishes in a later minute than the previous one, + * so its list is anchored to `prevTS`, not to now: `data[i]` is minute(prevTS) - 1 - i, and + * jobs finished in minute(prevTS) are still held as `count - prevCount`. Reading the list as + * if index 0 were the current minute shifts every point by however long the queue has been + * quiet. + */ +export function alignBullMetrics(metrics: BullMetrics, nowMs: number): number[] { + const currentMinute = minuteOf(nowMs) + const flushedMinute = minuteOf(metrics.meta.prevTS) + const pending = Math.max(0, metrics.meta.count - metrics.meta.prevCount) + const series: number[] = [] + + for (let offset = MONITORING_WINDOW_MINUTES - 1; offset >= 0; offset--) { + const minute = currentMinute - offset + if (!metrics.meta.prevTS || minute > flushedMinute) { + series.push(0) + } else if (minute === flushedMinute) { + series.push(pending) + } else { + const point = metrics.data[flushedMinute - 1 - minute] + series.push(point === undefined ? 0 : Number(point) || 0) + } + } + return series +} diff --git a/backend/src/queue/queue.module.ts b/backend/src/queue/queue.module.ts index eb8167681..d7ea910ae 100644 --- a/backend/src/queue/queue.module.ts +++ b/backend/src/queue/queue.module.ts @@ -1,10 +1,20 @@ -import { Module, OnModuleInit, Inject, Logger, Optional, forwardRef } from '@nestjs/common' +import { + Module, + OnModuleInit, + OnModuleDestroy, + Inject, + Logger, + Optional, + forwardRef, +} from '@nestjs/common' import { ConfigService } from '@nestjs/config' import { InjectRepository, TypeOrmModule } from '@nestjs/typeorm' import { Repository } from 'typeorm' import type Bull from 'bull' +import type Redis from 'ioredis' import { QueueName } from '../enum/queue-name.enum' import { createQueue, createRedisClient } from './redis-connection' +import { WorkerHeartbeat } from './worker-heartbeat' import { ProviderToken } from '../enum/provider-token.enum' import { IngestionWorker } from './workers/ingestion.worker' import { EmailDeliveryWorker } from './workers/email-delivery.worker' @@ -157,8 +167,9 @@ import { PhoneNumberService } from '../api/notify/services/phone-number.service' QueueName.WEBHOOK_DELIVERY, ], }) -export class QueueModule implements OnModuleInit { +export class QueueModule implements OnModuleInit, OnModuleDestroy { private readonly logger = new Logger(QueueModule.name) + private heartbeat?: WorkerHeartbeat constructor( @Inject(QueueName.INGESTION) private ingestionQueue?: Bull.Queue, @@ -183,8 +194,15 @@ export class QueueModule implements OnModuleInit { private readonly webhookService?: WebhookService, private readonly webhookDeliveryLogRepository?: WebhookDeliveryLogRepository, @Optional() private readonly structuredLogger?: StructuredLoggerService, + @Optional() + @Inject(ProviderToken.REDIS_CLIENT) + private readonly redisClient?: Redis | null, ) {} + onModuleDestroy(): void { + this.heartbeat?.stop() + } + async onModuleInit() { // Skip queue initialization if queues are not available (e.g., in tests without Redis) if (!this.ingestionQueue || !this.emailQueue || !this.smsQueue) { @@ -201,6 +219,9 @@ export class QueueModule implements OnModuleInit { const concurrency = this.configService?.get('queue.ingestionWorkerConcurrency') || 1 this.logger.debug(`Ingestion worker concurrency: ${concurrency}`) + // Published per pod so the admin monitoring page can list every pod's workers. + this.heartbeat = this.redisClient ? new WorkerHeartbeat(this.redisClient) : undefined + // Initialize workers in background - don't block app startup // Workers will be ready when first job is queued try { @@ -218,6 +239,7 @@ export class QueueModule implements OnModuleInit { this.attachmentService, this.phoneNumberService, ) + this.heartbeat?.track(this.ingestionQueue, concurrency) this.logger.debug('Ingestion worker initialization started') this.logger.debug('About to initialize email delivery worker...') @@ -240,6 +262,7 @@ export class QueueModule implements OnModuleInit { this.structuredLogger, this.tenantSettingsService, ) + this.heartbeat?.track(this.emailQueue, emailConcurrency) this.logger.log('Email delivery worker initialization started') this.logger.log('About to initialize SMS delivery worker...') @@ -260,6 +283,7 @@ export class QueueModule implements OnModuleInit { smsConcurrency, this.structuredLogger, ) + this.heartbeat?.track(this.smsQueue, smsConcurrency) this.logger.log('SMS delivery worker initialization started') // Initialize webhook delivery worker — handles HTTP POST callbacks @@ -272,9 +296,12 @@ export class QueueModule implements OnModuleInit { this.webhookDeliveryLogRepository, webhookConcurrency, ) + this.heartbeat?.track(this.webhookQueue, webhookConcurrency) this.logger.log('Webhook delivery worker initialization started') } + this.heartbeat?.start() + this.logger.log('Queue workers initialized successfully') } catch (error) { this.logger.error( diff --git a/backend/src/queue/redis-connection.spec.ts b/backend/src/queue/redis-connection.spec.ts new file mode 100644 index 000000000..adfa3f488 --- /dev/null +++ b/backend/src/queue/redis-connection.spec.ts @@ -0,0 +1,34 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { QueueName } from '../enum/queue-name.enum' + +const bullConstructor = vi.fn() + +vi.mock('bull', () => ({ + default: vi.fn().mockImplementation(function (name: string, opts: unknown) { + bullConstructor(name, opts) + return { name, on: vi.fn(), add: vi.fn() } + }), +})) + +import { createQueue, QUEUE_METRICS_MAX_DATA_POINTS } from './redis-connection' + +describe('createQueue', () => { + beforeEach(() => { + bullConstructor.mockClear() + }) + + it('enables Bull metrics so completed/failed throughput can be read back per minute', () => { + createQueue(QueueName.EMAIL_DELIVERY, { host: 'localhost', port: 6379, db: 0 }) + + expect(bullConstructor).toHaveBeenCalledWith( + QueueName.EMAIL_DELIVERY, + expect.objectContaining({ + metrics: { maxDataPoints: QUEUE_METRICS_MAX_DATA_POINTS }, + }), + ) + }) + + it('keeps a day of one-minute data points by default', () => { + expect(QUEUE_METRICS_MAX_DATA_POINTS).toBe(1440) + }) +}) diff --git a/backend/src/queue/redis-connection.ts b/backend/src/queue/redis-connection.ts index 17939b97b..e7a44de5a 100644 --- a/backend/src/queue/redis-connection.ts +++ b/backend/src/queue/redis-connection.ts @@ -3,6 +3,7 @@ import Redis from 'ioredis' import { attachRedisErrorLogging } from '../common/redis/redis-error.util' import { REDIS_KEY_PREFIX } from '../common/redis/redis-namespace' import { QueueName } from '../enum/queue-name.enum' +import { countAddedJobs } from './queue-metrics' /** Shape of the `redis` block in configuration.ts. */ export interface RedisConfig { @@ -32,6 +33,15 @@ export function buildRedisOptions( return options } +/** + * Minutes of completed/failed counts Bull keeps per queue, read back with `queue.getMetrics()`. + * Each point is one integer in a capped Redis list, so a day of history is a few KB per queue. + */ +export const QUEUE_METRICS_MAX_DATA_POINTS = parseInt( + process.env.QUEUE_METRICS_MAX_DATA_POINTS || '1440', + 10, +) + /** * Create a Bull queue with an error listener attached. * @@ -48,8 +58,10 @@ export function createQueue(name: QueueName, redisConfig: RedisConfig): Bull.Que enableReadyCheck: false, maxRetriesPerRequest: null, }), + metrics: { maxDataPoints: QUEUE_METRICS_MAX_DATA_POINTS }, }) attachRedisErrorLogging(queue, `Queue[${name}]`) + countAddedJobs(queue) return queue } diff --git a/backend/src/queue/worker-heartbeat.spec.ts b/backend/src/queue/worker-heartbeat.spec.ts new file mode 100644 index 000000000..2b508d5c6 --- /dev/null +++ b/backend/src/queue/worker-heartbeat.spec.ts @@ -0,0 +1,75 @@ +import { describe, it, expect, vi } from 'vitest' +import { EventEmitter } from 'events' +import type Bull from 'bull' +import type Redis from 'ioredis' +import { + HEARTBEAT_TTL_SECONDS, + WORKER_INDEX_KEY, + WorkerHeartbeat, + workerHeartbeatKey, +} from './worker-heartbeat' +import type { WorkerHeartbeatPayload } from './worker-heartbeat' + +function fakeRedis() { + const multi = { set: vi.fn(), zadd: vi.fn(), exec: vi.fn().mockResolvedValue([]) } + multi.set.mockReturnValue(multi) + multi.zadd.mockReturnValue(multi) + return { redis: { multi: vi.fn(() => multi) } as unknown as Redis, multi } +} + +function fakeQueue(name: string) { + return Object.assign(new EventEmitter(), { name }) as unknown as Bull.Queue +} + +describe('WorkerHeartbeat', () => { + it('publishes per-queue activity with an expiring key and indexes the pod', async () => { + const { redis, multi } = fakeRedis() + const heartbeat = new WorkerHeartbeat(redis) + const queue = fakeQueue('email-delivery') + heartbeat.track(queue, 2) + + queue.emit('active') + queue.emit('active') + queue.emit('completed') + queue.emit('active') + queue.emit('failed') + await heartbeat.beat() + + const [key, json, mode, ttl] = multi.set.mock.calls[0] + const payload = JSON.parse(json) as WorkerHeartbeatPayload + expect(key).toBe(workerHeartbeatKey(payload.podId)) + expect(mode).toBe('EX') + expect(ttl).toBe(HEARTBEAT_TTL_SECONDS) + expect(payload.queues).toEqual([ + { + queue: 'email-delivery', + concurrency: 2, + active: 1, + completed: 1, + failed: 1, + lastFinishedAt: expect.any(Number), + }, + ]) + expect(multi.zadd).toHaveBeenCalledWith(WORKER_INDEX_KEY, payload.heartbeatAt, payload.podId) + }) + + it('never lets the active count go negative', async () => { + const { redis, multi } = fakeRedis() + const heartbeat = new WorkerHeartbeat(redis) + const queue = fakeQueue('sms-delivery') + heartbeat.track(queue, 1) + + // A job that was already active before this pod started tracking. + queue.emit('completed') + await heartbeat.beat() + + const payload = JSON.parse(multi.set.mock.calls[0][1]) as WorkerHeartbeatPayload + expect(payload.queues[0].active).toBe(0) + }) + + it('swallows Redis errors so a heartbeat never crashes the pod', async () => { + const { redis, multi } = fakeRedis() + multi.exec.mockRejectedValue(new Error('connection lost')) + await expect(new WorkerHeartbeat(redis).beat()).resolves.toBeUndefined() + }) +}) diff --git a/backend/src/queue/worker-heartbeat.ts b/backend/src/queue/worker-heartbeat.ts new file mode 100644 index 000000000..9a4692955 --- /dev/null +++ b/backend/src/queue/worker-heartbeat.ts @@ -0,0 +1,114 @@ +import type Bull from 'bull' +import type Redis from 'ioredis' +import { hostname } from 'os' +import { Logger } from '@nestjs/common' +import { redisKey } from '../common/redis/redis-namespace' + +/** How often each pod republishes its worker state. */ +export const HEARTBEAT_INTERVAL_MS = 10_000 + +/** A pod that misses this many intervals drops out of the live list. */ +export const HEARTBEAT_TTL_SECONDS = 30 + +/** Index of pods that have published a heartbeat, scored by the time of their last one. */ +export const WORKER_INDEX_KEY = redisKey('monitoring:workers') + +export function workerHeartbeatKey(podId: string): string { + return redisKey(`monitoring:workers:${podId}`) +} + +export interface WorkerQueueState { + queue: string + concurrency: number + active: number + completed: number + failed: number + lastFinishedAt: number | null +} + +export interface WorkerHeartbeatPayload { + podId: string + startedAt: number + heartbeatAt: number + queues: WorkerQueueState[] +} + +const logger = new Logger('WorkerHeartbeat') + +/** + * Publishes what this pod's Bull workers are doing so the admin monitoring page can list every + * pod's workers, not just those of the pod that served the request. + * + * The counts are this pod's own and are meant to disappear with it: the key expires + * HEARTBEAT_TTL_SECONDS after the last write, so a crashed or scaled-down pod falls off the + * list on its own. `completed` and `failed` are since this pod started, not lifetime totals. + * Bull's `getWorkers()` is not used because it depends on CLIENT LIST, and returns undefined + * rather than failing when a managed Redis disallows it. + */ +export class WorkerHeartbeat { + private readonly podId = process.env.HOSTNAME || hostname() + private readonly startedAt = Date.now() + private readonly states = new Map() + private timer?: NodeJS.Timeout + + constructor(private readonly redis: Redis) {} + + /** Track a queue this pod processes. Call once per queue, after its worker is registered. */ + track(queue: Bull.Queue, concurrency: number): void { + const state: WorkerQueueState = { + queue: queue.name, + concurrency, + active: 0, + completed: 0, + failed: 0, + lastFinishedAt: null, + } + this.states.set(queue.name, state) + + queue.on('active', () => { + state.active++ + }) + queue.on('completed', () => { + state.active = Math.max(0, state.active - 1) + state.completed++ + state.lastFinishedAt = Date.now() + }) + queue.on('failed', () => { + state.active = Math.max(0, state.active - 1) + state.failed++ + state.lastFinishedAt = Date.now() + }) + } + + start(): void { + if (this.timer) return + void this.beat() + this.timer = setInterval(() => void this.beat(), HEARTBEAT_INTERVAL_MS) + // Never hold the process open just to send a heartbeat. + this.timer.unref() + } + + stop(): void { + if (this.timer) clearInterval(this.timer) + this.timer = undefined + } + + async beat(): Promise { + const now = Date.now() + const payload: WorkerHeartbeatPayload = { + podId: this.podId, + startedAt: this.startedAt, + heartbeatAt: now, + queues: [...this.states.values()], + } + try { + await this.redis + .multi() + .set(workerHeartbeatKey(this.podId), JSON.stringify(payload), 'EX', HEARTBEAT_TTL_SECONDS) + .zadd(WORKER_INDEX_KEY, now, this.podId) + .exec() + } catch (error) { + logger.warn(`Failed to publish worker heartbeat: ${(error as Error).message}`) + } + } +} diff --git a/frontend/src/api/monitoring.api.ts b/frontend/src/api/monitoring.api.ts new file mode 100644 index 000000000..273dcf5fa --- /dev/null +++ b/frontend/src/api/monitoring.api.ts @@ -0,0 +1,24 @@ +import type { AxiosError } from 'axios' +import { get, generateApiParameters, STATUS_CODES } from '@/common/api' +import type { QueueMonitoring } from '@/interfaces/queueMonitoring.interface' + +/** + * Queue, worker and Redis health for the admin monitoring page. Requires NOTIFY_ADMIN. + * Polled by the page itself; nothing else reads it, so it does not go through a slice. + */ +export async function getQueueMonitoring(): Promise { + try { + const params = generateApiParameters('/api/v1/frontend/admin/monitoring/queues') + return await get(params) + } catch (error) { + const status = (error as AxiosError).response?.status + if (status === STATUS_CODES.Unauthorized || status === STATUS_CODES.Forbidden) { + throw new Error('You do not have permission to view queue monitoring') + } + const message = ((error as AxiosError).response?.data as { message?: string } | undefined) + ?.message + throw new Error( + `Failed to load queue monitoring: ${message || (error instanceof Error ? error.message : 'Unknown error')}`, + ) + } +} diff --git a/frontend/src/components/MeterBar.tsx b/frontend/src/components/MeterBar.tsx new file mode 100644 index 000000000..97398ef57 --- /dev/null +++ b/frontend/src/components/MeterBar.tsx @@ -0,0 +1,35 @@ +import type { FC } from 'react' +import type { HealthStatus } from '@/interfaces/queueMonitoring.interface' +import '@/scss/components/meter-bar.scss' + +interface MeterBarProps { + label: string + /** 0–100. */ + percent: number + /** What a screen reader announces instead of the bare percentage. */ + valueText: string + status?: HealthStatus +} + +/** A horizontal fill gauge for a value against a ceiling, coloured by health. */ +const MeterBar: FC = ({ label, percent, valueText, status = 'healthy' }) => { + const clamped = Math.min(100, Math.max(0, percent)) + return ( +
+
+
+ ) +} + +export default MeterBar diff --git a/frontend/src/components/Sidebar.spec.tsx b/frontend/src/components/Sidebar.spec.tsx index 4795e2cff..3578f2625 100644 --- a/frontend/src/components/Sidebar.spec.tsx +++ b/frontend/src/components/Sidebar.spec.tsx @@ -184,6 +184,29 @@ describe('Sidebar', () => { expect(screen.getByRole('link', { name: /feature flags/i })).toBeInTheDocument() }) + it('shows the Monitoring link to NOTIFY_ADMIN users', async () => { + const user = userEvent.setup() + const UserService = (await import('@/service/user-service')).default + vi.mocked(UserService.hasRole).mockReturnValue(true) + + renderSidebar() + await user.click(screen.getByRole('button', { name: /admin/i })) + + expect(screen.getByRole('link', { name: 'Monitoring' })).toHaveAttribute( + 'href', + '/admin/monitoring', + ) + }) + + it('does not show the Monitoring link without the NOTIFY_ADMIN role', async () => { + const UserService = (await import('@/service/user-service')).default + vi.mocked(UserService.hasRole).mockReturnValue(false) + + renderSidebar() + + expect(screen.queryByRole('link', { name: 'Monitoring' })).not.toBeInTheDocument() + }) + it('renders the logged-in user display name', () => { renderSidebar(mockUser) diff --git a/frontend/src/components/Sidebar.tsx b/frontend/src/components/Sidebar.tsx index 128872260..e885dc204 100644 --- a/frontend/src/components/Sidebar.tsx +++ b/frontend/src/components/Sidebar.tsx @@ -77,6 +77,10 @@ const adminItems = { label: 'Usage & Limits', to: '/admin/usage', }, + { + label: 'Monitoring', + to: '/admin/monitoring', + }, ], } as const @@ -210,6 +214,15 @@ const Sidebar: FC = () => { Usage & Limits )} + {isAdmin && ( + + Monitoring + + )}
)} diff --git a/frontend/src/components/Sparkline.tsx b/frontend/src/components/Sparkline.tsx new file mode 100644 index 000000000..655350b64 --- /dev/null +++ b/frontend/src/components/Sparkline.tsx @@ -0,0 +1,54 @@ +import type { FC } from 'react' +import '@/scss/components/sparkline.scss' + +interface SparklineSeries { + values: number[] + /** BEM modifier for the stroke colour, e.g. 'in' or 'out'. */ + variant: string +} + +interface SparklineProps { + series: SparklineSeries[] + /** Text alternative; the chart itself is decorative to assistive tech. */ + label: string + width?: number + height?: number +} + +/** + * A tiny trend line for one or more series sharing a y-axis. It is a glance aid only: the + * figures it summarises must also be shown as text beside it. + */ +const Sparkline: FC = ({ series, label, width = 120, height = 28 }) => { + const max = Math.max(1, ...series.flatMap((s) => s.values)) + const points = (values: number[]) => + values + .map((value, i) => { + const x = values.length > 1 ? (i / (values.length - 1)) * width : 0 + const y = height - 1 - (value / max) * (height - 2) + return `${x.toFixed(1)},${y.toFixed(1)}` + }) + .join(' ') + + return ( + + {series.map((s) => ( + + ))} + + ) +} + +export default Sparkline diff --git a/frontend/src/components/StatTile.tsx b/frontend/src/components/StatTile.tsx new file mode 100644 index 000000000..959f8dfd7 --- /dev/null +++ b/frontend/src/components/StatTile.tsx @@ -0,0 +1,20 @@ +import type { FC, ReactNode } from 'react' +import '@/scss/components/stat-tile.scss' + +interface StatTileProps { + label: string + value: ReactNode + /** Secondary line under the value, e.g. a limit or a comparison. */ + hint?: ReactNode +} + +/** A single labelled figure. Use inside a `.stat-tile-grid` to lay several out in a row. */ +const StatTile: FC = ({ label, value, hint }) => ( +
+
{label}
+
{value}
+ {hint != null &&
{hint}
} +
+) + +export default StatTile diff --git a/frontend/src/components/StatusBadge.tsx b/frontend/src/components/StatusBadge.tsx index d660ddb13..ba9d6393e 100644 --- a/frontend/src/components/StatusBadge.tsx +++ b/frontend/src/components/StatusBadge.tsx @@ -27,8 +27,12 @@ export function StatusBadge({ case 'completed': case 'sent': case 'delivered': + case 'healthy': dotClassName = 'status-badge__dot--success' break + case 'critical': + dotClassName = 'status-badge__dot--failed' + break case 'partially_completed': dotClassName = 'status-badge__dot--partial' break diff --git a/frontend/src/hooks/usePolling.ts b/frontend/src/hooks/usePolling.ts new file mode 100644 index 000000000..7c5cd5217 --- /dev/null +++ b/frontend/src/hooks/usePolling.ts @@ -0,0 +1,65 @@ +import { useCallback, useEffect, useRef, useState } from 'react' + +interface PollingState { + data: T | null + error: string | null + /** True only until the first response (or error) arrives. */ + isLoading: boolean + isRefreshing: boolean + refresh: () => void +} + +interface PollingOptions { + intervalMs: number + /** Stop the timer but keep the data; `refresh()` still works. */ + paused?: boolean + /** Make no requests at all, e.g. while the user lacks the role the endpoint needs. */ + skip?: boolean +} + +/** + * Fetch `fetcher` on mount and then every `intervalMs`, skipping ticks while the tab is + * hidden. The last good data is kept when a refresh fails, so a single failed poll shows an + * error without blanking the screen. Pass a stable `fetcher` (a module function, or one + * wrapped in useCallback) or every render restarts the timer. + */ +export function usePolling( + fetcher: () => Promise, + { intervalMs, paused = false, skip = false }: PollingOptions, +): PollingState { + const [data, setData] = useState(null) + const [error, setError] = useState(null) + const [isLoading, setIsLoading] = useState(true) + const [isRefreshing, setIsRefreshing] = useState(false) + const inFlight = useRef(false) + + const load = useCallback(async () => { + if (inFlight.current) return + inFlight.current = true + setIsRefreshing(true) + try { + setData(await fetcher()) + setError(null) + } catch (err) { + setError(err instanceof Error ? err.message : 'Request failed') + } finally { + inFlight.current = false + setIsLoading(false) + setIsRefreshing(false) + } + }, [fetcher]) + + useEffect(() => { + if (!skip) void load() + }, [skip, load]) + + useEffect(() => { + if (skip || paused) return + const id = window.setInterval(() => { + if (!document.hidden) void load() + }, intervalMs) + return () => window.clearInterval(id) + }, [skip, paused, intervalMs, load]) + + return { data, error, isLoading, isRefreshing, refresh: load } +} diff --git a/frontend/src/interfaces/queueMonitoring.interface.ts b/frontend/src/interfaces/queueMonitoring.interface.ts new file mode 100644 index 000000000..c99695bae --- /dev/null +++ b/frontend/src/interfaces/queueMonitoring.interface.ts @@ -0,0 +1,78 @@ +/** + * Response of GET /api/v1/frontend/admin/monitoring/queues. + * Mirrors backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts. + */ + +export type HealthStatus = 'healthy' | 'warning' | 'critical' + +export interface RedisStats { + status: HealthStatus + reasons: string[] + usedMemoryBytes: number + peakMemoryBytes: number + maxMemoryBytes: number | null + usedMemoryPercent: number | null + maxMemoryPolicy: string + fragmentationRatio: number | null + evictedKeys: number + rejectedConnections: number + connectedClients: number + blockedClients: number + opsPerSecond: number + uptimeSeconds: number + version: string +} + +export interface QueueStats { + name: string + status: HealthStatus + reasons: string[] + isPaused: boolean + counts: { waiting: number; active: number; delayed: number; failed: number; paused: number } + oldestWaitingAgeMs: number | null + throughput: { + /** Per-minute counts, oldest first; the last entry is the current, partial minute. */ + added: number[] + completed: number[] + failed: number[] + inPerMinute: number + outPerMinute: number + failureRatePercent: number + } + estimatedDrainMinutes: number | null + liveWorkerPods: number +} + +export interface WorkerPod { + podId: string + startedAt: string + lastHeartbeatAt: string + queues: Array<{ + queue: string + concurrency: number + active: number + completed: number + failed: number + lastFinishedAt: string | null + }> +} + +export interface RecentFailure { + queue: string + jobId: string + notificationId: string | null + tenantId: string | null + tenantName: string | null + reason: string + attemptsMade: number + failedAt: string | null +} + +export interface QueueMonitoring { + generatedAt: string + status: HealthStatus + redis: RedisStats + queues: QueueStats[] + workers: WorkerPod[] + recentFailures: RecentFailure[] +} diff --git a/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx b/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx new file mode 100644 index 000000000..03f1c2bfa --- /dev/null +++ b/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx @@ -0,0 +1,240 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' +import { act, render, screen, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import QueueMonitoring, { REFRESH_INTERVAL_MS } from './QueueMonitoring' +import { getQueueMonitoring } from '@/api/monitoring.api' +import UserService from '@/service/user-service' +import type { QueueMonitoring as QueueMonitoringData } from '@/interfaces/queueMonitoring.interface' + +vi.mock('@/api/monitoring.api', () => ({ getQueueMonitoring: vi.fn() })) +vi.mock('@/service/user-service', () => ({ default: { hasRole: vi.fn(() => true) } })) + +const MB = 1024 * 1024 +const generatedAt = '2026-10-01T18:00:00.000Z' +const now = new Date(generatedAt).getTime() + +const series = (value: number) => Array(60).fill(value) + +function snapshot(overrides: Partial = {}): QueueMonitoringData { + return { + generatedAt, + status: 'healthy', + redis: { + status: 'healthy', + reasons: [], + usedMemoryBytes: 300 * MB, + peakMemoryBytes: 400 * MB, + maxMemoryBytes: 768 * MB, + usedMemoryPercent: 39.1, + maxMemoryPolicy: 'noeviction', + fragmentationRatio: 1.12, + evictedKeys: 0, + rejectedConnections: 0, + connectedClients: 12, + blockedClients: 4, + opsPerSecond: 250, + uptimeSeconds: 7200, + version: '7.2.4', + }, + queues: [ + { + name: 'email-delivery', + status: 'healthy', + reasons: [], + isPaused: false, + counts: { waiting: 120, active: 2, delayed: 3, failed: 1, paused: 0 }, + oldestWaitingAgeMs: 45_000, + throughput: { + added: series(10), + completed: series(30), + failed: series(0), + inPerMinute: 10, + outPerMinute: 30, + failureRatePercent: 0, + }, + estimatedDrainMinutes: 6, + liveWorkerPods: 1, + }, + ], + workers: [ + { + podId: 'notify-backend-abc12', + startedAt: new Date(now - 3_600_000).toISOString(), + lastHeartbeatAt: new Date(now - 4_000).toISOString(), + queues: [ + { + queue: 'email-delivery', + concurrency: 2, + active: 2, + completed: 900, + failed: 1, + lastFinishedAt: new Date(now - 2_000).toISOString(), + }, + ], + }, + ], + recentFailures: [ + { + queue: 'email-delivery', + jobId: '77', + notificationId: 'notif-77', + tenantId: 't-1', + tenantName: 'Health Ministry', + reason: 'CHES rejected [email]', + attemptsMade: 3, + failedAt: new Date(now - 60_000).toISOString(), + }, + ], + ...overrides, + } +} + +const mockedGet = vi.mocked(getQueueMonitoring) + +describe('QueueMonitoring', () => { + beforeEach(() => { + vi.mocked(UserService.hasRole).mockReturnValue(true) + mockedGet.mockReset() + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('shows Redis memory against its cap', async () => { + mockedGet.mockResolvedValue(snapshot()) + render() + + const meter = await screen.findByRole('meter', { name: 'Redis memory used' }) + expect(meter).toHaveAttribute('aria-valuenow', '39.1') + expect(meter).toHaveAttribute('aria-valuetext', '300 MB of 768 MB (39.1%)') + expect(screen.getByText('noeviction')).toBeInTheDocument() + }) + + it('shows each queue with its backlog, rates and drain estimate', async () => { + mockedGet.mockResolvedValue(snapshot()) + render() + + const table = await screen.findByRole('table', { name: 'Queues' }) + const row = within(table).getByRole('row', { name: /Email delivery/ }) + expect(within(row).getByText('120')).toBeInTheDocument() + expect(within(row).getByText('45s')).toBeInTheDocument() + expect(within(row).getByText('In 10/min')).toBeInTheDocument() + expect(within(row).getByText('Out 30/min')).toBeInTheDocument() + expect(within(row).getByText('~6 min')).toBeInTheDocument() + expect( + within(row).getByRole('img', { name: /600 added, 1,800 completed, 0 failed/ }), + ).toBeInTheDocument() + }) + + it('explains why a queue is not healthy, in text as well as colour', async () => { + const data = snapshot({ status: 'critical' }) + data.queues[0] = { + ...data.queues[0], + status: 'critical', + reasons: ['Jobs are waiting but no pod is processing this queue'], + estimatedDrainMinutes: null, + liveWorkerPods: 0, + } + mockedGet.mockResolvedValue(data) + render() + + expect( + await screen.findByText('Jobs are waiting but no pod is processing this queue'), + ).toBeInTheDocument() + expect(screen.getAllByText('Critical').length).toBeGreaterThan(0) + expect(screen.getByText('Not draining: Email delivery')).toBeInTheDocument() + }) + + it('lists worker pods and recent failures', async () => { + mockedGet.mockResolvedValue(snapshot()) + render() + + const workers = await screen.findByRole('table', { name: 'Workers' }) + expect(within(workers).getByText('notify-backend-abc12')).toBeInTheDocument() + expect(within(workers).getByText('2 of 2')).toBeInTheDocument() + + const failures = screen.getByRole('table', { name: 'Recent failed jobs' }) + expect(within(failures).getByText('Health Ministry')).toBeInTheDocument() + expect(within(failures).getByText('CHES rejected [email]')).toBeInTheDocument() + }) + + it('shows empty states when there are no workers or failures', async () => { + mockedGet.mockResolvedValue(snapshot({ workers: [], recentFailures: [] })) + render() + + expect(await screen.findAllByText(/No worker heartbeats received/)).not.toHaveLength(0) + expect(screen.getAllByText('No failed jobs.')).not.toHaveLength(0) + }) + + it('shows a loading message until the first snapshot arrives', () => { + mockedGet.mockReturnValue(new Promise(() => {})) + render() + + expect(screen.getByText('Loading queue health…')).toBeInTheDocument() + }) + + it('shows the error when the first load fails', async () => { + mockedGet.mockRejectedValue( + new Error('Failed to load queue monitoring: Redis is not configured'), + ) + render() + + expect(await screen.findByRole('alert')).toHaveTextContent('Redis is not configured') + expect(screen.queryByRole('table')).not.toBeInTheDocument() + }) + + it('keeps the last snapshot on screen when a refresh fails', async () => { + const user = userEvent.setup() + mockedGet.mockResolvedValueOnce(snapshot()).mockRejectedValueOnce(new Error('Network Error')) + render() + + await screen.findByRole('table', { name: 'Queues' }) + await user.click(screen.getByRole('button', { name: 'Refresh' })) + + expect(await screen.findByRole('alert')).toHaveTextContent( + 'Network Error Showing the last successful snapshot.', + ) + expect(screen.getByRole('table', { name: 'Queues' })).toBeInTheDocument() + }) + + it('announces a change in overall health but not an unchanged refresh', async () => { + const user = userEvent.setup() + mockedGet + .mockResolvedValueOnce(snapshot()) + .mockResolvedValueOnce(snapshot()) + .mockResolvedValueOnce(snapshot({ status: 'warning' })) + render() + await screen.findByRole('table', { name: 'Queues' }) + + await user.click(screen.getByRole('button', { name: 'Refresh' })) + expect(screen.queryByText(/Queue health changed/)).not.toBeInTheDocument() + + await user.click(screen.getByRole('button', { name: 'Refresh' })) + expect(await screen.findByText('Queue health changed to Warning')).toBeInTheDocument() + }) + + it('polls on an interval and stops when auto-refresh is turned off', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }) + mockedGet.mockResolvedValue(snapshot()) + render() + await screen.findByRole('table', { name: 'Queues' }) + expect(mockedGet).toHaveBeenCalledTimes(1) + + await act(() => vi.advanceTimersByTimeAsync(REFRESH_INTERVAL_MS)) + expect(mockedGet).toHaveBeenCalledTimes(2) + + await user.click(screen.getByRole('switch', { name: /Auto-refresh/ })) + await act(() => vi.advanceTimersByTimeAsync(REFRESH_INTERVAL_MS * 3)) + expect(mockedGet).toHaveBeenCalledTimes(2) + }) + + it('is not available to users without the NOTIFY_ADMIN role, and never calls the API', () => { + vi.mocked(UserService.hasRole).mockReturnValue(false) + render() + + expect(screen.queryByRole('heading', { name: 'Monitoring' })).not.toBeInTheDocument() + expect(mockedGet).not.toHaveBeenCalled() + }) +}) diff --git a/frontend/src/pages/admin/monitoring/QueueMonitoring.tsx b/frontend/src/pages/admin/monitoring/QueueMonitoring.tsx new file mode 100644 index 000000000..8e63b66c2 --- /dev/null +++ b/frontend/src/pages/admin/monitoring/QueueMonitoring.tsx @@ -0,0 +1,122 @@ +import { useState } from 'react' +import type { FC } from 'react' +import { Button, Switch } from '@bcgov/design-system-react-components' +import PageHeading from '@/components/PageHeading' +import { StatusBadge } from '@/components/StatusBadge' +import Alert from '@/components/Alert' +import NotAuthorized from '@/components/NotAuthorized' +import UserService from '@/service/user-service' +import { SsoRole } from '@/enum/sso-role.enum' +import { usePolling } from '@/hooks/usePolling' +import { getQueueMonitoring } from '@/api/monitoring.api' +import type { HealthStatus } from '@/interfaces/queueMonitoring.interface' +import { HEALTH_LABELS, formatDuration, formatQueueName } from '@/utils/monitoring' +import RedisSection from './sections/RedisSection' +import QueuesSection from './sections/QueuesSection' +import WorkersSection from './sections/WorkersSection' +import RecentFailuresSection from './sections/RecentFailuresSection' +import '@/scss/components/queue-monitoring.scss' + +export const REFRESH_INTERVAL_MS = 15_000 + +/** + * Admin view of the Bull queues, the pods working them and the Redis instance they live in. + * Read-only: it shows whether notifications are flowing; acting on failures is out of scope. + */ +const QueueMonitoring: FC = () => { + const isAdmin = UserService.hasRole(SsoRole.NOTIFY_ADMIN) + const [autoRefresh, setAutoRefresh] = useState(true) + const { data, error, isLoading, isRefreshing, refresh } = usePolling(getQueueMonitoring, { + intervalMs: REFRESH_INTERVAL_MS, + paused: !autoRefresh, + skip: !isAdmin, + }) + + // Announce only changes in overall health, not every 15-second refresh. + const [announcement, setAnnouncement] = useState('') + const [announcedStatus, setAnnouncedStatus] = useState(null) + if (data && data.status !== announcedStatus) { + if (announcedStatus !== null) { + setAnnouncement(`Queue health changed to ${HEALTH_LABELS[data.status]}`) + } + setAnnouncedStatus(data.status) + } + + if (!isAdmin) { + return + } + + const now = data ? new Date(data.generatedAt).getTime() : 0 + const backlog = data?.queues.reduce((sum, q) => sum + q.counts.waiting + q.counts.paused, 0) ?? 0 + const slowest = data?.queues.reduce((longest, q) => { + if (q.estimatedDrainMinutes === null || q.estimatedDrainMinutes === 0) return longest + return Math.max(longest ?? 0, q.estimatedDrainMinutes) + }, null) + const notDraining = data?.queues.filter( + (q) => q.estimatedDrainMinutes === null && q.counts.waiting + q.counts.paused > 0, + ) + + return ( +
+ + +
+ {announcement} +
+ +
+
+ {data && ( + <> + + {backlog.toLocaleString()} waiting + {notDraining && notDraining.length > 0 ? ( + + Not draining: {notDraining.map((q) => formatQueueName(q.name)).join(', ')} + + ) : ( + slowest != null && Clears in ~{slowest.toLocaleString()} min + )} + + )} +
+
+ {data && Updated {new Date(data.generatedAt).toLocaleTimeString()}} + + Auto-refresh every {formatDuration(REFRESH_INTERVAL_MS)} + + +
+
+ + {error && ( + + {error} + {data && ' Showing the last successful snapshot.'} + + )} + + {isLoading && !data && ( +

+ Loading queue health… +

+ )} + + {data && ( + <> + + + + + + )} +
+ ) +} + +export default QueueMonitoring diff --git a/frontend/src/pages/admin/monitoring/sections/QueuesSection.tsx b/frontend/src/pages/admin/monitoring/sections/QueuesSection.tsx new file mode 100644 index 000000000..d3bf833f8 --- /dev/null +++ b/frontend/src/pages/admin/monitoring/sections/QueuesSection.tsx @@ -0,0 +1,118 @@ +import type { FC } from 'react' +import PageSubHeading from '@/components/PageSubHeading' +import { StatusBadge } from '@/components/StatusBadge' +import Sparkline from '@/components/Sparkline' +import DataTable from '@/components/DataTable/DataTable' +import type { TableColumn } from '@/components/DataTable/DataTable' +import type { QueueStats } from '@/interfaces/queueMonitoring.interface' +import { HEALTH_LABELS, formatDuration, formatQueueName, formatRate } from '@/utils/monitoring' + +type QueueRow = QueueStats & { + waiting: number + active: number + delayed: number + failedCount: number +} + +function drainText(row: QueueRow): string { + if (row.estimatedDrainMinutes === 0) return 'Empty' + if (row.estimatedDrainMinutes === null) return 'Not draining' + return `~${row.estimatedDrainMinutes.toLocaleString()} min` +} + +const columns: TableColumn[] = [ + { + key: 'name', + label: 'Queue', + render: (_, row) => ( +
+ {formatQueueName(row.name)} + {row.reasons.length > 0 && ( +
    + {row.reasons.map((reason) => ( +
  • {reason}
    • + ))} +
+ )} +
+ ), + }, + { + key: 'status', + label: 'Status', + render: (_, row) => , + }, + { key: 'waiting', label: 'Waiting', render: (_, row) => row.waiting.toLocaleString() }, + { key: 'active', label: 'Active', render: (_, row) => row.active.toLocaleString() }, + { key: 'delayed', label: 'Delayed', render: (_, row) => row.delayed.toLocaleString() }, + { key: 'failedCount', label: 'Failed', render: (_, row) => row.failedCount.toLocaleString() }, + { + key: 'oldestWaitingAgeMs', + label: 'Oldest waiting', + render: (_, row) => + row.oldestWaitingAgeMs === null ? '—' : formatDuration(row.oldestWaitingAgeMs), + }, + { + key: 'throughput', + label: 'In / out', + render: (_, row) => ( +
+ In {formatRate(row.throughput.inPerMinute)} + Out {formatRate(row.throughput.outPerMinute)} + a + b, 0).toLocaleString()} added, ${row.throughput.completed.reduce((a, b) => a + b, 0).toLocaleString()} completed, ${row.throughput.failed.reduce((a, b) => a + b, 0).toLocaleString()} failed`} + series={[ + { variant: 'in', values: row.throughput.added }, + { + variant: 'out', + values: row.throughput.completed.map((value, i) => value + row.throughput.failed[i]), + }, + ]} + /> +
+ ), + }, + { key: 'estimatedDrainMinutes', label: 'Time to drain', render: (_, row) => drainText(row) }, + { + key: 'liveWorkerPods', + label: 'Worker pods', + render: (_, row) => row.liveWorkerPods.toLocaleString(), + }, +] + +interface QueuesSectionProps { + queues: QueueStats[] +} + +const QueuesSection: FC = ({ queues }) => { + const rows: QueueRow[] = queues.map((queue) => ({ + ...queue, + waiting: queue.counts.waiting + queue.counts.paused, + active: queue.counts.active, + delayed: queue.counts.delayed, + failedCount: queue.counts.failed, + })) + + return ( +
+ + row.name} + label="Queues" + emptyMessage="No queues are configured." + multiline + /> +

+

+
+ ) +} + +export default QueuesSection diff --git a/frontend/src/pages/admin/monitoring/sections/RecentFailuresSection.tsx b/frontend/src/pages/admin/monitoring/sections/RecentFailuresSection.tsx new file mode 100644 index 000000000..1855f2cef --- /dev/null +++ b/frontend/src/pages/admin/monitoring/sections/RecentFailuresSection.tsx @@ -0,0 +1,56 @@ +import type { FC } from 'react' +import PageSubHeading from '@/components/PageSubHeading' +import DataTable from '@/components/DataTable/DataTable' +import type { TableColumn } from '@/components/DataTable/DataTable' +import type { RecentFailure } from '@/interfaces/queueMonitoring.interface' +import { formatAgo, formatQueueName } from '@/utils/monitoring' + +interface RecentFailuresSectionProps { + failures: RecentFailure[] + now: number +} + +const RecentFailuresSection: FC = ({ failures, now }) => { + const columns: TableColumn[] = [ + { + key: 'failedAt', + label: 'Failed', + render: (_, row) => ( + {formatAgo(row.failedAt, now)} + ), + }, + { key: 'queue', label: 'Queue', render: (_, row) => formatQueueName(row.queue) }, + { + key: 'tenantName', + label: 'Tenant', + render: (_, row) => row.tenantName ?? row.tenantId ?? '—', + }, + { + key: 'notificationId', + label: 'Notification ID', + render: (_, row) => {row.notificationId ?? '—'}, + }, + { key: 'attemptsMade', label: 'Attempts' }, + { + key: 'reason', + label: 'Reason', + render: (_, row) => {row.reason}, + }, + ] + + return ( +
+ + `${row.queue}-${row.jobId}`} + label="Recent failed jobs" + emptyMessage="No failed jobs." + multiline + /> +
+ ) +} + +export default RecentFailuresSection diff --git a/frontend/src/pages/admin/monitoring/sections/RedisSection.tsx b/frontend/src/pages/admin/monitoring/sections/RedisSection.tsx new file mode 100644 index 000000000..c562ca211 --- /dev/null +++ b/frontend/src/pages/admin/monitoring/sections/RedisSection.tsx @@ -0,0 +1,79 @@ +import type { FC } from 'react' +import PageSubHeading from '@/components/PageSubHeading' +import { StatusBadge } from '@/components/StatusBadge' +import MeterBar from '@/components/MeterBar' +import StatTile from '@/components/StatTile' +import type { RedisStats } from '@/interfaces/queueMonitoring.interface' +import { HEALTH_LABELS, formatBytes, formatDuration } from '@/utils/monitoring' + +interface RedisSectionProps { + redis: RedisStats +} + +const RedisSection: FC = ({ redis }) => { + const used = formatBytes(redis.usedMemoryBytes) + const memoryText = + redis.maxMemoryBytes === null + ? `${used} used (no memory cap set)` + : `${used} of ${formatBytes(redis.maxMemoryBytes)} (${redis.usedMemoryPercent}%)` + + return ( +
+ +
+
+
+ Memory: {memoryText} + +
+ {redis.usedMemoryPercent !== null && ( + + )} + {redis.reasons.length > 0 && ( +
    + {redis.reasons.map((reason) => ( +
  • {reason}
    • + ))} +
+ )} +
+ +
+ + + + + + + + +
+
+
+ ) +} + +export default RedisSection diff --git a/frontend/src/pages/admin/monitoring/sections/WorkersSection.tsx b/frontend/src/pages/admin/monitoring/sections/WorkersSection.tsx new file mode 100644 index 000000000..ba68b6b6a --- /dev/null +++ b/frontend/src/pages/admin/monitoring/sections/WorkersSection.tsx @@ -0,0 +1,80 @@ +import type { FC } from 'react' +import PageSubHeading from '@/components/PageSubHeading' +import DataTable from '@/components/DataTable/DataTable' +import type { TableColumn } from '@/components/DataTable/DataTable' +import type { WorkerPod } from '@/interfaces/queueMonitoring.interface' +import { formatAgo, formatQueueName } from '@/utils/monitoring' + +interface WorkerRow { + key: string + podId: string + queue: string + concurrency: number + active: number + completed: number + failed: number + lastFinishedAt: string | null + lastHeartbeatAt: string +} + +interface WorkersSectionProps { + workers: WorkerPod[] + /** Reference time for "ago" labels: when the snapshot was taken. */ + now: number +} + +const WorkersSection: FC = ({ workers, now }) => { + const rows: WorkerRow[] = workers.flatMap((pod) => + pod.queues.map((queue) => ({ + key: `${pod.podId}-${queue.queue}`, + podId: pod.podId, + lastHeartbeatAt: pod.lastHeartbeatAt, + ...queue, + })), + ) + + const columns: TableColumn[] = [ + { key: 'podId', label: 'Pod' }, + { key: 'queue', label: 'Queue', render: (_, row) => formatQueueName(row.queue) }, + { + key: 'active', + label: 'Busy', + render: (_, row) => `${row.active} of ${row.concurrency}`, + }, + { + key: 'completed', + label: 'Completed', + render: (_, row) => row.completed.toLocaleString(), + }, + { key: 'failed', label: 'Failed', render: (_, row) => row.failed.toLocaleString() }, + { + key: 'lastFinishedAt', + label: 'Last job finished', + render: (_, row) => formatAgo(row.lastFinishedAt, now), + }, + { + key: 'lastHeartbeatAt', + label: 'Last heartbeat', + render: (_, row) => formatAgo(row.lastHeartbeatAt, now), + }, + ] + + return ( +
+ + row.key} + label="Workers" + emptyMessage="No worker heartbeats received. Either no pod is processing jobs, or the running pods predate worker heartbeats." + /> +

+ Completed and failed counts are since each pod started. A pod drops off this list 30 seconds + after its last heartbeat. +

+
+ ) +} + +export default WorkersSection diff --git a/frontend/src/routeTree.gen.ts b/frontend/src/routeTree.gen.ts index 677276ab7..8723deab4 100644 --- a/frontend/src/routeTree.gen.ts +++ b/frontend/src/routeTree.gen.ts @@ -22,6 +22,7 @@ import { Route as TemplateEditTemplateIdRouteImport } from './routes/template-ed import { Route as RequestStatusNotificationRequestIdRouteImport } from './routes/request-status/$notificationRequestId' import { Route as EventsCreateRouteImport } from './routes/events/create' import { Route as AdminUsageRouteImport } from './routes/admin/usage' +import { Route as AdminMonitoringRouteImport } from './routes/admin/monitoring' import { Route as AdminFeatureFlagsRouteImport } from './routes/admin/feature-flags' import { Route as EventsEventIdIndexRouteImport } from './routes/events/$eventId/index' import { Route as EventsEventIdSavedRouteImport } from './routes/events/$eventId/saved' @@ -92,6 +93,11 @@ const AdminUsageRoute = AdminUsageRouteImport.update({ path: '/admin/usage', getParentRoute: () => rootRouteImport, } as any) +const AdminMonitoringRoute = AdminMonitoringRouteImport.update({ + id: '/admin/monitoring', + path: '/admin/monitoring', + getParentRoute: () => rootRouteImport, +} as any) const AdminFeatureFlagsRoute = AdminFeatureFlagsRouteImport.update({ id: '/admin/feature-flags', path: '/admin/feature-flags', @@ -117,6 +123,7 @@ export interface FileRoutesByFullPath { '/templates': typeof TemplatesRoute '/usage': typeof UsageRoute '/admin/feature-flags': typeof AdminFeatureFlagsRoute + '/admin/monitoring': typeof AdminMonitoringRoute '/admin/usage': typeof AdminUsageRoute '/events/create': typeof EventsCreateRoute '/request-status/$notificationRequestId': typeof RequestStatusNotificationRequestIdRoute @@ -135,6 +142,7 @@ export interface FileRoutesByTo { '/templates': typeof TemplatesRoute '/usage': typeof UsageRoute '/admin/feature-flags': typeof AdminFeatureFlagsRoute + '/admin/monitoring': typeof AdminMonitoringRoute '/admin/usage': typeof AdminUsageRoute '/events/create': typeof EventsCreateRoute '/request-status/$notificationRequestId': typeof RequestStatusNotificationRequestIdRoute @@ -154,6 +162,7 @@ export interface FileRoutesById { '/templates': typeof TemplatesRoute '/usage': typeof UsageRoute '/admin/feature-flags': typeof AdminFeatureFlagsRoute + '/admin/monitoring': typeof AdminMonitoringRoute '/admin/usage': typeof AdminUsageRoute '/events/create': typeof EventsCreateRoute '/request-status/$notificationRequestId': typeof RequestStatusNotificationRequestIdRoute @@ -174,6 +183,7 @@ export interface FileRouteTypes { | '/templates' | '/usage' | '/admin/feature-flags' + | '/admin/monitoring' | '/admin/usage' | '/events/create' | '/request-status/$notificationRequestId' @@ -192,6 +202,7 @@ export interface FileRouteTypes { | '/templates' | '/usage' | '/admin/feature-flags' + | '/admin/monitoring' | '/admin/usage' | '/events/create' | '/request-status/$notificationRequestId' @@ -210,6 +221,7 @@ export interface FileRouteTypes { | '/templates' | '/usage' | '/admin/feature-flags' + | '/admin/monitoring' | '/admin/usage' | '/events/create' | '/request-status/$notificationRequestId' @@ -229,6 +241,7 @@ export interface RootRouteChildren { TemplatesRoute: typeof TemplatesRoute UsageRoute: typeof UsageRoute AdminFeatureFlagsRoute: typeof AdminFeatureFlagsRoute + AdminMonitoringRoute: typeof AdminMonitoringRoute AdminUsageRoute: typeof AdminUsageRoute EventsCreateRoute: typeof EventsCreateRoute RequestStatusNotificationRequestIdRoute: typeof RequestStatusNotificationRequestIdRoute @@ -332,6 +345,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AdminUsageRouteImport parentRoute: typeof rootRouteImport } + '/admin/monitoring': { + id: '/admin/monitoring' + path: '/admin/monitoring' + fullPath: '/admin/monitoring' + preLoaderRoute: typeof AdminMonitoringRouteImport + parentRoute: typeof rootRouteImport + } '/admin/feature-flags': { id: '/admin/feature-flags' path: '/admin/feature-flags' @@ -365,6 +385,7 @@ const rootRouteChildren: RootRouteChildren = { TemplatesRoute: TemplatesRoute, UsageRoute: UsageRoute, AdminFeatureFlagsRoute: AdminFeatureFlagsRoute, + AdminMonitoringRoute: AdminMonitoringRoute, AdminUsageRoute: AdminUsageRoute, EventsCreateRoute: EventsCreateRoute, RequestStatusNotificationRequestIdRoute: diff --git a/frontend/src/routes/admin/monitoring.tsx b/frontend/src/routes/admin/monitoring.tsx new file mode 100644 index 000000000..a9838eee9 --- /dev/null +++ b/frontend/src/routes/admin/monitoring.tsx @@ -0,0 +1,17 @@ +import { createFileRoute, redirect } from '@tanstack/react-router' +import QueueMonitoring from '@/pages/admin/monitoring/QueueMonitoring' +import UserService from '@/service/user-service' +import { SsoRole } from '@/enum/sso-role.enum' + +export const Route = createFileRoute('/admin/monitoring')({ + beforeLoad: () => { + if (!UserService.hasRole(SsoRole.NOTIFY_ADMIN)) { + throw redirect({ to: '/not-authorized' }) + } + }, + component: MonitoringPage, +}) + +function MonitoringPage() { + return +} diff --git a/frontend/src/scss/components/meter-bar.scss b/frontend/src/scss/components/meter-bar.scss new file mode 100644 index 000000000..ec00c63b0 --- /dev/null +++ b/frontend/src/scss/components/meter-bar.scss @@ -0,0 +1,26 @@ +@import '../variables'; + +.meter-bar { + width: 100%; + height: 12px; + border: 1px solid $border-medium; + border-radius: 6px; + background-color: $light-bg; + overflow: hidden; + + .meter-bar__fill { + height: 100%; + + &--healthy { + background-color: $success-color; + } + + &--warning { + background-color: $warning-color; + } + + &--critical { + background-color: $danger-color; + } + } +} diff --git a/frontend/src/scss/components/queue-monitoring.scss b/frontend/src/scss/components/queue-monitoring.scss new file mode 100644 index 000000000..e97d20fe7 --- /dev/null +++ b/frontend/src/scss/components/queue-monitoring.scss @@ -0,0 +1,64 @@ +@import '../variables'; + +.queue-monitoring__toolbar { + @include flex-between; + flex-wrap: wrap; + gap: $spacing-md; +} + +.queue-monitoring__overall { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: $spacing-sm $spacing-md; + font-weight: 700; +} + +.queue-monitoring__controls { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: $spacing-md; + font-size: $font-size-sm; + color: $text-secondary; +} + +.queue-monitoring__memory { + display: flex; + flex-direction: column; + gap: $spacing-sm; + width: 100%; + max-width: 640px; +} + +.queue-monitoring__memory-caption { + @include flex-between; + font-size: $font-size-sm; + color: $text-secondary; +} + +.queue-monitoring__reasons { + margin: 0; + padding-left: $spacing-lg; + font-size: $font-size-sm; + color: $text-secondary; +} + +.queue-monitoring__rates { + display: flex; + flex-direction: column; + gap: $spacing-xs; + font-size: $font-size-sm; + white-space: nowrap; +} + +.queue-monitoring__legend { + margin: 0; + font-size: $font-size-sm; + color: $text-secondary; +} + +.queue-monitoring__reason-text { + max-width: 420px; + word-break: break-word; +} diff --git a/frontend/src/scss/components/sparkline.scss b/frontend/src/scss/components/sparkline.scss new file mode 100644 index 000000000..518cca50f --- /dev/null +++ b/frontend/src/scss/components/sparkline.scss @@ -0,0 +1,38 @@ +@import '../variables'; + +.sparkline { + display: block; + overflow: visible; + + .sparkline__line { + fill: none; + stroke-width: 1.5; + stroke-linejoin: round; + vector-effect: non-scaling-stroke; + + &--in { + stroke: $border-active; + } + + &--out { + stroke: $success-color; + } + } +} + +// Legend swatches for pages that explain the sparkline colours in text. +.sparkline-key { + display: inline-block; + width: 12px; + height: 3px; + margin-right: $spacing-xs; + vertical-align: middle; + + &--in { + background-color: $border-active; + } + + &--out { + background-color: $success-color; + } +} diff --git a/frontend/src/scss/components/stat-tile.scss b/frontend/src/scss/components/stat-tile.scss new file mode 100644 index 000000000..d6c2cefbf --- /dev/null +++ b/frontend/src/scss/components/stat-tile.scss @@ -0,0 +1,39 @@ +@import '../variables'; + +// Lay out on a
; each StatTile is a
holding its
/
pair. +.stat-tile-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(160px, 1fr)); + gap: $spacing-md; + width: 100%; + margin: 0; +} + +.stat-tile { + display: flex; + flex-direction: column; + gap: $spacing-xs; + padding: $spacing-md; + border: 1px solid $border-default; + border-radius: 4px; + background-color: $white; + + .stat-tile__label { + font-size: $font-size-sm; + font-weight: 400; + color: $text-secondary; + } + + .stat-tile__value { + margin: 0; + font-size: $font-size-xl; + font-weight: 700; + color: $dark-text; + } + + .stat-tile__hint { + margin: 0; + font-size: $font-size-sm; + color: $text-secondary; + } +} diff --git a/frontend/src/utils/monitoring.spec.ts b/frontend/src/utils/monitoring.spec.ts new file mode 100644 index 000000000..52f29932d --- /dev/null +++ b/frontend/src/utils/monitoring.spec.ts @@ -0,0 +1,35 @@ +import { describe, it, expect } from 'vitest' +import { formatAgo, formatBytes, formatDuration, formatQueueName, formatRate } from './monitoring' + +describe('monitoring formatters', () => { + it('formats bytes in the largest sensible unit', () => { + expect(formatBytes(512)).toBe('512 B') + expect(formatBytes(1536)).toBe('1.5 KB') + expect(formatBytes(768 * 1024 * 1024)).toBe('768 MB') + expect(formatBytes(2 * 1024 ** 3)).toBe('2.0 GB') + }) + + it('formats durations using their two largest units', () => { + expect(formatDuration(45_000)).toBe('45s') + expect(formatDuration(190_000)).toBe('3m 10s') + expect(formatDuration(7_500_000)).toBe('2h 5m') + expect(formatDuration(4 * 86_400_000 + 7_200_000)).toBe('4d 2h') + expect(formatDuration(-5)).toBe('0s') + }) + + it('formats relative times against a reference time', () => { + expect(formatAgo('2026-10-01T18:00:00.000Z', Date.parse('2026-10-01T18:01:30.000Z'))).toBe( + '1m 30s ago', + ) + expect(formatAgo(null, 0)).toBe('—') + }) + + it('names known queues and passes unknown ones through', () => { + expect(formatQueueName('notification-ingestion')).toBe('Ingestion') + expect(formatQueueName('something-new')).toBe('something-new') + }) + + it('formats per-minute rates', () => { + expect(formatRate(12.345)).toBe('12.3/min') + }) +}) diff --git a/frontend/src/utils/monitoring.ts b/frontend/src/utils/monitoring.ts new file mode 100644 index 000000000..26cae729d --- /dev/null +++ b/frontend/src/utils/monitoring.ts @@ -0,0 +1,52 @@ +import type { HealthStatus } from '@/interfaces/queueMonitoring.interface' + +const QUEUE_LABELS: Record = { + 'notification-ingestion': 'Ingestion', + 'email-delivery': 'Email delivery', + 'sms-delivery': 'SMS delivery', + 'webhook-delivery': 'Webhook delivery', +} + +export function formatQueueName(name: string): string { + return QUEUE_LABELS[name] ?? name +} + +export const HEALTH_LABELS: Record = { + healthy: 'Healthy', + warning: 'Warning', + critical: 'Critical', +} + +export function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B` + const units = ['KB', 'MB', 'GB'] + let value = bytes / 1024 + let unit = 0 + while (value >= 1024 && unit < units.length - 1) { + value /= 1024 + unit++ + } + return `${value.toFixed(value >= 100 ? 0 : 1)} ${units[unit]}` +} + +/** "45s", "3m 10s", "2h 5m", "4d 2h" — the two largest units, for ages and uptimes. */ +export function formatDuration(ms: number): string { + const seconds = Math.max(0, Math.floor(ms / 1000)) + const days = Math.floor(seconds / 86400) + const hours = Math.floor((seconds % 86400) / 3600) + const minutes = Math.floor((seconds % 3600) / 60) + const secs = seconds % 60 + if (days > 0) return `${days}d ${hours}h` + if (hours > 0) return `${hours}h ${minutes}m` + if (minutes > 0) return `${minutes}m ${secs}s` + return `${secs}s` +} + +export function formatAgo(iso: string | null, now: number): string { + if (!iso) return '—' + return `${formatDuration(now - new Date(iso).getTime())} ago` +} + +export function formatRate(perMinute: number): string { + return `${perMinute.toLocaleString(undefined, { maximumFractionDigits: 1 })}/min` +} From e5a3d5ba8764c229471f992b1063a716233a46c7 Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Thu, 1 Oct 2026 15:18:42 -0700 Subject: [PATCH 02/23] Update app.module.ts --- backend/src/app.module.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index 7d684e210..903d8af56 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -12,6 +12,10 @@ import { MetricsController } from './metrics.controller' import { HealthController } from './health.controller' import { AuthModule } from './auth/auth.module' import { QueueModule } from './queue/queue.module' +// Imported after QueueModule, never from AdminModule: monitoring.module imports queue.module, +// and loading that file first from the admin chain leaves TemplatesModule undefined inside +// NotificationModule. test/app.e2e-spec.ts fails if the order breaks. +import { MonitoringModule } from './api/admin/monitoring/monitoring.module' import { NotificationModule } from './api/notification/notification.module' import configuration from './config/configuration' import { AdaptersModule } from './adapters' @@ -32,6 +36,7 @@ import { LoggerModule } from './common/logger' DatabaseModule, QueueModule, AdminModule, + MonitoringModule, ApiModule, AuthModule, NotificationModule, From 574381f40c049ac928e93b9afc87dd2fc83af5c1 Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Thu, 1 Oct 2026 16:28:29 -0700 Subject: [PATCH 03/23] Add live queue monitoring and batch progress Adds admin queue monitoring for recipient-level backlog and service health, plus live SSE change signals when queue state moves. The backend now aggregates per-channel message throughput, tracks stalled active jobs, exposes active merge-batch progress from Bull jobs, and includes a V72 index to support the hourly stats query. The frontend dashboard shows message backlog, active batches, and drill-down metrics, with matching specs covering the new monitoring views and progress reporting. No public API contract changes; this is admin-only monitoring. --- api-gateway/templates/routes.yaml | 23 ++ .../monitoring-events.service.spec.ts | 80 ++++++ .../monitoring/monitoring-events.service.ts | 47 ++++ .../monitoring/monitoring.controller.spec.ts | 5 + .../admin/monitoring/monitoring.controller.ts | 29 ++- .../api/admin/monitoring/monitoring.module.ts | 6 +- .../monitoring/monitoring.service.spec.ts | 159 +++++++++++- .../admin/monitoring/monitoring.service.ts | 231 +++++++++++++++--- .../schemas/queue-monitoring.dto.ts | 34 +++ backend/src/queue/batch-progress.spec.ts | 50 ++++ backend/src/queue/batch-progress.ts | 39 +++ .../workers/email-delivery.worker.spec.ts | 17 ++ .../queue/workers/email-delivery.worker.ts | 5 + .../src/queue/workers/sms-delivery.worker.ts | 7 + frontend/src/api/monitoring.api.ts | 56 +++++ frontend/src/hooks/useLiveUpdates.ts | 49 ++++ frontend/src/hooks/usePolling.ts | 21 +- .../interfaces/queueMonitoring.interface.ts | 30 +++ .../admin/monitoring/QueueMonitoring.spec.tsx | 167 ++++++++++++- .../admin/monitoring/QueueMonitoring.tsx | 66 +++-- .../sections/ActiveBatchesSection.tsx | 68 ++++++ .../monitoring/sections/MessagesSection.tsx | 77 ++++++ .../monitoring/sections/RedisSection.tsx | 4 +- .../src/scss/components/queue-monitoring.scss | 32 +++ frontend/src/scss/components/sparkline.scss | 8 + frontend/src/utils/monitoring.ts | 10 + ...fication_request_detail_finished_index.sql | 6 + 27 files changed, 1257 insertions(+), 69 deletions(-) create mode 100644 backend/src/api/admin/monitoring/monitoring-events.service.spec.ts create mode 100644 backend/src/api/admin/monitoring/monitoring-events.service.ts create mode 100644 backend/src/queue/batch-progress.spec.ts create mode 100644 backend/src/queue/batch-progress.ts create mode 100644 frontend/src/hooks/useLiveUpdates.ts create mode 100644 frontend/src/pages/admin/monitoring/sections/ActiveBatchesSection.tsx create mode 100644 frontend/src/pages/admin/monitoring/sections/MessagesSection.tsx create mode 100644 migrations/sql/V72__notification_request_detail_finished_index.sql diff --git a/api-gateway/templates/routes.yaml b/api-gateway/templates/routes.yaml index 0589be2ae..1f0e9cfba 100644 --- a/api-gateway/templates/routes.yaml +++ b/api-gateway/templates/routes.yaml @@ -646,6 +646,22 @@ routes: request_buffering: true response_buffering: true + # /api/v1/frontend/admin/monitoring/queues/events (SSE change signals) + # Buffering off, as for notification-request-events: a buffered stream never reaches the page. + - name: ${ROUTE_PREFIX}admin-monitoring-queue-events-frontend + tags: [${CONFIG_TAG}] + hosts: + - ${GATEWAY_HOSTNAME} + paths: + - ${PATH_PREFIX}/api/v1/frontend/admin/monitoring/queues/events + methods: + - GET + strip_path: true + https_redirect_status_code: 426 + path_handling: v0 + request_buffering: false + response_buffering: false + # ============================================================================ # FEATURE FLAG ADMIN ROUTES (admin only) # ============================================================================ @@ -1707,6 +1723,13 @@ plugins: route: ${ROUTE_PREFIX}admin-monitoring-queues-frontend config: *jwtConfig + # Admin Monitoring Queue Events (SSE) - Frontend Admin Token + - name: jwt-keycloak + tags: [${CONFIG_TAG}] + enabled: true + route: ${ROUTE_PREFIX}admin-monitoring-queue-events-frontend + config: *jwtConfig + # ============================================================================ # FEATURE FLAG ADMIN ROUTES PLUGINS # ============================================================================ diff --git a/backend/src/api/admin/monitoring/monitoring-events.service.spec.ts b/backend/src/api/admin/monitoring/monitoring-events.service.spec.ts new file mode 100644 index 000000000..c7a9f1f92 --- /dev/null +++ b/backend/src/api/admin/monitoring/monitoring-events.service.spec.ts @@ -0,0 +1,80 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' +import { EventEmitter } from 'events' +import type { ConfigService } from '@nestjs/config' +import { CHANGE_THROTTLE_MS, MonitoringEventsService } from './monitoring-events.service' +import { createRedisClient } from '../../../queue/redis-connection' +import { QueueName } from '../../../enum/queue-name.enum' +import { bullKey } from '../../../queue/queue-metrics' + +vi.mock('../../../queue/redis-connection', () => ({ createRedisClient: vi.fn() })) + +function fakeRedis() { + return Object.assign(new EventEmitter(), { + psubscribe: vi.fn().mockResolvedValue(4), + quit: vi.fn().mockResolvedValue('OK'), + disconnect: vi.fn(), + }) +} + +const config = (redis: unknown) => ({ get: vi.fn(() => redis) }) as unknown as ConfigService + +describe('MonitoringEventsService', () => { + let redis: ReturnType + + beforeEach(() => { + vi.useFakeTimers() + redis = fakeRedis() + vi.mocked(createRedisClient) + .mockReset() + .mockReturnValue(redis as never) + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('opens no Redis connection until someone watches', () => { + new MonitoringEventsService(config({ host: 'localhost', port: 6379, db: 0 })) + expect(createRedisClient).not.toHaveBeenCalled() + }) + + it('subscribes to every queue and shares one connection between viewers', () => { + const service = new MonitoringEventsService(config({ host: 'localhost', port: 6379, db: 0 })) + const a = service.changes$.subscribe() + const b = service.changes$.subscribe() + + expect(createRedisClient).toHaveBeenCalledTimes(1) + expect(redis.psubscribe).toHaveBeenCalledWith( + ...Object.values(QueueName).map((name) => bullKey(name, '*')), + ) + + a.unsubscribe() + expect(redis.quit).not.toHaveBeenCalled() + b.unsubscribe() + expect(redis.quit).toHaveBeenCalledTimes(1) + }) + + it('signals the first change at once and coalesces a burst into one trailing signal', () => { + const service = new MonitoringEventsService(config({ host: 'localhost', port: 6379, db: 0 })) + const received = vi.fn() + const subscription = service.changes$.subscribe(received) + + for (let i = 0; i < 500; i++) redis.emit('pmessage', 'p', 'c', 'm') + expect(received).toHaveBeenCalledTimes(1) + + vi.advanceTimersByTime(CHANGE_THROTTLE_MS) + expect(received).toHaveBeenCalledTimes(2) + + vi.advanceTimersByTime(CHANGE_THROTTLE_MS * 5) + expect(received).toHaveBeenCalledTimes(2) + subscription.unsubscribe() + }) + + it('emits nothing when Redis is not configured', () => { + const service = new MonitoringEventsService(config(undefined)) + const complete = vi.fn() + service.changes$.subscribe({ complete }) + expect(complete).toHaveBeenCalled() + expect(createRedisClient).not.toHaveBeenCalled() + }) +}) diff --git a/backend/src/api/admin/monitoring/monitoring-events.service.ts b/backend/src/api/admin/monitoring/monitoring-events.service.ts new file mode 100644 index 000000000..7c4d25527 --- /dev/null +++ b/backend/src/api/admin/monitoring/monitoring-events.service.ts @@ -0,0 +1,47 @@ +import { Injectable, Logger } from '@nestjs/common' +import { ConfigService } from '@nestjs/config' +import { EMPTY, Observable, asyncScheduler, share, throttleTime } from 'rxjs' +import { QueueName } from '../../../enum/queue-name.enum' +import { createRedisClient } from '../../../queue/redis-connection' +import type { RedisConfig } from '../../../queue/redis-connection' +import { bullKey } from '../../../queue/queue-metrics' + +/** Coalesces bursts: a 5,000-recipient batch reports progress per recipient. */ +export const CHANGE_THROTTLE_MS = 2_000 + +/** + * Signals that queue state has changed so the admin monitoring page can refetch its snapshot. + * + * Bull publishes every add, start, progress update, completion and failure on channels under + * `::`, from whichever pod did the work, so one pattern subscription per queue + * hears all of them. The subscriber connection exists only while at least one admin has the page + * open: share() opens it for the first stream and closes it when the last one ends. + */ +@Injectable() +export class MonitoringEventsService { + private readonly logger = new Logger(MonitoringEventsService.name) + readonly changes$: Observable + + constructor(configService: ConfigService) { + const redisConfig = configService.get('redis') + if (!redisConfig) { + this.changes$ = EMPTY + return + } + + const patterns = Object.values(QueueName).map((name) => bullKey(name, '*')) + this.changes$ = new Observable((subscriber) => { + const redis = createRedisClient(redisConfig, 'MonitoringEvents') + redis.on('pmessage', () => subscriber.next()) + redis.psubscribe(...patterns).catch((error: Error) => { + this.logger.error(`Failed to subscribe to queue events: ${error.message}`) + }) + return () => { + redis.quit().catch(() => redis.disconnect()) + } + }).pipe( + throttleTime(CHANGE_THROTTLE_MS, asyncScheduler, { leading: true, trailing: true }), + share(), + ) + } +} diff --git a/backend/src/api/admin/monitoring/monitoring.controller.spec.ts b/backend/src/api/admin/monitoring/monitoring.controller.spec.ts index 596d25653..56f69061f 100644 --- a/backend/src/api/admin/monitoring/monitoring.controller.spec.ts +++ b/backend/src/api/admin/monitoring/monitoring.controller.spec.ts @@ -14,4 +14,9 @@ describe('MonitoringController', () => { const roles = Reflect.getMetadata(ROLES_KEY, MonitoringController.prototype.getQueues) expect(roles).toEqual([SsoRole.NOTIFY_ADMIN]) }) + + it('requires the NOTIFY_ADMIN SSO role for the event stream', () => { + const roles = Reflect.getMetadata(ROLES_KEY, MonitoringController.prototype.streamQueueEvents) + expect(roles).toEqual([SsoRole.NOTIFY_ADMIN]) + }) }) diff --git a/backend/src/api/admin/monitoring/monitoring.controller.ts b/backend/src/api/admin/monitoring/monitoring.controller.ts index 99e2fb397..7f136eb47 100644 --- a/backend/src/api/admin/monitoring/monitoring.controller.ts +++ b/backend/src/api/admin/monitoring/monitoring.controller.ts @@ -1,11 +1,15 @@ -import { Controller, Get, UseGuards, Version } from '@nestjs/common' +import { Controller, Get, MessageEvent, Sse, UseGuards, Version } from '@nestjs/common' +import { Observable, interval, map, merge } from 'rxjs' import { ApiBearerAuth, ApiExcludeController, ApiOperation, ApiTags } from '@nestjs/swagger' import { NotifyAdminGuard } from '../../../common/guards/notify-admin.guard' import { Roles } from '../../../common/decorators/roles.decorator' import { SsoRole } from '../../../enum/sso-role.enum' import { MonitoringService } from './monitoring.service' +import { MonitoringEventsService } from './monitoring-events.service' import type { QueueMonitoringResponseDto } from './schemas/queue-monitoring.dto' +const KEEPALIVE_MS = 25_000 + /** * Admin monitoring of the Bull queues, their workers and Redis. * @@ -19,7 +23,10 @@ import type { QueueMonitoringResponseDto } from './schemas/queue-monitoring.dto' @UseGuards(NotifyAdminGuard) @ApiBearerAuth() export class MonitoringController { - constructor(private readonly monitoringService: MonitoringService) {} + constructor( + private readonly monitoringService: MonitoringService, + private readonly monitoringEvents: MonitoringEventsService, + ) {} @Version('1') @Get('queues') @@ -28,4 +35,22 @@ export class MonitoringController { getQueues(): Promise { return this.monitoringService.getQueueMonitoring() } + + /** + * Emits `changed` when queue state moves (at most every 2s) so the page refetches /queues, + * instead of polling it. A named keepalive every 25s stops proxies closing an idle stream. + */ + @Version('1') + @Sse('queues/events') + @Roles(SsoRole.NOTIFY_ADMIN) + @ApiOperation({ summary: 'Stream queue change signals for the monitoring page (admin only)' }) + streamQueueEvents(): Observable { + const changes$ = this.monitoringEvents.changes$.pipe( + map(() => ({ type: 'changed', data: '' }) as MessageEvent), + ) + const keepalive$ = interval(KEEPALIVE_MS).pipe( + map(() => ({ type: 'keepalive', data: '' }) as MessageEvent), + ) + return merge(changes$, keepalive$) + } } diff --git a/backend/src/api/admin/monitoring/monitoring.module.ts b/backend/src/api/admin/monitoring/monitoring.module.ts index d85c21ed0..92e1d14e6 100644 --- a/backend/src/api/admin/monitoring/monitoring.module.ts +++ b/backend/src/api/admin/monitoring/monitoring.module.ts @@ -1,14 +1,16 @@ import { Module } from '@nestjs/common' import { TypeOrmModule } from '@nestjs/typeorm' import { Tenant } from '../tenants/entities/tenant.entity' +import { NotificationRequestDetail } from '../../notification/entities/notification-request-detail.entity' import { QueueModule } from '../../../queue/queue.module' import { NotifyAdminGuard } from '../../../common/guards/notify-admin.guard' import { MonitoringController } from './monitoring.controller' import { MonitoringService } from './monitoring.service' +import { MonitoringEventsService } from './monitoring-events.service' @Module({ - imports: [TypeOrmModule.forFeature([Tenant]), QueueModule], + imports: [TypeOrmModule.forFeature([Tenant, NotificationRequestDetail]), QueueModule], controllers: [MonitoringController], - providers: [MonitoringService, NotifyAdminGuard], + providers: [MonitoringService, MonitoringEventsService, NotifyAdminGuard], }) export class MonitoringModule {} diff --git a/backend/src/api/admin/monitoring/monitoring.service.spec.ts b/backend/src/api/admin/monitoring/monitoring.service.spec.ts index a280c9caa..7df15e1a2 100644 --- a/backend/src/api/admin/monitoring/monitoring.service.spec.ts +++ b/backend/src/api/admin/monitoring/monitoring.service.spec.ts @@ -7,6 +7,7 @@ import { MonitoringService, averageOfWholeMinutes, buildRedisStats, + countUnheldActiveJobs, evaluateQueue, maskPersonalData, parseRedisInfo, @@ -14,6 +15,7 @@ import { } from './monitoring.service' import { MONITORING_WINDOW_MINUTES } from '../../../queue/queue-metrics' import type { Tenant } from '../tenants/entities/tenant.entity' +import type { NotificationRequestDetail } from '../../notification/entities/notification-request-detail.entity' const MB = 1024 * 1024 @@ -57,6 +59,7 @@ const baseQueue = { failureRatePercent: 0, }, liveWorkerPods: 2, + unheldActiveJobs: 0, } describe('parseRedisInfo', () => { @@ -154,6 +157,47 @@ describe('evaluateQueue', () => { }) }) +describe('countUnheldActiveJobs', () => { + const now = 1_000_000 + const pod = (active: number) => ({ + podId: 'pod', + startedAt: '', + lastHeartbeatAt: '', + queues: [ + { + queue: 'email-delivery', + concurrency: 2, + active, + completed: 0, + failed: 0, + lastFinishedAt: null, + }, + ], + }) + const job = (ageMs: number) => ({ processedOn: now - ageMs }) as unknown as Bull.Job + + it('is zero when live workers report holding every active job', () => { + expect(countUnheldActiveJobs('email-delivery', [job(60_000)], [pod(1)], now)).toBe(0) + }) + + it('counts settled active jobs that no live worker holds', () => { + expect(countUnheldActiveJobs('email-delivery', [job(60_000), job(90_000)], [pod(1)], now)).toBe( + 1, + ) + expect(countUnheldActiveJobs('email-delivery', [job(60_000)], [], now)).toBe(1) + }) + + it('allows for heartbeat lag on jobs that only just started', () => { + expect(countUnheldActiveJobs('email-delivery', [job(5_000)], [pod(0)], now)).toBe(0) + }) + + it('warns on the queue when a job looks stalled', () => { + const queue = evaluateQueue({ ...baseQueue, unheldActiveJobs: 1 }) + expect(queue.status).toBe('warning') + expect(queue.reasons[0]).toMatch(/not held by any live worker/) + }) +}) + describe('helpers', () => { it('averages whole minutes and leaves out the current partial minute', () => { expect(averageOfWholeMinutes([1, 2, 4, 6, 8, 10, 999], 5)).toBe(6) @@ -193,6 +237,7 @@ describe('MonitoringService', () => { }), getJob: vi.fn().mockResolvedValue({ timestamp: now - 30_000, opts: {} }), getFailed: vi.fn().mockResolvedValue([]), + getActive: vi.fn().mockResolvedValue([]), client: { lindex: vi.fn().mockResolvedValue('job-1'), mget: vi.fn((...keys: string[]) => Promise.resolve(keys.map(() => null))), @@ -214,6 +259,30 @@ describe('MonitoringService', () => { } as unknown as Redis } + /** Query builders resolve in creation order: the pending query, then the finished query. */ + function fakeDetailRepository(pendingRows: object[] = [], finishedRows: object[] = []) { + const results = [pendingRows, finishedRows] + return { + createQueryBuilder: vi.fn(() => { + const rows = results.shift() ?? [] + const builder: Record = {} + for (const method of [ + 'innerJoin', + 'select', + 'addSelect', + 'where', + 'andWhere', + 'groupBy', + 'addGroupBy', + ]) { + builder[method] = vi.fn(() => builder) + } + builder.getRawMany = vi.fn().mockResolvedValue(rows) + return builder + }), + } as unknown as Repository + } + const tenantRepository = { find: vi.fn().mockResolvedValue([{ id: 't-1', name: 'Health Ministry' }]), } as unknown as Repository @@ -242,6 +311,7 @@ describe('MonitoringService', () => { null, null, tenantRepository, + fakeDetailRepository(), ) const result = await service.getQueueMonitoring() @@ -272,6 +342,7 @@ describe('MonitoringService', () => { null, null, tenantRepository, + fakeDetailRepository(), ) const { recentFailures } = await service.getQueueMonitoring() @@ -300,14 +371,100 @@ describe('MonitoringService', () => { null, null, tenantRepository, + fakeDetailRepository(), ) const { workers } = await service.getQueueMonitoring() expect(workers.map((worker) => worker.podId)).toEqual(['pod-0']) }) + it('counts pending recipients and per-minute sends by channel', async () => { + const minuteStart = (offsetMinutes: number) => + new Date((Math.floor(now / 60_000) - offsetMinutes) * 60_000) + const service = new MonitoringService( + fakeRedis([livePod]), + null, + fakeQueue('email-delivery'), + null, + null, + tenantRepository, + fakeDetailRepository( + [{ channel: 'EMAIL', pending: '60', oldest: new Date(now - 90_000) }], + [ + { channel: 'EMAIL', status: 'sent', minute: minuteStart(1), count: '20' }, + { channel: 'EMAIL', status: 'failed', minute: minuteStart(1), count: '5' }, + { channel: 'EMAIL', status: 'sent', minute: minuteStart(0), count: '7' }, + // Outside the hour: ignored rather than written past the end of the series. + { channel: 'EMAIL', status: 'sent', minute: minuteStart(90), count: '999' }, + ], + ), + ) + + const { messages } = await service.getQueueMonitoring() + const email = messages.find((m) => m.channel === 'EMAIL')! + const sms = messages.find((m) => m.channel === 'SMS')! + + expect(email.pending).toBe(60) + expect(email.oldestPendingAgeMs).toBeGreaterThanOrEqual(90_000) + expect(email.sent.at(-1)).toBe(7) + expect(email.sent.at(-2)).toBe(20) + expect(email.failed.at(-2)).toBe(5) + expect(email.sent.reduce((a, b) => a + b, 0)).toBe(27) + // 25 finished in the one busy minute of the last five -> 5/min -> 60 pending clears in 12. + expect(email.sentPerMinute).toBe(4) + expect(email.failedPerMinute).toBe(1) + expect(email.estimatedClearMinutes).toBe(12) + expect(sms).toMatchObject({ pending: 0, oldestPendingAgeMs: null, estimatedClearMinutes: 0 }) + }) + + it('reports progress for active merge batches and ignores other active jobs', async () => { + const batch = { + id: 'req-1-EMAIL-0', + processedOn: now - 20_000, + data: { notifyId: 'req-1', tenantId: 't-1', mailMergeData: { recipients: ['secret'] } }, + progress: () => ({ sent: 37, failed: 2, total: 100 }), + } + const single = { id: '9', processedOn: now, data: {}, progress: () => 0 } + const queue = fakeQueue('email-delivery', { + getActive: vi.fn().mockResolvedValue([batch, single]), + }) + const service = new MonitoringService( + fakeRedis([livePod]), + null, + queue, + null, + null, + tenantRepository, + fakeDetailRepository(), + ) + + const { activeBatches } = await service.getQueueMonitoring() + + expect(activeBatches).toEqual([ + { + queue: 'email-delivery', + jobId: 'req-1-EMAIL-0', + notificationId: 'req-1', + tenantId: 't-1', + tenantName: 'Health Ministry', + sent: 37, + failed: 2, + total: 100, + startedAt: new Date(now - 20_000).toISOString(), + }, + ]) + }) + it('is unavailable when Redis is not configured', async () => { - const service = new MonitoringService(null, null, null, null, null, tenantRepository) + const service = new MonitoringService( + null, + null, + null, + null, + null, + tenantRepository, + fakeDetailRepository(), + ) await expect(service.getQueueMonitoring()).rejects.toBeInstanceOf(ServiceUnavailableException) }) }) diff --git a/backend/src/api/admin/monitoring/monitoring.service.ts b/backend/src/api/admin/monitoring/monitoring.service.ts index dcbd26abb..2c984fdf4 100644 --- a/backend/src/api/admin/monitoring/monitoring.service.ts +++ b/backend/src/api/admin/monitoring/monitoring.service.ts @@ -6,6 +6,11 @@ import type Redis from 'ioredis' import { QueueName } from '../../../enum/queue-name.enum' import { ProviderToken } from '../../../enum/provider-token.enum' import { Tenant } from '../tenants/entities/tenant.entity' +import { NotificationRequestDetail } from '../../notification/entities/notification-request-detail.entity' +import { NotificationStatus } from '../../../enum/notification-status.enum' +import { NotificationChannel } from '../../../enum/notification-channel.enum' +import { readBatchProgress } from '../../../queue/batch-progress' +import type { BatchProgress } from '../../../queue/batch-progress' import { MONITORING_WINDOW_MINUTES, addedCounterKey, @@ -15,6 +20,7 @@ import { } from '../../../queue/queue-metrics' import type { BullMetrics } from '../../../queue/queue-metrics' import { + HEARTBEAT_INTERVAL_MS, HEARTBEAT_TTL_SECONDS, WORKER_INDEX_KEY, workerHeartbeatKey, @@ -23,14 +29,25 @@ import type { WorkerHeartbeatPayload } from '../../../queue/worker-heartbeat' import { MONITORING_THRESHOLDS } from './monitoring-thresholds' import type { HealthStatus, + MessageChannelStatsDto, QueueMonitoringResponseDto, QueueStatsDto, - RecentFailureDto, RedisStatsDto, WorkerPodDto, } from './schemas/queue-monitoring.dto' const FAILURES_PER_QUEUE = 10 +const ACTIVE_JOBS_PER_QUEUE = 50 +/** + * A worker's heartbeat can lag what it is doing by one interval, so an active job younger than + * two intervals may be held by a live worker that has not reported it yet. + */ +const UNHELD_GRACE_MS = HEARTBEAT_INTERVAL_MS * 2 +/** Detail statuses between acceptance and a final sent/failed. */ +const IN_FLIGHT_DETAIL_STATUSES = ['pending', 'queued', 'processing', 'sending'] +/** Older in-flight rows are abandoned, not backlog, and would pin "oldest pending" forever. */ +const PENDING_LOOKBACK_MS = 24 * 60 * 60 * 1000 +const DEFAULT_MESSAGE_CHANNELS = [NotificationChannel.EMAIL, NotificationChannel.SMS] const RECENT_FAILURES_LIMIT = 20 /** A failure rate over a handful of jobs is noise; below this many finished jobs it is ignored. */ const MIN_JOBS_FOR_FAILURE_RATE = 10 @@ -141,6 +158,12 @@ export function evaluateQueue(input: QueueEvaluationInput): QueueStatsDto { statuses.push('warning') reasons.push('Queue is paused') } + if (input.unheldActiveJobs > 0) { + statuses.push('warning') + reasons.push( + `${input.unheldActiveJobs} active job(s) not held by any live worker (possibly stalled; Bull retries stalled jobs within about a minute)`, + ) + } if (backlog > 0 && liveWorkerPods === 0) { statuses.push('critical') reasons.push('Jobs are waiting but no pod is processing this queue') @@ -181,9 +204,56 @@ export function evaluateQueue(input: QueueEvaluationInput): QueueStatsDto { } type JobIdentifiers = { notifyId?: unknown; notificationId?: unknown; tenantId?: unknown } +type QueuedJob = { queue: string; job: Bull.Job } const asString = (value: unknown): string | null => (typeof value === 'string' ? value : null) +/** Only identifiers leave the job: its data also holds recipients and message content. */ +function jobIdentifiers(job: Bull.Job, tenantNames: Map) { + const data = (job.data ?? {}) as JobIdentifiers + const tenantId = asString(data.tenantId) + return { + notificationId: asString(data.notifyId) ?? asString(data.notificationId), + tenantId, + tenantName: tenantId ? (tenantNames.get(tenantId) ?? null) : null, + } +} + +/** Active jobs that report merge-batch progress; other jobs finish too fast to need a bar. */ +function activeBatches( + activeByQueue: Map, +): Array { + return [...activeByQueue.entries()] + .flatMap(([queue, jobs]) => + jobs.flatMap((job) => { + const progress = readBatchProgress(job.progress()) + return progress ? [{ queue, job, progress }] : [] + }), + ) + .sort((a, b) => (a.job.processedOn ?? 0) - (b.job.processedOn ?? 0)) +} + +/** + * Active jobs no live worker reports holding. A job stays "active" in Redis after the pod + * processing it dies, until Bull's stalled-job check moves it back to waiting, so these are + * likely stalled. Only jobs older than UNHELD_GRACE_MS are counted, to allow for heartbeat lag. + */ +export function countUnheldActiveJobs( + queueName: string, + activeJobs: Bull.Job[], + workers: WorkerPodDto[], + now: number, +): number { + const settled = activeJobs.filter( + (job) => job.processedOn && job.processedOn <= now - UNHELD_GRACE_MS, + ).length + const held = workers + .flatMap((pod) => pod.queues) + .filter((q) => q.queue === queueName) + .reduce((sum, q) => sum + q.active, 0) + return Math.max(0, settled - held) +} + /** * Read-only view of the Bull queues, their workers and the Redis instance behind them, for the * admin monitoring page. Everything is read from Redis on each request, so the answer is the @@ -200,6 +270,8 @@ export class MonitoringService { @Optional() @Inject(QueueName.SMS_DELIVERY) smsQueue: Bull.Queue | null, @Optional() @Inject(QueueName.WEBHOOK_DELIVERY) webhookQueue: Bull.Queue | null, @InjectRepository(Tenant) private readonly tenantRepository: Repository, + @InjectRepository(NotificationRequestDetail) + private readonly detailRepository: Repository, ) { this.queues = [ingestionQueue, emailQueue, smsQueue, webhookQueue].filter( (queue): queue is Bull.Queue => !!queue, @@ -214,26 +286,55 @@ export class MonitoringService { } const now = Date.now() - const [info, workers] = await Promise.all([this.redis.info(), this.getWorkers(now)]) + const [info, workers, messages, failedJobs, activeByQueue] = await Promise.all([ + this.redis.info(), + this.getWorkers(now), + this.getMessageStats(now), + this.getRecentFailedJobs(), + this.getActiveJobs(), + ]) const redis = buildRedisStats(parseRedisInfo(info)) const queues = await Promise.all( - this.queues.map((queue) => this.getQueueStats(queue, workers, now)), + this.queues.map((queue) => + this.getQueueStats(queue, workers, activeByQueue.get(queue.name) ?? [], now), + ), + ) + const activeJobs = activeBatches(activeByQueue) + const tenantNames = await this.getTenantNames( + [...failedJobs, ...activeJobs].map(({ job }) => + asString((job.data as JobIdentifiers)?.tenantId), + ), ) - const recentFailures = await this.getRecentFailures() return { generatedAt: new Date(now).toISOString(), status: worstStatus([redis.status, ...queues.map((queue) => queue.status)]), redis, + messages, queues, + activeBatches: activeJobs.map(({ queue, job, progress }) => ({ + queue, + jobId: String(job.id), + ...jobIdentifiers(job, tenantNames), + ...progress, + startedAt: job.processedOn ? new Date(job.processedOn).toISOString() : null, + })), workers, - recentFailures, + recentFailures: failedJobs.map(({ queue, job }) => ({ + queue, + jobId: String(job.id), + ...jobIdentifiers(job, tenantNames), + reason: maskPersonalData(job.failedReason ?? 'No reason recorded'), + attemptsMade: job.attemptsMade, + failedAt: job.finishedOn ? new Date(job.finishedOn).toISOString() : null, + })), } } private async getQueueStats( queue: Bull.Queue, workers: WorkerPodDto[], + activeJobs: Bull.Job[], now: number, ): Promise { const [counts, isPaused, oldestWaitingAgeMs, added, completed, failed] = await Promise.all([ @@ -275,6 +376,7 @@ export class MonitoringService { }, liveWorkerPods: workers.filter((pod) => pod.queues.some((q) => q.queue === queue.name)) .length, + unheldActiveJobs: countUnheldActiveJobs(queue.name, activeJobs, workers, now), }) } @@ -327,46 +429,113 @@ export class MonitoringService { .sort((a, b) => a.podId.localeCompare(b.podId)) } - private async getRecentFailures(): Promise { + private async getRecentFailedJobs(): Promise { const perQueue = await Promise.all( this.queues.map(async (queue) => { const jobs = await queue.getFailed(0, FAILURES_PER_QUEUE - 1) return jobs.filter(Boolean).map((job) => ({ queue: queue.name, job })) }), ) - - const failures = perQueue + return perQueue .flat() .sort((a, b) => (b.job.finishedOn ?? 0) - (a.job.finishedOn ?? 0)) .slice(0, RECENT_FAILURES_LIMIT) + } - const tenantIds = [ - ...new Set( - failures - .map(({ job }) => asString((job.data as JobIdentifiers)?.tenantId)) - .filter((id): id is string => !!id), + private async getActiveJobs(): Promise> { + const perQueue = await Promise.all( + this.queues.map( + async (queue) => + [ + queue.name, + (await queue.getActive(0, ACTIVE_JOBS_PER_QUEUE - 1)).filter(Boolean), + ] as const, ), - ] - const tenants = tenantIds.length - ? await this.tenantRepository.find({ - where: { id: In(tenantIds) }, - select: { id: true, name: true }, + ) + return new Map(perQueue) + } + + private async getTenantNames(ids: Array): Promise> { + const tenantIds = [...new Set(ids.filter((id): id is string => !!id))] + if (tenantIds.length === 0) return new Map() + const tenants = await this.tenantRepository.find({ + where: { id: In(tenantIds) }, + select: { id: true, name: true }, + }) + return new Map(tenants.map((tenant) => [tenant.id, tenant.name])) + } + + /** + * Recipient-level backlog and throughput per channel. Pending rows use the status index; + * sent/failed rows use the partial index on last_attempt_at (V72), whose predicate the + * status filter repeats literally so the planner can use it. + */ + private async getMessageStats(now: number): Promise { + const since = new Date((minuteOf(now) - (MONITORING_WINDOW_MINUTES - 1)) * 60_000) + + const [pendingRows, finishedRows] = await Promise.all([ + this.detailRepository + .createQueryBuilder('detail') + .innerJoin('detail.notificationRequest', 'request') + .select('detail.channel', 'channel') + .addSelect('COUNT(*)', 'pending') + .addSelect('MIN(detail.created_at)', 'oldest') + .where('detail.status IN (:...statuses)', { statuses: IN_FLIGHT_DETAIL_STATUSES }) + .andWhere('request.status <> :scheduled', { scheduled: NotificationStatus.SCHEDULED }) + .andWhere('detail.created_at > :pendingSince', { + pendingSince: new Date(now - PENDING_LOOKBACK_MS), }) - : [] - const tenantNames = new Map(tenants.map((tenant) => [tenant.id, tenant.name])) + .groupBy('detail.channel') + .getRawMany<{ channel: string; pending: string; oldest: Date | string | null }>(), + this.detailRepository + .createQueryBuilder('detail') + .select('detail.channel', 'channel') + .addSelect('detail.status', 'status') + .addSelect("date_trunc('minute', detail.last_attempt_at)", 'minute') + .addSelect('COUNT(*)', 'count') + .where("detail.status IN ('sent', 'failed')") + .andWhere('detail.last_attempt_at >= :since', { since }) + .groupBy('detail.channel') + .addGroupBy('detail.status') + .addGroupBy('minute') + .getRawMany<{ channel: string; status: string; minute: Date | string; count: string }>(), + ]) + + const channels = new Set(DEFAULT_MESSAGE_CHANNELS) + pendingRows.forEach((row) => channels.add(row.channel)) + finishedRows.forEach((row) => channels.add(row.channel)) + + const firstMinute = minuteOf(since.getTime()) + const window = MONITORING_THRESHOLDS.rateWindowMinutes + + return [...channels].map((channel) => { + const sent = Array(MONITORING_WINDOW_MINUTES).fill(0) + const failed = Array(MONITORING_WINDOW_MINUTES).fill(0) + for (const row of finishedRows) { + if (row.channel !== channel) continue + const index = minuteOf(new Date(row.minute).getTime()) - firstMinute + if (index < 0 || index >= MONITORING_WINDOW_MINUTES) continue + const series = row.status === 'sent' ? sent : failed + series[index] += Number(row.count) || 0 + } + + const pendingRow = pendingRows.find((row) => row.channel === channel) + const pending = Number(pendingRow?.pending ?? 0) || 0 + const oldest = pendingRow?.oldest ? new Date(pendingRow.oldest).getTime() : null + const sentPerMinute = averageOfWholeMinutes(sent, window) + const failedPerMinute = averageOfWholeMinutes(failed, window) + const finishedPerMinute = sentPerMinute + failedPerMinute - return failures.map(({ queue, job }) => { - const data = (job.data ?? {}) as JobIdentifiers - const tenantId = asString(data.tenantId) return { - queue, - jobId: String(job.id), - notificationId: asString(data.notifyId) ?? asString(data.notificationId), - tenantId, - tenantName: tenantId ? (tenantNames.get(tenantId) ?? null) : null, - reason: maskPersonalData(job.failedReason ?? 'No reason recorded'), - attemptsMade: job.attemptsMade, - failedAt: job.finishedOn ? new Date(job.finishedOn).toISOString() : null, + channel, + pending, + oldestPendingAgeMs: pending > 0 && oldest !== null ? Math.max(0, now - oldest) : null, + sent, + failed, + sentPerMinute, + failedPerMinute, + estimatedClearMinutes: + pending === 0 ? 0 : finishedPerMinute > 0 ? Math.ceil(pending / finishedPerMinute) : null, } }) } diff --git a/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts b/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts index 628aa4260..7d0a3b58a 100644 --- a/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts +++ b/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts @@ -57,6 +57,8 @@ export interface QueueStatsDto extends StatusReason { /** Minutes until the backlog clears at the current rates; null when it is not shrinking. */ estimatedDrainMinutes: number | null liveWorkerPods: number + /** Active jobs no live worker reports holding: likely stalled after a pod died. */ + unheldActiveJobs: number } export interface WorkerQueueDto { @@ -87,11 +89,43 @@ export interface RecentFailureDto { failedAt: string | null } +/** + * Individual recipients for one channel, counted from notification_request_detail. A merge + * batch is one job but up to BATCH_SIZE messages, so job counts alone understate the backlog. + */ +export interface MessageChannelStatsDto { + channel: string + /** Recipients accepted in the last 24 hours and not yet sent or failed; scheduled sends excluded. */ + pending: number + oldestPendingAgeMs: number | null + /** Per-minute counts, oldest first; the last entry is the current, partial minute. */ + sent: number[] + failed: number[] + sentPerMinute: number + failedPerMinute: number + /** Minutes until pending reaches zero at the current send rate; null when nothing is sending. */ + estimatedClearMinutes: number | null +} + +export interface ActiveBatchDto { + queue: string + jobId: string + notificationId: string | null + tenantId: string | null + tenantName: string | null + sent: number + failed: number + total: number + startedAt: string | null +} + export interface QueueMonitoringResponseDto { generatedAt: string status: HealthStatus redis: RedisStatsDto + messages: MessageChannelStatsDto[] queues: QueueStatsDto[] + activeBatches: ActiveBatchDto[] workers: WorkerPodDto[] recentFailures: RecentFailureDto[] } diff --git a/backend/src/queue/batch-progress.spec.ts b/backend/src/queue/batch-progress.spec.ts new file mode 100644 index 000000000..8e6dfff37 --- /dev/null +++ b/backend/src/queue/batch-progress.spec.ts @@ -0,0 +1,50 @@ +import { describe, it, expect, vi } from 'vitest' +import type Bull from 'bull' +import { batchProgressReporter, readBatchProgress } from './batch-progress' + +const flush = () => new Promise((resolve) => setTimeout(resolve, 0)) + +describe('batchProgressReporter', () => { + it('writes the progress to the job', async () => { + const job = { id: '1', progress: vi.fn().mockResolvedValue(undefined) } + batchProgressReporter(job as unknown as Bull.Job)({ sent: 3, failed: 1, total: 10 }) + await flush() + expect(job.progress).toHaveBeenCalledWith({ sent: 3, failed: 1, total: 10 }) + }) + + it('never throws into the send loop, even when the write fails synchronously', async () => { + const job = { + id: '1', + progress: vi.fn(() => { + throw new Error('not a function in this test double') + }), + } + expect(() => + batchProgressReporter(job as unknown as Bull.Job)({ sent: 0, failed: 0, total: 1 }), + ).not.toThrow() + await flush() + }) + + it('swallows a rejected write', async () => { + const job = { id: '1', progress: vi.fn().mockRejectedValue(new Error('redis down')) } + batchProgressReporter(job as unknown as Bull.Job)({ sent: 0, failed: 0, total: 1 }) + await flush() + expect(job.progress).toHaveBeenCalled() + }) +}) + +describe('readBatchProgress', () => { + it('reads a merge batch progress object', () => { + expect(readBatchProgress({ sent: 1, failed: 0, total: 5 })).toEqual({ + sent: 1, + failed: 0, + total: 5, + }) + }) + + it('returns null for Bull default progress and malformed values', () => { + expect(readBatchProgress(0)).toBeNull() + expect(readBatchProgress({ sent: '1', failed: 0, total: 5 })).toBeNull() + expect(readBatchProgress(null)).toBeNull() + }) +}) diff --git a/backend/src/queue/batch-progress.ts b/backend/src/queue/batch-progress.ts new file mode 100644 index 000000000..f2fd183ef --- /dev/null +++ b/backend/src/queue/batch-progress.ts @@ -0,0 +1,39 @@ +import type Bull from 'bull' +import { Logger } from '@nestjs/common' + +/** How far a merge batch has got, stored as the Bull job's progress. */ +export interface BatchProgress { + sent: number + failed: number + total: number +} + +export type BatchProgressReporter = (progress: BatchProgress) => void + +const logger = new Logger('BatchProgress') + +/** + * Report a merge batch's per-recipient progress on its Bull job, so the admin monitoring page + * can show "37 of 100" for a batch that is one job but many emails. Best-effort: the write is + * not awaited and a failure is only logged, so progress never slows or fails a send. + */ +export function batchProgressReporter(job: Bull.Job): BatchProgressReporter { + return (progress) => { + // Deferred through then() so even a synchronous throw lands in catch, not in the send loop. + Promise.resolve() + .then(() => job.progress(progress)) + .catch((error: Error) => { + logger.warn(`Failed to record progress for job ${job.id}: ${error.message}`) + }) + } +} + +/** Read a job's progress back, or null when it is not a merge batch's. */ +export function readBatchProgress(value: unknown): BatchProgress | null { + if (!value || typeof value !== 'object') return null + const { sent, failed, total } = value as Record + if (typeof sent !== 'number' || typeof failed !== 'number' || typeof total !== 'number') { + return null + } + return { sent, failed, total } +} diff --git a/backend/src/queue/workers/email-delivery.worker.spec.ts b/backend/src/queue/workers/email-delivery.worker.spec.ts index 83f80d5aa..5badbe265 100644 --- a/backend/src/queue/workers/email-delivery.worker.spec.ts +++ b/backend/src/queue/workers/email-delivery.worker.spec.ts @@ -1513,6 +1513,23 @@ describe('EmailDeliveryWorker', () => { ) }) + it('should report batch progress before the first send and after each recipient', async () => { + vi.mocked(mockEmailAdapter.send) + .mockResolvedValueOnce({ messageId: 'ext-1' }) + .mockRejectedValueOnce(new Error('rejected')) + const progress = vi.fn().mockResolvedValue(undefined) + const job = { ...makeBulkJob(['alice@example.com', 'bob@example.com']), progress } + + await processHandler(job as unknown as Bull.Job) + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect(progress.mock.calls.map(([value]) => value)).toEqual([ + { sent: 0, failed: 0, total: 2 }, + { sent: 1, failed: 0, total: 2 }, + { sent: 1, failed: 1, total: 2 }, + ]) + }) + it('should render inline content per recipient when no templateId is given', async () => { mockInlineRenderingService.renderEmail.mockResolvedValue({ subject: 'Hi', diff --git a/backend/src/queue/workers/email-delivery.worker.ts b/backend/src/queue/workers/email-delivery.worker.ts index 13c19e243..8e9fbaf08 100644 --- a/backend/src/queue/workers/email-delivery.worker.ts +++ b/backend/src/queue/workers/email-delivery.worker.ts @@ -2,6 +2,7 @@ import { HttpException, Logger, NotFoundException } from '@nestjs/common' import Bull from 'bull' import { ConfigService } from '@nestjs/config' import { DeliveryJobPayload, MailMergeJobData } from '../queue.types' +import { batchProgressReporter, type BatchProgressReporter } from '../batch-progress' import { NotificationService } from '../../api/notification/notification.service' import { NotificationRequestDetailService } from '../../api/notification/notification-request-detail.service' import { TemplatesRepository } from '../../api/templates/templates.repository' @@ -134,6 +135,7 @@ export class EmailDeliveryWorker { requestDetailService, notificationService, fromAddress, + batchProgressReporter(job), ) } @@ -477,6 +479,7 @@ export class EmailDeliveryWorker { requestDetailService: NotificationRequestDetailService, notificationService: NotificationService, fromAddress: string | null, + reportProgress?: BatchProgressReporter, ): Promise<{ success: boolean; batchId: string; sent: number; failed: number }> { const { content, params, recipients } = mailMergeData const templateId = content?.templateId @@ -512,6 +515,7 @@ export class EmailDeliveryWorker { let sent = 0 let failed = 0 + reportProgress?.({ sent, failed, total: recipients.length }) for (const recipient of recipients) { try { @@ -575,6 +579,7 @@ export class EmailDeliveryWorker { ) failed++ } + reportProgress?.({ sent, failed, total: recipients.length }) } logger.log(`[${notifyId}] Mail merge batch ${batchId} complete: sent=${sent}, failed=${failed}`) diff --git a/backend/src/queue/workers/sms-delivery.worker.ts b/backend/src/queue/workers/sms-delivery.worker.ts index 9854cb3ce..5dec206ce 100644 --- a/backend/src/queue/workers/sms-delivery.worker.ts +++ b/backend/src/queue/workers/sms-delivery.worker.ts @@ -2,6 +2,7 @@ import { BadRequestException, HttpException, Logger, NotFoundException } from '@ import Bull from 'bull' import { ConfigService } from '@nestjs/config' import { DeliveryJobPayload, MailMergeJobData } from '../queue.types' +import { batchProgressReporter, type BatchProgressReporter } from '../batch-progress' import { NotificationService } from '../../api/notification/notification.service' import { NotificationRequestDetailService } from '../../api/notification/notification-request-detail.service' import { TemplatesRepository } from '../../api/templates/templates.repository' @@ -97,6 +98,7 @@ export class SmsDeliveryWorker { smsAdapter, requestDetailService, notificationService, + batchProgressReporter(job), ) } @@ -405,6 +407,7 @@ export class SmsDeliveryWorker { smsAdapter: ISmsTransport, requestDetailService: NotificationRequestDetailService, notificationService: NotificationService, + reportProgress?: BatchProgressReporter, ): Promise<{ success: boolean; batchId: string; sent: number; failed: number }> { const { content, params, recipients } = mailMergeData const templateId = content?.templateId @@ -452,6 +455,9 @@ export class SmsDeliveryWorker { let sent = 0 let failed = 0 + // Recipients delivered by an earlier attempt count as sent, so a retried batch resumes its bar. + const progress = () => ({ sent: alreadySent.size + sent, failed, total: recipients.length }) + reportProgress?.(progress()) for (const recipient of pending) { try { @@ -490,6 +496,7 @@ export class SmsDeliveryWorker { ) failed++ } + reportProgress?.(progress()) } logger.log(`[${notifyId}] SMS merge batch ${batchId} complete: sent=${sent}, failed=${failed}`) diff --git a/frontend/src/api/monitoring.api.ts b/frontend/src/api/monitoring.api.ts index 273dcf5fa..6324aaff9 100644 --- a/frontend/src/api/monitoring.api.ts +++ b/frontend/src/api/monitoring.api.ts @@ -1,6 +1,8 @@ import type { AxiosError } from 'axios' +import { fetchEventSource, type EventSourceMessage } from '@microsoft/fetch-event-source' import { get, generateApiParameters, STATUS_CODES } from '@/common/api' import type { QueueMonitoring } from '@/interfaces/queueMonitoring.interface' +import UserService from '@/service/user-service' /** * Queue, worker and Redis health for the admin monitoring page. Requires NOTIFY_ADMIN. @@ -22,3 +24,57 @@ export async function getQueueMonitoring(): Promise { ) } } + +export interface QueueMonitoringStreamHandlers { + onOpen: () => void + /** Queue state changed; refetch the snapshot. */ + onChange: () => void + /** The stream dropped. `fatal` means it was refused and will not retry. */ + onError: (fatal: boolean) => void +} + +/** A refusal retrying cannot fix, such as 401 or 403. Thrown to stop fetch-event-source. */ +class StreamRefusedError extends Error {} + +/** + * Open the monitoring change stream. It carries no data, only `changed` signals (at most every + * 2s) and keepalives, so the page refetches the snapshot when something moves. Closes while the + * tab is hidden and reconnects when it is shown. Call abort() on the result to close it. + */ +export function connectQueueMonitoringStream(handlers: QueueMonitoringStreamHandlers) { + const controller = new AbortController() + const url = generateApiParameters('/api/v1/frontend/admin/monitoring/queues/events').url + + const fetchWithFreshToken = async (input: RequestInfo | URL, init?: RequestInit) => { + const token = await UserService.getToken() + return fetch(input, { + ...init, + headers: { ...(init?.headers ?? {}), Authorization: `Bearer ${token}` }, + }) + } + + fetchEventSource(url, { + fetch: fetchWithFreshToken, + signal: controller.signal, + async onopen(response) { + if (response.status === 401 || response.status === 403) { + throw new StreamRefusedError(`Monitoring stream refused with ${response.status}`) + } + if (!response.ok) throw new Error(`Monitoring stream failed with ${response.status}`) + handlers.onOpen() + }, + onmessage(event: EventSourceMessage) { + if (event.event === 'changed') handlers.onChange() + }, + onerror(error: unknown) { + const fatal = error instanceof StreamRefusedError + handlers.onError(fatal) + // Throwing stops the retry loop; returning lets it reconnect with backoff. + if (fatal) throw error + }, + }).catch(() => { + // Rejects only after a fatal error, which onError has already reported. + }) + + return controller +} diff --git a/frontend/src/hooks/useLiveUpdates.ts b/frontend/src/hooks/useLiveUpdates.ts new file mode 100644 index 000000000..73e18101d --- /dev/null +++ b/frontend/src/hooks/useLiveUpdates.ts @@ -0,0 +1,49 @@ +import { useEffect, useRef, useState } from 'react' + +export type LiveStatus = 'off' | 'connecting' | 'live' | 'reconnecting' | 'unavailable' + +interface StreamHandlers { + onOpen: () => void + onChange: () => void + onError: (fatal: boolean) => void +} + +/** + * Keep a server-sent change stream open while `enabled`, calling `onChange` for each signal. + * `connect` opens the stream and returns its AbortController; it must be stable. + */ +export function useLiveUpdates( + connect: (handlers: StreamHandlers) => AbortController, + enabled: boolean, + onChange: () => void, +): LiveStatus { + const [status, setStatus] = useState('connecting') + const onChangeRef = useRef(onChange) + + useEffect(() => { + onChangeRef.current = onChange + }, [onChange]) + + useEffect(() => { + if (!enabled) return + let opened = false + const controller = connect({ + onOpen: () => { + setStatus('live') + // Changes made while the stream was down were never signalled; catch up on reconnect. + if (opened) onChangeRef.current() + opened = true + }, + onChange: () => { + onChangeRef.current() + }, + onError: (fatal) => setStatus(fatal ? 'unavailable' : 'reconnecting'), + }) + return () => { + controller.abort() + setStatus('connecting') + } + }, [connect, enabled]) + + return enabled ? status : 'off' +} diff --git a/frontend/src/hooks/usePolling.ts b/frontend/src/hooks/usePolling.ts index 7c5cd5217..f6191cb90 100644 --- a/frontend/src/hooks/usePolling.ts +++ b/frontend/src/hooks/usePolling.ts @@ -32,16 +32,27 @@ export function usePolling( const [isLoading, setIsLoading] = useState(true) const [isRefreshing, setIsRefreshing] = useState(false) const inFlight = useRef(false) + const rerun = useRef(false) const load = useCallback(async () => { - if (inFlight.current) return + // A request made mid-fetch (e.g. a live-update signal) runs once more afterwards rather + // than being dropped, so the last change is never missed. + if (inFlight.current) { + rerun.current = true + return + } inFlight.current = true setIsRefreshing(true) try { - setData(await fetcher()) - setError(null) - } catch (err) { - setError(err instanceof Error ? err.message : 'Request failed') + do { + rerun.current = false + try { + setData(await fetcher()) + setError(null) + } catch (err) { + setError(err instanceof Error ? err.message : 'Request failed') + } + } while (rerun.current) } finally { inFlight.current = false setIsLoading(false) diff --git a/frontend/src/interfaces/queueMonitoring.interface.ts b/frontend/src/interfaces/queueMonitoring.interface.ts index c99695bae..4b58ba710 100644 --- a/frontend/src/interfaces/queueMonitoring.interface.ts +++ b/frontend/src/interfaces/queueMonitoring.interface.ts @@ -41,6 +41,8 @@ export interface QueueStats { } estimatedDrainMinutes: number | null liveWorkerPods: number + /** Active jobs no live worker reports holding: likely stalled after a pod died. */ + unheldActiveJobs: number } export interface WorkerPod { @@ -68,11 +70,39 @@ export interface RecentFailure { failedAt: string | null } +/** Individual recipients for one channel; a merge batch is one job but many messages. */ +export interface MessageChannelStats { + channel: string + /** Accepted in the last 24 hours and not yet sent or failed; scheduled sends excluded. */ + pending: number + oldestPendingAgeMs: number | null + /** Per-minute counts, oldest first; the last entry is the current, partial minute. */ + sent: number[] + failed: number[] + sentPerMinute: number + failedPerMinute: number + estimatedClearMinutes: number | null +} + +export interface ActiveBatch { + queue: string + jobId: string + notificationId: string | null + tenantId: string | null + tenantName: string | null + sent: number + failed: number + total: number + startedAt: string | null +} + export interface QueueMonitoring { generatedAt: string status: HealthStatus redis: RedisStats + messages: MessageChannelStats[] queues: QueueStats[] + activeBatches: ActiveBatch[] workers: WorkerPod[] recentFailures: RecentFailure[] } diff --git a/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx b/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx index 03f1c2bfa..b0d235b56 100644 --- a/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx +++ b/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx @@ -1,12 +1,16 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' -import { act, render, screen, within } from '@testing-library/react' +import { act, render, screen, waitFor, within } from '@testing-library/react' import userEvent from '@testing-library/user-event' -import QueueMonitoring, { REFRESH_INTERVAL_MS } from './QueueMonitoring' -import { getQueueMonitoring } from '@/api/monitoring.api' +import QueueMonitoring, { BACKSTOP_INTERVAL_MS, FALLBACK_INTERVAL_MS } from './QueueMonitoring' +import { connectQueueMonitoringStream, getQueueMonitoring } from '@/api/monitoring.api' +import type { QueueMonitoringStreamHandlers } from '@/api/monitoring.api' import UserService from '@/service/user-service' import type { QueueMonitoring as QueueMonitoringData } from '@/interfaces/queueMonitoring.interface' -vi.mock('@/api/monitoring.api', () => ({ getQueueMonitoring: vi.fn() })) +vi.mock('@/api/monitoring.api', () => ({ + getQueueMonitoring: vi.fn(), + connectQueueMonitoringStream: vi.fn(), +})) vi.mock('@/service/user-service', () => ({ default: { hasRole: vi.fn(() => true) } })) const MB = 1024 * 1024 @@ -36,6 +40,41 @@ function snapshot(overrides: Partial = {}): QueueMonitoring uptimeSeconds: 7200, version: '7.2.4', }, + messages: [ + { + channel: 'EMAIL', + pending: 63, + oldestPendingAgeMs: 50_000, + sent: series(1), + failed: series(0), + sentPerMinute: 21, + failedPerMinute: 0, + estimatedClearMinutes: 3, + }, + { + channel: 'SMS', + pending: 0, + oldestPendingAgeMs: null, + sent: series(0), + failed: series(0), + sentPerMinute: 0, + failedPerMinute: 0, + estimatedClearMinutes: 0, + }, + ], + activeBatches: [ + { + queue: 'email-delivery', + jobId: 'req-1-EMAIL-0', + notificationId: 'req-1', + tenantId: 't-1', + tenantName: 'Health Ministry', + sent: 37, + failed: 2, + total: 100, + startedAt: new Date(now - 30_000).toISOString(), + }, + ], queues: [ { name: 'email-delivery', @@ -54,6 +93,7 @@ function snapshot(overrides: Partial = {}): QueueMonitoring }, estimatedDrainMinutes: 6, liveWorkerPods: 1, + unheldActiveJobs: 0, }, ], workers: [ @@ -90,11 +130,21 @@ function snapshot(overrides: Partial = {}): QueueMonitoring } const mockedGet = vi.mocked(getQueueMonitoring) +const mockedConnect = vi.mocked(connectQueueMonitoringStream) + +/** The handlers the page passed to the most recently opened stream. */ +let stream: QueueMonitoringStreamHandlers +let streamController: AbortController describe('QueueMonitoring', () => { beforeEach(() => { vi.mocked(UserService.hasRole).mockReturnValue(true) mockedGet.mockReset() + mockedConnect.mockReset().mockImplementation((handlers) => { + stream = handlers + streamController = new AbortController() + return streamController + }) }) afterEach(() => { @@ -111,6 +161,44 @@ describe('QueueMonitoring', () => { expect(screen.getByText('noeviction')).toBeInTheDocument() }) + it('counts individual messages, not just jobs', async () => { + mockedGet.mockResolvedValue(snapshot()) + render() + + expect(await screen.findByText('63 messages pending')).toBeInTheDocument() + expect(screen.getByText('120 jobs waiting')).toBeInTheDocument() + expect(screen.getByText('Clears in ~3 min')).toBeInTheDocument() + + const table = screen.getByRole('table', { name: 'Messages' }) + const email = within(table).getByRole('row', { name: /Email/ }) + expect(within(email).getByText('63')).toBeInTheDocument() + expect(within(email).getByText('50s')).toBeInTheDocument() + expect(within(email).getByText('Sent 21/min')).toBeInTheDocument() + expect(within(email).getByText('~3 min')).toBeInTheDocument() + const sms = within(table).getByRole('row', { name: /SMS/ }) + expect(within(sms).getByText('Nothing pending')).toBeInTheDocument() + }) + + it('shows per-recipient progress for a batch that is one job', async () => { + mockedGet.mockResolvedValue(snapshot()) + render() + + const table = await screen.findByRole('table', { name: 'Batches in progress' }) + const meter = within(table).getByRole('meter', { name: 'Batch req-1-EMAIL-0 progress' }) + expect(meter).toHaveAttribute('aria-valuenow', '39') + expect(meter).toHaveAttribute('aria-valuetext', '37 of 100 sent, 2 failed') + expect(within(table).getByText('Health Ministry')).toBeInTheDocument() + }) + + it('says so when no batch is sending', async () => { + mockedGet.mockResolvedValue(snapshot({ activeBatches: [] })) + render() + + expect( + (await screen.findAllByText('No batches are sending right now.')).length, + ).toBeGreaterThan(0) + }) + it('shows each queue with its backlog, rates and drain estimate', async () => { mockedGet.mockResolvedValue(snapshot()) render() @@ -214,19 +302,77 @@ describe('QueueMonitoring', () => { expect(await screen.findByText('Queue health changed to Warning')).toBeInTheDocument() }) - it('polls on an interval and stops when auto-refresh is turned off', async () => { - vi.useFakeTimers({ shouldAdvanceTime: true }) - const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }) + it('refetches when the stream signals a change', async () => { mockedGet.mockResolvedValue(snapshot()) render() await screen.findByRole('table', { name: 'Queues' }) + act(() => stream.onOpen()) + expect(screen.getByText('Live')).toBeInTheDocument() expect(mockedGet).toHaveBeenCalledTimes(1) - await act(() => vi.advanceTimersByTimeAsync(REFRESH_INTERVAL_MS)) + mockedGet.mockResolvedValue(snapshot({ status: 'warning' })) + act(() => stream.onChange()) + + expect(await screen.findByText('Queue health changed to Warning')).toBeInTheDocument() + expect(mockedGet).toHaveBeenCalledTimes(2) + }) + + it('polls slowly while live and falls back to faster polling when the stream drops', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + mockedGet.mockResolvedValue(snapshot()) + render() + await screen.findByRole('table', { name: 'Queues' }) + act(() => stream.onOpen()) + + await act(() => vi.advanceTimersByTimeAsync(FALLBACK_INTERVAL_MS)) + expect(mockedGet).toHaveBeenCalledTimes(1) + await act(() => vi.advanceTimersByTimeAsync(BACKSTOP_INTERVAL_MS - FALLBACK_INTERVAL_MS)) expect(mockedGet).toHaveBeenCalledTimes(2) - await user.click(screen.getByRole('switch', { name: /Auto-refresh/ })) - await act(() => vi.advanceTimersByTimeAsync(REFRESH_INTERVAL_MS * 3)) + act(() => stream.onError(false)) + expect(screen.getByText(/Reconnecting/)).toBeInTheDocument() + await act(() => vi.advanceTimersByTimeAsync(FALLBACK_INTERVAL_MS)) + expect(mockedGet).toHaveBeenCalledTimes(3) + }) + + it('catches up once when the stream reconnects', async () => { + mockedGet.mockResolvedValue(snapshot()) + render() + await screen.findByRole('table', { name: 'Queues' }) + act(() => stream.onOpen()) + act(() => stream.onError(false)) + + act(() => stream.onOpen()) + + await waitFor(() => expect(mockedGet).toHaveBeenCalledTimes(2)) + }) + + it('says when live updates are refused, and keeps polling', async () => { + mockedGet.mockResolvedValue(snapshot()) + render() + await screen.findByRole('table', { name: 'Queues' }) + + act(() => stream.onError(true)) + + expect(screen.getByText(/Live updates unavailable/)).toBeInTheDocument() + }) + + it('closes the stream and stops polling when live updates are turned off', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }) + mockedGet.mockResolvedValue(snapshot()) + render() + await screen.findByRole('table', { name: 'Queues' }) + const opened = streamController + + await user.click(screen.getByRole('switch', { name: 'Live updates' })) + + expect(opened.signal.aborted).toBe(true) + expect(screen.getByText('Paused')).toBeInTheDocument() + await act(() => vi.advanceTimersByTimeAsync(BACKSTOP_INTERVAL_MS * 2)) + expect(mockedGet).toHaveBeenCalledTimes(1) + + await user.click(screen.getByRole('button', { name: 'Refresh' })) expect(mockedGet).toHaveBeenCalledTimes(2) }) @@ -236,5 +382,6 @@ describe('QueueMonitoring', () => { expect(screen.queryByRole('heading', { name: 'Monitoring' })).not.toBeInTheDocument() expect(mockedGet).not.toHaveBeenCalled() + expect(mockedConnect).not.toHaveBeenCalled() }) }) diff --git a/frontend/src/pages/admin/monitoring/QueueMonitoring.tsx b/frontend/src/pages/admin/monitoring/QueueMonitoring.tsx index 8e63b66c2..9a910c7f5 100644 --- a/frontend/src/pages/admin/monitoring/QueueMonitoring.tsx +++ b/frontend/src/pages/admin/monitoring/QueueMonitoring.tsx @@ -1,4 +1,4 @@ -import { useState } from 'react' +import { useCallback, useEffect, useRef, useState } from 'react' import type { FC } from 'react' import { Button, Switch } from '@bcgov/design-system-react-components' import PageHeading from '@/components/PageHeading' @@ -8,16 +8,31 @@ import NotAuthorized from '@/components/NotAuthorized' import UserService from '@/service/user-service' import { SsoRole } from '@/enum/sso-role.enum' import { usePolling } from '@/hooks/usePolling' -import { getQueueMonitoring } from '@/api/monitoring.api' +import { useLiveUpdates } from '@/hooks/useLiveUpdates' +import type { LiveStatus } from '@/hooks/useLiveUpdates' +import { connectQueueMonitoringStream, getQueueMonitoring } from '@/api/monitoring.api' import type { HealthStatus } from '@/interfaces/queueMonitoring.interface' -import { HEALTH_LABELS, formatDuration, formatQueueName } from '@/utils/monitoring' +import { HEALTH_LABELS, formatQueueName } from '@/utils/monitoring' +import MessagesSection from './sections/MessagesSection' +import ActiveBatchesSection from './sections/ActiveBatchesSection' import RedisSection from './sections/RedisSection' import QueuesSection from './sections/QueuesSection' import WorkersSection from './sections/WorkersSection' import RecentFailuresSection from './sections/RecentFailuresSection' import '@/scss/components/queue-monitoring.scss' -export const REFRESH_INTERVAL_MS = 15_000 +/** Polled while the live stream is up: catches time-based changes (ages, stale heartbeats). */ +export const BACKSTOP_INTERVAL_MS = 60_000 +/** Polled while the stream is down or refused. */ +export const FALLBACK_INTERVAL_MS = 15_000 + +const LIVE_STATUS_TEXT: Record = { + off: 'Paused', + connecting: 'Connecting…', + live: 'Live', + reconnecting: `Reconnecting… (updating every ${FALLBACK_INTERVAL_MS / 1000}s)`, + unavailable: `Live updates unavailable (updating every ${FALLBACK_INTERVAL_MS / 1000}s)`, +} /** * Admin view of the Bull queues, the pods working them and the Redis instance they live in. @@ -25,12 +40,23 @@ export const REFRESH_INTERVAL_MS = 15_000 */ const QueueMonitoring: FC = () => { const isAdmin = UserService.hasRole(SsoRole.NOTIFY_ADMIN) - const [autoRefresh, setAutoRefresh] = useState(true) + const [liveUpdates, setLiveUpdates] = useState(true) + + // The stream only says "something changed"; the snapshot still comes from the endpoint. + const refreshRef = useRef<() => void>(() => {}) + const onChange = useCallback(() => { + refreshRef.current() + }, []) + const liveStatus = useLiveUpdates(connectQueueMonitoringStream, isAdmin && liveUpdates, onChange) + const { data, error, isLoading, isRefreshing, refresh } = usePolling(getQueueMonitoring, { - intervalMs: REFRESH_INTERVAL_MS, - paused: !autoRefresh, + intervalMs: liveStatus === 'live' ? BACKSTOP_INTERVAL_MS : FALLBACK_INTERVAL_MS, + paused: !liveUpdates, skip: !isAdmin, }) + useEffect(() => { + refreshRef.current = refresh + }, [refresh]) // Announce only changes in overall health, not every 15-second refresh. const [announcement, setAnnouncement] = useState('') @@ -47,10 +73,12 @@ const QueueMonitoring: FC = () => { } const now = data ? new Date(data.generatedAt).getTime() : 0 - const backlog = data?.queues.reduce((sum, q) => sum + q.counts.waiting + q.counts.paused, 0) ?? 0 - const slowest = data?.queues.reduce((longest, q) => { - if (q.estimatedDrainMinutes === null || q.estimatedDrainMinutes === 0) return longest - return Math.max(longest ?? 0, q.estimatedDrainMinutes) + const pendingMessages = data?.messages.reduce((sum, m) => sum + m.pending, 0) ?? 0 + const jobsWaiting = + data?.queues.reduce((sum, q) => sum + q.counts.waiting + q.counts.paused, 0) ?? 0 + const slowest = data?.messages.reduce((longest, m) => { + if (m.estimatedClearMinutes === null || m.estimatedClearMinutes === 0) return longest + return Math.max(longest ?? 0, m.estimatedClearMinutes) }, null) const notDraining = data?.queues.filter( (q) => q.estimatedDrainMinutes === null && q.counts.waiting + q.counts.paused > 0, @@ -60,7 +88,7 @@ const QueueMonitoring: FC = () => {
@@ -72,7 +100,8 @@ const QueueMonitoring: FC = () => { {data && ( <> - {backlog.toLocaleString()} waiting + {pendingMessages.toLocaleString()} messages pending + {jobsWaiting.toLocaleString()} jobs waiting {notDraining && notDraining.length > 0 ? ( Not draining: {notDraining.map((q) => formatQueueName(q.name)).join(', ')} @@ -85,8 +114,11 @@ const QueueMonitoring: FC = () => {
{data && Updated {new Date(data.generatedAt).toLocaleTimeString()}} - - Auto-refresh every {formatDuration(REFRESH_INTERVAL_MS)} + + {LIVE_STATUS_TEXT[liveStatus]} + + + Live updates
+ {warnings.length > 0 && ( + +
    + {warnings.map((warning) => ( +
  • {warning}
    • + ))} +
+ {tab !== 'system' && ( + + )} + + )} + {error && ( {error} @@ -141,12 +171,25 @@ const QueueMonitoring: FC = () => { {data && ( <> - - - - - - + + + {tab === 'overview' && ( + <> + + + + + )} + + {tab === 'failures' && } + + {tab === 'system' && ( + <> + + + + + )} )}
diff --git a/frontend/src/pages/admin/monitoring/sections/MessagesSection.tsx b/frontend/src/pages/admin/monitoring/sections/MessagesSection.tsx index 4ff81c4a1..38d27728a 100644 --- a/frontend/src/pages/admin/monitoring/sections/MessagesSection.tsx +++ b/frontend/src/pages/admin/monitoring/sections/MessagesSection.tsx @@ -27,21 +27,19 @@ const columns: TableColumn[] = [ render: (_, row) => row.oldestPendingAgeMs === null ? '—' : formatDuration(row.oldestPendingAgeMs), }, + { key: 'sentPerMinute', label: 'Sending', render: (_, row) => formatRate(row.sentPerMinute) }, + { key: 'failedPerMinute', label: 'Failing', render: (_, row) => formatRate(row.failedPerMinute) }, { - key: 'sentPerMinute', - label: 'Sent / failed', + key: 'sent', + label: 'Last 60 min', render: (_, row) => ( -
- Sent {formatRate(row.sentPerMinute)} - Failed {formatRate(row.failedPerMinute)} - -
+ ), }, { key: 'estimatedClearMinutes', label: 'Time to clear', render: (_, row) => clearText(row) }, @@ -54,22 +52,21 @@ interface MessagesSectionProps { /** Individual emails and texts, as opposed to the jobs that carry them. */ const MessagesSection: FC = ({ messages }) => (
- + row.channel} - label="Messages" + label="Right now" emptyMessage="No messages in the last 24 hours." multiline />

- One row per recipient. Pending counts messages accepted in the last 24 hours that have not - been sent or failed yet; scheduled sends are not counted until they are due. + Rates average the last 5 minutes.

) diff --git a/frontend/src/pages/admin/monitoring/sections/OverviewCards.tsx b/frontend/src/pages/admin/monitoring/sections/OverviewCards.tsx new file mode 100644 index 000000000..d56eb08f6 --- /dev/null +++ b/frontend/src/pages/admin/monitoring/sections/OverviewCards.tsx @@ -0,0 +1,94 @@ +import type { FC } from 'react' +import SummaryCard from '@/components/SummaryCard' +import MeterBar from '@/components/MeterBar' +import { StatusBadge } from '@/components/StatusBadge' +import type { + MessageChannelStats, + MonitoringOverview, + RedisStats, +} from '@/interfaces/queueMonitoring.interface' +import { HEALTH_LABELS, formatBytes, formatDuration } from '@/utils/monitoring' + +interface OverviewCardsProps { + overview: MonitoringOverview + messages: MessageChannelStats[] + redis: RedisStats +} + +/** The four headline numbers: how fast, how much, how far behind, and how close to full. */ +const OverviewCards: FC = ({ overview, messages, redis }) => { + const pending = messages.reduce((sum, m) => sum + m.pending, 0) + const oldest = messages.reduce( + (max, m) => (m.oldestPendingAgeMs === null ? max : Math.max(max ?? 0, m.oldestPendingAgeMs)), + null, + ) + const clears = messages.reduce((max, m) => { + if (m.estimatedClearMinutes === null || m.estimatedClearMinutes === 0) return max + return Math.max(max ?? 0, m.estimatedClearMinutes) + }, null) + const stuck = messages.some((m) => m.pending > 0 && m.estimatedClearMinutes === null) + const memoryText = + redis.maxMemoryBytes === null + ? `${formatBytes(redis.usedMemoryBytes)} used, no cap set` + : `${formatBytes(redis.usedMemoryBytes)} of ${formatBytes(redis.maxMemoryBytes)}` + + return ( +
+ + {overview.deliveryTime ? ( + <> +

95% within {formatDuration(overview.deliveryTime.p95Ms)}

+

+ Accepted to handed to the provider, {overview.deliveryTime.messages.toLocaleString()}{' '} + messages in the last hour +

+ + ) : ( +

Nothing sent in the last hour

+ )} + + + +

{overview.sentPerMinute.toLocaleString()}/min on average

+

{overview.requestsReceived.toLocaleString()} requests received

+

+ {overview.messagesFailed.toLocaleString()} failed ({overview.failurePercent}%) +

+ + + +

Oldest {oldest === null ? '—' : formatDuration(oldest)}

+

+ {pending === 0 + ? 'Nothing waiting' + : stuck + ? 'Not sending' + : `Clears in ~${(clears ?? 0).toLocaleString()} min`} +

+ + + + {redis.usedMemoryPercent !== null && ( + + )} +

{memoryText}

+ + +
+ ) +} + +export default OverviewCards diff --git a/frontend/src/routes/admin/monitoring.tsx b/frontend/src/routes/admin/monitoring.tsx index a9838eee9..512bad228 100644 --- a/frontend/src/routes/admin/monitoring.tsx +++ b/frontend/src/routes/admin/monitoring.tsx @@ -1,5 +1,6 @@ import { createFileRoute, redirect } from '@tanstack/react-router' -import QueueMonitoring from '@/pages/admin/monitoring/QueueMonitoring' +import QueueMonitoring, { MONITORING_TABS } from '@/pages/admin/monitoring/QueueMonitoring' +import type { MonitoringTab } from '@/pages/admin/monitoring/QueueMonitoring' import UserService from '@/service/user-service' import { SsoRole } from '@/enum/sso-role.enum' @@ -9,9 +10,23 @@ export const Route = createFileRoute('/admin/monitoring')({ throw redirect({ to: '/not-authorized' }) } }, + // The tab is in the URL so a refresh or a shared link opens the same view. + validateSearch: (search: Record): { tab?: MonitoringTab } => + typeof search.tab === 'string' && MONITORING_TABS.includes(search.tab as MonitoringTab) + ? { tab: search.tab as MonitoringTab } + : {}, component: MonitoringPage, }) function MonitoringPage() { - return + const { tab = 'overview' } = Route.useSearch() + const navigate = Route.useNavigate() + return ( + + navigate({ search: next === 'overview' ? {} : { tab: next }, replace: true }) + } + /> + ) } diff --git a/frontend/src/routes/waf-routes.spec.ts b/frontend/src/routes/waf-routes.spec.ts new file mode 100644 index 000000000..7fd0b2316 --- /dev/null +++ b/frontend/src/routes/waf-routes.spec.ts @@ -0,0 +1,54 @@ +import { describe, it, expect } from 'vitest' +import { readFileSync, readdirSync, statSync } from 'fs' +import { join, resolve } from 'path' + +/** + * Caddy runs the Coraza WAF in front of the SPA, so a refresh or direct link to any page is a + * real request that must get past the URI rules before index.html is served. A route whose + * path matches one is reachable by clicking but 403s on refresh - which is how /admin/* broke. + */ +const frontendRoot = resolve(__dirname, '../..') +const coraza = readFileSync(join(frontendRoot, 'coraza.conf'), 'utf8') + +/** The `@rx` pattern of a REQUEST_URI rule, by id. */ +function uriRule(id: number): RegExp { + // Anchored to real rule lines: the comments above rule 1004 quote an example rule with its id. + const rules = coraza.matchAll(/^SecRule REQUEST_URI\S*\s+"@rx (.+?)"\s*\\\n\s*"id:(\d+),/gm) + const pattern = [...rules].find((match) => Number(match[2]) === id)?.[1] + if (!pattern) throw new Error(`REQUEST_URI rule ${id} not found in coraza.conf`) + // Coraza patterns are RE2; these use no syntax JavaScript reads differently. + return new RegExp(pattern.replace(/^\(\?i\)/, ''), pattern.startsWith('(?i)') ? 'i' : '') +} + +function routeFiles(dir: string): string[] { + return readdirSync(dir).flatMap((name) => { + const path = join(dir, name) + if (statSync(path).isDirectory()) return routeFiles(path) + return name.endsWith('.tsx') && !name.endsWith('.spec.tsx') ? [path] : [] + }) +} + +const routePaths = routeFiles(join(frontendRoot, 'src/routes')) + .map((file) => readFileSync(file, 'utf8').match(/createFileRoute\('([^']+)'\)/)?.[1]) + .filter((path): path is string => !!path) + // Route params become a plausible value, as a real URL would have. + .map((path) => path.replace(/\$[A-Za-z]+/g, '0d2c6a3e-1f4b-4c8d-9e7a-5b6c7d8e9f01')) + +describe('WAF URI rules', () => { + it('finds the app routes to check', () => { + expect(routePaths).toContain('/admin/monitoring') + expect(routePaths.length).toBeGreaterThan(10) + }) + + it.each([1004, 1007])('rule %i lets every app route through on refresh', (id) => { + const rule = uriRule(id) + expect(routePaths.filter((path) => rule.test(path))).toEqual([]) + }) + + it('still blocks the probe paths rule 1004 exists for', () => { + const rule = uriRule(1004) + for (const probe of ['/.env', '/.git/config', '/wp-admin', '/config']) { + expect(rule.test(probe)).toBe(true) + } + }) +}) diff --git a/frontend/src/scss/components/queue-monitoring.scss b/frontend/src/scss/components/queue-monitoring.scss index 8c973f0cb..69e3dfbdd 100644 --- a/frontend/src/scss/components/queue-monitoring.scss +++ b/frontend/src/scss/components/queue-monitoring.scss @@ -94,3 +94,15 @@ background-color: $warning-color; } } + +.queue-monitoring__warnings { + margin: 0 0 $spacing-xs; + padding-left: $spacing-lg; +} + +// A text-styled button: it switches tabs rather than navigating, so it is not a link. +.queue-monitoring__link { + @include button-reset; + color: $border-active; + text-decoration: underline; +} diff --git a/frontend/src/scss/components/summary-card.scss b/frontend/src/scss/components/summary-card.scss new file mode 100644 index 000000000..35b53064c --- /dev/null +++ b/frontend/src/scss/components/summary-card.scss @@ -0,0 +1,45 @@ +@import '../variables'; + +.summary-card-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); + gap: $spacing-md; + width: 100%; +} + +.summary-card { + display: flex; + flex-direction: column; + gap: $spacing-sm; + padding: $spacing-md $spacing-lg; + border: 1px solid $border-default; + border-radius: 4px; + background-color: $white; + + .summary-card__title { + margin: 0; + font-size: $font-size-sm; + font-weight: 700; + color: $text-secondary; + } + + .summary-card__value { + margin: 0; + font-size: 1.75rem; + font-weight: 700; + line-height: 1.2; + color: $dark-text; + } + + .summary-card__body { + display: flex; + flex-direction: column; + gap: $spacing-xs; + font-size: $font-size-sm; + color: $text-secondary; + + p { + margin: 0; + } + } +} diff --git a/frontend/src/scss/components/tab-bar.scss b/frontend/src/scss/components/tab-bar.scss new file mode 100644 index 000000000..9d423d5ed --- /dev/null +++ b/frontend/src/scss/components/tab-bar.scss @@ -0,0 +1,15 @@ +@import '../variables'; + +// Same overrides as .events__tabs, so every tab bar in the app looks alike. +.tab-bar { + .bcds-react-aria-ToggleButtonGroup { + width: fit-content; + flex-wrap: wrap; + gap: $spacing-sm; + } + + // The DS ToggleButton hardcodes 2px 8px regardless of `size`. + .bcds-react-aria-ToggleButton { + padding: 6px 12px; + } +} From 879feb2b5962e123562db7b72699f165beae55ca Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Thu, 1 Oct 2026 19:03:50 -0700 Subject: [PATCH 05/23] Prevent load-test auto-bind key hijacking Harden `autoBindApiKeyForLoadTest` so it no longer rebinds an existing credential from another tenant to the load-test tenant. The service now logs and throws a `ConflictException` when a key is already owned by a different tenant, while still allowing idempotent reuse of keys already bound to the load-test tenant. Added focused unit tests covering new bind, idempotent behavior, cross-tenant rejection, and environment guardrails in test/prod namespaces. --- .../src/api/api-keys/api-keys.service.spec.ts | 68 ++++++++++++++++++- backend/src/api/api-keys/api-keys.service.ts | 14 ++-- 2 files changed, 77 insertions(+), 5 deletions(-) diff --git a/backend/src/api/api-keys/api-keys.service.spec.ts b/backend/src/api/api-keys/api-keys.service.spec.ts index 4fced1b4f..42f887f06 100644 --- a/backend/src/api/api-keys/api-keys.service.spec.ts +++ b/backend/src/api/api-keys/api-keys.service.spec.ts @@ -1,5 +1,5 @@ import { In, Repository } from 'typeorm' -import { vi } from 'vitest' +import { vi, afterAll } from 'vitest' import { Tenant } from '../admin/tenants/entities/tenant.entity' import { ApiKeyConsumer } from './entities/api-key-consumer.entity' import { ApiKeyLimit } from './entities/api-key-limit.entity' @@ -112,3 +112,69 @@ describe('ApiKeysService.ensureDefaults', () => { ) }) }) + +describe('ApiKeysService.autoBindApiKeyForLoadTest', () => { + const loadtestTenant = { id: 'loadtest-tenant-id', slug: 'loadtest-tenant' } + const apiKeyConsumerRepository = { + findOne: vi.fn(), + create: vi.fn((mapping: Partial) => mapping), + save: vi.fn((mapping: Partial) => Promise.resolve({ ...mapping, id: 'c-1' })), + } + const tenantRepository = { findOne: vi.fn(), create: vi.fn(), save: vi.fn() } + const service = new ApiKeysService( + apiKeyConsumerRepository as unknown as Repository, + {} as Repository, + {} as Repository, + tenantRepository as unknown as Repository, + ) + const originalNamespace = process.env.NAMESPACE + + beforeEach(() => { + vi.clearAllMocks() + process.env.NAMESPACE = 'fe8c5-dev' + tenantRepository.findOne.mockResolvedValue(loadtestTenant) + }) + + afterAll(() => { + process.env.NAMESPACE = originalNamespace + }) + + it('binds a new key to the load-test tenant', async () => { + apiKeyConsumerRepository.findOne.mockResolvedValue(null) + + await service.autoBindApiKeyForLoadTest('cred-new', 'consumer-1') + + expect(apiKeyConsumerRepository.save).toHaveBeenCalledWith( + expect.objectContaining({ credentialIdentifier: 'cred-new', tenantId: loadtestTenant.id }), + ) + }) + + it('leaves the load-test key alone on a later run', async () => { + const existing = { credentialIdentifier: 'cred-lt', tenantId: loadtestTenant.id } + apiKeyConsumerRepository.findOne.mockResolvedValue(existing) + + await expect(service.autoBindApiKeyForLoadTest('cred-lt', '')).resolves.toBe(existing) + expect(apiKeyConsumerRepository.save).not.toHaveBeenCalled() + }) + + it("refuses to move a real tenant's key onto the load-test tenant", async () => { + apiKeyConsumerRepository.findOne.mockResolvedValue({ + credentialIdentifier: 'cred-real', + tenantId: 'real-tenant-id', + }) + + await expect(service.autoBindApiKeyForLoadTest('cred-real', '')).rejects.toThrow( + /belongs to another tenant/, + ) + expect(apiKeyConsumerRepository.save).not.toHaveBeenCalled() + }) + + it.each(['fe8c5-test', 'fe8c5-prod'])('refuses to run at all in %s', async (namespace) => { + process.env.NAMESPACE = namespace + + await expect(service.autoBindApiKeyForLoadTest('cred-new', '')).rejects.toThrow( + /disabled in this environment/, + ) + expect(apiKeyConsumerRepository.findOne).not.toHaveBeenCalled() + }) +}) diff --git a/backend/src/api/api-keys/api-keys.service.ts b/backend/src/api/api-keys/api-keys.service.ts index 5651c75ea..84f935f8b 100644 --- a/backend/src/api/api-keys/api-keys.service.ts +++ b/backend/src/api/api-keys/api-keys.service.ts @@ -1,4 +1,4 @@ -import { Injectable, Logger, ForbiddenException } from '@nestjs/common' +import { Injectable, Logger, ForbiddenException, ConflictException } from '@nestjs/common' import { InjectRepository } from '@nestjs/typeorm' import { In, Repository } from 'typeorm' import { ApiKeyConsumer } from './entities/api-key-consumer.entity' @@ -158,10 +158,16 @@ export class ApiKeysService { where: { credentialIdentifier }, }) if (existing) { + // Never move a key off a real tenant: that would let anyone holding a tenant's DEV key + // re-point it at the load-test tenant while auto-bind is on. Only a new key, or the + // load-test key on a later run, gets through. if (existing.tenantId !== tenant.id) { - existing.tenantId = tenant.id - existing.updatedAt = now - await this.apiKeyConsumerRepository.save(existing) + this.logger.error( + `[LOADTEST] Refusing to re-bind credential ${credentialIdentifier}: bound to tenant ${existing.tenantId}`, + ) + throw new ConflictException( + 'This API key belongs to another tenant and cannot be used for load testing. Use a dedicated load-test key.', + ) } return existing } From d01afdb2303b97b6c5e4303f97eabb8635212b5d Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Fri, 2 Oct 2026 07:55:51 -0700 Subject: [PATCH 06/23] Fix queue heartbeat and live refresh behavior Update worker heartbeat tracking to use per-job IDs instead of a simple counter so local stalled-job `failed` events for jobs from other pods do not corrupt active/failed metrics. Added backend tests for unknown finished jobs and cross-pod stalled-job scenarios. In Queue Monitoring, hide the manual Refresh button while the live stream is connected (it already auto-refetches) and show it again when the stream drops. Added a frontend test to cover this visibility toggle. --- backend/src/queue/worker-heartbeat.spec.ts | 35 ++++++++++++++----- backend/src/queue/worker-heartbeat.ts | 25 +++++++------ .../admin/monitoring/QueueMonitoring.spec.tsx | 12 +++++++ .../admin/monitoring/QueueMonitoring.tsx | 9 +++-- 4 files changed, 58 insertions(+), 23 deletions(-) diff --git a/backend/src/queue/worker-heartbeat.spec.ts b/backend/src/queue/worker-heartbeat.spec.ts index 2b508d5c6..dde24b1b0 100644 --- a/backend/src/queue/worker-heartbeat.spec.ts +++ b/backend/src/queue/worker-heartbeat.spec.ts @@ -17,6 +17,8 @@ function fakeRedis() { return { redis: { multi: vi.fn(() => multi) } as unknown as Redis, multi } } +const job = (id: string | number) => ({ id }) as unknown as Bull.Job + function fakeQueue(name: string) { return Object.assign(new EventEmitter(), { name }) as unknown as Bull.Queue } @@ -28,11 +30,11 @@ describe('WorkerHeartbeat', () => { const queue = fakeQueue('email-delivery') heartbeat.track(queue, 2) - queue.emit('active') - queue.emit('active') - queue.emit('completed') - queue.emit('active') - queue.emit('failed') + queue.emit('active', job(1)) + queue.emit('active', job(2)) + queue.emit('completed', job(1)) + queue.emit('active', job(3)) + queue.emit('failed', job(2)) await heartbeat.beat() const [key, json, mode, ttl] = multi.set.mock.calls[0] @@ -53,18 +55,33 @@ describe('WorkerHeartbeat', () => { expect(multi.zadd).toHaveBeenCalledWith(WORKER_INDEX_KEY, payload.heartbeatAt, payload.podId) }) - it('never lets the active count go negative', async () => { + it('ignores a job this pod never ran finishing', async () => { const { redis, multi } = fakeRedis() const heartbeat = new WorkerHeartbeat(redis) const queue = fakeQueue('sms-delivery') heartbeat.track(queue, 1) - // A job that was already active before this pod started tracking. - queue.emit('completed') + queue.emit('completed', job(9)) + await heartbeat.beat() + + const payload = JSON.parse(multi.set.mock.calls[0][1]) as WorkerHeartbeatPayload + expect(payload.queues[0]).toMatchObject({ active: 0, completed: 0 }) + }) + + it("keeps a running job busy when the stalled-job check fails another pod's job", async () => { + // Bull's stalled-job check runs on every worker and emits a local `failed` for a job it + // gives up on, even one a dead pod was running. That must not end this pod's own job. + const { redis, multi } = fakeRedis() + const heartbeat = new WorkerHeartbeat(redis) + const queue = fakeQueue('email-delivery') + heartbeat.track(queue, 2) + + queue.emit('active', job('mine')) + queue.emit('failed', job('abandoned-elsewhere')) await heartbeat.beat() const payload = JSON.parse(multi.set.mock.calls[0][1]) as WorkerHeartbeatPayload - expect(payload.queues[0].active).toBe(0) + expect(payload.queues[0]).toMatchObject({ active: 1, failed: 0 }) }) it('swallows Redis errors so a heartbeat never crashes the pod', async () => { diff --git a/backend/src/queue/worker-heartbeat.ts b/backend/src/queue/worker-heartbeat.ts index 9a4692955..8766f8299 100644 --- a/backend/src/queue/worker-heartbeat.ts +++ b/backend/src/queue/worker-heartbeat.ts @@ -65,19 +65,22 @@ export class WorkerHeartbeat { } this.states.set(queue.name, state) - queue.on('active', () => { - state.active++ - }) - queue.on('completed', () => { - state.active = Math.max(0, state.active - 1) - state.completed++ - state.lastFinishedAt = Date.now() - }) - queue.on('failed', () => { - state.active = Math.max(0, state.active - 1) - state.failed++ + // Keyed by job id, not counted: Bull's stalled-job check emits a local `failed` for jobs + // another pod abandoned, which a plain counter would take as one of this pod's jobs ending. + const running = new Set() + const finish = (job: Bull.Job | undefined, outcome: 'completed' | 'failed') => { + if (!job || !running.delete(job.id)) return + state.active = running.size + state[outcome]++ state.lastFinishedAt = Date.now() + } + + queue.on('active', (job: Bull.Job) => { + running.add(job.id) + state.active = running.size }) + queue.on('completed', (job: Bull.Job) => finish(job, 'completed')) + queue.on('failed', (job: Bull.Job) => finish(job, 'failed')) } start(): void { diff --git a/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx b/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx index 15976c8a3..c5ce1a285 100644 --- a/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx +++ b/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx @@ -393,6 +393,18 @@ describe('QueueMonitoring', () => { expect(mockedGet).toHaveBeenCalledTimes(2) }) + it('hides Refresh while the stream is live and brings it back when it drops', async () => { + mockedGet.mockResolvedValue(snapshot()) + renderPage() + await screen.findByRole('table', { name: 'Right now' }) + + act(() => stream.onOpen()) + expect(screen.queryByRole('button', { name: 'Refresh' })).not.toBeInTheDocument() + + act(() => stream.onError(false)) + expect(screen.getByRole('button', { name: 'Refresh' })).toBeInTheDocument() + }) + it('polls slowly while live and falls back to faster polling when the stream drops', async () => { vi.useFakeTimers({ shouldAdvanceTime: true }) mockedGet.mockResolvedValue(snapshot()) diff --git a/frontend/src/pages/admin/monitoring/QueueMonitoring.tsx b/frontend/src/pages/admin/monitoring/QueueMonitoring.tsx index 2fb2fb869..5f6403b45 100644 --- a/frontend/src/pages/admin/monitoring/QueueMonitoring.tsx +++ b/frontend/src/pages/admin/monitoring/QueueMonitoring.tsx @@ -131,9 +131,12 @@ const QueueMonitoring: FC = ({ tab, onTabChange }) => { Live updates - + {/* A live stream already refetches on every change; manual refresh is for when it isn't. */} + {liveStatus !== 'live' && ( + + )}
From a70e5e36a51a4762bb789162b3cfec1b0e5198d4 Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Fri, 2 Oct 2026 08:05:08 -0700 Subject: [PATCH 07/23] Use active-minute send rate in monitoring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the monitoring overview’s `sentPerMinute` with `sendingRate` (`perMinute` + `activeMinutes`) on both backend and frontend DTOs/interfaces. The new backend calculation averages only minutes with sends and ignores the current partial minute unless it is the only active minute, preventing idle time from diluting throughput; UI copy and specs were updated to show the new metric and cover null/aggregation behavior. --- .../monitoring/monitoring.service.spec.ts | 38 ++++++++++- .../admin/monitoring/monitoring.service.ts | 68 ++++++++++++++++--- .../schemas/queue-monitoring.dto.ts | 7 +- .../interfaces/queueMonitoring.interface.ts | 7 +- .../admin/monitoring/QueueMonitoring.spec.tsx | 5 +- .../monitoring/sections/OverviewCards.tsx | 6 +- 6 files changed, 115 insertions(+), 16 deletions(-) diff --git a/backend/src/api/admin/monitoring/monitoring.service.spec.ts b/backend/src/api/admin/monitoring/monitoring.service.spec.ts index fc074371c..c7dd05d2b 100644 --- a/backend/src/api/admin/monitoring/monitoring.service.spec.ts +++ b/backend/src/api/admin/monitoring/monitoring.service.spec.ts @@ -8,6 +8,7 @@ import { averageOfWholeMinutes, buildOverview, buildRedisStats, + sendingRateFrom, countUnheldActiveJobs, evaluateQueue, maskPersonalData, @@ -199,6 +200,19 @@ describe('countUnheldActiveJobs', () => { }) }) +describe('sendingRateFrom', () => { + it('scales busy sending up to a per-minute rate', () => { + // 20 messages over 30 seconds of sending is a 40/min pace. + expect(sendingRateFrom(20, 30)).toEqual({ perMinute: 40, messages: 20, busySeconds: 30 }) + }) + + it('is null when there is too little sending to extrapolate from', () => { + expect(sendingRateFrom(9, 60)).toBeNull() + expect(sendingRateFrom(50, 4)).toBeNull() + expect(sendingRateFrom(0, 0)).toBeNull() + }) +}) + describe('buildOverview', () => { const channel = (sent: number, failed: number) => ({ channel: 'EMAIL', @@ -221,20 +235,21 @@ describe('buildOverview', () => { [channel(1200, 6), { ...channel(40, 2), channel: 'SMS' }], [evaluateQueue(ingestion)], null, + null, ) expect(overview).toMatchObject({ windowMinutes: 60, messagesSent: 1240, messagesFailed: 8, - sentPerMinute: 20.7, + sendingRate: null, failurePercent: 0.6, requestsReceived: 38, }) }) it('reports a zero failure rate when nothing finished', () => { - expect(buildOverview([channel(0, 0)], [], null).failurePercent).toBe(0) + expect(buildOverview([channel(0, 0)], [], null, null).failurePercent).toBe(0) }) }) @@ -304,9 +319,12 @@ describe('MonitoringService', () => { pendingRows: object[] = [], finishedRows: object[] = [], deliveryRow: object = { median: null, p95: null, messages: '0' }, + rateRow: object = { busy_messages: '0', busy_seconds: '0' }, ) { const results: unknown[] = [pendingRows, finishedRows, deliveryRow] return { + metadata: { tablePath: 'notify.notification_request_detail' }, + query: vi.fn().mockResolvedValue([rateRow]), createQueryBuilder: vi.fn(() => { const rows = results.shift() ?? [] const builder: Record = {} @@ -516,6 +534,22 @@ describe('MonitoringService', () => { expect(overview.deliveryTime).toEqual({ medianMs: 42000, p95Ms: 130000, messages: 1240 }) }) + it('reports the sustained sending speed from busy time', async () => { + const service = new MonitoringService( + fakeRedis([livePod]), + null, + fakeQueue('email-delivery'), + null, + null, + tenantRepository, + fakeDetailRepository([], [], undefined, { busy_messages: '28', busy_seconds: '46.5' }), + ) + + const { overview } = await service.getQueueMonitoring() + + expect(overview.sendingRate).toEqual({ perMinute: 36.1, messages: 28, busySeconds: 47 }) + }) + it('reports no delivery time when nothing was sent in the last hour', async () => { const service = new MonitoringService( fakeRedis([livePod]), diff --git a/backend/src/api/admin/monitoring/monitoring.service.ts b/backend/src/api/admin/monitoring/monitoring.service.ts index 936ccea1f..51b72b411 100644 --- a/backend/src/api/admin/monitoring/monitoring.service.ts +++ b/backend/src/api/admin/monitoring/monitoring.service.ts @@ -48,6 +48,10 @@ const UNHELD_GRACE_MS = HEARTBEAT_INTERVAL_MS * 2 const IN_FLIGHT_DETAIL_STATUSES = ['pending', 'queued', 'processing', 'sending'] /** Older in-flight rows are abandoned, not backlog, and would pin "oldest pending" forever. */ const PENDING_LOOKBACK_MS = 24 * 60 * 60 * 1000 +/** A longer pause between consecutive sends is idle time, not part of how fast sending runs. */ +const IDLE_GAP_SECONDS = 30 +const MIN_RATE_MESSAGES = 10 +const MIN_RATE_BUSY_SECONDS = 5 const DEFAULT_MESSAGE_CHANNELS = [NotificationChannel.EMAIL, NotificationChannel.SMS] const RECENT_FAILURES_LIMIT = 20 /** A failure rate over a handful of jobs is noise; below this many finished jobs it is ignored. */ @@ -255,10 +259,27 @@ export function countUnheldActiveJobs( return Math.max(0, settled - held) } +/** + * Turn busy sending time into a per-minute rate, or null when there is too little to go on: a + * handful of messages, or a few seconds, extrapolate to a minute badly. + */ +export function sendingRateFrom( + busyMessages: number, + busySeconds: number, +): MonitoringOverviewDto['sendingRate'] { + if (busyMessages < MIN_RATE_MESSAGES || busySeconds < MIN_RATE_BUSY_SECONDS) return null + return { + perMinute: Math.round((busyMessages / busySeconds) * 60 * 10) / 10, + messages: busyMessages, + busySeconds: Math.round(busySeconds), + } +} + export function buildOverview( messages: MessageChannelStatsDto[], queues: QueueStatsDto[], deliveryTime: MonitoringOverviewDto['deliveryTime'], + sendingRate: MonitoringOverviewDto['sendingRate'], ): MonitoringOverviewDto { const sumSeries = (pick: (m: MessageChannelStatsDto) => number[]) => messages.reduce((total, m) => total + pick(m).reduce((a, b) => a + b, 0), 0) @@ -272,7 +293,7 @@ export function buildOverview( deliveryTime, messagesSent, messagesFailed, - sentPerMinute: Math.round((messagesSent / MONITORING_WINDOW_MINUTES) * 10) / 10, + sendingRate, failurePercent: finished > 0 ? Math.round((messagesFailed / finished) * 1000) / 10 : 0, requestsReceived: ingestion ? ingestion.throughput.added.reduce((a, b) => a + b, 0) : 0, } @@ -310,14 +331,16 @@ export class MonitoringService { } const now = Date.now() - const [info, workers, messages, failedJobs, activeByQueue, deliveryTime] = await Promise.all([ - this.redis.info(), - this.getWorkers(now), - this.getMessageStats(now), - this.getRecentFailedJobs(), - this.getActiveJobs(), - this.getDeliveryTime(now), - ]) + const [info, workers, messages, failedJobs, activeByQueue, deliveryTime, sendingRate] = + await Promise.all([ + this.redis.info(), + this.getWorkers(now), + this.getMessageStats(now), + this.getRecentFailedJobs(), + this.getActiveJobs(), + this.getDeliveryTime(now), + this.getSendingRate(now), + ]) const redis = buildRedisStats(parseRedisInfo(info)) const queues = await Promise.all( this.queues.map((queue) => @@ -334,7 +357,7 @@ export class MonitoringService { return { generatedAt: new Date(now).toISOString(), status: worstStatus([redis.status, ...queues.map((queue) => queue.status)]), - overview: buildOverview(messages, queues, deliveryTime), + overview: buildOverview(messages, queues, deliveryTime, sendingRate), redis, messages, queues, @@ -512,6 +535,31 @@ export class MonitoringService { } } + /** + * Sustained sending speed over the last hour, measured from the gaps between consecutive sends + * rather than per-minute buckets, so a 30-second burst reads as its own pace instead of being + * split across, or diluted by, the minutes around it. Concurrent workers shrink the gaps, so + * this is the whole system's throughput, not one worker's. + */ + private async getSendingRate(now: number): Promise { + const since = new Date(now - MONITORING_WINDOW_MINUTES * 60_000) + const table = this.detailRepository.metadata.tablePath + // The status filter repeats the partial index predicate (V72) so the planner uses it. + const [row] = (await this.detailRepository.query( + `SELECT COUNT(*) FILTER (WHERE gap <= $2) AS busy_messages, + COALESCE(SUM(gap) FILTER (WHERE gap <= $2), 0) AS busy_seconds + FROM ( + SELECT EXTRACT(EPOCH FROM last_attempt_at - lag(last_attempt_at) + OVER (ORDER BY last_attempt_at)) AS gap + FROM ${table} + WHERE status IN ('sent', 'failed') AND status = 'sent' AND last_attempt_at >= $1 + ) gaps`, + [since, IDLE_GAP_SECONDS], + )) as Array<{ busy_messages: string; busy_seconds: string }> + + return sendingRateFrom(Number(row?.busy_messages ?? 0), Number(row?.busy_seconds ?? 0)) + } + private async getTenantNames(ids: Array): Promise> { const tenantIds = [...new Set(ids.filter((id): id is string => !!id))] if (tenantIds.length === 0) return new Map() diff --git a/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts b/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts index c080e9f5c..8fd2fe978 100644 --- a/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts +++ b/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts @@ -126,7 +126,12 @@ export interface MonitoringOverviewDto { deliveryTime: { medianMs: number; p95Ms: number; messages: number } | null messagesSent: number messagesFailed: number - sentPerMinute: number + /** + * Sustained sending speed: messages sent while busy, divided by time spent busy, as a + * per-minute rate. Gaps between sends longer than an idle threshold are not counted as busy. + * Null when there was too little sending to estimate from. + */ + sendingRate: { perMinute: number; messages: number; busySeconds: number } | null failurePercent: number /** Notification requests accepted: one ingestion job each. */ requestsReceived: number diff --git a/frontend/src/interfaces/queueMonitoring.interface.ts b/frontend/src/interfaces/queueMonitoring.interface.ts index 2e806ba41..fd66be539 100644 --- a/frontend/src/interfaces/queueMonitoring.interface.ts +++ b/frontend/src/interfaces/queueMonitoring.interface.ts @@ -103,7 +103,12 @@ export interface MonitoringOverview { deliveryTime: { medianMs: number; p95Ms: number; messages: number } | null messagesSent: number messagesFailed: number - sentPerMinute: number + /** + * Sustained sending speed: messages sent while busy, divided by time spent busy, as a + * per-minute rate. Gaps between sends longer than an idle threshold are not counted as busy. + * Null when there was too little sending to estimate from. + */ + sendingRate: { perMinute: number; messages: number; busySeconds: number } | null failurePercent: number requestsReceived: number } diff --git a/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx b/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx index c5ce1a285..e15409879 100644 --- a/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx +++ b/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx @@ -30,7 +30,7 @@ function snapshot(overrides: Partial = {}): QueueMonitoring deliveryTime: { medianMs: 42_000, p95Ms: 130_000, messages: 1240 }, messagesSent: 1240, messagesFailed: 8, - sentPerMinute: 20.7, + sendingRate: { perMinute: 36.1, messages: 1180, busySeconds: 1962 }, failurePercent: 0.6, requestsReceived: 38, }, @@ -192,6 +192,9 @@ describe('QueueMonitoring', () => { const hour = screen.getByRole('region', { name: 'Last hour' }) expect(within(hour).getByText('1,240 sent')).toBeInTheDocument() + expect( + within(hour).getByText('Sends ~36.1/min (measured over 32m 42s of sending)'), + ).toBeInTheDocument() expect(within(hour).getByText('38 requests received')).toBeInTheDocument() expect(within(hour).getByText('8 failed (0.6%)')).toBeInTheDocument() diff --git a/frontend/src/pages/admin/monitoring/sections/OverviewCards.tsx b/frontend/src/pages/admin/monitoring/sections/OverviewCards.tsx index d56eb08f6..7ffddbc62 100644 --- a/frontend/src/pages/admin/monitoring/sections/OverviewCards.tsx +++ b/frontend/src/pages/admin/monitoring/sections/OverviewCards.tsx @@ -54,7 +54,11 @@ const OverviewCards: FC = ({ overview, messages, redis }) => -

{overview.sentPerMinute.toLocaleString()}/min on average

+

+ {overview.sendingRate + ? `Sends ~${overview.sendingRate.perMinute.toLocaleString()}/min (measured over ${formatDuration(overview.sendingRate.busySeconds * 1000)} of sending)` + : 'Not enough sending to estimate a rate'} +

{overview.requestsReceived.toLocaleString()} requests received

{overview.messagesFailed.toLocaleString()} failed ({overview.failurePercent}%) From 83a57c8ccc8847787e366a80162b0b233dab2533 Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Fri, 2 Oct 2026 11:24:29 -0700 Subject: [PATCH 08/23] Refine Redis fragmentation signal in monitoring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Improve the Redis stats UI to avoid misleading fragmentation ratios on small instances. Fragmentation is now hidden with a “Not meaningful at low memory use” hint below 100MB used memory, and high fragmentation (>1.5) is explicitly flagged with guidance. Added/updated QueueMonitoring tests to cover normal, low-memory, and high-fragmentation scenarios. --- .../admin/monitoring/QueueMonitoring.spec.tsx | 25 +++++++++++++++++++ .../monitoring/sections/RedisSection.tsx | 24 +++++++++++++++--- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx b/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx index e15409879..0615a40c7 100644 --- a/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx +++ b/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx @@ -310,6 +310,31 @@ describe('QueueMonitoring', () => { expect(screen.getByText('4 idle workers listening')).toBeInTheDocument() }) + it('hides fragmentation at low memory use and flags it when high on a busy instance', async () => { + mockedGet.mockResolvedValue(snapshot()) + const { unmount } = renderPage('system') + // 300 MB used in the default snapshot: meaningful, and 1.12 is normal. + expect(await screen.findByText('1.12')).toBeInTheDocument() + unmount() + + const small = snapshot() + small.redis = { ...small.redis, usedMemoryBytes: 45 * MB, fragmentationRatio: 2.76 } + mockedGet.mockResolvedValue(small) + const second = renderPage('system') + expect(await screen.findByText('Not meaningful at low memory use')).toBeInTheDocument() + expect(screen.queryByText('2.76')).not.toBeInTheDocument() + second.unmount() + + const fragmented = snapshot() + fragmented.redis = { ...fragmented.redis, fragmentationRatio: 2.4 } + mockedGet.mockResolvedValue(fragmented) + renderPage('system') + expect(await screen.findByText('2.40')).toBeInTheDocument() + expect( + screen.getByText('High: Redis holds more memory than its data needs'), + ).toBeInTheDocument() + }) + it('lists recent failures on the Failures tab, reached from the tab bar', async () => { mockedGet.mockResolvedValue(snapshot()) renderPage() diff --git a/frontend/src/pages/admin/monitoring/sections/RedisSection.tsx b/frontend/src/pages/admin/monitoring/sections/RedisSection.tsx index 115dbade4..cfbcc9b0a 100644 --- a/frontend/src/pages/admin/monitoring/sections/RedisSection.tsx +++ b/frontend/src/pages/admin/monitoring/sections/RedisSection.tsx @@ -10,7 +10,26 @@ interface RedisSectionProps { redis: RedisStats } +/** + * Below this much memory Redis's fixed overhead dominates the ratio, so a small, healthy + * instance routinely reads 2-5. Only above it does a high ratio mean memory is being wasted. + */ +export const FRAGMENTATION_MIN_BYTES = 100 * 1024 * 1024 +const FRAGMENTATION_HIGH_RATIO = 1.5 + +function describeFragmentation(redis: RedisStats): { value: string; hint?: string } { + if (redis.fragmentationRatio === null) return { value: '—' } + if (redis.usedMemoryBytes < FRAGMENTATION_MIN_BYTES) { + return { value: '—', hint: 'Not meaningful at low memory use' } + } + const value = redis.fragmentationRatio.toFixed(2) + return redis.fragmentationRatio > FRAGMENTATION_HIGH_RATIO + ? { value, hint: 'High: Redis holds more memory than its data needs' } + : { value } +} + const RedisSection: FC = ({ redis }) => { + const fragmentation = describeFragmentation(redis) const used = formatBytes(redis.usedMemoryBytes) const memoryText = redis.maxMemoryBytes === null @@ -61,10 +80,7 @@ const RedisSection: FC = ({ redis }) => { value={redis.rejectedConnections.toLocaleString()} /> - + Date: Sun, 4 Oct 2026 09:19:36 -0700 Subject: [PATCH 09/23] Drain queue workers on shutdown Add graceful queue shutdown so workers mark themselves as draining, finish in-flight jobs within the pod grace period, and then remove their heartbeat instead of appearing stalled. Expose the new draining state through backend monitoring and the admin UI. Also make retried email merge batches skip recipients already sent by an earlier attempt to avoid duplicate delivery during pod replacement or shutdown. --- .../monitoring/monitoring.service.spec.ts | 2 + .../admin/monitoring/monitoring.service.ts | 2 + .../schemas/queue-monitoring.dto.ts | 2 + .../src/queue/queue.module.shutdown.spec.ts | 60 +++++++++++++++++++ backend/src/queue/queue.module.ts | 45 +++++++++++++- backend/src/queue/worker-heartbeat.spec.ts | 28 ++++++++- backend/src/queue/worker-heartbeat.ts | 27 +++++++++ .../workers/email-delivery.worker.spec.ts | 29 +++++++++ .../queue/workers/email-delivery.worker.ts | 21 ++++++- .../backend/templates/deployment.yaml | 5 ++ charts/app/values.yaml | 3 + .../interfaces/queueMonitoring.interface.ts | 2 + .../admin/monitoring/QueueMonitoring.spec.tsx | 11 ++++ .../monitoring/sections/WorkersSection.tsx | 22 ++++++- 14 files changed, 250 insertions(+), 9 deletions(-) create mode 100644 backend/src/queue/queue.module.shutdown.spec.ts diff --git a/backend/src/api/admin/monitoring/monitoring.service.spec.ts b/backend/src/api/admin/monitoring/monitoring.service.spec.ts index c7dd05d2b..2106f3a2d 100644 --- a/backend/src/api/admin/monitoring/monitoring.service.spec.ts +++ b/backend/src/api/admin/monitoring/monitoring.service.spec.ts @@ -384,6 +384,8 @@ describe('MonitoringService', () => { expect(result.queues[0].liveWorkerPods).toBe(1) expect(result.queues[0].oldestWaitingAgeMs).toBeGreaterThanOrEqual(30_000) expect(result.workers[0].podId).toBe('pod-0') + // Heartbeats from a build that predates draining carry no flag. + expect(result.workers[0].draining).toBe(false) expect(result.status).toBe('healthy') }) diff --git a/backend/src/api/admin/monitoring/monitoring.service.ts b/backend/src/api/admin/monitoring/monitoring.service.ts index 51b72b411..bccd607c3 100644 --- a/backend/src/api/admin/monitoring/monitoring.service.ts +++ b/backend/src/api/admin/monitoring/monitoring.service.ts @@ -470,6 +470,8 @@ export class MonitoringService { podId: pod.podId, startedAt: new Date(pod.startedAt).toISOString(), lastHeartbeatAt: new Date(pod.heartbeatAt).toISOString(), + // Absent from pods running a build from before draining was reported. + draining: pod.draining ?? false, queues: pod.queues.map((q) => ({ ...q, lastFinishedAt: q.lastFinishedAt ? new Date(q.lastFinishedAt).toISOString() : null, diff --git a/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts b/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts index 8fd2fe978..ce7d2b651 100644 --- a/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts +++ b/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts @@ -74,6 +74,8 @@ export interface WorkerPodDto { podId: string startedAt: string lastHeartbeatAt: string + /** Shutting down: finishing the jobs it holds, taking no new ones. */ + draining: boolean queues: WorkerQueueDto[] } diff --git a/backend/src/queue/queue.module.shutdown.spec.ts b/backend/src/queue/queue.module.shutdown.spec.ts new file mode 100644 index 000000000..ac799cb19 --- /dev/null +++ b/backend/src/queue/queue.module.shutdown.spec.ts @@ -0,0 +1,60 @@ +import { describe, it, expect, vi, afterEach } from 'vitest' +import type Bull from 'bull' +import { QueueModule } from './queue.module' +import type { WorkerHeartbeat } from './worker-heartbeat' + +type Heartbeat = Pick + +function moduleWith(close: () => Promise) { + const queue = { close: vi.fn(close) } as unknown as Bull.Queue + const heartbeat: Heartbeat = { + markDraining: vi.fn(), + remove: vi.fn().mockResolvedValue(undefined), + } + const module = new QueueModule(queue, queue, queue, null as unknown as Bull.Queue) + // Set by onModuleInit when workers start; that path needs the whole worker graph. + ;(module as unknown as { heartbeat: Heartbeat }).heartbeat = heartbeat + return { module, queue, heartbeat } +} + +describe('QueueModule shutdown', () => { + afterEach(() => { + vi.useRealTimers() + delete process.env.QUEUE_SHUTDOWN_DRAIN_MS + }) + + it('keeps reporting as draining instead of going silent when shutdown starts', () => { + const { module, heartbeat } = moduleWith(() => Promise.resolve()) + module.onModuleDestroy() + expect(heartbeat.markDraining).toHaveBeenCalled() + }) + + it('waits for in-flight jobs, then drops the pod off the live list', async () => { + const { module, queue, heartbeat } = moduleWith(() => Promise.resolve()) + + await module.beforeApplicationShutdown() + + // Three queues present (webhook absent), each closed so it finishes its active jobs. + expect(queue.close).toHaveBeenCalledTimes(3) + expect(heartbeat.remove).toHaveBeenCalled() + }) + + it('gives up at the drain limit and keeps the heartbeat for jobs still running', async () => { + vi.useFakeTimers() + process.env.QUEUE_SHUTDOWN_DRAIN_MS = '5000' + const { module, heartbeat } = moduleWith(() => new Promise(() => {})) + + const shutdown = module.beforeApplicationShutdown() + await vi.advanceTimersByTimeAsync(5000) + await shutdown + + expect(heartbeat.remove).not.toHaveBeenCalled() + }) + + it('does not throw out of shutdown when closing a queue fails', async () => { + const { module, heartbeat } = moduleWith(() => Promise.reject(new Error('connection lost'))) + + await expect(module.beforeApplicationShutdown()).resolves.toBeUndefined() + expect(heartbeat.remove).not.toHaveBeenCalled() + }) +}) diff --git a/backend/src/queue/queue.module.ts b/backend/src/queue/queue.module.ts index d7ea910ae..721a64aaa 100644 --- a/backend/src/queue/queue.module.ts +++ b/backend/src/queue/queue.module.ts @@ -1,4 +1,5 @@ import { + BeforeApplicationShutdown, Module, OnModuleInit, OnModuleDestroy, @@ -167,7 +168,7 @@ import { PhoneNumberService } from '../api/notify/services/phone-number.service' QueueName.WEBHOOK_DELIVERY, ], }) -export class QueueModule implements OnModuleInit, OnModuleDestroy { +export class QueueModule implements OnModuleInit, OnModuleDestroy, BeforeApplicationShutdown { private readonly logger = new Logger(QueueModule.name) private heartbeat?: WorkerHeartbeat @@ -200,7 +201,47 @@ export class QueueModule implements OnModuleInit, OnModuleDestroy { ) {} onModuleDestroy(): void { - this.heartbeat?.stop() + // Not stopped yet: the workers keep running until beforeApplicationShutdown drains them, + // and the page should show their jobs as held by a pod shutting down, not as stalled. + this.heartbeat?.markDraining() + } + + /** + * Let in-flight jobs finish before the pod exits. Without this a deploy kills a merge batch + * part-way through and Bull retries it on another pod about a minute later. + * + * Runs before onApplicationShutdown, where TypeORM closes its connection, so a finishing job + * can still record what it sent. Bull's close() stops taking new jobs and resolves once the + * active ones end. The wait is capped below the pod's termination grace period; a batch still + * running then is left for Bull to retry, and the merge workers skip recipients already sent. + */ + async beforeApplicationShutdown(): Promise { + const queues = [this.ingestionQueue, this.emailQueue, this.smsQueue, this.webhookQueue].filter( + (queue): queue is Bull.Queue => !!queue, + ) + if (queues.length === 0) return + + const drainMs = parseInt(process.env.QUEUE_SHUTDOWN_DRAIN_MS || '100000', 10) + this.logger.log(`Draining queue workers (up to ${drainMs / 1000}s)`) + let timer: NodeJS.Timeout | undefined + const drained = await Promise.race([ + Promise.all(queues.map((queue) => queue.close())).then(() => true), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), drainMs) + }), + ]).catch((error: Error) => { + this.logger.error(`Queue drain failed: ${error.message}`) + return false + }) + if (timer) clearTimeout(timer) + + if (drained) { + this.logger.log('Queue workers drained') + await this.heartbeat?.remove() + } else { + // Still holding jobs: keep the heartbeat until the pod is killed so they read as held. + this.logger.warn('Queue drain timed out; unfinished jobs will be retried on another pod') + } } async onModuleInit() { diff --git a/backend/src/queue/worker-heartbeat.spec.ts b/backend/src/queue/worker-heartbeat.spec.ts index dde24b1b0..716616cd3 100644 --- a/backend/src/queue/worker-heartbeat.spec.ts +++ b/backend/src/queue/worker-heartbeat.spec.ts @@ -11,9 +11,17 @@ import { import type { WorkerHeartbeatPayload } from './worker-heartbeat' function fakeRedis() { - const multi = { set: vi.fn(), zadd: vi.fn(), exec: vi.fn().mockResolvedValue([]) } + const multi = { + set: vi.fn(), + zadd: vi.fn(), + del: vi.fn(), + zrem: vi.fn(), + exec: vi.fn().mockResolvedValue([]), + } multi.set.mockReturnValue(multi) multi.zadd.mockReturnValue(multi) + multi.del.mockReturnValue(multi) + multi.zrem.mockReturnValue(multi) return { redis: { multi: vi.fn(() => multi) } as unknown as Redis, multi } } @@ -89,4 +97,22 @@ describe('WorkerHeartbeat', () => { multi.exec.mockRejectedValue(new Error('connection lost')) await expect(new WorkerHeartbeat(redis).beat()).resolves.toBeUndefined() }) + + it('reports draining while the pod shuts down, then removes itself', async () => { + const { redis, multi } = fakeRedis() + const heartbeat = new WorkerHeartbeat(redis) + heartbeat.track(fakeQueue('email-delivery'), 2) + + await heartbeat.beat() + expect(JSON.parse(multi.set.mock.calls[0][1]).draining).toBe(false) + + heartbeat.markDraining() + await vi.waitFor(() => expect(multi.set).toHaveBeenCalledTimes(2)) + const payload = JSON.parse(multi.set.mock.calls[1][1]) as WorkerHeartbeatPayload + expect(payload.draining).toBe(true) + + await heartbeat.remove() + expect(multi.del).toHaveBeenCalledWith(workerHeartbeatKey(payload.podId)) + expect(multi.zrem).toHaveBeenCalledWith(WORKER_INDEX_KEY, payload.podId) + }) }) diff --git a/backend/src/queue/worker-heartbeat.ts b/backend/src/queue/worker-heartbeat.ts index 8766f8299..779321a28 100644 --- a/backend/src/queue/worker-heartbeat.ts +++ b/backend/src/queue/worker-heartbeat.ts @@ -30,6 +30,8 @@ export interface WorkerHeartbeatPayload { podId: string startedAt: number heartbeatAt: number + /** The pod is shutting down: it takes no new jobs and is finishing the ones it holds. */ + draining: boolean queues: WorkerQueueState[] } @@ -49,6 +51,7 @@ export class WorkerHeartbeat { private readonly podId = process.env.HOSTNAME || hostname() private readonly startedAt = Date.now() private readonly states = new Map() + private draining = false private timer?: NodeJS.Timeout constructor(private readonly redis: Redis) {} @@ -96,12 +99,36 @@ export class WorkerHeartbeat { this.timer = undefined } + /** + * Keep reporting while the pod shuts down, flagged as draining, so the jobs it is still + * finishing show as held rather than as stalled. + */ + markDraining(): void { + this.draining = true + void this.beat() + } + + /** Drop off the live list straight away instead of waiting for the key to expire. */ + async remove(): Promise { + this.stop() + try { + await this.redis + .multi() + .del(workerHeartbeatKey(this.podId)) + .zrem(WORKER_INDEX_KEY, this.podId) + .exec() + } catch (error) { + logger.warn(`Failed to remove worker heartbeat: ${(error as Error).message}`) + } + } + async beat(): Promise { const now = Date.now() const payload: WorkerHeartbeatPayload = { podId: this.podId, startedAt: this.startedAt, heartbeatAt: now, + draining: this.draining, queues: [...this.states.values()], } try { diff --git a/backend/src/queue/workers/email-delivery.worker.spec.ts b/backend/src/queue/workers/email-delivery.worker.spec.ts index 5badbe265..985e9614c 100644 --- a/backend/src/queue/workers/email-delivery.worker.spec.ts +++ b/backend/src/queue/workers/email-delivery.worker.spec.ts @@ -40,6 +40,7 @@ describe('EmailDeliveryWorker', () => { markRecipientSent: vi.fn().mockResolvedValue(undefined), markRecipientFailed: vi.fn().mockResolvedValue(undefined), countByStatus: vi.fn().mockResolvedValue(0), + findSentAddresses: vi.fn().mockResolvedValue(new Set()), } // Mock the notification service @@ -1530,6 +1531,34 @@ describe('EmailDeliveryWorker', () => { ]) }) + it('should not re-send to recipients an earlier attempt of the batch delivered', async () => { + // A pod replaced mid-batch leaves the job to be retried from the top. + mockRequestDetailService.findSentAddresses.mockResolvedValue(new Set(['alice@example.com'])) + const progress = vi.fn().mockResolvedValue(undefined) + const job = { ...makeBulkJob(['alice@example.com', 'bob@example.com']), progress } + + const result = await processHandler(job as unknown as Bull.Job) + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect(mockRequestDetailService.findSentAddresses).toHaveBeenCalledWith( + 'notify-bulk', + 'notify-bulk-EMAIL-0', + ) + expect(mockEmailAdapter.send).toHaveBeenCalledTimes(1) + expect(mockRequestDetailService.markRecipientSent).toHaveBeenCalledWith( + 'notify-bulk', + 'notify-bulk-EMAIL-0', + 'bob@example.com', + 'ext-123', + ) + expect(result).toMatchObject({ success: true, sent: 1, failed: 0 }) + // The bar resumes where the earlier attempt left off. + expect(progress.mock.calls.map(([value]) => value)).toEqual([ + { sent: 1, failed: 0, total: 2 }, + { sent: 2, failed: 0, total: 2 }, + ]) + }) + it('should render inline content per recipient when no templateId is given', async () => { mockInlineRenderingService.renderEmail.mockResolvedValue({ subject: 'Hi', diff --git a/backend/src/queue/workers/email-delivery.worker.ts b/backend/src/queue/workers/email-delivery.worker.ts index 8e9fbaf08..9fe7c73c2 100644 --- a/backend/src/queue/workers/email-delivery.worker.ts +++ b/backend/src/queue/workers/email-delivery.worker.ts @@ -513,11 +513,26 @@ export class EmailDeliveryWorker { `[${notifyId}] Processing mail merge batch ${batchId}: ${recipients.length} recipient(s)`, ) + // A retried batch (a pod replaced mid-send, a crash, a provider error) starts again from + // the top. Addresses an earlier attempt delivered keep their rows and are skipped, or they + // would get the email twice. Mirrors SmsDeliveryWorker.processMergeBatch. + const alreadySent = await requestDetailService.findSentAddresses(notifyId, batchId) + const pending = alreadySent.size + ? recipients.filter((recipient) => !alreadySent.has(recipient.address)) + : recipients + + if (alreadySent.size > 0) { + logger.log( + `[${notifyId}] Batch ${batchId}: skipping ${alreadySent.size} recipient(s) delivered on an earlier attempt`, + ) + } + let sent = 0 let failed = 0 - reportProgress?.({ sent, failed, total: recipients.length }) + const progress = () => ({ sent: alreadySent.size + sent, failed, total: recipients.length }) + reportProgress?.(progress()) - for (const recipient of recipients) { + for (const recipient of pending) { try { // Merge global params with per-recipient params; per-recipient takes precedence const mergedParams = { ...(params || {}), ...recipient.params } @@ -579,7 +594,7 @@ export class EmailDeliveryWorker { ) failed++ } - reportProgress?.({ sent, failed, total: recipients.length }) + reportProgress?.(progress()) } logger.log(`[${notifyId}] Mail merge batch ${batchId} complete: sent=${sent}, failed=${failed}`) diff --git a/charts/app/templates/backend/templates/deployment.yaml b/charts/app/templates/backend/templates/deployment.yaml index 2e6c058c3..eef6f150b 100644 --- a/charts/app/templates/backend/templates/deployment.yaml +++ b/charts/app/templates/backend/templates/deployment.yaml @@ -25,6 +25,9 @@ spec: labels: {{- include "backend.labels" . | nindent 8 }} spec: + # Time for queue workers to finish in-flight jobs on shutdown (QueueModule + # beforeApplicationShutdown). QUEUE_SHUTDOWN_DRAIN_MS below stops waiting 15s short of it. + terminationGracePeriodSeconds: {{ .Values.backend.terminationGracePeriodSeconds | default 120 }} securityContext: runAsNonRoot: true initContainers: @@ -119,6 +122,8 @@ spec: value: "0" - name: INGESTION_WORKER_CONCURRENCY value: "1" + - name: QUEUE_SHUTDOWN_DRAIN_MS + value: {{ mul (sub (.Values.backend.terminationGracePeriodSeconds | default 120) 15) 1000 | quote }} - name: NOTIFICATION_DEDUP_WINDOW_SECONDS value: {{ .Values.backend.dedupWindowSeconds | quote }} - name: CLAMAV_HOST diff --git a/charts/app/values.yaml b/charts/app/values.yaml index bac120e4c..1f26d38a5 100644 --- a/charts/app/values.yaml +++ b/charts/app/values.yaml @@ -30,6 +30,9 @@ backend: enabled: true #-- the deployment strategy, can be "Recreate" or "RollingUpdate" deploymentStrategy: Recreate + #-- seconds a stopping pod gets to finish in-flight queue jobs before it is killed. With + #-- Recreate, a deploy waits this long only while a pod is mid-batch; idle pods exit at once. + terminationGracePeriodSeconds: 120 #-- the number of replicas when autoscaling is disabled replicaCount: 1 #-- autoscaling for the component. it is optional and is an object. diff --git a/frontend/src/interfaces/queueMonitoring.interface.ts b/frontend/src/interfaces/queueMonitoring.interface.ts index fd66be539..8663329a4 100644 --- a/frontend/src/interfaces/queueMonitoring.interface.ts +++ b/frontend/src/interfaces/queueMonitoring.interface.ts @@ -49,6 +49,8 @@ export interface WorkerPod { podId: string startedAt: string lastHeartbeatAt: string + /** Shutting down: finishing the jobs it holds, taking no new ones. */ + draining: boolean queues: Array<{ queue: string concurrency: number diff --git a/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx b/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx index 0615a40c7..526f086b0 100644 --- a/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx +++ b/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx @@ -112,6 +112,7 @@ function snapshot(overrides: Partial = {}): QueueMonitoring podId: 'notify-backend-abc12', startedAt: new Date(now - 3_600_000).toISOString(), lastHeartbeatAt: new Date(now - 4_000).toISOString(), + draining: false, queues: [ { queue: 'email-delivery', @@ -335,6 +336,16 @@ describe('QueueMonitoring', () => { ).toBeInTheDocument() }) + it('labels a pod that is shutting down', async () => { + const data = snapshot() + data.workers = [{ ...data.workers[0], draining: true }] + mockedGet.mockResolvedValue(data) + renderPage('system') + + const workers = await screen.findByRole('table', { name: 'Workers' }) + expect(within(workers).getByText('Shutting down')).toBeInTheDocument() + }) + it('lists recent failures on the Failures tab, reached from the tab bar', async () => { mockedGet.mockResolvedValue(snapshot()) renderPage() diff --git a/frontend/src/pages/admin/monitoring/sections/WorkersSection.tsx b/frontend/src/pages/admin/monitoring/sections/WorkersSection.tsx index ba68b6b6a..8d5862f59 100644 --- a/frontend/src/pages/admin/monitoring/sections/WorkersSection.tsx +++ b/frontend/src/pages/admin/monitoring/sections/WorkersSection.tsx @@ -1,5 +1,6 @@ import type { FC } from 'react' import PageSubHeading from '@/components/PageSubHeading' +import { StatusBadge } from '@/components/StatusBadge' import DataTable from '@/components/DataTable/DataTable' import type { TableColumn } from '@/components/DataTable/DataTable' import type { WorkerPod } from '@/interfaces/queueMonitoring.interface' @@ -8,6 +9,7 @@ import { formatAgo, formatQueueName } from '@/utils/monitoring' interface WorkerRow { key: string podId: string + draining: boolean queue: string concurrency: number active: number @@ -28,13 +30,27 @@ const WorkersSection: FC = ({ workers, now }) => { pod.queues.map((queue) => ({ key: `${pod.podId}-${queue.queue}`, podId: pod.podId, + draining: pod.draining, lastHeartbeatAt: pod.lastHeartbeatAt, ...queue, })), ) const columns: TableColumn[] = [ - { key: 'podId', label: 'Pod' }, + { + key: 'podId', + label: 'Pod', + render: (_, row) => ( + <> + {row.podId} + {row.draining && ( +

+ +
+ )} + + ), + }, { key: 'queue', label: 'Queue', render: (_, row) => formatQueueName(row.queue) }, { key: 'active', @@ -70,8 +86,8 @@ const WorkersSection: FC = ({ workers, now }) => { emptyMessage="No worker heartbeats received. Either no pod is processing jobs, or the running pods predate worker heartbeats." />

- Completed and failed counts are since each pod started. A pod drops off this list 30 seconds - after its last heartbeat. + Completed and failed counts are since each pod started. A pod shutting down finishes the + jobs it holds before it exits, and drops off this list once it has.

) From 697fa8a1049b9c0a79c4ad650ecabc5a4b2e1714 Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Sun, 4 Oct 2026 11:18:47 -0700 Subject: [PATCH 10/23] Deduplicate GC Notify module imports Remove unnecessary `forwardRef()` wrappers from `GcNotifyModule.forRoot()` when importing `NotifyModule` and `QueueModule`, preventing duplicate static module instantiation. Add an e2e guard test that scans the Nest `ModulesContainer` (excluding `TypeOrmModule` dynamic `forFeature` entries) to ensure non-TypeORM modules are only created once, protecting against double worker/heartbeat startup and related false stalled-job monitoring. --- backend/src/api/gc-notify/gc-notify.module.ts | 6 +++--- backend/test/app.e2e-spec.ts | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/backend/src/api/gc-notify/gc-notify.module.ts b/backend/src/api/gc-notify/gc-notify.module.ts index 9807d41f2..349489f39 100644 --- a/backend/src/api/gc-notify/gc-notify.module.ts +++ b/backend/src/api/gc-notify/gc-notify.module.ts @@ -1,4 +1,4 @@ -import { DynamicModule, Module, forwardRef } from '@nestjs/common' +import { DynamicModule, Module } from '@nestjs/common' import { TypeOrmModule } from '@nestjs/typeorm' import { GcNotifyController } from './gc-notify.controller' import { TenantsModule } from '../admin/tenants/tenants.module' @@ -43,8 +43,8 @@ export class GcNotifyModule { TypeOrmModule.forFeature([NotifyConfiguration, MimeTypeCode]), SafelistModule, TenantSettingsModule, - forwardRef(() => NotifyModule), - forwardRef(() => QueueModule), + NotifyModule, + QueueModule, ], controllers: [GcNotifyController], providers: [ diff --git a/backend/test/app.e2e-spec.ts b/backend/test/app.e2e-spec.ts index 081c715ea..b51d1fe0f 100644 --- a/backend/test/app.e2e-spec.ts +++ b/backend/test/app.e2e-spec.ts @@ -5,6 +5,8 @@ import { ConfigService } from '@nestjs/config' import { AppModule } from '../src/app.module' import { PendingNotificationRetryService } from '../src/queue/services/pending-notification-retry.service' import { NotificationPubSubService } from '../src/api/notification/notification-pubsub.service' +import { ModulesContainer } from '@nestjs/core' +import { TypeOrmModule } from '@nestjs/typeorm' describe('AppController (e2e)', () => { let app: INestApplication @@ -69,6 +71,23 @@ describe('AppController (e2e)', () => { } }) + it('creates each module once', () => { + // A static module registered twice runs its lifecycle hooks twice. QueueModule did: every + // pod started two sets of Bull workers and two heartbeats under one pod name, which made + // the monitoring page report running jobs as stalled. forwardRef() inside a global dynamic + // module (GcNotifyModule.forRoot) produced the orphaned copies. + // TypeOrmModule repeats legitimately: each forFeature() call is its own dynamic module. + const counts = new Map() + for (const module of app.get(ModulesContainer).values()) { + if (module.metatype === TypeOrmModule) continue + counts.set(module.metatype, (counts.get(module.metatype) ?? 0) + 1) + } + const duplicated = [...counts] + .filter(([, count]) => count > 1) + .map(([metatype]) => (metatype as { name?: string })?.name) + expect(duplicated).toEqual([]) + }) + it('/ (GET)', () => request(app.getHttpServer()).get('/').expect(200).expect({ message: 'Hello Backend!' })) }) From 110145c527180ddd37dd03677d655ad089f749aa Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Sun, 4 Oct 2026 14:05:13 -0700 Subject: [PATCH 11/23] Allow queue jobs to stall up to three times before failing Bull's default maxStalledCount of 1 fails a job the second time its pod dies mid-run, e.g. a pod deleted and then restarted during one bulk send. The two merge batches in PR-253 ended in the failed set that way, leaving 80 recipients pending. Retries are safe now that merge batches skip recipients already sent. Overridable with QUEUE_MAX_STALLED_COUNT. Co-Authored-By: Claude Opus 5.5 --- backend/src/queue/redis-connection.spec.ts | 18 +++++++++++++++++- backend/src/queue/redis-connection.ts | 8 ++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/backend/src/queue/redis-connection.spec.ts b/backend/src/queue/redis-connection.spec.ts index adfa3f488..199760c77 100644 --- a/backend/src/queue/redis-connection.spec.ts +++ b/backend/src/queue/redis-connection.spec.ts @@ -10,7 +10,11 @@ vi.mock('bull', () => ({ }), })) -import { createQueue, QUEUE_METRICS_MAX_DATA_POINTS } from './redis-connection' +import { + createQueue, + QUEUE_MAX_STALLED_COUNT, + QUEUE_METRICS_MAX_DATA_POINTS, +} from './redis-connection' describe('createQueue', () => { beforeEach(() => { @@ -31,4 +35,16 @@ describe('createQueue', () => { it('keeps a day of one-minute data points by default', () => { expect(QUEUE_METRICS_MAX_DATA_POINTS).toBe(1440) }) + + it('lets a job stall more than once before Bull fails it', () => { + // A batch can be interrupted twice in one send (a pod deleted, then a deploy). Bull's default + // of 1 fails it on the second; retries are safe because merge batches skip sent recipients. + createQueue(QueueName.EMAIL_DELIVERY, { host: 'localhost', port: 6379, db: 0 }) + + expect(QUEUE_MAX_STALLED_COUNT).toBe(3) + expect(bullConstructor).toHaveBeenCalledWith( + QueueName.EMAIL_DELIVERY, + expect.objectContaining({ settings: { maxStalledCount: 3 } }), + ) + }) }) diff --git a/backend/src/queue/redis-connection.ts b/backend/src/queue/redis-connection.ts index e7a44de5a..f4aa95d9c 100644 --- a/backend/src/queue/redis-connection.ts +++ b/backend/src/queue/redis-connection.ts @@ -42,6 +42,13 @@ export const QUEUE_METRICS_MAX_DATA_POINTS = parseInt( 10, ) +/** + * Times a job may stall (its pod dies or loses its lock mid-job) before Bull fails it instead + * of retrying. Bull's default of 1 fails a batch interrupted twice, e.g. a pod deleted and then + * restarted during the same send. A retry is safe: merge batches skip recipients already sent. + */ +export const QUEUE_MAX_STALLED_COUNT = parseInt(process.env.QUEUE_MAX_STALLED_COUNT || '3', 10) + /** * Create a Bull queue with an error listener attached. * @@ -59,6 +66,7 @@ export function createQueue(name: QueueName, redisConfig: RedisConfig): Bull.Que maxRetriesPerRequest: null, }), metrics: { maxDataPoints: QUEUE_METRICS_MAX_DATA_POINTS }, + settings: { maxStalledCount: QUEUE_MAX_STALLED_COUNT }, }) attachRedisErrorLogging(queue, `Queue[${name}]`) countAddedJobs(queue) From e2daf877cc5096d09cb056885d0907e5ebfc541d Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Sun, 4 Oct 2026 15:03:44 -0700 Subject: [PATCH 12/23] Harden merge retries and batch recovery Refactors mail-merge queue payload handling so jobs carry shared content/params only, while per-recipient params are persisted on `notification_request_detail` (with migration `V73`). Adds merge reconstruction helpers and updates pending-notification retry to rebuild proper merge ingestion payloads and enqueue unnamed jobs so workers can process them. Introduces `BatchReconcilerService` to detect stale merge batches, retry/requeue missing or failed batch jobs from stored request data, and fail remaining recipients after a capped number of recovery attempts to prevent batches from staying in-flight indefinitely. Includes new unit coverage for merge batch builders, batch reconciliation behavior, and pending retry merge rebuilding. --- .../notification-request-detail.entity.ts | 4 + .../notification-request-detail.service.ts | 46 ++- .../common/decorators/queueable.decorator.ts | 16 +- backend/src/config/configuration.ts | 5 +- backend/src/queue/merge-batch-builder.spec.ts | 62 ++++ backend/src/queue/merge-batch-builder.ts | 69 +++++ backend/src/queue/queue.module.ts | 2 + backend/src/queue/queue.types.ts | 22 +- .../services/batch-reconciler.service.spec.ts | 202 +++++++++++++ .../services/batch-reconciler.service.ts | 270 ++++++++++++++++++ ...pending-notification-retry.service.spec.ts | 63 +++- .../pending-notification-retry.service.ts | 57 ++-- .../workers/email-delivery.worker.spec.ts | 24 ++ .../queue/workers/email-delivery.worker.ts | 10 +- .../queue/workers/ingestion.worker.spec.ts | 165 +++++++++-- backend/src/queue/workers/ingestion.worker.ts | 60 +++- .../queue/workers/merge-batch-recipients.ts | 33 +++ .../src/queue/workers/sms-delivery.worker.ts | 10 +- ...73__notification_request_detail_params.sql | 7 + 19 files changed, 1030 insertions(+), 97 deletions(-) create mode 100644 backend/src/queue/merge-batch-builder.spec.ts create mode 100644 backend/src/queue/merge-batch-builder.ts create mode 100644 backend/src/queue/services/batch-reconciler.service.spec.ts create mode 100644 backend/src/queue/services/batch-reconciler.service.ts create mode 100644 backend/src/queue/workers/merge-batch-recipients.ts create mode 100644 migrations/sql/V73__notification_request_detail_params.sql diff --git a/backend/src/api/notification/entities/notification-request-detail.entity.ts b/backend/src/api/notification/entities/notification-request-detail.entity.ts index 4f3780748..d3bee9ccd 100644 --- a/backend/src/api/notification/entities/notification-request-detail.entity.ts +++ b/backend/src/api/notification/entities/notification-request-detail.entity.ts @@ -42,6 +42,10 @@ export class NotificationRequestDetail { @Column({ name: 'provider_response', type: 'jsonb', nullable: true }) providerResponse?: any + /** Mail merge personalisation for this recipient; null for non-merge sends. */ + @Column({ type: 'jsonb', nullable: true }) + params?: Record | null + @Column({ name: 'error_message', type: 'text', nullable: true }) errorMessage?: string diff --git a/backend/src/api/notification/notification-request-detail.service.ts b/backend/src/api/notification/notification-request-detail.service.ts index 129338cbf..d024bc535 100644 --- a/backend/src/api/notification/notification-request-detail.service.ts +++ b/backend/src/api/notification/notification-request-detail.service.ts @@ -93,24 +93,26 @@ export class NotificationRequestDetailService { } /** - * Create pending request detail records for one batch of a mail merge email send. - * Each record is tagged with the shared batchId so a delivery worker can scope updates to its batch. + * Create pending request detail records for one batch of a mail merge send. + * Each record is tagged with the shared batchId so a delivery worker can scope updates to its + * batch, and keeps the recipient's personalisation, which the worker renders from. */ async createMergePending( notificationRequestId: string, batchId: string, - addresses: string[], + recipients: Array<{ address: string; params?: Record }>, channel: NotificationChannel = NotificationChannel.EMAIL, createdBy?: string, ): Promise { - if (addresses.length === 0) return + if (recipients.length === 0) return const now = new Date() const isEmail = channel === NotificationChannel.EMAIL - const entities = addresses.map((address) => + const entities = recipients.map(({ address, params }) => this.detailRepository.create({ notificationRequestId, batchId, recipientAddress: address, + params: params ?? null, channel, // Only meaningful for email, where a recipient can be a to/cc/bcc. ...(isEmail && { emailAddressType: 'primary' }), @@ -124,6 +126,40 @@ export class NotificationRequestDetailService { await this.detailRepository.save(entities) } + /** + * A merge batch's recipients as stored at ingestion, in insertion order, with their status so + * a retried batch can skip those already sent. + */ + async findBatchRecipients( + notificationRequestId: string, + batchId: string, + ): Promise; status: string }>> { + const rows = await this.detailRepository.find({ + where: { notificationRequestId, batchId }, + select: { id: true, recipientAddress: true, params: true, status: true }, + order: { createdAt: 'ASC', id: 'ASC' }, + }) + return rows.map((row) => ({ + address: row.recipientAddress, + params: row.params ?? {}, + status: row.status, + })) + } + + /** Rows already written for a merge batch; non-zero means ingestion created it on an earlier run. */ + async countBatch(notificationRequestId: string, batchId: string): Promise { + return this.detailRepository.count({ where: { notificationRequestId, batchId } }) + } + + /** Addresses of a request's recipients in one status, e.g. those the safelist blocked. */ + async findAddressesByStatus(notificationRequestId: string, status: string): Promise> { + const rows = await this.detailRepository.find({ + where: { notificationRequestId, status }, + select: { recipientAddress: true }, + }) + return new Set(rows.map((row) => row.recipientAddress)) + } + /** * Record recipients that were never attempted because they are not on the tenant safelist. * Written at accept time, so these rows carry no attempt count and no batchId. diff --git a/backend/src/common/decorators/queueable.decorator.ts b/backend/src/common/decorators/queueable.decorator.ts index 405847c2d..287417048 100644 --- a/backend/src/common/decorators/queueable.decorator.ts +++ b/backend/src/common/decorators/queueable.decorator.ts @@ -28,6 +28,7 @@ import type { DuplicateNotification, NotificationDedupService, } from '../../api/notify/services/notification-dedup.service' +import { mergeRequestChannel } from '../../queue/merge-batch-builder' import { COMPLETED_JOB_RETENTION, FAILED_JOB_RETENTION } from '../../queue/job-retention' interface AcceptedUsageResult extends RecordedUsageResult { @@ -282,18 +283,6 @@ function isValidTenantContext(tenant: unknown): tenant is { id: string } { ) } -/** - * Detect a mail-merge email payload: an email channel whose recipients use `mergeArray`. The global - * ValidationPipe has already validated the body, so the presence of `email.recipients.mergeArray` is - * sufficient to route the request through the mail merge fan-out flow. - */ -function mergeRequestChannel(payload: unknown): NotificationChannel | null { - const request = payload as NotifySimpleRequest | undefined - if (Array.isArray(request?.email?.recipients?.mergeArray)) return NotificationChannel.EMAIL - if (Array.isArray(request?.sms?.recipients?.mergeArray)) return NotificationChannel.SMS - return null -} - /** * Billable message count for a merge. * @@ -540,10 +529,11 @@ export async function handleMerge( requestedAt: new Date().toISOString(), mailMerge: true, mailMergeChannel: channel, + // Recipients are not carried: ingestion reads them from the request stored above, so a + // large merge is not held in Redis. mailMergeData: { content: channelPayload.content, params: globalParams, - recipients, }, ...(delayedSendTimestamp && { scheduledFor: delayedSendTimestamp }), } diff --git a/backend/src/config/configuration.ts b/backend/src/config/configuration.ts index 43af1845e..eb6f54545 100644 --- a/backend/src/config/configuration.ts +++ b/backend/src/config/configuration.ts @@ -180,8 +180,9 @@ export default () => { // Job Queue Worker Configuration queue: { ingestionWorkerConcurrency: parseInt(process.env.INGESTION_WORKER_CONCURRENCY || '1', 10), - // Number of recipients per mail merge delivery batch - batchSize: parseInt(process.env.BATCH_SIZE || '100', 10), + // Recipients per merge delivery job. Small enough that a job finishes in seconds, so a + // pod stopping mid-send replays little and its drain fits the termination grace period. + batchSize: parseInt(process.env.BATCH_SIZE || '25', 10), emailDeliveryWorkerConcurrency: parseInt( process.env.EMAIL_DELIVERY_WORKER_CONCURRENCY || '20', 10, diff --git a/backend/src/queue/merge-batch-builder.spec.ts b/backend/src/queue/merge-batch-builder.spec.ts new file mode 100644 index 000000000..7d9c530b5 --- /dev/null +++ b/backend/src/queue/merge-batch-builder.spec.ts @@ -0,0 +1,62 @@ +import { describe, it, expect } from 'vitest' +import { NotificationChannel } from '../enum/notification-channel.enum' +import { mergeJobDataFromRequest, mergeRequestChannel } from './merge-batch-builder' + +const parse = (rows: string[][]) => + rows.slice(1).map(([address, name]) => ({ address, params: { name } })) + +const emailMerge = { + params: { org: 'BC', team: 'global' }, + email: { + content: { templateId: 'tpl-1' }, + params: { team: 'email' }, + recipients: { + mergeArray: [ + ['email', 'name'], + ['a@example.com', 'Ann'], + ['b@example.com', 'Bo'], + ], + }, + }, +} + +describe('mergeRequestChannel', () => { + it('detects email and SMS merges and nothing else', () => { + expect(mergeRequestChannel(emailMerge)).toBe(NotificationChannel.EMAIL) + expect(mergeRequestChannel({ sms: { recipients: { mergeArray: [] } } })).toBe( + NotificationChannel.SMS, + ) + expect(mergeRequestChannel({ email: { recipients: { to: ['a@example.com'] } } })).toBeNull() + expect(mergeRequestChannel(null)).toBeNull() + }) +}) + +describe('mergeJobDataFromRequest', () => { + it('rebuilds content, merged global params and every recipient', () => { + expect(mergeJobDataFromRequest(emailMerge, NotificationChannel.EMAIL, parse)).toEqual({ + content: { templateId: 'tpl-1' }, + // Channel params win over request-level ones, as at acceptance. + params: { org: 'BC', team: 'email' }, + recipients: [ + { address: 'a@example.com', params: { name: 'Ann' } }, + { address: 'b@example.com', params: { name: 'Bo' } }, + ], + }) + }) + + it('narrows to the recipients kept', () => { + const data = mergeJobDataFromRequest( + emailMerge, + NotificationChannel.EMAIL, + parse, + (address) => address === 'b@example.com', + ) + expect(data.recipients).toEqual([{ address: 'b@example.com', params: { name: 'Bo' } }]) + }) + + it('refuses a stored request that is not a merge on that channel', () => { + expect(() => mergeJobDataFromRequest(emailMerge, NotificationChannel.SMS, parse)).toThrow( + /not a SMS merge/, + ) + }) +}) diff --git a/backend/src/queue/merge-batch-builder.ts b/backend/src/queue/merge-batch-builder.ts new file mode 100644 index 000000000..acf5d673d --- /dev/null +++ b/backend/src/queue/merge-batch-builder.ts @@ -0,0 +1,69 @@ +import { NotificationChannel } from '../enum/notification-channel.enum' +import type { NotifySimpleRequest } from '../api/notify/schemas/notify-simple-request' +import type { MailMergeJobData } from './queue.types' + +/** NotificationService.parseMailMergeRecipients, passed in so this stays a plain function. */ +export type ParseMergeRecipients = ( + mergeArray: string[][], + channel: NotificationChannel, +) => Array<{ address: string; params: Record }> + +/** + * Detect a mail-merge payload: a channel whose recipients use `mergeArray`. The global + * ValidationPipe has already validated the body, so the presence of `recipients.mergeArray` is + * sufficient to route the request through the mail merge fan-out flow. + */ +export function mergeRequestChannel(payload: unknown): NotificationChannel | null { + const request = payload as NotifySimpleRequest | undefined + if (Array.isArray(request?.email?.recipients?.mergeArray)) return NotificationChannel.EMAIL + if (Array.isArray(request?.sms?.recipients?.mergeArray)) return NotificationChannel.SMS + return null +} + +/** + * The parts of a merge every batch shares - content and global params - without parsing its + * recipient list, which for a large merge is the expensive part. + */ +export function mergeContentFromRequest( + payload: unknown, + channel: NotificationChannel, +): Pick { + const request = payload as NotifySimpleRequest + const channelPayload = channel === NotificationChannel.SMS ? request?.sms : request?.email + if (!channelPayload || !Array.isArray(channelPayload.recipients?.mergeArray)) { + throw new Error(`Stored request is not a ${channel} merge`) + } + return { + content: channelPayload.content as MailMergeJobData['content'], + params: { ...request.params, ...channelPayload.params }, + } +} + +/** + * Rebuild a merge's job data from the request as accepted, which notification_request.payload + * stores in full. Postgres, not the job, is the record of a send, so anything Redis loses - a + * failed batch, a job never queued - can be recreated from it. + * + * `keep` narrows the recipients - to one batch's, or to those the safelist did not block at + * acceptance; the rest of a merge's shape (content, global params) is shared. + */ +export function mergeJobDataFromRequest( + payload: unknown, + channel: NotificationChannel, + parseRecipients: ParseMergeRecipients, + keep?: (address: string) => boolean, +): MailMergeJobData { + const request = payload as NotifySimpleRequest + const channelPayload = channel === NotificationChannel.SMS ? request?.sms : request?.email + const mergeArray = channelPayload?.recipients?.mergeArray + if (!channelPayload || !Array.isArray(mergeArray)) { + throw new Error(`Stored request is not a ${channel} merge`) + } + + const recipients = parseRecipients(mergeArray, channel) + return { + content: channelPayload.content as MailMergeJobData['content'], + params: { ...request.params, ...channelPayload.params }, + recipients: keep ? recipients.filter(({ address }) => keep(address)) : recipients, + } +} diff --git a/backend/src/queue/queue.module.ts b/backend/src/queue/queue.module.ts index 721a64aaa..7ec8cee49 100644 --- a/backend/src/queue/queue.module.ts +++ b/backend/src/queue/queue.module.ts @@ -22,6 +22,7 @@ import { EmailDeliveryWorker } from './workers/email-delivery.worker' import { SmsDeliveryWorker } from './workers/sms-delivery.worker' import { WebhookDeliveryWorker } from './workers/webhook-delivery.worker' import { PendingNotificationRetryService } from './services/pending-notification-retry.service' +import { BatchReconcilerService } from './services/batch-reconciler.service' import { WebhookTriggerService } from './services/webhook-trigger.service' import { NotificationRequest } from '../api/notification/entities/notification-request.entity' import { NotificationRequestDetail } from '../api/notification/entities/notification-request-detail.entity' @@ -75,6 +76,7 @@ import { PhoneNumberService } from '../api/notify/services/phone-number.service' ], providers: [ PendingNotificationRetryService, + BatchReconcilerService, NotificationService, NotificationRequestDetailService, NotificationPubSubService, diff --git a/backend/src/queue/queue.types.ts b/backend/src/queue/queue.types.ts index c01914fe9..6d8ca89c6 100644 --- a/backend/src/queue/queue.types.ts +++ b/backend/src/queue/queue.types.ts @@ -42,12 +42,12 @@ export interface NotificationRequest { } /** - * Mail-merge payload carried by ingestion and delivery jobs for the - * /notifysimple/email merge flow (a request whose email recipients use `mergeArray`). - * On the ingestion job `recipients` holds every recipient; on a delivery job it holds only - * the recipients of a single batch (identified by `batchId`). Content is rendered from either a - * server `templateId` or inline `content`. `params` are global params applied when rendering; - * per-recipient `params` override them on a per-key basis. + * Mail-merge payload carried by ingestion and delivery jobs for the merge flow (a request whose + * recipients use `mergeArray`). It holds what every batch shares: content, rendered from either + * a server `templateId` or inline `content`, and global `params`. Recipients are not in it - + * they live in Postgres (the stored request, then one detail row per recipient with its own + * `params`), so a merge of any size is a few hundred bytes per job in Redis. Per-recipient + * params override the global ones on a per-key basis. */ export interface MailMergeJobData { content?: { @@ -58,7 +58,11 @@ export interface MailMergeJobData { renderer?: 'handlebars' | 'mustache' | 'legacy_gc_notify' | 'mjml' } params?: Record - recipients: Array<{ address: string; params: Record }> + /** + * Absent on jobs queued now: ingestion reads recipients from the stored request, and delivery + * workers from their batch's detail rows. Only jobs queued before that change carry them. + */ + recipients?: Array<{ address: string; params: Record }> } /** @@ -70,7 +74,7 @@ export interface IngestionJobPayload { request: NotifyRequest requestedAt: string scheduledFor?: string // ISO datetime for delayed sends (optional). Works by delaying the ingestion job, which in turn delays all downstream delivery jobs. This simplifies handling of scheduled notifications by centralizing the scheduling logic in one place (ingestion worker) rather than needing to handle scheduling in each delivery worker. - mailMerge?: boolean // When true, this is a merge send and `mailMergeData` carries the recipients + mailMerge?: boolean // When true, this is a merge send; recipients are read from the stored request mailMergeChannel?: NotificationChannel // Which channel the merge fans out to (defaults to EMAIL) mailMergeData?: MailMergeJobData } @@ -87,7 +91,7 @@ export interface DeliveryJobPayload { attempt: number mailMerge?: boolean // When true, this is one batch of a merge send batchId?: string // Identifies the batch within the parent notification_request (mail merge only) - mailMergeData?: MailMergeJobData // Recipients for this batch (mail merge only) + mailMergeData?: MailMergeJobData // Shared content and params; recipients are the batch's detail rows } /** diff --git a/backend/src/queue/services/batch-reconciler.service.spec.ts b/backend/src/queue/services/batch-reconciler.service.spec.ts new file mode 100644 index 000000000..3d14db3a6 --- /dev/null +++ b/backend/src/queue/services/batch-reconciler.service.spec.ts @@ -0,0 +1,202 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import type Bull from 'bull' +import type Redis from 'ioredis' +import type { Repository } from 'typeorm' +import { BatchReconcilerService } from './batch-reconciler.service' +import { NotificationStatus } from '../../enum/notification-status.enum' +import type { NotificationRequestDetail } from '../../api/notification/entities/notification-request-detail.entity' +import type { NotificationRequest } from '../../api/notification/entities/notification-request.entity' +import type { NotificationService } from '../../api/notification/notification.service' + +const BATCH = { + batchId: 'req-1-EMAIL-0', + notifyId: 'req-1', + tenantId: 'tenant-1', + channel: 'EMAIL', +} + +const storedRequest = { + id: 'req-1', + payload: { + email: { + content: { subject: 'Hi', body: 'Hello {{name}}', bodyType: 'text' }, + recipients: { + mergeArray: [ + ['email', 'name'], + ['a@example.com', 'Ann'], + ['b@example.com', 'Bo'], + ['other-batch@example.com', 'Cy'], + ], + }, + }, + }, +} + +function setup({ + job, + locked = 'OK', + attempt = 1, + stale = [BATCH], +}: { + job?: Partial | null + locked?: string | null + attempt?: number + stale?: object[] +}) { + const builder: Record = {} + for (const m of [ + 'innerJoin', + 'select', + 'addSelect', + 'where', + 'andWhere', + 'groupBy', + 'addGroupBy', + 'orderBy', + 'limit', + ]) { + builder[m] = vi.fn(() => builder) + } + builder.getRawMany = vi.fn().mockResolvedValue(stale) + + const detailRepository = { + createQueryBuilder: vi.fn(() => builder), + find: vi + .fn() + .mockResolvedValue([ + { recipientAddress: 'a@example.com' }, + { recipientAddress: 'b@example.com' }, + ]), + update: vi.fn().mockResolvedValue({ affected: 2 }), + count: vi.fn().mockResolvedValue(0), + } + const requestRepository = { findOne: vi.fn().mockResolvedValue(storedRequest) } + const notificationService = { + parseMailMergeRecipients: (rows: string[][]) => + rows.slice(1).map(([address, name]) => ({ address, params: { name } })), + update: vi.fn().mockResolvedValue(undefined), + } + const redis = { + set: vi.fn().mockResolvedValue(locked), + eval: vi.fn().mockResolvedValue(1), + incr: vi.fn().mockResolvedValue(attempt), + expire: vi.fn().mockResolvedValue(1), + } + const queue = { + getJob: vi.fn().mockResolvedValue(job ?? null), + add: vi.fn().mockResolvedValue({}), + } + const service = new BatchReconcilerService( + detailRepository as unknown as Repository, + requestRepository as unknown as Repository, + notificationService as unknown as NotificationService, + redis as unknown as Redis, + queue as unknown as Bull.Queue, + null, + ) + return { service, detailRepository, notificationService, redis, queue } +} + +const jobIn = (state: string, extra: Partial = {}) => + ({ + getState: vi.fn().mockResolvedValue(state), + retry: vi.fn().mockResolvedValue(undefined), + remove: vi.fn().mockResolvedValue(undefined), + failedReason: 'job stalled more than allowable limit', + ...extra, + }) as unknown as Bull.Job + +describe('BatchReconcilerService', () => { + beforeEach(() => vi.clearAllMocks()) + + it.each(['waiting', 'active', 'delayed'])('leaves a %s batch alone', async (state) => { + const job = jobIn(state) + const { service, queue, redis } = setup({ job }) + + expect(await service.reconcile()).toEqual({ 'in-progress': 1 }) + expect(job.retry).not.toHaveBeenCalled() + expect(queue.add).not.toHaveBeenCalled() + // A long batch still running must not use up its recovery attempts. + expect(redis.incr).not.toHaveBeenCalled() + }) + + it('retries a batch whose job failed, keeping its original payload', async () => { + const job = jobIn('failed') + const { service, queue } = setup({ job }) + + expect(await service.reconcile()).toEqual({ retried: 1 }) + expect(job.retry).toHaveBeenCalled() + expect(queue.add).not.toHaveBeenCalled() + }) + + it('rebuilds a batch with no job from the stored request, leaving recipients in its rows', async () => { + const { service, queue } = setup({ job: null }) + + expect(await service.reconcile()).toEqual({ requeued: 1 }) + const [payload, options] = queue.add.mock.calls[0] + expect(payload).toMatchObject({ + notifyId: 'req-1', + tenantId: 'tenant-1', + channel: 'EMAIL', + mailMerge: true, + batchId: 'req-1-EMAIL-0', + mailMergeData: { + content: { subject: 'Hi', body: 'Hello {{name}}', bodyType: 'text' }, + params: {}, + }, + }) + expect(payload.mailMergeData).not.toHaveProperty('recipients') + expect(options).toMatchObject({ jobId: 'req-1-EMAIL-0', attempts: 3 }) + }) + + it('replaces a job that finished while recipients are still owed', async () => { + const job = jobIn('completed') + const { service, queue } = setup({ job }) + + expect(await service.reconcile()).toEqual({ requeued: 1 }) + expect(job.remove).toHaveBeenCalled() + expect(queue.add).toHaveBeenCalled() + }) + + it('gives up after the recovery limit: marks what is owed failed and settles the request', async () => { + const { service, detailRepository, notificationService, queue } = setup({ + job: jobIn('failed'), + attempt: 6, + }) + detailRepository.count.mockResolvedValueOnce(0).mockResolvedValueOnce(120) + + expect(await service.reconcile()).toEqual({ 'gave-up': 1 }) + expect(detailRepository.update).toHaveBeenCalledWith( + expect.objectContaining({ notificationRequestId: 'req-1', batchId: 'req-1-EMAIL-0' }), + expect.objectContaining({ + status: 'failed', + errorMessage: expect.stringMatching(/5 recovery/), + }), + ) + expect(notificationService.update).toHaveBeenCalledWith('req-1', 'tenant-1', { + status: NotificationStatus.PARTIALLY_COMPLETED, + updatedBy: 'batch-reconciler', + }) + expect(queue.add).not.toHaveBeenCalled() + }) + + it('does nothing when another pod holds the lock', async () => { + const { service, queue, detailRepository } = setup({ job: null, locked: null }) + + expect(await service.reconcile()).toEqual({}) + expect(detailRepository.createQueryBuilder).not.toHaveBeenCalled() + expect(queue.add).not.toHaveBeenCalled() + }) + + it('keeps going past a batch it cannot reconcile, and releases the lock', async () => { + const { service, queue, redis } = setup({ + job: null, + stale: [{ ...BATCH, channel: 'SMS' }, BATCH], + }) + + // No SMS queue was given, so the first batch throws; the second is still requeued. + expect(await service.reconcile()).toEqual({ requeued: 1 }) + expect(queue.add).toHaveBeenCalledTimes(1) + expect(redis.eval).toHaveBeenCalled() + }) +}) diff --git a/backend/src/queue/services/batch-reconciler.service.ts b/backend/src/queue/services/batch-reconciler.service.ts new file mode 100644 index 000000000..1acf12812 --- /dev/null +++ b/backend/src/queue/services/batch-reconciler.service.ts @@ -0,0 +1,270 @@ +import { + Inject, + Injectable, + Logger, + OnApplicationBootstrap, + OnModuleDestroy, + Optional, +} from '@nestjs/common' +import { InjectRepository } from '@nestjs/typeorm' +import { In, Repository } from 'typeorm' +import { randomUUID } from 'crypto' +import type Bull from 'bull' +import type Redis from 'ioredis' +import { NotificationRequestDetail } from '../../api/notification/entities/notification-request-detail.entity' +import { NotificationRequest } from '../../api/notification/entities/notification-request.entity' +import { NotificationService } from '../../api/notification/notification.service' +import { NotificationChannel } from '../../enum/notification-channel.enum' +import { NotificationStatus } from '../../enum/notification-status.enum' +import { ProviderToken } from '../../enum/provider-token.enum' +import { QueueName } from '../../enum/queue-name.enum' +import { redisKey } from '../../common/redis/redis-namespace' +import { FAILED_JOB_RETENTION } from '../job-retention' +import { mergeContentFromRequest } from '../merge-batch-builder' +import type { DeliveryJobPayload } from '../queue.types' + +const intFromEnv = (name: string, fallback: number) => { + const value = parseInt(process.env[name] || '', 10) + return Number.isFinite(value) ? value : fallback +} + +const RECONCILE_INTERVAL_MS = intFromEnv('BATCH_RECONCILE_INTERVAL_MS', 60_000) +/** How long a recipient may stay in flight before its batch is checked for a live job. */ +const STALE_AFTER_MS = intFromEnv('BATCH_RECONCILE_STALE_MS', 10 * 60_000) +/** Recovery attempts per batch before its remaining recipients are marked failed. */ +const MAX_RECOVERY_ATTEMPTS = intFromEnv('BATCH_RECONCILE_MAX_ATTEMPTS', 5) +const BATCHES_PER_PASS = 50 + +const LOCK_KEY = redisKey('batch-reconcile:lock') +const attemptsKey = (batchId: string) => redisKey(`batch-reconcile:attempts:${batchId}`) +const ATTEMPTS_TTL_SECONDS = 7 * 24 * 60 * 60 + +/** Release only our own lock; see PendingNotificationRetryService for why. */ +const RELEASE_LOCK_SCRIPT = ` +if redis.call("get", KEYS[1]) == ARGV[1] then + return redis.call("del", KEYS[1]) +end +return 0 +` + +const IN_FLIGHT_STATUSES = ['pending', 'queued', 'processing', 'sending'] +/** Bull states in which the batch's job will still run without help. */ +const LIVE_JOB_STATES = new Set(['waiting', 'active', 'delayed', 'paused']) + +export type ReconcileOutcome = 'in-progress' | 'retried' | 'requeued' | 'gave-up' + +interface StaleBatch { + batchId: string + notifyId: string + tenantId: string + channel: string +} + +/** + * Finds merge batches whose recipients are still in flight with no job left to send them, and + * puts them back on the queue. + * + * The queue's own recovery covers a worker dying mid-job: Bull notices the lapsed lock and + * retries. It does not cover a job that ends up failed (out of retries, or stalled more often + * than maxStalledCount) or one that never reached Redis; its recipients would stay pending + * forever. Postgres is the record of what is owed, so this compares it with what the queue + * holds: + * - a live job (waiting, active, delayed) is left alone - a long batch is not stuck; + * - a failed job is retried with its original payload; + * - a missing or finished job is rebuilt from the stored request and queued again. + * Merge workers skip recipients already sent, so any of these is safe to repeat. After + * MAX_RECOVERY_ATTEMPTS a batch's remaining recipients are marked failed, so a send that can + * never succeed (a deleted template, a rejected sender) ends visibly instead of looping. + * + * One pod at a time, behind a Redis lock, every RECONCILE_INTERVAL_MS. + */ +@Injectable() +export class BatchReconcilerService implements OnApplicationBootstrap, OnModuleDestroy { + private readonly logger = new Logger(BatchReconcilerService.name) + private readonly instanceId = randomUUID() + private timer?: NodeJS.Timeout + + constructor( + @InjectRepository(NotificationRequestDetail) + private readonly detailRepository: Repository, + @InjectRepository(NotificationRequest) + private readonly requestRepository: Repository, + private readonly notificationService: NotificationService, + @Optional() @Inject(ProviderToken.REDIS_CLIENT) private readonly redis?: Redis | null, + @Optional() @Inject(QueueName.EMAIL_DELIVERY) private readonly emailQueue?: Bull.Queue | null, + @Optional() @Inject(QueueName.SMS_DELIVERY) private readonly smsQueue?: Bull.Queue | null, + ) {} + + onApplicationBootstrap(): void { + if (!this.redis || !this.emailQueue) return + this.timer = setInterval(() => void this.reconcile(), RECONCILE_INTERVAL_MS) + this.timer.unref() + } + + onModuleDestroy(): void { + if (this.timer) clearInterval(this.timer) + } + + /** One pass. Never throws: a failed pass is logged and the next interval tries again. */ + async reconcile(now = Date.now()): Promise>> { + const outcomes: Partial> = {} + if (!this.redis) return outcomes + + try { + const locked = await this.redis.set( + LOCK_KEY, + this.instanceId, + 'PX', + RECONCILE_INTERVAL_MS, + 'NX', + ) + if (locked !== 'OK') return outcomes + } catch (error) { + this.logger.warn(`Could not take the reconcile lock: ${(error as Error).message}`) + return outcomes + } + + try { + for (const batch of await this.findStaleBatches(now)) { + try { + const outcome = await this.reconcileBatch(batch) + outcomes[outcome] = (outcomes[outcome] ?? 0) + 1 + } catch (error) { + this.logger.error( + `[${batch.notifyId}] Could not reconcile batch ${batch.batchId}: ${(error as Error).message}`, + ) + } + } + const acted = (outcomes.retried ?? 0) + (outcomes.requeued ?? 0) + (outcomes['gave-up'] ?? 0) + if (acted > 0) this.logger.warn(`Batch reconcile: ${JSON.stringify(outcomes)}`) + } catch (error) { + this.logger.error(`Batch reconcile pass failed: ${(error as Error).message}`) + } finally { + await this.redis + .eval(RELEASE_LOCK_SCRIPT, 1, LOCK_KEY, this.instanceId) + .catch(() => undefined) + } + return outcomes + } + + /** Merge batches with a recipient in flight longer than STALE_AFTER_MS; scheduled sends excluded. */ + private findStaleBatches(now: number): Promise { + return this.detailRepository + .createQueryBuilder('detail') + .innerJoin('detail.notificationRequest', 'request') + .select('detail.batch_id', 'batchId') + .addSelect('detail.notification_request_id', 'notifyId') + .addSelect('request.tenant_id', 'tenantId') + .addSelect('detail.channel', 'channel') + .where('detail.batch_id IS NOT NULL') + .andWhere('detail.status IN (:...statuses)', { statuses: IN_FLIGHT_STATUSES }) + .andWhere('detail.last_attempt_at < :staleBefore', { + staleBefore: new Date(now - STALE_AFTER_MS), + }) + .andWhere('request.status <> :scheduled', { scheduled: NotificationStatus.SCHEDULED }) + .groupBy('detail.batch_id') + .addGroupBy('detail.notification_request_id') + .addGroupBy('request.tenant_id') + .addGroupBy('detail.channel') + .orderBy('MIN(detail.last_attempt_at)', 'ASC') + .limit(BATCHES_PER_PASS) + .getRawMany() + } + + async reconcileBatch(batch: StaleBatch): Promise { + const queue = batch.channel === NotificationChannel.SMS ? this.smsQueue : this.emailQueue + if (!queue || !this.redis) throw new Error(`No queue for channel ${batch.channel}`) + + const job = await queue.getJob(batch.batchId) + const state = job ? await job.getState() : null + if (state && LIVE_JOB_STATES.has(state)) return 'in-progress' + + const key = attemptsKey(batch.batchId) + const attempt = await this.redis.incr(key) + await this.redis.expire(key, ATTEMPTS_TTL_SECONDS) + if (attempt > MAX_RECOVERY_ATTEMPTS) { + await this.giveUp(batch, attempt - 1) + return 'gave-up' + } + + if (job && state === 'failed') { + this.logger.warn( + `[${batch.notifyId}] Retrying failed batch ${batch.batchId} (recovery ${attempt}/${MAX_RECOVERY_ATTEMPTS}): ${job.failedReason ?? 'no reason'}`, + ) + await job.retry() + return 'retried' + } + + // Finished or gone while recipients are still owed: replace it with one built from Postgres. + if (job) await job.remove().catch(() => undefined) + await queue.add(await this.rebuildJob(batch), { + jobId: batch.batchId, + attempts: 3, + backoff: { type: 'exponential', delay: 2000 }, + removeOnComplete: true, + removeOnFail: FAILED_JOB_RETENTION, + }) + this.logger.warn( + `[${batch.notifyId}] Re-queued batch ${batch.batchId} with no job (recovery ${attempt}/${MAX_RECOVERY_ATTEMPTS})`, + ) + return 'requeued' + } + + /** The batch's job as ingestion queued it: shared content and params, recipients left in Postgres. */ + private async rebuildJob(batch: StaleBatch): Promise { + const request = await this.requestRepository.findOne({ + where: { id: batch.notifyId }, + select: { id: true, payload: true }, + }) + if (!request) throw new Error('Parent notification request not found') + + const channel = batch.channel as NotificationChannel + const mailMergeData = mergeContentFromRequest(request.payload, channel) + return { + notifyId: batch.notifyId, + tenantId: batch.tenantId, + channel, + request: { templateId: mailMergeData.content?.templateId } as DeliveryJobPayload['request'], + payload: {} as DeliveryJobPayload['payload'], + attempt: 0, + mailMerge: true, + batchId: batch.batchId, + mailMergeData, + } + } + + /** + * Stop recovering a batch that keeps failing: mark what it still owes failed and settle the + * parent's status the way a finishing batch would, so the send ends visibly. + */ + private async giveUp(batch: StaleBatch, attempts: number): Promise { + await this.detailRepository.update( + { + notificationRequestId: batch.notifyId, + batchId: batch.batchId, + status: In(IN_FLIGHT_STATUSES), + }, + { + status: 'failed', + errorMessage: `Not delivered: the batch failed ${attempts} recovery attempt(s)`, + lastAttemptAt: new Date(), + updatedBy: 'batch-reconciler', + }, + ) + this.logger.error( + `[${batch.notifyId}] Gave up on batch ${batch.batchId} after ${attempts} recovery attempt(s); remaining recipients marked failed`, + ) + + const remaining = await this.detailRepository.count({ + where: { notificationRequestId: batch.notifyId, status: In(IN_FLIGHT_STATUSES) }, + }) + if (remaining > 0) return + const sent = await this.detailRepository.count({ + where: { notificationRequestId: batch.notifyId, status: 'sent' }, + }) + await this.notificationService.update(batch.notifyId, batch.tenantId, { + status: sent > 0 ? NotificationStatus.PARTIALLY_COMPLETED : NotificationStatus.FAILED, + updatedBy: 'batch-reconciler', + }) + } +} diff --git a/backend/src/queue/services/pending-notification-retry.service.spec.ts b/backend/src/queue/services/pending-notification-retry.service.spec.ts index e4d19d525..fb0147359 100644 --- a/backend/src/queue/services/pending-notification-retry.service.spec.ts +++ b/backend/src/queue/services/pending-notification-retry.service.spec.ts @@ -18,7 +18,12 @@ describe('PendingNotificationRetryService', () => { beforeEach(async () => { mockRepository = { find: vi.fn() } - mockNotificationService = { update: vi.fn() } + mockNotificationService = { + update: vi.fn(), + parseMailMergeRecipients: vi.fn((rows: string[][]) => + rows.slice(1).map(([address, firstName]) => ({ address, params: { firstName } })), + ), + } // The sweep takes a lock through the queue's own Redis connection; 'OK' means acquired. mockRedisClient = { set: vi.fn().mockResolvedValue('OK'), @@ -153,12 +158,60 @@ describe('PendingNotificationRetryService', () => { mockNotificationService.update.mockResolvedValue({}) await service.retryPendingNotifications() expect(mockQueue.add).toHaveBeenCalledWith( - 'process', expect.objectContaining({ request: {} }), expect.any(Object), ) }) + it('queues the job unnamed, so the ingestion worker has a handler for it', async () => { + // A named job ('process') matches no handler - the worker registers Bull's default one - + // and fails the moment it is picked up. + mockRepository.find.mockResolvedValue([ + { id: 'notify-1', tenantId: 'tenant-1', payload: {}, createdAt: new Date() }, + ]) + + await service.retryPendingNotifications() + + expect(typeof mockQueue.add.mock.calls[0][0]).toBe('object') + }) + + const storedMerge = { + id: 'notify-merge', + tenantId: 'tenant-1', + createdAt: new Date('2026-10-04T12:00:00Z'), + payload: { + params: { org: 'BC' }, + email: { + content: { subject: 'Hi {{firstName}}', body: 'Body', bodyType: 'text' }, + params: { team: 'Notify' }, + recipients: { + mergeArray: [ + ['email', 'firstName'], + ['a@example.com', 'Ann'], + ['blocked@example.com', 'Bo'], + ], + }, + }, + }, + } + + it('re-queues a merge as a merge, leaving its recipients for ingestion to read', async () => { + mockRepository.find.mockResolvedValue([storedMerge]) + + await service.retryPendingNotifications() + + const [payload] = mockQueue.add.mock.calls[0] + expect(payload.mailMergeData).toEqual({ + content: { subject: 'Hi {{firstName}}', body: 'Body', bodyType: 'text' }, + params: { org: 'BC', team: 'Notify' }, + }) + expect(payload).toMatchObject({ + notifyId: 'notify-merge', + mailMerge: true, + mailMergeChannel: 'EMAIL', + }) + }) + it('should log stats', async () => { const notifications = [ { @@ -296,7 +349,7 @@ describe('PendingNotificationRetryService', () => { mockQueue.add.mockResolvedValue({ id: 'job-123' }) mockNotificationService.update.mockResolvedValue({}) await service.retryPendingNotifications() - const jobConfig = mockQueue.add.mock.calls[0][2] + const jobConfig = mockQueue.add.mock.calls[0][1] expect(jobConfig.attempts).toBe(3) expect(jobConfig.backoff).toEqual({ type: 'exponential', delay: 2000 }) // Bounded by age and count: an unbounded completed set eventually fills a Redis that @@ -319,7 +372,7 @@ describe('PendingNotificationRetryService', () => { mockQueue.add.mockResolvedValue({ id: 'job-123' }) mockNotificationService.update.mockResolvedValue({}) await service.retryPendingNotifications() - const jobConfig = mockQueue.add.mock.calls[0][2] + const jobConfig = mockQueue.add.mock.calls[0][1] expect(jobConfig.jobId).toBe('my-unique-id') }) @@ -337,7 +390,7 @@ describe('PendingNotificationRetryService', () => { mockQueue.add.mockResolvedValue({ id: 'job-123' }) mockNotificationService.update.mockResolvedValue({}) await service.retryPendingNotifications() - const jobData = mockQueue.add.mock.calls[0][1] + const jobData = mockQueue.add.mock.calls[0][0] expect(jobData).toHaveProperty('notifyId', 'notify-1') expect(jobData).toHaveProperty('tenantId', 'tenant-1') expect(jobData).toHaveProperty('request') diff --git a/backend/src/queue/services/pending-notification-retry.service.ts b/backend/src/queue/services/pending-notification-retry.service.ts index 47d15fd59..e28ea209a 100644 --- a/backend/src/queue/services/pending-notification-retry.service.ts +++ b/backend/src/queue/services/pending-notification-retry.service.ts @@ -4,6 +4,8 @@ import { Repository } from 'typeorm' import { InjectRepository } from '@nestjs/typeorm' import Bull from 'bull' import { NotificationRequest } from '../../api/notification/entities/notification-request.entity' +import { mergeContentFromRequest, mergeRequestChannel } from '../merge-batch-builder' +import type { IngestionJobPayload } from '../queue.types' import { NotificationService } from '../../api/notification/notification.service' import { NotificationStatus } from '../../enum/notification-status.enum' import { QueueName } from '../../enum/queue-name.enum' @@ -116,26 +118,18 @@ export class PendingNotificationRetryService implements OnApplicationBootstrap, queueName: this.ingestionQueue?.name, }) - // Try to add to queue with stored payload - await this.ingestionQueue.add( - 'process', - { - notifyId: notification.id, - tenantId: notification.tenantId, - request: notification.payload || {}, // Use stored payload - requestedAt: notification.createdAt.toISOString(), + // Unnamed, like every other add: the ingestion worker registers Bull's default + // handler, and a named job finds no handler and fails as soon as it is picked up. + await this.ingestionQueue.add(this.ingestionPayloadFor(notification), { + jobId: notification.id, + attempts: 3, + backoff: { + type: 'exponential', + delay: 2000, }, - { - jobId: notification.id, - attempts: 3, - backoff: { - type: 'exponential', - delay: 2000, - }, - removeOnComplete: COMPLETED_JOB_RETENTION, - removeOnFail: FAILED_JOB_RETENTION, - }, - ) + removeOnComplete: COMPLETED_JOB_RETENTION, + removeOnFail: FAILED_JOB_RETENTION, + }) // Update status to QUEUED await this.notificationService.update(notification.id, notification.tenantId, { @@ -167,6 +161,31 @@ export class PendingNotificationRetryService implements OnApplicationBootstrap, } } + /** + * The ingestion job the request would have been queued with when it was accepted. A merge + * needs its fan-out fields, or ingestion treats it as a plain send; its recipients are read + * by ingestion from the stored request, as for any merge. + */ + private ingestionPayloadFor(notification: NotificationRequest): IngestionJobPayload { + const base = { + notifyId: notification.id, + tenantId: notification.tenantId, + request: (notification.payload || {}) as IngestionJobPayload['request'], + requestedAt: notification.createdAt.toISOString(), + } + const channel = mergeRequestChannel(notification.payload) + if (!channel) return base + + const mailMergeData = mergeContentFromRequest(notification.payload, channel) + return { + ...base, + request: { templateId: mailMergeData.content?.templateId } as IngestionJobPayload['request'], + mailMerge: true, + mailMergeChannel: channel, + mailMergeData, + } + } + /** * Claim the sweep for this pod. The TTL is a crash guard: the lock is released in a finally * block on the happy path, and expires on its own if the holder dies mid-sweep. diff --git a/backend/src/queue/workers/email-delivery.worker.spec.ts b/backend/src/queue/workers/email-delivery.worker.spec.ts index 985e9614c..26aa3286e 100644 --- a/backend/src/queue/workers/email-delivery.worker.spec.ts +++ b/backend/src/queue/workers/email-delivery.worker.spec.ts @@ -41,6 +41,7 @@ describe('EmailDeliveryWorker', () => { markRecipientFailed: vi.fn().mockResolvedValue(undefined), countByStatus: vi.fn().mockResolvedValue(0), findSentAddresses: vi.fn().mockResolvedValue(new Set()), + findBatchRecipients: vi.fn().mockResolvedValue([]), } // Mock the notification service @@ -1559,6 +1560,29 @@ describe('EmailDeliveryWorker', () => { ]) }) + it("reads a batch's recipients and their params from its rows when the job carries none", async () => { + mockRequestDetailService.findBatchRecipients.mockResolvedValue([ + { address: 'alice@example.com', params: { name: 'Alice' }, status: 'sent' }, + { address: 'bob@example.com', params: { name: 'Bob' }, status: 'pending' }, + ]) + const job = makeBulkJob([]) + delete (job.data!.mailMergeData as { recipients?: unknown }).recipients + + const result = await processHandler(job as Bull.Job) + + expect(mockRequestDetailService.findBatchRecipients).toHaveBeenCalledWith( + 'notify-bulk', + 'notify-bulk-EMAIL-0', + ) + // Alice was sent by an earlier attempt; only Bob goes out, rendered with his own params. + expect(mockEmailAdapter.send).toHaveBeenCalledTimes(1) + expect(mockTemplatesService.renderTemplateContent).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ name: 'Bob' }), + ) + expect(result).toMatchObject({ success: true, sent: 1, failed: 0 }) + }) + it('should render inline content per recipient when no templateId is given', async () => { mockInlineRenderingService.renderEmail.mockResolvedValue({ subject: 'Hi', diff --git a/backend/src/queue/workers/email-delivery.worker.ts b/backend/src/queue/workers/email-delivery.worker.ts index 9fe7c73c2..ee457db90 100644 --- a/backend/src/queue/workers/email-delivery.worker.ts +++ b/backend/src/queue/workers/email-delivery.worker.ts @@ -3,6 +3,7 @@ import Bull from 'bull' import { ConfigService } from '@nestjs/config' import { DeliveryJobPayload, MailMergeJobData } from '../queue.types' import { batchProgressReporter, type BatchProgressReporter } from '../batch-progress' +import { loadBatchRecipients } from './merge-batch-recipients' import { NotificationService } from '../../api/notification/notification.service' import { NotificationRequestDetailService } from '../../api/notification/notification-request-detail.service' import { TemplatesRepository } from '../../api/templates/templates.repository' @@ -481,7 +482,13 @@ export class EmailDeliveryWorker { fromAddress: string | null, reportProgress?: BatchProgressReporter, ): Promise<{ success: boolean; batchId: string; sent: number; failed: number }> { - const { content, params, recipients } = mailMergeData + const { content, params } = mailMergeData + const { recipients, alreadySent } = await loadBatchRecipients( + requestDetailService, + notifyId, + batchId, + mailMergeData.recipients, + ) const templateId = content?.templateId const hasInlineContent = !!(content && (content.subject || content.body)) @@ -516,7 +523,6 @@ export class EmailDeliveryWorker { // A retried batch (a pod replaced mid-send, a crash, a provider error) starts again from // the top. Addresses an earlier attempt delivered keep their rows and are skipped, or they // would get the email twice. Mirrors SmsDeliveryWorker.processMergeBatch. - const alreadySent = await requestDetailService.findSentAddresses(notifyId, batchId) const pending = alreadySent.size ? recipients.filter((recipient) => !alreadySent.has(recipient.address)) : recipients diff --git a/backend/src/queue/workers/ingestion.worker.spec.ts b/backend/src/queue/workers/ingestion.worker.spec.ts index 42e859de1..04d42fce6 100644 --- a/backend/src/queue/workers/ingestion.worker.spec.ts +++ b/backend/src/queue/workers/ingestion.worker.spec.ts @@ -1,6 +1,6 @@ import { Logger } from '@nestjs/common' import Bull from 'bull' -import { vi, describe, it, expect, beforeEach, afterEach } from 'vitest' +import { vi, describe, it, expect, beforeEach } from 'vitest' import { IngestionWorker } from './ingestion.worker' import { IngestionJobPayload, DeliveryJobPayload } from '../queue.types' import { NotificationChannel } from '../../enum/notification-channel.enum' @@ -29,6 +29,8 @@ describe('IngestionWorker', () => { createPending: vi.fn().mockResolvedValue(undefined), createMergePending: vi.fn().mockResolvedValue(undefined), updateStatus: vi.fn().mockResolvedValue(undefined), + countBatch: vi.fn().mockResolvedValue(0), + findAddressesByStatus: vi.fn().mockResolvedValue(new Set()), } mockConfigService = { @@ -828,10 +830,11 @@ describe('IngestionWorker', () => { expect(result).toEqual({ success: true, deliveryJobsQueued: 1 }) expect(mockRequestDetailService.createMergePending).toHaveBeenCalledTimes(1) + // Each recipient's params go on its row; the worker renders from there. expect(mockRequestDetailService.createMergePending).toHaveBeenCalledWith( 'notify-bulk', 'notify-bulk-EMAIL-0', - ['alice@example.com', 'bob@example.com'], + recipients, 'EMAIL', 'tenant-bulk', ) @@ -843,7 +846,7 @@ describe('IngestionWorker', () => { channel: NotificationChannel.EMAIL, mailMerge: true, batchId: 'notify-bulk-EMAIL-0', - mailMergeData: expect.objectContaining({ recipients }), + mailMergeData: { params: { key: 'val' } }, }), expect.objectContaining({ jobId: 'notify-bulk-EMAIL-0', @@ -903,35 +906,31 @@ describe('IngestionWorker', () => { expect(result).toEqual({ success: true, deliveryJobsQueued: 2 }) expect(mockRequestDetailService.createMergePending).toHaveBeenCalledTimes(2) expect(mockEmailQueue.add).toHaveBeenCalledTimes(2) - expect(mockEmailQueue.add).toHaveBeenCalledWith( - expect.objectContaining({ - channel: NotificationChannel.EMAIL, - mailMerge: true, - batchId: 'notify-bulk-multi-EMAIL-0', - mailMergeData: expect.objectContaining({ - recipients: [ - { address: 'a@example.com', params: {} }, - { address: 'b@example.com', params: {} }, - ], - }), - }), - expect.anything(), + expect(mockRequestDetailService.createMergePending).toHaveBeenCalledWith( + 'notify-bulk-multi', + 'notify-bulk-multi-EMAIL-0', + [ + { address: 'a@example.com', params: {} }, + { address: 'b@example.com', params: {} }, + ], + 'EMAIL', + 'tenant-bulk', ) - expect(mockEmailQueue.add).toHaveBeenCalledWith( - expect.objectContaining({ - channel: NotificationChannel.EMAIL, - mailMerge: true, - batchId: 'notify-bulk-multi-EMAIL-1', - mailMergeData: expect.objectContaining({ - recipients: [{ address: 'c@example.com', params: {} }], - }), - }), - expect.anything(), + expect(mockRequestDetailService.createMergePending).toHaveBeenCalledWith( + 'notify-bulk-multi', + 'notify-bulk-multi-EMAIL-1', + [{ address: 'c@example.com', params: {} }], + 'EMAIL', + 'tenant-bulk', ) + // Jobs carry the batch id, not its recipients. + for (const [payload] of mockEmailQueue.add.mock.calls) { + expect(payload.mailMergeData).not.toHaveProperty('recipients') + } }) - it('should default to batchSize 100 when not configured', async () => { - // mockConfigService.get returns undefined for queue.batchSize → defaults to 100 + it('should default to batchSize 25 when not configured', async () => { + // mockConfigService.get returns undefined for queue.batchSize → defaults to 25 const recipients = Array.from({ length: 150 }, (_, i) => ({ address: `user${i}@example.com`, params: {}, @@ -964,8 +963,114 @@ describe('IngestionWorker', () => { const result = await processHandler(job as Bull.Job) - // 150 addresses with default batchSize=100 → 2 batches - expect(result).toEqual({ success: true, deliveryJobsQueued: 2 }) + // 150 addresses with default batchSize=25 → 6 batches + expect(result).toEqual({ success: true, deliveryJobsQueued: 6 }) + expect(mockEmailQueue.add).toHaveBeenCalledTimes(6) + }) + + it('reads recipients from the stored request when the job carries none, less blocked ones', async () => { + mockNotificationService.findOne = vi.fn().mockResolvedValue({ + payload: { + email: { + content: { templateId: 'template-uuid' }, + recipients: { + mergeArray: [ + ['email', 'name'], + ['a@example.com', 'Ann'], + ['blocked@example.com', 'Bo'], + ], + }, + }, + }, + }) + mockNotificationService.parseMailMergeRecipients = vi.fn((rows: string[][]) => + rows.slice(1).map(([address, name]) => ({ address, params: { name } })), + ) + mockRequestDetailService.findAddressesByStatus.mockResolvedValue( + new Set(['blocked@example.com']), + ) + + await IngestionWorker.initialize( + mockIngestionQueue as Bull.Queue, + mockEmailQueue as Bull.Queue, + mockSmsQueue as Bull.Queue, + mockNotificationService, + mockRequestDetailService, + mockConfigService, + mockClamavService, + ) + + const result = await processHandler({ + data: { + notifyId: 'notify-db', + tenantId: 'tenant-bulk', + request: {} as any, + requestedAt: new Date().toISOString(), + mailMerge: true, + mailMergeData: { content: { templateId: 'template-uuid' }, params: {} }, + }, + } as Bull.Job) + + expect(result).toEqual({ success: true, deliveryJobsQueued: 1 }) + expect(mockNotificationService.findOne).toHaveBeenCalledWith('notify-db', 'tenant-bulk') + expect(mockRequestDetailService.findAddressesByStatus).toHaveBeenCalledWith( + 'notify-db', + 'blocked', + ) + expect(mockRequestDetailService.createMergePending).toHaveBeenCalledWith( + 'notify-db', + 'notify-db-EMAIL-0', + [{ address: 'a@example.com', params: { name: 'Ann' } }], + 'EMAIL', + 'tenant-bulk', + ) + }) + + it('does not write rows again for a batch an earlier run of the job created', async () => { + // An ingestion retry re-runs the fan-out; duplicate rows would send twice. + mockRequestDetailService.countBatch.mockResolvedValueOnce(2).mockResolvedValue(0) + mockConfigService.get.mockImplementation((key: string) => + key === 'queue.batchSize' ? 2 : undefined, + ) + + await IngestionWorker.initialize( + mockIngestionQueue as Bull.Queue, + mockEmailQueue as Bull.Queue, + mockSmsQueue as Bull.Queue, + mockNotificationService, + mockRequestDetailService, + mockConfigService, + mockClamavService, + ) + + await processHandler({ + data: { + notifyId: 'notify-retry', + tenantId: 'tenant-bulk', + request: {} as any, + requestedAt: new Date().toISOString(), + mailMerge: true, + mailMergeData: { + params: {}, + recipients: [ + { address: 'a@example.com', params: {} }, + { address: 'b@example.com', params: {} }, + { address: 'c@example.com', params: {} }, + ], + }, + }, + } as Bull.Job) + + // Batch 0 already had rows; only batch 1 is written. Both are (re)queued - Bull ignores a + // jobId it already holds. + expect(mockRequestDetailService.createMergePending).toHaveBeenCalledTimes(1) + expect(mockRequestDetailService.createMergePending).toHaveBeenCalledWith( + 'notify-retry', + 'notify-retry-EMAIL-1', + [{ address: 'c@example.com', params: {} }], + 'EMAIL', + 'tenant-bulk', + ) expect(mockEmailQueue.add).toHaveBeenCalledTimes(2) }) }) diff --git a/backend/src/queue/workers/ingestion.worker.ts b/backend/src/queue/workers/ingestion.worker.ts index f11fb75df..caaef196f 100644 --- a/backend/src/queue/workers/ingestion.worker.ts +++ b/backend/src/queue/workers/ingestion.worker.ts @@ -10,6 +10,7 @@ import { ClamavService } from '../../services/clamav.service' import { QuarantineDetails } from '../../api/notification/entities/notification-request.entity' import { AttachmentService } from '../../api/attachment/attachment.service' import { PhoneNumberService } from '../../api/notify/services/phone-number.service' +import { mergeJobDataFromRequest } from '../merge-batch-builder' import { FAILED_JOB_RETENTION } from '../job-retention' /** @@ -27,6 +28,31 @@ import { FAILED_JOB_RETENTION } from '../job-retention' export class IngestionWorker { private readonly logger = new Logger(IngestionWorker.name) + /** + * Every recipient of a merge, from the request as stored at acceptance, less those the + * safelist blocked then (recorded as `blocked` rows). Read here rather than carried on the + * job, so a large merge is not held in Redis. + */ + static async loadMergeRecipients( + notifyId: string, + tenantId: string, + channel: NotificationChannel, + notificationService: NotificationService, + requestDetailService: NotificationRequestDetailService, + ): Promise }>> { + const [stored, blocked] = await Promise.all([ + notificationService.findOne(notifyId, tenantId), + requestDetailService.findAddressesByStatus(notifyId, 'blocked'), + ]) + return mergeJobDataFromRequest( + stored.payload, + channel, + (mergeArray, mergeChannel) => + notificationService.parseMailMergeRecipients(mergeArray, mergeChannel), + (address) => !blocked.has(address), + ).recipients! + } + private static normalizeSmsRecipients( request: IngestionJobPayload['request'], phoneNumberService: PhoneNumberService, @@ -119,10 +145,19 @@ export class IngestionWorker { // Mail merge email send: split recipients into fixed-size batches and fan out one // email-delivery job per batch. Detail rows are created here, tagged with a batchId. if (job.data.mailMerge && job.data.mailMergeData) { - const { content, params, recipients } = job.data.mailMergeData - const batchSize = configService?.get('queue.batchSize') || 100 + const { content, params } = job.data.mailMergeData + const batchSize = configService?.get('queue.batchSize') || 25 // Older jobs queued before SMS merge existed carry no channel and are email. const mergeChannel = job.data.mailMergeChannel ?? NotificationChannel.EMAIL + const recipients = + job.data.mailMergeData.recipients ?? + (await IngestionWorker.loadMergeRecipients( + notifyId, + tenantId, + mergeChannel, + notificationService, + requestDetailService, + )) const isSmsMerge = mergeChannel === NotificationChannel.SMS const mergeQueue = isSmsMerge ? smsQueue : emailQueue @@ -137,13 +172,17 @@ export class IngestionWorker { // so a failed batch is easy to identify and retry. const batchId = `${notifyId}-${mergeChannel}-${batchIndex}` - await requestDetailService.createMergePending( - notifyId, - batchId, - chunk.map((r) => r.address), - mergeChannel, - tenantId, - ) + // A retried ingestion job re-runs this loop. Rows already written for a batch mean an + // earlier run created it; writing them again would send those recipients twice. + if ((await requestDetailService.countBatch(notifyId, batchId)) === 0) { + await requestDetailService.createMergePending( + notifyId, + batchId, + chunk, + mergeChannel, + tenantId, + ) + } const deliveryPayload: DeliveryJobPayload = { notifyId, @@ -154,7 +193,8 @@ export class IngestionWorker { attempt: 0, mailMerge: true, batchId, - mailMergeData: { content, params, recipients: chunk }, + // Recipients stay in Postgres; the worker reads this batch's detail rows. + mailMergeData: { content, params }, } await mergeQueue.add(deliveryPayload, { diff --git a/backend/src/queue/workers/merge-batch-recipients.ts b/backend/src/queue/workers/merge-batch-recipients.ts new file mode 100644 index 000000000..a856bc6ac --- /dev/null +++ b/backend/src/queue/workers/merge-batch-recipients.ts @@ -0,0 +1,33 @@ +import type { NotificationRequestDetailService } from '../../api/notification/notification-request-detail.service' +import type { MailMergeJobData } from '../queue.types' + +export interface BatchRecipients { + recipients: NonNullable + /** Delivered by an earlier attempt at this batch; a retry skips them. */ + alreadySent: Set +} + +/** + * The recipients a merge batch owes, read from its detail rows - written at ingestion with each + * recipient's params - rather than carried on the job. Jobs queued before that change still carry + * their recipients, and those are used as they are. + */ +export async function loadBatchRecipients( + requestDetailService: NotificationRequestDetailService, + notifyId: string, + batchId: string, + carried: MailMergeJobData['recipients'], +): Promise { + if (carried) { + return { + recipients: carried, + alreadySent: await requestDetailService.findSentAddresses(notifyId, batchId), + } + } + + const rows = await requestDetailService.findBatchRecipients(notifyId, batchId) + return { + recipients: rows.map(({ address, params }) => ({ address, params })), + alreadySent: new Set(rows.filter((row) => row.status === 'sent').map((row) => row.address)), + } +} diff --git a/backend/src/queue/workers/sms-delivery.worker.ts b/backend/src/queue/workers/sms-delivery.worker.ts index 5dec206ce..b6ab983e4 100644 --- a/backend/src/queue/workers/sms-delivery.worker.ts +++ b/backend/src/queue/workers/sms-delivery.worker.ts @@ -3,6 +3,7 @@ import Bull from 'bull' import { ConfigService } from '@nestjs/config' import { DeliveryJobPayload, MailMergeJobData } from '../queue.types' import { batchProgressReporter, type BatchProgressReporter } from '../batch-progress' +import { loadBatchRecipients } from './merge-batch-recipients' import { NotificationService } from '../../api/notification/notification.service' import { NotificationRequestDetailService } from '../../api/notification/notification-request-detail.service' import { TemplatesRepository } from '../../api/templates/templates.repository' @@ -409,7 +410,13 @@ export class SmsDeliveryWorker { notificationService: NotificationService, reportProgress?: BatchProgressReporter, ): Promise<{ success: boolean; batchId: string; sent: number; failed: number }> { - const { content, params, recipients } = mailMergeData + const { content, params } = mailMergeData + const { recipients, alreadySent } = await loadBatchRecipients( + requestDetailService, + notifyId, + batchId, + mailMergeData.recipients, + ) const templateId = content?.templateId const hasInlineContent = !!content?.body @@ -442,7 +449,6 @@ export class SmsDeliveryWorker { // The merge branch returns before the retry handling above, so a redelivered batch filters // here instead. Addresses delivered by an earlier attempt keep their rows and are skipped. - const alreadySent = await requestDetailService.findSentAddresses(notifyId, batchId) const pending = alreadySent.size ? recipients.filter((recipient) => !alreadySent.has(recipient.address)) : recipients diff --git a/migrations/sql/V73__notification_request_detail_params.sql b/migrations/sql/V73__notification_request_detail_params.sql new file mode 100644 index 000000000..f3f563bc5 --- /dev/null +++ b/migrations/sql/V73__notification_request_detail_params.sql @@ -0,0 +1,7 @@ +-- Per-recipient personalisation for mail merge sends. Delivery jobs carry only a batch id, and +-- the worker reads each recipient's params from here, so a large merge is not held in Redis. +ALTER TABLE notify.notification_request_detail + ADD COLUMN params JSONB; + +COMMENT ON COLUMN notify.notification_request_detail.params IS + 'Mail merge personalisation for this recipient, merged over the request''s global params when rendering; null for non-merge sends'; From d48d48ab59d3444eef69a90c78d289f565f33014 Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Sun, 4 Oct 2026 15:05:59 -0700 Subject: [PATCH 13/23] Add shutdown diagnostics for nonzero exits Introduce a shutdown diagnostics helper that logs synchronous stderr markers when SIGTERM is received and when the process exits with a non-zero code. It now records the last app error from `StructuredLoggerService` so failed shutdowns include likely cause/context, wires diagnostics into app bootstrap, and adds unit tests covering SIGTERM, non-zero exit reporting, and clean exits. --- backend/src/app.ts | 2 + .../logger/structured-logger.service.ts | 6 ++- .../src/common/shutdown-diagnostics.spec.ts | 41 +++++++++++++++++++ backend/src/common/shutdown-diagnostics.ts | 40 ++++++++++++++++++ 4 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 backend/src/common/shutdown-diagnostics.spec.ts create mode 100644 backend/src/common/shutdown-diagnostics.ts diff --git a/backend/src/app.ts b/backend/src/app.ts index 80296b458..bb64514a9 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -11,6 +11,7 @@ import bodyParser from 'body-parser' import { Router } from 'express' import { ValidationExceptionFilter } from './common/filters/validation.filter' import { JwtGuard } from './common/guards/auth.jwt-guard' +import { installShutdownDiagnostics } from './common/shutdown-diagnostics' import { applyNotifySchemaConstraints } from './api/notify/schemas/schema-constraints' /** @@ -53,6 +54,7 @@ export async function bootstrap() { app.set('trust proxy', 1) app.use(metricsMiddleware) app.enableShutdownHooks() + installShutdownDiagnostics() // Health check at root level (before global prefix) for Kong's health probe const rootRouter = Router() diff --git a/backend/src/common/logger/structured-logger.service.ts b/backend/src/common/logger/structured-logger.service.ts index 10bed02dd..7f883bb20 100644 --- a/backend/src/common/logger/structured-logger.service.ts +++ b/backend/src/common/logger/structured-logger.service.ts @@ -1,6 +1,7 @@ import { Injectable, LoggerService } from '@nestjs/common' import * as winston from 'winston' import LokiTransport from 'winston-loki' +import { recordError } from '../shutdown-diagnostics' export interface LogContext { notificationId?: string @@ -218,10 +219,13 @@ export class StructuredLoggerService implements LoggerService { */ error(message: string, context?: LogContext | string, nestContext?: string) { if (typeof context === 'string') { + recordError({ message, context: nestContext, stack: context }) this.writeLog('error', message, { context: nestContext, stack: context }) return } - this.writeLog('error', message, this.toContext(context)) + const logContext = this.toContext(context) + recordError({ message, context: logContext?.context as string | undefined }) + this.writeLog('error', message, logContext) } /** diff --git a/backend/src/common/shutdown-diagnostics.spec.ts b/backend/src/common/shutdown-diagnostics.spec.ts new file mode 100644 index 000000000..0113be3b1 --- /dev/null +++ b/backend/src/common/shutdown-diagnostics.spec.ts @@ -0,0 +1,41 @@ +import { describe, it, expect, vi } from 'vitest' +import { EventEmitter } from 'events' +import { installShutdownDiagnostics, recordError } from './shutdown-diagnostics' + +function install() { + const target = new EventEmitter() + const write = vi.fn() + installShutdownDiagnostics(target, write) + return { target, write } +} + +describe('installShutdownDiagnostics', () => { + it('notes when SIGTERM arrives', () => { + const { target, write } = install() + target.emit('SIGTERM') + expect(write).toHaveBeenCalledWith(expect.stringMatching(/^\[shutdown\] SIGTERM received at /)) + }) + + it('names the last error logged when the process exits non-zero', () => { + // As Nest does on a failed shutdown: log through the app logger, then process.exit(1). + recordError({ + message: 'Error happened during shutdown', + context: 'NestApplicationContext', + stack: 'Error: Connection is closed.\n at quit (ioredis)', + }) + const { target, write } = install() + + target.emit('exit', 1) + + const line = write.mock.calls[0][0] as string + expect(line).toMatch(/process exiting with code 1/) + expect(line).toMatch(/\[NestApplicationContext\] Error happened during shutdown/) + expect(line).toMatch(/Connection is closed/) + }) + + it('stays quiet on a clean exit', () => { + const { target, write } = install() + target.emit('exit', 0) + expect(write).not.toHaveBeenCalled() + }) +}) diff --git a/backend/src/common/shutdown-diagnostics.ts b/backend/src/common/shutdown-diagnostics.ts new file mode 100644 index 000000000..ca26a41ce --- /dev/null +++ b/backend/src/common/shutdown-diagnostics.ts @@ -0,0 +1,40 @@ +import type { EventEmitter } from 'events' + +/** The most recent error logged, kept so a failed shutdown can name its cause. */ +export interface RecordedError { + message: string + context?: string + stack?: string + at: string +} + +let lastError: RecordedError | undefined + +export function recordError(error: Omit): void { + lastError = { ...error, at: new Date().toISOString() } +} + +/** + * Make a failed shutdown explain itself. + * + * When a shutdown hook throws, Nest logs the error and calls process.exit(1) straight after. The + * log goes through winston, which writes asynchronously, so it never reaches the pod log - the + * pod just disappears a second into its grace period. stderr writes are synchronous, so these + * lines survive an immediate exit: one when SIGTERM arrives, and on a non-zero exit the code + * and the last error logged. + */ +export function installShutdownDiagnostics( + target: Pick = process, + write: (line: string) => void = (line) => process.stderr.write(line), +): void { + target.once('SIGTERM', () => { + write(`[shutdown] SIGTERM received at ${new Date().toISOString()}\n`) + }) + target.on('exit', (code: number) => { + if (code === 0) return + const cause = lastError + ? `last error logged at ${lastError.at}: [${lastError.context ?? 'unknown'}] ${lastError.message}${lastError.stack ? `\n${lastError.stack}` : ''}` + : 'no error was logged' + write(`[shutdown] process exiting with code ${code}; ${cause}\n`) + }) +} From 87888c8c881a0eb61ad233482726fd0d2200e068 Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Sun, 4 Oct 2026 16:36:25 -0700 Subject: [PATCH 14/23] Trigger PR deploy From 080b868758b24a46aa23d62524e12627896de8a5 Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Sun, 4 Oct 2026 16:42:10 -0700 Subject: [PATCH 15/23] Make backend heap and memory request configurable Moves the Node heap limit out of the backend image and into Helm so environments can tune it without rebuilding. The deployment now sets NODE_OPTIONS from `backend.heapMb` (default 512), and backend memory requests are templated via `backend.memoryRequest` (default 200Mi). TEST and PROD deploy params in `merge.yml` now override memory request to 384Mi while keeping autoscaling settings unchanged. --- .github/workflows/merge.yml | 4 ++-- backend/Dockerfile | 5 +++-- charts/app/templates/backend/templates/deployment.yaml | 6 +++++- charts/app/values.yaml | 6 ++++++ 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 308db6aeb..317349a0b 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -213,7 +213,7 @@ jobs: environment: TEST oc_server: ${{ vars.oc_server }} db_user: appproxy # appproxy is the user which works with pgbouncer. - params: --set backend.deploymentStrategy=RollingUpdate --set frontend.deploymentStrategy=RollingUpdate --set global.autoscaling=true --set backend.autoscaling.minReplicas=2 --set backend.autoscaling.maxReplicas=2 --set frontend.autoscaling.minReplicas=2 --set frontend.autoscaling.maxReplicas=2 --set frontend.secrets.enabled=true + params: --set backend.deploymentStrategy=RollingUpdate --set frontend.deploymentStrategy=RollingUpdate --set global.autoscaling=true --set backend.autoscaling.minReplicas=2 --set backend.autoscaling.maxReplicas=2 --set backend.memoryRequest=384Mi --set frontend.autoscaling.minReplicas=2 --set frontend.autoscaling.maxReplicas=2 --set frontend.secrets.enabled=true db_triggers: ('charts/crunchy/') redis_triggers: ('charts/redis/') tag: test @@ -288,7 +288,7 @@ jobs: environment: PROD oc_server: ${{ vars.oc_server }} db_user: appproxy # appproxy is the user which works with pgbouncer. - params: --set backend.deploymentStrategy=RollingUpdate --set frontend.deploymentStrategy=RollingUpdate --set global.autoscaling=true --set backend.autoscaling.minReplicas=5 --set backend.autoscaling.maxReplicas=5 --set frontend.autoscaling.minReplicas=5 --set frontend.autoscaling.maxReplicas=5 --set frontend.pdb.enabled=true --set backend.pdb.enabled=true + params: --set backend.deploymentStrategy=RollingUpdate --set frontend.deploymentStrategy=RollingUpdate --set global.autoscaling=true --set backend.autoscaling.minReplicas=5 --set backend.autoscaling.maxReplicas=5 --set backend.memoryRequest=384Mi --set frontend.autoscaling.minReplicas=5 --set frontend.autoscaling.maxReplicas=5 --set frontend.pdb.enabled=true --set backend.pdb.enabled=true db_triggers: ('charts/crunchy/') redis_triggers: ('charts/redis/') tag: prod diff --git a/backend/Dockerfile b/backend/Dockerfile index bd91258cb..ab6d6ed0c 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -27,6 +27,7 @@ COPY --from=build /app/dist ./dist EXPOSE 3000 HEALTHCHECK --interval=30s --timeout=3s CMD curl -f http://localhost:3000/api -# Nonroot user, limit heap size to 150 MB +# Nonroot user. The heap limit is set per environment by the Helm chart (NODE_OPTIONS from +# backend.heapMb), not baked in here, so it can be raised without rebuilding the image. USER nonroot -CMD ["--max-old-space-size=150", "/app/dist/main"] +CMD ["/app/dist/main"] diff --git a/charts/app/templates/backend/templates/deployment.yaml b/charts/app/templates/backend/templates/deployment.yaml index eef6f150b..6a859e72b 100644 --- a/charts/app/templates/backend/templates/deployment.yaml +++ b/charts/app/templates/backend/templates/deployment.yaml @@ -122,6 +122,10 @@ spec: value: "0" - name: INGESTION_WORKER_CONCURRENCY value: "1" + # V8 heap cap. Exceeding it crashes the whole pod rather than failing one request; a + # 50,000-recipient merge holds ~40 MB+ while it is accepted. Reserves nothing up front. + - name: NODE_OPTIONS + value: {{ printf "--max-old-space-size=%v" (.Values.backend.heapMb | default 512) | quote }} - name: QUEUE_SHUTDOWN_DRAIN_MS value: {{ mul (sub (.Values.backend.terminationGracePeriodSeconds | default 120) 15) 1000 | quote }} - name: NOTIFICATION_DEDUP_WINDOW_SECONDS @@ -268,7 +272,7 @@ spec: resources: # this is optional requests: cpu: 50m - memory: 200Mi + memory: {{ .Values.backend.memoryRequest | default "200Mi" }} limits: cpu: 4 memory: 8Gi diff --git a/charts/app/values.yaml b/charts/app/values.yaml index 1f26d38a5..5dea52445 100644 --- a/charts/app/values.yaml +++ b/charts/app/values.yaml @@ -30,6 +30,12 @@ backend: enabled: true #-- the deployment strategy, can be "Recreate" or "RollingUpdate" deploymentStrategy: Recreate + #-- V8 heap limit in MB, passed as NODE_OPTIONS. Only used memory counts against the pod, so + #-- raising it costs no quota; it must stay well under the container's memory limit. + heapMb: 512 + #-- memory the scheduler reserves for each backend pod. Kept low by default because DEV + #-- hosts every PR environment under one quota; TEST and PROD raise it in merge.yml. + memoryRequest: 200Mi #-- seconds a stopping pod gets to finish in-flight queue jobs before it is killed. With #-- Recreate, a deploy waits this long only while a pod is mid-batch; idle pods exit at once. terminationGracePeriodSeconds: 120 From 5d51dd4a4587d98a8fbed27809c00c382258d29f Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Mon, 5 Oct 2026 09:47:50 -0700 Subject: [PATCH 16/23] Unify delivery recovery and add CHES timeouts Replace the separate pending-notification and batch recovery services with a single DeliveryReconciler that restores stuck pending, ingestion, batch, and plain-delivery work from Postgres. It rebuilds scheduled sends correctly, avoids duplicating detail rows or re-sending finished channels on ingestion reruns, and adds CHES request deadlines so hung token or email calls fail fast with clear 504 errors. --- .../email/ches/ches-email.adapter.spec.ts | 69 ++- .../delivery/email/ches/ches-email.adapter.ts | 32 +- .../notification-request-detail.service.ts | 22 +- .../common/decorators/queueable.decorator.ts | 4 +- backend/src/config/configuration.ts | 4 +- backend/src/queue/job-retention.ts | 7 +- backend/src/queue/merge-batch-builder.spec.ts | 59 ++- backend/src/queue/merge-batch-builder.ts | 46 +- backend/src/queue/queue.module.ts | 6 +- .../services/batch-reconciler.service.spec.ts | 202 --------- .../services/batch-reconciler.service.ts | 270 ----------- .../delivery-reconciler.service.spec.ts | 392 ++++++++++++++++ .../services/delivery-reconciler.service.ts | 424 ++++++++++++++++++ ...pending-notification-retry.service.spec.ts | 400 ----------------- .../pending-notification-retry.service.ts | 253 ----------- .../queue/workers/ingestion.worker.spec.ts | 43 ++ backend/src/queue/workers/ingestion.worker.ts | 27 +- backend/test/app.e2e-spec.ts | 5 - 18 files changed, 1114 insertions(+), 1151 deletions(-) delete mode 100644 backend/src/queue/services/batch-reconciler.service.spec.ts delete mode 100644 backend/src/queue/services/batch-reconciler.service.ts create mode 100644 backend/src/queue/services/delivery-reconciler.service.spec.ts create mode 100644 backend/src/queue/services/delivery-reconciler.service.ts delete mode 100644 backend/src/queue/services/pending-notification-retry.service.spec.ts delete mode 100644 backend/src/queue/services/pending-notification-retry.service.ts diff --git a/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.spec.ts b/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.spec.ts index 541600963..4d83f1c30 100644 --- a/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.spec.ts +++ b/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.spec.ts @@ -1,6 +1,6 @@ import { Test, TestingModule } from '@nestjs/testing' import { ConfigService } from '@nestjs/config' -import { BadGatewayException } from '@nestjs/common' +import { BadGatewayException, GatewayTimeoutException } from '@nestjs/common' import { ChesEmailTransport } from '../../../../../../src/adapters/implementations/delivery/email/ches/ches-email.adapter' import type { SendEmailOptions, SendEmailResult } from '../../../../../../src/adapters/interfaces' @@ -795,4 +795,71 @@ describe('ChesEmailTransport', () => { await expect(transport.send(options)).rejects.toThrow(BadGatewayException) }) }) + + describe('request timeout', () => { + const configured = (timeoutMs?: number) => + configGetMock.mockImplementation((key: string) => { + const map: Record = { + 'ches.baseUrl': 'https://ches.example.com/api/v1', + 'ches.clientId': 'client-id', + 'ches.clientSecret': 'client-secret', + 'ches.tokenUrl': 'https://auth.example.com/token', + 'ches.from': 'noreply@gov.bc.ca', + 'ches.timeoutMs': timeoutMs, + } + return map[key] + }) + const tokenOk = () => + fetchMock.mockResolvedValueOnce({ + ok: true, + json: async () => ({ access_token: 'tok', expires_in: 300 }), + }) + const timedOut = () => + Object.assign(new Error('The operation was aborted due to timeout'), { name: 'TimeoutError' }) + + it('gives every CHES request a deadline', async () => { + configured(15_000) + tokenOk() + fetchMock.mockResolvedValueOnce({ + ok: true, + json: async () => ({ messages: [{ msgId: 'm1' }], txId: 't1' }), + }) + + await transport.send({ to: 'user@example.com', subject: 'Hi', body: 'Hello' }) + + for (const [, init] of fetchMock.mock.calls) { + expect(init.signal).toBeInstanceOf(AbortSignal) + } + }) + + it('reports a CHES that never answers as a 504 naming CHES', async () => { + configured(15_000) + tokenOk() + fetchMock.mockRejectedValueOnce(timedOut()) + + const sending = transport.send({ to: 'user@example.com', subject: 'Hi', body: 'Hello' }) + + await expect(sending).rejects.toBeInstanceOf(GatewayTimeoutException) + await expect(sending).rejects.toThrow('CHES email request timed out after 15s') + }) + + it('times out the token request too', async () => { + configured(15_000) + fetchMock.mockRejectedValueOnce(timedOut()) + + await expect( + transport.send({ to: 'user@example.com', subject: 'Hi', body: 'Hello' }), + ).rejects.toThrow('CHES token request timed out after 15s') + }) + + it('leaves other network errors as they are', async () => { + configured(15_000) + tokenOk() + fetchMock.mockRejectedValueOnce(new TypeError('fetch failed')) + + await expect( + transport.send({ to: 'user@example.com', subject: 'Hi', body: 'Hello' }), + ).rejects.toThrow('fetch failed') + }) + }) }) diff --git a/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.ts b/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.ts index 482bb64f1..26324a7d8 100644 --- a/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.ts +++ b/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.ts @@ -1,6 +1,7 @@ import { BadGatewayException, BadRequestException, + GatewayTimeoutException, Injectable, Logger, NotFoundException, @@ -170,7 +171,7 @@ export class ChesEmailTransport implements IEmailTransport { })}`, ) - const response = await fetch(`${baseUrl.replace(/\/$/, '')}/email`, { + const response = await this.fetchWithTimeout(`${baseUrl.replace(/\/$/, '')}/email`, 'email', { method: 'POST', headers: { Authorization: `Bearer ${token}`, @@ -207,6 +208,33 @@ export class ChesEmailTransport implements IEmailTransport { } } + /** + * fetch with a deadline. A timeout becomes a 504 naming CHES, so it reads as the provider being + * slow - a retryable failure for the job, or one failed recipient in a merge batch - rather than + * a bare AbortError. + */ + private async fetchWithTimeout( + url: string, + operation: 'email' | 'token', + init: RequestInit, + ): Promise { + const timeoutMs = this.configService.get('ches.timeoutMs') ?? 30_000 + try { + return await fetch(url, { ...init, signal: AbortSignal.timeout(timeoutMs) }) + } catch (caught: unknown) { + if ( + caught instanceof Error && + (caught.name === 'TimeoutError' || caught.name === 'AbortError') + ) { + this.logger.error(`[CHES] ${operation} request timed out after ${timeoutMs}ms`) + throw new GatewayTimeoutException( + `CHES ${operation} request timed out after ${timeoutMs / 1000}s`, + ) + } + throw caught + } + } + private async getAccessToken( tokenUrl: string, clientId: string, @@ -223,7 +251,7 @@ export class ChesEmailTransport implements IEmailTransport { client_secret: clientSecret, }) - const response = await fetch(tokenUrl, { + const response = await this.fetchWithTimeout(tokenUrl, 'token', { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: params.toString(), diff --git a/backend/src/api/notification/notification-request-detail.service.ts b/backend/src/api/notification/notification-request-detail.service.ts index d024bc535..29ceb84b9 100644 --- a/backend/src/api/notification/notification-request-detail.service.ts +++ b/backend/src/api/notification/notification-request-detail.service.ts @@ -1,7 +1,7 @@ import { NotificationChannel } from '../../enum/notification-channel.enum' import { Injectable } from '@nestjs/common' import { InjectRepository } from '@nestjs/typeorm' -import { FindOptionsWhere, In, Not, Repository } from 'typeorm' +import { FindOptionsWhere, In, IsNull, Not, Repository } from 'typeorm' import { NotificationRequestDetail } from './entities/notification-request-detail.entity' import { ProcessedNotifySimpleRequest } from '../notify/schemas/stored-notify-attachment' import { NotifySimpleRequest } from '../notify/schemas/notify-simple-request' @@ -42,6 +42,9 @@ export const notificationRequestDetailListQueryConfig: QueryableFieldsConfig = { */ const RETRY_PRESERVED_STATUSES = ['sent', 'blocked'] +/** Detail statuses between acceptance and a final sent/failed. */ +export const IN_FLIGHT_DETAIL_STATUSES = ['pending', 'queued', 'processing', 'sending'] + @Injectable() export class NotificationRequestDetailService { constructor( @@ -151,6 +154,23 @@ export class NotificationRequestDetailService { return this.detailRepository.count({ where: { notificationRequestId, batchId } }) } + /** + * Rows for a request outside any merge batch, ignoring safelist-blocked ones. Non-zero means an + * earlier run of ingestion already wrote them. + */ + async countUnbatched(notificationRequestId: string): Promise { + return this.detailRepository.count({ + where: { notificationRequestId, batchId: IsNull(), status: Not('blocked') }, + }) + } + + /** A request's recipients on one channel that are not yet sent or failed. */ + async countInFlight(notificationRequestId: string, channel: string): Promise { + return this.detailRepository.count({ + where: { notificationRequestId, channel, status: In(IN_FLIGHT_DETAIL_STATUSES) }, + }) + } + /** Addresses of a request's recipients in one status, e.g. those the safelist blocked. */ async findAddressesByStatus(notificationRequestId: string, status: string): Promise> { const rows = await this.detailRepository.find({ diff --git a/backend/src/common/decorators/queueable.decorator.ts b/backend/src/common/decorators/queueable.decorator.ts index 287417048..e7b7b7c18 100644 --- a/backend/src/common/decorators/queueable.decorator.ts +++ b/backend/src/common/decorators/queueable.decorator.ts @@ -876,7 +876,7 @@ export function Queueable( // Fire off queueing asynchronously - don't block the response // If queuing succeeds, status updates to QUEUED - // If queuing fails, PendingNotificationRetryService will pick it up and retry + // If queuing fails, the record stays PENDING and DeliveryReconcilerService queues it setImmediate(async () => { try { const jobPayload = { @@ -933,7 +933,7 @@ export function Queueable( } } catch (queueError) { // Redis unavailable... that's OK, status stays PENDING - // PendingNotificationRetryService will pick it up once Redis is back + // DeliveryReconcilerService queues it once Redis is back logger.warn(`Failed to enqueue job (will be retried): ${notificationRecord.id}`, { tenantId, error: (queueError as Error).message, diff --git a/backend/src/config/configuration.ts b/backend/src/config/configuration.ts index eb6f54545..14aa86d9a 100644 --- a/backend/src/config/configuration.ts +++ b/backend/src/config/configuration.ts @@ -59,6 +59,9 @@ export default () => { clientSecret: process.env.CHES_CLIENT_SECRET, tokenUrl: process.env.CHES_TOKEN_URL, from: process.env.DEFAULT_EMAIL_FROM || defaultEmailFrom, + // Per request. Without one a CHES that accepts the connection but never answers holds a + // worker until Node's own ~5-minute timeout, per message. + timeoutMs: parseInt(process.env.CHES_TIMEOUT_MS || '30000', 10), }, // Notification events @@ -193,7 +196,6 @@ export default () => { ), jobRetries: parseInt(process.env.JOB_RETRIES || '3', 10), jobBackoffDelay: parseInt(process.env.JOB_BACKOFF_DELAY || '2000', 10), - pendingRetryInterval: parseInt(process.env.PENDING_RETRY_INTERVAL || '30000', 10), }, // Identical sends (same tenant, recipients and content) within this window are answered with diff --git a/backend/src/queue/job-retention.ts b/backend/src/queue/job-retention.ts index 583d372ec..3564283c9 100644 --- a/backend/src/queue/job-retention.ts +++ b/backend/src/queue/job-retention.ts @@ -6,10 +6,9 @@ * stops enqueue altogether, not just this queue. Both sets are bounded by age and count. * * The completed window is not an audit log; notification_request and its history table are. - * It exists so that re-adding the same jobId within the window is deduplicated by Bull, which - * is what stops the pending-notification sweep from queueing a request the original request - * path has already queued but not yet marked QUEUED. An hour is far longer than that race - * (the sweep runs every 30s) while still bounding what Redis holds. + * It exists so that re-adding the same jobId within the window is deduplicated by Bull - a + * request queued twice in quick succession (the original path, then a retry of the same add) runs + * once. An hour is far longer than that race while still bounding what Redis holds. */ export interface JobRetention { /** Maximum age in seconds. */ diff --git a/backend/src/queue/merge-batch-builder.spec.ts b/backend/src/queue/merge-batch-builder.spec.ts index 7d9c530b5..3be9dfcaa 100644 --- a/backend/src/queue/merge-batch-builder.spec.ts +++ b/backend/src/queue/merge-batch-builder.spec.ts @@ -1,6 +1,11 @@ import { describe, it, expect } from 'vitest' import { NotificationChannel } from '../enum/notification-channel.enum' -import { mergeJobDataFromRequest, mergeRequestChannel } from './merge-batch-builder' +import { + delayUntilScheduled, + ingestionJobFromRequest, + mergeJobDataFromRequest, + mergeRequestChannel, +} from './merge-batch-builder' const parse = (rows: string[][]) => rows.slice(1).map(([address, name]) => ({ address, params: { name } })) @@ -60,3 +65,55 @@ describe('mergeJobDataFromRequest', () => { ) }) }) + +describe('ingestionJobFromRequest', () => { + const createdAt = new Date('2026-10-05T12:00:00Z') + + it('rebuilds a plain send with its stored request', () => { + const payload = { + email: { recipients: { to: ['a@example.com'] }, content: { subject: 'S', body: 'B' } }, + } + expect(ingestionJobFromRequest({ id: 'r1', tenantId: 't1', payload, createdAt })).toEqual({ + notifyId: 'r1', + tenantId: 't1', + request: payload, + requestedAt: createdAt.toISOString(), + }) + }) + + it('rebuilds a merge with its fan-out fields and no recipients', () => { + const job = ingestionJobFromRequest({ + id: 'r1', + tenantId: 't1', + payload: emailMerge, + createdAt, + }) + expect(job).toMatchObject({ mailMerge: true, mailMergeChannel: NotificationChannel.EMAIL }) + expect(job.mailMergeData).not.toHaveProperty('recipients') + }) + + it('keeps a scheduled send time', () => { + const payload = { + sms: { + recipients: { to: ['+12505550123'] }, + content: { body: 'B' }, + delayedSend: '2026-10-06T09:00:00Z', + }, + } + const job = ingestionJobFromRequest({ id: 'r1', tenantId: 't1', payload, createdAt }) + expect(job.scheduledFor).toBe('2026-10-06T09:00:00Z') + }) +}) + +describe('delayUntilScheduled', () => { + const now = Date.parse('2026-10-05T12:00:00Z') + + it('is the time left until the send', () => { + expect(delayUntilScheduled({ scheduledFor: '2026-10-05T12:10:00Z' }, now)).toBe(10 * 60_000) + }) + + it('is zero when unscheduled or already due', () => { + expect(delayUntilScheduled({}, now)).toBe(0) + expect(delayUntilScheduled({ scheduledFor: '2026-10-05T11:00:00Z' }, now)).toBe(0) + }) +}) diff --git a/backend/src/queue/merge-batch-builder.ts b/backend/src/queue/merge-batch-builder.ts index acf5d673d..9c6243989 100644 --- a/backend/src/queue/merge-batch-builder.ts +++ b/backend/src/queue/merge-batch-builder.ts @@ -1,6 +1,6 @@ import { NotificationChannel } from '../enum/notification-channel.enum' import type { NotifySimpleRequest } from '../api/notify/schemas/notify-simple-request' -import type { MailMergeJobData } from './queue.types' +import type { IngestionJobPayload, MailMergeJobData } from './queue.types' /** NotificationService.parseMailMergeRecipients, passed in so this stays a plain function. */ export type ParseMergeRecipients = ( @@ -67,3 +67,47 @@ export function mergeJobDataFromRequest( recipients: keep ? recipients.filter(({ address }) => keep(address)) : recipients, } } + +/** + * The ingestion job a request would have been queued with when it was accepted, rebuilt from + * notification_request. A merge gets its fan-out fields - without them ingestion treats it as a + * plain send - and leaves its recipients for ingestion to read from the stored request. A + * scheduled send keeps its send time in `scheduledFor`; queue it with that delay (see + * delayUntilScheduled), or it goes out immediately. + */ +export function ingestionJobFromRequest(request: { + id: string + tenantId: string + payload?: unknown + createdAt: Date +}): IngestionJobPayload { + const stored = request.payload as NotifySimpleRequest | undefined + const scheduledFor = + stored?.email?.delayedSend || stored?.sms?.delayedSend || stored?.msgApp?.delayedSend + const base = { + notifyId: request.id, + tenantId: request.tenantId, + request: (request.payload || {}) as IngestionJobPayload['request'], + requestedAt: request.createdAt.toISOString(), + ...(scheduledFor && { scheduledFor }), + } + const channel = mergeRequestChannel(request.payload) + if (!channel) return base + + const mailMergeData = mergeContentFromRequest(request.payload, channel) + return { + ...base, + request: { templateId: mailMergeData.content?.templateId } as IngestionJobPayload['request'], + mailMerge: true, + mailMergeChannel: channel, + mailMergeData, + } +} + +/** Milliseconds until a job's scheduled send time; 0 when it is unscheduled or already due. */ +export function delayUntilScheduled( + job: Pick, + now = Date.now(), +) { + return job.scheduledFor ? Math.max(0, new Date(job.scheduledFor).getTime() - now) : 0 +} diff --git a/backend/src/queue/queue.module.ts b/backend/src/queue/queue.module.ts index 7ec8cee49..f651eaf84 100644 --- a/backend/src/queue/queue.module.ts +++ b/backend/src/queue/queue.module.ts @@ -21,8 +21,7 @@ import { IngestionWorker } from './workers/ingestion.worker' import { EmailDeliveryWorker } from './workers/email-delivery.worker' import { SmsDeliveryWorker } from './workers/sms-delivery.worker' import { WebhookDeliveryWorker } from './workers/webhook-delivery.worker' -import { PendingNotificationRetryService } from './services/pending-notification-retry.service' -import { BatchReconcilerService } from './services/batch-reconciler.service' +import { DeliveryReconcilerService } from './services/delivery-reconciler.service' import { WebhookTriggerService } from './services/webhook-trigger.service' import { NotificationRequest } from '../api/notification/entities/notification-request.entity' import { NotificationRequestDetail } from '../api/notification/entities/notification-request-detail.entity' @@ -75,8 +74,7 @@ import { PhoneNumberService } from '../api/notify/services/phone-number.service' forwardRef(() => NotifyModule), ], providers: [ - PendingNotificationRetryService, - BatchReconcilerService, + DeliveryReconcilerService, NotificationService, NotificationRequestDetailService, NotificationPubSubService, diff --git a/backend/src/queue/services/batch-reconciler.service.spec.ts b/backend/src/queue/services/batch-reconciler.service.spec.ts deleted file mode 100644 index 3d14db3a6..000000000 --- a/backend/src/queue/services/batch-reconciler.service.spec.ts +++ /dev/null @@ -1,202 +0,0 @@ -import { describe, it, expect, vi, beforeEach } from 'vitest' -import type Bull from 'bull' -import type Redis from 'ioredis' -import type { Repository } from 'typeorm' -import { BatchReconcilerService } from './batch-reconciler.service' -import { NotificationStatus } from '../../enum/notification-status.enum' -import type { NotificationRequestDetail } from '../../api/notification/entities/notification-request-detail.entity' -import type { NotificationRequest } from '../../api/notification/entities/notification-request.entity' -import type { NotificationService } from '../../api/notification/notification.service' - -const BATCH = { - batchId: 'req-1-EMAIL-0', - notifyId: 'req-1', - tenantId: 'tenant-1', - channel: 'EMAIL', -} - -const storedRequest = { - id: 'req-1', - payload: { - email: { - content: { subject: 'Hi', body: 'Hello {{name}}', bodyType: 'text' }, - recipients: { - mergeArray: [ - ['email', 'name'], - ['a@example.com', 'Ann'], - ['b@example.com', 'Bo'], - ['other-batch@example.com', 'Cy'], - ], - }, - }, - }, -} - -function setup({ - job, - locked = 'OK', - attempt = 1, - stale = [BATCH], -}: { - job?: Partial | null - locked?: string | null - attempt?: number - stale?: object[] -}) { - const builder: Record = {} - for (const m of [ - 'innerJoin', - 'select', - 'addSelect', - 'where', - 'andWhere', - 'groupBy', - 'addGroupBy', - 'orderBy', - 'limit', - ]) { - builder[m] = vi.fn(() => builder) - } - builder.getRawMany = vi.fn().mockResolvedValue(stale) - - const detailRepository = { - createQueryBuilder: vi.fn(() => builder), - find: vi - .fn() - .mockResolvedValue([ - { recipientAddress: 'a@example.com' }, - { recipientAddress: 'b@example.com' }, - ]), - update: vi.fn().mockResolvedValue({ affected: 2 }), - count: vi.fn().mockResolvedValue(0), - } - const requestRepository = { findOne: vi.fn().mockResolvedValue(storedRequest) } - const notificationService = { - parseMailMergeRecipients: (rows: string[][]) => - rows.slice(1).map(([address, name]) => ({ address, params: { name } })), - update: vi.fn().mockResolvedValue(undefined), - } - const redis = { - set: vi.fn().mockResolvedValue(locked), - eval: vi.fn().mockResolvedValue(1), - incr: vi.fn().mockResolvedValue(attempt), - expire: vi.fn().mockResolvedValue(1), - } - const queue = { - getJob: vi.fn().mockResolvedValue(job ?? null), - add: vi.fn().mockResolvedValue({}), - } - const service = new BatchReconcilerService( - detailRepository as unknown as Repository, - requestRepository as unknown as Repository, - notificationService as unknown as NotificationService, - redis as unknown as Redis, - queue as unknown as Bull.Queue, - null, - ) - return { service, detailRepository, notificationService, redis, queue } -} - -const jobIn = (state: string, extra: Partial = {}) => - ({ - getState: vi.fn().mockResolvedValue(state), - retry: vi.fn().mockResolvedValue(undefined), - remove: vi.fn().mockResolvedValue(undefined), - failedReason: 'job stalled more than allowable limit', - ...extra, - }) as unknown as Bull.Job - -describe('BatchReconcilerService', () => { - beforeEach(() => vi.clearAllMocks()) - - it.each(['waiting', 'active', 'delayed'])('leaves a %s batch alone', async (state) => { - const job = jobIn(state) - const { service, queue, redis } = setup({ job }) - - expect(await service.reconcile()).toEqual({ 'in-progress': 1 }) - expect(job.retry).not.toHaveBeenCalled() - expect(queue.add).not.toHaveBeenCalled() - // A long batch still running must not use up its recovery attempts. - expect(redis.incr).not.toHaveBeenCalled() - }) - - it('retries a batch whose job failed, keeping its original payload', async () => { - const job = jobIn('failed') - const { service, queue } = setup({ job }) - - expect(await service.reconcile()).toEqual({ retried: 1 }) - expect(job.retry).toHaveBeenCalled() - expect(queue.add).not.toHaveBeenCalled() - }) - - it('rebuilds a batch with no job from the stored request, leaving recipients in its rows', async () => { - const { service, queue } = setup({ job: null }) - - expect(await service.reconcile()).toEqual({ requeued: 1 }) - const [payload, options] = queue.add.mock.calls[0] - expect(payload).toMatchObject({ - notifyId: 'req-1', - tenantId: 'tenant-1', - channel: 'EMAIL', - mailMerge: true, - batchId: 'req-1-EMAIL-0', - mailMergeData: { - content: { subject: 'Hi', body: 'Hello {{name}}', bodyType: 'text' }, - params: {}, - }, - }) - expect(payload.mailMergeData).not.toHaveProperty('recipients') - expect(options).toMatchObject({ jobId: 'req-1-EMAIL-0', attempts: 3 }) - }) - - it('replaces a job that finished while recipients are still owed', async () => { - const job = jobIn('completed') - const { service, queue } = setup({ job }) - - expect(await service.reconcile()).toEqual({ requeued: 1 }) - expect(job.remove).toHaveBeenCalled() - expect(queue.add).toHaveBeenCalled() - }) - - it('gives up after the recovery limit: marks what is owed failed and settles the request', async () => { - const { service, detailRepository, notificationService, queue } = setup({ - job: jobIn('failed'), - attempt: 6, - }) - detailRepository.count.mockResolvedValueOnce(0).mockResolvedValueOnce(120) - - expect(await service.reconcile()).toEqual({ 'gave-up': 1 }) - expect(detailRepository.update).toHaveBeenCalledWith( - expect.objectContaining({ notificationRequestId: 'req-1', batchId: 'req-1-EMAIL-0' }), - expect.objectContaining({ - status: 'failed', - errorMessage: expect.stringMatching(/5 recovery/), - }), - ) - expect(notificationService.update).toHaveBeenCalledWith('req-1', 'tenant-1', { - status: NotificationStatus.PARTIALLY_COMPLETED, - updatedBy: 'batch-reconciler', - }) - expect(queue.add).not.toHaveBeenCalled() - }) - - it('does nothing when another pod holds the lock', async () => { - const { service, queue, detailRepository } = setup({ job: null, locked: null }) - - expect(await service.reconcile()).toEqual({}) - expect(detailRepository.createQueryBuilder).not.toHaveBeenCalled() - expect(queue.add).not.toHaveBeenCalled() - }) - - it('keeps going past a batch it cannot reconcile, and releases the lock', async () => { - const { service, queue, redis } = setup({ - job: null, - stale: [{ ...BATCH, channel: 'SMS' }, BATCH], - }) - - // No SMS queue was given, so the first batch throws; the second is still requeued. - expect(await service.reconcile()).toEqual({ requeued: 1 }) - expect(queue.add).toHaveBeenCalledTimes(1) - expect(redis.eval).toHaveBeenCalled() - }) -}) diff --git a/backend/src/queue/services/batch-reconciler.service.ts b/backend/src/queue/services/batch-reconciler.service.ts deleted file mode 100644 index 1acf12812..000000000 --- a/backend/src/queue/services/batch-reconciler.service.ts +++ /dev/null @@ -1,270 +0,0 @@ -import { - Inject, - Injectable, - Logger, - OnApplicationBootstrap, - OnModuleDestroy, - Optional, -} from '@nestjs/common' -import { InjectRepository } from '@nestjs/typeorm' -import { In, Repository } from 'typeorm' -import { randomUUID } from 'crypto' -import type Bull from 'bull' -import type Redis from 'ioredis' -import { NotificationRequestDetail } from '../../api/notification/entities/notification-request-detail.entity' -import { NotificationRequest } from '../../api/notification/entities/notification-request.entity' -import { NotificationService } from '../../api/notification/notification.service' -import { NotificationChannel } from '../../enum/notification-channel.enum' -import { NotificationStatus } from '../../enum/notification-status.enum' -import { ProviderToken } from '../../enum/provider-token.enum' -import { QueueName } from '../../enum/queue-name.enum' -import { redisKey } from '../../common/redis/redis-namespace' -import { FAILED_JOB_RETENTION } from '../job-retention' -import { mergeContentFromRequest } from '../merge-batch-builder' -import type { DeliveryJobPayload } from '../queue.types' - -const intFromEnv = (name: string, fallback: number) => { - const value = parseInt(process.env[name] || '', 10) - return Number.isFinite(value) ? value : fallback -} - -const RECONCILE_INTERVAL_MS = intFromEnv('BATCH_RECONCILE_INTERVAL_MS', 60_000) -/** How long a recipient may stay in flight before its batch is checked for a live job. */ -const STALE_AFTER_MS = intFromEnv('BATCH_RECONCILE_STALE_MS', 10 * 60_000) -/** Recovery attempts per batch before its remaining recipients are marked failed. */ -const MAX_RECOVERY_ATTEMPTS = intFromEnv('BATCH_RECONCILE_MAX_ATTEMPTS', 5) -const BATCHES_PER_PASS = 50 - -const LOCK_KEY = redisKey('batch-reconcile:lock') -const attemptsKey = (batchId: string) => redisKey(`batch-reconcile:attempts:${batchId}`) -const ATTEMPTS_TTL_SECONDS = 7 * 24 * 60 * 60 - -/** Release only our own lock; see PendingNotificationRetryService for why. */ -const RELEASE_LOCK_SCRIPT = ` -if redis.call("get", KEYS[1]) == ARGV[1] then - return redis.call("del", KEYS[1]) -end -return 0 -` - -const IN_FLIGHT_STATUSES = ['pending', 'queued', 'processing', 'sending'] -/** Bull states in which the batch's job will still run without help. */ -const LIVE_JOB_STATES = new Set(['waiting', 'active', 'delayed', 'paused']) - -export type ReconcileOutcome = 'in-progress' | 'retried' | 'requeued' | 'gave-up' - -interface StaleBatch { - batchId: string - notifyId: string - tenantId: string - channel: string -} - -/** - * Finds merge batches whose recipients are still in flight with no job left to send them, and - * puts them back on the queue. - * - * The queue's own recovery covers a worker dying mid-job: Bull notices the lapsed lock and - * retries. It does not cover a job that ends up failed (out of retries, or stalled more often - * than maxStalledCount) or one that never reached Redis; its recipients would stay pending - * forever. Postgres is the record of what is owed, so this compares it with what the queue - * holds: - * - a live job (waiting, active, delayed) is left alone - a long batch is not stuck; - * - a failed job is retried with its original payload; - * - a missing or finished job is rebuilt from the stored request and queued again. - * Merge workers skip recipients already sent, so any of these is safe to repeat. After - * MAX_RECOVERY_ATTEMPTS a batch's remaining recipients are marked failed, so a send that can - * never succeed (a deleted template, a rejected sender) ends visibly instead of looping. - * - * One pod at a time, behind a Redis lock, every RECONCILE_INTERVAL_MS. - */ -@Injectable() -export class BatchReconcilerService implements OnApplicationBootstrap, OnModuleDestroy { - private readonly logger = new Logger(BatchReconcilerService.name) - private readonly instanceId = randomUUID() - private timer?: NodeJS.Timeout - - constructor( - @InjectRepository(NotificationRequestDetail) - private readonly detailRepository: Repository, - @InjectRepository(NotificationRequest) - private readonly requestRepository: Repository, - private readonly notificationService: NotificationService, - @Optional() @Inject(ProviderToken.REDIS_CLIENT) private readonly redis?: Redis | null, - @Optional() @Inject(QueueName.EMAIL_DELIVERY) private readonly emailQueue?: Bull.Queue | null, - @Optional() @Inject(QueueName.SMS_DELIVERY) private readonly smsQueue?: Bull.Queue | null, - ) {} - - onApplicationBootstrap(): void { - if (!this.redis || !this.emailQueue) return - this.timer = setInterval(() => void this.reconcile(), RECONCILE_INTERVAL_MS) - this.timer.unref() - } - - onModuleDestroy(): void { - if (this.timer) clearInterval(this.timer) - } - - /** One pass. Never throws: a failed pass is logged and the next interval tries again. */ - async reconcile(now = Date.now()): Promise>> { - const outcomes: Partial> = {} - if (!this.redis) return outcomes - - try { - const locked = await this.redis.set( - LOCK_KEY, - this.instanceId, - 'PX', - RECONCILE_INTERVAL_MS, - 'NX', - ) - if (locked !== 'OK') return outcomes - } catch (error) { - this.logger.warn(`Could not take the reconcile lock: ${(error as Error).message}`) - return outcomes - } - - try { - for (const batch of await this.findStaleBatches(now)) { - try { - const outcome = await this.reconcileBatch(batch) - outcomes[outcome] = (outcomes[outcome] ?? 0) + 1 - } catch (error) { - this.logger.error( - `[${batch.notifyId}] Could not reconcile batch ${batch.batchId}: ${(error as Error).message}`, - ) - } - } - const acted = (outcomes.retried ?? 0) + (outcomes.requeued ?? 0) + (outcomes['gave-up'] ?? 0) - if (acted > 0) this.logger.warn(`Batch reconcile: ${JSON.stringify(outcomes)}`) - } catch (error) { - this.logger.error(`Batch reconcile pass failed: ${(error as Error).message}`) - } finally { - await this.redis - .eval(RELEASE_LOCK_SCRIPT, 1, LOCK_KEY, this.instanceId) - .catch(() => undefined) - } - return outcomes - } - - /** Merge batches with a recipient in flight longer than STALE_AFTER_MS; scheduled sends excluded. */ - private findStaleBatches(now: number): Promise { - return this.detailRepository - .createQueryBuilder('detail') - .innerJoin('detail.notificationRequest', 'request') - .select('detail.batch_id', 'batchId') - .addSelect('detail.notification_request_id', 'notifyId') - .addSelect('request.tenant_id', 'tenantId') - .addSelect('detail.channel', 'channel') - .where('detail.batch_id IS NOT NULL') - .andWhere('detail.status IN (:...statuses)', { statuses: IN_FLIGHT_STATUSES }) - .andWhere('detail.last_attempt_at < :staleBefore', { - staleBefore: new Date(now - STALE_AFTER_MS), - }) - .andWhere('request.status <> :scheduled', { scheduled: NotificationStatus.SCHEDULED }) - .groupBy('detail.batch_id') - .addGroupBy('detail.notification_request_id') - .addGroupBy('request.tenant_id') - .addGroupBy('detail.channel') - .orderBy('MIN(detail.last_attempt_at)', 'ASC') - .limit(BATCHES_PER_PASS) - .getRawMany() - } - - async reconcileBatch(batch: StaleBatch): Promise { - const queue = batch.channel === NotificationChannel.SMS ? this.smsQueue : this.emailQueue - if (!queue || !this.redis) throw new Error(`No queue for channel ${batch.channel}`) - - const job = await queue.getJob(batch.batchId) - const state = job ? await job.getState() : null - if (state && LIVE_JOB_STATES.has(state)) return 'in-progress' - - const key = attemptsKey(batch.batchId) - const attempt = await this.redis.incr(key) - await this.redis.expire(key, ATTEMPTS_TTL_SECONDS) - if (attempt > MAX_RECOVERY_ATTEMPTS) { - await this.giveUp(batch, attempt - 1) - return 'gave-up' - } - - if (job && state === 'failed') { - this.logger.warn( - `[${batch.notifyId}] Retrying failed batch ${batch.batchId} (recovery ${attempt}/${MAX_RECOVERY_ATTEMPTS}): ${job.failedReason ?? 'no reason'}`, - ) - await job.retry() - return 'retried' - } - - // Finished or gone while recipients are still owed: replace it with one built from Postgres. - if (job) await job.remove().catch(() => undefined) - await queue.add(await this.rebuildJob(batch), { - jobId: batch.batchId, - attempts: 3, - backoff: { type: 'exponential', delay: 2000 }, - removeOnComplete: true, - removeOnFail: FAILED_JOB_RETENTION, - }) - this.logger.warn( - `[${batch.notifyId}] Re-queued batch ${batch.batchId} with no job (recovery ${attempt}/${MAX_RECOVERY_ATTEMPTS})`, - ) - return 'requeued' - } - - /** The batch's job as ingestion queued it: shared content and params, recipients left in Postgres. */ - private async rebuildJob(batch: StaleBatch): Promise { - const request = await this.requestRepository.findOne({ - where: { id: batch.notifyId }, - select: { id: true, payload: true }, - }) - if (!request) throw new Error('Parent notification request not found') - - const channel = batch.channel as NotificationChannel - const mailMergeData = mergeContentFromRequest(request.payload, channel) - return { - notifyId: batch.notifyId, - tenantId: batch.tenantId, - channel, - request: { templateId: mailMergeData.content?.templateId } as DeliveryJobPayload['request'], - payload: {} as DeliveryJobPayload['payload'], - attempt: 0, - mailMerge: true, - batchId: batch.batchId, - mailMergeData, - } - } - - /** - * Stop recovering a batch that keeps failing: mark what it still owes failed and settle the - * parent's status the way a finishing batch would, so the send ends visibly. - */ - private async giveUp(batch: StaleBatch, attempts: number): Promise { - await this.detailRepository.update( - { - notificationRequestId: batch.notifyId, - batchId: batch.batchId, - status: In(IN_FLIGHT_STATUSES), - }, - { - status: 'failed', - errorMessage: `Not delivered: the batch failed ${attempts} recovery attempt(s)`, - lastAttemptAt: new Date(), - updatedBy: 'batch-reconciler', - }, - ) - this.logger.error( - `[${batch.notifyId}] Gave up on batch ${batch.batchId} after ${attempts} recovery attempt(s); remaining recipients marked failed`, - ) - - const remaining = await this.detailRepository.count({ - where: { notificationRequestId: batch.notifyId, status: In(IN_FLIGHT_STATUSES) }, - }) - if (remaining > 0) return - const sent = await this.detailRepository.count({ - where: { notificationRequestId: batch.notifyId, status: 'sent' }, - }) - await this.notificationService.update(batch.notifyId, batch.tenantId, { - status: sent > 0 ? NotificationStatus.PARTIALLY_COMPLETED : NotificationStatus.FAILED, - updatedBy: 'batch-reconciler', - }) - } -} diff --git a/backend/src/queue/services/delivery-reconciler.service.spec.ts b/backend/src/queue/services/delivery-reconciler.service.spec.ts new file mode 100644 index 000000000..45917117b --- /dev/null +++ b/backend/src/queue/services/delivery-reconciler.service.spec.ts @@ -0,0 +1,392 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import type Bull from 'bull' +import type Redis from 'ioredis' +import type { Repository } from 'typeorm' +import { DeliveryReconcilerService } from './delivery-reconciler.service' +import { NotificationStatus } from '../../enum/notification-status.enum' +import type { NotificationRequestDetail } from '../../api/notification/entities/notification-request-detail.entity' +import type { NotificationRequest } from '../../api/notification/entities/notification-request.entity' +import type { NotificationService } from '../../api/notification/notification.service' + +const BATCH = { + batchId: 'req-1-EMAIL-0', + notifyId: 'req-1', + tenantId: 'tenant-1', + channel: 'EMAIL', +} + +const storedRequest = { + id: 'req-1', + tenantId: 'tenant-1', + createdAt: new Date('2026-10-04T12:00:00Z'), + payload: { + email: { + content: { subject: 'Hi', body: 'Hello {{name}}', bodyType: 'text' }, + recipients: { + mergeArray: [ + ['email', 'name'], + ['a@example.com', 'Ann'], + ['b@example.com', 'Bo'], + ['other-batch@example.com', 'Cy'], + ], + }, + }, + }, +} + +function setup({ + job, + locked = 'OK', + attempt = 1, + stale = [BATCH], + deliveries = [], + ingestions = [], + pending = [], + ingestionJob = null, +}: { + job?: Partial | null + locked?: string | null + attempt?: number + /** Stale merge batches, then stale plain-send deliveries: the two detail queries, in order. */ + stale?: object[] + deliveries?: object[] + /** Requests stuck at QUEUED. */ + ingestions?: object[] + /** Requests still PENDING; queried before the QUEUED ones. */ + pending?: object[] + ingestionJob?: Partial | null +}) { + const results = [stale, deliveries] + const builder: Record = {} + for (const m of [ + 'innerJoin', + 'select', + 'addSelect', + 'where', + 'andWhere', + 'groupBy', + 'addGroupBy', + 'orderBy', + 'limit', + ]) { + builder[m] = vi.fn(() => builder) + } + builder.getRawMany = vi.fn(() => Promise.resolve(results.shift() ?? [])) + + const detailRepository = { + createQueryBuilder: vi.fn(() => builder), + find: vi + .fn() + .mockResolvedValue([ + { recipientAddress: 'a@example.com' }, + { recipientAddress: 'b@example.com' }, + ]), + update: vi.fn().mockResolvedValue({ affected: 2 }), + count: vi.fn().mockResolvedValue(0), + } + const requestBuilder: Record = {} + for (const m of ['select', 'addSelect', 'where', 'andWhere', 'orderBy', 'limit']) { + requestBuilder[m] = vi.fn(() => requestBuilder) + } + const requestResults = [pending, ingestions] + requestBuilder.getRawMany = vi.fn(() => Promise.resolve(requestResults.shift() ?? [])) + const requestRepository = { + findOne: vi.fn().mockResolvedValue(storedRequest), + createQueryBuilder: vi.fn(() => requestBuilder), + } + const ingestionQueue = { + getJob: vi.fn().mockResolvedValue(ingestionJob), + add: vi.fn().mockResolvedValue({}), + } + const notificationService = { + parseMailMergeRecipients: (rows: string[][]) => + rows.slice(1).map(([address, name]) => ({ address, params: { name } })), + update: vi.fn().mockResolvedValue(undefined), + } + const redis = { + set: vi.fn().mockResolvedValue(locked), + eval: vi.fn().mockResolvedValue(1), + incr: vi.fn().mockResolvedValue(attempt), + expire: vi.fn().mockResolvedValue(1), + } + const queue = { + getJob: vi.fn().mockResolvedValue(job ?? null), + add: vi.fn().mockResolvedValue({}), + } + const service = new DeliveryReconcilerService( + detailRepository as unknown as Repository, + requestRepository as unknown as Repository, + notificationService as unknown as NotificationService, + redis as unknown as Redis, + queue as unknown as Bull.Queue, + null, + ingestionQueue as unknown as Bull.Queue, + ) + return { + service, + detailRepository, + requestRepository, + notificationService, + redis, + queue, + ingestionQueue, + } +} + +const jobIn = (state: string, extra: Partial = {}) => + ({ + getState: vi.fn().mockResolvedValue(state), + retry: vi.fn().mockResolvedValue(undefined), + remove: vi.fn().mockResolvedValue(undefined), + failedReason: 'job stalled more than allowable limit', + ...extra, + }) as unknown as Bull.Job + +describe('DeliveryReconcilerService', () => { + beforeEach(() => vi.clearAllMocks()) + + it.each(['waiting', 'active', 'delayed'])('leaves a %s batch alone', async (state) => { + const job = jobIn(state) + const { service, queue, redis } = setup({ job }) + + expect(await service.reconcile()).toEqual({ 'in-progress': 1 }) + expect(job.retry).not.toHaveBeenCalled() + expect(queue.add).not.toHaveBeenCalled() + // A long batch still running must not use up its recovery attempts. + expect(redis.incr).not.toHaveBeenCalled() + }) + + it('retries a batch whose job failed, keeping its original payload', async () => { + const job = jobIn('failed') + const { service, queue } = setup({ job }) + + expect(await service.reconcile()).toEqual({ retried: 1 }) + expect(job.retry).toHaveBeenCalled() + expect(queue.add).not.toHaveBeenCalled() + }) + + it('rebuilds a batch with no job from the stored request, leaving recipients in its rows', async () => { + const { service, queue } = setup({ job: null }) + + expect(await service.reconcile()).toEqual({ requeued: 1 }) + const [payload, options] = queue.add.mock.calls[0] + expect(payload).toMatchObject({ + notifyId: 'req-1', + tenantId: 'tenant-1', + channel: 'EMAIL', + mailMerge: true, + batchId: 'req-1-EMAIL-0', + mailMergeData: { + content: { subject: 'Hi', body: 'Hello {{name}}', bodyType: 'text' }, + params: {}, + }, + }) + expect(payload.mailMergeData).not.toHaveProperty('recipients') + expect(options).toMatchObject({ jobId: 'req-1-EMAIL-0', attempts: 3 }) + }) + + it('replaces a job that finished while recipients are still owed', async () => { + const job = jobIn('completed') + const { service, queue } = setup({ job }) + + expect(await service.reconcile()).toEqual({ requeued: 1 }) + expect(job.remove).toHaveBeenCalled() + expect(queue.add).toHaveBeenCalled() + }) + + it('gives up after the recovery limit: marks what is owed failed and settles the request', async () => { + const { service, detailRepository, notificationService, queue } = setup({ + job: jobIn('failed'), + attempt: 6, + }) + detailRepository.count.mockResolvedValueOnce(0).mockResolvedValueOnce(120) + + expect(await service.reconcile()).toEqual({ 'gave-up': 1 }) + expect(detailRepository.update).toHaveBeenCalledWith( + expect.objectContaining({ notificationRequestId: 'req-1', batchId: 'req-1-EMAIL-0' }), + expect.objectContaining({ + status: 'failed', + errorMessage: expect.stringMatching(/5 recovery/), + }), + ) + expect(notificationService.update).toHaveBeenCalledWith('req-1', 'tenant-1', { + status: NotificationStatus.PARTIALLY_COMPLETED, + updatedBy: 'delivery-reconciler', + }) + expect(queue.add).not.toHaveBeenCalled() + }) + + it('does nothing when another pod holds the lock', async () => { + const { service, queue, detailRepository } = setup({ job: null, locked: null }) + + expect(await service.reconcile()).toEqual({}) + expect(detailRepository.createQueryBuilder).not.toHaveBeenCalled() + expect(queue.add).not.toHaveBeenCalled() + }) + + it('keeps going past a batch it cannot reconcile, and releases the lock', async () => { + const { service, queue, redis } = setup({ + job: null, + stale: [{ ...BATCH, channel: 'SMS' }, BATCH], + }) + + // No SMS queue was given, so the first batch throws; the second is still requeued. + expect(await service.reconcile()).toEqual({ requeued: 1 }) + expect(queue.add).toHaveBeenCalledTimes(1) + expect(redis.eval).toHaveBeenCalled() + }) + + describe('plain sends', () => { + const DELIVERY = { notifyId: 'req-1', tenantId: 'tenant-1', channel: 'EMAIL' } + + it('retries a failed delivery job', async () => { + const job = jobIn('failed') + const { service, queue } = setup({ job, stale: [], deliveries: [DELIVERY] }) + + expect(await service.reconcile()).toEqual({ retried: 1 }) + expect(queue.getJob).toHaveBeenCalledWith('req-1_EMAIL') + expect(job.retry).toHaveBeenCalled() + }) + + it('replaces a lost delivery job by running ingestion for the request again', async () => { + const { service, queue, ingestionQueue } = setup({ + job: null, + stale: [], + deliveries: [DELIVERY], + }) + + expect(await service.reconcile()).toEqual({ requeued: 1 }) + expect(queue.add).not.toHaveBeenCalled() + expect(ingestionQueue.add).toHaveBeenCalledWith( + expect.objectContaining({ notifyId: 'req-1', tenantId: 'tenant-1' }), + expect.objectContaining({ jobId: 'req-1', attempts: 3 }), + ) + }) + + it('leaves a lost delivery alone while ingestion is already running for the request', async () => { + const { service, ingestionQueue, redis } = setup({ + job: null, + stale: [], + deliveries: [DELIVERY], + ingestionJob: jobIn('active'), + }) + + expect(await service.reconcile()).toEqual({ 'in-progress': 1 }) + expect(ingestionQueue.add).not.toHaveBeenCalled() + expect(redis.incr).not.toHaveBeenCalled() + }) + + it("gives up on one channel's recipients without touching merge batches", async () => { + const { service, detailRepository } = setup({ + job: jobIn('failed'), + stale: [], + deliveries: [DELIVERY], + attempt: 6, + }) + + expect(await service.reconcile()).toEqual({ 'gave-up': 1 }) + expect(detailRepository.update).toHaveBeenCalledWith( + expect.objectContaining({ + notificationRequestId: 'req-1', + channel: 'EMAIL', + batchId: expect.anything(), + }), + expect.objectContaining({ status: 'failed' }), + ) + }) + }) + + describe('requests never ingested', () => { + it('re-queues ingestion for a request stuck at QUEUED with no live job', async () => { + const { service, ingestionQueue } = setup({ + job: null, + stale: [], + ingestions: [{ notifyId: 'req-1', tenantId: 'tenant-1' }], + }) + + expect(await service.reconcile()).toEqual({ requeued: 1 }) + expect(ingestionQueue.add).toHaveBeenCalledWith( + expect.objectContaining({ notifyId: 'req-1', mailMerge: true, mailMergeChannel: 'EMAIL' }), + expect.objectContaining({ jobId: 'req-1' }), + ) + }) + + it('retries a failed ingestion job', async () => { + const ingestionJob = jobIn('failed') + const { service } = setup({ + stale: [], + ingestions: [{ notifyId: 'req-1', tenantId: 'tenant-1' }], + ingestionJob, + }) + + expect(await service.reconcile()).toEqual({ retried: 1 }) + expect(ingestionJob.retry).toHaveBeenCalled() + }) + }) + + describe('requests never queued (PENDING)', () => { + const PENDING = [{ notifyId: 'req-1', tenantId: 'tenant-1' }] + + it('queues ingestion from the stored request, unnamed, and moves the request to QUEUED', async () => { + const { service, ingestionQueue, notificationService } = setup({ + stale: [], + pending: PENDING, + }) + + expect(await service.reconcile()).toEqual({ requeued: 1 }) + const [payload, options] = ingestionQueue.add.mock.calls[0] + // Unnamed: the ingestion worker registers Bull's default handler; a named job finds none. + expect(typeof payload).toBe('object') + expect(payload).toMatchObject({ + notifyId: 'req-1', + mailMerge: true, + mailMergeChannel: 'EMAIL', + }) + expect(options).toMatchObject({ jobId: 'req-1', attempts: 3 }) + expect(options).not.toHaveProperty('delay') + expect(notificationService.update).toHaveBeenCalledWith('req-1', 'tenant-1', { + status: NotificationStatus.QUEUED, + updatedBy: 'delivery-reconciler', + }) + }) + + it('keeps a scheduled send scheduled instead of sending it now', async () => { + const { service, ingestionQueue, notificationService, requestRepository } = setup({ + stale: [], + pending: PENDING, + }) + const sendAt = new Date(Date.now() + 60 * 60_000).toISOString() + requestRepository.findOne.mockResolvedValue({ + ...storedRequest, + payload: { + email: { + recipients: { to: ['a@example.com'] }, + content: { subject: 'S', body: 'B' }, + delayedSend: sendAt, + }, + }, + }) + + await service.reconcile() + + const [payload, options] = ingestionQueue.add.mock.calls[0] + expect(payload.scheduledFor).toBe(sendAt) + expect(options.delay).toBeGreaterThan(59 * 60_000) + expect(notificationService.update).toHaveBeenCalledWith('req-1', 'tenant-1', { + status: NotificationStatus.SCHEDULED, + updatedBy: 'delivery-reconciler', + }) + }) + + it('leaves a PENDING request alone while its ingestion job is queued', async () => { + const { service, ingestionQueue } = setup({ + stale: [], + pending: PENDING, + ingestionJob: jobIn('waiting'), + }) + + expect(await service.reconcile()).toEqual({ 'in-progress': 1 }) + expect(ingestionQueue.add).not.toHaveBeenCalled() + }) + }) +}) diff --git a/backend/src/queue/services/delivery-reconciler.service.ts b/backend/src/queue/services/delivery-reconciler.service.ts new file mode 100644 index 000000000..38f536d46 --- /dev/null +++ b/backend/src/queue/services/delivery-reconciler.service.ts @@ -0,0 +1,424 @@ +import { + Inject, + Injectable, + Logger, + OnApplicationBootstrap, + OnModuleDestroy, + Optional, +} from '@nestjs/common' +import { InjectRepository } from '@nestjs/typeorm' +import { FindOptionsWhere, In, IsNull, Repository } from 'typeorm' +import { randomUUID } from 'crypto' +import type Bull from 'bull' +import type Redis from 'ioredis' +import { NotificationRequestDetail } from '../../api/notification/entities/notification-request-detail.entity' +import { NotificationRequest } from '../../api/notification/entities/notification-request.entity' +import { NotificationService } from '../../api/notification/notification.service' +import { IN_FLIGHT_DETAIL_STATUSES } from '../../api/notification/notification-request-detail.service' +import { NotificationChannel } from '../../enum/notification-channel.enum' +import { NotificationStatus } from '../../enum/notification-status.enum' +import { ProviderToken } from '../../enum/provider-token.enum' +import { QueueName } from '../../enum/queue-name.enum' +import { redisKey } from '../../common/redis/redis-namespace' +import { COMPLETED_JOB_RETENTION, FAILED_JOB_RETENTION } from '../job-retention' +import { + delayUntilScheduled, + ingestionJobFromRequest, + mergeContentFromRequest, +} from '../merge-batch-builder' +import type { DeliveryJobPayload } from '../queue.types' + +const intFromEnv = (name: string, fallback: number) => { + const value = parseInt(process.env[name] || '', 10) + return Number.isFinite(value) ? value : fallback +} + +const RECONCILE_INTERVAL_MS = intFromEnv('DELIVERY_RECONCILE_INTERVAL_MS', 60_000) +/** How long work may sit unfinished before its job is checked. */ +const STALE_AFTER_MS = intFromEnv('DELIVERY_RECONCILE_STALE_MS', 10 * 60_000) +/** Recovery attempts per job before what it still owes is marked failed. */ +const MAX_RECOVERY_ATTEMPTS = intFromEnv('DELIVERY_RECONCILE_MAX_ATTEMPTS', 5) +/** + * How long a request may stay PENDING - stored, but its ingestion job not yet queued - before it + * is queued from here. Short: PENDING normally lasts milliseconds, and when it does not, Redis + * was unreachable at acceptance and the request has nothing else to move it. + */ +const PENDING_STALE_MS = intFromEnv('DELIVERY_RECONCILE_PENDING_STALE_MS', 30_000) +const ITEMS_PER_KIND_PER_PASS = 50 + +const LOCK_KEY = redisKey('delivery-reconcile:lock') +const attemptsKey = (item: StuckWork) => + redisKey(`delivery-reconcile:attempts:${item.kind}:${item.jobId}`) +const ATTEMPTS_TTL_SECONDS = 7 * 24 * 60 * 60 + +/** + * Release only our own lock. A pass that overran its TTL must not delete the lock a different + * pod has since taken, or two pods would reconcile at once. + */ +const RELEASE_LOCK_SCRIPT = ` +if redis.call("get", KEYS[1]) == ARGV[1] then + return redis.call("del", KEYS[1]) +end +return 0 +` + +/** Bull states in which a job will still run without help. */ +const LIVE_JOB_STATES = new Set(['waiting', 'active', 'delayed', 'paused']) +const DELIVERY_CHANNELS = [NotificationChannel.EMAIL, NotificationChannel.SMS] + +const JOB_OPTIONS = { + attempts: 3, + backoff: { type: 'exponential', delay: 2000 }, +} as const + +export type ReconcileOutcome = 'in-progress' | 'retried' | 'requeued' | 'gave-up' + +/** + * Work Postgres says is owed, keyed by the Bull job that should be doing it: + * - `pending`: a request stored but never queued (Redis unreachable at acceptance), job + * `` on ingestion; + * - `ingestion`: a request queued but never fanned out, job `` on ingestion; + * - `batch`: a merge batch's recipients, job `` on the channel's delivery queue; + * - `delivery`: a plain send's recipients on one channel, job `_`. + */ +interface StuckWork { + kind: 'pending' | 'ingestion' | 'batch' | 'delivery' + jobId: string + notifyId: string + tenantId: string + channel?: string + batchId?: string +} + +/** + * Finds work Postgres says is still owed but no job is left to do, and puts it back on a queue - + * from a request that never reached Redis at all to one batch of a merge that lost its job. + * + * The queue's own recovery covers a worker dying mid-job: Bull notices the lapsed lock and + * retries. It does not cover a job that ends up failed (out of retries, or stalled more often + * than maxStalledCount) or one that never reached Redis; what it owed would stay pending forever. + * Postgres is the record of what is owed, so this compares it with what the queue holds: + * - a live job (waiting, active, delayed) is left alone - a long batch is not stuck; + * - a failed job is retried with its original payload; + * - a missing or finished job is replaced: a merge batch rebuilt from the stored request, and + * anything else by running ingestion for the request again. + * Every path is safe to repeat: merge batches skip recipients already sent, and a re-run of + * ingestion writes no rows twice and skips channels already finished. After + * MAX_RECOVERY_ATTEMPTS the work still owed is marked failed and the request settled, so a send + * that can never succeed (a deleted template, a rejected sender) ends visibly instead of looping. + * + * One pod at a time, behind a Redis lock, every RECONCILE_INTERVAL_MS. + */ +@Injectable() +export class DeliveryReconcilerService implements OnApplicationBootstrap, OnModuleDestroy { + private readonly logger = new Logger(DeliveryReconcilerService.name) + private readonly instanceId = randomUUID() + private timer?: NodeJS.Timeout + + constructor( + @InjectRepository(NotificationRequestDetail) + private readonly detailRepository: Repository, + @InjectRepository(NotificationRequest) + private readonly requestRepository: Repository, + private readonly notificationService: NotificationService, + @Optional() @Inject(ProviderToken.REDIS_CLIENT) private readonly redis?: Redis | null, + @Optional() @Inject(QueueName.EMAIL_DELIVERY) private readonly emailQueue?: Bull.Queue | null, + @Optional() @Inject(QueueName.SMS_DELIVERY) private readonly smsQueue?: Bull.Queue | null, + @Optional() @Inject(QueueName.INGESTION) private readonly ingestionQueue?: Bull.Queue | null, + ) {} + + onApplicationBootstrap(): void { + if (!this.redis || !this.emailQueue) return + this.timer = setInterval(() => void this.reconcile(), RECONCILE_INTERVAL_MS) + this.timer.unref() + } + + onModuleDestroy(): void { + if (this.timer) clearInterval(this.timer) + } + + /** One pass. Never throws: a failed pass is logged and the next interval tries again. */ + async reconcile(now = Date.now()): Promise>> { + const outcomes: Partial> = {} + if (!this.redis) return outcomes + + try { + const locked = await this.redis.set( + LOCK_KEY, + this.instanceId, + 'PX', + RECONCILE_INTERVAL_MS, + 'NX', + ) + if (locked !== 'OK') return outcomes + } catch (error) { + this.logger.warn(`Could not take the reconcile lock: ${(error as Error).message}`) + return outcomes + } + + try { + const staleBefore = new Date(now - STALE_AFTER_MS) + const items = [ + ...(await this.findPendingRequests(new Date(now - PENDING_STALE_MS))), + ...(await this.findStuckIngestions(staleBefore)), + ...(await this.findStaleBatches(staleBefore)), + ...(await this.findStaleDeliveries(staleBefore)), + ] + for (const item of items) { + try { + const outcome = await this.reconcileItem(item) + outcomes[outcome] = (outcomes[outcome] ?? 0) + 1 + } catch (error) { + this.logger.error( + `[${item.notifyId}] Could not reconcile ${item.kind} ${item.jobId}: ${(error as Error).message}`, + ) + } + } + const acted = (outcomes.retried ?? 0) + (outcomes.requeued ?? 0) + (outcomes['gave-up'] ?? 0) + if (acted > 0) this.logger.warn(`Delivery reconcile: ${JSON.stringify(outcomes)}`) + } catch (error) { + this.logger.error(`Delivery reconcile pass failed: ${(error as Error).message}`) + } finally { + await this.redis + .eval(RELEASE_LOCK_SCRIPT, 1, LOCK_KEY, this.instanceId) + .catch(() => undefined) + } + return outcomes + } + + /** Merge batches with a recipient in flight longer than the stale window; scheduled sends excluded. */ + private async findStaleBatches(staleBefore: Date): Promise { + const rows = await this.inFlightRows(staleBefore) + .andWhere('detail.batch_id IS NOT NULL') + .select('detail.batch_id', 'batchId') + .addSelect('detail.notification_request_id', 'notifyId') + .addSelect('request.tenant_id', 'tenantId') + .addSelect('detail.channel', 'channel') + .groupBy('detail.batch_id') + .addGroupBy('detail.notification_request_id') + .addGroupBy('request.tenant_id') + .addGroupBy('detail.channel') + .orderBy('MIN(detail.last_attempt_at)', 'ASC') + .limit(ITEMS_PER_KIND_PER_PASS) + .getRawMany<{ batchId: string; notifyId: string; tenantId: string; channel: string }>() + return rows.map((row) => ({ kind: 'batch', jobId: row.batchId, ...row })) + } + + /** Plain sends: recipients outside any batch, in flight on one channel past the stale window. */ + private async findStaleDeliveries(staleBefore: Date): Promise { + const rows = await this.inFlightRows(staleBefore) + .andWhere('detail.batch_id IS NULL') + .andWhere('detail.channel IN (:...channels)', { channels: DELIVERY_CHANNELS }) + .select('detail.notification_request_id', 'notifyId') + .addSelect('request.tenant_id', 'tenantId') + .addSelect('detail.channel', 'channel') + .groupBy('detail.notification_request_id') + .addGroupBy('request.tenant_id') + .addGroupBy('detail.channel') + .orderBy('MIN(detail.last_attempt_at)', 'ASC') + .limit(ITEMS_PER_KIND_PER_PASS) + .getRawMany<{ notifyId: string; tenantId: string; channel: string }>() + return rows.map((row) => ({ + kind: 'delivery', + jobId: `${row.notifyId}_${row.channel}`, + ...row, + })) + } + + /** + * Requests still QUEUED past the stale window: accepted, but ingestion never fanned them out. + * A plain send that already has rows is left to findStaleDeliveries - ingestion got that far, + * and its recipients may already be sent. A merge is safe to re-ingest either way. + */ + private async findStuckIngestions(staleBefore: Date): Promise { + if (!this.ingestionQueue) return [] + const rows = await this.requestRepository + .createQueryBuilder('request') + .select('request.id', 'notifyId') + .addSelect('request.tenant_id', 'tenantId') + .where('request.status = :queued', { queued: NotificationStatus.QUEUED }) + .andWhere('request.updated_at < :staleBefore', { staleBefore }) + .andWhere( + `(jsonb_exists(request.payload -> 'email' -> 'recipients', 'mergeArray') + OR jsonb_exists(request.payload -> 'sms' -> 'recipients', 'mergeArray') + OR NOT EXISTS ( + SELECT 1 FROM notify.notification_request_detail detail + WHERE detail.notification_request_id = request.id AND detail.status <> 'blocked'))`, + ) + .orderBy('request.updated_at', 'ASC') + .limit(ITEMS_PER_KIND_PER_PASS) + .getRawMany<{ notifyId: string; tenantId: string }>() + return rows.map((row) => ({ kind: 'ingestion', jobId: row.notifyId, ...row })) + } + + /** Requests still PENDING: stored at acceptance, but their ingestion job never queued. Oldest first. */ + private async findPendingRequests(pendingBefore: Date): Promise { + if (!this.ingestionQueue) return [] + const rows = await this.requestRepository + .createQueryBuilder('request') + .select('request.id', 'notifyId') + .addSelect('request.tenant_id', 'tenantId') + .where('request.status = :pending', { pending: NotificationStatus.PENDING }) + .andWhere('request.created_at < :pendingBefore', { pendingBefore }) + .orderBy('request.created_at', 'ASC') + .limit(ITEMS_PER_KIND_PER_PASS) + .getRawMany<{ notifyId: string; tenantId: string }>() + return rows.map((row) => ({ kind: 'pending', jobId: row.notifyId, ...row })) + } + + private inFlightRows(staleBefore: Date) { + return this.detailRepository + .createQueryBuilder('detail') + .innerJoin('detail.notificationRequest', 'request') + .where('detail.status IN (:...statuses)', { statuses: IN_FLIGHT_DETAIL_STATUSES }) + .andWhere('detail.last_attempt_at < :staleBefore', { staleBefore }) + .andWhere('request.status <> :scheduled', { scheduled: NotificationStatus.SCHEDULED }) + } + + private queueFor(item: StuckWork): Bull.Queue | null | undefined { + if (item.kind === 'ingestion' || item.kind === 'pending') return this.ingestionQueue + return item.channel === NotificationChannel.SMS ? this.smsQueue : this.emailQueue + } + + async reconcileItem(item: StuckWork): Promise { + const queue = this.queueFor(item) + if (!queue || !this.redis) + throw new Error(`No queue for ${item.kind} on ${item.channel ?? '-'}`) + + const job = await queue.getJob(item.jobId) + const state = job ? await job.getState() : null + if (state && LIVE_JOB_STATES.has(state)) return 'in-progress' + + // A plain send is replaced by re-running ingestion; while that runs, it is in hand. + if (item.kind === 'delivery' && (await this.ingestionLive(item.notifyId))) return 'in-progress' + + const key = attemptsKey(item) + const attempt = await this.redis.incr(key) + await this.redis.expire(key, ATTEMPTS_TTL_SECONDS) + if (attempt > MAX_RECOVERY_ATTEMPTS) { + await this.giveUp(item, attempt - 1) + return 'gave-up' + } + + if (job && state === 'failed') { + this.logger.warn( + `[${item.notifyId}] Retrying failed ${item.kind} ${item.jobId} (recovery ${attempt}/${MAX_RECOVERY_ATTEMPTS}): ${job.failedReason ?? 'no reason'}`, + ) + await job.retry() + return 'retried' + } + + // Finished or gone while work is still owed: replace it. + if (job) await job.remove().catch(() => undefined) + if (item.kind === 'batch') { + await queue.add(await this.rebuildBatchJob(item), { + jobId: item.jobId, + ...JOB_OPTIONS, + removeOnComplete: true, + removeOnFail: FAILED_JOB_RETENTION, + }) + } else { + await this.reingest(item) + } + this.logger.warn( + `[${item.notifyId}] Re-queued ${item.kind} ${item.jobId} with no job (recovery ${attempt}/${MAX_RECOVERY_ATTEMPTS})`, + ) + return 'requeued' + } + + private async ingestionLive(notifyId: string): Promise { + const job = await this.ingestionQueue?.getJob(notifyId) + return !!job && LIVE_JOB_STATES.has(await job.getState()) + } + + /** + * Run ingestion for the request again, from the stored request. A finished ingestion job of the + * same id is removed first, or Bull would treat the add as a duplicate and do nothing. A + * scheduled send keeps its delay. A request recovered from PENDING then moves on as acceptance + * would have moved it - to SCHEDULED or QUEUED - so it leaves this finder. + */ + private async reingest(item: StuckWork): Promise { + if (!this.ingestionQueue) throw new Error('No ingestion queue') + const request = await this.requestRepository.findOne({ + where: { id: item.notifyId }, + select: { id: true, tenantId: true, payload: true, createdAt: true }, + }) + if (!request) throw new Error('Notification request not found') + + const job = ingestionJobFromRequest(request) + const delay = delayUntilScheduled(job) + const previous = await this.ingestionQueue.getJob(item.notifyId) + if (previous) await previous.remove().catch(() => undefined) + await this.ingestionQueue.add(job, { + jobId: item.notifyId, + ...JOB_OPTIONS, + ...(delay > 0 && { delay }), + removeOnComplete: COMPLETED_JOB_RETENTION, + removeOnFail: FAILED_JOB_RETENTION, + }) + + if (item.kind === 'pending') { + await this.notificationService.update(item.notifyId, item.tenantId, { + status: delay > 0 ? NotificationStatus.SCHEDULED : NotificationStatus.QUEUED, + updatedBy: 'delivery-reconciler', + }) + } + } + + /** The batch's job as ingestion queued it: shared content and params, recipients left in Postgres. */ + private async rebuildBatchJob(item: StuckWork): Promise { + const request = await this.requestRepository.findOne({ + where: { id: item.notifyId }, + select: { id: true, payload: true }, + }) + if (!request) throw new Error('Parent notification request not found') + + const channel = item.channel as NotificationChannel + const mailMergeData = mergeContentFromRequest(request.payload, channel) + return { + notifyId: item.notifyId, + tenantId: item.tenantId, + channel, + request: { templateId: mailMergeData.content?.templateId } as DeliveryJobPayload['request'], + payload: {} as DeliveryJobPayload['payload'], + attempt: 0, + mailMerge: true, + batchId: item.batchId, + mailMergeData, + } + } + + /** + * Stop recovering work that keeps failing: mark what it still owes failed and settle the + * request's status the way a finishing job would, so the send ends visibly. + */ + private async giveUp(item: StuckWork, attempts: number): Promise { + const owed: FindOptionsWhere = { + notificationRequestId: item.notifyId, + status: In(IN_FLIGHT_DETAIL_STATUSES), + ...(item.kind === 'batch' && { batchId: item.batchId }), + ...(item.kind === 'delivery' && { batchId: IsNull(), channel: item.channel }), + } + await this.detailRepository.update(owed, { + status: 'failed', + errorMessage: `Not delivered: the ${item.kind} failed ${attempts} recovery attempt(s)`, + lastAttemptAt: new Date(), + updatedBy: 'delivery-reconciler', + }) + this.logger.error( + `[${item.notifyId}] Gave up on ${item.kind} ${item.jobId} after ${attempts} recovery attempt(s); what it owed is marked failed`, + ) + + const remaining = await this.detailRepository.count({ + where: { notificationRequestId: item.notifyId, status: In(IN_FLIGHT_DETAIL_STATUSES) }, + }) + if (remaining > 0) return + const sent = await this.detailRepository.count({ + where: { notificationRequestId: item.notifyId, status: 'sent' }, + }) + await this.notificationService.update(item.notifyId, item.tenantId, { + status: sent > 0 ? NotificationStatus.PARTIALLY_COMPLETED : NotificationStatus.FAILED, + updatedBy: 'delivery-reconciler', + }) + } +} diff --git a/backend/src/queue/services/pending-notification-retry.service.spec.ts b/backend/src/queue/services/pending-notification-retry.service.spec.ts deleted file mode 100644 index fb0147359..000000000 --- a/backend/src/queue/services/pending-notification-retry.service.spec.ts +++ /dev/null @@ -1,400 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing' -import { getRepositoryToken } from '@nestjs/typeorm' -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' -import { PendingNotificationRetryService } from './pending-notification-retry.service' -import { NotificationRequest } from '../../api/notification/entities/notification-request.entity' -import { NotificationService } from '../../api/notification/notification.service' -import { NotificationStatus } from '../../enum/notification-status.enum' -import { QueueName } from '../../enum/queue-name.enum' -import { COMPLETED_JOB_RETENTION, FAILED_JOB_RETENTION } from '../job-retention' - -describe('PendingNotificationRetryService', () => { - let service: PendingNotificationRetryService - let module: TestingModule - let mockRepository: any - let mockNotificationService: any - let mockQueue: any - let mockRedisClient: any - - beforeEach(async () => { - mockRepository = { find: vi.fn() } - mockNotificationService = { - update: vi.fn(), - parseMailMergeRecipients: vi.fn((rows: string[][]) => - rows.slice(1).map(([address, firstName]) => ({ address, params: { firstName } })), - ), - } - // The sweep takes a lock through the queue's own Redis connection; 'OK' means acquired. - mockRedisClient = { - set: vi.fn().mockResolvedValue('OK'), - eval: vi.fn().mockResolvedValue(1), - } - mockQueue = { - add: vi.fn().mockResolvedValue({ id: 'job-123' }), - name: 'ingestion', - client: mockRedisClient, - } - - module = await Test.createTestingModule({ - providers: [ - PendingNotificationRetryService, - { provide: getRepositoryToken(NotificationRequest), useValue: mockRepository }, - { provide: NotificationService, useValue: mockNotificationService }, - { provide: QueueName.INGESTION, useValue: mockQueue }, - ], - }).compile() - - service = module.get(PendingNotificationRetryService) - vi.spyOn(service['logger'], 'debug').mockImplementation(() => {}) - vi.spyOn(service['logger'], 'log').mockImplementation(() => {}) - vi.spyOn(service['logger'], 'warn').mockImplementation(() => {}) - vi.spyOn(service['logger'], 'error').mockImplementation(() => {}) - }) - - afterEach(async () => { - if (service?.['retryInterval']) clearInterval(service['retryInterval']) - await module.close() - }) - - describe('retryPendingNotifications', () => { - it('should handle no pending notifications', async () => { - mockRepository.find.mockResolvedValue([]) - await service.retryPendingNotifications() - expect(mockRepository.find).toHaveBeenCalledWith({ - where: { status: NotificationStatus.PENDING }, - order: { createdAt: 'ASC' }, - take: 100, - }) - expect(mockQueue.add).not.toHaveBeenCalled() - }) - - it('should retry all pending notifications', async () => { - const notifications = [ - { - id: 'notify-1', - tenantId: 'tenant-1', - status: NotificationStatus.PENDING, - payload: { email: 'test@example.com' }, - createdAt: new Date('2024-01-01T10:00:00Z'), - }, - { - id: 'notify-2', - tenantId: 'tenant-2', - status: NotificationStatus.PENDING, - payload: { phone: '+11234567890' }, - createdAt: new Date('2024-01-01T11:00:00Z'), - }, - ] - mockRepository.find.mockResolvedValue(notifications) - mockQueue.add.mockResolvedValue({ id: 'job-123' }) - mockNotificationService.update.mockResolvedValue({}) - await service.retryPendingNotifications() - expect(mockQueue.add).toHaveBeenCalledTimes(2) - expect(mockNotificationService.update).toHaveBeenCalledTimes(2) - }) - - it('should handle partial failures', async () => { - const notifications = [ - { - id: 'notify-1', - tenantId: 'tenant-1', - status: NotificationStatus.PENDING, - payload: { email: 'test@example.com' }, - createdAt: new Date('2024-01-01T10:00:00Z'), - }, - { - id: 'notify-2', - tenantId: 'tenant-2', - status: NotificationStatus.PENDING, - payload: { phone: '+11234567890' }, - createdAt: new Date('2024-01-01T11:00:00Z'), - }, - ] - mockRepository.find.mockResolvedValue(notifications) - mockQueue.add.mockResolvedValueOnce({ id: 'job-123' }) - mockQueue.add.mockRejectedValueOnce(new Error('Redis error')) - mockNotificationService.update.mockResolvedValue({}) - await service.retryPendingNotifications() - expect(mockQueue.add).toHaveBeenCalledTimes(2) - expect(mockNotificationService.update).toHaveBeenCalledTimes(1) - expect(service['logger'].warn).toHaveBeenCalled() - }) - - it('should handle update failure', async () => { - const notifications = [ - { - id: 'notify-1', - tenantId: 'tenant-1', - status: NotificationStatus.PENDING, - payload: { email: 'test@example.com' }, - createdAt: new Date('2024-01-01T10:00:00Z'), - }, - ] - mockRepository.find.mockResolvedValue(notifications) - mockQueue.add.mockResolvedValue({ id: 'job-123' }) - mockNotificationService.update.mockRejectedValue(new Error('DB error')) - await service.retryPendingNotifications() - expect(service['logger'].warn).toHaveBeenCalled() - }) - - it('should handle repository error', async () => { - mockRepository.find.mockRejectedValue(new Error('Connection failed')) - await service.retryPendingNotifications() - expect(service['logger'].error).toHaveBeenCalled() - }) - - it('should handle null payload', async () => { - const notifications = [ - { - id: 'notify-1', - tenantId: 'tenant-1', - status: NotificationStatus.PENDING, - payload: null, - createdAt: new Date('2024-01-01T10:00:00Z'), - }, - ] - mockRepository.find.mockResolvedValue(notifications) - mockQueue.add.mockResolvedValue({ id: 'job-123' }) - mockNotificationService.update.mockResolvedValue({}) - await service.retryPendingNotifications() - expect(mockQueue.add).toHaveBeenCalledWith( - expect.objectContaining({ request: {} }), - expect.any(Object), - ) - }) - - it('queues the job unnamed, so the ingestion worker has a handler for it', async () => { - // A named job ('process') matches no handler - the worker registers Bull's default one - - // and fails the moment it is picked up. - mockRepository.find.mockResolvedValue([ - { id: 'notify-1', tenantId: 'tenant-1', payload: {}, createdAt: new Date() }, - ]) - - await service.retryPendingNotifications() - - expect(typeof mockQueue.add.mock.calls[0][0]).toBe('object') - }) - - const storedMerge = { - id: 'notify-merge', - tenantId: 'tenant-1', - createdAt: new Date('2026-10-04T12:00:00Z'), - payload: { - params: { org: 'BC' }, - email: { - content: { subject: 'Hi {{firstName}}', body: 'Body', bodyType: 'text' }, - params: { team: 'Notify' }, - recipients: { - mergeArray: [ - ['email', 'firstName'], - ['a@example.com', 'Ann'], - ['blocked@example.com', 'Bo'], - ], - }, - }, - }, - } - - it('re-queues a merge as a merge, leaving its recipients for ingestion to read', async () => { - mockRepository.find.mockResolvedValue([storedMerge]) - - await service.retryPendingNotifications() - - const [payload] = mockQueue.add.mock.calls[0] - expect(payload.mailMergeData).toEqual({ - content: { subject: 'Hi {{firstName}}', body: 'Body', bodyType: 'text' }, - params: { org: 'BC', team: 'Notify' }, - }) - expect(payload).toMatchObject({ - notifyId: 'notify-merge', - mailMerge: true, - mailMergeChannel: 'EMAIL', - }) - }) - - it('should log stats', async () => { - const notifications = [ - { - id: 'notify-1', - tenantId: 'tenant-1', - status: NotificationStatus.PENDING, - payload: { email: 'test@example.com' }, - createdAt: new Date('2024-01-01T10:00:00Z'), - }, - ] - mockRepository.find.mockResolvedValue(notifications) - mockQueue.add.mockResolvedValue({ id: 'job-123' }) - mockNotificationService.update.mockResolvedValue({}) - await service.retryPendingNotifications() - expect(service['logger'].log).toHaveBeenCalledWith( - expect.stringMatching(/1 succeeded, 0 failed/), - ) - }) - }) - - describe('onApplicationBootstrap', () => { - it('should start interval with default config', () => { - mockRepository.find.mockResolvedValue([]) - service.onApplicationBootstrap() - expect(service['retryInterval']).toBeDefined() - expect(service['logger'].log).toHaveBeenCalledWith( - expect.stringContaining('Starting pending notification retry job'), - ) - }) - - it('should use custom interval from env', () => { - const originalEnv = process.env.PENDING_RETRY_INTERVAL - process.env.PENDING_RETRY_INTERVAL = '60000' - mockRepository.find.mockResolvedValue([]) - service.onApplicationBootstrap() - expect(service['logger'].log).toHaveBeenCalledWith(expect.stringContaining('60000')) - process.env.PENDING_RETRY_INTERVAL = originalEnv - }) - }) - - describe('onModuleDestroy', () => { - it('should clear interval when one exists', () => { - mockRepository.find.mockResolvedValue([]) - service.onApplicationBootstrap() - expect(service['retryInterval']).toBeDefined() - service.onModuleDestroy() - expect(service['logger'].log).toHaveBeenCalledWith(expect.stringContaining('stopped')) - }) - - it('should handle destroy when no interval', () => { - service.onModuleDestroy() - expect(true).toBe(true) - }) - }) - - describe('sweep lock', () => { - // Every pod runs this timer, so without the lock three to seven replicas pull the same - // PENDING rows every 30 seconds and race to queue them. - it('does not sweep when another pod holds the lock', async () => { - mockRedisClient.set.mockResolvedValue(null) - - await service.retryPendingNotifications() - - expect(mockRepository.find).not.toHaveBeenCalled() - expect(mockQueue.add).not.toHaveBeenCalled() - }) - - it('takes the lock with NX and a TTL, so a dead holder cannot block the sweep forever', async () => { - mockRepository.find.mockResolvedValue([]) - - await service.retryPendingNotifications() - - expect(mockRedisClient.set).toHaveBeenCalledWith( - 'notify:pending-retry:lock', - expect.any(String), - 'PX', - 60000, - 'NX', - ) - }) - - it('releases the lock after sweeping', async () => { - mockRepository.find.mockResolvedValue([]) - - await service.retryPendingNotifications() - - expect(mockRedisClient.eval).toHaveBeenCalledWith( - expect.stringContaining('redis.call("del", KEYS[1])'), - 1, - 'notify:pending-retry:lock', - expect.any(String), - ) - }) - - it('releases the lock even when the sweep throws', async () => { - mockRepository.find.mockRejectedValue(new Error('database gone')) - - await service.retryPendingNotifications() - - expect(mockRedisClient.eval).toHaveBeenCalled() - }) - - it('releases only its own lock, so an overrun does not free another pod', async () => { - mockRepository.find.mockResolvedValue([]) - - await service.retryPendingNotifications() - - const [script, , , heldBy] = mockRedisClient.eval.mock.calls[0] - expect(script).toContain('redis.call("get", KEYS[1]) == ARGV[1]') - expect(heldBy).toBe(mockRedisClient.set.mock.calls[0][1]) - }) - - it('skips the pass when Redis cannot be reached', async () => { - // Queueing needs the same Redis, so there is nothing useful to do this pass. - mockRedisClient.set.mockRejectedValue(new Error('ECONNREFUSED')) - - await service.retryPendingNotifications() - - expect(mockRepository.find).not.toHaveBeenCalled() - }) - }) - - describe('job configuration', () => { - it('should configure jobs with retry and backoff', async () => { - const notifications = [ - { - id: 'notify-1', - tenantId: 'tenant-1', - status: NotificationStatus.PENDING, - payload: { email: 'test@example.com' }, - createdAt: new Date('2024-01-01T10:00:00Z'), - }, - ] - mockRepository.find.mockResolvedValue(notifications) - mockQueue.add.mockResolvedValue({ id: 'job-123' }) - mockNotificationService.update.mockResolvedValue({}) - await service.retryPendingNotifications() - const jobConfig = mockQueue.add.mock.calls[0][1] - expect(jobConfig.attempts).toBe(3) - expect(jobConfig.backoff).toEqual({ type: 'exponential', delay: 2000 }) - // Bounded by age and count: an unbounded completed set eventually fills a Redis that - // is configured to reject writes rather than evict. - expect(jobConfig.removeOnComplete).toEqual(COMPLETED_JOB_RETENTION) - expect(jobConfig.removeOnFail).toEqual(FAILED_JOB_RETENTION) - }) - - it('should use notification id as job id', async () => { - const notifications = [ - { - id: 'my-unique-id', - tenantId: 'tenant-1', - status: NotificationStatus.PENDING, - payload: { email: 'test@example.com' }, - createdAt: new Date('2024-01-01T10:00:00Z'), - }, - ] - mockRepository.find.mockResolvedValue(notifications) - mockQueue.add.mockResolvedValue({ id: 'job-123' }) - mockNotificationService.update.mockResolvedValue({}) - await service.retryPendingNotifications() - const jobConfig = mockQueue.add.mock.calls[0][1] - expect(jobConfig.jobId).toBe('my-unique-id') - }) - - it('should include all required fields in job data', async () => { - const notifications = [ - { - id: 'notify-1', - tenantId: 'tenant-1', - status: NotificationStatus.PENDING, - payload: { email: 'test@example.com', subject: 'Test' }, - createdAt: new Date('2024-01-01T10:00:00Z'), - }, - ] - mockRepository.find.mockResolvedValue(notifications) - mockQueue.add.mockResolvedValue({ id: 'job-123' }) - mockNotificationService.update.mockResolvedValue({}) - await service.retryPendingNotifications() - const jobData = mockQueue.add.mock.calls[0][0] - expect(jobData).toHaveProperty('notifyId', 'notify-1') - expect(jobData).toHaveProperty('tenantId', 'tenant-1') - expect(jobData).toHaveProperty('request') - expect(jobData).toHaveProperty('requestedAt') - }) - }) -}) diff --git a/backend/src/queue/services/pending-notification-retry.service.ts b/backend/src/queue/services/pending-notification-retry.service.ts deleted file mode 100644 index e28ea209a..000000000 --- a/backend/src/queue/services/pending-notification-retry.service.ts +++ /dev/null @@ -1,253 +0,0 @@ -import { Injectable, Logger, Inject, OnApplicationBootstrap, OnModuleDestroy } from '@nestjs/common' -import { randomUUID } from 'crypto' -import { Repository } from 'typeorm' -import { InjectRepository } from '@nestjs/typeorm' -import Bull from 'bull' -import { NotificationRequest } from '../../api/notification/entities/notification-request.entity' -import { mergeContentFromRequest, mergeRequestChannel } from '../merge-batch-builder' -import type { IngestionJobPayload } from '../queue.types' -import { NotificationService } from '../../api/notification/notification.service' -import { NotificationStatus } from '../../enum/notification-status.enum' -import { QueueName } from '../../enum/queue-name.enum' -import { redisKey } from '../../common/redis/redis-namespace' -import { COMPLETED_JOB_RETENTION, FAILED_JOB_RETENTION } from '../job-retention' - -/** - * Redis key held by whichever pod is currently sweeping. Namespaced per deployment: the Redis - * instance is shared, and one deployment must not hold a lock that stops another sweeping its - * own database. - */ -const SWEEP_LOCK_KEY = redisKey('pending-retry:lock') - -/** - * Release only our own lock. A sweep that overran its TTL must not delete the lock a - * different pod has since taken, or two pods would sweep at once. - */ -const RELEASE_LOCK_SCRIPT = ` -if redis.call("get", KEYS[1]) == ARGV[1] then - return redis.call("del", KEYS[1]) -end -return 0 -` - -/** - * Pending Notification Retry Service - * - * Periodically retries PENDING notifications that couldn't be queued. - * This handles the case where Redis was temporarily unavailable when the - * notification was received. - * - * Flow: - * 1. Take the sweep lock, so only one pod sweeps at a time - * 2. Find a batch of notifications with status PENDING, oldest first - * 3. Try to add each one to the queue - * 4. Update to QUEUED if successful - * 5. Leave as PENDING if Redis still unavailable (will retry next interval) - * - * Schedule: Every 30 seconds (configurable via PENDING_RETRY_INTERVAL env var) - * - * The lock matters because every pod runs this timer: without it, three to seven replicas - * each pull the same PENDING rows every 30 seconds and race to queue them. Bull's jobId - * deduplication hides the duplicate sends, but the wasted work is real and it is at its worst - * during the backlog this job exists to drain. The batch limit bounds that further - a - * backlog is drained over several passes rather than loaded into memory in one. - */ -@Injectable() -export class PendingNotificationRetryService implements OnApplicationBootstrap, OnModuleDestroy { - private readonly logger = new Logger(PendingNotificationRetryService.name) - private retryInterval: NodeJS.Timeout | null = null - /** Identifies this pod's hold on the lock, so it only ever releases its own. */ - private readonly instanceId = randomUUID() - - constructor( - @InjectRepository(NotificationRequest) - private readonly notificationRepository: Repository, - private readonly notificationService: NotificationService, - @Inject(QueueName.INGESTION) private readonly ingestionQueue: Bull.Queue, - ) {} - - /** - * Retry pending notifications on a fixed schedule (every 30 seconds) - * Started automatically when application boots - */ - async retryPendingNotifications(): Promise { - this.logger.debug('Running pending notification retry job...') - - const lockTtlMs = parseInt(process.env.PENDING_RETRY_LOCK_TTL || '60000', 10) - if (!(await this.acquireSweepLock(lockTtlMs))) { - this.logger.debug('Another pod holds the pending notification sweep lock, skipping') - return - } - - try { - await this.sweep() - } finally { - await this.releaseSweepLock() - } - } - - private async sweep(): Promise { - try { - const batchSize = parseInt(process.env.PENDING_RETRY_BATCH_SIZE || '100', 10) - - // Oldest first, so a backlog drains in arrival order across passes rather than the - // same head of the table being re-read while later rows starve. - const pendingNotifications = await this.notificationRepository.find({ - where: { status: NotificationStatus.PENDING }, - order: { createdAt: 'ASC' }, - take: batchSize, - }) - - if (pendingNotifications.length === 0) { - this.logger.debug('No pending notifications to retry') - return - } - - this.logger.log( - `Found ${pendingNotifications.length} pending notifications, attempting to queue...`, - ) - - let successCount = 0 - let failureCount = 0 - - for (const notification of pendingNotifications) { - try { - this.logger.debug(`Processing pending notification: ${notification.id}`, { - payload: notification.payload, - hasQueue: !!this.ingestionQueue, - queueName: this.ingestionQueue?.name, - }) - - // Unnamed, like every other add: the ingestion worker registers Bull's default - // handler, and a named job finds no handler and fails as soon as it is picked up. - await this.ingestionQueue.add(this.ingestionPayloadFor(notification), { - jobId: notification.id, - attempts: 3, - backoff: { - type: 'exponential', - delay: 2000, - }, - removeOnComplete: COMPLETED_JOB_RETENTION, - removeOnFail: FAILED_JOB_RETENTION, - }) - - // Update status to QUEUED - await this.notificationService.update(notification.id, notification.tenantId, { - status: NotificationStatus.QUEUED, - updatedBy: 'retry-scheduler', - }) - - successCount++ - this.logger.log(`Successfully queued retry: ${notification.id}`) - } catch (error) { - failureCount++ - this.logger.warn( - `Failed to queue retry for notification ${notification.id}: ${(error as Error).message}`, - { errorStack: (error as Error).stack }, - ) - // Don't throw - continue with next notification - } - } - - this.logger.log( - `Pending notification retry completed: ${successCount} succeeded, ${failureCount} failed`, - ) - } catch (error) { - this.logger.error( - `Error in pending notification retry job: ${(error as Error).message}`, - (error as Error).stack, - ) - // Don't throw - scheduler should continue running - } - } - - /** - * The ingestion job the request would have been queued with when it was accepted. A merge - * needs its fan-out fields, or ingestion treats it as a plain send; its recipients are read - * by ingestion from the stored request, as for any merge. - */ - private ingestionPayloadFor(notification: NotificationRequest): IngestionJobPayload { - const base = { - notifyId: notification.id, - tenantId: notification.tenantId, - request: (notification.payload || {}) as IngestionJobPayload['request'], - requestedAt: notification.createdAt.toISOString(), - } - const channel = mergeRequestChannel(notification.payload) - if (!channel) return base - - const mailMergeData = mergeContentFromRequest(notification.payload, channel) - return { - ...base, - request: { templateId: mailMergeData.content?.templateId } as IngestionJobPayload['request'], - mailMerge: true, - mailMergeChannel: channel, - mailMergeData, - } - } - - /** - * Claim the sweep for this pod. The TTL is a crash guard: the lock is released in a finally - * block on the happy path, and expires on its own if the holder dies mid-sweep. - * - * A Redis failure here means no sweep this pass, which is the right answer - queueing is - * what the sweep does, and that needs the same Redis. - */ - private async acquireSweepLock(ttlMs: number): Promise { - try { - const result = await this.ingestionQueue.client.set( - SWEEP_LOCK_KEY, - this.instanceId, - 'PX', - ttlMs, - 'NX', - ) - return result === 'OK' - } catch (error) { - this.logger.warn( - `Could not reach Redis to take the sweep lock, skipping this pass: ${(error as Error).message}`, - ) - return false - } - } - - private async releaseSweepLock(): Promise { - try { - await this.ingestionQueue.client.eval(RELEASE_LOCK_SCRIPT, 1, SWEEP_LOCK_KEY, this.instanceId) - } catch (error) { - // The lock expires on its own, so a failed release costs at most one skipped pass. - this.logger.warn(`Failed to release the sweep lock: ${(error as Error).message}`) - } - } - - /** - * Start the retry interval when application boots - * Runs retryPendingNotifications every 30 seconds - */ - onApplicationBootstrap(): void { - const retryIntervalMs = parseInt(process.env.PENDING_RETRY_INTERVAL || '30000', 10) - this.logger.log(`Starting pending notification retry job (interval: ${retryIntervalMs}ms)`) - - // Run immediately on startup - this.retryPendingNotifications().catch((error) => { - this.logger.error(`Initial retry job failed: ${(error as Error).message}`) - }) - - // Then run on interval - this.retryInterval = setInterval(() => { - this.retryPendingNotifications().catch((error) => { - this.logger.error(`Scheduled retry job failed: ${(error as Error).message}`) - }) - }, retryIntervalMs) - } - - /** - * Clean up the interval when application shuts down (optional onModuleDestroy) - */ - onModuleDestroy(): void { - if (this.retryInterval) { - clearInterval(this.retryInterval) - this.logger.log('Pending notification retry job stopped') - } - } -} diff --git a/backend/src/queue/workers/ingestion.worker.spec.ts b/backend/src/queue/workers/ingestion.worker.spec.ts index 04d42fce6..968cfc323 100644 --- a/backend/src/queue/workers/ingestion.worker.spec.ts +++ b/backend/src/queue/workers/ingestion.worker.spec.ts @@ -31,6 +31,9 @@ describe('IngestionWorker', () => { updateStatus: vi.fn().mockResolvedValue(undefined), countBatch: vi.fn().mockResolvedValue(0), findAddressesByStatus: vi.fn().mockResolvedValue(new Set()), + // A first run: no rows yet, so every channel is queued. + countUnbatched: vi.fn().mockResolvedValue(0), + countInFlight: vi.fn().mockResolvedValue(1), } mockConfigService = { @@ -795,6 +798,46 @@ describe('IngestionWorker', () => { expect(mockEmailQueue.add).not.toHaveBeenCalled() }) + describe('re-run of a plain send', () => { + it('writes no rows twice, skips a finished channel and leaves row statuses alone', async () => { + // An earlier run wrote the rows and email finished; only SMS is still owed. + mockRequestDetailService.countUnbatched.mockResolvedValue(3) + mockRequestDetailService.countInFlight.mockImplementation((_id: string, channel: string) => + Promise.resolve(channel === NotificationChannel.EMAIL ? 0 : 1), + ) + await IngestionWorker.initialize( + mockIngestionQueue as Bull.Queue, + mockEmailQueue as Bull.Queue, + mockSmsQueue as Bull.Queue, + mockNotificationService, + mockRequestDetailService, + mockConfigService, + mockClamavService, + ) + + await processHandler({ + data: { + notifyId: 'notify-rerun', + tenantId: 'tenant-1', + requestedAt: new Date().toISOString(), + request: { + email: { + recipients: { to: ['a@example.com'] }, + content: { subject: 'S', body: 'B' }, + }, + sms: { recipients: { to: ['+12505550123'] }, content: { body: 'B' } }, + } as any, + }, + } as Bull.Job) + + expect(mockRequestDetailService.createPending).not.toHaveBeenCalled() + expect(mockEmailQueue.add).not.toHaveBeenCalled() + expect(mockSmsQueue.add).toHaveBeenCalledTimes(1) + // A blanket PROCESSING would put the sent email rows back in flight. + expect(mockRequestDetailService.updateStatus).not.toHaveBeenCalled() + }) + }) + describe('bulk email fan-out', () => { it('should fan out a bulk job into one batch and update status to PROCESSING', async () => { await IngestionWorker.initialize( diff --git a/backend/src/queue/workers/ingestion.worker.ts b/backend/src/queue/workers/ingestion.worker.ts index caaef196f..d6e2bb008 100644 --- a/backend/src/queue/workers/ingestion.worker.ts +++ b/backend/src/queue/workers/ingestion.worker.ts @@ -122,6 +122,8 @@ export class IngestionWorker { logger.debug(`[${notifyId}] Processing ingestion job for tenant=${tenantId}`) + // Set once ingestion finds rows an earlier run wrote; read by both the success and error paths. + let resumed = false try { logger.log(`[${notifyId}] [IngestionWorker] Starting to process job ${job.id}`) @@ -221,8 +223,12 @@ export class IngestionWorker { } else { processedRequest = IngestionWorker.normalizeSmsRecipients(request, phoneNumberService) - // Create notification request detail entries for regular notification request - await requestDetailService.createPending(notifyId, processedRequest, tenantId) + // A re-run (Bull retry, or recovery by the delivery reconciler) finds its rows already + // written; writing them again would double every recipient. + resumed = (await requestDetailService.countUnbatched(notifyId)) > 0 + if (!resumed) { + await requestDetailService.createPending(notifyId, processedRequest, tenantId) + } } const channelAttachments = [ @@ -358,6 +364,13 @@ export class IngestionWorker { // Queue delivery jobs with idempotency and tracing for (const { queue, channel, payload } of deliveryJobs) { + // On a re-run, a channel whose recipients are all sent or failed is done; queueing it + // again would re-send it while recovering the other channel. + if (resumed && (await requestDetailService.countInFlight(notifyId, channel)) === 0) { + logger.log(`[${notifyId}] Skipping ${channel}: delivered on an earlier run`) + continue + } + const deliveryPayload: DeliveryJobPayload = { notifyId, tenantId, @@ -406,7 +419,11 @@ export class IngestionWorker { updatedBy: 'ingestion-worker', }) - await requestDetailService.updateStatus(notifyId, NotificationStatus.PROCESSING) + // Not on a re-run: some rows are already sent or failed, and a blanket update would mark + // them in flight again - and the reconciler would then send them twice. + if (!resumed) { + await requestDetailService.updateStatus(notifyId, NotificationStatus.PROCESSING) + } return { success: true, deliveryJobsQueued: deliveryJobs.length } } catch (error) { @@ -421,7 +438,9 @@ export class IngestionWorker { status: NotificationStatus.FAILED, updatedBy: 'ingestion-worker', }) - await requestDetailService.updateStatus(notifyId, NotificationStatus.FAILED) + await requestDetailService.updateStatus(notifyId, NotificationStatus.FAILED, { + preserveCompleted: resumed, + }) // Re-throw to trigger BullMQ retry logic throw error diff --git a/backend/test/app.e2e-spec.ts b/backend/test/app.e2e-spec.ts index b51d1fe0f..a6572061b 100644 --- a/backend/test/app.e2e-spec.ts +++ b/backend/test/app.e2e-spec.ts @@ -3,7 +3,6 @@ import { Test } from '@nestjs/testing' import type { INestApplication } from '@nestjs/common' import { ConfigService } from '@nestjs/config' import { AppModule } from '../src/app.module' -import { PendingNotificationRetryService } from '../src/queue/services/pending-notification-retry.service' import { NotificationPubSubService } from '../src/api/notification/notification-pubsub.service' import { ModulesContainer } from '@nestjs/core' import { TypeOrmModule } from '@nestjs/typeorm' @@ -50,10 +49,6 @@ describe('AppController (e2e)', () => { }) .overrideProvider(ConfigService) .useValue(configMock) - .overrideProvider(PendingNotificationRetryService) - .useValue({ - onModuleInit: () => Promise.resolve(), - }) .overrideProvider(NotificationPubSubService) .useValue({ publish: () => Promise.resolve(), From 1d51d403fb09960e9333de8eebfe536f7054ce46 Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Mon, 5 Oct 2026 12:23:25 -0700 Subject: [PATCH 17/23] Add delivery reconciler monitoring section Adds Redis-backed reconcile activity tracking (last pass, action counts, recent actions) and exposes it through the admin monitoring API with new reconciler DTOs and status evaluation. Updates the admin Monitoring UI to show a Delivery recovery panel, include reconciler warnings in the top banner, and display recent recoveries. Also extends delivery reconciliation to handle overdue scheduled sends and adds comprehensive backend/frontend test coverage for the new behavior. --- .../monitoring/monitoring.service.spec.ts | 119 +++++++++++++++++- .../admin/monitoring/monitoring.service.ts | 91 +++++++++++--- .../schemas/queue-monitoring.dto.ts | 28 +++++ backend/src/queue/reconcile-activity.ts | 90 +++++++++++++ .../delivery-reconciler.service.spec.ts | 106 +++++++++++++++- .../services/delivery-reconciler.service.ts | 46 ++++++- .../interfaces/queueMonitoring.interface.ts | 29 +++++ .../admin/monitoring/QueueMonitoring.spec.tsx | 58 +++++++++ .../admin/monitoring/QueueMonitoring.tsx | 3 + .../monitoring/sections/ReconcilerSection.tsx | 119 ++++++++++++++++++ 10 files changed, 671 insertions(+), 18 deletions(-) create mode 100644 backend/src/queue/reconcile-activity.ts create mode 100644 frontend/src/pages/admin/monitoring/sections/ReconcilerSection.tsx diff --git a/backend/src/api/admin/monitoring/monitoring.service.spec.ts b/backend/src/api/admin/monitoring/monitoring.service.spec.ts index 2106f3a2d..12fbe69ad 100644 --- a/backend/src/api/admin/monitoring/monitoring.service.spec.ts +++ b/backend/src/api/admin/monitoring/monitoring.service.spec.ts @@ -8,6 +8,7 @@ import { averageOfWholeMinutes, buildOverview, buildRedisStats, + buildReconcilerStats, sendingRateFrom, countUnheldActiveJobs, evaluateQueue, @@ -301,8 +302,32 @@ describe('MonitoringService', () => { } as unknown as Bull.Queue } - function fakeRedis(workers: object[] = []) { + /** Reconcile activity as the pipeline returns it: last pass, recent actions, then minute counts. */ + function fakeReconcilePipeline( + lastPass: object | null = null, + recent: object[] = [], + lastMinuteCounts: Record = {}, + ) { + const minutes = Array.from({ length: MONITORING_WINDOW_MINUTES }, (_, i) => + i === MONITORING_WINDOW_MINUTES - 1 ? lastMinuteCounts : {}, + ) + const pipeline: Record = {} + for (const method of ['get', 'lrange', 'hgetall']) pipeline[method] = vi.fn(() => pipeline) + pipeline.exec = vi + .fn() + .mockResolvedValue( + [ + lastPass && JSON.stringify(lastPass), + recent.map((action) => JSON.stringify(action)), + ...minutes, + ].map((value) => [null, value]), + ) + return pipeline + } + + function fakeRedis(workers: object[] = [], pipeline = fakeReconcilePipeline()) { return { + pipeline: vi.fn(() => pipeline), info: vi .fn() .mockResolvedValue( @@ -389,6 +414,48 @@ describe('MonitoringService', () => { expect(result.status).toBe('healthy') }) + it('reports what the delivery reconciler recovered, with tenant names', async () => { + const pipeline = fakeReconcilePipeline( + { at: now - 20_000, durationMs: 40, intervalMs: 60_000, found: 3, outcomes: { requeued: 1 } }, + [ + { + at: now - 20_000, + kind: 'batch', + action: 'gave-up', + jobId: 'n-1-EMAIL-0', + notifyId: 'n-1', + tenantId: 't-1', + }, + ], + { requeued: '2', 'gave-up': '1' }, + ) + const service = new MonitoringService( + fakeRedis([livePod], pipeline), + null, + fakeQueue('email-delivery'), + null, + null, + tenantRepository, + fakeDetailRepository(), + ) + + const result = await service.getQueueMonitoring() + + expect(result.reconciler).toMatchObject({ + lastPassFound: 3, + requeued: 2, + gaveUp: 1, + retried: 0, + status: 'warning', + }) + expect(result.reconciler.recentActions[0]).toMatchObject({ + notificationId: 'n-1', + tenantName: 'Health Ministry', + }) + // A give-up is surfaced in the page's overall status, not only on its own panel. + expect(result.status).toBe('warning') + }) + it('returns recent failures newest first with tenant names and masked reasons', async () => { const failedJob = (id: string, finishedOn: number) => ({ id, @@ -579,3 +646,53 @@ describe('MonitoringService', () => { await expect(service.getQueueMonitoring()).rejects.toBeInstanceOf(ServiceUnavailableException) }) }) + +describe('buildReconcilerStats', () => { + const now = Date.parse('2026-10-05T12:00:00Z') + const counts = { retried: 0, requeued: 0, 'gave-up': 0 } + const pass = (ageMs: number) => ({ + at: now - ageMs, + durationMs: 12, + intervalMs: 60_000, + found: 0, + outcomes: {}, + }) + + it('is healthy when passes are running and nothing was given up', () => { + const stats = buildReconcilerStats( + { lastPass: pass(30_000), counts, recent: [] }, + now, + new Map(), + ) + expect(stats.status).toBe('healthy') + expect(stats.reasons).toEqual([]) + expect(stats.lastPassAt).toBe('2026-10-05T11:59:30.000Z') + }) + + it('warns when the reconciler has missed several passes', () => { + const stats = buildReconcilerStats( + { lastPass: pass(5 * 60_000), counts, recent: [] }, + now, + new Map(), + ) + expect(stats.status).toBe('warning') + expect(stats.reasons[0]).toMatch(/Has not run for 5 min/) + }) + + it('treats no recorded pass as not yet run rather than a fault', () => { + const stats = buildReconcilerStats({ lastPass: null, counts, recent: [] }, now, new Map()) + expect(stats.status).toBe('healthy') + expect(stats.lastPassAt).toBeNull() + expect(stats.intervalMs).toBeNull() + }) + + it('warns when work was given up on', () => { + const stats = buildReconcilerStats( + { lastPass: pass(1_000), counts: { ...counts, 'gave-up': 2 }, recent: [] }, + now, + new Map(), + ) + expect(stats.status).toBe('warning') + expect(stats.reasons[0]).toMatch(/Gave up on 2 stuck send/) + }) +}) diff --git a/backend/src/api/admin/monitoring/monitoring.service.ts b/backend/src/api/admin/monitoring/monitoring.service.ts index bccd607c3..e0dafa0b0 100644 --- a/backend/src/api/admin/monitoring/monitoring.service.ts +++ b/backend/src/api/admin/monitoring/monitoring.service.ts @@ -26,6 +26,8 @@ import { workerHeartbeatKey, } from '../../../queue/worker-heartbeat' import type { WorkerHeartbeatPayload } from '../../../queue/worker-heartbeat' +import { readReconcileActivity } from '../../../queue/reconcile-activity' +import type { ReconcileActivity } from '../../../queue/reconcile-activity' import { MONITORING_THRESHOLDS } from './monitoring-thresholds' import type { HealthStatus, @@ -33,6 +35,7 @@ import type { MonitoringOverviewDto, QueueMonitoringResponseDto, QueueStatsDto, + ReconcilerStatsDto, RedisStatsDto, WorkerPodDto, } from './schemas/queue-monitoring.dto' @@ -57,6 +60,9 @@ const RECENT_FAILURES_LIMIT = 20 /** A failure rate over a handful of jobs is noise; below this many finished jobs it is ignored. */ const MIN_JOBS_FOR_FAILURE_RATE = 10 +/** Passes missed before the reconciler is reported as not running. */ +const RECONCILER_MISSED_PASSES = 3 + const SEVERITY: Record = { healthy: 0, warning: 1, critical: 2 } export function worstStatus(statuses: HealthStatus[]): HealthStatus { @@ -299,6 +305,47 @@ export function buildOverview( } } +export function buildReconcilerStats( + activity: ReconcileActivity, + now: number, + tenantNames: Map, +): ReconcilerStatsDto { + const { lastPass, counts, recent } = activity + const reasons: string[] = [] + // No pass recorded yet is a fresh Redis or a first deploy, not a fault; one follows within a minute. + if (lastPass && now - lastPass.at > lastPass.intervalMs * RECONCILER_MISSED_PASSES) { + reasons.push( + `Has not run for ${Math.round((now - lastPass.at) / 60_000)} min; stuck sends are not being recovered`, + ) + } + if (counts['gave-up'] > 0) { + reasons.push( + `Gave up on ${counts['gave-up'].toLocaleString()} stuck send(s) in the last ${MONITORING_WINDOW_MINUTES} min; what they owed is marked failed`, + ) + } + return { + status: reasons.length > 0 ? 'warning' : 'healthy', + reasons, + lastPassAt: lastPass ? new Date(lastPass.at).toISOString() : null, + intervalMs: lastPass?.intervalMs ?? null, + lastPassDurationMs: lastPass?.durationMs ?? null, + lastPassFound: lastPass?.found ?? null, + windowMinutes: MONITORING_WINDOW_MINUTES, + retried: counts.retried, + requeued: counts.requeued, + gaveUp: counts['gave-up'], + recentActions: recent.map((action) => ({ + at: new Date(action.at).toISOString(), + kind: action.kind, + action: action.action, + jobId: action.jobId, + notificationId: action.notifyId, + tenantId: action.tenantId, + tenantName: tenantNames.get(action.tenantId) ?? null, + })), + } +} + /** * Read-only view of the Bull queues, their workers and the Redis instance behind them, for the * admin monitoring page. Everything is read from Redis on each request, so the answer is the @@ -331,16 +378,25 @@ export class MonitoringService { } const now = Date.now() - const [info, workers, messages, failedJobs, activeByQueue, deliveryTime, sendingRate] = - await Promise.all([ - this.redis.info(), - this.getWorkers(now), - this.getMessageStats(now), - this.getRecentFailedJobs(), - this.getActiveJobs(), - this.getDeliveryTime(now), - this.getSendingRate(now), - ]) + const [ + info, + workers, + messages, + failedJobs, + activeByQueue, + deliveryTime, + sendingRate, + reconcileActivity, + ] = await Promise.all([ + this.redis.info(), + this.getWorkers(now), + this.getMessageStats(now), + this.getRecentFailedJobs(), + this.getActiveJobs(), + this.getDeliveryTime(now), + this.getSendingRate(now), + readReconcileActivity(this.redis, now, MONITORING_WINDOW_MINUTES), + ]) const redis = buildRedisStats(parseRedisInfo(info)) const queues = await Promise.all( this.queues.map((queue) => @@ -348,15 +404,21 @@ export class MonitoringService { ), ) const activeJobs = activeBatches(activeByQueue) - const tenantNames = await this.getTenantNames( - [...failedJobs, ...activeJobs].map(({ job }) => + const tenantNames = await this.getTenantNames([ + ...[...failedJobs, ...activeJobs].map(({ job }) => asString((job.data as JobIdentifiers)?.tenantId), ), - ) + ...reconcileActivity.recent.map((action) => action.tenantId), + ]) + const reconciler = buildReconcilerStats(reconcileActivity, now, tenantNames) return { generatedAt: new Date(now).toISOString(), - status: worstStatus([redis.status, ...queues.map((queue) => queue.status)]), + status: worstStatus([ + redis.status, + reconciler.status, + ...queues.map((queue) => queue.status), + ]), overview: buildOverview(messages, queues, deliveryTime, sendingRate), redis, messages, @@ -369,6 +431,7 @@ export class MonitoringService { startedAt: job.processedOn ? new Date(job.processedOn).toISOString() : null, })), workers, + reconciler, recentFailures: failedJobs.map(({ queue, job }) => ({ queue, jobId: String(job.id), diff --git a/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts b/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts index ce7d2b651..73129134f 100644 --- a/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts +++ b/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts @@ -139,6 +139,33 @@ export interface MonitoringOverviewDto { requestsReceived: number } +export interface ReconcilerActionDto { + at: string + /** What was owed: pending, ingestion, scheduled, batch or delivery. */ + kind: string + action: 'retried' | 'requeued' | 'gave-up' + jobId: string + notificationId: string + tenantId: string + tenantName: string | null +} + +/** The delivery reconciler: when it last ran and what it recovered. */ +export interface ReconcilerStatsDto { + status: HealthStatus + reasons: string[] + lastPassAt: string | null + intervalMs: number | null + lastPassDurationMs: number | null + /** Work items the last pass checked, including those whose job was still running. */ + lastPassFound: number | null + windowMinutes: number + retried: number + requeued: number + gaveUp: number + recentActions: ReconcilerActionDto[] +} + export interface QueueMonitoringResponseDto { generatedAt: string status: HealthStatus @@ -148,5 +175,6 @@ export interface QueueMonitoringResponseDto { queues: QueueStatsDto[] activeBatches: ActiveBatchDto[] workers: WorkerPodDto[] + reconciler: ReconcilerStatsDto recentFailures: RecentFailureDto[] } diff --git a/backend/src/queue/reconcile-activity.ts b/backend/src/queue/reconcile-activity.ts new file mode 100644 index 000000000..fef23a043 --- /dev/null +++ b/backend/src/queue/reconcile-activity.ts @@ -0,0 +1,90 @@ +import type Redis from 'ioredis' +import { redisKey } from '../common/redis/redis-namespace' +import { minuteOf } from './queue-metrics' + +/** + * What the delivery reconciler has done, kept in Redis so the monitoring page reads the same + * answer from any pod: the last pass, per-minute counts of what it acted on, and the most recent + * actions. Only the pod holding the reconcile lock writes, once per pass. + */ + +export type ReconcileKind = 'pending' | 'ingestion' | 'scheduled' | 'batch' | 'delivery' +export type ReconcileAction = 'retried' | 'requeued' | 'gave-up' +export const RECONCILE_ACTIONS: ReconcileAction[] = ['retried', 'requeued', 'gave-up'] + +export interface ReconcilePassRecord { + at: number + durationMs: number + intervalMs: number + /** Work items the finders returned, including those left alone because their job was live. */ + found: number + outcomes: Partial> +} + +export interface ReconcileActionRecord { + at: number + kind: ReconcileKind + action: ReconcileAction + jobId: string + notifyId: string + tenantId: string +} + +export const RECONCILE_LAST_PASS_KEY = redisKey('delivery-reconcile:last-pass') +export const RECONCILE_RECENT_KEY = redisKey('delivery-reconcile:recent') +export const reconcileCountKey = (minute: number) => redisKey(`delivery-reconcile:count:${minute}`) + +export const RECENT_ACTIONS_KEPT = 50 +const COUNT_TTL_SECONDS = 2 * 60 * 60 + +/** Record one pass. Best-effort: the caller logs a failure and recovery itself is unaffected. */ +export async function recordReconcilePass( + redis: Redis, + pass: ReconcilePassRecord, + actions: ReconcileActionRecord[], +): Promise { + const multi = redis.multi().set(RECONCILE_LAST_PASS_KEY, JSON.stringify(pass)) + if (actions.length > 0) { + const countKey = reconcileCountKey(minuteOf(pass.at)) + for (const action of actions) multi.hincrby(countKey, action.action, 1) + multi + .expire(countKey, COUNT_TTL_SECONDS) + .lpush(RECONCILE_RECENT_KEY, ...actions.map((action) => JSON.stringify(action))) + .ltrim(RECONCILE_RECENT_KEY, 0, RECENT_ACTIONS_KEPT - 1) + } + await multi.exec() +} + +export interface ReconcileActivity { + lastPass: ReconcilePassRecord | null + /** Actions per type over the last `windowMinutes`, including the current minute. */ + counts: Record + recent: ReconcileActionRecord[] +} + +export async function readReconcileActivity( + redis: Redis, + now: number, + windowMinutes: number, +): Promise { + const current = minuteOf(now) + const pipeline = redis.pipeline().get(RECONCILE_LAST_PASS_KEY).lrange(RECONCILE_RECENT_KEY, 0, -1) + for (let minute = current - windowMinutes + 1; minute <= current; minute++) { + pipeline.hgetall(reconcileCountKey(minute)) + } + const results = (await pipeline.exec()) ?? [] + const [lastPass, recent, ...minutes] = results.map(([error, value]) => { + if (error) throw error + return value + }) + + const counts: Record = { retried: 0, requeued: 0, 'gave-up': 0 } + for (const minute of minutes as Array>) { + for (const action of RECONCILE_ACTIONS) counts[action] += Number(minute?.[action] ?? 0) || 0 + } + return { + lastPass: lastPass ? (JSON.parse(lastPass as string) as ReconcilePassRecord) : null, + counts, + recent: (recent as string[]).map((entry) => JSON.parse(entry) as ReconcileActionRecord), + } +} diff --git a/backend/src/queue/services/delivery-reconciler.service.spec.ts b/backend/src/queue/services/delivery-reconciler.service.spec.ts index 45917117b..7694c5de7 100644 --- a/backend/src/queue/services/delivery-reconciler.service.spec.ts +++ b/backend/src/queue/services/delivery-reconciler.service.spec.ts @@ -42,6 +42,7 @@ function setup({ deliveries = [], ingestions = [], pending = [], + scheduled = [], ingestionJob = null, }: { job?: Partial | null @@ -54,6 +55,8 @@ function setup({ ingestions?: object[] /** Requests still PENDING; queried before the QUEUED ones. */ pending?: object[] + /** Scheduled sends past their send time; queried after the QUEUED ones. */ + scheduled?: object[] ingestionJob?: Partial | null }) { const results = [stale, deliveries] @@ -88,7 +91,7 @@ function setup({ for (const m of ['select', 'addSelect', 'where', 'andWhere', 'orderBy', 'limit']) { requestBuilder[m] = vi.fn(() => requestBuilder) } - const requestResults = [pending, ingestions] + const requestResults = [pending, ingestions, scheduled] requestBuilder.getRawMany = vi.fn(() => Promise.resolve(requestResults.shift() ?? [])) const requestRepository = { findOne: vi.fn().mockResolvedValue(storedRequest), @@ -103,7 +106,13 @@ function setup({ rows.slice(1).map(([address, name]) => ({ address, params: { name } })), update: vi.fn().mockResolvedValue(undefined), } + const multi: Record> = {} + for (const m of ['set', 'hincrby', 'expire', 'lpush', 'ltrim']) { + multi[m] = vi.fn(() => multi) + } + multi.exec = vi.fn().mockResolvedValue([]) const redis = { + multi: vi.fn(() => multi), set: vi.fn().mockResolvedValue(locked), eval: vi.fn().mockResolvedValue(1), incr: vi.fn().mockResolvedValue(attempt), @@ -128,6 +137,7 @@ function setup({ requestRepository, notificationService, redis, + multi, queue, ingestionQueue, } @@ -216,6 +226,37 @@ describe('DeliveryReconcilerService', () => { expect(queue.add).not.toHaveBeenCalled() }) + it('records the pass and what it acted on for the monitoring page', async () => { + const { service, multi } = setup({ job: jobIn('failed') }) + + await service.reconcile() + + const [, pass] = multi.set.mock.calls[0] + expect(JSON.parse(pass)).toMatchObject({ found: 1, outcomes: { retried: 1 } }) + expect(multi.hincrby).toHaveBeenCalledWith(expect.any(String), 'retried', 1) + expect(JSON.parse(multi.lpush.mock.calls[0][1])).toMatchObject({ + kind: 'batch', + action: 'retried', + jobId: 'req-1-EMAIL-0', + notifyId: 'req-1', + }) + }) + + it('records a pass that found only live work without logging any action', async () => { + const { service, multi } = setup({ job: jobIn('active') }) + + expect(await service.reconcile()).toEqual({ 'in-progress': 1 }) + expect(multi.set).toHaveBeenCalled() + expect(multi.lpush).not.toHaveBeenCalled() + }) + + it('still reconciles when the pass cannot be recorded', async () => { + const { service, multi } = setup({ job: jobIn('failed') }) + multi.exec.mockRejectedValue(new Error('READONLY')) + + expect(await service.reconcile()).toEqual({ retried: 1 }) + }) + it('does nothing when another pod holds the lock', async () => { const { service, queue, detailRepository } = setup({ job: null, locked: null }) @@ -389,4 +430,67 @@ describe('DeliveryReconcilerService', () => { expect(ingestionQueue.add).not.toHaveBeenCalled() }) }) + + describe('scheduled sends past their send time', () => { + const SCHEDULED = [{ notifyId: 'req-1', tenantId: 'tenant-1' }] + const overdueRequest = { + ...storedRequest, + payload: { + email: { + recipients: { to: ['a@example.com'] }, + content: { subject: 'S', body: 'B' }, + delayedSend: '2026-10-04T13:00:00Z', + }, + }, + } + + it('runs ingestion now when the delayed job was lost, leaving the status to ingestion', async () => { + const { service, ingestionQueue, notificationService, requestRepository } = setup({ + stale: [], + scheduled: SCHEDULED, + }) + requestRepository.findOne.mockResolvedValue(overdueRequest) + + expect(await service.reconcile()).toEqual({ requeued: 1 }) + const [payload, options] = ingestionQueue.add.mock.calls[0] + expect(payload).toMatchObject({ notifyId: 'req-1', scheduledFor: '2026-10-04T13:00:00Z' }) + expect(options).toMatchObject({ jobId: 'req-1' }) + expect(options).not.toHaveProperty('delay') + expect(notificationService.update).not.toHaveBeenCalled() + }) + + it('leaves a send whose job is still delayed alone', async () => { + const { service, ingestionQueue } = setup({ + stale: [], + scheduled: SCHEDULED, + ingestionJob: jobIn('delayed'), + }) + + expect(await service.reconcile()).toEqual({ 'in-progress': 1 }) + expect(ingestionQueue.add).not.toHaveBeenCalled() + }) + + it('retries a delayed job that failed', async () => { + const ingestionJob = jobIn('failed') + const { service } = setup({ stale: [], scheduled: SCHEDULED, ingestionJob }) + + expect(await service.reconcile()).toEqual({ retried: 1 }) + expect(ingestionJob.retry).toHaveBeenCalled() + }) + + it('settles the request as failed after the recovery limit', async () => { + const { service, ingestionQueue, notificationService } = setup({ + stale: [], + scheduled: SCHEDULED, + attempt: 6, + }) + + expect(await service.reconcile()).toEqual({ 'gave-up': 1 }) + expect(ingestionQueue.add).not.toHaveBeenCalled() + expect(notificationService.update).toHaveBeenCalledWith('req-1', 'tenant-1', { + status: NotificationStatus.FAILED, + updatedBy: 'delivery-reconciler', + }) + }) + }) }) diff --git a/backend/src/queue/services/delivery-reconciler.service.ts b/backend/src/queue/services/delivery-reconciler.service.ts index 38f536d46..fe8b826a9 100644 --- a/backend/src/queue/services/delivery-reconciler.service.ts +++ b/backend/src/queue/services/delivery-reconciler.service.ts @@ -27,6 +27,8 @@ import { mergeContentFromRequest, } from '../merge-batch-builder' import type { DeliveryJobPayload } from '../queue.types' +import { recordReconcilePass } from '../reconcile-activity' +import type { ReconcileActionRecord, ReconcileKind } from '../reconcile-activity' const intFromEnv = (name: string, fallback: number) => { const value = parseInt(process.env[name] || '', 10) @@ -78,11 +80,13 @@ export type ReconcileOutcome = 'in-progress' | 'retried' | 'requeued' | 'gave-up * - `pending`: a request stored but never queued (Redis unreachable at acceptance), job * `` on ingestion; * - `ingestion`: a request queued but never fanned out, job `` on ingestion; + * - `scheduled`: a scheduled send past its send time but never fanned out, job `` on + * ingestion (its delayed job was lost or failed); * - `batch`: a merge batch's recipients, job `` on the channel's delivery queue; * - `delivery`: a plain send's recipients on one channel, job `_`. */ interface StuckWork { - kind: 'pending' | 'ingestion' | 'batch' | 'delivery' + kind: ReconcileKind jobId: string notifyId: string tenantId: string @@ -156,18 +160,25 @@ export class DeliveryReconcilerService implements OnApplicationBootstrap, OnModu return outcomes } + const startedAt = Date.now() try { const staleBefore = new Date(now - STALE_AFTER_MS) const items = [ ...(await this.findPendingRequests(new Date(now - PENDING_STALE_MS))), ...(await this.findStuckIngestions(staleBefore)), + ...(await this.findOverdueScheduled(staleBefore)), ...(await this.findStaleBatches(staleBefore)), ...(await this.findStaleDeliveries(staleBefore)), ] + const actions: ReconcileActionRecord[] = [] for (const item of items) { try { const outcome = await this.reconcileItem(item) outcomes[outcome] = (outcomes[outcome] ?? 0) + 1 + if (outcome !== 'in-progress') { + const { kind, jobId, notifyId, tenantId } = item + actions.push({ at: Date.now(), kind, action: outcome, jobId, notifyId, tenantId }) + } } catch (error) { this.logger.error( `[${item.notifyId}] Could not reconcile ${item.kind} ${item.jobId}: ${(error as Error).message}`, @@ -176,6 +187,16 @@ export class DeliveryReconcilerService implements OnApplicationBootstrap, OnModu } const acted = (outcomes.retried ?? 0) + (outcomes.requeued ?? 0) + (outcomes['gave-up'] ?? 0) if (acted > 0) this.logger.warn(`Delivery reconcile: ${JSON.stringify(outcomes)}`) + const pass = { + at: now, + durationMs: Date.now() - startedAt, + intervalMs: RECONCILE_INTERVAL_MS, + found: items.length, + outcomes, + } + await recordReconcilePass(this.redis, pass, actions).catch((error: Error) => + this.logger.warn(`Could not record the reconcile pass: ${error.message}`), + ) } catch (error) { this.logger.error(`Delivery reconcile pass failed: ${(error as Error).message}`) } finally { @@ -251,6 +272,26 @@ export class DeliveryReconcilerService implements OnApplicationBootstrap, OnModu return rows.map((row) => ({ kind: 'ingestion', jobId: row.notifyId, ...row })) } + /** + * Scheduled sends still SCHEDULED a stale window after their send time. Ingestion moves a + * scheduled send to PROCESSING when its delayed job runs, so one still waiting this long has a + * job that was lost or failed. Before its send time a scheduled send is never checked: its job + * is meant to be sitting delayed, and checking every future send each pass would not scale. + */ + private async findOverdueScheduled(staleBefore: Date): Promise { + if (!this.ingestionQueue) return [] + const rows = await this.requestRepository + .createQueryBuilder('request') + .select('request.id', 'notifyId') + .addSelect('request.tenant_id', 'tenantId') + .where('request.status = :scheduled', { scheduled: NotificationStatus.SCHEDULED }) + .andWhere('request.delayed_send_time < :staleBefore', { staleBefore }) + .orderBy('request.delayed_send_time', 'ASC') + .limit(ITEMS_PER_KIND_PER_PASS) + .getRawMany<{ notifyId: string; tenantId: string }>() + return rows.map((row) => ({ kind: 'scheduled', jobId: row.notifyId, ...row })) + } + /** Requests still PENDING: stored at acceptance, but their ingestion job never queued. Oldest first. */ private async findPendingRequests(pendingBefore: Date): Promise { if (!this.ingestionQueue) return [] @@ -276,7 +317,8 @@ export class DeliveryReconcilerService implements OnApplicationBootstrap, OnModu } private queueFor(item: StuckWork): Bull.Queue | null | undefined { - if (item.kind === 'ingestion' || item.kind === 'pending') return this.ingestionQueue + if (item.kind === 'ingestion' || item.kind === 'pending' || item.kind === 'scheduled') + return this.ingestionQueue return item.channel === NotificationChannel.SMS ? this.smsQueue : this.emailQueue } diff --git a/frontend/src/interfaces/queueMonitoring.interface.ts b/frontend/src/interfaces/queueMonitoring.interface.ts index 8663329a4..3cf5a43d8 100644 --- a/frontend/src/interfaces/queueMonitoring.interface.ts +++ b/frontend/src/interfaces/queueMonitoring.interface.ts @@ -115,6 +115,34 @@ export interface MonitoringOverview { requestsReceived: number } +export type ReconcilerActionType = 'retried' | 'requeued' | 'gave-up' + +export interface ReconcilerAction { + at: string + /** What was owed: pending, ingestion, scheduled, batch or delivery. */ + kind: string + action: ReconcilerActionType + jobId: string + notificationId: string + tenantId: string + tenantName: string | null +} + +/** The delivery reconciler: when it last ran and what it recovered. */ +export interface ReconcilerStats { + status: HealthStatus + reasons: string[] + lastPassAt: string | null + intervalMs: number | null + lastPassDurationMs: number | null + lastPassFound: number | null + windowMinutes: number + retried: number + requeued: number + gaveUp: number + recentActions: ReconcilerAction[] +} + export interface QueueMonitoring { generatedAt: string status: HealthStatus @@ -124,5 +152,6 @@ export interface QueueMonitoring { queues: QueueStats[] activeBatches: ActiveBatch[] workers: WorkerPod[] + reconciler: ReconcilerStats recentFailures: RecentFailure[] } diff --git a/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx b/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx index 526f086b0..92596d79c 100644 --- a/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx +++ b/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx @@ -125,6 +125,29 @@ function snapshot(overrides: Partial = {}): QueueMonitoring ], }, ], + reconciler: { + status: 'healthy', + reasons: [], + lastPassAt: new Date(now - 20_000).toISOString(), + intervalMs: 60_000, + lastPassDurationMs: 35, + lastPassFound: 4, + windowMinutes: 60, + retried: 1, + requeued: 3, + gaveUp: 0, + recentActions: [ + { + at: new Date(now - 120_000).toISOString(), + kind: 'scheduled', + action: 'requeued', + jobId: 'notif-90', + notificationId: 'notif-90', + tenantId: 't-1', + tenantName: 'Health Ministry', + }, + ], + }, recentFailures: [ { queue: 'email-delivery', @@ -336,6 +359,41 @@ describe('QueueMonitoring', () => { ).toBeInTheDocument() }) + it('shows what the delivery reconciler recovered on the System tab', async () => { + mockedGet.mockResolvedValue(snapshot()) + renderPage('system') + + const recoveries = await screen.findByRole('table', { name: 'Recent recoveries' }) + expect(within(recoveries).getByText('Re-queued')).toBeInTheDocument() + expect(within(recoveries).getByText('Scheduled send')).toBeInTheDocument() + expect(within(recoveries).getByText('notif-90')).toBeInTheDocument() + expect(screen.getByText('Every 1m 0s; checked 4')).toBeInTheDocument() + }) + + it('raises reconciler warnings in the banner and says when it has not run yet', async () => { + const data = snapshot() + data.reconciler = { + ...data.reconciler, + status: 'warning', + reasons: ['Gave up on 2 stuck send(s) in the last 60 min; what they owed is marked failed'], + lastPassAt: null, + intervalMs: null, + lastPassFound: null, + recentActions: [], + } + mockedGet.mockResolvedValue(data) + renderPage() + + expect( + await screen.findByText( + 'Delivery recovery: Gave up on 2 stuck send(s) in the last 60 min; what they owed is marked failed', + ), + ).toBeInTheDocument() + await userEvent.setup().click(screen.getByText('View details on the System tab')) + expect(await screen.findByText('Not run yet')).toBeInTheDocument() + expect(screen.getAllByText('Nothing has needed recovering.')).not.toHaveLength(0) + }) + it('labels a pod that is shutting down', async () => { const data = snapshot() data.workers = [{ ...data.workers[0], draining: true }] diff --git a/frontend/src/pages/admin/monitoring/QueueMonitoring.tsx b/frontend/src/pages/admin/monitoring/QueueMonitoring.tsx index 5f6403b45..fe7232574 100644 --- a/frontend/src/pages/admin/monitoring/QueueMonitoring.tsx +++ b/frontend/src/pages/admin/monitoring/QueueMonitoring.tsx @@ -21,6 +21,7 @@ import ActiveBatchesSection from './sections/ActiveBatchesSection' import RedisSection from './sections/RedisSection' import QueuesSection from './sections/QueuesSection' import WorkersSection from './sections/WorkersSection' +import ReconcilerSection from './sections/ReconcilerSection' import RecentFailuresSection from './sections/RecentFailuresSection' import '@/scss/components/queue-monitoring.scss' @@ -93,6 +94,7 @@ const QueueMonitoring: FC = ({ tab, onTabChange }) => { const warnings = data ? [ ...data.redis.reasons.map((reason) => `Redis: ${reason}`), + ...data.reconciler.reasons.map((reason) => `Delivery recovery: ${reason}`), ...data.queues.flatMap((queue) => queue.reasons.map((reason) => `${formatQueueName(queue.name)}: ${reason}`), ), @@ -190,6 +192,7 @@ const QueueMonitoring: FC = ({ tab, onTabChange }) => { <> + )} diff --git a/frontend/src/pages/admin/monitoring/sections/ReconcilerSection.tsx b/frontend/src/pages/admin/monitoring/sections/ReconcilerSection.tsx new file mode 100644 index 000000000..a8055d1a4 --- /dev/null +++ b/frontend/src/pages/admin/monitoring/sections/ReconcilerSection.tsx @@ -0,0 +1,119 @@ +import type { FC } from 'react' +import PageSubHeading from '@/components/PageSubHeading' +import { StatusBadge } from '@/components/StatusBadge' +import StatTile from '@/components/StatTile' +import DataTable from '@/components/DataTable/DataTable' +import type { TableColumn } from '@/components/DataTable/DataTable' +import type { + ReconcilerAction, + ReconcilerActionType, + ReconcilerStats, +} from '@/interfaces/queueMonitoring.interface' +import { HEALTH_LABELS, formatAgo, formatDuration } from '@/utils/monitoring' + +const KIND_LABELS: Record = { + pending: 'Never queued', + ingestion: 'Never fanned out', + scheduled: 'Scheduled send', + batch: 'Merge batch', + delivery: 'Send', +} + +const ACTION_LABELS: Record = { + retried: 'Retried', + requeued: 'Re-queued', + 'gave-up': 'Gave up', +} + +interface ReconcilerSectionProps { + reconciler: ReconcilerStats + /** Reference time for "ago" labels: when the snapshot was taken. */ + now: number +} + +const ReconcilerSection: FC = ({ reconciler, now }) => { + const windowLabel = `Last ${reconciler.windowMinutes} min` + const lastPassHint = + reconciler.intervalMs === null + ? 'Not run yet' + : `Every ${formatDuration(reconciler.intervalMs)}; checked ${(reconciler.lastPassFound ?? 0).toLocaleString()}` + + const columns: TableColumn[] = [ + { + key: 'at', + label: 'When', + render: (_, row) => {formatAgo(row.at, now)}, + }, + { + key: 'action', + label: 'Action', + render: (_, row) => + row.action === 'gave-up' ? ( + + ) : ( + ACTION_LABELS[row.action] + ), + }, + { key: 'kind', label: 'What was stuck', render: (_, row) => KIND_LABELS[row.kind] ?? row.kind }, + { key: 'tenantName', label: 'Tenant', render: (_, row) => row.tenantName ?? row.tenantId }, + { + key: 'notificationId', + label: 'Notification ID', + render: (_, row) => {row.notificationId}, + }, + ] + + return ( +
+ +
+
+ Finds sends that stopped moving and puts them back on the queue + +
+ {reconciler.reasons.length > 0 && ( +
    + {reconciler.reasons.map((reason) => ( +
  • {reason}
    • + ))} +
+ )} + +
+ + + + +
+ + `${row.at}-${row.kind}-${row.jobId}`} + label="Recent recoveries" + emptyMessage="Nothing has needed recovering." + /> +

+ Every pass compares what Postgres says is still owed with what the queues hold, and puts + stuck work back: a failed job is retried, a lost one re-queued. After repeated attempts it + gives up and marks what was owed as failed, so a send that cannot succeed ends visibly. + Recent recoveries are listed newest first. +

+
+
+ ) +} + +export default ReconcilerSection From 51360ea76869521f38c76e313185b2d0f2e6b885 Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Mon, 5 Oct 2026 13:49:39 -0700 Subject: [PATCH 18/23] Limit CHES email concurrency via Redis Add a Redis-backed concurrency limiter and apply it to CHES email POST requests so in-flight CHES calls are capped across pods. The limiter uses expiring leases, releases slots on success/failure, and fails open if Redis is unavailable. Update CHES config defaults by increasing `CHES_TIMEOUT_MS` to 120000 and adding `CHES_MAX_CONCURRENT_REQUESTS` (default 5, `0` disables limiting). Extend tests to cover limiter behavior and verify CHES adapter uses the shared limit only when enabled. --- .../email/ches/ches-email.adapter.spec.ts | 57 +++++++++++ .../delivery/email/ches/ches-email.adapter.ts | 59 +++++++++--- .../common/redis/concurrency-limiter.spec.ts | 84 ++++++++++++++++ .../src/common/redis/concurrency-limiter.ts | 95 +++++++++++++++++++ backend/src/config/configuration.ts | 5 +- 5 files changed, 288 insertions(+), 12 deletions(-) create mode 100644 backend/src/common/redis/concurrency-limiter.spec.ts create mode 100644 backend/src/common/redis/concurrency-limiter.ts diff --git a/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.spec.ts b/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.spec.ts index 4d83f1c30..5ac85c25e 100644 --- a/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.spec.ts +++ b/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.spec.ts @@ -3,6 +3,11 @@ import { ConfigService } from '@nestjs/config' import { BadGatewayException, GatewayTimeoutException } from '@nestjs/common' import { ChesEmailTransport } from '../../../../../../src/adapters/implementations/delivery/email/ches/ches-email.adapter' import type { SendEmailOptions, SendEmailResult } from '../../../../../../src/adapters/interfaces' +import { RedisConcurrencyLimiter } from '../../../../../../src/common/redis/concurrency-limiter' + +vi.mock('../../../../../../src/queue/redis-connection', () => ({ + createRedisClient: vi.fn(() => ({ quit: vi.fn().mockResolvedValue('OK') })), +})) const fetchMock = vi.fn() global.fetch = fetchMock @@ -862,4 +867,56 @@ describe('ChesEmailTransport', () => { ).rejects.toThrow('fetch failed') }) }) + + describe('concurrency limit', () => { + const config = (overrides: Record) => + ({ + get: (key: string) => + ({ + 'ches.baseUrl': 'https://ches.example.com/api/v1', + 'ches.clientId': 'client-id', + 'ches.clientSecret': 'client-secret', + 'ches.tokenUrl': 'https://auth.example.com/token', + 'ches.from': 'noreply@gov.bc.ca', + redis: { host: 'localhost', port: 6379 }, + ...overrides, + })[key], + }) as unknown as ConfigService + const respond = () => { + fetchMock.mockResolvedValueOnce({ + ok: true, + json: async () => ({ access_token: 'tok', expires_in: 300 }), + }) + fetchMock.mockResolvedValueOnce({ + ok: true, + json: async () => ({ messages: [{ msgId: 'm1' }], txId: 't1' }), + }) + } + + it('sends each email through the shared limit', async () => { + const run = vi + .spyOn(RedisConcurrencyLimiter.prototype, 'run') + .mockImplementation((fn) => fn()) + respond() + const limited = new ChesEmailTransport(config({ 'ches.maxConcurrentRequests': 5 })) + + await expect( + limited.send({ to: 'user@example.com', subject: 'Hi', body: 'Hello' }), + ).resolves.toMatchObject({ messageId: 'm1' }) + // The email POST only: the token request is cached and rare. + expect(run).toHaveBeenCalledTimes(1) + run.mockRestore() + }) + + it('sends unlimited when the limit is 0', async () => { + const run = vi.spyOn(RedisConcurrencyLimiter.prototype, 'run') + respond() + const unlimited = new ChesEmailTransport(config({ 'ches.maxConcurrentRequests': 0 })) + + await unlimited.send({ to: 'user@example.com', subject: 'Hi', body: 'Hello' }) + + expect(run).not.toHaveBeenCalled() + run.mockRestore() + }) + }) }) diff --git a/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.ts b/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.ts index 26324a7d8..5ad899728 100644 --- a/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.ts +++ b/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.ts @@ -5,9 +5,14 @@ import { Injectable, Logger, NotFoundException, + OnModuleDestroy, UnauthorizedException, } from '@nestjs/common' import { ConfigService } from '@nestjs/config' +import type Redis from 'ioredis' +import { RedisConcurrencyLimiter } from '../../../../../common/redis/concurrency-limiter' +import { redisKey } from '../../../../../common/redis/redis-namespace' +import { createRedisClient, type RedisConfig } from '../../../../../queue/redis-connection' import { toEmailHtml } from '../../../../../services/rendering/email-body-html' import { IEmailTransport, SendEmailOptions, SendEmailResult } from '../../../../interfaces' @@ -45,13 +50,42 @@ interface ChesErrorResponse { } @Injectable() -export class ChesEmailTransport implements IEmailTransport { +export class ChesEmailTransport implements IEmailTransport, OnModuleDestroy { readonly name = 'ches' private readonly logger = new Logger(ChesEmailTransport.name) private tokenCache: { token: string; expiresAt: number } | null = null + private readonly redis: Redis | null = null + /** + * Caps how many emails this service is sending to CHES at once, counted across every pod + * (CHES_MAX_CONCURRENT_REQUESTS). Each send waits here for a free slot before it calls CHES. + * Null - no cap - when Redis is not configured (tests) or the limit is 0. + */ + private readonly limiter: RedisConcurrencyLimiter | null = null + + constructor(private readonly configService: ConfigService) { + const redisConfig = this.configService.get('redis') + const limit = this.configService.get('ches.maxConcurrentRequests') ?? 0 + if (!redisConfig || !(limit > 0)) return + + this.redis = createRedisClient(redisConfig, ChesEmailTransport.name, { + // On the send path: a Redis outage has to fall back to unlimited sending in milliseconds. + lazyConnect: true, + enableOfflineQueue: false, + maxRetriesPerRequest: 1, + commandTimeout: 250, + }) + const timeoutMs = this.configService.get('ches.timeoutMs') ?? 120_000 + this.limiter = new RedisConcurrencyLimiter(this.redis, redisKey('ches:in-flight'), { + limit, + // Outlasts the request's own timeout, so a slot is only reclaimed from a pod that died. + leaseMs: timeoutMs + 15_000, + }) + } - constructor(private readonly configService: ConfigService) {} + async onModuleDestroy(): Promise { + await this.redis?.quit().catch(() => this.redis?.disconnect()) + } async send(options: SendEmailOptions): Promise { // Handle both flat (SendEmailOptions) and nested (NotifyEmailChannel) structures @@ -171,14 +205,17 @@ export class ChesEmailTransport implements IEmailTransport { })}`, ) - const response = await this.fetchWithTimeout(`${baseUrl.replace(/\/$/, '')}/email`, 'email', { - method: 'POST', - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(payload), - }) + const post = () => + this.fetchWithTimeout(`${baseUrl.replace(/\/$/, '')}/email`, 'email', { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(payload), + }) + // The wait for a slot sits outside the timeout: it measures CHES, not our own queue. + const response = this.limiter ? await this.limiter.run(post) : await post() if (!response.ok) { const errText = await response.text() @@ -218,7 +255,7 @@ export class ChesEmailTransport implements IEmailTransport { operation: 'email' | 'token', init: RequestInit, ): Promise { - const timeoutMs = this.configService.get('ches.timeoutMs') ?? 30_000 + const timeoutMs = this.configService.get('ches.timeoutMs') ?? 120_000 try { return await fetch(url, { ...init, signal: AbortSignal.timeout(timeoutMs) }) } catch (caught: unknown) { diff --git a/backend/src/common/redis/concurrency-limiter.spec.ts b/backend/src/common/redis/concurrency-limiter.spec.ts new file mode 100644 index 000000000..96f4e3494 --- /dev/null +++ b/backend/src/common/redis/concurrency-limiter.spec.ts @@ -0,0 +1,84 @@ +import { describe, it, expect, vi } from 'vitest' +import type Redis from 'ioredis' +import { RedisConcurrencyLimiter } from './concurrency-limiter' + +/** The acquire script's behaviour over an in-memory set of leases: token -> expiry. */ +function fakeRedis(leases = new Map()) { + return { + leases, + eval: vi.fn( + async ( + _script: string, + _keys: number, + _key: string, + token: string, + leaseMs: number, + limit: number, + ) => { + const now = Date.now() + for (const [held, expiresAt] of leases) if (expiresAt <= now) leases.delete(held) + if (leases.size >= limit) return 0 + leases.set(token, now + leaseMs) + return 1 + }, + ), + zrem: vi.fn(async (_key: string, token: string) => (leases.delete(token) ? 1 : 0)), + } +} + +const limiterOn = (redis: ReturnType, limit = 2) => + new RedisConcurrencyLimiter(redis as unknown as Redis, 'notify:test:in-flight', { + limit, + leaseMs: 60_000, + pollMs: 1, + }) + +describe('RedisConcurrencyLimiter', () => { + it('never runs more than the limit at once, and runs every call', async () => { + const limiter = limiterOn(fakeRedis(), 3) + let running = 0 + let peak = 0 + const call = async (i: number) => { + running++ + peak = Math.max(peak, running) + await new Promise((resolve) => setTimeout(resolve, 5)) + running-- + return i + } + + const results = await Promise.all( + Array.from({ length: 10 }, (_, i) => limiter.run(() => call(i))), + ) + + expect(results).toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + expect(peak).toBe(3) + }) + + it('frees the slot when the call throws', async () => { + const redis = fakeRedis() + const limiter = limiterOn(redis, 1) + + await expect(limiter.run(() => Promise.reject(new Error('CHES down')))).rejects.toThrow( + 'CHES down', + ) + + expect(redis.leases.size).toBe(0) + await expect(limiter.run(() => Promise.resolve('next'))).resolves.toBe('next') + }) + + it('reclaims a slot whose holder never released it', async () => { + const redis = fakeRedis(new Map([['pod-that-died', Date.now() - 1]])) + + await expect(limiterOn(redis, 1).run(() => Promise.resolve('ran'))).resolves.toBe('ran') + }) + + it('runs the call unlimited when Redis cannot be reached', async () => { + const redis = fakeRedis() + redis.eval.mockRejectedValue(new Error('Connection is closed.')) + const limiter = limiterOn(redis, 1) + + await expect(limiter.run(() => Promise.resolve('sent'))).resolves.toBe('sent') + await expect(limiter.run(() => Promise.resolve('sent'))).resolves.toBe('sent') + expect(redis.zrem).not.toHaveBeenCalled() + }) +}) diff --git a/backend/src/common/redis/concurrency-limiter.ts b/backend/src/common/redis/concurrency-limiter.ts new file mode 100644 index 000000000..b2738dff9 --- /dev/null +++ b/backend/src/common/redis/concurrency-limiter.ts @@ -0,0 +1,95 @@ +import { Logger } from '@nestjs/common' +import { randomUUID } from 'crypto' +import type Redis from 'ioredis' + +/** + * Take a slot if fewer than ARGV[3] are held. Each held slot is a sorted-set member scored by + * when it expires; expired ones are dropped first. Uses Redis's clock so every pod agrees. + */ +const ACQUIRE_SCRIPT = ` +local time = redis.call('TIME') +local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000) +redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now) +if redis.call('ZCARD', KEYS[1]) < tonumber(ARGV[3]) then + redis.call('ZADD', KEYS[1], now + tonumber(ARGV[2]), ARGV[1]) + redis.call('PEXPIRE', KEYS[1], tonumber(ARGV[2])) + return 1 +end +return 0 +` + +export interface ConcurrencyLimiterOptions { + /** Most calls in flight at once, across every pod sharing `key`. */ + limit: number + /** How long a slot is held if never released. Longer than the slowest call it guards. */ + leaseMs: number + /** Wait between attempts to take a slot; each wait is jittered up to double this. */ + pollMs?: number +} + +const FAIL_OPEN_WARN_INTERVAL_MS = 60_000 + +/** + * "At most N of these calls at once, across every pod." `run(fn)` waits for a free slot, runs + * `fn`, and gives the slot back. The slots live in Redis, so all pods share one count. + * + * Used for CHES. CHES accepts about one email a second however many we send at once. A large + * mail merge runs many batches in parallel on several pods, each sending one recipient at a + * time, so dozens of requests reach CHES together. They do not finish sooner; they queue + * inside CHES, each request takes longer, and some pass our request timeout and are marked + * failed. Waiting for a slot on our side keeps the queue here, where nothing times out, and + * delivers just as many emails a minute. + * + * A slot is a lease with an expiry, so a pod that dies mid-call cannot hold one forever. + * Fails open: if Redis cannot be reached, calls run unlimited rather than stopping delivery. + */ +export class RedisConcurrencyLimiter { + private readonly logger = new Logger(RedisConcurrencyLimiter.name) + private lastFailOpenWarnAt = 0 + + constructor( + private readonly client: Redis, + private readonly key: string, + private readonly options: ConcurrencyLimiterOptions, + ) {} + + async run(fn: () => Promise): Promise { + const token = await this.acquire() + try { + return await fn() + } finally { + if (token) await this.client.zrem(this.key, token).catch(() => undefined) + } + } + + /** The lease token once a slot is held, or null when Redis failed and the call runs unlimited. */ + private async acquire(): Promise { + const token = randomUUID() + const pollMs = this.options.pollMs ?? 200 + for (;;) { + let acquired: unknown + try { + acquired = await this.client.eval( + ACQUIRE_SCRIPT, + 1, + this.key, + token, + this.options.leaseMs, + this.options.limit, + ) + } catch (error) { + this.warnFailOpen(error as Error) + return null + } + if (acquired === 1) return token + await new Promise((resolve) => setTimeout(resolve, pollMs + Math.random() * pollMs)) + } + } + + private warnFailOpen(error: Error): void { + const now = Date.now() + if (now - this.lastFailOpenWarnAt < FAIL_OPEN_WARN_INTERVAL_MS) return + this.lastFailOpenWarnAt = now + this.logger.warn(`Concurrency limit on ${this.key} not applied: ${error.message}`) + } +} diff --git a/backend/src/config/configuration.ts b/backend/src/config/configuration.ts index 14aa86d9a..3415f8dec 100644 --- a/backend/src/config/configuration.ts +++ b/backend/src/config/configuration.ts @@ -61,7 +61,10 @@ export default () => { from: process.env.DEFAULT_EMAIL_FROM || defaultEmailFrom, // Per request. Without one a CHES that accepts the connection but never answers holds a // worker until Node's own ~5-minute timeout, per message. - timeoutMs: parseInt(process.env.CHES_TIMEOUT_MS || '30000', 10), + timeoutMs: parseInt(process.env.CHES_TIMEOUT_MS || '120000', 10), + // Across every pod. CHES accepts messages at a fixed rate, so more callers only make each + // call wait longer, until they pass timeoutMs. 0 turns the limit off. + maxConcurrentRequests: parseInt(process.env.CHES_MAX_CONCURRENT_REQUESTS || '5', 10), }, // Notification events From 7442b08e424574603090d6117194008cc5b55efc Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Mon, 5 Oct 2026 17:15:47 -0700 Subject: [PATCH 19/23] Add provider circuit breakers and owed-send flow Introduces shared transient delivery error handling and a Redis-backed circuit breaker, then applies it to CHES and SMS so provider outages pause sends instead of failing recipients immediately. SMS adapter wiring now always runs through a breaker, provider/HTTP/network failures are classified consistently, and delivery workers keep outage-affected recipients owed (including merge batches) while only failing unsent rows when retries are exhausted for non-transient errors. Monitoring was expanded to show provider circuit state and sends-in-progress rollups (replacing active batch rows), with backend/frontend DTO, service, UI, and test updates to match. --- backend/src/adapters/adapters.module.spec.ts | 18 +- backend/src/adapters/adapters.module.ts | 9 +- backend/src/adapters/delivery-errors.ts | 75 +++++ backend/src/adapters/delivery-keys.ts | 8 + .../email/ches/ches-email.adapter.spec.ts | 236 +++++++++++++- .../delivery/email/ches/ches-email.adapter.ts | 123 ++++++-- .../delivery/sms/acs/acs-sms.adapter.spec.ts | 43 +++ .../delivery/sms/acs/acs-sms.adapter.ts | 50 ++- .../delivery/sms/sms-circuit-breaker.spec.ts | 44 +++ .../delivery/sms/sms-circuit-breaker.ts | 65 ++++ .../sms/twilio/twilio-sms.adapter.spec.ts | 45 +++ .../delivery/sms/twilio/twilio-sms.adapter.ts | 22 +- .../interfaces/delivery/sms.interface.ts | 5 + .../monitoring/monitoring.service.spec.ts | 247 +++++++++++++-- .../admin/monitoring/monitoring.service.ts | 294 +++++++++++++++--- .../schemas/queue-monitoring.dto.ts | 49 ++- .../notification-request-detail.service.ts | 15 + .../src/common/redis/circuit-breaker.spec.ts | 148 +++++++++ backend/src/common/redis/circuit-breaker.ts | 197 ++++++++++++ .../workers/email-delivery.worker.spec.ts | 116 +++++++ .../queue/workers/email-delivery.worker.ts | 64 +++- .../queue/workers/sms-delivery.worker.spec.ts | 136 ++++++++ .../src/queue/workers/sms-delivery.worker.ts | 69 +++- .../interfaces/queueMonitoring.interface.ts | 43 ++- .../admin/monitoring/QueueMonitoring.spec.tsx | 115 ++++++- .../admin/monitoring/QueueMonitoring.tsx | 9 +- .../sections/ActiveBatchesSection.tsx | 68 ---- .../monitoring/sections/ProvidersSection.tsx | 81 +++++ .../sections/SendsInProgressSection.tsx | 87 ++++++ .../src/scss/components/queue-monitoring.scss | 7 + 30 files changed, 2263 insertions(+), 225 deletions(-) create mode 100644 backend/src/adapters/delivery-errors.ts create mode 100644 backend/src/adapters/delivery-keys.ts create mode 100644 backend/src/adapters/implementations/delivery/sms/sms-circuit-breaker.spec.ts create mode 100644 backend/src/adapters/implementations/delivery/sms/sms-circuit-breaker.ts create mode 100644 backend/src/common/redis/circuit-breaker.spec.ts create mode 100644 backend/src/common/redis/circuit-breaker.ts delete mode 100644 frontend/src/pages/admin/monitoring/sections/ActiveBatchesSection.tsx create mode 100644 frontend/src/pages/admin/monitoring/sections/ProvidersSection.tsx create mode 100644 frontend/src/pages/admin/monitoring/sections/SendsInProgressSection.tsx diff --git a/backend/src/adapters/adapters.module.spec.ts b/backend/src/adapters/adapters.module.spec.ts index 8cbc31e01..88dfd9edb 100644 --- a/backend/src/adapters/adapters.module.spec.ts +++ b/backend/src/adapters/adapters.module.spec.ts @@ -11,6 +11,7 @@ import { NodemailerEmailTransport } from '../../src/adapters/implementations/del import { LogEmailTransport } from '../../src/adapters/implementations/delivery/email/log/log-email.adapter' import { TwilioSmsTransport } from '../../src/adapters/implementations/delivery/sms/twilio/twilio-sms.adapter' import { AcsSmsTransport } from '../../src/adapters/implementations/delivery/sms/acs/acs-sms.adapter' +import { SmsCircuitBreaker } from '../../src/adapters/implementations/delivery/sms/sms-circuit-breaker' describe('AdaptersModule', () => { it('forRoot returns dynamic module with all adapters and maps', () => { @@ -18,7 +19,8 @@ describe('AdaptersModule', () => { expect(dynamic.module).toBe(AdaptersModule) expect(dynamic.global).toBe(true) - expect(dynamic.providers).toHaveLength(11) + expect(dynamic.providers).toHaveLength(12) + expect(dynamic.providers).toContain(SmsCircuitBreaker) expect(dynamic.exports).toContain(EMAIL_ADAPTER) expect(dynamic.exports).toContain(SMS_ADAPTER) expect(dynamic.exports).toContain(EMAIL_ADAPTER_MAP) @@ -39,4 +41,18 @@ describe('AdaptersModule', () => { expect(adapterProviders).toContain(TwilioSmsTransport) expect(adapterProviders).toContain(AcsSmsTransport) }) + + it('sends SMS through the circuit breaker, whichever provider is configured', () => { + const provider = (AdaptersModule.forRoot().providers ?? []).find( + (p) => (p as { provide?: unknown }).provide === SMS_ADAPTER, + ) as { useFactory: (...args: unknown[]) => unknown } + const acs = { name: 'acs', send: vi.fn() } + const wrapped = { name: 'acs', send: vi.fn() } + const breaker = { wrap: vi.fn(() => wrapped) } + + const adapter = provider.useFactory({ acs }, { get: () => 'acs' }, breaker) + + expect(breaker.wrap).toHaveBeenCalledWith(acs) + expect(adapter).toBe(wrapped) + }) }) diff --git a/backend/src/adapters/adapters.module.ts b/backend/src/adapters/adapters.module.ts index b374aeef0..31790096d 100644 --- a/backend/src/adapters/adapters.module.ts +++ b/backend/src/adapters/adapters.module.ts @@ -5,6 +5,7 @@ import { NodemailerEmailTransport } from './implementations/delivery/email/nodem import { LogEmailTransport } from './implementations/delivery/email/log/log-email.adapter' import { TwilioSmsTransport } from './implementations/delivery/sms/twilio/twilio-sms.adapter' import { AcsSmsTransport } from './implementations/delivery/sms/acs/acs-sms.adapter' +import { SmsCircuitBreaker } from './implementations/delivery/sms/sms-circuit-breaker' import { EMAIL_ADAPTER, EMAIL_ADAPTER_MAP, @@ -37,6 +38,7 @@ export class AdaptersModule { LogEmailTransport, TwilioSmsTransport, AcsSmsTransport, + SmsCircuitBreaker, { provide: EMAIL_ADAPTER_MAP, useFactory: ( @@ -83,16 +85,17 @@ export class AdaptersModule { useFactory: ( map: Record, configService: ConfigService, + breaker: SmsCircuitBreaker, ): ISmsTransport => { const key = configService.get('delivery.sms') ?? 'acs' // As above: a `:passthrough` suffix names no ISmsTransport, so fall back for DI. if (key?.includes(':passthrough')) { - return map['acs'] + return breaker.wrap(map['acs']) } const provider = key?.includes(':') ? key.split(':')[0] : key - return map[provider] ?? map['acs'] + return breaker.wrap(map[provider] ?? map['acs']) }, - inject: [SMS_ADAPTER_MAP, ConfigService], + inject: [SMS_ADAPTER_MAP, ConfigService, SmsCircuitBreaker], }, InMemoryTemplateStore, { provide: SENDER_STORE, useClass: InMemorySenderStore }, diff --git a/backend/src/adapters/delivery-errors.ts b/backend/src/adapters/delivery-errors.ts new file mode 100644 index 000000000..8954bd4e7 --- /dev/null +++ b/backend/src/adapters/delivery-errors.ts @@ -0,0 +1,75 @@ +import { HttpException, HttpStatus } from '@nestjs/common' + +/** + * The provider could not take the message right now - it is down, restarting, overloaded or + * rate limiting - and it accepted nothing. The message itself is fine: send it again later. + * + * Workers must not mark a recipient failed for this. A provider outage of a few seconds would + * otherwise fail every recipient that reached it in those seconds, and since such errors come + * back instantly, a large send would burn through its whole backlog. The work is left owed and + * retried; the circuit breaker in front of the provider stops calls while it recovers. + * + * Not used when the outcome is unknown - a timeout after the request was sent may have been + * delivered, and sending it again could deliver it twice. + */ +export class TransientDeliveryError extends HttpException { + readonly transient = true + + constructor(message: string, status: number = HttpStatus.SERVICE_UNAVAILABLE) { + super(message, status) + } +} + +export function isTransientDeliveryError(error: unknown): error is TransientDeliveryError { + return error instanceof TransientDeliveryError +} + +/** HTTP statuses that mean "not now" rather than "never": the request was not acted on. */ +export function isTransientHttpStatus(status: number): boolean { + return status === 408 || status === 429 || status >= 500 +} + +/** + * Network errors raised before a request reached the provider: nothing was sent. A connection + * reset or a body cut off mid-response is not here - the provider may already have acted. + */ +const NOT_SENT_NETWORK_CODES = new Set(['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN', 'EHOSTUNREACH']) + +/** + * A network error raised before the request reached the provider. The code is on the error itself + * (Azure SDK, axios) or on its `cause` (fetch's `TypeError('fetch failed')`). + */ +export function isNotSentNetworkError(error: unknown): boolean { + const { code, cause } = (error ?? {}) as { code?: unknown; cause?: { code?: unknown } } + return [code, cause?.code].some( + (value) => typeof value === 'string' && NOT_SENT_NETWORK_CODES.has(value), + ) +} + +/** + * An SDK error from a provider, as a TransientDeliveryError when it means the provider took + * nothing: an HTTP status of 408, 429 or 5xx (`statusCode` on Azure's RestError, `status` on + * Twilio's RestException), or a network error before the request was sent. Null otherwise. + */ +export function transientFromProviderError( + error: unknown, + label: string, +): TransientDeliveryError | null { + if (isTransientDeliveryError(error)) return error + // Our own exceptions have a runtime `status` too; only a raw SDK error is classified here. + if (error instanceof HttpException) return null + const { statusCode, status, message } = (error ?? {}) as { + statusCode?: unknown + status?: unknown + message?: unknown + } + const httpStatus = typeof statusCode === 'number' ? statusCode : status + const text = typeof message === 'string' ? message : String(error) + if (typeof httpStatus === 'number' && isTransientHttpStatus(httpStatus)) { + return new TransientDeliveryError(`${label}: upstream ${httpStatus} - ${text}`, 502) + } + if (isNotSentNetworkError(error)) { + return new TransientDeliveryError(`${label} unreachable: ${text}`) + } + return null +} diff --git a/backend/src/adapters/delivery-keys.ts b/backend/src/adapters/delivery-keys.ts new file mode 100644 index 000000000..b4bda111e --- /dev/null +++ b/backend/src/adapters/delivery-keys.ts @@ -0,0 +1,8 @@ +import { redisKey } from '../common/redis/redis-namespace' + +/** CHES email requests in flight across all pods (RedisConcurrencyLimiter). */ +export const CHES_IN_FLIGHT_KEY = redisKey('ches:in-flight') +/** The circuit breaker in front of CHES (RedisCircuitBreaker); read by admin monitoring. */ +export const CHES_CIRCUIT_KEY = redisKey('ches:circuit') +/** The circuit breaker in front of the SMS provider; read by admin monitoring. */ +export const SMS_CIRCUIT_KEY = redisKey('sms:circuit') diff --git a/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.spec.ts b/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.spec.ts index 5ac85c25e..2175cb032 100644 --- a/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.spec.ts +++ b/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.spec.ts @@ -1,9 +1,17 @@ import { Test, TestingModule } from '@nestjs/testing' import { ConfigService } from '@nestjs/config' -import { BadGatewayException, GatewayTimeoutException } from '@nestjs/common' +import { + BadGatewayException, + BadRequestException, + GatewayTimeoutException, + UnauthorizedException, +} from '@nestjs/common' import { ChesEmailTransport } from '../../../../../../src/adapters/implementations/delivery/email/ches/ches-email.adapter' import type { SendEmailOptions, SendEmailResult } from '../../../../../../src/adapters/interfaces' import { RedisConcurrencyLimiter } from '../../../../../../src/common/redis/concurrency-limiter' +import { RedisCircuitBreaker } from '../../../../../../src/common/redis/circuit-breaker' +import { TransientDeliveryError } from '../../../../../../src/adapters/delivery-errors' +import { createRedisClient } from '../../../../../../src/queue/redis-connection' vi.mock('../../../../../../src/queue/redis-connection', () => ({ createRedisClient: vi.fn(() => ({ quit: vi.fn().mockResolvedValue('OK') })), @@ -908,6 +916,39 @@ describe('ChesEmailTransport', () => { run.mockRestore() }) + it('holds startup until the limiter can reach Redis, so the first sends are limited', async () => { + const handlers: Record void> = {} + vi.mocked(createRedisClient).mockReturnValueOnce({ + status: 'connecting', + once: vi.fn((event: string, handler: () => void) => (handlers[event] = handler)), + quit: vi.fn().mockResolvedValue('OK'), + } as never) + const limited = new ChesEmailTransport(config({ 'ches.maxConcurrentRequests': 5 })) + + let started = false + const init = limited.onModuleInit().then(() => (started = true)) + await Promise.resolve() + expect(started).toBe(false) + + handlers.ready() + await init + expect(started).toBe(true) + }) + + it('gets the token once it holds a slot, so a long wait cannot expire it', async () => { + const run = vi.spyOn(RedisConcurrencyLimiter.prototype, 'run').mockImplementation((fn) => { + expect(fetchMock).not.toHaveBeenCalled() + return fn() + }) + respond() + const limited = new ChesEmailTransport(config({ 'ches.maxConcurrentRequests': 5 })) + + await limited.send({ to: 'user@example.com', subject: 'Hi', body: 'Hello' }) + + expect(fetchMock).toHaveBeenCalledTimes(2) + run.mockRestore() + }) + it('sends unlimited when the limit is 0', async () => { const run = vi.spyOn(RedisConcurrencyLimiter.prototype, 'run') respond() @@ -919,4 +960,197 @@ describe('ChesEmailTransport', () => { run.mockRestore() }) }) + + describe('rejected token', () => { + const unlimited = () => + new ChesEmailTransport({ + get: (key: string) => + ({ + 'ches.baseUrl': 'https://ches.example.com/api/v1', + 'ches.clientId': 'client-id', + 'ches.clientSecret': 'client-secret', + 'ches.tokenUrl': 'https://auth.example.com/token', + 'ches.from': 'noreply@gov.bc.ca', + })[key], + } as unknown as ConfigService) + const token = (value: string) => + fetchMock.mockResolvedValueOnce({ + ok: true, + json: async () => ({ access_token: value, expires_in: 300 }), + }) + const unauthorized = () => + fetchMock.mockResolvedValueOnce({ + ok: false, + status: 401, + text: async () => '{"detail":"Token claims invalid: [\\"exp\\"]=\\"token expired\\""}', + }) + const authHeaders = () => + fetchMock.mock.calls + .filter(([url]) => String(url).endsWith('/email')) + .map(([, init]) => (init.headers as Record).Authorization) + + it('retries once with a new token when CHES rejects the cached one', async () => { + token('stale') + unauthorized() + token('fresh') + fetchMock.mockResolvedValueOnce({ + ok: true, + json: async () => ({ messages: [{ msgId: 'm1' }], txId: 't1' }), + }) + + await expect( + unlimited().send({ to: 'user@example.com', subject: 'Hi', body: 'Hello' }), + ).resolves.toMatchObject({ messageId: 'm1' }) + expect(authHeaders()).toEqual(['Bearer stale', 'Bearer fresh']) + }) + + it('reports a token CHES still rejects after the retry', async () => { + token('stale') + unauthorized() + token('fresh') + unauthorized() + + await expect( + unlimited().send({ to: 'user@example.com', subject: 'Hi', body: 'Hello' }), + ).rejects.toBeInstanceOf(UnauthorizedException) + expect(authHeaders()).toHaveLength(2) + }) + }) + + describe('error classification', () => { + const transport = () => + new ChesEmailTransport({ + get: (key: string) => + ({ + 'ches.baseUrl': 'https://ches.example.com/api/v1', + 'ches.clientId': 'client-id', + 'ches.clientSecret': 'client-secret', + 'ches.tokenUrl': 'https://auth.example.com/token', + 'ches.from': 'noreply@gov.bc.ca', + 'ches.timeoutMs': 15_000, + })[key], + } as unknown as ConfigService) + const send = (t = transport()) => + t.send({ to: 'user@example.com', subject: 'Hi', body: 'Hello' }) + const tokenOk = () => + fetchMock.mockResolvedValueOnce({ + ok: true, + json: async () => ({ access_token: 'tok', expires_in: 300 }), + }) + const emailStatus = (status: number, body = '{"detail":"nope"}') => + fetchMock.mockResolvedValueOnce({ ok: false, status, text: async () => body }) + + it.each([503, 502, 500, 429, 408])( + 'treats a %i from CHES as transient: nothing was accepted', + async (status) => { + tokenOk() + emailStatus(status) + await expect(send()).rejects.toBeInstanceOf(TransientDeliveryError) + }, + ) + + it.each([400, 404, 422])('treats a %i as a problem with this message', async (status) => { + tokenOk() + emailStatus(status) + const error = await send().catch((caught: unknown) => caught) + expect(error).not.toBeInstanceOf(TransientDeliveryError) + }) + + it('treats CHES being unreachable as transient', async () => { + tokenOk() + fetchMock.mockRejectedValueOnce( + Object.assign(new TypeError('fetch failed'), { cause: { code: 'ECONNREFUSED' } }), + ) + await expect(send()).rejects.toThrow('CHES email unreachable: ECONNREFUSED') + }) + + it('does not retry a connection cut mid-request: CHES may have taken it', async () => { + tokenOk() + fetchMock.mockRejectedValueOnce( + Object.assign(new TypeError('terminated'), { cause: { code: 'ECONNRESET' } }), + ) + const error = await send().catch((caught: unknown) => caught) + expect(error).not.toBeInstanceOf(TransientDeliveryError) + }) + + it('keeps an email timeout as an unknown outcome, not transient', async () => { + tokenOk() + fetchMock.mockRejectedValueOnce(Object.assign(new Error('aborted'), { name: 'TimeoutError' })) + const error = await send().catch((caught: unknown) => caught) + expect(error).toBeInstanceOf(GatewayTimeoutException) + expect(error).not.toBeInstanceOf(TransientDeliveryError) + }) + + it('treats a token timeout as transient: no email was sent', async () => { + fetchMock.mockRejectedValueOnce(Object.assign(new Error('aborted'), { name: 'TimeoutError' })) + await expect(send()).rejects.toBeInstanceOf(TransientDeliveryError) + }) + + it('raises a 503 inside the circuit breaker, where it counts against CHES', async () => { + let seen: unknown + vi.spyOn(RedisCircuitBreaker.prototype, 'run').mockImplementation(async (fn) => { + try { + return await fn() + } catch (error) { + seen = error + throw error + } + }) + vi.spyOn(RedisConcurrencyLimiter.prototype, 'run').mockImplementation((fn) => fn()) + const limited = new ChesEmailTransport({ + get: (key: string) => + ({ + 'ches.baseUrl': 'https://ches.example.com/api/v1', + 'ches.clientId': 'client-id', + 'ches.clientSecret': 'client-secret', + 'ches.tokenUrl': 'https://auth.example.com/token', + 'ches.maxConcurrentRequests': 5, + redis: { host: 'localhost', port: 6379 }, + })[key], + } as unknown as ConfigService) + tokenOk() + emailStatus(503, '{"detail":"Server is shutting down"}') + + await expect(send(limited)).rejects.toBeInstanceOf(TransientDeliveryError) + expect(seen).toBeInstanceOf(TransientDeliveryError) + vi.restoreAllMocks() + }) + + it('sends through the circuit breaker, and counts timeouts as CHES being unwell', async () => { + let isFailure: ((error: unknown) => boolean) | undefined + const run = vi.spyOn(RedisCircuitBreaker.prototype, 'run').mockImplementation(function ( + this: RedisCircuitBreaker, + fn, + ) { + isFailure = (this as unknown as { options: { isFailure: typeof isFailure } }).options + .isFailure + return fn() + }) + const limited = new ChesEmailTransport({ + get: (key: string) => + ({ + 'ches.baseUrl': 'https://ches.example.com/api/v1', + 'ches.clientId': 'client-id', + 'ches.clientSecret': 'client-secret', + 'ches.tokenUrl': 'https://auth.example.com/token', + 'ches.maxConcurrentRequests': 5, + redis: { host: 'localhost', port: 6379 }, + })[key], + } as unknown as ConfigService) + vi.spyOn(RedisConcurrencyLimiter.prototype, 'run').mockImplementation((fn) => fn()) + tokenOk() + fetchMock.mockResolvedValueOnce({ + ok: true, + json: async () => ({ messages: [{ msgId: 'm1' }], txId: 't1' }), + }) + + await send(limited) + + expect(run).toHaveBeenCalledTimes(1) + expect(isFailure?.(new TransientDeliveryError('503'))).toBe(true) + expect(isFailure?.(new GatewayTimeoutException('slow'))).toBe(true) + expect(isFailure?.(new BadRequestException('bad address'))).toBe(false) + vi.restoreAllMocks() + }) + }) }) diff --git a/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.ts b/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.ts index 5ad899728..a8220af5b 100644 --- a/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.ts +++ b/backend/src/adapters/implementations/delivery/email/ches/ches-email.adapter.ts @@ -6,12 +6,20 @@ import { Logger, NotFoundException, OnModuleDestroy, + OnModuleInit, UnauthorizedException, } from '@nestjs/common' import { ConfigService } from '@nestjs/config' import type Redis from 'ioredis' +import { RedisCircuitBreaker } from '../../../../../common/redis/circuit-breaker' import { RedisConcurrencyLimiter } from '../../../../../common/redis/concurrency-limiter' -import { redisKey } from '../../../../../common/redis/redis-namespace' +import { + TransientDeliveryError, + isNotSentNetworkError, + isTransientDeliveryError, + isTransientHttpStatus, +} from '../../../../delivery-errors' +import { CHES_CIRCUIT_KEY, CHES_IN_FLIGHT_KEY } from '../../../../delivery-keys' import { createRedisClient, type RedisConfig } from '../../../../../queue/redis-connection' import { toEmailHtml } from '../../../../../services/rendering/email-body-html' import { IEmailTransport, SendEmailOptions, SendEmailResult } from '../../../../interfaces' @@ -49,8 +57,17 @@ interface ChesErrorResponse { errors?: Array<{ message: string }> } +/** + * Five failures in ten seconds is CHES unwell rather than one bad message: at its usual pace + * that is most calls failing. Thirty seconds then covers a pod restart without probing it. + */ +const CHES_CIRCUIT = { failureThreshold: 5, windowMs: 10_000, cooldownMs: 30_000 } + +/** How long startup waits for the limiter's Redis connection before sending without it. */ +const LIMITER_CONNECT_WAIT_MS = 2000 + @Injectable() -export class ChesEmailTransport implements IEmailTransport, OnModuleDestroy { +export class ChesEmailTransport implements IEmailTransport, OnModuleInit, OnModuleDestroy { readonly name = 'ches' private readonly logger = new Logger(ChesEmailTransport.name) @@ -62,6 +79,11 @@ export class ChesEmailTransport implements IEmailTransport, OnModuleDestroy { * Null - no cap - when Redis is not configured (tests) or the limit is 0. */ private readonly limiter: RedisConcurrencyLimiter | null = null + /** + * Stops every pod calling CHES while it is failing (TransientDeliveryError or a timeout): + * sends wait instead of failing, and resume once a probe send succeeds. Null with the limiter. + */ + private readonly breaker: RedisCircuitBreaker | null = null constructor(private readonly configService: ConfigService) { const redisConfig = this.configService.get('redis') @@ -70,17 +92,40 @@ export class ChesEmailTransport implements IEmailTransport, OnModuleDestroy { this.redis = createRedisClient(redisConfig, ChesEmailTransport.name, { // On the send path: a Redis outage has to fall back to unlimited sending in milliseconds. - lazyConnect: true, + // Not lazy: see onModuleInit. enableOfflineQueue: false, maxRetriesPerRequest: 1, commandTimeout: 250, }) const timeoutMs = this.configService.get('ches.timeoutMs') ?? 120_000 - this.limiter = new RedisConcurrencyLimiter(this.redis, redisKey('ches:in-flight'), { + this.limiter = new RedisConcurrencyLimiter(this.redis, CHES_IN_FLIGHT_KEY, { limit, // Outlasts the request's own timeout, so a slot is only reclaimed from a pod that died. leaseMs: timeoutMs + 15_000, }) + this.breaker = new RedisCircuitBreaker(this.redis, CHES_CIRCUIT_KEY, { + ...CHES_CIRCUIT, + probeTimeoutMs: timeoutMs + 15_000, + isFailure: (error) => + isTransientDeliveryError(error) || error instanceof GatewayTimeoutException, + }) + } + + /** + * Hold startup until the limiter's connection is up. With no offline queue a command sent + * mid-handshake is rejected, so without this a pod's first sends would skip the limit. Never + * fatal: after LIMITER_CONNECT_WAIT_MS the pod starts anyway and fails open until Redis is up. + */ + async onModuleInit(): Promise { + const redis = this.redis + if (!redis || redis.status === 'ready') return + await new Promise((resolve) => { + const timer = setTimeout(resolve, LIMITER_CONNECT_WAIT_MS) + redis.once('ready', () => { + clearTimeout(timer) + resolve() + }) + }) } async onModuleDestroy(): Promise { @@ -160,7 +205,6 @@ export class ChesEmailTransport implements IEmailTransport, OnModuleDestroy { throw new Error('CHES adapter: "body" is required and must be a string') } - const token = await this.getAccessToken(tokenUrl, clientId, clientSecret) const from = opts.from ?? this.configService.get('ches.from') ?? @@ -205,23 +249,41 @@ export class ChesEmailTransport implements IEmailTransport, OnModuleDestroy { })}`, ) - const post = () => - this.fetchWithTimeout(`${baseUrl.replace(/\/$/, '')}/email`, 'email', { + const url = `${baseUrl.replace(/\/$/, '')}/email` + const requestBody = JSON.stringify(payload) + const post = async () => + this.fetchWithTimeout(url, 'email', { method: 'POST', headers: { - Authorization: `Bearer ${token}`, + Authorization: `Bearer ${await this.getAccessToken(tokenUrl, clientId, clientSecret)}`, 'Content-Type': 'application/json', }, - body: JSON.stringify(payload), + body: requestBody, }) - // The wait for a slot sits outside the timeout: it measures CHES, not our own queue. - const response = this.limiter ? await this.limiter.run(post) : await post() - - if (!response.ok) { - const errText = await response.text() - this.logger.error(`[CHES] Response not OK: ${response.status} - ${errText}`) - this.throwForChesApiFailure(response.status, errText, 'email') + // The token is taken inside the slot: one taken before a long wait for a slot can expire + // before it is used. A 401 means CHES accepted nothing, so one retry with a fresh token + // cannot send twice; it covers a token that expired in flight or was revoked. + const send = async () => { + let response = await post() + if (response.status === 401) { + await response.text().catch(() => undefined) + this.logger.warn('[CHES] Token rejected; retrying once with a new token') + this.tokenCache = null + response = await post() + } + // Thrown here, inside the circuit breaker, so a 503 counts against CHES: fetch itself + // resolves for any status. + if (!response.ok) { + const errText = await response.text() + this.logger.error(`[CHES] Response not OK: ${response.status} - ${errText}`) + this.throwForChesApiFailure(response.status, errText, 'email') + } + return response } + // Waiting for a closed circuit and then a slot sits outside the timeout: that measures CHES, + // not our own queue. The circuit is checked first, so an open one holds no slots. + const limited = () => (this.limiter ? this.limiter.run(send) : send()) + const response = this.breaker ? await this.breaker.run(limited) : await limited() let data: ChesEmailResponse try { @@ -246,9 +308,9 @@ export class ChesEmailTransport implements IEmailTransport, OnModuleDestroy { } /** - * fetch with a deadline. A timeout becomes a 504 naming CHES, so it reads as the provider being - * slow - a retryable failure for the job, or one failed recipient in a merge batch - rather than - * a bare AbortError. + * fetch with a deadline. An email timeout becomes a 504 naming CHES: the outcome is unknown - + * CHES may have queued it - so it is not retried as transient. A token timeout, or a network + * error before the request reached CHES, sent nothing, so it is transient. */ private async fetchWithTimeout( url: string, @@ -264,9 +326,13 @@ export class ChesEmailTransport implements IEmailTransport, OnModuleDestroy { (caught.name === 'TimeoutError' || caught.name === 'AbortError') ) { this.logger.error(`[CHES] ${operation} request timed out after ${timeoutMs}ms`) - throw new GatewayTimeoutException( - `CHES ${operation} request timed out after ${timeoutMs / 1000}s`, - ) + const message = `CHES ${operation} request timed out after ${timeoutMs / 1000}s` + if (operation === 'token') throw new TransientDeliveryError(message, 504) + throw new GatewayTimeoutException(message) + } + if (isNotSentNetworkError(caught)) { + const code = (caught as { cause: { code: string } }).cause.code + throw new TransientDeliveryError(`CHES ${operation} unreachable: ${code}`) } throw caught } @@ -341,6 +407,13 @@ export class ChesEmailTransport implements IEmailTransport, OnModuleDestroy { `${label} request failed`, ) + // CHES down, restarting, overloaded or rate limiting: it accepted nothing, so send it later. + if (isTransientHttpStatus(status)) { + throw new TransientDeliveryError( + `${label}: upstream ${status} - ${message}`, + status >= 500 ? 502 : status, + ) + } if (status === 400) { throw new BadRequestException(`${label}: ${message}`) } @@ -353,12 +426,6 @@ export class ChesEmailTransport implements IEmailTransport, OnModuleDestroy { if (status === 422) { throw new BadRequestException(`${label} validation: ${message}`) } - if (status === 429) { - throw new BadRequestException(`${label} rate limit: ${message}`) - } - if (status >= 500) { - throw new BadGatewayException(`${label}: upstream ${status} - ${message}`) - } throw new BadGatewayException(`${label}: ${status} - ${message}`) } diff --git a/backend/src/adapters/implementations/delivery/sms/acs/acs-sms.adapter.spec.ts b/backend/src/adapters/implementations/delivery/sms/acs/acs-sms.adapter.spec.ts index 9fc6e677e..19df0e641 100644 --- a/backend/src/adapters/implementations/delivery/sms/acs/acs-sms.adapter.spec.ts +++ b/backend/src/adapters/implementations/delivery/sms/acs/acs-sms.adapter.spec.ts @@ -3,6 +3,7 @@ import { ConfigService } from '@nestjs/config' import { Logger } from '@nestjs/common' import { AcsSmsTransport } from '../../../../../../src/adapters/implementations/delivery/sms/acs/acs-sms.adapter' import type { SendSmsResult } from '../../../../../../src/adapters/interfaces' +import { TransientDeliveryError } from '../../../../../../src/adapters/delivery-errors' const mockSend = vi.fn() vi.mock('@azure/communication-sms', () => ({ @@ -195,4 +196,46 @@ describe('AcsSmsTransport', () => { transport.send({ to: ['+15559876543', '+15550000001'], body: 'Hello' }), ).rejects.toThrow('ACS send failed for all 2 recipient(s)') }) + + describe('provider outages', () => { + const configured = () => + createModule({ connectionString: 'endpoint=https://example;accesskey=key' }) + const send = (t: AcsSmsTransport, to: string[] = ['+15559876543']) => + t.send({ to, body: 'Hello' } as never) + + it.each([503, 429, 500])('treats a %i from ACS as transient', async (statusCode) => { + mockSend.mockRejectedValue(Object.assign(new Error('Service Unavailable'), { statusCode })) + await expect(send(await configured())).rejects.toBeInstanceOf(TransientDeliveryError) + }) + + it('treats ACS being unreachable as transient', async () => { + mockSend.mockRejectedValue( + Object.assign(new Error('connect ECONNREFUSED'), { code: 'ECONNREFUSED' }), + ) + await expect(send(await configured())).rejects.toBeInstanceOf(TransientDeliveryError) + }) + + it('leaves a 400 from ACS as a problem with the request', async () => { + mockSend.mockRejectedValue(Object.assign(new Error('Bad request'), { statusCode: 400 })) + const error = await send(await configured()).catch((caught: unknown) => caught) + expect(error).not.toBeInstanceOf(TransientDeliveryError) + }) + + it('flags a throttled recipient as owed, not failed, alongside one that was sent', async () => { + mockSend.mockResolvedValue([ + { to: '+15559876543', successful: true, messageId: 'm1', httpStatusCode: 202 }, + { to: '+15550000000', successful: false, httpStatusCode: 429, errorMessage: 'throttled' }, + ]) + const result = await send(await configured(), ['+15559876543', '+15550000000']) + expect(result.results?.[1]).toMatchObject({ success: false, transient: true }) + expect(result.results?.[0]).not.toHaveProperty('transient') + }) + + it('throws transient when ACS could take none of the recipients', async () => { + mockSend.mockResolvedValue([ + { to: '+15559876543', successful: false, httpStatusCode: 503, errorMessage: 'down' }, + ]) + await expect(send(await configured())).rejects.toBeInstanceOf(TransientDeliveryError) + }) + }) }) diff --git a/backend/src/adapters/implementations/delivery/sms/acs/acs-sms.adapter.ts b/backend/src/adapters/implementations/delivery/sms/acs/acs-sms.adapter.ts index b909d2cda..8a9d319c8 100644 --- a/backend/src/adapters/implementations/delivery/sms/acs/acs-sms.adapter.ts +++ b/backend/src/adapters/implementations/delivery/sms/acs/acs-sms.adapter.ts @@ -7,6 +7,11 @@ import { SendSmsResult, SmsRecipientResult, } from '../../../../interfaces' +import { + TransientDeliveryError, + isTransientHttpStatus, + transientFromProviderError, +} from '../../../../delivery-errors' @Injectable() export class AcsSmsTransport implements ISmsTransport { @@ -63,22 +68,28 @@ export class AcsSmsTransport implements ISmsTransport { } } - const results = await this.client.send( - { - from: resolvedFrom, - to: toNumbers, - message: body, - }, - { - // ACS only emits SMSDeliveryReportReceived events when this is set at send time, so it has - // to be on now for those reports to exist later. Nothing consumes them yet: acceptance by - // ACS is not delivery, and a number pending regulatory approval is accepted and then - // dropped by the carrier with no signal back to us. - enableDeliveryReport: true, - // Comes back on the delivery report, so it can be matched to our notification. - tag: opts.tag, - }, - ) + // The SDK has already retried 429, 5xx and network errors by the time this throws. One that + // means ACS took nothing is transient, so the worker leaves the recipients owed. + const results = await this.client + .send( + { + from: resolvedFrom, + to: toNumbers, + message: body, + }, + { + // ACS only emits SMSDeliveryReportReceived events when this is set at send time, so it has + // to be on now for those reports to exist later. Nothing consumes them yet: acceptance by + // ACS is not delivery, and a number pending regulatory approval is accepted and then + // dropped by the carrier with no signal back to us. + enableDeliveryReport: true, + // Comes back on the delivery report, so it can be matched to our notification. + tag: opts.tag, + }, + ) + .catch((error: unknown) => { + throw transientFromProviderError(error, 'ACS SMS') ?? error + }) // ACS reports each recipient separately, so report them onward rather than collapsing the // send into one success or failure. Throwing on a partial failure used to fail the whole job, @@ -90,6 +101,9 @@ export class AcsSmsTransport implements ISmsTransport { error: result.successful ? undefined : (result.errorMessage ?? `ACS returned HTTP ${result.httpStatusCode}`), + // ACS throttling or failing for this recipient: not the number's fault. + ...(!result.successful && + isTransientHttpStatus(result.httpStatusCode) && { transient: true }), })) const failed = recipientResults.filter((result) => !result.success) @@ -102,7 +116,9 @@ export class AcsSmsTransport implements ISmsTransport { // Every recipient failing is systemic - a bad credential, a disabled number - and worth a // retry of the whole job. A partial failure is not: the successes must not be repeated. if (failed.length === recipientResults.length && recipientResults.length > 0) { - throw new Error(`ACS send failed for all ${recipientResults.length} recipient(s)`) + const message = `ACS send failed for all ${recipientResults.length} recipient(s)` + if (failed.every((result) => result.transient)) throw new TransientDeliveryError(message, 502) + throw new Error(message) } const succeeded = recipientResults.filter((result) => result.success) diff --git a/backend/src/adapters/implementations/delivery/sms/sms-circuit-breaker.spec.ts b/backend/src/adapters/implementations/delivery/sms/sms-circuit-breaker.spec.ts new file mode 100644 index 000000000..90b7b91b7 --- /dev/null +++ b/backend/src/adapters/implementations/delivery/sms/sms-circuit-breaker.spec.ts @@ -0,0 +1,44 @@ +import { describe, it, expect, vi } from 'vitest' +import type { ConfigService } from '@nestjs/config' +import { SmsCircuitBreaker } from './sms-circuit-breaker' +import { RedisCircuitBreaker } from '../../../../common/redis/circuit-breaker' +import { TransientDeliveryError } from '../../../delivery-errors' + +vi.mock('../../../../queue/redis-connection', () => ({ + createRedisClient: vi.fn(() => ({ quit: vi.fn().mockResolvedValue('OK') })), +})) + +const config = (redis: unknown) => ({ get: () => redis }) as unknown as ConfigService +const transport = () => ({ name: 'acs', send: vi.fn().mockResolvedValue({ messageId: 'm1' }) }) + +describe('SmsCircuitBreaker', () => { + it('sends every SMS through the breaker, keeping the provider name', async () => { + let isFailure: ((error: unknown) => boolean) | undefined + const run = vi.spyOn(RedisCircuitBreaker.prototype, 'run').mockImplementation(function ( + this: RedisCircuitBreaker, + fn, + ) { + isFailure = (this as unknown as { options: { isFailure: typeof isFailure } }).options + .isFailure + return fn() + }) + const inner = transport() + + const wrapped = new SmsCircuitBreaker(config({ host: 'localhost', port: 6379 })).wrap(inner) + + await expect(wrapped.send({ to: '+15550000001', body: 'Hi' })).resolves.toEqual({ + messageId: 'm1', + }) + expect(wrapped.name).toBe('acs') + expect(run).toHaveBeenCalledTimes(1) + // Only an outage counts against the provider, not a bad number. + expect(isFailure?.(new TransientDeliveryError('503'))).toBe(true) + expect(isFailure?.(new Error('Invalid number'))).toBe(false) + run.mockRestore() + }) + + it('leaves the transport as it is without Redis', () => { + const inner = transport() + expect(new SmsCircuitBreaker(config(undefined)).wrap(inner)).toBe(inner) + }) +}) diff --git a/backend/src/adapters/implementations/delivery/sms/sms-circuit-breaker.ts b/backend/src/adapters/implementations/delivery/sms/sms-circuit-breaker.ts new file mode 100644 index 000000000..3daf5f87e --- /dev/null +++ b/backend/src/adapters/implementations/delivery/sms/sms-circuit-breaker.ts @@ -0,0 +1,65 @@ +import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common' +import { ConfigService } from '@nestjs/config' +import type Redis from 'ioredis' +import { RedisCircuitBreaker } from '../../../../common/redis/circuit-breaker' +import { createRedisClient, type RedisConfig } from '../../../../queue/redis-connection' +import { isTransientDeliveryError } from '../../../delivery-errors' +import { SMS_CIRCUIT_KEY } from '../../../delivery-keys' +import type { ISmsTransport } from '../../../interfaces' + +/** How long startup waits for the breaker's Redis connection before sending without it. */ +const CONNECT_WAIT_MS = 2000 + +/** + * Stops every pod sending SMS while the provider is failing, whichever provider is configured. + * Five transient failures in ten seconds open it; sends wait 30s, then one probe send decides. + * See RedisCircuitBreaker. Without Redis (tests) sends go straight through. + */ +@Injectable() +export class SmsCircuitBreaker implements OnModuleInit, OnModuleDestroy { + private readonly redis: Redis | null = null + private readonly breaker: RedisCircuitBreaker | null = null + + constructor(configService: ConfigService) { + const redisConfig = configService.get('redis') + if (!redisConfig) return + this.redis = createRedisClient(redisConfig, SmsCircuitBreaker.name, { + // On the send path: a Redis outage has to fall back to sending in milliseconds. + enableOfflineQueue: false, + maxRetriesPerRequest: 1, + commandTimeout: 250, + }) + this.breaker = new RedisCircuitBreaker(this.redis, SMS_CIRCUIT_KEY, { + failureThreshold: 5, + windowMs: 10_000, + cooldownMs: 30_000, + // Provider SDKs retry internally before giving up; allow for that. + probeTimeoutMs: 120_000, + isFailure: isTransientDeliveryError, + }) + } + + /** The transport, with every send passing the breaker. */ + wrap(transport: ISmsTransport): ISmsTransport { + const breaker = this.breaker + if (!breaker) return transport + return { name: transport.name, send: (options) => breaker.run(() => transport.send(options)) } + } + + /** With no offline queue a command sent mid-handshake is rejected; see ChesEmailTransport. */ + async onModuleInit(): Promise { + const redis = this.redis + if (!redis || redis.status === 'ready') return + await new Promise((resolve) => { + const timer = setTimeout(resolve, CONNECT_WAIT_MS) + redis.once('ready', () => { + clearTimeout(timer) + resolve() + }) + }) + } + + async onModuleDestroy(): Promise { + await this.redis?.quit().catch(() => this.redis?.disconnect()) + } +} diff --git a/backend/src/adapters/implementations/delivery/sms/twilio/twilio-sms.adapter.spec.ts b/backend/src/adapters/implementations/delivery/sms/twilio/twilio-sms.adapter.spec.ts index 4bd77eb19..0187d7900 100644 --- a/backend/src/adapters/implementations/delivery/sms/twilio/twilio-sms.adapter.spec.ts +++ b/backend/src/adapters/implementations/delivery/sms/twilio/twilio-sms.adapter.spec.ts @@ -3,6 +3,7 @@ import { Test, TestingModule } from '@nestjs/testing' import { ConfigService } from '@nestjs/config' import { TwilioSmsTransport } from '../../../../../../src/adapters/implementations/delivery/sms/twilio/twilio-sms.adapter' import type { SendSmsResult } from '../../../../../../src/adapters/interfaces' +import { TransientDeliveryError } from '../../../../../../src/adapters/delivery-errors' const mockMessagesCreate = vi.fn() vi.mock('twilio', () => ({ @@ -151,4 +152,48 @@ describe('TwilioSmsTransport', () => { expect.objectContaining({ from: '+15559999999' }), ) }) + + describe('provider outages', () => { + const configured = () => createModule({ accountSid: 'AC123', authToken: 'token' }) + const unavailable = () => Object.assign(new Error('Service Unavailable'), { status: 503 }) + + it('stops at the first outage, reporting who was sent and leaving the rest owed', async () => { + mockMessagesCreate.mockResolvedValueOnce({ sid: 'SM1' }).mockRejectedValueOnce(unavailable()) + const result = await ( + await configured() + ).send({ + to: ['+15550000001', '+15550000002', '+15550000003'], + body: 'Hello', + } as never) + + expect(mockMessagesCreate).toHaveBeenCalledTimes(2) + expect(result.results).toEqual([ + { to: '+15550000001', success: true, messageId: 'SM1' }, + expect.objectContaining({ to: '+15550000002', success: false, transient: true }), + expect.objectContaining({ to: '+15550000003', success: false, transient: true }), + ]) + }) + + it('throws transient when Twilio took nobody', async () => { + mockMessagesCreate.mockRejectedValue(unavailable()) + await expect( + (await configured()).send({ to: ['+15550000001'], body: 'Hello' } as never), + ).rejects.toBeInstanceOf(TransientDeliveryError) + }) + + it('still fails a recipient Twilio rejects, and carries on', async () => { + mockMessagesCreate + .mockRejectedValueOnce(Object.assign(new Error('Invalid number'), { status: 400 })) + .mockResolvedValueOnce({ sid: 'SM2' }) + const result = await ( + await configured() + ).send({ + to: ['+15550000001', '+15550000002'], + body: 'Hello', + } as never) + expect(result.results?.[0]).toMatchObject({ success: false, error: 'Invalid number' }) + expect(result.results?.[0]).not.toHaveProperty('transient') + expect(result.results?.[1]).toMatchObject({ success: true }) + }) + }) }) diff --git a/backend/src/adapters/implementations/delivery/sms/twilio/twilio-sms.adapter.ts b/backend/src/adapters/implementations/delivery/sms/twilio/twilio-sms.adapter.ts index a7070d5ec..22bf92b05 100644 --- a/backend/src/adapters/implementations/delivery/sms/twilio/twilio-sms.adapter.ts +++ b/backend/src/adapters/implementations/delivery/sms/twilio/twilio-sms.adapter.ts @@ -7,6 +7,7 @@ import { SendSmsResult, SmsRecipientResult, } from '../../../../interfaces' +import { TransientDeliveryError, transientFromProviderError } from '../../../../delivery-errors' @Injectable() export class TwilioSmsTransport implements ISmsTransport { @@ -69,7 +70,7 @@ export class TwilioSmsTransport implements ISmsTransport { // part-way through threw away the knowledge that earlier recipients had already been sent to, // and the queue's retry then messaged them a second time. const recipientResults: SmsRecipientResult[] = [] - for (const recipient of toNumbers) { + for (const [index, recipient] of toNumbers.entries()) { try { const message = await this.client.messages.create({ body, @@ -78,6 +79,21 @@ export class TwilioSmsTransport implements ISmsTransport { }) recipientResults.push({ to: recipient, success: true, messageId: message.sid }) } catch (error) { + // Twilio unavailable: stop, and leave this recipient and the rest owed. Carrying on + // would only fail them too, and earlier successes must still be reported. + const transient = transientFromProviderError(error, 'Twilio SMS') + if (transient) { + this.logger.warn(`Twilio unavailable, ${toNumbers.length - index} recipient(s) left owed`) + for (const owed of toNumbers.slice(index)) { + recipientResults.push({ + to: owed, + success: false, + error: transient.message, + transient: true, + }) + } + break + } const errorMessage = error instanceof Error ? error.message : String(error) this.logger.error(`Twilio send failed for one recipient: ${errorMessage}`) recipientResults.push({ to: recipient, success: false, error: errorMessage }) @@ -85,6 +101,10 @@ export class TwilioSmsTransport implements ISmsTransport { } const succeeded = recipientResults.filter((result) => result.success) + const owed = recipientResults.filter((result) => result.transient) + if (succeeded.length === 0 && owed.length > 0 && owed.length === recipientResults.length) { + throw new TransientDeliveryError(owed[0].error ?? 'Twilio unavailable', 502) + } // Every recipient failing is systemic and worth a retry; a partial failure is not. if (succeeded.length === 0 && recipientResults.length > 0) { diff --git a/backend/src/adapters/interfaces/delivery/sms.interface.ts b/backend/src/adapters/interfaces/delivery/sms.interface.ts index 97541b37e..a9f21b995 100644 --- a/backend/src/adapters/interfaces/delivery/sms.interface.ts +++ b/backend/src/adapters/interfaces/delivery/sms.interface.ts @@ -25,6 +25,11 @@ export interface SmsRecipientResult { messageId?: string /** Why this recipient failed, when it did. */ error?: string + /** + * Not sent because the provider was unavailable, not because of this recipient: still owed, + * to be sent again later rather than marked failed. + */ + transient?: boolean } export interface SendSmsResult { diff --git a/backend/src/api/admin/monitoring/monitoring.service.spec.ts b/backend/src/api/admin/monitoring/monitoring.service.spec.ts index 12fbe69ad..63452d3f0 100644 --- a/backend/src/api/admin/monitoring/monitoring.service.spec.ts +++ b/backend/src/api/admin/monitoring/monitoring.service.spec.ts @@ -8,7 +8,9 @@ import { averageOfWholeMinutes, buildOverview, buildRedisStats, + buildProviderStats, buildReconcilerStats, + buildSendsInProgress, sendingRateFrom, countUnheldActiveJobs, evaluateQueue, @@ -116,16 +118,73 @@ describe('evaluateQueue', () => { expect(queue.estimatedDrainMinutes).toBe(5) }) - it('warns when jobs arrive faster than they finish', () => { + /** Per-minute series ending with the current, partial minute. */ + const minutes = (...values: number[]) => [...Array(60 - values.length).fill(0), ...values] + /** Nothing added or finished in the last hour. */ + const idle = { + ...baseQueue.throughput, + added: minutes(), + completed: minutes(), + failed: minutes(), + inPerMinute: 0, + outPerMinute: 0, + } + + it('warns when jobs arrive faster than they finish minute after minute', () => { const queue = evaluateQueue({ ...baseQueue, counts: { ...baseQueue.counts, waiting: 100 }, - throughput: { ...baseQueue.throughput, inPerMinute: 30, outPerMinute: 10 }, + throughput: { + ...baseQueue.throughput, + added: minutes(30, 30, 30, 0), + completed: minutes(10, 10, 10, 0), + inPerMinute: 30, + outPerMinute: 10, + }, }) expect(queue.status).toBe('warning') + expect(queue.reasons).toContain( + 'Jobs have arrived faster than they finish for 3 minutes running', + ) expect(queue.estimatedDrainMinutes).toBeNull() }) + it('does not warn about one burst of arrivals, such as a large send queueing its batches', () => { + const queue = evaluateQueue({ + ...baseQueue, + counts: { ...baseQueue.counts, waiting: 60, active: 20 }, + throughput: { + ...baseQueue.throughput, + added: minutes(80, 0, 0, 0), + completed: minutes(0, 2, 3, 1), + inPerMinute: 16, + outPerMinute: 1, + }, + }) + expect(queue.status).toBe('healthy') + }) + + it('does not flag long-waiting jobs while the queue is moving', () => { + const waiting = { ...baseQueue.counts, waiting: 60 } + const oldest = 15 * 60_000 + // A batch finished in the last two minutes. + expect( + evaluateQueue({ + ...baseQueue, + counts: waiting, + oldestWaitingAgeMs: oldest, + throughput: { ...idle, completed: minutes(1, 0) }, + }).status, + ).toBe('healthy') + // No batch finished yet, but their recipients are being sent. + expect( + evaluateQueue( + { ...baseQueue, counts: waiting, oldestWaitingAgeMs: oldest, throughput: idle }, + true, + ).status, + ).toBe('healthy') + }) + it('is critical when jobs are waiting and no pod is processing the queue', () => { const queue = evaluateQueue({ ...baseQueue, @@ -135,14 +194,12 @@ describe('evaluateQueue', () => { expect(queue.status).toBe('critical') }) - it('grades the oldest waiting job against both thresholds', () => { - const waiting = { ...baseQueue.counts, waiting: 1 } - expect( - evaluateQueue({ ...baseQueue, counts: waiting, oldestWaitingAgeMs: 3 * 60_000 }).status, - ).toBe('warning') - expect( - evaluateQueue({ ...baseQueue, counts: waiting, oldestWaitingAgeMs: 11 * 60_000 }).status, - ).toBe('critical') + it('grades the oldest waiting job against both thresholds when nothing is finishing', () => { + const stuck = { ...baseQueue, counts: { ...baseQueue.counts, waiting: 1 }, throughput: idle } + expect(evaluateQueue({ ...stuck, oldestWaitingAgeMs: 3 * 60_000 }).status).toBe('warning') + const critical = evaluateQueue({ ...stuck, oldestWaitingAgeMs: 11 * 60_000 }) + expect(critical.status).toBe('critical') + expect(critical.reasons[0]).toMatch(/nothing has finished in the last 2 minutes/) }) it('ignores the failure rate until enough jobs have finished', () => { @@ -166,6 +223,7 @@ describe('countUnheldActiveJobs', () => { podId: 'pod', startedAt: '', lastHeartbeatAt: '', + draining: false, queues: [ { queue: 'email-delivery', @@ -328,6 +386,9 @@ describe('MonitoringService', () => { function fakeRedis(workers: object[] = [], pipeline = fakeReconcilePipeline()) { return { pipeline: vi.fn(() => pipeline), + get: vi.fn().mockResolvedValue(null), + exists: vi.fn().mockResolvedValue(0), + zcount: vi.fn().mockResolvedValue(3), info: vi .fn() .mockResolvedValue( @@ -345,11 +406,19 @@ describe('MonitoringService', () => { finishedRows: object[] = [], deliveryRow: object = { median: null, p95: null, messages: '0' }, rateRow: object = { busy_messages: '0', busy_seconds: '0' }, + sendRows: object[] = [], ) { const results: unknown[] = [pendingRows, finishedRows, deliveryRow] return { - metadata: { tablePath: 'notify.notification_request_detail' }, - query: vi.fn().mockResolvedValue([rateRow]), + metadata: { + tablePath: 'notify.notification_request_detail', + findRelationWithPropertyPath: () => ({ + inverseEntityMetadata: { tablePath: 'notify.notification_request' }, + }), + }, + query: vi.fn((sql: string) => + Promise.resolve(sql.includes('busy_messages') ? [rateRow] : sendRows), + ), createQueryBuilder: vi.fn(() => { const rows = results.shift() ?? [] const builder: Record = {} @@ -456,6 +525,47 @@ describe('MonitoringService', () => { expect(result.status).toBe('warning') }) + it('reports each provider, with CHES calls in flight, and closed circuits as healthy', async () => { + const service = new MonitoringService( + fakeRedis([livePod]), + null, + fakeQueue('email-delivery'), + null, + null, + tenantRepository, + fakeDetailRepository(), + { + get: (key: string) => ({ 'ches.maxConcurrentRequests': 5, 'delivery.sms': 'acs' })[key], + } as never, + ) + + const { providers, status } = await service.getQueueMonitoring() + + expect(providers).toEqual([ + { + name: 'CHES', + channel: 'EMAIL', + status: 'healthy', + reasons: [], + circuit: 'closed', + reopensAt: null, + inFlight: 3, + limit: 5, + }, + { + name: 'ACS', + channel: 'SMS', + status: 'healthy', + reasons: [], + circuit: 'closed', + reopensAt: null, + inFlight: null, + limit: null, + }, + ]) + expect(status).toBe('healthy') + }) + it('returns recent failures newest first with tenant names and masked reasons', async () => { const failedJob = (id: string, finishedOn: number) => ({ id, @@ -549,7 +659,7 @@ describe('MonitoringService', () => { expect(sms).toMatchObject({ pending: 0, oldestPendingAgeMs: null, estimatedClearMinutes: 0 }) }) - it('reports progress for active merge batches and ignores other active jobs', async () => { + it('rolls sends up per request, counting the merge batches being worked now', async () => { const batch = { id: 'req-1-EMAIL-0', processedOn: now - 20_000, @@ -567,23 +677,36 @@ describe('MonitoringService', () => { null, null, tenantRepository, - fakeDetailRepository(), + fakeDetailRepository([], [], undefined, undefined, [ + { + id: 'req-1', + tenant_id: 't-1', + accepted_at: new Date(now - 60_000), + channels: 'EMAIL', + total: '100', + sent: '37', + failed: '2', + remaining: '61', + batches: '4', + batches_open: '3', + recent_finished: '39', + }, + ]), ) - const { activeBatches } = await service.getQueueMonitoring() + const { sendsInProgress } = await service.getQueueMonitoring() - expect(activeBatches).toEqual([ - { - queue: 'email-delivery', - jobId: 'req-1-EMAIL-0', + expect(sendsInProgress).toEqual([ + expect.objectContaining({ notificationId: 'req-1', - tenantId: 't-1', tenantName: 'Health Ministry', sent: 37, failed: 2, total: 100, - startedAt: new Date(now - 20_000).toISOString(), - }, + batchesDone: 1, + // The plain job on the same queue is not a batch. + batchesSending: 1, + }), ]) }) @@ -696,3 +819,83 @@ describe('buildReconcilerStats', () => { expect(stats.reasons[0]).toMatch(/Gave up on 2 stuck send/) }) }) + +describe('buildSendsInProgress', () => { + const row = { + id: 'n-1', + tenant_id: 't-1', + accepted_at: '2026-10-05T22:36:30.000Z', + channels: 'EMAIL', + total: '2000', + sent: '400', + failed: '2', + remaining: '1598', + batches: '80', + batches_open: '64', + recent_finished: '325', + } + const activeBatch = (notifyId: string) => + ({ + queue: 'email-delivery', + job: { data: { notifyId } }, + progress: { sent: 3, failed: 0, total: 25 }, + }) as unknown as Parameters[1][number] + + it('rolls a merge up into one row with its batches and a time left', () => { + const [send] = buildSendsInProgress( + [row], + [activeBatch('n-1'), activeBatch('n-1'), activeBatch('other')], + new Map([['t-1', 'Health Ministry']]), + ) + expect(send).toEqual({ + notificationId: 'n-1', + tenantId: 't-1', + tenantName: 'Health Ministry', + channels: ['EMAIL'], + acceptedAt: '2026-10-05T22:36:30.000Z', + total: 2000, + sent: 400, + failed: 2, + remaining: 1598, + batches: 80, + batchesDone: 16, + batchesSending: 2, + perMinute: 65, + estimatedMinutesLeft: 25, + }) + }) + + it('gives no time left when nothing has finished recently', () => { + const [send] = buildSendsInProgress( + [{ ...row, recent_finished: '0', batches: '0', batches_open: '0' }], + [], + new Map(), + ) + expect(send).toMatchObject({ perMinute: 0, estimatedMinutesLeft: null, batches: 0 }) + }) +}) + +describe('buildProviderStats', () => { + const ches = { name: 'CHES', channel: 'EMAIL' as const, inFlight: 0, limit: 5 } + + it('warns while the circuit is open, saying sends are waiting rather than failing', () => { + const stats = buildProviderStats(ches, { + state: 'open', + reopensAt: '2026-10-05T22:43:48.000Z', + }) + expect(stats).toMatchObject({ + status: 'warning', + circuit: 'open', + reopensAt: '2026-10-05T22:43:48.000Z', + }) + expect(stats.reasons[0]).toMatch(/sends are waiting/) + }) + + it('warns while a probe is testing whether the provider has recovered', () => { + expect(buildProviderStats(ches, { state: 'half-open' }).reasons[0]).toMatch(/one test send/) + }) + + it('reports an unreadable circuit without raising an alarm', () => { + expect(buildProviderStats(ches, null)).toMatchObject({ status: 'healthy', circuit: null }) + }) +}) diff --git a/backend/src/api/admin/monitoring/monitoring.service.ts b/backend/src/api/admin/monitoring/monitoring.service.ts index e0dafa0b0..a1581afad 100644 --- a/backend/src/api/admin/monitoring/monitoring.service.ts +++ b/backend/src/api/admin/monitoring/monitoring.service.ts @@ -1,5 +1,6 @@ import { Inject, Injectable, Optional, ServiceUnavailableException } from '@nestjs/common' import { InjectRepository } from '@nestjs/typeorm' +import { ConfigService } from '@nestjs/config' import { In, Repository } from 'typeorm' import type Bull from 'bull' import type Redis from 'ioredis' @@ -27,6 +28,13 @@ import { } from '../../../queue/worker-heartbeat' import type { WorkerHeartbeatPayload } from '../../../queue/worker-heartbeat' import { readReconcileActivity } from '../../../queue/reconcile-activity' +import { readCircuitState } from '../../../common/redis/circuit-breaker' +import type { CircuitState } from '../../../common/redis/circuit-breaker' +import { + CHES_CIRCUIT_KEY, + CHES_IN_FLIGHT_KEY, + SMS_CIRCUIT_KEY, +} from '../../../adapters/delivery-keys' import type { ReconcileActivity } from '../../../queue/reconcile-activity' import { MONITORING_THRESHOLDS } from './monitoring-thresholds' import type { @@ -36,7 +44,9 @@ import type { QueueMonitoringResponseDto, QueueStatsDto, ReconcilerStatsDto, + ProviderStatsDto, RedisStatsDto, + SendInProgressDto, WorkerPodDto, } from './schemas/queue-monitoring.dto' @@ -57,6 +67,7 @@ const MIN_RATE_MESSAGES = 10 const MIN_RATE_BUSY_SECONDS = 5 const DEFAULT_MESSAGE_CHANNELS = [NotificationChannel.EMAIL, NotificationChannel.SMS] const RECENT_FAILURES_LIMIT = 20 +const SENDS_IN_PROGRESS_LIMIT = 20 /** A failure rate over a handful of jobs is noise; below this many finished jobs it is ignored. */ const MIN_JOBS_FOR_FAILURE_RATE = 10 @@ -158,7 +169,49 @@ export function averageOfWholeMinutes(series: number[], window: number): number type QueueEvaluationInput = Omit -export function evaluateQueue(input: QueueEvaluationInput): QueueStatsDto { +/** Minutes, current one included, in which something finishing counts as the queue moving. */ +const MOVING_WINDOW_MINUTES = 2 +/** Whole minutes in a row that arrivals must beat completions before it is reported. */ +const SUSTAINED_GROWTH_MINUTES = 3 + +/** The delivery queue's channel, whose sent messages show progress inside a long batch job. */ +const QUEUE_CHANNELS: Record = { + 'email-delivery': NotificationChannel.EMAIL, + 'sms-delivery': NotificationChannel.SMS, +} + +const sumLast = (series: number[], minutes: number) => + series.slice(-minutes).reduce((sum, value) => sum + value, 0) + +/** + * Arrivals beat completions in each of the last few whole minutes. One burst - a large send + * queueing all its batches at once - is a single minute and does not count. + */ +export function sustainedGrowth(throughput: QueueStatsDto['throughput']): boolean { + const { added, completed, failed } = throughput + const end = added.length - 1 + if (end < SUSTAINED_GROWTH_MINUTES) return false + for (let minute = end - SUSTAINED_GROWTH_MINUTES; minute < end; minute++) { + if (added[minute] <= (completed[minute] ?? 0) + (failed[minute] ?? 0)) return false + } + return true +} + +/** + * @param messagesMoving recipients on this queue's channel were sent or failed recently. A merge + * batch finishes only after its last recipient, so a queue of them can be sending steadily for + * minutes before any job completes. + */ +function messagesMoving(queueName: string, messages: MessageChannelStatsDto[]): boolean { + const channel = QUEUE_CHANNELS[queueName] + const stats = channel && messages.find((m) => m.channel === channel) + return ( + !!stats && + sumLast(stats.sent, MOVING_WINDOW_MINUTES) + sumLast(stats.failed, MOVING_WINDOW_MINUTES) > 0 + ) +} + +export function evaluateQueue(input: QueueEvaluationInput, messagesMoving = false): QueueStatsDto { const { counts, throughput, oldestWaitingAgeMs, isPaused, liveWorkerPods } = input const t = MONITORING_THRESHOLDS const statuses: HealthStatus[] = [] @@ -179,12 +232,24 @@ export function evaluateQueue(input: QueueEvaluationInput): QueueStatsDto { statuses.push('critical') reasons.push('Jobs are waiting but no pod is processing this queue') } - if (oldestWaitingAgeMs !== null && oldestWaitingAgeMs >= t.oldestWaitingCriticalMs) { + // A waiting job is only a problem when nothing is moving: during a large send, batches wait + // their turn for many minutes while the queue works steadily through them. + const moving = + messagesMoving || + sumLast(throughput.completed, MOVING_WINDOW_MINUTES) + + sumLast(throughput.failed, MOVING_WINDOW_MINUTES) > + 0 + const stalledFor = `nothing has finished in the last ${MOVING_WINDOW_MINUTES} minutes` + if (!moving && oldestWaitingAgeMs !== null && oldestWaitingAgeMs >= t.oldestWaitingCriticalMs) { statuses.push('critical') - reasons.push('Oldest job has waited longer than the critical threshold') - } else if (oldestWaitingAgeMs !== null && oldestWaitingAgeMs >= t.oldestWaitingWarningMs) { + reasons.push(`Oldest job has waited longer than the critical threshold and ${stalledFor}`) + } else if ( + !moving && + oldestWaitingAgeMs !== null && + oldestWaitingAgeMs >= t.oldestWaitingWarningMs + ) { statuses.push('warning') - reasons.push('Oldest job has waited longer than the warning threshold') + reasons.push(`Oldest job has waited longer than the warning threshold and ${stalledFor}`) } const recentFinished = throughput.outPerMinute * t.rateWindowMinutes @@ -198,10 +263,11 @@ export function evaluateQueue(input: QueueEvaluationInput): QueueStatsDto { } } - const growing = throughput.inPerMinute > throughput.outPerMinute - if (backlog > 0 && growing) { + if (backlog > 0 && sustainedGrowth(throughput)) { statuses.push('warning') - reasons.push('Jobs are arriving faster than they are finishing') + reasons.push( + `Jobs have arrived faster than they finish for ${SUSTAINED_GROWTH_MINUTES} minutes running`, + ) } let estimatedDrainMinutes: number | null = null @@ -244,6 +310,55 @@ function activeBatches( .sort((a, b) => (a.job.processedOn ?? 0) - (b.job.processedOn ?? 0)) } +export interface SendInProgressRow { + id: string + tenant_id: string + accepted_at: Date | string + channels: string | null + total: string + sent: string + failed: string + remaining: string + batches: string + batches_open: string + recent_finished: string +} + +/** Roll recipient counts up per request, with how many of its batches are being worked now. */ +export function buildSendsInProgress( + rows: SendInProgressRow[], + activeJobs: Array, + tenantNames: Map, +): SendInProgressDto[] { + const sending = new Map() + for (const { job } of activeJobs) { + const notifyId = asString((job.data as JobIdentifiers)?.notifyId) + if (notifyId) sending.set(notifyId, (sending.get(notifyId) ?? 0) + 1) + } + const window = MONITORING_THRESHOLDS.rateWindowMinutes + return rows.map((row) => { + const remaining = Number(row.remaining) || 0 + const batches = Number(row.batches) || 0 + const perMinute = Math.round(((Number(row.recent_finished) || 0) / window) * 10) / 10 + return { + notificationId: row.id, + tenantId: row.tenant_id, + tenantName: tenantNames.get(row.tenant_id) ?? null, + channels: row.channels ? row.channels.split(',') : [], + acceptedAt: new Date(row.accepted_at).toISOString(), + total: Number(row.total) || 0, + sent: Number(row.sent) || 0, + failed: Number(row.failed) || 0, + remaining, + batches, + batchesDone: batches - (Number(row.batches_open) || 0), + batchesSending: sending.get(row.id) ?? 0, + perMinute, + estimatedMinutesLeft: perMinute > 0 ? Math.ceil(remaining / perMinute) : null, + } + }) +} + /** * Active jobs no live worker reports holding. A job stays "active" in Redis after the pod * processing it dies, until Bull's stalled-job check moves it back to waiting, so these are @@ -305,6 +420,27 @@ export function buildOverview( } } +const SMS_PROVIDER_NAMES: Record = { acs: 'ACS', twilio: 'Twilio' } + +export function buildProviderStats( + provider: Pick, + circuit: CircuitState | null, +): ProviderStatsDto { + const reasons: string[] = [] + if (circuit?.state === 'open') { + reasons.push('Failing; sends are waiting, and resume once a test send succeeds') + } else if (circuit?.state === 'half-open') { + reasons.push('Was failing; one test send is checking whether it has recovered') + } + return { + ...provider, + status: reasons.length > 0 ? 'warning' : 'healthy', + reasons, + circuit: circuit?.state ?? null, + reopensAt: circuit?.state === 'open' ? circuit.reopensAt : null, + } +} + export function buildReconcilerStats( activity: ReconcileActivity, now: number, @@ -364,6 +500,7 @@ export class MonitoringService { @InjectRepository(Tenant) private readonly tenantRepository: Repository, @InjectRepository(NotificationRequestDetail) private readonly detailRepository: Repository, + @Optional() private readonly configService?: ConfigService, ) { this.queues = [ingestionQueue, emailQueue, smsQueue, webhookQueue].filter( (queue): queue is Bull.Queue => !!queue, @@ -387,6 +524,8 @@ export class MonitoringService { deliveryTime, sendingRate, reconcileActivity, + sendRows, + providers, ] = await Promise.all([ this.redis.info(), this.getWorkers(now), @@ -396,19 +535,20 @@ export class MonitoringService { this.getDeliveryTime(now), this.getSendingRate(now), readReconcileActivity(this.redis, now, MONITORING_WINDOW_MINUTES), + this.getSendsInProgress(now), + this.getProviderStats(now), ]) const redis = buildRedisStats(parseRedisInfo(info)) const queues = await Promise.all( this.queues.map((queue) => - this.getQueueStats(queue, workers, activeByQueue.get(queue.name) ?? [], now), + this.getQueueStats(queue, workers, activeByQueue.get(queue.name) ?? [], messages, now), ), ) const activeJobs = activeBatches(activeByQueue) const tenantNames = await this.getTenantNames([ - ...[...failedJobs, ...activeJobs].map(({ job }) => - asString((job.data as JobIdentifiers)?.tenantId), - ), + ...failedJobs.map(({ job }) => asString((job.data as JobIdentifiers)?.tenantId)), ...reconcileActivity.recent.map((action) => action.tenantId), + ...sendRows.map((row) => row.tenant_id), ]) const reconciler = buildReconcilerStats(reconcileActivity, now, tenantNames) @@ -417,21 +557,17 @@ export class MonitoringService { status: worstStatus([ redis.status, reconciler.status, + ...providers.map((provider) => provider.status), ...queues.map((queue) => queue.status), ]), overview: buildOverview(messages, queues, deliveryTime, sendingRate), redis, messages, queues, - activeBatches: activeJobs.map(({ queue, job, progress }) => ({ - queue, - jobId: String(job.id), - ...jobIdentifiers(job, tenantNames), - ...progress, - startedAt: job.processedOn ? new Date(job.processedOn).toISOString() : null, - })), + sendsInProgress: buildSendsInProgress(sendRows, activeJobs, tenantNames), workers, reconciler, + providers, recentFailures: failedJobs.map(({ queue, job }) => ({ queue, jobId: String(job.id), @@ -447,6 +583,7 @@ export class MonitoringService { queue: Bull.Queue, workers: WorkerPodDto[], activeJobs: Bull.Job[], + messages: MessageChannelStatsDto[], now: number, ): Promise { const [counts, isPaused, oldestWaitingAgeMs, added, completed, failed] = await Promise.all([ @@ -467,29 +604,32 @@ export class MonitoringService { const failureRatePercent = outPerMinute > 0 ? Math.round((failedPerMinute / outPerMinute) * 1000) / 10 : 0 - return evaluateQueue({ - name: queue.name, - isPaused, - counts: { - waiting: counts.waiting, - active: counts.active, - delayed: counts.delayed, - failed: counts.failed, - paused: (counts as Bull.JobCounts & { paused?: number }).paused ?? 0, + return evaluateQueue( + { + name: queue.name, + isPaused, + counts: { + waiting: counts.waiting, + active: counts.active, + delayed: counts.delayed, + failed: counts.failed, + paused: (counts as Bull.JobCounts & { paused?: number }).paused ?? 0, + }, + oldestWaitingAgeMs, + throughput: { + added, + completed: completedSeries, + failed: failedSeries, + inPerMinute: averageOfWholeMinutes(added, window), + outPerMinute, + failureRatePercent, + }, + liveWorkerPods: workers.filter((pod) => pod.queues.some((q) => q.queue === queue.name)) + .length, + unheldActiveJobs: countUnheldActiveJobs(queue.name, activeJobs, workers, now), }, - oldestWaitingAgeMs, - throughput: { - added, - completed: completedSeries, - failed: failedSeries, - inPerMinute: averageOfWholeMinutes(added, window), - outPerMinute, - failureRatePercent, - }, - liveWorkerPods: workers.filter((pod) => pod.queues.some((q) => q.queue === queue.name)) - .length, - unheldActiveJobs: countUnheldActiveJobs(queue.name, activeJobs, workers, now), - }) + messagesMoving(queue.name, messages), + ) } /** @@ -625,6 +765,78 @@ export class MonitoringService { return sendingRateFrom(Number(row?.busy_messages ?? 0), Number(row?.busy_seconds ?? 0)) } + /** + * Requests with recipients still in flight, oldest first, each counted across all its rows. + * Requests are found through in-flight rows in the lookback window - the same rows, and index, + * the pending count uses - and scheduled sends are left out until they fall due. + */ + private async getSendsInProgress(now: number): Promise { + const detail = this.detailRepository.metadata + const request = + detail.findRelationWithPropertyPath('notificationRequest')?.inverseEntityMetadata.tablePath + if (!request) return [] + return (await this.detailRepository.query( + `SELECT d.notification_request_id AS id, + r.tenant_id, + r.created_at AS accepted_at, + string_agg(DISTINCT d.channel, ',') AS channels, + COUNT(*) FILTER (WHERE d.status <> 'blocked') AS total, + COUNT(*) FILTER (WHERE d.status = 'sent') AS sent, + COUNT(*) FILTER (WHERE d.status = 'failed') AS failed, + COUNT(*) FILTER (WHERE d.status = ANY($2)) AS remaining, + COUNT(DISTINCT d.batch_id) AS batches, + COUNT(DISTINCT d.batch_id) FILTER (WHERE d.status = ANY($2)) AS batches_open, + COUNT(*) FILTER (WHERE d.status IN ('sent', 'failed') AND d.last_attempt_at >= $3) + AS recent_finished + FROM ${detail.tablePath} d + JOIN ${request} r ON r.id = d.notification_request_id + WHERE d.notification_request_id IN ( + SELECT notification_request_id FROM ${detail.tablePath} + WHERE status = ANY($2) AND created_at > $1) + AND r.status <> $4 + GROUP BY d.notification_request_id, r.tenant_id, r.created_at + ORDER BY r.created_at ASC + LIMIT ${SENDS_IN_PROGRESS_LIMIT}`, + [ + new Date(now - PENDING_LOOKBACK_MS), + IN_FLIGHT_DETAIL_STATUSES, + new Date(now - MONITORING_THRESHOLDS.rateWindowMinutes * 60_000), + NotificationStatus.SCHEDULED, + ], + )) as SendInProgressRow[] + } + + private async getProviderStats(now: number): Promise { + const redis = this.redis as Redis + const [chesCircuit, smsCircuit, chesInFlight] = await Promise.all([ + readCircuitState(redis, CHES_CIRCUIT_KEY), + readCircuitState(redis, SMS_CIRCUIT_KEY), + // Leases are scored by expiry; an expired one is a pod that died, not a call in flight. + redis.zcount(CHES_IN_FLIGHT_KEY, now, '+inf').catch(() => 0), + ]) + const smsAdapter = (this.configService?.get('delivery.sms') ?? 'acs').split(':')[0] + return [ + buildProviderStats( + { + name: 'CHES', + channel: 'EMAIL', + inFlight: chesInFlight, + limit: this.configService?.get('ches.maxConcurrentRequests') ?? 0, + }, + chesCircuit, + ), + buildProviderStats( + { + name: SMS_PROVIDER_NAMES[smsAdapter] ?? smsAdapter, + channel: 'SMS', + inFlight: null, + limit: null, + }, + smsCircuit, + ), + ] + } + private async getTenantNames(ids: Array): Promise> { const tenantIds = [...new Set(ids.filter((id): id is string => !!id))] if (tenantIds.length === 0) return new Map() diff --git a/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts b/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts index 73129134f..91e4bad44 100644 --- a/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts +++ b/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts @@ -109,16 +109,32 @@ export interface MessageChannelStatsDto { estimatedClearMinutes: number | null } -export interface ActiveBatchDto { - queue: string - jobId: string - notificationId: string | null - tenantId: string | null +/** + * One notification request with recipients still to send: a whole mail merge, however many + * batches it was split into, or a plain send. Counted from its recipient rows. + */ +export interface SendInProgressDto { + notificationId: string + tenantId: string tenantName: string | null + /** Channels the request is sending on, e.g. ["EMAIL"]. */ + channels: string[] + acceptedAt: string + /** Recipients to send to, blocked recipients excluded. */ + total: number sent: number failed: number - total: number - startedAt: string | null + /** Accepted, queued or sending: not yet sent or failed. */ + remaining: number + /** Merge batches; 0 for a plain send. */ + batches: number + batchesDone: number + /** Batches a worker is on right now. */ + batchesSending: number + /** Recipients sent or failed per minute over the last few minutes. */ + perMinute: number + /** At perMinute; null when nothing finished recently. */ + estimatedMinutesLeft: number | null } /** Totals over the last hour, for the summary cards. */ @@ -166,6 +182,22 @@ export interface ReconcilerStatsDto { recentActions: ReconcilerActionDto[] } +/** A delivery provider as this service sees it: the shared circuit breaker in front of it. */ +export interface ProviderStatsDto { + /** e.g. "CHES", "ACS". */ + name: string + channel: 'EMAIL' | 'SMS' + status: HealthStatus + reasons: string[] + /** closed: sending; open: provider failing, sends wait; half-open: one probe send in progress. Null when unreadable. */ + circuit: 'closed' | 'open' | 'half-open' | null + /** When an open circuit lets its probe through. */ + reopensAt: string | null + /** Requests in flight and the cap on them, where one is applied (CHES); null where not. */ + inFlight: number | null + limit: number | null +} + export interface QueueMonitoringResponseDto { generatedAt: string status: HealthStatus @@ -173,8 +205,9 @@ export interface QueueMonitoringResponseDto { redis: RedisStatsDto messages: MessageChannelStatsDto[] queues: QueueStatsDto[] - activeBatches: ActiveBatchDto[] + sendsInProgress: SendInProgressDto[] workers: WorkerPodDto[] reconciler: ReconcilerStatsDto + providers: ProviderStatsDto[] recentFailures: RecentFailureDto[] } diff --git a/backend/src/api/notification/notification-request-detail.service.ts b/backend/src/api/notification/notification-request-detail.service.ts index 29ceb84b9..554dd15cd 100644 --- a/backend/src/api/notification/notification-request-detail.service.ts +++ b/backend/src/api/notification/notification-request-detail.service.ts @@ -251,6 +251,21 @@ export class NotificationRequestDetailService { ) } + /** + * Mark what one merge batch still owes failed, leaving its sent rows and every other batch + * alone. For a batch that can never succeed (its template deleted, say), once retries are spent. + */ + async markBatchUnsentFailed( + notificationRequestId: string, + batchId: string, + errorMessage: string, + ): Promise { + await this.detailRepository.update( + { notificationRequestId, batchId, status: In(IN_FLIGHT_DETAIL_STATUSES) }, + { status: 'failed', errorMessage, lastAttemptAt: new Date(), updatedBy: 'system' }, + ) + } + /** * Count detail records for a request in a given status (used to reconcile the parent request). */ diff --git a/backend/src/common/redis/circuit-breaker.spec.ts b/backend/src/common/redis/circuit-breaker.spec.ts new file mode 100644 index 000000000..328943687 --- /dev/null +++ b/backend/src/common/redis/circuit-breaker.spec.ts @@ -0,0 +1,148 @@ +import { describe, it, expect, vi } from 'vitest' +import type Redis from 'ioredis' +import { RedisCircuitBreaker, circuitKeys, readCircuitState } from './circuit-breaker' + +const BASE = 'notify:test:circuit' +const keys = circuitKeys(BASE) + +/** The breaker's two scripts over an in-memory store: key -> { value, expiresAt }. */ +function fakeRedis() { + const store = new Map() + const failures: number[] = [] + const live = (key: string) => { + const entry = store.get(key) + if (entry && entry.expiresAt <= Date.now()) store.delete(key) + return store.get(key) + } + const set = (key: string, value: string, ttlMs: number) => + store.set(key, { value, expiresAt: Date.now() + Number(ttlMs) }) + return { + store, + eval: vi.fn(async (_script: string, numKeys: number, ...args: unknown[]) => { + const [open, halfOpen, probe] = args as string[] + if (numKeys === 3) { + const [, , , token, probeMs] = args as [string, string, string, string, number] + if (live(open)) return 'wait' + if (live(halfOpen)) { + if (live(probe)) return 'wait' + set(probe, token, probeMs) + return 'probe' + } + return 'go' + } + const [, , , , , windowMs, threshold, cooldownMs, isProbe, halfOpenTtl] = args as [ + string, + string, + string, + string, + string, + number, + number, + number, + string, + number, + ] + const now = Date.now() + if (isProbe === '1') { + set(open, String(now + cooldownMs), cooldownMs) + store.delete(probe) + return 'reopened' + } + if (live(open) || live(halfOpen)) return 'ignored' + failures.push(now) + while (failures.length && failures[0] <= now - windowMs) failures.shift() + if (failures.length >= threshold) { + set(open, String(now + cooldownMs), cooldownMs) + set(halfOpen, '1', halfOpenTtl) + failures.length = 0 + return 'opened' + } + return 'counted' + }), + del: vi.fn(async (...deleted: string[]) => { + deleted.forEach((key) => store.delete(key)) + return deleted.length + }), + get: vi.fn(async (key: string) => live(key)?.value ?? null), + exists: vi.fn(async (key: string) => (live(key) ? 1 : 0)), + } +} + +class Outage extends Error {} + +const breakerOn = (redis: ReturnType) => + new RedisCircuitBreaker(redis as unknown as Redis, BASE, { + failureThreshold: 3, + windowMs: 10_000, + cooldownMs: 30, + probeTimeoutMs: 1_000, + isFailure: (error) => error instanceof Outage, + pollMs: 5, + }) + +const fail = () => Promise.reject(new Outage('503')) +const ok = () => Promise.resolve('sent') + +describe('RedisCircuitBreaker', () => { + it('lets calls through while the dependency is healthy', async () => { + await expect(breakerOn(fakeRedis()).run(ok)).resolves.toBe('sent') + }) + + it('opens after the threshold of failures, so further calls wait instead of being made', async () => { + const redis = fakeRedis() + const breaker = breakerOn(redis) + for (let i = 0; i < 3; i++) await expect(breaker.run(fail)).rejects.toThrow('503') + + expect(await readCircuitState(redis as unknown as Redis, BASE)).toMatchObject({ + state: 'open', + }) + const call = vi.fn(ok) + const waiting = breaker.run(call) + await new Promise((resolve) => setTimeout(resolve, 10)) + expect(call).not.toHaveBeenCalled() + + // After the cooldown the waiting call is the probe; it succeeds and closes the circuit. + await expect(waiting).resolves.toBe('sent') + expect(await readCircuitState(redis as unknown as Redis, BASE)).toEqual({ state: 'closed' }) + }) + + it('lets one probe through after the cooldown, and reopens if it fails', async () => { + const redis = fakeRedis() + const breaker = breakerOn(redis) + for (let i = 0; i < 3; i++) await breaker.run(fail).catch(() => undefined) + await new Promise((resolve) => setTimeout(resolve, 40)) + + await expect(breaker.run(fail)).rejects.toThrow('503') + + expect(await readCircuitState(redis as unknown as Redis, BASE)).toMatchObject({ + state: 'open', + }) + }) + + it('does not count errors that are not the dependency failing', async () => { + const redis = fakeRedis() + const breaker = breakerOn(redis) + for (let i = 0; i < 5; i++) { + await expect(breaker.run(() => Promise.reject(new Error('bad address')))).rejects.toThrow() + } + expect(await readCircuitState(redis as unknown as Redis, BASE)).toEqual({ state: 'closed' }) + }) + + it('closes when the probe gets any answer from the dependency, even a rejection', async () => { + const redis = fakeRedis() + const breaker = breakerOn(redis) + for (let i = 0; i < 3; i++) await breaker.run(fail).catch(() => undefined) + await new Promise((resolve) => setTimeout(resolve, 40)) + + await expect(breaker.run(() => Promise.reject(new Error('bad address')))).rejects.toThrow() + + expect(redis.store.has(keys.halfOpen)).toBe(false) + }) + + it('lets calls through when Redis cannot be reached', async () => { + const redis = fakeRedis() + redis.eval.mockRejectedValue(new Error('Connection is closed.')) + await expect(breakerOn(redis).run(ok)).resolves.toBe('sent') + await expect(breakerOn(redis).run(fail)).rejects.toThrow('503') + }) +}) diff --git a/backend/src/common/redis/circuit-breaker.ts b/backend/src/common/redis/circuit-breaker.ts new file mode 100644 index 000000000..d505f2e86 --- /dev/null +++ b/backend/src/common/redis/circuit-breaker.ts @@ -0,0 +1,197 @@ +import { Logger } from '@nestjs/common' +import { randomUUID } from 'crypto' +import type Redis from 'ioredis' + +/** + * Whether a call may go ahead. Open: wait. Half-open (cooldown over, not yet confirmed healthy): + * one caller - the one that takes the probe lock - goes ahead, the rest wait. Otherwise closed. + */ +const ENTER_SCRIPT = ` +if redis.call('EXISTS', KEYS[1]) == 1 then return 'wait' end +if redis.call('EXISTS', KEYS[2]) == 1 then + if redis.call('SET', KEYS[3], ARGV[1], 'NX', 'PX', ARGV[2]) then return 'probe' end + return 'wait' +end +return 'go' +` + +/** + * Record a failed call. A failed probe reopens straight away. Otherwise the failure is counted in + * a sliding window, on Redis's clock, and reaching the threshold opens the circuit. + */ +const FAILURE_SCRIPT = ` +local time = redis.call('TIME') +local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000) +local window, threshold, cooldown = tonumber(ARGV[2]), tonumber(ARGV[3]), tonumber(ARGV[4]) +if ARGV[5] == '1' then + redis.call('SET', KEYS[1], now + cooldown, 'PX', cooldown) + redis.call('DEL', KEYS[3], KEYS[4]) + return 'reopened' +end +if redis.call('EXISTS', KEYS[1]) == 1 or redis.call('EXISTS', KEYS[2]) == 1 then return 'ignored' end +redis.call('ZADD', KEYS[4], now, ARGV[1]) +redis.call('ZREMRANGEBYSCORE', KEYS[4], '-inf', now - window) +redis.call('PEXPIRE', KEYS[4], window) +if redis.call('ZCARD', KEYS[4]) >= threshold then + redis.call('SET', KEYS[1], now + cooldown, 'PX', cooldown) + redis.call('SET', KEYS[2], '1', 'PX', ARGV[6]) + redis.call('DEL', KEYS[4]) + return 'opened' +end +return 'counted' +` + +export interface CircuitBreakerOptions { + /** Failures within `windowMs` that open the circuit. */ + failureThreshold: number + windowMs: number + /** How long the circuit stays open before one probe call is let through. */ + cooldownMs: number + /** How long a probe may run before another caller may probe. Longer than the slowest call. */ + probeTimeoutMs: number + /** Which errors mean the dependency is unwell. Anything else proves it answered. */ + isFailure: (error: unknown) => boolean + /** Wait between checks while open; each wait is jittered up to double this. */ + pollMs?: number +} + +export type CircuitState = + | { state: 'closed' } + | { state: 'open'; reopensAt: string } + | { state: 'half-open' } + +/** How long the half-open flag outlives its last use, so an abandoned one cannot linger. */ +const HALF_OPEN_TTL_MS = 24 * 60 * 60 * 1000 +const FAIL_OPEN_WARN_INTERVAL_MS = 60_000 + +export const circuitKeys = (base: string) => ({ + open: `${base}:open`, + halfOpen: `${base}:half-open`, + probe: `${base}:probe`, + failures: `${base}:failures`, +}) + +/** The circuit's state, for display. Null when Redis cannot be read. */ +export async function readCircuitState(redis: Redis, base: string): Promise { + const keys = circuitKeys(base) + try { + const [openUntil, halfOpen] = await Promise.all([ + redis.get(keys.open), + redis.exists(keys.halfOpen), + ]) + if (openUntil) return { state: 'open', reopensAt: new Date(Number(openUntil)).toISOString() } + return halfOpen ? { state: 'half-open' } : { state: 'closed' } + } catch { + return null + } +} + +/** + * Stops every pod calling a dependency that is failing, until it recovers. + * + * Without it, an outage is met with a call for every piece of work waiting: each fails at once, + * so the backlog drains into failures, and the dependency is hit hardest while it is trying to + * come back. With it, a few failures in quick succession open the circuit and calls wait instead + * of being made. After a cooldown one probe call goes through; if it succeeds the circuit closes + * and the waiting calls proceed, and if it fails the cooldown starts again. + * + * State lives in Redis, so all pods open and close together. Fails open: if Redis cannot be + * reached, calls go ahead as if the circuit were closed. + */ +export class RedisCircuitBreaker { + private readonly logger = new Logger(RedisCircuitBreaker.name) + private readonly keys: ReturnType + private lastFailOpenWarnAt = 0 + + constructor( + private readonly client: Redis, + private readonly base: string, + private readonly options: CircuitBreakerOptions, + ) { + this.keys = circuitKeys(base) + } + + async run(fn: () => Promise): Promise { + const token = randomUUID() + const probe = await this.enter(token) + try { + const result = await fn() + if (probe) await this.closeAfterProbe() + return result + } catch (error) { + if (this.options.isFailure(error)) await this.recordFailure(token, probe) + else if (probe) await this.closeAfterProbe() + throw error + } + } + + /** Waits while the circuit is open. True when this call is the half-open probe. */ + private async enter(token: string): Promise { + const pollMs = this.options.pollMs ?? 1000 + for (;;) { + let verdict: unknown + try { + verdict = await this.client.eval( + ENTER_SCRIPT, + 3, + this.keys.open, + this.keys.halfOpen, + this.keys.probe, + token, + this.options.probeTimeoutMs, + ) + } catch (error) { + this.warnFailOpen(error as Error) + return false + } + if (verdict === 'go') return false + if (verdict === 'probe') return true + await new Promise((resolve) => setTimeout(resolve, pollMs + Math.random() * pollMs)) + } + } + + private async recordFailure(token: string, probe: boolean): Promise { + const { failureThreshold, windowMs, cooldownMs } = this.options + try { + const outcome = await this.client.eval( + FAILURE_SCRIPT, + 4, + this.keys.open, + this.keys.halfOpen, + this.keys.probe, + this.keys.failures, + token, + windowMs, + failureThreshold, + cooldownMs, + probe ? '1' : '0', + HALF_OPEN_TTL_MS, + ) + if (outcome === 'opened') { + this.logger.warn( + `Circuit ${this.base} opened after ${failureThreshold} failures in ${windowMs / 1000}s; calls wait ${cooldownMs / 1000}s, then one probe`, + ) + } else if (outcome === 'reopened') { + this.logger.warn(`Circuit ${this.base} probe failed; waiting another ${cooldownMs / 1000}s`) + } + } catch (error) { + this.warnFailOpen(error as Error) + } + } + + private async closeAfterProbe(): Promise { + try { + await this.client.del(this.keys.halfOpen, this.keys.probe, this.keys.failures) + this.logger.log(`Circuit ${this.base} closed: probe succeeded`) + } catch (error) { + this.warnFailOpen(error as Error) + } + } + + private warnFailOpen(error: Error): void { + const now = Date.now() + if (now - this.lastFailOpenWarnAt < FAIL_OPEN_WARN_INTERVAL_MS) return + this.lastFailOpenWarnAt = now + this.logger.warn(`Circuit ${this.base} not applied: ${error.message}`) + } +} diff --git a/backend/src/queue/workers/email-delivery.worker.spec.ts b/backend/src/queue/workers/email-delivery.worker.spec.ts index 26aa3286e..1f7ec7970 100644 --- a/backend/src/queue/workers/email-delivery.worker.spec.ts +++ b/backend/src/queue/workers/email-delivery.worker.spec.ts @@ -1,4 +1,5 @@ import { BadRequestException, Logger } from '@nestjs/common' +import { TransientDeliveryError } from '../../adapters/delivery-errors' import Bull from 'bull' import { vi, describe, it, expect, beforeEach, afterEach } from 'vitest' import { EmailDeliveryWorker } from './email-delivery.worker' @@ -39,6 +40,7 @@ describe('EmailDeliveryWorker', () => { markFailed: vi.fn().mockResolvedValue(undefined), markRecipientSent: vi.fn().mockResolvedValue(undefined), markRecipientFailed: vi.fn().mockResolvedValue(undefined), + markBatchUnsentFailed: vi.fn().mockResolvedValue(undefined), countByStatus: vi.fn().mockResolvedValue(0), findSentAddresses: vi.fn().mockResolvedValue(new Set()), findBatchRecipients: vi.fn().mockResolvedValue([]), @@ -945,6 +947,47 @@ describe('EmailDeliveryWorker', () => { ) }) + it('leaves a plain send owed after a CHES outage outlasts every attempt', async () => { + await EmailDeliveryWorker.initialize( + mockEmailQueue as Bull.Queue, + mockNotificationService, + mockConfigService, + mockTemplatesRepository, + mockTemplatesService, + mockInlineRenderingService, + mockAttachmentResolverService as AttachmentResolverService, + mockEmailAdapter, + mockRequestDetailService, + ) + const outage = new TransientDeliveryError('CHES email: upstream 503', 502) + vi.mocked(mockEmailAdapter.send).mockRejectedValue(outage) + const job: Partial> = { + data: { + notifyId: 'notify-outage', + tenantId: 'tenant-1', + channel: NotificationChannel.EMAIL, + request: {}, + payload: { + recipients: { to: ['test@example.com'] }, + content: { subject: 'Hi', body: 'Hello', bodyType: 'html' }, + }, + attempt: 2, + } as DeliveryJobPayload, + opts: { attempts: 3 } as any, + attemptsMade: 2, + } + + await expect(processHandler(job as Bull.Job)).rejects.toBe(outage) + await failedCallback({ ...job, attemptsMade: 3 } as Bull.Job, outage) + + expect(mockRequestDetailService.markFailed).not.toHaveBeenCalled() + expect(mockNotificationService.update).not.toHaveBeenCalledWith( + 'notify-outage', + 'tenant-1', + expect.objectContaining({ status: NotificationStatus.FAILED }), + ) + }) + it('should NOT mark notification as FAILED on non-final attempts', async () => { await EmailDeliveryWorker.initialize( mockEmailQueue as Bull.Queue, @@ -1717,6 +1760,79 @@ describe('EmailDeliveryWorker', () => { expect(mockEmailAdapter.send).not.toHaveBeenCalled() }) + it('pauses the batch on a CHES outage, leaving unsent recipients owed rather than failed', async () => { + vi.mocked(mockEmailAdapter.send) + .mockResolvedValueOnce({ messageId: 'ext-1' } as any) + .mockRejectedValueOnce(new TransientDeliveryError('CHES email: upstream 503', 502)) + + const job = makeBulkJob(['alice@example.com', 'bob@example.com', 'carol@example.com']) + + await expect(processHandler(job as Bull.Job)).rejects.toBeInstanceOf( + TransientDeliveryError, + ) + expect(mockRequestDetailService.markRecipientSent).toHaveBeenCalledTimes(1) + expect(mockRequestDetailService.markRecipientFailed).not.toHaveBeenCalled() + expect(mockRequestDetailService.markBatchUnsentFailed).not.toHaveBeenCalled() + // Carol is never tried: the batch stops at the first sign of trouble. + expect(mockEmailAdapter.send).toHaveBeenCalledTimes(2) + expect(mockNotificationService.update).not.toHaveBeenCalled() + }) + + it('leaves a CHES outage owed even on the final attempt, for the reconciler', async () => { + vi.mocked(mockEmailAdapter.send).mockRejectedValue( + new TransientDeliveryError('CHES email: upstream 503', 502), + ) + const job = { ...makeBulkJob(['alice@example.com']), attemptsMade: 2, discard: vi.fn() } + + await expect(processHandler(job as Bull.Job)).rejects.toThrow('503') + expect(mockRequestDetailService.markBatchUnsentFailed).not.toHaveBeenCalled() + expect(mockRequestDetailService.markFailed).not.toHaveBeenCalled() + }) + + it('fails only this batch, never the whole send, once a systemic error exhausts retries', async () => { + mockTemplatesRepository.findById.mockResolvedValue(null) + // pending=0, failed=1, sent=24 once this batch is settled + mockRequestDetailService.countByStatus + .mockResolvedValueOnce(0) + .mockResolvedValueOnce(1) + .mockResolvedValueOnce(24) + const job = { ...makeBulkJob(['alice@example.com']), attemptsMade: 2, discard: vi.fn() } + + await expect(processHandler(job as Bull.Job)).rejects.toThrow( + 'not found', + ) + expect(mockRequestDetailService.markBatchUnsentFailed).toHaveBeenCalledWith( + 'notify-bulk', + 'notify-bulk-EMAIL-0', + expect.stringContaining('not found'), + ) + expect(mockRequestDetailService.markFailed).not.toHaveBeenCalled() + expect(mockNotificationService.update).toHaveBeenCalledWith('notify-bulk', 'tenant-bulk', { + status: NotificationStatus.PARTIALLY_COMPLETED, + updatedBy: 'email-delivery-worker', + }) + }) + + it('leaves a systemic error to be retried before the final attempt', async () => { + mockTemplatesRepository.findById.mockResolvedValue(null) + const job = makeBulkJob(['alice@example.com']) + + await expect(processHandler(job as Bull.Job)).rejects.toThrow() + expect(mockRequestDetailService.markBatchUnsentFailed).not.toHaveBeenCalled() + }) + + it('does not fail the whole send when a batch job is exhausted', async () => { + const job = { ...makeBulkJob(['alice@example.com']), attemptsMade: 3 } + + await failedCallback( + job as Bull.Job, + new Error('job stalled more than allowable limit'), + ) + + expect(mockRequestDetailService.markFailed).not.toHaveBeenCalled() + expect(mockNotificationService.update).not.toHaveBeenCalled() + }) + it('should resolve template once and render content per recipient', async () => { const job = makeBulkJob(['a@example.com', 'b@example.com', 'c@example.com']) diff --git a/backend/src/queue/workers/email-delivery.worker.ts b/backend/src/queue/workers/email-delivery.worker.ts index ee457db90..c22314fb2 100644 --- a/backend/src/queue/workers/email-delivery.worker.ts +++ b/backend/src/queue/workers/email-delivery.worker.ts @@ -4,6 +4,7 @@ import { ConfigService } from '@nestjs/config' import { DeliveryJobPayload, MailMergeJobData } from '../queue.types' import { batchProgressReporter, type BatchProgressReporter } from '../batch-progress' import { loadBatchRecipients } from './merge-batch-recipients' +import { isTransientDeliveryError } from '../../adapters/delivery-errors' import { NotificationService } from '../../api/notification/notification.service' import { NotificationRequestDetailService } from '../../api/notification/notification-request-detail.service' import { TemplatesRepository } from '../../api/templates/templates.repository' @@ -352,11 +353,36 @@ export class EmailDeliveryWorker { } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error) const attempt = (job.attemptsMade || 0) + 1 + const finalAttempt = (job.attemptsMade || 0) >= (job.opts.attempts || 3) - 1 logger.error( `[${notifyId}] Failed to send email delivery job (attempt ${attempt}/3): ${errorMessage}`, error instanceof Error ? error.stack : '', ) + // CHES could not take it right now. Nothing is marked failed, even on the last attempt: + // the recipients stay owed, and the delivery reconciler re-queues the job once Bull's + // retries are spent, until its own attempts run out. + if (isTransientDeliveryError(error)) throw error + + // A merge batch settles only itself: its sent rows, and every other batch, are not its + // to fail. + const batchId = job.data.mailMerge ? job.data.batchId : undefined + if (batchId) { + if (finalAttempt || EmailDeliveryWorker.isPermanentValidationError(error)) { + await requestDetailService.markBatchUnsentFailed(notifyId, batchId, errorMessage) + await EmailDeliveryWorker.settleMergeRequest( + notifyId, + tenantId, + requestDetailService, + notificationService, + logger, + ) + job.discard() + logger.error(`[${notifyId}] Batch ${batchId} failed; its unsent recipients are FAILED`) + } + throw error + } + if (EmailDeliveryWorker.isPermanentValidationError(error)) { await notificationService.update(notifyId, tenantId, { status: NotificationStatus.FAILED, @@ -372,7 +398,7 @@ export class EmailDeliveryWorker { } // Update status to FAILED only on final attempt (when no retries left) - if ((job.attemptsMade || 0) >= (job.opts.attempts || 3) - 1) { + if (finalAttempt) { await notificationService.update(notifyId, tenantId, { status: NotificationStatus.FAILED, updatedBy: 'system', @@ -431,6 +457,9 @@ export class EmailDeliveryWorker { // send error always reaches a terminal status — even if the in-processor final-attempt // handler didn't run (e.g. a job enqueued without a retry count, or a different worker // processed the last attempt). + // Not for a transient error - the recipients stay owed for the reconciler - nor for a merge + // batch, which the processor settled itself, scoped to that batch. + if (isTransientDeliveryError(err) || (job.data.mailMerge && job.data.batchId)) return const attemptsMade = job.attemptsMade ?? 0 const maxAttempts = job.opts.attempts ?? 1 if (attemptsMade >= maxAttempts) { @@ -587,6 +616,14 @@ export class EmailDeliveryWorker { ) sent++ } catch (recipientError) { + // CHES, not this recipient: stop the batch and leave the rest owed. The job is retried, + // skipping whoever was sent, and the circuit breaker holds the retry until CHES is back. + if (isTransientDeliveryError(recipientError)) { + logger.warn( + `[${notifyId}] Batch ${batchId} paused after ${sent} sent: ${recipientError.message}`, + ) + throw recipientError + } const errorMessage = recipientError instanceof Error ? recipientError.message : String(recipientError) logger.error( @@ -604,9 +641,28 @@ export class EmailDeliveryWorker { } logger.log(`[${notifyId}] Mail merge batch ${batchId} complete: sent=${sent}, failed=${failed}`) + await EmailDeliveryWorker.settleMergeRequest( + notifyId, + tenantId, + requestDetailService, + notificationService, + logger, + ) + + return { success: true, batchId, sent, failed } + } - // Reconcile the parent request once no recipients remain pending across all batches. - // all sent → COMPLETED; some sent + some failed → PARTIALLY_COMPLETED; all failed → FAILED. + /** + * Settle the parent request once no recipients remain pending across all batches. + * all sent → COMPLETED; some sent + some failed → PARTIALLY_COMPLETED; all failed → FAILED. + */ + private static async settleMergeRequest( + notifyId: string, + tenantId: string, + requestDetailService: NotificationRequestDetailService, + notificationService: NotificationService, + logger: Logger, + ): Promise { const pendingRemaining = await requestDetailService.countByStatus(notifyId, 'pending') if (pendingRemaining === 0) { const failedRemaining = await requestDetailService.countByStatus(notifyId, 'failed') @@ -627,8 +683,6 @@ export class EmailDeliveryWorker { `[${notifyId}] All mail merge batches complete; parent marked ${finalStatus.toUpperCase()}`, ) } - - return { success: true, batchId, sent, failed } } /** diff --git a/backend/src/queue/workers/sms-delivery.worker.spec.ts b/backend/src/queue/workers/sms-delivery.worker.spec.ts index 56cf61ef8..b1a8ce49a 100644 --- a/backend/src/queue/workers/sms-delivery.worker.spec.ts +++ b/backend/src/queue/workers/sms-delivery.worker.spec.ts @@ -6,6 +6,7 @@ import { DeliveryJobPayload } from '../queue.types' import { NotificationChannel } from '../../enum/notification-channel.enum' import { NotificationStatus } from '../../enum/notification-status.enum' import { ISmsTransport } from '../../adapters' +import { TransientDeliveryError } from '../../adapters/delivery-errors' describe('SmsDeliveryWorker', () => { let mockSmsQueue: Partial> @@ -77,6 +78,9 @@ describe('SmsDeliveryWorker', () => { markFailed: vi.fn().mockResolvedValue(undefined), markRecipientSent: vi.fn().mockResolvedValue(undefined), markRecipientFailed: vi.fn().mockResolvedValue(undefined), + markBatchUnsentFailed: vi.fn().mockResolvedValue(undefined), + findSentAddresses: vi.fn().mockResolvedValue(new Set()), + countByStatus: vi.fn().mockResolvedValue(1), updateStatus: vi.fn().mockResolvedValue(undefined), } @@ -962,4 +966,136 @@ describe('SmsDeliveryWorker', () => { expect(Logger.prototype.error).toHaveBeenCalled() }) }) + + describe('provider outages', () => { + const init = () => + SmsDeliveryWorker.initialize( + mockSmsQueue as Bull.Queue, + mockNotificationService, + mockConfigService, + mockTemplatesRepository, + mockTemplatesService, + mockInlineRenderingService, + mockSmsAdapter, + mockRequestDetailService, + ) + const outage = () => new TransientDeliveryError('ACS SMS: upstream 503 - unavailable', 502) + const mergeJob = (attemptsMade = 0) => + ({ + data: { + notifyId: 'notify-merge', + tenantId: 'tenant-123', + channel: NotificationChannel.SMS, + mailMerge: true, + batchId: 'notify-merge-SMS-0', + mailMergeData: { + content: { body: 'Hi {{name}}' }, + params: {}, + recipients: ['+16135550001', '+16135550002', '+16135550003'].map((address) => ({ + address, + params: { name: 'A' }, + })), + }, + request: {}, + payload: {} as never, + attempt: 0, + }, + opts: { attempts: 3 }, + attemptsMade, + discard: vi.fn(), + }) as unknown as Bull.Job + const plainJob = (attemptsMade = 0) => + ({ + data: { + notifyId: 'notify-plain', + tenantId: 'tenant-123', + channel: NotificationChannel.SMS, + request: {}, + payload: { + recipients: { to: ['+16135551234', '+16135559999'] }, + content: { body: 'Test SMS' }, + }, + attempt: 0, + }, + opts: { attempts: 3 }, + attemptsMade, + }) as unknown as Bull.Job + + beforeEach(() => { + mockInlineRenderingService.renderSms = vi.fn().mockResolvedValue({ body: 'Hi A' }) + }) + + it('pauses a merge batch on an outage, leaving unsent recipients owed rather than failed', async () => { + mockSmsAdapter.send = vi + .fn() + .mockResolvedValueOnce({ messageId: 'SM1' }) + .mockRejectedValueOnce(outage()) + await init() + + await expect(processHandler(mergeJob())).rejects.toBeInstanceOf(TransientDeliveryError) + + expect(mockRequestDetailService.markRecipientSent).toHaveBeenCalledTimes(1) + expect(mockRequestDetailService.markRecipientFailed).not.toHaveBeenCalled() + expect(mockSmsAdapter.send).toHaveBeenCalledTimes(2) + expect(mockRequestDetailService.markBatchUnsentFailed).not.toHaveBeenCalled() + }) + + it('leaves an outage owed on the final attempt, for the reconciler', async () => { + mockSmsAdapter.send = vi.fn().mockRejectedValue(outage()) + await init() + + await expect(processHandler(mergeJob(2))).rejects.toBeInstanceOf(TransientDeliveryError) + await expect(processHandler(plainJob(2))).rejects.toBeInstanceOf(TransientDeliveryError) + + expect(mockRequestDetailService.markBatchUnsentFailed).not.toHaveBeenCalled() + expect(mockRequestDetailService.markFailed).not.toHaveBeenCalled() + expect(mockNotificationService.update).not.toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + expect.objectContaining({ status: NotificationStatus.FAILED }), + ) + }) + + it('records who was sent, then retries for recipients the provider could not take', async () => { + mockSmsAdapter.send = vi.fn().mockResolvedValue({ + messageId: 'SM1', + results: [ + { to: '+16135551234', success: true, messageId: 'SM1' }, + { to: '+16135559999', success: false, error: 'throttled', transient: true }, + ], + }) + await init() + + await expect(processHandler(plainJob())).rejects.toBeInstanceOf(TransientDeliveryError) + + expect(mockRequestDetailService.markRecipientSent).toHaveBeenCalledWith( + 'notify-plain', + null, + '+16135551234', + 'SM1', + ) + expect(mockRequestDetailService.markRecipientFailed).not.toHaveBeenCalled() + }) + + it('fails only this batch once a systemic error exhausts retries', async () => { + mockTemplatesRepository.findById = vi.fn().mockResolvedValue(null) + const job = mergeJob(2) + ;(job.data.mailMergeData as { content: object }).content = { templateId: 'gone' } + mockRequestDetailService.countByStatus = vi + .fn() + .mockResolvedValueOnce(0) + .mockResolvedValueOnce(3) + .mockResolvedValueOnce(0) + await init() + + await expect(processHandler(job)).rejects.toThrow('not found') + + expect(mockRequestDetailService.markBatchUnsentFailed).toHaveBeenCalledWith( + 'notify-merge', + 'notify-merge-SMS-0', + expect.stringContaining('not found'), + ) + expect(mockRequestDetailService.markFailed).not.toHaveBeenCalled() + }) + }) }) diff --git a/backend/src/queue/workers/sms-delivery.worker.ts b/backend/src/queue/workers/sms-delivery.worker.ts index b6ab983e4..00c1c95de 100644 --- a/backend/src/queue/workers/sms-delivery.worker.ts +++ b/backend/src/queue/workers/sms-delivery.worker.ts @@ -4,6 +4,7 @@ import { ConfigService } from '@nestjs/config' import { DeliveryJobPayload, MailMergeJobData } from '../queue.types' import { batchProgressReporter, type BatchProgressReporter } from '../batch-progress' import { loadBatchRecipients } from './merge-batch-recipients' +import { TransientDeliveryError, isTransientDeliveryError } from '../../adapters/delivery-errors' import { NotificationService } from '../../api/notification/notification.service' import { NotificationRequestDetailService } from '../../api/notification/notification-request-detail.service' import { TemplatesRepository } from '../../api/templates/templates.repository' @@ -262,6 +263,8 @@ export class SmsDeliveryWorker { let finalStatus = NotificationStatus.COMPLETED if (result.results && result.results.length > 0) { for (const recipient of result.results) { + // The provider could not take it: still owed, not failed. Sent again below. + if (recipient.transient) continue if (recipient.success) { await requestDetailService.markRecipientSent( notifyId, @@ -279,6 +282,16 @@ export class SmsDeliveryWorker { } } + // Who was sent is recorded above, so the retry skips them; the rest are sent again once + // the provider is back. + const owed = result.results.filter((recipient) => recipient.transient) + if (owed.length > 0) { + throw new TransientDeliveryError( + `${owed.length} recipient(s) not sent, provider unavailable: ${owed[0].error ?? ''}`, + 502, + ) + } + const failedCount = result.results.filter((recipient) => !recipient.success).length if (failedCount > 0) { finalStatus = NotificationStatus.PARTIALLY_COMPLETED @@ -309,11 +322,36 @@ export class SmsDeliveryWorker { } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error) const attempt = (job.attemptsMade || 0) + 1 + const finalAttempt = (job.attemptsMade || 0) >= (job.opts.attempts || 3) - 1 logger.error( `[${notifyId}] Failed to send SMS delivery job (attempt ${attempt}/3): ${errorMessage}`, error instanceof Error ? error.stack : '', ) + // The provider could not take it right now. Nothing is marked failed, even on the last + // attempt: the recipients stay owed, and the delivery reconciler re-queues the job once + // Bull's retries are spent. + if (isTransientDeliveryError(error)) throw error + + // A merge batch settles only itself: its sent rows, and every other batch, are not its + // to fail. + const batchId = job.data.mailMerge ? job.data.batchId : undefined + if (batchId) { + if (finalAttempt || SmsDeliveryWorker.isPermanentValidationError(error)) { + await requestDetailService.markBatchUnsentFailed(notifyId, batchId, errorMessage) + await SmsDeliveryWorker.settleMergeRequest( + notifyId, + tenantId, + requestDetailService, + notificationService, + logger, + ) + job.discard() + logger.error(`[${notifyId}] Batch ${batchId} failed; its unsent recipients are FAILED`) + } + throw error + } + if (SmsDeliveryWorker.isPermanentValidationError(error)) { await notificationService.update(notifyId, tenantId, { status: NotificationStatus.FAILED, @@ -329,7 +367,7 @@ export class SmsDeliveryWorker { } // Update status to FAILED only on final attempt (when no retries left) - if ((job.attemptsMade || 0) >= (job.opts.attempts || 3) - 1) { + if (finalAttempt) { await notificationService.update(notifyId, tenantId, { status: NotificationStatus.FAILED, updatedBy: 'system', @@ -489,6 +527,14 @@ export class SmsDeliveryWorker { ) sent++ } catch (recipientError) { + // The provider, not this recipient: stop the batch and leave the rest owed. The job is + // retried, skipping whoever was sent, and the circuit breaker holds it until it is back. + if (isTransientDeliveryError(recipientError)) { + logger.warn( + `[${notifyId}] Batch ${batchId} paused after ${sent} sent: ${recipientError.message}`, + ) + throw recipientError + } const errorMessage = recipientError instanceof Error ? recipientError.message : String(recipientError) logger.error( @@ -506,8 +552,25 @@ export class SmsDeliveryWorker { } logger.log(`[${notifyId}] SMS merge batch ${batchId} complete: sent=${sent}, failed=${failed}`) + await SmsDeliveryWorker.settleMergeRequest( + notifyId, + tenantId, + requestDetailService, + notificationService, + logger, + ) + + return { success: failed === 0, batchId, sent, failed } + } - // Reconcile the parent request once no recipients remain pending across all batches. + /** Settle the parent request once no recipients remain pending across all batches. */ + private static async settleMergeRequest( + notifyId: string, + tenantId: string, + requestDetailService: NotificationRequestDetailService, + notificationService: NotificationService, + logger: Logger, + ): Promise { const pendingRemaining = await requestDetailService.countByStatus(notifyId, 'pending') if (pendingRemaining === 0) { const failedRemaining = await requestDetailService.countByStatus(notifyId, 'failed') @@ -528,8 +591,6 @@ export class SmsDeliveryWorker { `[${notifyId}] All SMS merge batches complete; parent marked ${finalStatus.toUpperCase()}`, ) } - - return { success: failed === 0, batchId, sent, failed } } private static async sendSmsViaAdapter( diff --git a/frontend/src/interfaces/queueMonitoring.interface.ts b/frontend/src/interfaces/queueMonitoring.interface.ts index 3cf5a43d8..28338f1a0 100644 --- a/frontend/src/interfaces/queueMonitoring.interface.ts +++ b/frontend/src/interfaces/queueMonitoring.interface.ts @@ -86,16 +86,27 @@ export interface MessageChannelStats { estimatedClearMinutes: number | null } -export interface ActiveBatch { - queue: string - jobId: string - notificationId: string | null - tenantId: string | null +/** + * One notification request with recipients still to send: a whole mail merge, however many + * batches it was split into, or a plain send. + */ +export interface SendInProgress { + notificationId: string + tenantId: string tenantName: string | null + channels: string[] + acceptedAt: string + total: number sent: number failed: number - total: number - startedAt: string | null + remaining: number + /** Merge batches; 0 for a plain send. */ + batches: number + batchesDone: number + batchesSending: number + /** Recipients sent or failed per minute over the last few minutes. */ + perMinute: number + estimatedMinutesLeft: number | null } /** Totals over the last hour, for the summary cards. */ @@ -143,6 +154,21 @@ export interface ReconcilerStats { recentActions: ReconcilerAction[] } +/** A delivery provider as the service sees it: the shared circuit breaker in front of it. */ +export interface ProviderStats { + /** e.g. "CHES", "ACS". */ + name: string + channel: 'EMAIL' | 'SMS' + status: HealthStatus + reasons: string[] + /** closed: sending; open: provider failing, sends wait; half-open: one test send in progress. */ + circuit: 'closed' | 'open' | 'half-open' | null + reopensAt: string | null + /** Requests in flight and their cap, where one is applied; null where not. */ + inFlight: number | null + limit: number | null +} + export interface QueueMonitoring { generatedAt: string status: HealthStatus @@ -150,8 +176,9 @@ export interface QueueMonitoring { redis: RedisStats messages: MessageChannelStats[] queues: QueueStats[] - activeBatches: ActiveBatch[] + sendsInProgress: SendInProgress[] workers: WorkerPod[] reconciler: ReconcilerStats + providers: ProviderStats[] recentFailures: RecentFailure[] } diff --git a/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx b/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx index 92596d79c..34ae3ed91 100644 --- a/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx +++ b/frontend/src/pages/admin/monitoring/QueueMonitoring.spec.tsx @@ -73,17 +73,22 @@ function snapshot(overrides: Partial = {}): QueueMonitoring estimatedClearMinutes: 0, }, ], - activeBatches: [ + sendsInProgress: [ { - queue: 'email-delivery', - jobId: 'req-1-EMAIL-0', notificationId: 'req-1', tenantId: 't-1', tenantName: 'Health Ministry', - sent: 37, + channels: ['EMAIL'], + acceptedAt: new Date(now - 300_000).toISOString(), + total: 2000, + sent: 398, failed: 2, - total: 100, - startedAt: new Date(now - 30_000).toISOString(), + remaining: 1600, + batches: 80, + batchesDone: 16, + batchesSending: 20, + perMinute: 64, + estimatedMinutesLeft: 25, }, ], queues: [ @@ -125,6 +130,28 @@ function snapshot(overrides: Partial = {}): QueueMonitoring ], }, ], + providers: [ + { + name: 'CHES', + channel: 'EMAIL', + status: 'healthy', + reasons: [], + circuit: 'closed', + reopensAt: null, + inFlight: 3, + limit: 5, + }, + { + name: 'ACS', + channel: 'SMS', + status: 'healthy', + reasons: [], + circuit: 'closed', + reopensAt: null, + inFlight: null, + limit: null, + }, + ], reconciler: { status: 'healthy', reasons: [], @@ -254,24 +281,44 @@ describe('QueueMonitoring', () => { expect(within(sms).getByText('Nothing pending')).toBeInTheDocument() }) - it('shows per-recipient progress for a batch that is one job', async () => { + it('rolls a merge send up into one row with its batches and time left', async () => { mockedGet.mockResolvedValue(snapshot()) renderPage() - const table = await screen.findByRole('table', { name: 'Batches in progress' }) - const meter = within(table).getByRole('meter', { name: 'Batch req-1-EMAIL-0 progress' }) - expect(meter).toHaveAttribute('aria-valuenow', '39') - expect(meter).toHaveAttribute('aria-valuetext', '37 of 100 sent, 2 failed') + const table = await screen.findByRole('table', { name: 'Sends in progress' }) + const meter = within(table).getByRole('meter', { name: 'Notification req-1 progress' }) + expect(meter).toHaveAttribute('aria-valuenow', '20') + expect(meter).toHaveAttribute('aria-valuetext', '398 of 2,000 sent, 2 failed') + expect(within(table).getByText('16 of 80 done, 20 sending, 44 waiting')).toBeInTheDocument() + expect(within(table).getByText('~25 min at 64/min')).toBeInTheDocument() expect(within(table).getByText('Health Ministry')).toBeInTheDocument() }) - it('says so when no batch is sending', async () => { - mockedGet.mockResolvedValue(snapshot({ activeBatches: [] })) + it('shows a plain send without batches, and one not sending yet', async () => { + const data = snapshot() + data.sendsInProgress = [ + { + ...data.sendsInProgress[0], + batches: 0, + batchesDone: 0, + batchesSending: 0, + perMinute: 0, + estimatedMinutesLeft: null, + }, + ] + mockedGet.mockResolvedValue(data) renderPage() - expect( - (await screen.findAllByText('No batches are sending right now.')).length, - ).toBeGreaterThan(0) + const table = await screen.findByRole('table', { name: 'Sends in progress' }) + expect(within(table).getByText('Not sending yet')).toBeInTheDocument() + expect(within(table).getAllByText('—').length).toBeGreaterThan(0) + }) + + it('says so when nothing is sending', async () => { + mockedGet.mockResolvedValue(snapshot({ sendsInProgress: [] })) + renderPage() + + expect((await screen.findAllByText('Nothing is sending right now.')).length).toBeGreaterThan(0) }) it('shows each queue with its backlog, rates and drain estimate on the System tab', async () => { @@ -394,6 +441,42 @@ describe('QueueMonitoring', () => { expect(screen.getAllByText('Nothing has needed recovering.')).not.toHaveLength(0) }) + it('shows each provider sending normally, with CHES requests in flight, on the System tab', async () => { + mockedGet.mockResolvedValue(snapshot()) + renderPage('system') + + expect(await screen.findByText('CHES (Email)')).toBeInTheDocument() + expect(screen.getByText('ACS (SMS)')).toBeInTheDocument() + expect(screen.getAllByText('Sending')).toHaveLength(2) + expect(screen.getByText('3 of 5')).toBeInTheDocument() + // ACS has no concurrency cap, so no in-flight tile. + expect(screen.getAllByText('Requests in flight')).toHaveLength(1) + }) + + it('raises a paused provider in the banner and on the System tab', async () => { + const data = snapshot() + data.providers = [ + data.providers[0], + { + ...data.providers[1], + status: 'warning', + reasons: ['Failing; sends are waiting, and resume once a test send succeeds'], + circuit: 'open', + reopensAt: '2026-10-01T18:00:30.000Z', + }, + ] + mockedGet.mockResolvedValue(data) + renderPage('system') + + expect( + await screen.findByText( + 'ACS (SMS): Failing; sends are waiting, and resume once a test send succeeds', + ), + ).toBeInTheDocument() + expect(screen.getByText('Paused: provider failing')).toBeInTheDocument() + expect(screen.getByText(/^Test send at /)).toBeInTheDocument() + }) + it('labels a pod that is shutting down', async () => { const data = snapshot() data.workers = [{ ...data.workers[0], draining: true }] diff --git a/frontend/src/pages/admin/monitoring/QueueMonitoring.tsx b/frontend/src/pages/admin/monitoring/QueueMonitoring.tsx index fe7232574..b9a1a059d 100644 --- a/frontend/src/pages/admin/monitoring/QueueMonitoring.tsx +++ b/frontend/src/pages/admin/monitoring/QueueMonitoring.tsx @@ -17,11 +17,12 @@ import type { HealthStatus } from '@/interfaces/queueMonitoring.interface' import { HEALTH_LABELS, formatQueueName } from '@/utils/monitoring' import OverviewCards from './sections/OverviewCards' import MessagesSection from './sections/MessagesSection' -import ActiveBatchesSection from './sections/ActiveBatchesSection' +import SendsInProgressSection from './sections/SendsInProgressSection' import RedisSection from './sections/RedisSection' import QueuesSection from './sections/QueuesSection' import WorkersSection from './sections/WorkersSection' import ReconcilerSection from './sections/ReconcilerSection' +import ProvidersSection, { providerLabel } from './sections/ProvidersSection' import RecentFailuresSection from './sections/RecentFailuresSection' import '@/scss/components/queue-monitoring.scss' @@ -95,6 +96,9 @@ const QueueMonitoring: FC = ({ tab, onTabChange }) => { ? [ ...data.redis.reasons.map((reason) => `Redis: ${reason}`), ...data.reconciler.reasons.map((reason) => `Delivery recovery: ${reason}`), + ...data.providers.flatMap((provider) => + provider.reasons.map((reason) => `${providerLabel(provider)}: ${reason}`), + ), ...data.queues.flatMap((queue) => queue.reasons.map((reason) => `${formatQueueName(queue.name)}: ${reason}`), ), @@ -182,7 +186,7 @@ const QueueMonitoring: FC = ({ tab, onTabChange }) => { <> - + )} @@ -192,6 +196,7 @@ const QueueMonitoring: FC = ({ tab, onTabChange }) => { <> + diff --git a/frontend/src/pages/admin/monitoring/sections/ActiveBatchesSection.tsx b/frontend/src/pages/admin/monitoring/sections/ActiveBatchesSection.tsx deleted file mode 100644 index 5073c2072..000000000 --- a/frontend/src/pages/admin/monitoring/sections/ActiveBatchesSection.tsx +++ /dev/null @@ -1,68 +0,0 @@ -import type { FC } from 'react' -import PageSubHeading from '@/components/PageSubHeading' -import MeterBar from '@/components/MeterBar' -import DataTable from '@/components/DataTable/DataTable' -import type { TableColumn } from '@/components/DataTable/DataTable' -import type { ActiveBatch } from '@/interfaces/queueMonitoring.interface' -import { formatAgo, formatQueueName } from '@/utils/monitoring' - -interface ActiveBatchesSectionProps { - batches: ActiveBatch[] - now: number -} - -function progressText(batch: ActiveBatch): string { - const failed = batch.failed > 0 ? `, ${batch.failed.toLocaleString()} failed` : '' - return `${batch.sent.toLocaleString()} of ${batch.total.toLocaleString()} sent${failed}` -} - -/** Merge batches being worked right now: one job each, with per-recipient progress. */ -const ActiveBatchesSection: FC = ({ batches, now }) => { - const columns: TableColumn[] = [ - { key: 'queue', label: 'Queue', render: (_, row) => formatQueueName(row.queue) }, - { - key: 'tenantName', - label: 'Tenant', - render: (_, row) => row.tenantName ?? row.tenantId ?? '—', - }, - { - key: 'notificationId', - label: 'Notification ID', - render: (_, row) => {row.notificationId ?? '—'}, - }, - { - key: 'sent', - label: 'Progress', - render: (_, row) => { - const done = row.total > 0 ? ((row.sent + row.failed) / row.total) * 100 : 0 - return ( -
- 0 ? 'warning' : 'healthy'} - /> - {progressText(row)} -
- ) - }, - }, - { key: 'startedAt', label: 'Started', render: (_, row) => formatAgo(row.startedAt, now) }, - ] - - return ( -
- - `${row.queue}-${row.jobId}`} - label="Batches in progress" - emptyMessage="No batches are sending right now." - /> -
- ) -} - -export default ActiveBatchesSection diff --git a/frontend/src/pages/admin/monitoring/sections/ProvidersSection.tsx b/frontend/src/pages/admin/monitoring/sections/ProvidersSection.tsx new file mode 100644 index 000000000..01983540d --- /dev/null +++ b/frontend/src/pages/admin/monitoring/sections/ProvidersSection.tsx @@ -0,0 +1,81 @@ +import type { FC } from 'react' +import PageSubHeading from '@/components/PageSubHeading' +import { StatusBadge } from '@/components/StatusBadge' +import StatTile from '@/components/StatTile' +import type { ProviderStats } from '@/interfaces/queueMonitoring.interface' +import { HEALTH_LABELS, formatMessageChannel } from '@/utils/monitoring' + +const CIRCUIT_LABELS: Record, string> = { + closed: 'Sending', + open: 'Paused: provider failing', + 'half-open': 'Testing recovery', +} + +/** "CHES (Email)". */ +export function providerLabel(provider: ProviderStats): string { + return `${provider.name} (${formatMessageChannel(provider.channel)})` +} + +function inFlightText(provider: ProviderStats): string | null { + if (provider.inFlight === null) return null + return provider.limit + ? `${provider.inFlight.toLocaleString()} of ${provider.limit.toLocaleString()}` + : provider.inFlight.toLocaleString() +} + +interface ProvidersSectionProps { + providers: ProviderStats[] +} + +/** Each delivery provider: whether sends are flowing to it, and how many are in flight. */ +const ProvidersSection: FC = ({ providers }) => ( +
+ +
+ {providers.map((provider) => { + const inFlight = inFlightText(provider) + return ( +
+
+ {providerLabel(provider)} + +
+ {provider.reasons.length > 0 && ( +
    + {provider.reasons.map((reason) => ( +
  • {reason}
    • + ))} +
+ )} +
+ + {inFlight !== null && ( + + )} +
+
+ ) + })} +

+ When a provider fails several sends in a row, every pod stops sending to it. Messages wait + rather than fail; after a pause one test send checks whether it has recovered, and sending + resumes when it succeeds. CHES requests are also capped across all pods, because CHES + accepts emails at a fixed pace and more at once only makes each one slower. +

+
+
+) + +export default ProvidersSection diff --git a/frontend/src/pages/admin/monitoring/sections/SendsInProgressSection.tsx b/frontend/src/pages/admin/monitoring/sections/SendsInProgressSection.tsx new file mode 100644 index 000000000..4707b7a1b --- /dev/null +++ b/frontend/src/pages/admin/monitoring/sections/SendsInProgressSection.tsx @@ -0,0 +1,87 @@ +import type { FC } from 'react' +import PageSubHeading from '@/components/PageSubHeading' +import MeterBar from '@/components/MeterBar' +import DataTable from '@/components/DataTable/DataTable' +import type { TableColumn } from '@/components/DataTable/DataTable' +import type { SendInProgress } from '@/interfaces/queueMonitoring.interface' +import { formatAgo, formatMessageChannel, formatRate } from '@/utils/monitoring' + +interface SendsInProgressSectionProps { + sends: SendInProgress[] + /** Reference time for "ago" labels: when the snapshot was taken. */ + now: number +} + +function progressText(send: SendInProgress): string { + const failed = send.failed > 0 ? `, ${send.failed.toLocaleString()} failed` : '' + return `${send.sent.toLocaleString()} of ${send.total.toLocaleString()} sent${failed}` +} + +function batchesText(send: SendInProgress): string { + if (send.batches === 0) return '—' + const waiting = send.batches - send.batchesDone - send.batchesSending + return [ + `${send.batchesDone.toLocaleString()} of ${send.batches.toLocaleString()} done`, + `${send.batchesSending.toLocaleString()} sending`, + ...(waiting > 0 ? [`${waiting.toLocaleString()} waiting`] : []), + ].join(', ') +} + +function timeLeftText(send: SendInProgress): string { + if (send.estimatedMinutesLeft === null) return 'Not sending yet' + return `~${send.estimatedMinutesLeft.toLocaleString()} min at ${formatRate(send.perMinute)}` +} + +/** Each request still sending, with its merge batches rolled up into one row. */ +const SendsInProgressSection: FC = ({ sends, now }) => { + const columns: TableColumn[] = [ + { + key: 'notificationId', + label: 'Notification ID', + render: (_, row) => {row.notificationId}, + }, + { key: 'tenantName', label: 'Tenant', render: (_, row) => row.tenantName ?? row.tenantId }, + { + key: 'channels', + label: 'Channel', + render: (_, row) => row.channels.map(formatMessageChannel).join(', ') || '—', + }, + { + key: 'sent', + label: 'Progress', + render: (_, row) => { + const done = row.total > 0 ? ((row.sent + row.failed) / row.total) * 100 : 0 + return ( +
+ 0 ? 'warning' : 'healthy'} + /> + {progressText(row)} +
+ ) + }, + }, + { key: 'batches', label: 'Batches', render: (_, row) => batchesText(row) }, + { key: 'estimatedMinutesLeft', label: 'Time left', render: (_, row) => timeLeftText(row) }, + { key: 'acceptedAt', label: 'Accepted', render: (_, row) => formatAgo(row.acceptedAt, now) }, + ] + + return ( +
+ + row.notificationId} + label="Sends in progress" + emptyMessage="Nothing is sending right now." + multiline + /> +
+ ) +} + +export default SendsInProgressSection diff --git a/frontend/src/scss/components/queue-monitoring.scss b/frontend/src/scss/components/queue-monitoring.scss index 69e3dfbdd..da8473849 100644 --- a/frontend/src/scss/components/queue-monitoring.scss +++ b/frontend/src/scss/components/queue-monitoring.scss @@ -31,6 +31,13 @@ max-width: 640px; } +// One delivery provider's caption, reasons and tiles, stacked like the Redis memory block. +.queue-monitoring__provider { + display: flex; + flex-direction: column; + gap: $spacing-sm; +} + .queue-monitoring__memory-caption { @include flex-between; font-size: $font-size-sm; From 48f837ee5cf30681e1eacb5932b26a4ebd507af0 Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Tue, 6 Oct 2026 12:02:43 -0700 Subject: [PATCH 20/23] Fix queue shutdown drain timing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change renames the shutdown timeout env var to QUEUE_SHUTDOWN_DRAIN_MS and tightens the graceful shutdown sequence. Queue workers are paused and closed in stages (ingestion → delivery → webhooks) with a bounded drain window before app shutdown, while timed-out jobs are logged as re-run elsewhere. This avoids partial merge batches being left mid-flight during deploys or scale-downs. --- backend/src/queue/queue.module.spec.ts | 4 +- backend/src/queue/queue.module.ts | 90 ++++++++------------------ 2 files changed, 29 insertions(+), 65 deletions(-) diff --git a/backend/src/queue/queue.module.spec.ts b/backend/src/queue/queue.module.spec.ts index c40bdf090..b73389558 100644 --- a/backend/src/queue/queue.module.spec.ts +++ b/backend/src/queue/queue.module.spec.ts @@ -110,7 +110,7 @@ describe('QueueModule', () => { }) afterEach(() => { - delete process.env.QUEUE_SHUTDOWN_TIMEOUT_MS + delete process.env.QUEUE_SHUTDOWN_DRAIN_MS vi.restoreAllMocks() }) @@ -133,7 +133,7 @@ describe('QueueModule', () => { }) it('gives up at the deadline rather than holding the pod past its grace period', async () => { - process.env.QUEUE_SHUTDOWN_TIMEOUT_MS = '20' + process.env.QUEUE_SHUTDOWN_DRAIN_MS = '20' const stuck = mockQueue('ingestion', () => new Promise(() => {})) const module = new QueueModule(stuck, mockQueue('email'), mockQueue('sms'), undefined) diff --git a/backend/src/queue/queue.module.ts b/backend/src/queue/queue.module.ts index c49e9c30c..6c8b347e1 100644 --- a/backend/src/queue/queue.module.ts +++ b/backend/src/queue/queue.module.ts @@ -207,25 +207,39 @@ export class QueueModule implements OnModuleInit, OnModuleDestroy, BeforeApplica } /** - * Let in-flight jobs finish before the pod exits. Without this a deploy kills a merge batch - * part-way through and Bull retries it on another pod about a minute later. + * Let in-flight jobs finish before the pod exits. Without this a deploy or an HPA scale-down + * kills a merge batch part-way through and Bull re-runs it on another pod once its lock lapses. * - * Runs before onApplicationShutdown, where TypeORM closes its connection, so a finishing job - * can still record what it sent. Bull's close() stops taking new jobs and resolves once the - * active ones end. The wait is capped below the pod's termination grace period; a batch still - * running then is left for Bull to retry, and the merge workers skip recipients already sent. + * Every queue stops taking new work on this pod first; then ingestion drains ahead of the + * delivery queues it feeds, and delivery ahead of the webhooks it triggers. Runs before + * onApplicationShutdown, where TypeORM closes its connection, so a finishing job can still + * record what it sent. The wait is capped below the pod's termination grace period + * (QUEUE_SHUTDOWN_DRAIN_MS, set by the chart); a batch still running then is re-run elsewhere, + * and the delivery workers skip recipients already sent. */ async beforeApplicationShutdown(): Promise { - const queues = [this.ingestionQueue, this.emailQueue, this.smsQueue, this.webhookQueue].filter( - (queue): queue is Bull.Queue => !!queue, - ) + const stages = [ + [this.ingestionQueue], + [this.emailQueue, this.smsQueue], + [this.webhookQueue], + ].map((stage) => stage.filter((queue): queue is Bull.Queue => !!queue)) + const queues = stages.flat() if (queues.length === 0) return const drainMs = parseInt(process.env.QUEUE_SHUTDOWN_DRAIN_MS || '100000', 10) this.logger.log(`Draining queue workers (up to ${drainMs / 1000}s)`) + + const drain = async (): Promise => { + // Local pause without waiting: stops fetching new jobs on this pod only. + await Promise.all(queues.map((queue) => queue.pause(true, true))) + for (const stage of stages) { + await Promise.all(stage.map((queue) => queue.close())) + } + } + let timer: NodeJS.Timeout | undefined const drained = await Promise.race([ - Promise.all(queues.map((queue) => queue.close())).then(() => true), + drain().then(() => true), new Promise((resolve) => { timer = setTimeout(() => resolve(false), drainMs) }), @@ -240,7 +254,9 @@ export class QueueModule implements OnModuleInit, OnModuleDestroy, BeforeApplica await this.heartbeat?.remove() } else { // Still holding jobs: keep the heartbeat until the pod is killed so they read as held. - this.logger.warn('Queue drain timed out; unfinished jobs will be retried on another pod') + this.logger.warn( + `Queue drain did not finish within ${drainMs}ms; unfinished jobs will be re-run by another pod`, + ) } } @@ -351,56 +367,4 @@ export class QueueModule implements OnModuleInit, OnModuleDestroy, BeforeApplica this.logger.debug(error) } } - - /** - * Let this pod's running jobs finish before it exits. Without this a stopping pod (a rolling - * deploy, an HPA scale-down) abandons its active jobs, and Bull re-runs them on another pod once - * their lock lapses. Every queue stops taking new work first; then ingestion drains ahead of the - * delivery queues it feeds, and delivery ahead of the webhooks it triggers. - * - * Bounded below Kubernetes' default 30s termination grace period. A job still running at the - * deadline is re-run elsewhere, which the delivery workers' already-sent checks make safe. - */ - async beforeApplicationShutdown(): Promise { - const stages = [ - [this.ingestionQueue], - [this.emailQueue, this.smsQueue], - [this.webhookQueue], - ].map((stage) => stage.filter((queue): queue is Bull.Queue => !!queue)) - const queues = stages.flat() - if (queues.length === 0) return - - const timeoutMs = parseInt(process.env.QUEUE_SHUTDOWN_TIMEOUT_MS || '20000', 10) - this.logger.log(`Draining queue workers before shutdown (timeout ${timeoutMs}ms)`) - - const drain = async (): Promise => { - // Local pause without waiting: stops fetching new jobs on this pod only. - await Promise.all(queues.map((queue) => queue.pause(true, true))) - for (const stage of stages) { - await Promise.all(stage.map((queue) => queue.close())) - } - } - - let timer: NodeJS.Timeout | undefined - const deadline = new Promise<'timeout'>((resolve) => { - timer = setTimeout(() => resolve('timeout'), timeoutMs) - }) - - try { - const outcome = await Promise.race([drain().then(() => 'drained' as const), deadline]) - if (outcome === 'timeout') { - this.logger.warn( - `Queue drain did not finish within ${timeoutMs}ms; remaining jobs will be re-run by another pod`, - ) - } else { - this.logger.log('Queue workers drained') - } - } catch (error) { - this.logger.error( - `Queue drain failed: ${error instanceof Error ? error.message : String(error)}`, - ) - } finally { - clearTimeout(timer) - } - } } From b6d42bd44cc13d600419cfc7452f225c43c0111f Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Tue, 6 Oct 2026 12:03:00 -0700 Subject: [PATCH 21/23] Move TEST/PROD deploy params to chart values Removes long TEST and PROD `params` overrides from `merge.yml` so rollout strategy, autoscaling, PDB behavior, and memory sizing come from `values-test.yaml` and `values-prod.yaml` directly. Both env value files now explicitly set `backend.memoryRequest` to `384Mi` and update quota/headroom comments to match, keeping deployment shape and resource planning reviewable in git. --- .github/workflows/merge.yml | 2 -- charts/app/values-prod.yaml | 7 +++++-- charts/app/values-test.yaml | 7 +++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 587f0477f..a14f04fc3 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -229,7 +229,6 @@ jobs: environment: TEST oc_server: ${{ vars.oc_server }} db_user: appproxy # appproxy is the user which works with pgbouncer. - params: --set backend.deploymentStrategy=RollingUpdate --set frontend.deploymentStrategy=RollingUpdate --set global.autoscaling=true --set backend.autoscaling.minReplicas=2 --set backend.autoscaling.maxReplicas=2 --set frontend.autoscaling.minReplicas=2 --set frontend.autoscaling.maxReplicas=2 --set frontend.secrets.enabled=true --set frontend.promtail.enabled=true --set backend.memoryRequest=384Mi --set frontend.secrets.enabled=true # Replica counts, rollout strategy and PDBs now live in # charts/app/values-test.yaml so the deployed shape of TEST is reviewable # in git rather than spread across --set flags here. @@ -309,7 +308,6 @@ jobs: environment: PROD oc_server: ${{ vars.oc_server }} db_user: appproxy # appproxy is the user which works with pgbouncer. - params: --set backend.deploymentStrategy=RollingUpdate --set frontend.deploymentStrategy=RollingUpdate --set global.autoscaling=true --set backend.autoscaling.minReplicas=5 --set backend.autoscaling.maxReplicas=5 --set frontend.autoscaling.minReplicas=5 --set frontend.autoscaling.maxReplicas=5 --set frontend.pdb.enabled=true --set backend.pdb.enabled=true --set frontend.promtail.enabled=true --set backend.memoryRequest=384Mi --set backend.pdb.enabled=true # Replica counts, rollout strategy and PDBs now live in # charts/app/values-prod.yaml so the deployed shape of PROD is reviewable # in git rather than spread across --set flags here. diff --git a/charts/app/values-prod.yaml b/charts/app/values-prod.yaml index 115dccda0..3513c0c31 100644 --- a/charts/app/values-prod.yaml +++ b/charts/app/values-prod.yaml @@ -8,14 +8,17 @@ # reviewable in git. # # Quota headroom (f6bc3f-prod: 4000m CPU / 16Gi memory, ~2060m CPU in use): -# backend 50m + 200Mi per pod, frontend 30m + 50Mi per pod -# at maxReplicas 7+7 that is 560m CPU / 1750Mi memory total. +# backend 50m + 384Mi per pod, frontend 30m + 50Mi per pod +# at maxReplicas 7+7 that is 560m CPU / 3038Mi memory total. global: #-- required for the HPAs to be rendered at all; see hpa.yaml autoscaling: true backend: + #-- requested memory per pod: room for a pod's working set under bulk-send load, below the + #-- 512MB heap cap (backend.heapMb) so the scheduler places pods where they actually fit + memoryRequest: 384Mi #-- rolling, so a prod deploy never drops the service deploymentStrategy: RollingUpdate autoscaling: diff --git a/charts/app/values-test.yaml b/charts/app/values-test.yaml index fded8db18..7aa2efe23 100644 --- a/charts/app/values-test.yaml +++ b/charts/app/values-test.yaml @@ -8,14 +8,17 @@ # reviewable in git. # # Quota headroom (f6bc3f-test: 3000m CPU / 12Gi memory): -# backend 50m + 200Mi per pod, frontend 30m + 50Mi per pod -# at maxReplicas 4+4 that is 320m CPU / 1000Mi memory total. +# backend 50m + 384Mi per pod, frontend 30m + 50Mi per pod +# at maxReplicas 4+4 that is 320m CPU / 1736Mi memory total. global: #-- required for the HPAs to be rendered at all; see hpa.yaml autoscaling: true backend: + #-- requested memory per pod: room for a pod's working set under bulk-send load, below the + #-- 512MB heap cap (backend.heapMb) so the scheduler places pods where they actually fit + memoryRequest: 384Mi #-- rolling, so a test deploy does not drop the service between pods deploymentStrategy: RollingUpdate autoscaling: From adb955a7b9d2e3362a3e5189fe220026a8ccde12 Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Tue, 6 Oct 2026 12:03:09 -0700 Subject: [PATCH 22/23] Update queue.module.shutdown.spec.ts --- backend/src/queue/queue.module.shutdown.spec.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/backend/src/queue/queue.module.shutdown.spec.ts b/backend/src/queue/queue.module.shutdown.spec.ts index ac799cb19..a68deb86a 100644 --- a/backend/src/queue/queue.module.shutdown.spec.ts +++ b/backend/src/queue/queue.module.shutdown.spec.ts @@ -6,7 +6,10 @@ import type { WorkerHeartbeat } from './worker-heartbeat' type Heartbeat = Pick function moduleWith(close: () => Promise) { - const queue = { close: vi.fn(close) } as unknown as Bull.Queue + const queue = { + pause: vi.fn().mockResolvedValue(undefined), + close: vi.fn(close), + } as unknown as Bull.Queue const heartbeat: Heartbeat = { markDraining: vi.fn(), remove: vi.fn().mockResolvedValue(undefined), @@ -34,7 +37,9 @@ describe('QueueModule shutdown', () => { await module.beforeApplicationShutdown() - // Three queues present (webhook absent), each closed so it finishes its active jobs. + // Three queues present (webhook absent): each stops taking work, then closes once its + // active jobs finish. + expect(queue.pause).toHaveBeenCalledTimes(3) expect(queue.close).toHaveBeenCalledTimes(3) expect(heartbeat.remove).toHaveBeenCalled() }) From ab5130244947fbe9d6662f35dfa8b9f69eab9918 Mon Sep 17 00:00:00 2001 From: Barrett Falk Date: Tue, 6 Oct 2026 12:06:52 -0700 Subject: [PATCH 23/23] Potential fix for pull request finding 'Overwritten property' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- backend/src/queue/workers/sms-delivery.worker.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/queue/workers/sms-delivery.worker.spec.ts b/backend/src/queue/workers/sms-delivery.worker.spec.ts index 02401b960..cc8a1821f 100644 --- a/backend/src/queue/workers/sms-delivery.worker.spec.ts +++ b/backend/src/queue/workers/sms-delivery.worker.spec.ts @@ -79,7 +79,6 @@ describe('SmsDeliveryWorker', () => { markRecipientSent: vi.fn().mockResolvedValue(undefined), markRecipientFailed: vi.fn().mockResolvedValue(undefined), markBatchUnsentFailed: vi.fn().mockResolvedValue(undefined), - findSentAddresses: vi.fn().mockResolvedValue(new Set()), countByStatus: vi.fn().mockResolvedValue(1), updateStatus: vi.fn().mockResolvedValue(undefined), findSentAddresses: vi.fn().mockResolvedValue(new Set()),